From 4cd281112cdd16cf4f5ec80d23ac9bb12f6a580e Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Fri, 16 May 2025 10:55:31 +0200 Subject: [PATCH 001/110] Proof of concept --- conan/api/subapi/cache.py | 15 ++-- conan/internal/api/migrations.py | 2 + conan/internal/api/uploader.py | 28 ++------ conan/internal/cache/home_paths.py | 4 ++ conan/internal/rest/remote_manager.py | 14 ++-- conan/internal/util/compression.py | 69 +++++++++++++++++++ conan/internal/util/files.py | 11 --- test/unittests/util/files/tar_extract_test.py | 6 +- 8 files changed, 97 insertions(+), 52 deletions(-) create mode 100644 conan/internal/util/compression.py diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index b0d3ede2d10..ffa05bf3b66 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -18,6 +18,7 @@ from conan.api.model import RecipeReference from conan.internal.util.dates import revision_timestamp_now from conan.internal.util.files import rmdir, mkdir, remove +from conan.internal.util.compression import tar_extract class CacheAPI: @@ -133,6 +134,8 @@ def save(self, package_list, tgz_path, no_source=False): mkdir(os.path.dirname(tgz_path)) name = os.path.basename(tgz_path) compresslevel = global_conf.get("core.gzip:compresslevel", check_type=int) + # compress_fn = self._load_compress_plugin() + with open(tgz_path, "wb") as tgz_handle: tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) @@ -182,14 +185,10 @@ def restore(self, path): cache = PkgCache(self.conan_api.cache_folder, self.conan_api.config.global_conf) cache_folder = cache.store # Note, this is not the home, but the actual package cache - - with open(path, mode='rb') as file_handler: - the_tar = tarfile.open(fileobj=file_handler) - fileobj = the_tar.extractfile("pkglist.json") - pkglist = fileobj.read() - the_tar.extraction_filter = (lambda member, _: member) # fully_trusted (Py 3.14) - the_tar.extractall(path=cache_folder) - the_tar.close() + tar_extract(self.conan_api.cache_folder, path, cache_folder) + # Retrieve the package list from the already extracted archive + with open(os.path.join(cache_folder, "pkglist.json")) as file_handler: + pkglist = file_handler.read() # After unzipping the files, we need to update the DB that references these files out = ConanOutput() diff --git a/conan/internal/api/migrations.py b/conan/internal/api/migrations.py index fc54424e940..da3faf4d3cd 100644 --- a/conan/internal/api/migrations.py +++ b/conan/internal/api/migrations.py @@ -4,6 +4,7 @@ from conan.api.output import ConanOutput from conan.internal.default_settings import migrate_settings_file +from conan.internal.util.compression import migrate_compression_plugin from conans.migrations import Migrator from conan.internal.util.dates import timestamp_now from conan.internal.util.files import load, save @@ -51,6 +52,7 @@ def _apply_migrations(self, old_version): # Update profile plugin from conan.internal.api.profile.profile_loader import migrate_profile_plugin migrate_profile_plugin(self.cache_folder) + migrate_compression_plugin(self.cache_folder) if old_version and old_version < "2.0.14-": _migrate_pkg_db_lru(self.cache_folder, old_version) diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index b075e4f07c7..f8921fa1c5e 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -12,8 +12,9 @@ from conan.errors import ConanException from conan.internal.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO) +from conan.internal.util.compression import tar_compress from conan.internal.util.files import (clean_dirty, is_dirty, gather_files, - set_dirty_context_manager, mkdir, human_size) + mkdir, human_size) UPLOAD_POLICY_FORCE = "force-upload" UPLOAD_POLICY_SKIP = "skip-upload" @@ -156,8 +157,8 @@ def add_tgz(tgz_name, tgz_files): result[tgz_name] = tgz elif tgz_files: compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) - tgz = compress_files(tgz_files, tgz_name, download_export_folder, - compresslevel=compresslevel, ref=ref) + tgz = tar_compress(self._app.cache_folder, tgz_files, tgz_name, download_export_folder, + compresslevel=compresslevel, ref=ref) result[tgz_name] = tgz add_tgz(EXPORT_TGZ_NAME, files) @@ -203,8 +204,8 @@ def _compress_package_files(self, layout, pref): if not os.path.isfile(package_tgz): tgz_files = {f: path for f, path in files.items()} compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) - tgz_path = compress_files(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, - compresslevel=compresslevel, ref=pref) + tgz_path = tar_compress(self._app.cache_folder, tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, + compresslevel=compresslevel, ref=pref) assert tgz_path == package_tgz assert os.path.exists(package_tgz) @@ -271,23 +272,6 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): return t -def compress_files(files, name, dest_dir, compresslevel=None, ref=None): - t1 = time.time() - # FIXME, better write to disk sequentially and not keep tgz contents in memory - tgz_path = os.path.join(dest_dir, name) - ConanOutput(scope=str(ref)).info(f"Compressing {name}") - with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: - tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) - for filename, abs_path in sorted(files.items()): - # recursive is False in case it is a symlink to a folder - tgz.add(abs_path, filename, recursive=False) - tgz.close() - - duration = time.time() - t1 - ConanOutput().debug(f"{name} compressed in {duration} time") - return tgz_path - - def _total_size(cache_files): total_size = 0 for file in cache_files.values(): diff --git a/conan/internal/cache/home_paths.py b/conan/internal/cache/home_paths.py index 5674fbba145..5d27bfd5a0e 100644 --- a/conan/internal/cache/home_paths.py +++ b/conan/internal/cache/home_paths.py @@ -90,3 +90,7 @@ def settings_path_user(self): @property def config_version_path(self): return os.path.join(self._home, "config_version.json") + + @property + def compression_plugin_path(self): + return os.path.join(self._home, _EXTENSIONS_FOLDER, _PLUGINS, "compression.py") diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index d87f3d849bb..eeaf5e939b7 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -17,7 +17,8 @@ from conan.api.model import RecipeReference from conan.internal.util.files import rmdir, human_size from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME -from conan.internal.util.files import mkdir, tar_extract +from conan.internal.util.files import mkdir +from conan.internal.util.compression import tar_extract class RemoteManager: @@ -81,7 +82,7 @@ def get_recipe(self, ref, remote, metadata=None): tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None) if tgz_file: - uncompress_file(tgz_file, export_folder, scope=str(ref)) + uncompress_file(self._home_folder, tgz_file, export_folder, scope=str(ref)) mkdir(export_folder) for file_name, file_path in zipped_files.items(): # copy CONANFILE shutil.move(file_path, os.path.join(export_folder, file_name)) @@ -123,7 +124,7 @@ def get_recipe_sources(self, ref, layout, remote): self._signer.verify(ref, download_folder, files=zipped_files) tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME] - uncompress_file(tgz_file, export_sources_folder, scope=str(ref)) + uncompress_file(self._home_folder, tgz_file, export_sources_folder, scope=str(ref)) def get_package(self, pref, remote, metadata=None): output = ConanOutput(scope=str(pref.ref)) @@ -171,7 +172,7 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata): tgz_file = zipped_files.pop(PACKAGE_TGZ_NAME, None) package_folder = layout.package() - uncompress_file(tgz_file, package_folder, scope=str(pref.ref)) + uncompress_file(self._home_folder, tgz_file, package_folder, scope=str(pref.ref)) mkdir(package_folder) # Just in case it doesn't exist, because uncompress did nothing for file_name, file_path in zipped_files.items(): # copy CONANINFO and CONANMANIFEST shutil.move(file_path, os.path.join(package_folder, file_name)) @@ -281,15 +282,14 @@ def _call_remote(self, remote, method, *args, **kwargs): raise ConanException(exc, remote=remote) -def uncompress_file(src_path, dest_folder, scope=None): +def uncompress_file(cache_folder, src_path, dest_folder, scope=None): try: filesize = os.path.getsize(src_path) big_file = filesize > 10000000 # 10 MB if big_file: hs = human_size(filesize) ConanOutput(scope=scope).info(f"Decompressing {hs} {os.path.basename(src_path)}") - with open(src_path, mode='rb') as file_handler: - tar_extract(file_handler, dest_folder) + tar_extract(cache_folder, src_path, dest_folder) except Exception as e: error_msg = "Error while extracting downloaded file '%s' to %s\n%s\n"\ % (src_path, dest_folder, str(e)) diff --git a/conan/internal/util/compression.py b/conan/internal/util/compression.py new file mode 100644 index 00000000000..046e32d1407 --- /dev/null +++ b/conan/internal/util/compression.py @@ -0,0 +1,69 @@ +import os +from conan.internal.cache.home_paths import HomePaths +from conan.internal.loader import load_python_file +from conan.internal.errors import ConanException + +def tar_extract(cache_folder, src_path, destination_dir): + compress_plugin = _load_compress_plugin(cache_folder) + compress_plugin.tar_extract(src_path, destination_dir) + +def tar_compress(cache_folder, files, name, dest_dir, compresslevel=None, ref=None): + compress_plugin = _load_compress_plugin(cache_folder) + return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) + +def _load_compress_plugin(cache_folder): + compression_plugin_path = HomePaths(cache_folder).compression_plugin_path + if not os.path.exists(compression_plugin_path): + # TODO + raise ConanException("The 'compression.py' plugin file doesn't exist. If you want " + "to disable it, edit its contents instead of removing it") + + mod, _ = load_python_file(compression_plugin_path) + if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"): + raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions") + return mod + + +_default_compression_plugin = """\ +# This file was generated by Conan. Remove this comment if you edit this file or Conan +# will destroy your changes. + +import os +import time +import tarfile +from conan.api.output import ConanOutput + +def tar_compress(files, name, dest_dir, compresslevel=None, ref=None): + t1 = time.time() + # FIXME, better write to disk sequentially and not keep tgz contents in memory + tgz_path = os.path.join(dest_dir, name) + ConanOutput(scope=str(ref)).info(f"Compressing {name}") + with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: + tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) + for filename, abs_path in sorted(files.items()): + # recursive is False in case it is a symlink to a folder + tgz.add(abs_path, filename, recursive=False) + tgz.close() + + duration = time.time() - t1 + ConanOutput().debug(f"{name} compressed in {duration} time") + return tgz_path + + +def tar_extract(src_path, destination_dir): + with open(src_path, mode='rb') as file_handler: + the_tar = tarfile.open(fileobj=file_handler) + # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to + # "could not change modification time", with time=0 + # the_tar.errorlevel = 2 # raise exception if any error + the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break + the_tar.extractall(path=destination_dir) + the_tar.close() +""" + +def migrate_compression_plugin(cache_folder): + from conan.internal.api.migrations import update_file + + profile_plugin_file = HomePaths(cache_folder).compression_plugin_path + update_file(profile_plugin_file, _default_compression_plugin) + diff --git a/conan/internal/util/files.py b/conan/internal/util/files.py index 98746479ae0..01b9a76fe64 100644 --- a/conan/internal/util/files.py +++ b/conan/internal/util/files.py @@ -6,7 +6,6 @@ import shutil import stat import sys -import tarfile import time from contextlib import contextmanager @@ -256,16 +255,6 @@ def mkdir(path): os.makedirs(path) -def tar_extract(fileobj, destination_dir): - the_tar = tarfile.open(fileobj=fileobj) - # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to - # "could not change modification time", with time=0 - # the_tar.errorlevel = 2 # raise exception if any error - the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break - the_tar.extractall(path=destination_dir) - the_tar.close() - - def merge_directories(src, dst): from conan.tools.files import copy copy(None, pattern="*", src=src, dst=dst) diff --git a/test/unittests/util/files/tar_extract_test.py b/test/unittests/util/files/tar_extract_test.py index 6f031a56a13..9c71f5d07b8 100644 --- a/test/unittests/util/files/tar_extract_test.py +++ b/test/unittests/util/files/tar_extract_test.py @@ -50,13 +50,11 @@ def check_files(destination_dir): with chdir(working_dir): # Unpack and check destination_dir = os.path.join(self.tmp_folder, "dest") - with open(self.tgz_file, 'rb') as file_handler: - tar_extract(file_handler, destination_dir) + tar_extract(self.tgz_file, destination_dir) check_files(destination_dir) # Unpack and check (now we have a symlinked local folder) os.symlink(temp_folder(), "folder") destination_dir = os.path.join(self.tmp_folder, "dest2") - with open(self.tgz_file, 'rb') as file_handler: - tar_extract(file_handler, destination_dir) + tar_extract(self.tgz_file, destination_dir) check_files(destination_dir) From 156e036a882a3b1d7c084024eea67c603092fed0 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Fri, 16 May 2025 11:44:29 +0200 Subject: [PATCH 002/110] Simplify --- conan/api/subapi/cache.py | 6 +- conan/internal/api/migrations.py | 2 - conan/internal/api/uploader.py | 25 ++---- conan/internal/rest/remote_manager.py | 10 +-- conan/internal/util/compression.py | 84 ++++++++++--------- conan/test/utils/test_files.py | 2 +- .../integration/command/upload/upload_test.py | 2 +- .../util/files/strip_root_extract_test.py | 2 +- test/unittests/util/files/tar_extract_test.py | 5 +- 9 files changed, 61 insertions(+), 77 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index ffa05bf3b66..b27b9c06565 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -6,7 +6,6 @@ from conan.api.model import PackagesList from conan.api.output import ConanOutput -from conan.internal.api.uploader import gzopen_without_timestamps from conan.internal.cache.cache import PkgCache from conan.internal.cache.conan_reference_layout import EXPORT_SRC_FOLDER, EXPORT_FOLDER, SRC_FOLDER, \ METADATA, DOWNLOAD_EXPORT_FOLDER @@ -18,7 +17,7 @@ from conan.api.model import RecipeReference from conan.internal.util.dates import revision_timestamp_now from conan.internal.util.files import rmdir, mkdir, remove -from conan.internal.util.compression import tar_extract +from conan.internal.util.compression import gzopen_without_timestamps, tar_extract class CacheAPI: @@ -134,7 +133,6 @@ def save(self, package_list, tgz_path, no_source=False): mkdir(os.path.dirname(tgz_path)) name = os.path.basename(tgz_path) compresslevel = global_conf.get("core.gzip:compresslevel", check_type=int) - # compress_fn = self._load_compress_plugin() with open(tgz_path, "wb") as tgz_handle: tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, @@ -185,7 +183,7 @@ def restore(self, path): cache = PkgCache(self.conan_api.cache_folder, self.conan_api.config.global_conf) cache_folder = cache.store # Note, this is not the home, but the actual package cache - tar_extract(self.conan_api.cache_folder, path, cache_folder) + tar_extract(path, cache_folder, cache_folder=self.conan_api.cache_folder) # Retrieve the package list from the already extracted archive with open(os.path.join(cache_folder, "pkglist.json")) as file_handler: pkglist = file_handler.read() diff --git a/conan/internal/api/migrations.py b/conan/internal/api/migrations.py index da3faf4d3cd..fc54424e940 100644 --- a/conan/internal/api/migrations.py +++ b/conan/internal/api/migrations.py @@ -4,7 +4,6 @@ from conan.api.output import ConanOutput from conan.internal.default_settings import migrate_settings_file -from conan.internal.util.compression import migrate_compression_plugin from conans.migrations import Migrator from conan.internal.util.dates import timestamp_now from conan.internal.util.files import load, save @@ -52,7 +51,6 @@ def _apply_migrations(self, old_version): # Update profile plugin from conan.internal.api.profile.profile_loader import migrate_profile_plugin migrate_profile_plugin(self.cache_folder) - migrate_compression_plugin(self.cache_folder) if old_version and old_version < "2.0.14-": _migrate_pkg_db_lru(self.cache_folder, old_version) diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index f8921fa1c5e..06cf59da330 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -12,9 +12,9 @@ from conan.errors import ConanException from conan.internal.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO) -from conan.internal.util.compression import tar_compress from conan.internal.util.files import (clean_dirty, is_dirty, gather_files, mkdir, human_size) +from conan.internal.util.compression import tar_compress UPLOAD_POLICY_FORCE = "force-upload" UPLOAD_POLICY_SKIP = "skip-upload" @@ -157,8 +157,8 @@ def add_tgz(tgz_name, tgz_files): result[tgz_name] = tgz elif tgz_files: compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) - tgz = tar_compress(self._app.cache_folder, tgz_files, tgz_name, download_export_folder, - compresslevel=compresslevel, ref=ref) + tgz = tar_compress(tgz_files, tgz_name, download_export_folder, + compresslevel=compresslevel, ref=ref, cache_folder=self._app.cache_folder) result[tgz_name] = tgz add_tgz(EXPORT_TGZ_NAME, files) @@ -204,8 +204,8 @@ def _compress_package_files(self, layout, pref): if not os.path.isfile(package_tgz): tgz_files = {f: path for f, path in files.items()} compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) - tgz_path = tar_compress(self._app.cache_folder, tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, - compresslevel=compresslevel, ref=pref) + tgz_path = tar_compress(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, + compresslevel=compresslevel, ref=pref, cache_folder=self._app.cache_folder) assert tgz_path == package_tgz assert os.path.exists(package_tgz) @@ -257,21 +257,6 @@ def upload_package(self, pref, prev_bundle, remote): output.debug(f"Upload {pref} in {duration} time") -def gzopen_without_timestamps(name, fileobj, compresslevel=None): - """ !! Method overrided by laso to pass mtime=0 (!=None) to avoid time.time() was - setted in Gzip file causing md5 to change. Not possible using the - previous tarfile open because arguments are not passed to GzipFile constructor - """ - compresslevel = compresslevel if compresslevel is not None else 9 # default Gzip = 9 - fileobj = gzip.GzipFile(name, "w", compresslevel, fileobj, mtime=0) - # Format is forced because in Python3.8, it changed and it generates different tarfiles - # with different checksums, which break hashes of tgzs - # PAX_FORMAT is the default for Py38, lets make it explicit for older Python versions - t = tarfile.TarFile.taropen(name, "w", fileobj, format=tarfile.PAX_FORMAT) - t._extfileobj = False - return t - - def _total_size(cache_files): total_size = 0 for file in cache_files.values(): diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index eeaf5e939b7..9ebff0a07ab 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -82,7 +82,7 @@ def get_recipe(self, ref, remote, metadata=None): tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None) if tgz_file: - uncompress_file(self._home_folder, tgz_file, export_folder, scope=str(ref)) + uncompress_file(tgz_file, export_folder, scope=str(ref), cache_folder=self._home_folder) mkdir(export_folder) for file_name, file_path in zipped_files.items(): # copy CONANFILE shutil.move(file_path, os.path.join(export_folder, file_name)) @@ -124,7 +124,7 @@ def get_recipe_sources(self, ref, layout, remote): self._signer.verify(ref, download_folder, files=zipped_files) tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME] - uncompress_file(self._home_folder, tgz_file, export_sources_folder, scope=str(ref)) + uncompress_file(tgz_file, export_sources_folder, scope=str(ref), cache_folder=self._home_folder) def get_package(self, pref, remote, metadata=None): output = ConanOutput(scope=str(pref.ref)) @@ -172,7 +172,7 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata): tgz_file = zipped_files.pop(PACKAGE_TGZ_NAME, None) package_folder = layout.package() - uncompress_file(self._home_folder, tgz_file, package_folder, scope=str(pref.ref)) + uncompress_file(tgz_file, package_folder, scope=str(pref.ref), cache_folder=self._home_folder) mkdir(package_folder) # Just in case it doesn't exist, because uncompress did nothing for file_name, file_path in zipped_files.items(): # copy CONANINFO and CONANMANIFEST shutil.move(file_path, os.path.join(package_folder, file_name)) @@ -282,14 +282,14 @@ def _call_remote(self, remote, method, *args, **kwargs): raise ConanException(exc, remote=remote) -def uncompress_file(cache_folder, src_path, dest_folder, scope=None): +def uncompress_file(src_path, dest_folder, scope=None, cache_folder=None): try: filesize = os.path.getsize(src_path) big_file = filesize > 10000000 # 10 MB if big_file: hs = human_size(filesize) ConanOutput(scope=scope).info(f"Decompressing {hs} {os.path.basename(src_path)}") - tar_extract(cache_folder, src_path, dest_folder) + tar_extract(src_path, dest_folder, cache_folder=cache_folder) except Exception as e: error_msg = "Error while extracting downloaded file '%s' to %s\n%s\n"\ % (src_path, dest_folder, str(e)) diff --git a/conan/internal/util/compression.py b/conan/internal/util/compression.py index 046e32d1407..d72bfa023e2 100644 --- a/conan/internal/util/compression.py +++ b/conan/internal/util/compression.py @@ -1,39 +1,34 @@ -import os from conan.internal.cache.home_paths import HomePaths from conan.internal.loader import load_python_file from conan.internal.errors import ConanException -def tar_extract(cache_folder, src_path, destination_dir): - compress_plugin = _load_compress_plugin(cache_folder) - compress_plugin.tar_extract(src_path, destination_dir) +import os +import gzip +import time +import tarfile +from conan.api.output import ConanOutput +from conan.internal.util.files import set_dirty_context_manager -def tar_compress(cache_folder, files, name, dest_dir, compresslevel=None, ref=None): +def tar_extract(src_path, destination_dir, cache_folder=None): compress_plugin = _load_compress_plugin(cache_folder) - return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) - -def _load_compress_plugin(cache_folder): - compression_plugin_path = HomePaths(cache_folder).compression_plugin_path - if not os.path.exists(compression_plugin_path): - # TODO - raise ConanException("The 'compression.py' plugin file doesn't exist. If you want " - "to disable it, edit its contents instead of removing it") - - mod, _ = load_python_file(compression_plugin_path) - if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"): - raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions") - return mod + if compress_plugin: + return compress_plugin.tar_extract(src_path, destination_dir) + with open(src_path, mode='rb') as file_handler: + the_tar = tarfile.open(fileobj=file_handler) + # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to + # "could not change modification time", with time=0 + # the_tar.errorlevel = 2 # raise exception if any error + the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break + the_tar.extractall(path=destination_dir) + the_tar.close() -_default_compression_plugin = """\ -# This file was generated by Conan. Remove this comment if you edit this file or Conan -# will destroy your changes. -import os -import time -import tarfile -from conan.api.output import ConanOutput +def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, cache_folder=None): + compress_plugin = _load_compress_plugin(cache_folder) + if compress_plugin: + return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) -def tar_compress(files, name, dest_dir, compresslevel=None, ref=None): t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory tgz_path = os.path.join(dest_dir, name) @@ -50,20 +45,29 @@ def tar_compress(files, name, dest_dir, compresslevel=None, ref=None): return tgz_path -def tar_extract(src_path, destination_dir): - with open(src_path, mode='rb') as file_handler: - the_tar = tarfile.open(fileobj=file_handler) - # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to - # "could not change modification time", with time=0 - # the_tar.errorlevel = 2 # raise exception if any error - the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break - the_tar.extractall(path=destination_dir) - the_tar.close() -""" +def gzopen_without_timestamps(name, fileobj, compresslevel=None): + """ !! Method overrided by laso to pass mtime=0 (!=None) to avoid time.time() was + setted in Gzip file causing md5 to change. Not possible using the + previous tarfile open because arguments are not passed to GzipFile constructor + """ + compresslevel = compresslevel if compresslevel is not None else 9 # default Gzip = 9 + fileobj = gzip.GzipFile(name, "w", compresslevel, fileobj, mtime=0) + # Format is forced because in Python3.8, it changed and it generates different tarfiles + # with different checksums, which break hashes of tgzs + # PAX_FORMAT is the default for Py38, lets make it explicit for older Python versions + t = tarfile.TarFile.taropen(name, "w", fileobj, format=tarfile.PAX_FORMAT) + t._extfileobj = False + return t -def migrate_compression_plugin(cache_folder): - from conan.internal.api.migrations import update_file - profile_plugin_file = HomePaths(cache_folder).compression_plugin_path - update_file(profile_plugin_file, _default_compression_plugin) +def _load_compress_plugin(cache_folder): + if not cache_folder: + return None + compression_plugin_path = HomePaths(cache_folder).compression_plugin_path + if not os.path.exists(compression_plugin_path): + return None + mod, _ = load_python_file(compression_plugin_path) + if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"): + raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions") + return mod diff --git a/conan/test/utils/test_files.py b/conan/test/utils/test_files.py index 7b3f011116d..55b7c173fab 100644 --- a/conan/test/utils/test_files.py +++ b/conan/test/utils/test_files.py @@ -8,7 +8,7 @@ import time from io import BytesIO -from conan.internal.api.uploader import gzopen_without_timestamps +from conan.internal.util.compression import gzopen_without_timestamps from conan.tools.files.files import untargz from conan.internal.subsystems import get_cased_path from conan.errors import ConanException diff --git a/test/integration/command/upload/upload_test.py b/test/integration/command/upload/upload_test.py index 63368208747..1bb1044d534 100644 --- a/test/integration/command/upload/upload_test.py +++ b/test/integration/command/upload/upload_test.py @@ -6,6 +6,7 @@ import unittest from collections import OrderedDict +from conan.internal.util.compression import gzopen_without_timestamps import pytest from mock import patch from requests import Response @@ -13,7 +14,6 @@ from conan.errors import ConanException from conan.api.model import PkgReference from conan.api.model import RecipeReference -from conan.internal.api.uploader import gzopen_without_timestamps from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME, PACKAGE_TGZ_NAME from conan.test.utils.tools import NO_SETTINGS_PACKAGE_ID, TestClient, TestServer, \ GenConanfile, TestRequester, TestingResponse diff --git a/test/unittests/util/files/strip_root_extract_test.py b/test/unittests/util/files/strip_root_extract_test.py index 1e219be7802..0f86c4b1165 100644 --- a/test/unittests/util/files/strip_root_extract_test.py +++ b/test/unittests/util/files/strip_root_extract_test.py @@ -3,7 +3,7 @@ import unittest import zipfile -from conan.internal.api.uploader import gzopen_without_timestamps +from conan.internal.util.compression import gzopen_without_timestamps from conan.tools.files.files import untargz, unzip from conan.errors import ConanException from conan.internal.model.manifest import gather_files diff --git a/test/unittests/util/files/tar_extract_test.py b/test/unittests/util/files/tar_extract_test.py index 9c71f5d07b8..f00b1b825a6 100644 --- a/test/unittests/util/files/tar_extract_test.py +++ b/test/unittests/util/files/tar_extract_test.py @@ -2,12 +2,11 @@ import platform import tarfile import unittest - import pytest -from conan.internal.api.uploader import gzopen_without_timestamps +from conan.internal.util.compression import gzopen_without_timestamps, tar_extract from conan.test.utils.test_files import temp_folder -from conan.internal.util.files import tar_extract, save, gather_files, chdir +from conan.internal.util.files import save, gather_files, chdir class TarExtractTest(unittest.TestCase): From c7b5dff762d692e7e77b2f8dc3ae78c4ff813c6d Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Fri, 16 May 2025 12:29:31 +0200 Subject: [PATCH 003/110] Added tar_compressor --- conan/api/subapi/cache.py | 17 ++++++++--------- conan/internal/util/compression.py | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index b27b9c06565..32e9759efca 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -1,8 +1,6 @@ import json import os import shutil -import tarfile -from io import BytesIO from conan.api.model import PackagesList from conan.api.output import ConanOutput @@ -17,7 +15,7 @@ from conan.api.model import RecipeReference from conan.internal.util.dates import revision_timestamp_now from conan.internal.util.files import rmdir, mkdir, remove -from conan.internal.util.compression import gzopen_without_timestamps, tar_extract +from conan.internal.util.compression import tar_compressor, tar_extract class CacheAPI: @@ -135,8 +133,8 @@ def save(self, package_list, tgz_path, no_source=False): compresslevel = global_conf.get("core.gzip:compresslevel", check_type=int) with open(tgz_path, "wb") as tgz_handle: - tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, - compresslevel=compresslevel) + tgz = tar_compressor(name, fileobj=tgz_handle, compresslevel=compresslevel, + cache_path=self.conan_api.cache_folder) for ref, ref_bundle in package_list.refs().items(): ref_layout = cache.recipe_layout(ref) recipe_folder = os.path.relpath(ref_layout.base_folder, cache_folder) @@ -170,11 +168,12 @@ def save(self, package_list, tgz_path, no_source=False): out.info(f"Saving {pref} metadata: {metadata_folder}") tgz.add(os.path.join(cache_folder, metadata_folder), metadata_folder, recursive=True) + # Create pgklist.json to add it to the tgz serialized = json.dumps(package_list.serialize(), indent=2) - info = tarfile.TarInfo(name="pkglist.json") - data = serialized.encode('utf-8') - info.size = len(data) - tgz.addfile(tarinfo=info, fileobj=BytesIO(data)) + pkglist_path = os.path.join(cache_folder, "pkglist.json") + with open(pkglist_path, "w") as file_handler: + file_handler.write(serialized) + tgz.add(pkglist_path, "pkglist.json", recursive=False) tgz.close() def restore(self, path): diff --git a/conan/internal/util/compression.py b/conan/internal/util/compression.py index d72bfa023e2..e237b39ce13 100644 --- a/conan/internal/util/compression.py +++ b/conan/internal/util/compression.py @@ -44,6 +44,13 @@ def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, cache_fold ConanOutput().debug(f"{name} compressed in {duration} time") return tgz_path +def tar_compressor(name, fileobj, compresslevel, cache_path=None): + compress_plugin = _load_compress_plugin(cache_path) + if compress_plugin: + return compress_plugin.TarCompressor(name, fileobj, compresslevel) + else: + return gzopen_without_timestamps(name, fileobj, compresslevel) + def gzopen_without_timestamps(name, fileobj, compresslevel=None): """ !! Method overrided by laso to pass mtime=0 (!=None) to avoid time.time() was @@ -71,3 +78,14 @@ def _load_compress_plugin(cache_folder): if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"): raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions") return mod + + +""" +Plugin `compression.py` interface: + + def tar_extract(src_path, destination_dir) -> None + def tar_compress(files, name, dest_dir, compresslevel=None, ref=None) -> str + class TarCompressor(name, fileobj, compresslevel) + def add(self, abs_path, filename, recursive=True) -> None + def close() -> None +""" From be30ba2494347d974d4cc1567dccc5d9be9fb8bc Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Fri, 16 May 2025 13:05:10 +0200 Subject: [PATCH 004/110] Extra simplify --- conan/internal/api/uploader.py | 48 ++++++++++++++++--- conan/internal/rest/remote_manager.py | 11 +++-- conan/internal/util/compression.py | 8 ++-- conan/internal/util/files.py | 11 +++++ conan/test/utils/test_files.py | 2 +- .../integration/command/upload/upload_test.py | 2 +- .../util/files/strip_root_extract_test.py | 2 +- test/unittests/util/files/tar_extract_test.py | 11 +++-- 8 files changed, 75 insertions(+), 20 deletions(-) diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 06cf59da330..2d020f39827 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -12,9 +12,9 @@ from conan.errors import ConanException from conan.internal.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO) +from conan.internal.util.compression import load_compress_plugin from conan.internal.util.files import (clean_dirty, is_dirty, gather_files, - mkdir, human_size) -from conan.internal.util.compression import tar_compress + set_dirty_context_manager, mkdir, human_size) UPLOAD_POLICY_FORCE = "force-upload" UPLOAD_POLICY_SKIP = "skip-upload" @@ -157,8 +157,8 @@ def add_tgz(tgz_name, tgz_files): result[tgz_name] = tgz elif tgz_files: compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) - tgz = tar_compress(tgz_files, tgz_name, download_export_folder, - compresslevel=compresslevel, ref=ref, cache_folder=self._app.cache_folder) + tgz = compress_files(tgz_files, tgz_name, download_export_folder, + compresslevel=compresslevel, ref=ref, cache_folder=self._app.cache_folder) result[tgz_name] = tgz add_tgz(EXPORT_TGZ_NAME, files) @@ -204,8 +204,8 @@ def _compress_package_files(self, layout, pref): if not os.path.isfile(package_tgz): tgz_files = {f: path for f, path in files.items()} compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) - tgz_path = tar_compress(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, - compresslevel=compresslevel, ref=pref, cache_folder=self._app.cache_folder) + tgz_path = compress_files(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, + compresslevel=compresslevel, ref=pref, cache_folder=self._app.cache_folder) assert tgz_path == package_tgz assert os.path.exists(package_tgz) @@ -257,6 +257,42 @@ def upload_package(self, pref, prev_bundle, remote): output.debug(f"Upload {pref} in {duration} time") +def gzopen_without_timestamps(name, fileobj, compresslevel=None): + """ !! Method overrided by laso to pass mtime=0 (!=None) to avoid time.time() was + setted in Gzip file causing md5 to change. Not possible using the + previous tarfile open because arguments are not passed to GzipFile constructor + """ + compresslevel = compresslevel if compresslevel is not None else 9 # default Gzip = 9 + fileobj = gzip.GzipFile(name, "w", compresslevel, fileobj, mtime=0) + # Format is forced because in Python3.8, it changed and it generates different tarfiles + # with different checksums, which break hashes of tgzs + # PAX_FORMAT is the default for Py38, lets make it explicit for older Python versions + t = tarfile.TarFile.taropen(name, "w", fileobj, format=tarfile.PAX_FORMAT) + t._extfileobj = False + return t + + +def compress_files(files, name, dest_dir, compresslevel=None, ref=None, cache_folder=None): + compress_plugin = load_compress_plugin(cache_folder) + if compress_plugin: + return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) + + t1 = time.time() + # FIXME, better write to disk sequentially and not keep tgz contents in memory + tgz_path = os.path.join(dest_dir, name) + ConanOutput(scope=str(ref)).info(f"Compressing {name}") + with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: + tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) + for filename, abs_path in sorted(files.items()): + # recursive is False in case it is a symlink to a folder + tgz.add(abs_path, filename, recursive=False) + tgz.close() + + duration = time.time() - t1 + ConanOutput().debug(f"{name} compressed in {duration} time") + return tgz_path + + def _total_size(cache_files): total_size = 0 for file in cache_files.values(): diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index 9ebff0a07ab..19da098dbe6 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -15,10 +15,10 @@ from conan.internal.model.info import load_binary_info from conan.api.model import PkgReference from conan.api.model import RecipeReference +from conan.internal.util.compression import load_compress_plugin from conan.internal.util.files import rmdir, human_size from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME -from conan.internal.util.files import mkdir -from conan.internal.util.compression import tar_extract +from conan.internal.util.files import mkdir, tar_extract class RemoteManager: @@ -289,7 +289,12 @@ def uncompress_file(src_path, dest_folder, scope=None, cache_folder=None): if big_file: hs = human_size(filesize) ConanOutput(scope=scope).info(f"Decompressing {hs} {os.path.basename(src_path)}") - tar_extract(src_path, dest_folder, cache_folder=cache_folder) + + compression_plugin = load_compress_plugin(cache_folder) + if compression_plugin: + compression_plugin.tar_extract(src_path, dest_folder) + else: + tar_extract(src_path, dest_folder) except Exception as e: error_msg = "Error while extracting downloaded file '%s' to %s\n%s\n"\ % (src_path, dest_folder, str(e)) diff --git a/conan/internal/util/compression.py b/conan/internal/util/compression.py index e237b39ce13..e811c84f46e 100644 --- a/conan/internal/util/compression.py +++ b/conan/internal/util/compression.py @@ -10,7 +10,7 @@ from conan.internal.util.files import set_dirty_context_manager def tar_extract(src_path, destination_dir, cache_folder=None): - compress_plugin = _load_compress_plugin(cache_folder) + compress_plugin = load_compress_plugin(cache_folder) if compress_plugin: return compress_plugin.tar_extract(src_path, destination_dir) @@ -25,7 +25,7 @@ def tar_extract(src_path, destination_dir, cache_folder=None): def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, cache_folder=None): - compress_plugin = _load_compress_plugin(cache_folder) + compress_plugin = load_compress_plugin(cache_folder) if compress_plugin: return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) @@ -45,7 +45,7 @@ def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, cache_fold return tgz_path def tar_compressor(name, fileobj, compresslevel, cache_path=None): - compress_plugin = _load_compress_plugin(cache_path) + compress_plugin = load_compress_plugin(cache_path) if compress_plugin: return compress_plugin.TarCompressor(name, fileobj, compresslevel) else: @@ -67,7 +67,7 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): return t -def _load_compress_plugin(cache_folder): +def load_compress_plugin(cache_folder): if not cache_folder: return None compression_plugin_path = HomePaths(cache_folder).compression_plugin_path diff --git a/conan/internal/util/files.py b/conan/internal/util/files.py index 01b9a76fe64..98746479ae0 100644 --- a/conan/internal/util/files.py +++ b/conan/internal/util/files.py @@ -6,6 +6,7 @@ import shutil import stat import sys +import tarfile import time from contextlib import contextmanager @@ -255,6 +256,16 @@ def mkdir(path): os.makedirs(path) +def tar_extract(fileobj, destination_dir): + the_tar = tarfile.open(fileobj=fileobj) + # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to + # "could not change modification time", with time=0 + # the_tar.errorlevel = 2 # raise exception if any error + the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break + the_tar.extractall(path=destination_dir) + the_tar.close() + + def merge_directories(src, dst): from conan.tools.files import copy copy(None, pattern="*", src=src, dst=dst) diff --git a/conan/test/utils/test_files.py b/conan/test/utils/test_files.py index 55b7c173fab..7b3f011116d 100644 --- a/conan/test/utils/test_files.py +++ b/conan/test/utils/test_files.py @@ -8,7 +8,7 @@ import time from io import BytesIO -from conan.internal.util.compression import gzopen_without_timestamps +from conan.internal.api.uploader import gzopen_without_timestamps from conan.tools.files.files import untargz from conan.internal.subsystems import get_cased_path from conan.errors import ConanException diff --git a/test/integration/command/upload/upload_test.py b/test/integration/command/upload/upload_test.py index 1bb1044d534..63368208747 100644 --- a/test/integration/command/upload/upload_test.py +++ b/test/integration/command/upload/upload_test.py @@ -6,7 +6,6 @@ import unittest from collections import OrderedDict -from conan.internal.util.compression import gzopen_without_timestamps import pytest from mock import patch from requests import Response @@ -14,6 +13,7 @@ from conan.errors import ConanException from conan.api.model import PkgReference from conan.api.model import RecipeReference +from conan.internal.api.uploader import gzopen_without_timestamps from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME, PACKAGE_TGZ_NAME from conan.test.utils.tools import NO_SETTINGS_PACKAGE_ID, TestClient, TestServer, \ GenConanfile, TestRequester, TestingResponse diff --git a/test/unittests/util/files/strip_root_extract_test.py b/test/unittests/util/files/strip_root_extract_test.py index 0f86c4b1165..1e219be7802 100644 --- a/test/unittests/util/files/strip_root_extract_test.py +++ b/test/unittests/util/files/strip_root_extract_test.py @@ -3,7 +3,7 @@ import unittest import zipfile -from conan.internal.util.compression import gzopen_without_timestamps +from conan.internal.api.uploader import gzopen_without_timestamps from conan.tools.files.files import untargz, unzip from conan.errors import ConanException from conan.internal.model.manifest import gather_files diff --git a/test/unittests/util/files/tar_extract_test.py b/test/unittests/util/files/tar_extract_test.py index f00b1b825a6..6f031a56a13 100644 --- a/test/unittests/util/files/tar_extract_test.py +++ b/test/unittests/util/files/tar_extract_test.py @@ -2,11 +2,12 @@ import platform import tarfile import unittest + import pytest -from conan.internal.util.compression import gzopen_without_timestamps, tar_extract +from conan.internal.api.uploader import gzopen_without_timestamps from conan.test.utils.test_files import temp_folder -from conan.internal.util.files import save, gather_files, chdir +from conan.internal.util.files import tar_extract, save, gather_files, chdir class TarExtractTest(unittest.TestCase): @@ -49,11 +50,13 @@ def check_files(destination_dir): with chdir(working_dir): # Unpack and check destination_dir = os.path.join(self.tmp_folder, "dest") - tar_extract(self.tgz_file, destination_dir) + with open(self.tgz_file, 'rb') as file_handler: + tar_extract(file_handler, destination_dir) check_files(destination_dir) # Unpack and check (now we have a symlinked local folder) os.symlink(temp_folder(), "folder") destination_dir = os.path.join(self.tmp_folder, "dest2") - tar_extract(self.tgz_file, destination_dir) + with open(self.tgz_file, 'rb') as file_handler: + tar_extract(file_handler, destination_dir) check_files(destination_dir) From 1295d4f02c2736267410dee51faba4c134b585f0 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Mon, 19 May 2025 11:23:52 +0200 Subject: [PATCH 005/110] Cache the plugin load --- conan/api/subapi/cache.py | 89 +++++++++++++++--------------- conan/internal/util/compression.py | 21 ++----- 2 files changed, 50 insertions(+), 60 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index 32e9759efca..d019477b511 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -1,6 +1,7 @@ import json import os import shutil +import tempfile from conan.api.model import PackagesList from conan.api.output import ConanOutput @@ -14,8 +15,8 @@ from conan.api.model import PkgReference from conan.api.model import RecipeReference from conan.internal.util.dates import revision_timestamp_now -from conan.internal.util.files import rmdir, mkdir, remove -from conan.internal.util.compression import tar_compressor, tar_extract +from conan.internal.util.files import rmdir, mkdir, remove, save +from conan.internal.util.compression import tar_compress, tar_extract class CacheAPI: @@ -129,52 +130,50 @@ def save(self, package_list, tgz_path, no_source=False): cache_folder = cache.store # Note, this is not the home, but the actual package cache out = ConanOutput() mkdir(os.path.dirname(tgz_path)) - name = os.path.basename(tgz_path) compresslevel = global_conf.get("core.gzip:compresslevel", check_type=int) + tar_files: dict[str,str] = {} # {path_in_tar: abs_path} - with open(tgz_path, "wb") as tgz_handle: - tgz = tar_compressor(name, fileobj=tgz_handle, compresslevel=compresslevel, - cache_path=self.conan_api.cache_folder) - for ref, ref_bundle in package_list.refs().items(): - ref_layout = cache.recipe_layout(ref) - recipe_folder = os.path.relpath(ref_layout.base_folder, cache_folder) - recipe_folder = recipe_folder.replace("\\", "/") # make win paths portable - ref_bundle["recipe_folder"] = recipe_folder - out.info(f"Saving {ref}: {recipe_folder}") - # Package only selected folders, not DOWNLOAD one - for f in (EXPORT_FOLDER, EXPORT_SRC_FOLDER, SRC_FOLDER): - if f == SRC_FOLDER and no_source: - continue - path = os.path.join(cache_folder, recipe_folder, f) - if os.path.exists(path): - tgz.add(path, f"{recipe_folder}/{f}", recursive=True) - path = os.path.join(cache_folder, recipe_folder, DOWNLOAD_EXPORT_FOLDER, METADATA) + for ref, ref_bundle in package_list.refs().items(): + ref_layout = cache.recipe_layout(ref) + recipe_folder = os.path.relpath(ref_layout.base_folder, cache_folder) + recipe_folder = recipe_folder.replace("\\", "/") # make win paths portable + ref_bundle["recipe_folder"] = recipe_folder + out.info(f"Saving {ref}: {recipe_folder}") + # Package only selected folders, not DOWNLOAD one + for f in (EXPORT_FOLDER, EXPORT_SRC_FOLDER, SRC_FOLDER): + if f == SRC_FOLDER and no_source: + continue + path = os.path.join(cache_folder, recipe_folder, f) if os.path.exists(path): - tgz.add(path, f"{recipe_folder}/{DOWNLOAD_EXPORT_FOLDER}/{METADATA}", - recursive=True) - - for pref, pref_bundle in package_list.prefs(ref, ref_bundle).items(): - pref_layout = cache.pkg_layout(pref) - pkg_folder = pref_layout.package() - folder = os.path.relpath(pkg_folder, cache_folder) - folder = folder.replace("\\", "/") # make win paths portable - pref_bundle["package_folder"] = folder - out.info(f"Saving {pref}: {folder}") - tgz.add(os.path.join(cache_folder, folder), folder, recursive=True) - if os.path.exists(pref_layout.metadata()): - metadata_folder = os.path.relpath(pref_layout.metadata(), cache_folder) - metadata_folder = metadata_folder.replace("\\", "/") # make paths portable - pref_bundle["metadata_folder"] = metadata_folder - out.info(f"Saving {pref} metadata: {metadata_folder}") - tgz.add(os.path.join(cache_folder, metadata_folder), metadata_folder, - recursive=True) - # Create pgklist.json to add it to the tgz - serialized = json.dumps(package_list.serialize(), indent=2) - pkglist_path = os.path.join(cache_folder, "pkglist.json") - with open(pkglist_path, "w") as file_handler: - file_handler.write(serialized) - tgz.add(pkglist_path, "pkglist.json", recursive=False) - tgz.close() + tar_files[f"{recipe_folder}/{f}"] = path + path = os.path.join(cache_folder, recipe_folder, DOWNLOAD_EXPORT_FOLDER, METADATA) + if os.path.exists(path): + tar_files[f"{recipe_folder}/{DOWNLOAD_EXPORT_FOLDER}/{METADATA}"] = path + + for pref, pref_bundle in package_list.prefs(ref, ref_bundle).items(): + pref_layout = cache.pkg_layout(pref) + pkg_folder = pref_layout.package() + folder = os.path.relpath(pkg_folder, cache_folder) + folder = folder.replace("\\", "/") # make win paths portable + pref_bundle["package_folder"] = folder + out.info(f"Saving {pref}: {folder}") + tar_files[folder] = os.path.join(cache_folder, folder) + + if os.path.exists(pref_layout.metadata()): + metadata_folder = os.path.relpath(pref_layout.metadata(), cache_folder) + metadata_folder = metadata_folder.replace("\\", "/") # make paths portable + pref_bundle["metadata_folder"] = metadata_folder + out.info(f"Saving {pref} metadata: {metadata_folder}") + tar_files[metadata_folder] = os.path.join(cache_folder, metadata_folder) + + # Create a temporary file in order to reuse compress_files functionality + serialized = json.dumps(package_list.serialize(), indent=2) + pkglist_path = os.path.join(tempfile.gettempdir(), "pkglist.json") + save(pkglist_path, serialized) + tar_files["pkglist.json"] = pkglist_path + tar_compress(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), compresslevel, + recursive=True, ref=None, cache_folder=self.conan_api.cache_folder) + remove(pkglist_path) def restore(self, path): if not os.path.isfile(path): diff --git a/conan/internal/util/compression.py b/conan/internal/util/compression.py index e811c84f46e..6a4f3e993d0 100644 --- a/conan/internal/util/compression.py +++ b/conan/internal/util/compression.py @@ -1,3 +1,4 @@ +from functools import lru_cache from conan.internal.cache.home_paths import HomePaths from conan.internal.loader import load_python_file from conan.internal.errors import ConanException @@ -24,7 +25,7 @@ def tar_extract(src_path, destination_dir, cache_folder=None): the_tar.close() -def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, cache_folder=None): +def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, cache_folder=None): compress_plugin = load_compress_plugin(cache_folder) if compress_plugin: return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) @@ -37,21 +38,13 @@ def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, cache_fold tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) for filename, abs_path in sorted(files.items()): # recursive is False in case it is a symlink to a folder - tgz.add(abs_path, filename, recursive=False) + tgz.add(abs_path, filename, recursive=recursive) tgz.close() duration = time.time() - t1 ConanOutput().debug(f"{name} compressed in {duration} time") return tgz_path -def tar_compressor(name, fileobj, compresslevel, cache_path=None): - compress_plugin = load_compress_plugin(cache_path) - if compress_plugin: - return compress_plugin.TarCompressor(name, fileobj, compresslevel) - else: - return gzopen_without_timestamps(name, fileobj, compresslevel) - - def gzopen_without_timestamps(name, fileobj, compresslevel=None): """ !! Method overrided by laso to pass mtime=0 (!=None) to avoid time.time() was setted in Gzip file causing md5 to change. Not possible using the @@ -67,6 +60,7 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): return t +@lru_cache(maxsize=1) def load_compress_plugin(cache_folder): if not cache_folder: return None @@ -83,9 +77,6 @@ def load_compress_plugin(cache_folder): """ Plugin `compression.py` interface: - def tar_extract(src_path, destination_dir) -> None - def tar_compress(files, name, dest_dir, compresslevel=None, ref=None) -> str - class TarCompressor(name, fileobj, compresslevel) - def add(self, abs_path, filename, recursive=True) -> None - def close() -> None + def tar_extract(src_path: str, destination_dir: str) -> None + def tar_compress(files: List[str], name: str, dest_dir: str, compresslevel=None, ref: str=None, cache_folder:str, recursive: bool = False) -> str """ From cc4d3add4326f1de0d4e22579e3c9c34b46e177a Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Mon, 19 May 2025 17:23:54 +0200 Subject: [PATCH 006/110] WIP --- conan/api/subapi/config.py | 2 ++ conan/internal/util/compression.py | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index f7342e475b2..5485181fcd4 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -3,6 +3,7 @@ import platform import textwrap import yaml +from conan.internal.util.compression import load_compress_plugin from jinja2 import Environment, FileSystemLoader from conan import conan_version @@ -31,6 +32,7 @@ def __init__(self, conan_api): self._new_config = None self._cli_core_confs = None self.hook_manager = HookManager(HomePaths(conan_api.home_folder).hooks_path) + self.compress_plugin = load_compress_plugin(conan_api.cache_folder) def home(self): return self.conan_api.cache_folder diff --git a/conan/internal/util/compression.py b/conan/internal/util/compression.py index 6a4f3e993d0..d1090beb299 100644 --- a/conan/internal/util/compression.py +++ b/conan/internal/util/compression.py @@ -60,7 +60,6 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): return t -@lru_cache(maxsize=1) def load_compress_plugin(cache_folder): if not cache_folder: return None From f094d9aec5633d1caf77b584ac0ea8b5aff8b5e1 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Tue, 20 May 2025 13:51:25 +0200 Subject: [PATCH 007/110] Moved plugin load to ConfigAPI --- conan/api/subapi/cache.py | 4 ++-- conan/api/subapi/config.py | 11 +++++++++-- conan/internal/api/uploader.py | 12 +++++++----- conan/internal/rest/remote_manager.py | 13 +++++++------ conan/internal/util/compression.py | 10 ++++------ 5 files changed, 29 insertions(+), 21 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index d019477b511..a0a5fd99086 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -172,7 +172,7 @@ def save(self, package_list, tgz_path, no_source=False): save(pkglist_path, serialized) tar_files["pkglist.json"] = pkglist_path tar_compress(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), compresslevel, - recursive=True, ref=None, cache_folder=self.conan_api.cache_folder) + recursive=True, ref=None, compress_plugin=self.conan_api.config.compress_plugin) remove(pkglist_path) def restore(self, path): @@ -181,7 +181,7 @@ def restore(self, path): cache = PkgCache(self.conan_api.cache_folder, self.conan_api.config.global_conf) cache_folder = cache.store # Note, this is not the home, but the actual package cache - tar_extract(path, cache_folder, cache_folder=self.conan_api.cache_folder) + tar_extract(path, cache_folder, compress_plugin=self.conan_api.config.compress_plugin) # Retrieve the package list from the already extracted archive with open(os.path.join(cache_folder, "pkglist.json")) as file_handler: pkglist = file_handler.read() diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index 5485181fcd4..3ccb3b0545e 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -3,7 +3,7 @@ import platform import textwrap import yaml -from conan.internal.util.compression import load_compress_plugin +from conan.internal.util.compression import load_compression_plugin from jinja2 import Environment, FileSystemLoader from conan import conan_version @@ -32,7 +32,7 @@ def __init__(self, conan_api): self._new_config = None self._cli_core_confs = None self.hook_manager = HookManager(HomePaths(conan_api.home_folder).hooks_path) - self.compress_plugin = load_compress_plugin(conan_api.cache_folder) + self._compress_plugin = None def home(self): return self.conan_api.cache_folder @@ -238,4 +238,11 @@ def reinit(self): if self._new_config is not None: self._new_config.clear() self._populate_global_conf() + self._compress_plugin = None self.hook_manager = HookManager(HomePaths(self.conan_api.home_folder).hooks_path) + + @property + def compression_plugin(self): + if not self._compress_plugin: + self._compress_plugin = load_compression_plugin(self.conan_api.cache_folder) + return self._compress_plugin diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 1a9bb70ab5c..3aaf93ca5bb 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -12,7 +12,7 @@ from conan.errors import ConanException from conan.internal.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO) -from conan.internal.util.compression import load_compress_plugin +from conan.internal.util.compression import load_compression_plugin from conan.internal.util.files import (clean_dirty, is_dirty, gather_files, set_dirty_context_manager, mkdir, human_size) @@ -158,7 +158,8 @@ def add_tgz(tgz_name, tgz_files): elif tgz_files: compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) tgz = compress_files(tgz_files, tgz_name, download_export_folder, - compresslevel=compresslevel, ref=ref, cache_folder=self._app.cache_folder) + compresslevel=compresslevel, ref=ref, + compress_plugin=self._app.conan_api.config.compress_plugin) result[tgz_name] = tgz add_tgz(EXPORT_TGZ_NAME, files) @@ -205,7 +206,8 @@ def _compress_package_files(self, layout, pref): tgz_files = {f: path for f, path in files.items()} compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) tgz_path = compress_files(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, - compresslevel=compresslevel, ref=pref, cache_folder=self._app.cache_folder) + compresslevel=compresslevel, ref=pref, + compress_plugin=self._app.conan_api.config.compress_plugin) assert tgz_path == package_tgz assert os.path.exists(package_tgz) @@ -272,8 +274,8 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): return t -def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, cache_folder=None): - compress_plugin = load_compress_plugin(cache_folder) +def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, + compress_plugin=None): if compress_plugin: return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index 19da098dbe6..4eda8e1b154 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -1,3 +1,4 @@ +from gzip import compress import os import shutil from typing import List @@ -15,7 +16,7 @@ from conan.internal.model.info import load_binary_info from conan.api.model import PkgReference from conan.api.model import RecipeReference -from conan.internal.util.compression import load_compress_plugin +from conan.internal.util.compression import load_compression_plugin from conan.internal.util.files import rmdir, human_size from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME from conan.internal.util.files import mkdir, tar_extract @@ -28,6 +29,7 @@ def __init__(self, cache, auth_manager, home_folder): self._auth_manager = auth_manager self._signer = PkgSignaturesPlugin(cache, home_folder) self._home_folder = home_folder + self._compression_plugin = load_compression_plugin(home_folder) # TODO: should use the instantiated one in ConfigAPI def _local_folder_remote(self, remote): if remote.remote_type == LOCAL_RECIPES_INDEX: @@ -82,7 +84,7 @@ def get_recipe(self, ref, remote, metadata=None): tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None) if tgz_file: - uncompress_file(tgz_file, export_folder, scope=str(ref), cache_folder=self._home_folder) + uncompress_file(tgz_file, export_folder, scope=str(ref), compression_plugin=self._compression_plugin) mkdir(export_folder) for file_name, file_path in zipped_files.items(): # copy CONANFILE shutil.move(file_path, os.path.join(export_folder, file_name)) @@ -124,7 +126,7 @@ def get_recipe_sources(self, ref, layout, remote): self._signer.verify(ref, download_folder, files=zipped_files) tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME] - uncompress_file(tgz_file, export_sources_folder, scope=str(ref), cache_folder=self._home_folder) + uncompress_file(tgz_file, export_sources_folder, scope=str(ref), compression_plugin=self._compression_plugin) def get_package(self, pref, remote, metadata=None): output = ConanOutput(scope=str(pref.ref)) @@ -172,7 +174,7 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata): tgz_file = zipped_files.pop(PACKAGE_TGZ_NAME, None) package_folder = layout.package() - uncompress_file(tgz_file, package_folder, scope=str(pref.ref), cache_folder=self._home_folder) + uncompress_file(tgz_file, package_folder, scope=str(pref.ref), compression_plugin=self._compression_plugin) mkdir(package_folder) # Just in case it doesn't exist, because uncompress did nothing for file_name, file_path in zipped_files.items(): # copy CONANINFO and CONANMANIFEST shutil.move(file_path, os.path.join(package_folder, file_name)) @@ -282,7 +284,7 @@ def _call_remote(self, remote, method, *args, **kwargs): raise ConanException(exc, remote=remote) -def uncompress_file(src_path, dest_folder, scope=None, cache_folder=None): +def uncompress_file(src_path, dest_folder, scope=None, compression_plugin=None): try: filesize = os.path.getsize(src_path) big_file = filesize > 10000000 # 10 MB @@ -290,7 +292,6 @@ def uncompress_file(src_path, dest_folder, scope=None, cache_folder=None): hs = human_size(filesize) ConanOutput(scope=scope).info(f"Decompressing {hs} {os.path.basename(src_path)}") - compression_plugin = load_compress_plugin(cache_folder) if compression_plugin: compression_plugin.tar_extract(src_path, dest_folder) else: diff --git a/conan/internal/util/compression.py b/conan/internal/util/compression.py index d1090beb299..f8ddedf55e5 100644 --- a/conan/internal/util/compression.py +++ b/conan/internal/util/compression.py @@ -10,8 +10,7 @@ from conan.api.output import ConanOutput from conan.internal.util.files import set_dirty_context_manager -def tar_extract(src_path, destination_dir, cache_folder=None): - compress_plugin = load_compress_plugin(cache_folder) +def tar_extract(src_path, destination_dir, compress_plugin=None): if compress_plugin: return compress_plugin.tar_extract(src_path, destination_dir) @@ -25,15 +24,14 @@ def tar_extract(src_path, destination_dir, cache_folder=None): the_tar.close() -def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, cache_folder=None): - compress_plugin = load_compress_plugin(cache_folder) +def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, compress_plugin=None): if compress_plugin: return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory tgz_path = os.path.join(dest_dir, name) - ConanOutput(scope=str(ref)).info(f"Compressing {name}") + ConanOutput(scope=str(ref) if ref else "").info(f"Compressing {name}") with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) for filename, abs_path in sorted(files.items()): @@ -60,7 +58,7 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): return t -def load_compress_plugin(cache_folder): +def load_compression_plugin(cache_folder): if not cache_folder: return None compression_plugin_path = HomePaths(cache_folder).compression_plugin_path From fee5fab5cfeb655bf3dd83d91ad5211f2923097a Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Tue, 20 May 2025 16:23:01 +0200 Subject: [PATCH 008/110] Pass config_api to remote_manager and local_recipe_index --- conan/api/subapi/cache.py | 4 ++-- conan/api/subapi/config.py | 10 +++++----- conan/internal/api/uploader.py | 12 +++++------- conan/internal/conan_app.py | 6 +++--- conan/internal/rest/remote_manager.py | 13 ++++++------- .../internal/rest/rest_client_local_recipe_index.py | 4 ++-- conan/internal/util/compression.py | 12 ++++++------ 7 files changed, 29 insertions(+), 32 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index a0a5fd99086..b78ecd8e196 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -172,7 +172,7 @@ def save(self, package_list, tgz_path, no_source=False): save(pkglist_path, serialized) tar_files["pkglist.json"] = pkglist_path tar_compress(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), compresslevel, - recursive=True, ref=None, compress_plugin=self.conan_api.config.compress_plugin) + recursive=True, ref=None, compression_plugin=self.conan_api.config.compression_plugin) remove(pkglist_path) def restore(self, path): @@ -181,7 +181,7 @@ def restore(self, path): cache = PkgCache(self.conan_api.cache_folder, self.conan_api.config.global_conf) cache_folder = cache.store # Note, this is not the home, but the actual package cache - tar_extract(path, cache_folder, compress_plugin=self.conan_api.config.compress_plugin) + tar_extract(path, cache_folder, compression_plugin=self.conan_api.config.compression_plugin) # Retrieve the package list from the already extracted archive with open(os.path.join(cache_folder, "pkglist.json")) as file_handler: pkglist = file_handler.read() diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index 3ccb3b0545e..bc4ab0be776 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -32,7 +32,7 @@ def __init__(self, conan_api): self._new_config = None self._cli_core_confs = None self.hook_manager = HookManager(HomePaths(conan_api.home_folder).hooks_path) - self._compress_plugin = None + self._compression_plugin = None def home(self): return self.conan_api.cache_folder @@ -238,11 +238,11 @@ def reinit(self): if self._new_config is not None: self._new_config.clear() self._populate_global_conf() - self._compress_plugin = None + self._compression_plugin = None self.hook_manager = HookManager(HomePaths(self.conan_api.home_folder).hooks_path) @property def compression_plugin(self): - if not self._compress_plugin: - self._compress_plugin = load_compression_plugin(self.conan_api.cache_folder) - return self._compress_plugin + if not self._compression_plugin: + self._compression_plugin = load_compression_plugin(self.conan_api.cache_folder) + return self._compression_plugin diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 3aaf93ca5bb..e3cca0ea177 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -12,7 +12,6 @@ from conan.errors import ConanException from conan.internal.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO) -from conan.internal.util.compression import load_compression_plugin from conan.internal.util.files import (clean_dirty, is_dirty, gather_files, set_dirty_context_manager, mkdir, human_size) @@ -159,7 +158,7 @@ def add_tgz(tgz_name, tgz_files): compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) tgz = compress_files(tgz_files, tgz_name, download_export_folder, compresslevel=compresslevel, ref=ref, - compress_plugin=self._app.conan_api.config.compress_plugin) + compression_plugin=self._app.conan_api.config.compression_plugin) result[tgz_name] = tgz add_tgz(EXPORT_TGZ_NAME, files) @@ -207,7 +206,7 @@ def _compress_package_files(self, layout, pref): compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) tgz_path = compress_files(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, compresslevel=compresslevel, ref=pref, - compress_plugin=self._app.conan_api.config.compress_plugin) + compression_plugin=self._app.conan_api.config.compression_plugin) assert tgz_path == package_tgz assert os.path.exists(package_tgz) @@ -274,10 +273,9 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): return t -def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, - compress_plugin=None): - if compress_plugin: - return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) +def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, compression_plugin=None): + if compression_plugin: + return compression_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory diff --git a/conan/internal/conan_app.py b/conan/internal/conan_app.py index 0ca4df974e2..9e1aa07a053 100644 --- a/conan/internal/conan_app.py +++ b/conan/internal/conan_app.py @@ -52,7 +52,7 @@ def __init__(self, conan_api): localdb = LocalDB(cache_folder) auth_manager = ConanApiAuthManager(conan_api.remotes.requester, cache_folder, localdb, global_conf) # Handle remote connections - self.remote_manager = RemoteManager(self.cache, auth_manager, cache_folder) + self.remote_manager = RemoteManager(self.cache, auth_manager, cache_folder, self.conan_api.config) global_editables = conan_api.local.editable_packages ws_editables = conan_api.workspace.editable_packages self.editable_packages = global_editables.update_copy(ws_editables) @@ -81,10 +81,10 @@ class LocalRecipesIndexApp: - loader (for the export phase of local-recipes-index) The others are internally use by other collaborators """ - def __init__(self, cache_folder): + def __init__(self, cache_folder, config_api): self.global_conf = ConfDefinition() self.cache = PkgCache(cache_folder, self.global_conf) - self.remote_manager = RemoteManager(self.cache, auth_manager=None, home_folder=cache_folder) + self.remote_manager = RemoteManager(self.cache, auth_manager=None, home_folder=cache_folder, config_api=config_api) editable_packages = EditablePackages() self.proxy = ConanProxy(self, editable_packages) self.range_resolver = RangeResolver(self, self.global_conf, editable_packages) diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index 4eda8e1b154..d3894accdb4 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -16,7 +16,6 @@ from conan.internal.model.info import load_binary_info from conan.api.model import PkgReference from conan.api.model import RecipeReference -from conan.internal.util.compression import load_compression_plugin from conan.internal.util.files import rmdir, human_size from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME from conan.internal.util.files import mkdir, tar_extract @@ -24,16 +23,16 @@ class RemoteManager: """ Will handle the remotes to get recipes, packages etc """ - def __init__(self, cache, auth_manager, home_folder): + def __init__(self, cache, auth_manager, home_folder, config_api): self._cache = cache self._auth_manager = auth_manager self._signer = PkgSignaturesPlugin(cache, home_folder) self._home_folder = home_folder - self._compression_plugin = load_compression_plugin(home_folder) # TODO: should use the instantiated one in ConfigAPI + self._config_api = config_api def _local_folder_remote(self, remote): if remote.remote_type == LOCAL_RECIPES_INDEX: - return RestApiClientLocalRecipesIndex(remote, self._home_folder) + return RestApiClientLocalRecipesIndex(remote, self._home_folder, self._config_api) def check_credentials(self, remote, force_auth=False): self._call_remote(remote, "check_credentials", force_auth) @@ -84,7 +83,7 @@ def get_recipe(self, ref, remote, metadata=None): tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None) if tgz_file: - uncompress_file(tgz_file, export_folder, scope=str(ref), compression_plugin=self._compression_plugin) + uncompress_file(tgz_file, export_folder, scope=str(ref), compression_plugin=self._config_api.compression_plugin) mkdir(export_folder) for file_name, file_path in zipped_files.items(): # copy CONANFILE shutil.move(file_path, os.path.join(export_folder, file_name)) @@ -126,7 +125,7 @@ def get_recipe_sources(self, ref, layout, remote): self._signer.verify(ref, download_folder, files=zipped_files) tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME] - uncompress_file(tgz_file, export_sources_folder, scope=str(ref), compression_plugin=self._compression_plugin) + uncompress_file(tgz_file, export_sources_folder, scope=str(ref), compression_plugin=self._config_api.compression_plugin) def get_package(self, pref, remote, metadata=None): output = ConanOutput(scope=str(pref.ref)) @@ -174,7 +173,7 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata): tgz_file = zipped_files.pop(PACKAGE_TGZ_NAME, None) package_folder = layout.package() - uncompress_file(tgz_file, package_folder, scope=str(pref.ref), compression_plugin=self._compression_plugin) + uncompress_file(tgz_file, package_folder, scope=str(pref.ref), compression_plugin=self._config_api.compression_plugin) mkdir(package_folder) # Just in case it doesn't exist, because uncompress did nothing for file_name, file_path in zipped_files.items(): # copy CONANINFO and CONANMANIFEST shutil.move(file_path, os.path.join(package_folder, file_name)) diff --git a/conan/internal/rest/rest_client_local_recipe_index.py b/conan/internal/rest/rest_client_local_recipe_index.py index 33082ca1356..83af84a263a 100644 --- a/conan/internal/rest/rest_client_local_recipe_index.py +++ b/conan/internal/rest/rest_client_local_recipe_index.py @@ -58,14 +58,14 @@ class RestApiClientLocalRecipesIndex: a local folder assuming the conan-center-index repo layout """ - def __init__(self, remote, home_folder): + def __init__(self, remote, home_folder, config_api): self._remote = remote local_recipes_index_path = HomePaths(home_folder).local_recipes_index_path local_recipes_index_path = os.path.join(local_recipes_index_path, remote.name, ".conan") repo_folder = self._remote.url from conan.internal.conan_app import LocalRecipesIndexApp - self._app = LocalRecipesIndexApp(local_recipes_index_path) + self._app = LocalRecipesIndexApp(local_recipes_index_path, config_api) self._hook_manager = HookManager(HomePaths(local_recipes_index_path).hooks_path) self._layout = _LocalRecipesIndexLayout(repo_folder) diff --git a/conan/internal/util/compression.py b/conan/internal/util/compression.py index f8ddedf55e5..9ebe0af96ce 100644 --- a/conan/internal/util/compression.py +++ b/conan/internal/util/compression.py @@ -10,9 +10,9 @@ from conan.api.output import ConanOutput from conan.internal.util.files import set_dirty_context_manager -def tar_extract(src_path, destination_dir, compress_plugin=None): - if compress_plugin: - return compress_plugin.tar_extract(src_path, destination_dir) +def tar_extract(src_path, destination_dir, compression_plugin=None): + if compression_plugin: + return compression_plugin.tar_extract(src_path, destination_dir) with open(src_path, mode='rb') as file_handler: the_tar = tarfile.open(fileobj=file_handler) @@ -24,9 +24,9 @@ def tar_extract(src_path, destination_dir, compress_plugin=None): the_tar.close() -def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, compress_plugin=None): - if compress_plugin: - return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) +def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, compression_plugin=None): + if compression_plugin: + return compression_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory From 452f7746b5d793cdd32e3f66e3c26dcaf73d45fa Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Tue, 20 May 2025 16:34:52 +0200 Subject: [PATCH 009/110] Restore previous tar_extract fixing tests --- conan/internal/rest/remote_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index d3894accdb4..10f6617d068 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -294,7 +294,8 @@ def uncompress_file(src_path, dest_folder, scope=None, compression_plugin=None): if compression_plugin: compression_plugin.tar_extract(src_path, dest_folder) else: - tar_extract(src_path, dest_folder) + with open(src_path, mode='rb') as file_handler: + tar_extract(file_handler, dest_folder) except Exception as e: error_msg = "Error while extracting downloaded file '%s' to %s\n%s\n"\ % (src_path, dest_folder, str(e)) From c1a73204c088773d4f1f4b7be23bfc4336adb79b Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Thu, 22 May 2025 17:17:27 +0200 Subject: [PATCH 010/110] Removed compression.py module and minimize diff --- conan/api/subapi/cache.py | 18 ++- conan/api/subapi/config.py | 13 ++- conan/internal/api/uploader.py | 4 +- conan/internal/util/compression.py | 79 ------------- .../extensions/test_compression_plugin.py | 106 ++++++++++++++++++ 5 files changed, 132 insertions(+), 88 deletions(-) delete mode 100644 conan/internal/util/compression.py create mode 100644 test/integration/extensions/test_compression_plugin.py diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index b78ecd8e196..025f2b56d2a 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -5,6 +5,7 @@ from conan.api.model import PackagesList from conan.api.output import ConanOutput +from conan.internal.api.uploader import compress_files from conan.internal.cache.cache import PkgCache from conan.internal.cache.conan_reference_layout import EXPORT_SRC_FOLDER, EXPORT_FOLDER, SRC_FOLDER, \ METADATA, DOWNLOAD_EXPORT_FOLDER @@ -15,8 +16,7 @@ from conan.api.model import PkgReference from conan.api.model import RecipeReference from conan.internal.util.dates import revision_timestamp_now -from conan.internal.util.files import rmdir, mkdir, remove, save -from conan.internal.util.compression import tar_compress, tar_extract +from conan.internal.util.files import rmdir, mkdir, remove, save, tar_extract class CacheAPI: @@ -171,8 +171,9 @@ def save(self, package_list, tgz_path, no_source=False): pkglist_path = os.path.join(tempfile.gettempdir(), "pkglist.json") save(pkglist_path, serialized) tar_files["pkglist.json"] = pkglist_path - tar_compress(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), compresslevel, - recursive=True, ref=None, compression_plugin=self.conan_api.config.compression_plugin) + print(tar_files) + compress_files(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), compresslevel, + recursive=True, ref=None, compression_plugin=self.conan_api.config.compression_plugin) remove(pkglist_path) def restore(self, path): @@ -181,7 +182,14 @@ def restore(self, path): cache = PkgCache(self.conan_api.cache_folder, self.conan_api.config.global_conf) cache_folder = cache.store # Note, this is not the home, but the actual package cache - tar_extract(path, cache_folder, compression_plugin=self.conan_api.config.compression_plugin) + + compression_plugin = self.conan_api.config.compression_plugin + if compression_plugin: + compression_plugin.tar_extract(path, cache_folder) + else: + with open(path, mode='rb') as file_handler: + tar_extract(file_handler, cache_folder) + # Retrieve the package list from the already extracted archive with open(os.path.join(cache_folder, "pkglist.json")) as file_handler: pkglist = file_handler.read() diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index bc4ab0be776..9540da485d9 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -3,7 +3,7 @@ import platform import textwrap import yaml -from conan.internal.util.compression import load_compression_plugin +from conan.internal.loader import load_python_file from jinja2 import Environment, FileSystemLoader from conan import conan_version @@ -243,6 +243,15 @@ def reinit(self): @property def compression_plugin(self): + def load_compression_plugin(): + compression_plugin_path = HomePaths(self.conan_api.home_folder).compression_plugin_path + if not os.path.exists(compression_plugin_path): + return None + mod, _ = load_python_file(compression_plugin_path) + if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"): + raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions") + return mod if not self._compression_plugin: - self._compression_plugin = load_compression_plugin(self.conan_api.cache_folder) + self._compression_plugin = load_compression_plugin() return self._compression_plugin + diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index e3cca0ea177..31c14ef2a49 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -275,12 +275,12 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, compression_plugin=None): if compression_plugin: - return compression_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) + return compression_plugin.tar_compress(files, name, dest_dir, compresslevel, ref, recursive) t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory tgz_path = os.path.join(dest_dir, name) - ConanOutput(scope=str(ref) if ref else "").info(f"Compressing {name}") + ConanOutput(scope=str(ref or "")).info(f"Compressing {name}") with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) for filename, abs_path in sorted(files.items()): diff --git a/conan/internal/util/compression.py b/conan/internal/util/compression.py deleted file mode 100644 index 9ebe0af96ce..00000000000 --- a/conan/internal/util/compression.py +++ /dev/null @@ -1,79 +0,0 @@ -from functools import lru_cache -from conan.internal.cache.home_paths import HomePaths -from conan.internal.loader import load_python_file -from conan.internal.errors import ConanException - -import os -import gzip -import time -import tarfile -from conan.api.output import ConanOutput -from conan.internal.util.files import set_dirty_context_manager - -def tar_extract(src_path, destination_dir, compression_plugin=None): - if compression_plugin: - return compression_plugin.tar_extract(src_path, destination_dir) - - with open(src_path, mode='rb') as file_handler: - the_tar = tarfile.open(fileobj=file_handler) - # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to - # "could not change modification time", with time=0 - # the_tar.errorlevel = 2 # raise exception if any error - the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break - the_tar.extractall(path=destination_dir) - the_tar.close() - - -def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, compression_plugin=None): - if compression_plugin: - return compression_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) - - t1 = time.time() - # FIXME, better write to disk sequentially and not keep tgz contents in memory - tgz_path = os.path.join(dest_dir, name) - ConanOutput(scope=str(ref) if ref else "").info(f"Compressing {name}") - with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: - tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) - for filename, abs_path in sorted(files.items()): - # recursive is False in case it is a symlink to a folder - tgz.add(abs_path, filename, recursive=recursive) - tgz.close() - - duration = time.time() - t1 - ConanOutput().debug(f"{name} compressed in {duration} time") - return tgz_path - -def gzopen_without_timestamps(name, fileobj, compresslevel=None): - """ !! Method overrided by laso to pass mtime=0 (!=None) to avoid time.time() was - setted in Gzip file causing md5 to change. Not possible using the - previous tarfile open because arguments are not passed to GzipFile constructor - """ - compresslevel = compresslevel if compresslevel is not None else 9 # default Gzip = 9 - fileobj = gzip.GzipFile(name, "w", compresslevel, fileobj, mtime=0) - # Format is forced because in Python3.8, it changed and it generates different tarfiles - # with different checksums, which break hashes of tgzs - # PAX_FORMAT is the default for Py38, lets make it explicit for older Python versions - t = tarfile.TarFile.taropen(name, "w", fileobj, format=tarfile.PAX_FORMAT) - t._extfileobj = False - return t - - -def load_compression_plugin(cache_folder): - if not cache_folder: - return None - compression_plugin_path = HomePaths(cache_folder).compression_plugin_path - if not os.path.exists(compression_plugin_path): - return None - - mod, _ = load_python_file(compression_plugin_path) - if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"): - raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions") - return mod - - -""" -Plugin `compression.py` interface: - - def tar_extract(src_path: str, destination_dir: str) -> None - def tar_compress(files: List[str], name: str, dest_dir: str, compresslevel=None, ref: str=None, cache_folder:str, recursive: bool = False) -> str -""" diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py new file mode 100644 index 00000000000..6dc5ec21b5c --- /dev/null +++ b/test/integration/extensions/test_compression_plugin.py @@ -0,0 +1,106 @@ +import os +import textwrap + +from conan.test.assets.genconanfile import GenConanfile +from conan.test.utils.tools import TestClient + + +def test_compression_plugin_not_existing(): + """Test that the compression plugin is not used if it does not exist""" + c = TestClient() + c.save({"conanfile.py": GenConanfile("pkg", "1.0")}) + c.run("create .") + c.run("cache save 'pkg/*'") + assert "Compressing conan_cache_save.tgz\n" in c.out + c.run("cache restore conan_cache_save.tgz") + # Default decompress does not have any output + assert "Decompressing conan_cache_save.tgz" not in c.out + + +def test_compression_plugin_not_valid(): + """Test an error is raised if the compression plugin is not valid""" + + c = TestClient() + compression_plugin = textwrap.dedent( + """ + def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False): + return None + """ + ) + + c.save( + { + os.path.join( + c.cache_folder, "extensions", "plugins", "compression.py" + ): compression_plugin, + "conanfile.py": GenConanfile("pkg", "1.0"), + } + ) + c.run("create .") + c.run("cache save 'pkg/*'", assert_error=True) + assert ( + "ERROR: The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions" + in c.out + ) + + +def test_compression_plugin_correctly_load(): + """Test that the compression plugin is correctly loaded and used on: + - cache save/restore + - remote upload/download + """ + c = TestClient(default_server_user=True) + + compression_plugin = textwrap.dedent( + """ + import os + import tarfile + from conan.api.output import ConanOutput + + # xz compression + def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, *args, **kwargs): + tgz_path = os.path.join(dest_dir, name) + ConanOutput(scope=str(ref) if ref else "").info(f"Compressing {name} using compression plugin (xz)") + kwargs = {"preset": compresslevel} if compresslevel else {} + with tarfile.open(tgz_path, f"w:xz", **kwargs) as tgz: + for filename, abs_path in sorted(files.items()): + tgz.add(abs_path, filename, recursive=True) + return tgz_path + + def tar_extract(src_path, destination_dir, *args, **kwargs): + ConanOutput().info(f"Decompressing {os.path.basename(src_path)} using compression plugin (xz)") + with open(src_path, mode='rb') as file_handler: + the_tar = tarfile.open(fileobj=file_handler) + the_tar.extraction_filter = (lambda member, path: member) + the_tar.extractall(path=destination_dir) + the_tar.close() + """ + ) + + c.save( + { + os.path.join( + c.cache_folder, "extensions", "plugins", "compression.py" + ): compression_plugin, + "conanfile.py": GenConanfile("pkg", "1.0"), + } + ) + c.run("create .") + c.run("cache save 'pkg/*'") + assert "Compressing conan_cache_save.tgz using compression plugin (xz)" in c.out + c.run("remove pkg/* -c") + c.run("cache restore conan_cache_save.tgz") + assert "Decompressing conan_cache_save.tgz using compression plugin (xz)" in c.out + c.run("list pkg/1.0") + assert "Found 1 pkg/version recipes matching pkg/1.0 in local cache" in c.out + + # Remove pre existing tgz to force a recompression + c.run("remove pkg/* -c") + c.run("create .") + # Check the plugin is also used on remote interactions + c.run("upload * -r=default -c") + assert "Compressing conan_package.tgz using compression plugin (xz)" in c.out + assert "pkg/1.0: Uploading recipe" in c.out + c.run("remove pkg/* -c") + c.run("download 'pkg/*' -r=default") + assert "Decompressing conan_package.tgz using compression plugin (xz)" in c.out From a240c8c619766eb8a14d9f77343ce3821f604169 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Thu, 22 May 2025 17:25:30 +0200 Subject: [PATCH 011/110] Remove debug print --- conan/api/subapi/cache.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index 025f2b56d2a..1198ebe5924 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -171,7 +171,6 @@ def save(self, package_list, tgz_path, no_source=False): pkglist_path = os.path.join(tempfile.gettempdir(), "pkglist.json") save(pkglist_path, serialized) tar_files["pkglist.json"] = pkglist_path - print(tar_files) compress_files(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), compresslevel, recursive=True, ref=None, compression_plugin=self.conan_api.config.compression_plugin) remove(pkglist_path) From cf7de747bf3f66ce25ae95e7f9accfd4af8654ff Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Fri, 23 May 2025 10:03:37 +0200 Subject: [PATCH 012/110] Applied thread suggestions --- conan/api/subapi/config.py | 6 ++---- .../command/cache/test_cache_save_restore.py | 4 ++++ .../extensions/test_compression_plugin.py | 12 ------------ 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index 9540da485d9..9577d86c962 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -243,15 +243,13 @@ def reinit(self): @property def compression_plugin(self): - def load_compression_plugin(): + if not self._compression_plugin: compression_plugin_path = HomePaths(self.conan_api.home_folder).compression_plugin_path if not os.path.exists(compression_plugin_path): return None mod, _ = load_python_file(compression_plugin_path) if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"): raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions") - return mod - if not self._compression_plugin: - self._compression_plugin = load_compression_plugin() + self._compression_plugin = mod return self._compression_plugin diff --git a/test/integration/command/cache/test_cache_save_restore.py b/test/integration/command/cache/test_cache_save_restore.py index a6bc91b495d..113a0dd0ac5 100644 --- a/test/integration/command/cache/test_cache_save_restore.py +++ b/test/integration/command/cache/test_cache_save_restore.py @@ -158,9 +158,13 @@ def test_cache_save_excluded_folders(): # exclude source c.run("cache save * --no-source") + # Check default compression function is being used and not compression.py plugin one + assert "Compressing conan_cache_save.tgz\n" in c.out c3 = TestClient() shutil.copy2(cache_path, c3.current_folder) c3.run("cache restore conan_cache_save.tgz") + # Default decompress does not have any output + assert "Decompressing conan_cache_save.tgz" not in c3.out ref_layout = c3.get_latest_ref_layout(ref) assert not os.path.exists(os.path.join(ref_layout.source(), "mysrc.c")) diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py index 6dc5ec21b5c..cee4c3b7079 100644 --- a/test/integration/extensions/test_compression_plugin.py +++ b/test/integration/extensions/test_compression_plugin.py @@ -5,18 +5,6 @@ from conan.test.utils.tools import TestClient -def test_compression_plugin_not_existing(): - """Test that the compression plugin is not used if it does not exist""" - c = TestClient() - c.save({"conanfile.py": GenConanfile("pkg", "1.0")}) - c.run("create .") - c.run("cache save 'pkg/*'") - assert "Compressing conan_cache_save.tgz\n" in c.out - c.run("cache restore conan_cache_save.tgz") - # Default decompress does not have any output - assert "Decompressing conan_cache_save.tgz" not in c.out - - def test_compression_plugin_not_valid(): """Test an error is raised if the compression plugin is not valid""" From 557f8e001f42110087dbf760d1ffd2a64783f35e Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Fri, 23 May 2025 11:31:38 +0200 Subject: [PATCH 013/110] Remove created pkglist.json on cache restore after usage --- conan/api/subapi/cache.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index 1198ebe5924..024fa6de68b 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -190,8 +190,11 @@ def restore(self, path): tar_extract(file_handler, cache_folder) # Retrieve the package list from the already extracted archive - with open(os.path.join(cache_folder, "pkglist.json")) as file_handler: + pkglist_path = os.path.join(cache_folder, "pkglist.json") + with open(pkglist_path) as file_handler: pkglist = file_handler.read() + # Delete the pkglist.json file to keep cache clean + remove(pkglist_path) # After unzipping the files, we need to update the DB that references these files out = ConanOutput() From f82d764a2d86b4af76b105488de7834893782aa3 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Mon, 26 May 2025 11:09:27 +0200 Subject: [PATCH 014/110] Remove unused and avoid rechecking FS --- conan/api/subapi/config.py | 7 ++++--- conan/internal/rest/remote_manager.py | 1 - 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index 9577d86c962..c509753e832 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -32,7 +32,6 @@ def __init__(self, conan_api): self._new_config = None self._cli_core_confs = None self.hook_manager = HookManager(HomePaths(conan_api.home_folder).hooks_path) - self._compression_plugin = None def home(self): return self.conan_api.cache_folder @@ -238,14 +237,16 @@ def reinit(self): if self._new_config is not None: self._new_config.clear() self._populate_global_conf() - self._compression_plugin = None + if hasattr(self, "_compression_plugin"): + del self._compression_plugin self.hook_manager = HookManager(HomePaths(self.conan_api.home_folder).hooks_path) @property def compression_plugin(self): - if not self._compression_plugin: + if not hasattr(self, "_compression_plugin"): compression_plugin_path = HomePaths(self.conan_api.home_folder).compression_plugin_path if not os.path.exists(compression_plugin_path): + self._compression_plugin = None return None mod, _ = load_python_file(compression_plugin_path) if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"): diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index 10f6617d068..3dce1518b02 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -1,4 +1,3 @@ -from gzip import compress import os import shutil from typing import List From ac1fc45695fec2acd78f30588448c25a9c9a19fa Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Mon, 26 May 2025 19:47:44 +0200 Subject: [PATCH 015/110] Added test to check extract failure and issue #18259 test --- conan/api/subapi/config.py | 10 +- conan/internal/rest/remote_manager.py | 2 +- conan/internal/util/files.py | 19 +-- .../extensions/test_compression_plugin.py | 108 ++++++++++++++++++ test/unittests/util/files/tar_extract_test.py | 2 + 5 files changed, 128 insertions(+), 13 deletions(-) diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index c509753e832..d03c275b0c2 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -32,6 +32,7 @@ def __init__(self, conan_api): self._new_config = None self._cli_core_confs = None self.hook_manager = HookManager(HomePaths(conan_api.home_folder).hooks_path) + self._compression_plugin = None def home(self): return self.conan_api.cache_folder @@ -237,17 +238,16 @@ def reinit(self): if self._new_config is not None: self._new_config.clear() self._populate_global_conf() - if hasattr(self, "_compression_plugin"): - del self._compression_plugin + self._compression_plugin = None self.hook_manager = HookManager(HomePaths(self.conan_api.home_folder).hooks_path) @property def compression_plugin(self): - if not hasattr(self, "_compression_plugin"): + if self._compression_plugin is None: compression_plugin_path = HomePaths(self.conan_api.home_folder).compression_plugin_path if not os.path.exists(compression_plugin_path): - self._compression_plugin = None - return None + self._compression_plugin = False # Avoid FS re-check + return False mod, _ = load_python_file(compression_plugin_path) if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"): raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions") diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index 3dce1518b02..14b45d089d0 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -282,7 +282,7 @@ def _call_remote(self, remote, method, *args, **kwargs): raise ConanException(exc, remote=remote) -def uncompress_file(src_path, dest_folder, scope=None, compression_plugin=None): +def uncompress_file(src_path, dest_folder, scope="", compression_plugin=None): try: filesize = os.path.getsize(src_path) big_file = filesize > 10000000 # 10 MB diff --git a/conan/internal/util/files.py b/conan/internal/util/files.py index 98746479ae0..2cd89b90cb5 100644 --- a/conan/internal/util/files.py +++ b/conan/internal/util/files.py @@ -257,13 +257,18 @@ def mkdir(path): def tar_extract(fileobj, destination_dir): - the_tar = tarfile.open(fileobj=fileobj) - # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to - # "could not change modification time", with time=0 - # the_tar.errorlevel = 2 # raise exception if any error - the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break - the_tar.extractall(path=destination_dir) - the_tar.close() + try: + the_tar = tarfile.open(fileobj=fileobj) + # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to + # "could not change modification time", with time=0 + # the_tar.errorlevel = 2 # raise exception if any error + the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break + the_tar.extractall(path=destination_dir) + the_tar.close() + except tarfile.ReadError: + raise ConanException(f"Error while extracting {os.path.basename(fileobj.name)}. The file compression is not recogniced.\n" + "This file could have been compressed using a `compression` plugin.\n" + "If your organization uses this plugin, ensure it is correctly installed on your environment.") def merge_directories(src, dst): diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py index cee4c3b7079..31a1e206e9e 100644 --- a/test/integration/extensions/test_compression_plugin.py +++ b/test/integration/extensions/test_compression_plugin.py @@ -1,6 +1,7 @@ import os import textwrap +from conan.internal.util.files import tar_extract from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient @@ -92,3 +93,110 @@ def tar_extract(src_path, destination_dir, *args, **kwargs): c.run("remove pkg/* -c") c.run("download 'pkg/*' -r=default") assert "Decompressing conan_package.tgz using compression plugin (xz)" in c.out + + +def test_compression_plugin_tar_not_compatible_with_builtin(): + """ + Test that built in tar_extract function fails when uncompressing a non compatible file (a file + which has been compressed using the compression plugin with a different algorithm than the built-in one). + """ + c = TestClient(default_server_user=True) + + compression_plugin = textwrap.dedent( + """ + import os + import zipfile + from conan.api.output import ConanOutput + + # zip compression + def tar_extract(src_path, destination_dir): + # extract tar using zipfile library + ConanOutput().info(f"Decompressing {src_path} using compression plugin (zip)") + with zipfile.ZipFile(src_path, 'r') as zip_ref: + zip_ref.extractall(destination_dir) + + def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False): + # compress files using zipfile library taking into account recursive + zip_path = os.path.join(dest_dir, name) + ConanOutput(scope=str(ref) if ref else "").info(f"Compressing {name} using compression plugin (zip)") + with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED, compresslevel=compresslevel) as zipf: + for filename, abs_path in sorted(files.items()): + if recursive: + arcname = os.path.relpath(abs_path, start=os.path.dirname(abs_path)) + zipf.write(abs_path, arcname) + else: + zipf.write(abs_path, filename) + return zip_path + """ + ) + + c.save( + { + os.path.join( + c.cache_folder, "extensions", "plugins", "compression.py" + ): compression_plugin, + "conanfile.py": GenConanfile("pkg", "1.0"), + } + ) + c.run("create .") + c.run("cache save 'pkg/*'") + c.run("remove pkg/* -c") + os.unlink(os.path.join(c.cache_folder, "extensions", "plugins", "compression.py")) + c.run("cache restore conan_cache_save.tgz", assert_error=True) + assert ( + "Error while extracting conan_cache_save.tgz. The file compression is not recogniced.\n" + "This file could have been compressed using a `compression` plugin.\n" + "If your organization uses this plugin, ensure it is correctly installed on your environment." + ) in c.out + + +# https://github.com/conan-io/conan/issues/18259 +def test_compress_in_subdirectory(): + c = TestClient(default_server_user=True) + compression_plugin = textwrap.dedent( + """ + import os + import tarfile + from conan.api.output import ConanOutput + def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, *args, **kwargs): + # compress files using tarfile putting all content in a `conan/` subfolder + tgz_path = os.path.join(dest_dir, name) + ConanOutput(scope=str(ref or "")).info(f"Compressing {name} in conan subfolder") + with open(tgz_path, "wb") as tgz_handle: + tgz = tarfile.open(name, "w", fileobj=tgz_handle) + for filename, abs_path in sorted(files.items()): + tgz.add(abs_path, os.path.join("conan", filename), recursive=recursive) + tgz.close() + return tgz_path + + def tar_extract(src_path, destination_dir, *args, **kwargs): + ConanOutput().info(f"Decompressing {src_path} in conan subfolder") + with open(src_path, mode="rb") as file_handler: + the_tar = tarfile.open(fileobj=file_handler) + the_tar.extraction_filter = (lambda member, path: member) + for member in the_tar.getmembers(): + if member.name.startswith("conan/"): + member.name = member.name[len("conan/"):] # Strip 'conan/' prefix + the_tar.extract(member, path=destination_dir) + the_tar.close() + """ + ) + c.save( + { + os.path.join( + c.cache_folder, "extensions", "plugins", "compression.py" + ): compression_plugin, + "conanfile.py": GenConanfile("pkg", "1.0"), + } + ) + c.run("create .") + c.run("cache save 'pkg/*'") + c.run("remove pkg/* -c") + c.run("cache restore conan_cache_save.tgz") + with open(os.path.join(c.current_folder, "conan_cache_save.tgz"), 'rb') as file_handler: + destination_dir = os.path.join(c.cache_folder, "extracted") + tar_extract(file_handler, destination_dir) + assert os.listdir(destination_dir) == ["conan"] + assert os.path.exists(os.path.join(destination_dir, "conan", "pkglist.json")) + + diff --git a/test/unittests/util/files/tar_extract_test.py b/test/unittests/util/files/tar_extract_test.py index 6f031a56a13..9ca622dbb9d 100644 --- a/test/unittests/util/files/tar_extract_test.py +++ b/test/unittests/util/files/tar_extract_test.py @@ -60,3 +60,5 @@ def check_files(destination_dir): with open(self.tgz_file, 'rb') as file_handler: tar_extract(file_handler, destination_dir) check_files(destination_dir) + + From 181a7366336882124b7a333eae689f0bcc92d611 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Tue, 27 May 2025 11:24:54 +0200 Subject: [PATCH 016/110] Added config to compression.py interface and renamed parameters --- conan/api/subapi/cache.py | 3 +- conan/api/subapi/config.py | 5 +- conan/internal/api/uploader.py | 20 ++++-- conan/internal/rest/remote_manager.py | 13 ++-- .../extensions/test_compression_plugin.py | 70 +++++++++---------- 5 files changed, 58 insertions(+), 53 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index 024fa6de68b..ca2cf67c907 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -184,7 +184,8 @@ def restore(self, path): compression_plugin = self.conan_api.config.compression_plugin if compression_plugin: - compression_plugin.tar_extract(path, cache_folder) + compression_plugin.tar_extract(archive_path=path, dest_dir=cache_folder, + config=self.conan_api.config.global_conf) else: with open(path, mode='rb') as file_handler: tar_extract(file_handler, cache_folder) diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index d03c275b0c2..7b8aba3eb2c 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -247,10 +247,9 @@ def compression_plugin(self): compression_plugin_path = HomePaths(self.conan_api.home_folder).compression_plugin_path if not os.path.exists(compression_plugin_path): self._compression_plugin = False # Avoid FS re-check - return False + return None mod, _ = load_python_file(compression_plugin_path) if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"): raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions") self._compression_plugin = mod - return self._compression_plugin - + return self._compression_plugin or None diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 31c14ef2a49..44585516704 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -155,9 +155,8 @@ def add_tgz(tgz_name, tgz_files): if os.path.isfile(tgz): result[tgz_name] = tgz elif tgz_files: - compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) tgz = compress_files(tgz_files, tgz_name, download_export_folder, - compresslevel=compresslevel, ref=ref, + config=self._global_conf, ref=ref, compression_plugin=self._app.conan_api.config.compression_plugin) result[tgz_name] = tgz @@ -203,9 +202,8 @@ def _compress_package_files(self, layout, pref): if not os.path.isfile(package_tgz): tgz_files = {f: path for f, path in files.items()} - compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) tgz_path = compress_files(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, - compresslevel=compresslevel, ref=pref, + config=self._global_conf, ref=pref, compression_plugin=self._app.conan_api.config.compression_plugin) assert tgz_path == package_tgz assert os.path.exists(package_tgz) @@ -273,15 +271,23 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): return t -def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, compression_plugin=None): +def compress_files(files, name, dest_dir, config=None, ref=None, recursive=False, compression_plugin=None): + tgz_path = os.path.join(dest_dir, name) if compression_plugin: - return compression_plugin.tar_compress(files, name, dest_dir, compresslevel, ref, recursive) + compression_plugin.tar_compress( + archive_path=tgz_path, + files=files, + recursive=recursive, + config=config, + ref=ref, + ) + return tgz_path t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory - tgz_path = os.path.join(dest_dir, name) ConanOutput(scope=str(ref or "")).info(f"Compressing {name}") with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: + compresslevel = config.get("core.gzip:compresslevel", check_type=int) if config else None tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) for filename, abs_path in sorted(files.items()): # recursive is False by default in case it is a symlink to a folder diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index 14b45d089d0..01e7bfaa552 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -82,7 +82,7 @@ def get_recipe(self, ref, remote, metadata=None): tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None) if tgz_file: - uncompress_file(tgz_file, export_folder, scope=str(ref), compression_plugin=self._config_api.compression_plugin) + uncompress_file(tgz_file, export_folder, scope=str(ref), config_api=self._config_api) mkdir(export_folder) for file_name, file_path in zipped_files.items(): # copy CONANFILE shutil.move(file_path, os.path.join(export_folder, file_name)) @@ -124,7 +124,7 @@ def get_recipe_sources(self, ref, layout, remote): self._signer.verify(ref, download_folder, files=zipped_files) tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME] - uncompress_file(tgz_file, export_sources_folder, scope=str(ref), compression_plugin=self._config_api.compression_plugin) + uncompress_file(tgz_file, export_sources_folder, scope=str(ref), config_api=self._config_api) def get_package(self, pref, remote, metadata=None): output = ConanOutput(scope=str(pref.ref)) @@ -172,7 +172,7 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata): tgz_file = zipped_files.pop(PACKAGE_TGZ_NAME, None) package_folder = layout.package() - uncompress_file(tgz_file, package_folder, scope=str(pref.ref), compression_plugin=self._config_api.compression_plugin) + uncompress_file(tgz_file, package_folder, scope=str(pref.ref), config_api=self._config_api) mkdir(package_folder) # Just in case it doesn't exist, because uncompress did nothing for file_name, file_path in zipped_files.items(): # copy CONANINFO and CONANMANIFEST shutil.move(file_path, os.path.join(package_folder, file_name)) @@ -282,7 +282,7 @@ def _call_remote(self, remote, method, *args, **kwargs): raise ConanException(exc, remote=remote) -def uncompress_file(src_path, dest_folder, scope="", compression_plugin=None): +def uncompress_file(src_path, dest_folder, scope="", config_api=None): try: filesize = os.path.getsize(src_path) big_file = filesize > 10000000 # 10 MB @@ -290,8 +290,9 @@ def uncompress_file(src_path, dest_folder, scope="", compression_plugin=None): hs = human_size(filesize) ConanOutput(scope=scope).info(f"Decompressing {hs} {os.path.basename(src_path)}") - if compression_plugin: - compression_plugin.tar_extract(src_path, dest_folder) + if config_api and config_api.compression_plugin: + config_api.compression_plugin.tar_extract(archive_path=src_path, dest_dir=dest_folder, + config=config_api.global_conf) else: with open(src_path, mode='rb') as file_handler: tar_extract(file_handler, dest_folder) diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py index 31a1e206e9e..0bb21e3414b 100644 --- a/test/integration/extensions/test_compression_plugin.py +++ b/test/integration/extensions/test_compression_plugin.py @@ -12,8 +12,8 @@ def test_compression_plugin_not_valid(): c = TestClient() compression_plugin = textwrap.dedent( """ - def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False): - return None + def tar_compress(archive_path, files, recursive, config=None, *args, **kwargs): + pass """ ) @@ -47,21 +47,21 @@ def test_compression_plugin_correctly_load(): from conan.api.output import ConanOutput # xz compression - def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, *args, **kwargs): - tgz_path = os.path.join(dest_dir, name) - ConanOutput(scope=str(ref) if ref else "").info(f"Compressing {name} using compression plugin (xz)") + def tar_compress(archive_path, files, recursive, config=None, *args, **kwargs): + name = os.path.basename(archive_path) + ConanOutput().info(f"Compressing {name} using compression plugin (xz)") + compresslevel = config.get("core.gzip:compresslevel", check_type=int) if config else None kwargs = {"preset": compresslevel} if compresslevel else {} - with tarfile.open(tgz_path, f"w:xz", **kwargs) as tgz: + with tarfile.open(archive_path, f"w:xz", **kwargs) as tgz: for filename, abs_path in sorted(files.items()): tgz.add(abs_path, filename, recursive=True) - return tgz_path - def tar_extract(src_path, destination_dir, *args, **kwargs): - ConanOutput().info(f"Decompressing {os.path.basename(src_path)} using compression plugin (xz)") - with open(src_path, mode='rb') as file_handler: + def tar_extract(archive_path, dest_dir, config=None, *args, **kwargs): + ConanOutput().info(f"Decompressing {os.path.basename(archive_path)} using compression plugin (xz)") + with open(archive_path, mode='rb') as file_handler: the_tar = tarfile.open(fileobj=file_handler) the_tar.extraction_filter = (lambda member, path: member) - the_tar.extractall(path=destination_dir) + the_tar.extractall(path=dest_dir) the_tar.close() """ ) @@ -109,24 +109,24 @@ def test_compression_plugin_tar_not_compatible_with_builtin(): from conan.api.output import ConanOutput # zip compression - def tar_extract(src_path, destination_dir): - # extract tar using zipfile library - ConanOutput().info(f"Decompressing {src_path} using compression plugin (zip)") - with zipfile.ZipFile(src_path, 'r') as zip_ref: - zip_ref.extractall(destination_dir) - - def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False): + def tar_compress(archive_path, files, recursive, config=None, *args, **kwargs): # compress files using zipfile library taking into account recursive - zip_path = os.path.join(dest_dir, name) - ConanOutput(scope=str(ref) if ref else "").info(f"Compressing {name} using compression plugin (zip)") - with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED, compresslevel=compresslevel) as zipf: + name = os.path.basename(archive_path) + compresslevel = config.get("core.gzip:compresslevel", check_type=int) if config else None + ConanOutput().info(f"Compressing {name} using compression plugin (zip)") + with zipfile.ZipFile(archive_path, 'w', zipfile.ZIP_DEFLATED, compresslevel=compresslevel) as zipf: for filename, abs_path in sorted(files.items()): if recursive: arcname = os.path.relpath(abs_path, start=os.path.dirname(abs_path)) zipf.write(abs_path, arcname) else: zipf.write(abs_path, filename) - return zip_path + + def tar_extract(archive_path, dest_dir, config=None, *args, **kwargs): + # extract tar using zipfile library + ConanOutput().info(f"Decompressing {os.path.basename(archive_path)} using compression plugin (zip)") + with zipfile.ZipFile(archive_path, 'r') as zip_ref: + zip_ref.extractall(dest_dir) """ ) @@ -158,26 +158,25 @@ def test_compress_in_subdirectory(): import os import tarfile from conan.api.output import ConanOutput - def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, *args, **kwargs): + def tar_compress(archive_path, files, recursive, *args, **kwargs): # compress files using tarfile putting all content in a `conan/` subfolder - tgz_path = os.path.join(dest_dir, name) - ConanOutput(scope=str(ref or "")).info(f"Compressing {name} in conan subfolder") - with open(tgz_path, "wb") as tgz_handle: + name = os.path.basename(archive_path) + ConanOutput().info(f"Compressing {os.path.basename(name)} in conan subfolder") + with open(archive_path, "wb") as tgz_handle: tgz = tarfile.open(name, "w", fileobj=tgz_handle) for filename, abs_path in sorted(files.items()): tgz.add(abs_path, os.path.join("conan", filename), recursive=recursive) tgz.close() - return tgz_path - def tar_extract(src_path, destination_dir, *args, **kwargs): - ConanOutput().info(f"Decompressing {src_path} in conan subfolder") - with open(src_path, mode="rb") as file_handler: + def tar_extract(archive_path, dest_dir, *args, **kwargs): + ConanOutput().info(f"Decompressing {archive_path} in conan subfolder") + with open(archive_path, mode="rb") as file_handler: the_tar = tarfile.open(fileobj=file_handler) the_tar.extraction_filter = (lambda member, path: member) for member in the_tar.getmembers(): if member.name.startswith("conan/"): member.name = member.name[len("conan/"):] # Strip 'conan/' prefix - the_tar.extract(member, path=destination_dir) + the_tar.extract(member, path=dest_dir) the_tar.close() """ ) @@ -194,9 +193,8 @@ def tar_extract(src_path, destination_dir, *args, **kwargs): c.run("remove pkg/* -c") c.run("cache restore conan_cache_save.tgz") with open(os.path.join(c.current_folder, "conan_cache_save.tgz"), 'rb') as file_handler: - destination_dir = os.path.join(c.cache_folder, "extracted") - tar_extract(file_handler, destination_dir) - assert os.listdir(destination_dir) == ["conan"] - assert os.path.exists(os.path.join(destination_dir, "conan", "pkglist.json")) - + dest_dir = os.path.join(c.cache_folder, "extracted") + tar_extract(file_handler, dest_dir) + assert os.listdir(dest_dir) == ["conan"] + assert os.path.exists(os.path.join(dest_dir, "conan", "pkglist.json")) From 3b32ec20f991b1bee1c5b6e9f4e4c606062e241f Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Tue, 27 May 2025 11:54:31 +0200 Subject: [PATCH 017/110] Fix invokation --- conan/api/subapi/cache.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index ca2cf67c907..afa6d1195de 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -130,7 +130,6 @@ def save(self, package_list, tgz_path, no_source=False): cache_folder = cache.store # Note, this is not the home, but the actual package cache out = ConanOutput() mkdir(os.path.dirname(tgz_path)) - compresslevel = global_conf.get("core.gzip:compresslevel", check_type=int) tar_files: dict[str,str] = {} # {path_in_tar: abs_path} for ref, ref_bundle in package_list.refs().items(): @@ -171,7 +170,7 @@ def save(self, package_list, tgz_path, no_source=False): pkglist_path = os.path.join(tempfile.gettempdir(), "pkglist.json") save(pkglist_path, serialized) tar_files["pkglist.json"] = pkglist_path - compress_files(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), compresslevel, + compress_files(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), config=self.conan_api.config, recursive=True, ref=None, compression_plugin=self.conan_api.config.compression_plugin) remove(pkglist_path) From b52bce2ffd0be2376a427dd06e7ec643ff9226a4 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Tue, 27 May 2025 12:24:20 +0200 Subject: [PATCH 018/110] Rename config for conf --- conan/api/subapi/cache.py | 4 ++-- conan/internal/api/uploader.py | 10 +++++----- conan/internal/rest/remote_manager.py | 2 +- .../extensions/test_compression_plugin.py | 14 +++++++------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index afa6d1195de..753229ae15f 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -170,7 +170,7 @@ def save(self, package_list, tgz_path, no_source=False): pkglist_path = os.path.join(tempfile.gettempdir(), "pkglist.json") save(pkglist_path, serialized) tar_files["pkglist.json"] = pkglist_path - compress_files(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), config=self.conan_api.config, + compress_files(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), conf=self.conan_api.config, recursive=True, ref=None, compression_plugin=self.conan_api.config.compression_plugin) remove(pkglist_path) @@ -184,7 +184,7 @@ def restore(self, path): compression_plugin = self.conan_api.config.compression_plugin if compression_plugin: compression_plugin.tar_extract(archive_path=path, dest_dir=cache_folder, - config=self.conan_api.config.global_conf) + conf=self.conan_api.config.global_conf) else: with open(path, mode='rb') as file_handler: tar_extract(file_handler, cache_folder) diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 44585516704..5b0738ca509 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -156,7 +156,7 @@ def add_tgz(tgz_name, tgz_files): result[tgz_name] = tgz elif tgz_files: tgz = compress_files(tgz_files, tgz_name, download_export_folder, - config=self._global_conf, ref=ref, + conf=self._global_conf, ref=ref, compression_plugin=self._app.conan_api.config.compression_plugin) result[tgz_name] = tgz @@ -203,7 +203,7 @@ def _compress_package_files(self, layout, pref): if not os.path.isfile(package_tgz): tgz_files = {f: path for f, path in files.items()} tgz_path = compress_files(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, - config=self._global_conf, ref=pref, + conf=self._global_conf, ref=pref, compression_plugin=self._app.conan_api.config.compression_plugin) assert tgz_path == package_tgz assert os.path.exists(package_tgz) @@ -271,14 +271,14 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): return t -def compress_files(files, name, dest_dir, config=None, ref=None, recursive=False, compression_plugin=None): +def compress_files(files, name, dest_dir, conf=None, ref=None, recursive=False, compression_plugin=None): tgz_path = os.path.join(dest_dir, name) if compression_plugin: compression_plugin.tar_compress( archive_path=tgz_path, files=files, recursive=recursive, - config=config, + conf=conf, ref=ref, ) return tgz_path @@ -287,7 +287,7 @@ def compress_files(files, name, dest_dir, config=None, ref=None, recursive=False # FIXME, better write to disk sequentially and not keep tgz contents in memory ConanOutput(scope=str(ref or "")).info(f"Compressing {name}") with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: - compresslevel = config.get("core.gzip:compresslevel", check_type=int) if config else None + compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) for filename, abs_path in sorted(files.items()): # recursive is False by default in case it is a symlink to a folder diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index 01e7bfaa552..a95214826a7 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -292,7 +292,7 @@ def uncompress_file(src_path, dest_folder, scope="", config_api=None): if config_api and config_api.compression_plugin: config_api.compression_plugin.tar_extract(archive_path=src_path, dest_dir=dest_folder, - config=config_api.global_conf) + conf=config_api.global_conf) else: with open(src_path, mode='rb') as file_handler: tar_extract(file_handler, dest_folder) diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py index 0bb21e3414b..cb179d020fd 100644 --- a/test/integration/extensions/test_compression_plugin.py +++ b/test/integration/extensions/test_compression_plugin.py @@ -12,7 +12,7 @@ def test_compression_plugin_not_valid(): c = TestClient() compression_plugin = textwrap.dedent( """ - def tar_compress(archive_path, files, recursive, config=None, *args, **kwargs): + def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): pass """ ) @@ -47,16 +47,16 @@ def test_compression_plugin_correctly_load(): from conan.api.output import ConanOutput # xz compression - def tar_compress(archive_path, files, recursive, config=None, *args, **kwargs): + def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): name = os.path.basename(archive_path) ConanOutput().info(f"Compressing {name} using compression plugin (xz)") - compresslevel = config.get("core.gzip:compresslevel", check_type=int) if config else None + compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None kwargs = {"preset": compresslevel} if compresslevel else {} with tarfile.open(archive_path, f"w:xz", **kwargs) as tgz: for filename, abs_path in sorted(files.items()): tgz.add(abs_path, filename, recursive=True) - def tar_extract(archive_path, dest_dir, config=None, *args, **kwargs): + def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): ConanOutput().info(f"Decompressing {os.path.basename(archive_path)} using compression plugin (xz)") with open(archive_path, mode='rb') as file_handler: the_tar = tarfile.open(fileobj=file_handler) @@ -109,10 +109,10 @@ def test_compression_plugin_tar_not_compatible_with_builtin(): from conan.api.output import ConanOutput # zip compression - def tar_compress(archive_path, files, recursive, config=None, *args, **kwargs): + def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): # compress files using zipfile library taking into account recursive name = os.path.basename(archive_path) - compresslevel = config.get("core.gzip:compresslevel", check_type=int) if config else None + compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None ConanOutput().info(f"Compressing {name} using compression plugin (zip)") with zipfile.ZipFile(archive_path, 'w', zipfile.ZIP_DEFLATED, compresslevel=compresslevel) as zipf: for filename, abs_path in sorted(files.items()): @@ -122,7 +122,7 @@ def tar_compress(archive_path, files, recursive, config=None, *args, **kwargs): else: zipf.write(abs_path, filename) - def tar_extract(archive_path, dest_dir, config=None, *args, **kwargs): + def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): # extract tar using zipfile library ConanOutput().info(f"Decompressing {os.path.basename(archive_path)} using compression plugin (zip)") with zipfile.ZipFile(archive_path, 'r') as zip_ref: From 91da7a4b5e5b8478201d05273fcf23ddb9f3a433 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Tue, 27 May 2025 14:23:29 +0200 Subject: [PATCH 019/110] Added ref on test --- test/integration/extensions/test_compression_plugin.py | 4 ++-- test/unittests/util/files/tar_extract_test.py | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py index cb179d020fd..43b64b369dd 100644 --- a/test/integration/extensions/test_compression_plugin.py +++ b/test/integration/extensions/test_compression_plugin.py @@ -47,9 +47,9 @@ def test_compression_plugin_correctly_load(): from conan.api.output import ConanOutput # xz compression - def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): + def tar_compress(archive_path, files, recursive, conf=None, ref=None, *args, **kwargs): name = os.path.basename(archive_path) - ConanOutput().info(f"Compressing {name} using compression plugin (xz)") + ConanOutput(scope=ref).info(f"Compressing {name} using compression plugin (xz)") compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None kwargs = {"preset": compresslevel} if compresslevel else {} with tarfile.open(archive_path, f"w:xz", **kwargs) as tgz: diff --git a/test/unittests/util/files/tar_extract_test.py b/test/unittests/util/files/tar_extract_test.py index 9ca622dbb9d..6f031a56a13 100644 --- a/test/unittests/util/files/tar_extract_test.py +++ b/test/unittests/util/files/tar_extract_test.py @@ -60,5 +60,3 @@ def check_files(destination_dir): with open(self.tgz_file, 'rb') as file_handler: tar_extract(file_handler, destination_dir) check_files(destination_dir) - - From 989fde99776b773973b79e2a3ad6b2bd8a5e0679 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Fri, 20 Jun 2025 09:47:50 +0200 Subject: [PATCH 020/110] Move to different approach: tar encapsulation respecting extensions --- conan/api/subapi/cache.py | 14 ++--- conan/internal/api/uploader.py | 20 +++++-- conan/internal/rest/remote_manager.py | 13 ++--- conan/internal/util/files.py | 52 ++++++++++++++----- .../extensions/test_compression_plugin.py | 37 ++++++------- 5 files changed, 90 insertions(+), 46 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index 753229ae15f..addab4b6a85 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -181,13 +181,13 @@ def restore(self, path): cache = PkgCache(self.conan_api.cache_folder, self.conan_api.config.global_conf) cache_folder = cache.store # Note, this is not the home, but the actual package cache - compression_plugin = self.conan_api.config.compression_plugin - if compression_plugin: - compression_plugin.tar_extract(archive_path=path, dest_dir=cache_folder, - conf=self.conan_api.config.global_conf) - else: - with open(path, mode='rb') as file_handler: - tar_extract(file_handler, cache_folder) + with open(path, mode="rb") as file_handler: + tar_extract( + fileobj=file_handler, + destination_dir=cache_folder, + compression_plugin=self.conan_api.config.compression_plugin, + conf=self.conan_api.config.global_conf, + ) # Retrieve the package list from the already extracted archive pkglist_path = os.path.join(cache_folder, "pkglist.json") diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 5b0738ca509..abd1ce3322d 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -4,6 +4,7 @@ import shutil import tarfile import time +from pathlib import Path from conan.internal.conan_app import ConanApp from conan.api.output import ConanOutput @@ -12,7 +13,7 @@ from conan.errors import ConanException from conan.internal.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO) -from conan.internal.util.files import (clean_dirty, is_dirty, gather_files, +from conan.internal.util.files import (COMPRESSED_PLUGIN_TAR_NAME, clean_dirty, is_dirty, gather_files, remove, set_dirty_context_manager, mkdir, human_size) UPLOAD_POLICY_FORCE = "force-upload" @@ -274,13 +275,26 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): def compress_files(files, name, dest_dir, conf=None, ref=None, recursive=False, compression_plugin=None): tgz_path = os.path.join(dest_dir, name) if compression_plugin: - compression_plugin.tar_compress( - archive_path=tgz_path, + t1 = time.time() + compressed_path = compression_plugin.tar_compress( + archive_path=os.path.join(dest_dir, COMPRESSED_PLUGIN_TAR_NAME), files=files, recursive=recursive, conf=conf, ref=ref, ) + ConanOutput().debug(f"{name} compressed in {time.time() - t1} time in plugin") + ConanOutput(scope=str(ref or "")).info(f"Compressing {compressed_path}") + t1 = time.time() + ConanOutput().debug(f"Wrapping {compressed_path} in {name}") + with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: + tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=0) + tgz.add(compressed_path, arcname=os.path.basename(compressed_path), recursive=recursive) + tgz.close() + ConanOutput().debug(f"{name} wrapped in {time.time() - t1} time") + # Only remove wrapped if it is different from the tgz_path + if compressed_path != os.path.basename(tgz_path): + remove(compressed_path) return tgz_path t1 = time.time() diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index 13ccf6e5efe..9d4d70bf817 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -304,12 +304,13 @@ def uncompress_file(src_path, dest_folder, scope="", config_api=None): hs = human_size(filesize) ConanOutput(scope=scope).info(f"Decompressing {hs} {os.path.basename(src_path)}") - if config_api and config_api.compression_plugin: - config_api.compression_plugin.tar_extract(archive_path=src_path, dest_dir=dest_folder, - conf=config_api.global_conf) - else: - with open(src_path, mode='rb') as file_handler: - tar_extract(file_handler, dest_folder) + with open(src_path, mode='rb') as file_handler: + tar_extract( + fileobj=file_handler, + destination_dir=dest_folder, + compression_plugin=config_api.compression_plugin if config_api and config_api.compression_plugin else None, + conf=config_api.global_conf if config_api else None + ) except Exception as e: error_msg = "Error while extracting downloaded file '%s' to %s\n%s\n"\ % (src_path, dest_folder, str(e)) diff --git a/conan/internal/util/files.py b/conan/internal/util/files.py index 2cd89b90cb5..a5c172aae20 100644 --- a/conan/internal/util/files.py +++ b/conan/internal/util/files.py @@ -1,4 +1,6 @@ import errno +from pathlib import Path +import tempfile import gzip import hashlib import os @@ -11,10 +13,13 @@ from contextlib import contextmanager +from conan.api.output import ConanOutput from conan.errors import ConanException _DIRTY_FOLDER = ".dirty" +# Name (without extension) of the tar file to be created by the compression plugin +COMPRESSED_PLUGIN_TAR_NAME = "__conan_plugin_compressed_contents__" def set_dirty(folder): dirty_file = os.path.normpath(folder) + _DIRTY_FOLDER @@ -256,20 +261,43 @@ def mkdir(path): os.makedirs(path) -def tar_extract(fileobj, destination_dir): - try: - the_tar = tarfile.open(fileobj=fileobj) - # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to - # "could not change modification time", with time=0 - # the_tar.errorlevel = 2 # raise exception if any error - the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break - the_tar.extractall(path=destination_dir) - the_tar.close() - except tarfile.ReadError: - raise ConanException(f"Error while extracting {os.path.basename(fileobj.name)}. The file compression is not recogniced.\n" - "This file could have been compressed using a `compression` plugin.\n" +def tar_extract(fileobj, destination_dir, compression_plugin=None, conf=None): + if compression_plugin: + _tar_extract_with_plugin(fileobj, destination_dir, compression_plugin, conf) + return + the_tar = tarfile.open(fileobj=fileobj) + # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to + # "could not change modification time", with time=0 + # the_tar.errorlevel = 2 # raise exception if any error + the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break + the_tar.extractall(path=destination_dir) + the_tar.close() + if Path(destination_dir).glob(f"{COMPRESSED_PLUGIN_TAR_NAME}.*"): + raise ConanException(f"Error while extracting {os.path.basename(fileobj.name)}.\n" + "This file has been compressed using a `compression` plugin.\n" "If your organization uses this plugin, ensure it is correctly installed on your environment.") +def _tar_extract_with_plugin(fileobj, destination_dir, compression_plugin, conf): + """First remove tar.gz wrapper and then call the plugin to extract""" + with tempfile.TemporaryDirectory() as temp_dir: + t1 = time.time() + the_tar = tarfile.open(fileobj=fileobj) + the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break + the_tar.extractall(path=temp_dir) + # Check if the tar was compressed with the compression plugin by checking the existence of + # our constant COMPRESSED_PLUGIN_TAR_NAME (without extension as extension is added by the plugin) + if list(Path(temp_dir).glob(f"{COMPRESSED_PLUGIN_TAR_NAME}.*")): + # Get the only extracted file: the plugin tar + plugin_tar_path = os.path.join(temp_dir, the_tar.getnames()[0]) + the_tar.close() + ConanOutput().debug(f"Unwrapped in {time.time() - t1} time") + t1 = time.time() + compression_plugin.tar_extract(archive_path=plugin_tar_path, dest_dir=destination_dir, conf=conf) + ConanOutput().debug(f"Extracted in {time.time() - t1} time on plugin") + else: + # The tar was not compressed using the plugin, copy files to destination + from conan.tools.files import copy + copy(None, pattern="*", src=temp_dir, dst=destination_dir) def merge_directories(src, dst): from conan.tools.files import copy diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py index 43b64b369dd..82900518c22 100644 --- a/test/integration/extensions/test_compression_plugin.py +++ b/test/integration/extensions/test_compression_plugin.py @@ -1,7 +1,7 @@ import os import textwrap -from conan.internal.util.files import tar_extract +from conan.internal.util.files import COMPRESSED_PLUGIN_TAR_NAME, tar_extract from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient @@ -26,7 +26,7 @@ def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): } ) c.run("create .") - c.run("cache save 'pkg/*'", assert_error=True) + c.run("cache save 'pkg/*:*'", assert_error=True) assert ( "ERROR: The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions" in c.out @@ -48,6 +48,7 @@ def test_compression_plugin_correctly_load(): # xz compression def tar_compress(archive_path, files, recursive, conf=None, ref=None, *args, **kwargs): + archive_path += ".xz" name = os.path.basename(archive_path) ConanOutput(scope=ref).info(f"Compressing {name} using compression plugin (xz)") compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None @@ -55,6 +56,7 @@ def tar_compress(archive_path, files, recursive, conf=None, ref=None, *args, **k with tarfile.open(archive_path, f"w:xz", **kwargs) as tgz: for filename, abs_path in sorted(files.items()): tgz.add(abs_path, filename, recursive=True) + return archive_path def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): ConanOutput().info(f"Decompressing {os.path.basename(archive_path)} using compression plugin (xz)") @@ -75,11 +77,12 @@ def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): } ) c.run("create .") - c.run("cache save 'pkg/*'") - assert "Compressing conan_cache_save.tgz using compression plugin (xz)" in c.out + c.run("cache save 'pkg/*:*'") + print(c.out) + assert f"Compressing {COMPRESSED_PLUGIN_TAR_NAME}.xz using compression plugin (xz)" in c.out c.run("remove pkg/* -c") c.run("cache restore conan_cache_save.tgz") - assert "Decompressing conan_cache_save.tgz using compression plugin (xz)" in c.out + assert f"Decompressing {COMPRESSED_PLUGIN_TAR_NAME}.xz using compression plugin (xz)" in c.out c.run("list pkg/1.0") assert "Found 1 pkg/version recipes matching pkg/1.0 in local cache" in c.out @@ -87,12 +90,12 @@ def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): c.run("remove pkg/* -c") c.run("create .") # Check the plugin is also used on remote interactions - c.run("upload * -r=default -c") - assert "Compressing conan_package.tgz using compression plugin (xz)" in c.out + c.run("upload *:* -r=default -c") + assert f"Compressing {COMPRESSED_PLUGIN_TAR_NAME}.xz using compression plugin (xz)" in c.out assert "pkg/1.0: Uploading recipe" in c.out c.run("remove pkg/* -c") c.run("download 'pkg/*' -r=default") - assert "Decompressing conan_package.tgz using compression plugin (xz)" in c.out + assert f"Decompressing {COMPRESSED_PLUGIN_TAR_NAME}.xz using compression plugin (xz)" in c.out def test_compression_plugin_tar_not_compatible_with_builtin(): @@ -111,6 +114,7 @@ def test_compression_plugin_tar_not_compatible_with_builtin(): # zip compression def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): # compress files using zipfile library taking into account recursive + archive_path += ".zip" name = os.path.basename(archive_path) compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None ConanOutput().info(f"Compressing {name} using compression plugin (zip)") @@ -121,6 +125,7 @@ def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): zipf.write(abs_path, arcname) else: zipf.write(abs_path, filename) + return archive_path def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): # extract tar using zipfile library @@ -139,13 +144,13 @@ def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): } ) c.run("create .") - c.run("cache save 'pkg/*'") + c.run("cache save 'pkg/*:*'") c.run("remove pkg/* -c") os.unlink(os.path.join(c.cache_folder, "extensions", "plugins", "compression.py")) c.run("cache restore conan_cache_save.tgz", assert_error=True) assert ( - "Error while extracting conan_cache_save.tgz. The file compression is not recogniced.\n" - "This file could have been compressed using a `compression` plugin.\n" + "Error while extracting conan_cache_save.tgz.\n" + "This file has been compressed using a `compression` plugin.\n" "If your organization uses this plugin, ensure it is correctly installed on your environment." ) in c.out @@ -160,6 +165,7 @@ def test_compress_in_subdirectory(): from conan.api.output import ConanOutput def tar_compress(archive_path, files, recursive, *args, **kwargs): # compress files using tarfile putting all content in a `conan/` subfolder + archive_path += ".tgz" name = os.path.basename(archive_path) ConanOutput().info(f"Compressing {os.path.basename(name)} in conan subfolder") with open(archive_path, "wb") as tgz_handle: @@ -167,6 +173,7 @@ def tar_compress(archive_path, files, recursive, *args, **kwargs): for filename, abs_path in sorted(files.items()): tgz.add(abs_path, os.path.join("conan", filename), recursive=recursive) tgz.close() + return archive_path def tar_extract(archive_path, dest_dir, *args, **kwargs): ConanOutput().info(f"Decompressing {archive_path} in conan subfolder") @@ -189,12 +196,6 @@ def tar_extract(archive_path, dest_dir, *args, **kwargs): } ) c.run("create .") - c.run("cache save 'pkg/*'") + c.run("cache save 'pkg/*:*'") c.run("remove pkg/* -c") c.run("cache restore conan_cache_save.tgz") - with open(os.path.join(c.current_folder, "conan_cache_save.tgz"), 'rb') as file_handler: - dest_dir = os.path.join(c.cache_folder, "extracted") - tar_extract(file_handler, dest_dir) - assert os.listdir(dest_dir) == ["conan"] - assert os.path.exists(os.path.join(dest_dir, "conan", "pkglist.json")) - From f986651d0afe3867afd27d8376b469e43e315db9 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Mon, 23 Jun 2025 17:50:41 +0200 Subject: [PATCH 021/110] Fix condition error --- conan/internal/api/uploader.py | 1 - conan/internal/util/files.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index abd1ce3322d..c1f945fcf31 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -4,7 +4,6 @@ import shutil import tarfile import time -from pathlib import Path from conan.internal.conan_app import ConanApp from conan.api.output import ConanOutput diff --git a/conan/internal/util/files.py b/conan/internal/util/files.py index a5c172aae20..b7985c733fe 100644 --- a/conan/internal/util/files.py +++ b/conan/internal/util/files.py @@ -272,7 +272,7 @@ def tar_extract(fileobj, destination_dir, compression_plugin=None, conf=None): the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break the_tar.extractall(path=destination_dir) the_tar.close() - if Path(destination_dir).glob(f"{COMPRESSED_PLUGIN_TAR_NAME}.*"): + if list(Path(destination_dir).glob(f"{COMPRESSED_PLUGIN_TAR_NAME}.*")): raise ConanException(f"Error while extracting {os.path.basename(fileobj.name)}.\n" "This file has been compressed using a `compression` plugin.\n" "If your organization uses this plugin, ensure it is correctly installed on your environment.") @@ -297,6 +297,7 @@ def _tar_extract_with_plugin(fileobj, destination_dir, compression_plugin, conf) else: # The tar was not compressed using the plugin, copy files to destination from conan.tools.files import copy + ConanOutput().debug(f"Extracted in {time.time() - t1} time built in") copy(None, pattern="*", src=temp_dir, dst=destination_dir) def merge_directories(src, dst): From 8009bed44ed708357e4b6e5f915e45f6260aed12 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Wed, 25 Jun 2025 10:39:15 +0200 Subject: [PATCH 022/110] Addressed some issues --- conan/internal/api/uploader.py | 53 +++++++++++-------- conan/internal/util/files.py | 6 ++- .../extensions/test_compression_plugin.py | 31 ++++++++++- 3 files changed, 64 insertions(+), 26 deletions(-) diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index c1f945fcf31..24a49a6c911 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -272,30 +272,10 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): def compress_files(files, name, dest_dir, conf=None, ref=None, recursive=False, compression_plugin=None): - tgz_path = os.path.join(dest_dir, name) if compression_plugin: - t1 = time.time() - compressed_path = compression_plugin.tar_compress( - archive_path=os.path.join(dest_dir, COMPRESSED_PLUGIN_TAR_NAME), - files=files, - recursive=recursive, - conf=conf, - ref=ref, - ) - ConanOutput().debug(f"{name} compressed in {time.time() - t1} time in plugin") - ConanOutput(scope=str(ref or "")).info(f"Compressing {compressed_path}") - t1 = time.time() - ConanOutput().debug(f"Wrapping {compressed_path} in {name}") - with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: - tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=0) - tgz.add(compressed_path, arcname=os.path.basename(compressed_path), recursive=recursive) - tgz.close() - ConanOutput().debug(f"{name} wrapped in {time.time() - t1} time") - # Only remove wrapped if it is different from the tgz_path - if compressed_path != os.path.basename(tgz_path): - remove(compressed_path) - return tgz_path + return _compress_files_with_plugin(files, name, dest_dir, conf, ref, recursive, compression_plugin) + tgz_path = os.path.join(dest_dir, name) t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory ConanOutput(scope=str(ref or "")).info(f"Compressing {name}") @@ -311,6 +291,35 @@ def compress_files(files, name, dest_dir, conf=None, ref=None, recursive=False, ConanOutput().debug(f"{name} compressed in {duration} time") return tgz_path +def _compress_files_with_plugin(files, name, dest_dir, conf, ref, recursive, compression_plugin): + t1 = time.time() + abs_path_without_extension = os.path.join(dest_dir, COMPRESSED_PLUGIN_TAR_NAME) + ConanOutput(scope=str(ref or "")).info(f"Compressing {name} using compression plugin") + compressed_path = compression_plugin.tar_compress( + archive_path=abs_path_without_extension, + files=files, + recursive=recursive, + conf=conf, + ref=ref, + ) + ConanOutput().debug(f"Compressed {compressed_path} in {time.time() - t1} time") + # Check if compressed_path == abs_path_without_extension + .* (any extension) + path, extension = os.path.splitext(compressed_path) + if path != abs_path_without_extension or not extension: + raise ConanException("The 'compression.py' plugin returned an unexpected path. Plugin should return the 'archive_path' with an extra extension") + + t1 = time.time() + ConanOutput().debug(f"Wrapping {compressed_path} in {name}") + tgz_path = os.path.join(dest_dir, name) + with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: + tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=0) + tgz.add(compressed_path, arcname=os.path.basename(compressed_path), recursive=recursive) + tgz.close() + ConanOutput().debug(f"{name} wrapped in {time.time() - t1} time") + # Only remove wrapped if it is different from the tgz_path + if compressed_path != os.path.basename(tgz_path): + remove(compressed_path) + return tgz_path def _total_size(cache_files): total_size = 0 diff --git a/conan/internal/util/files.py b/conan/internal/util/files.py index b7985c733fe..36bbe80daaa 100644 --- a/conan/internal/util/files.py +++ b/conan/internal/util/files.py @@ -265,6 +265,7 @@ def tar_extract(fileobj, destination_dir, compression_plugin=None, conf=None): if compression_plugin: _tar_extract_with_plugin(fileobj, destination_dir, compression_plugin, conf) return + the_tar = tarfile.open(fileobj=fileobj) # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to # "could not change modification time", with time=0 @@ -284,12 +285,13 @@ def _tar_extract_with_plugin(fileobj, destination_dir, compression_plugin, conf) the_tar = tarfile.open(fileobj=fileobj) the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break the_tar.extractall(path=temp_dir) + extracted_file = the_tar.getnames()[0] + the_tar.close() # Check if the tar was compressed with the compression plugin by checking the existence of # our constant COMPRESSED_PLUGIN_TAR_NAME (without extension as extension is added by the plugin) if list(Path(temp_dir).glob(f"{COMPRESSED_PLUGIN_TAR_NAME}.*")): # Get the only extracted file: the plugin tar - plugin_tar_path = os.path.join(temp_dir, the_tar.getnames()[0]) - the_tar.close() + plugin_tar_path = os.path.join(temp_dir, extracted_file) ConanOutput().debug(f"Unwrapped in {time.time() - t1} time") t1 = time.time() compression_plugin.tar_extract(archive_path=plugin_tar_path, dest_dir=destination_dir, conf=conf) diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py index 82900518c22..96144d95aa9 100644 --- a/test/integration/extensions/test_compression_plugin.py +++ b/test/integration/extensions/test_compression_plugin.py @@ -1,7 +1,7 @@ import os import textwrap -from conan.internal.util.files import COMPRESSED_PLUGIN_TAR_NAME, tar_extract +from conan.internal.util.files import COMPRESSED_PLUGIN_TAR_NAME from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient @@ -32,6 +32,34 @@ def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): in c.out ) +def test_compression_plugin_returning_invalid_path(): + """Test an error is raised if the compression plugin does not return expected path""" + + c = TestClient() + compression_plugin = textwrap.dedent( + """ + def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): + return archive_path + def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): + pass + """ + ) + + c.save( + { + os.path.join( + c.cache_folder, "extensions", "plugins", "compression.py" + ): compression_plugin, + "conanfile.py": GenConanfile("pkg", "1.0"), + } + ) + c.run("create .") + c.run("cache save 'pkg/*:*'", assert_error=True) + assert ( + "ERROR: The 'compression.py' plugin returned an unexpected path. Plugin should return the 'archive_path' with an extra extension" + in c.out + ) + def test_compression_plugin_correctly_load(): """Test that the compression plugin is correctly loaded and used on: @@ -78,7 +106,6 @@ def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): ) c.run("create .") c.run("cache save 'pkg/*:*'") - print(c.out) assert f"Compressing {COMPRESSED_PLUGIN_TAR_NAME}.xz using compression plugin (xz)" in c.out c.run("remove pkg/* -c") c.run("cache restore conan_cache_save.tgz") From 0db1a7b2e38737ca398ab2f8e530775d35394ed2 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Wed, 25 Jun 2025 11:24:13 +0200 Subject: [PATCH 023/110] Make plugin return compressed extension --- conan/internal/api/uploader.py | 16 ++++++---------- .../extensions/test_compression_plugin.py | 19 +++++++++++-------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 24a49a6c911..04d8c5a6248 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -295,30 +295,26 @@ def _compress_files_with_plugin(files, name, dest_dir, conf, ref, recursive, com t1 = time.time() abs_path_without_extension = os.path.join(dest_dir, COMPRESSED_PLUGIN_TAR_NAME) ConanOutput(scope=str(ref or "")).info(f"Compressing {name} using compression plugin") - compressed_path = compression_plugin.tar_compress( + compressed_extension = compression_plugin.tar_compress( archive_path=abs_path_without_extension, files=files, recursive=recursive, conf=conf, ref=ref, ) - ConanOutput().debug(f"Compressed {compressed_path} in {time.time() - t1} time") - # Check if compressed_path == abs_path_without_extension + .* (any extension) - path, extension = os.path.splitext(compressed_path) - if path != abs_path_without_extension or not extension: - raise ConanException("The 'compression.py' plugin returned an unexpected path. Plugin should return the 'archive_path' with an extra extension") + ConanOutput().debug(f"Compressed in {time.time() - t1} time") + if not compressed_extension or not compressed_extension.startswith("."): + raise ConanException("The 'compression.py' did not return the compressed extension.") + compressed_path = abs_path_without_extension + compressed_extension t1 = time.time() - ConanOutput().debug(f"Wrapping {compressed_path} in {name}") tgz_path = os.path.join(dest_dir, name) with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=0) tgz.add(compressed_path, arcname=os.path.basename(compressed_path), recursive=recursive) tgz.close() ConanOutput().debug(f"{name} wrapped in {time.time() - t1} time") - # Only remove wrapped if it is different from the tgz_path - if compressed_path != os.path.basename(tgz_path): - remove(compressed_path) + remove(compressed_path) return tgz_path def _total_size(cache_files): diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py index 96144d95aa9..55bbd7257e6 100644 --- a/test/integration/extensions/test_compression_plugin.py +++ b/test/integration/extensions/test_compression_plugin.py @@ -39,7 +39,7 @@ def test_compression_plugin_returning_invalid_path(): compression_plugin = textwrap.dedent( """ def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): - return archive_path + return def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): pass """ @@ -56,7 +56,7 @@ def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): c.run("create .") c.run("cache save 'pkg/*:*'", assert_error=True) assert ( - "ERROR: The 'compression.py' plugin returned an unexpected path. Plugin should return the 'archive_path' with an extra extension" + "ERROR: The 'compression.py' did not return the compressed extension." in c.out ) @@ -76,7 +76,8 @@ def test_compression_plugin_correctly_load(): # xz compression def tar_compress(archive_path, files, recursive, conf=None, ref=None, *args, **kwargs): - archive_path += ".xz" + extension = ".xz" + archive_path += extension name = os.path.basename(archive_path) ConanOutput(scope=ref).info(f"Compressing {name} using compression plugin (xz)") compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None @@ -84,7 +85,7 @@ def tar_compress(archive_path, files, recursive, conf=None, ref=None, *args, **k with tarfile.open(archive_path, f"w:xz", **kwargs) as tgz: for filename, abs_path in sorted(files.items()): tgz.add(abs_path, filename, recursive=True) - return archive_path + return extension def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): ConanOutput().info(f"Decompressing {os.path.basename(archive_path)} using compression plugin (xz)") @@ -141,7 +142,8 @@ def test_compression_plugin_tar_not_compatible_with_builtin(): # zip compression def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): # compress files using zipfile library taking into account recursive - archive_path += ".zip" + extension = ".zip" + archive_path += extension name = os.path.basename(archive_path) compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None ConanOutput().info(f"Compressing {name} using compression plugin (zip)") @@ -152,7 +154,7 @@ def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): zipf.write(abs_path, arcname) else: zipf.write(abs_path, filename) - return archive_path + return extension def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): # extract tar using zipfile library @@ -192,7 +194,8 @@ def test_compress_in_subdirectory(): from conan.api.output import ConanOutput def tar_compress(archive_path, files, recursive, *args, **kwargs): # compress files using tarfile putting all content in a `conan/` subfolder - archive_path += ".tgz" + extension = ".tgz" + archive_path += extension name = os.path.basename(archive_path) ConanOutput().info(f"Compressing {os.path.basename(name)} in conan subfolder") with open(archive_path, "wb") as tgz_handle: @@ -200,7 +203,7 @@ def tar_compress(archive_path, files, recursive, *args, **kwargs): for filename, abs_path in sorted(files.items()): tgz.add(abs_path, os.path.join("conan", filename), recursive=recursive) tgz.close() - return archive_path + return extension def tar_extract(archive_path, dest_dir, *args, **kwargs): ConanOutput().info(f"Decompressing {archive_path} in conan subfolder") From dcbb29e5f71bbab0d1737108173d1403f25bb50e Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Mon, 14 Jul 2025 12:46:45 +0200 Subject: [PATCH 024/110] Adapt changes to support metadata in wrapped files Add new test case to check those files are correctly pruned after extraction Improve performance by extracting in the actual destination folder even when plugin is enabled --- conan/internal/api/uploader.py | 1 + conan/internal/util/files.py | 44 ++++++++++--------- .../extensions/test_compression_plugin.py | 42 +++++++++++++++++- 3 files changed, 65 insertions(+), 22 deletions(-) diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 04d8c5a6248..7960ebc1226 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -303,6 +303,7 @@ def _compress_files_with_plugin(files, name, dest_dir, conf, ref, recursive, com ref=ref, ) ConanOutput().debug(f"Compressed in {time.time() - t1} time") + ConanOutput().success(f"{time.time() - t1}") if not compressed_extension or not compressed_extension.startswith("."): raise ConanException("The 'compression.py' did not return the compressed extension.") diff --git a/conan/internal/util/files.py b/conan/internal/util/files.py index 36bbe80daaa..40a989cc5e7 100644 --- a/conan/internal/util/files.py +++ b/conan/internal/util/files.py @@ -278,29 +278,33 @@ def tar_extract(fileobj, destination_dir, compression_plugin=None, conf=None): "This file has been compressed using a `compression` plugin.\n" "If your organization uses this plugin, ensure it is correctly installed on your environment.") + def _tar_extract_with_plugin(fileobj, destination_dir, compression_plugin, conf): """First remove tar.gz wrapper and then call the plugin to extract""" - with tempfile.TemporaryDirectory() as temp_dir: - t1 = time.time() - the_tar = tarfile.open(fileobj=fileobj) - the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break - the_tar.extractall(path=temp_dir) - extracted_file = the_tar.getnames()[0] - the_tar.close() - # Check if the tar was compressed with the compression plugin by checking the existence of - # our constant COMPRESSED_PLUGIN_TAR_NAME (without extension as extension is added by the plugin) - if list(Path(temp_dir).glob(f"{COMPRESSED_PLUGIN_TAR_NAME}.*")): - # Get the only extracted file: the plugin tar - plugin_tar_path = os.path.join(temp_dir, extracted_file) - ConanOutput().debug(f"Unwrapped in {time.time() - t1} time") + t1 = time.time() + the_tar = tarfile.open(fileobj=fileobj) + the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break + the_tar.extractall(path=destination_dir) + extracted_files = the_tar.getnames() + the_tar.close() + # Check if the tar was compressed with the compression plugin by checking the existence of + # our constant COMPRESSED_PLUGIN_TAR_NAME (without extension as extension is added by the plugin) + ConanOutput().success(f"{time.time() - t1}") + for path in extracted_files: + if os.path.basename(path).startswith(COMPRESSED_PLUGIN_TAR_NAME): + # Extract the actual contents from the plugin tar (ignore other files present). + ConanOutput().debug(f"Unwrapped in {time.time() - t1}") t1 = time.time() - compression_plugin.tar_extract(archive_path=plugin_tar_path, dest_dir=destination_dir, conf=conf) - ConanOutput().debug(f"Extracted in {time.time() - t1} time on plugin") - else: - # The tar was not compressed using the plugin, copy files to destination - from conan.tools.files import copy - ConanOutput().debug(f"Extracted in {time.time() - t1} time built in") - copy(None, pattern="*", src=temp_dir, dst=destination_dir) + compression_plugin.tar_extract( + archive_path=os.path.join(destination_dir, path), + dest_dir=destination_dir, + conf=conf, + ) + # Remove extracted files from tar + for f in extracted_files: + remove(os.path.join(destination_dir, f)) + break + ConanOutput().debug(f"Extracted in {time.time() - t1}") def merge_directories(src, dst): from conan.tools.files import copy diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py index 55bbd7257e6..0090d03fd34 100644 --- a/test/integration/extensions/test_compression_plugin.py +++ b/test/integration/extensions/test_compression_plugin.py @@ -1,7 +1,8 @@ import os import textwrap +import tarfile -from conan.internal.util.files import COMPRESSED_PLUGIN_TAR_NAME +from conan.internal.util.files import COMPRESSED_PLUGIN_TAR_NAME, mkdir from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient @@ -227,5 +228,42 @@ def tar_extract(archive_path, dest_dir, *args, **kwargs): ) c.run("create .") c.run("cache save 'pkg/*:*'") + tgz = os.path.join(c.current_folder, "conan_cache_save.tgz") + assert os.path.exists(tgz) + + mkdir("extract_folder") + destination_dir = os.path.join(c.current_folder, "extract_folder") + extracted_files = _tar_extract(tgz, destination_dir) + assert extracted_files == [COMPRESSED_PLUGIN_TAR_NAME + ".tgz"] + # Create example files + c.save({os.path.join(destination_dir, "README.md"): "This is a readme file.", + os.path.join(destination_dir, "Cache-contents-graph-report.html"): "Cache contents graph report", + os.path.join(destination_dir, "Cache-contents-graph-report.exe"): "executable file...", + os.path.join(destination_dir, "conan_cache_save.tgz"): "this should also be ignored", + }) + # Recompress the file with metadata + _tar_compress(os.path.join(c.current_folder, "conan_cache_save_rearchived.tgz"), destination_dir) + c.run("remove pkg/* -c") - c.run("cache restore conan_cache_save.tgz") + c.run("cache restore conan_cache_save_rearchived.tgz") + + # Check any of the metadata are present in the cache + assert any(item not in os.listdir(os.path.join(c.cache_folder, "p")) for item in ("README.md", + "Cache-contents-graph-report.html", + "Cache-contents-graph-report.exe")) + + +def _tar_extract(tgz_path, destination_dir): + with open(tgz_path, "rb") as fileobj: + the_tar = tarfile.open(fileobj=fileobj) + the_tar.extraction_filter = (lambda member, path: member) + the_tar.extractall(path=destination_dir) + return the_tar.getnames() + + +def _tar_compress(archive_path, folder): + with open(archive_path, "wb") as tgz_handle: + tgz = tarfile.open(os.path.basename(archive_path), "w", fileobj=tgz_handle) + for filename in os.listdir(folder): + tgz.add(os.path.join(folder, filename), filename, recursive=True) + tgz.close() From 8b50ae079bd28ff0a0d899666e2bb8c075ed700c Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 13 Jan 2026 17:25:24 +0100 Subject: [PATCH 025/110] branching from Perseo --- .ci/docker/conan-tests | 22 +- .github/CONTRIBUTING.md | 14 +- .github/workflows/build-binaries.yml | 68 ++ .github/workflows/main.yml | 24 +- .github/workflows/osx-tests.yml | 55 +- .github/workflows/win-tests.yml | 10 +- README.md | 2 +- conan/__init__.py | 2 +- conan/api/conan_api.py | 61 +- conan/api/input.py | 2 +- conan/api/model/list.py | 159 ++- conan/api/model/remote.py | 21 +- conan/api/output.py | 42 +- conan/api/subapi/audit.py | 23 +- conan/api/subapi/cache.py | 98 +- conan/api/subapi/config.py | 232 +++- conan/api/subapi/download.py | 21 +- conan/api/subapi/export.py | 101 +- conan/api/subapi/graph.py | 30 +- conan/api/subapi/install.py | 21 +- conan/api/subapi/list.py | 116 +- conan/api/subapi/lockfile.py | 21 +- conan/api/subapi/new.py | 2 +- conan/api/subapi/profiles.py | 7 +- conan/api/subapi/remotes.py | 30 +- conan/api/subapi/report.py | 20 +- conan/api/subapi/upload.py | 99 +- conan/api/subapi/workspace.py | 149 ++- conan/cli/args.py | 121 +- conan/cli/cli.py | 13 +- conan/cli/command.py | 33 +- conan/cli/commands/audit.py | 11 +- conan/cli/commands/build.py | 4 +- conan/cli/commands/cache.py | 19 +- conan/cli/commands/config.py | 29 +- conan/cli/commands/create.py | 10 +- conan/cli/commands/download.py | 2 +- conan/cli/commands/editable.py | 13 +- conan/cli/commands/export.py | 6 +- conan/cli/commands/export_pkg.py | 51 +- conan/cli/commands/graph.py | 7 +- conan/cli/commands/inspect.py | 4 +- conan/cli/commands/install.py | 47 +- conan/cli/commands/list.py | 9 +- conan/cli/commands/lock.py | 74 +- conan/cli/commands/remote.py | 28 +- conan/cli/commands/remove.py | 54 +- conan/cli/commands/report.py | 3 +- conan/cli/commands/run.py | 55 + conan/cli/commands/search.py | 5 +- conan/cli/commands/source.py | 4 +- conan/cli/commands/test.py | 4 +- conan/cli/commands/upload.py | 39 +- conan/cli/commands/workspace.py | 72 +- conan/cli/formatters/audit/vulnerabilities.py | 172 ++- conan/cli/formatters/graph/graph_info_text.py | 18 +- conan/cli/formatters/graph/info_graph_html.py | 39 + conan/cli/formatters/report/diff.py | 74 +- conan/cli/formatters/report/diff_html.py | 890 +++++++++++++- conan/cps/cps.py | 66 +- conan/internal/api/audit/providers.py | 30 +- conan/internal/api/config/config_installer.py | 7 +- conan/internal/api/detect/detect_api.py | 12 +- conan/internal/api/detect/detect_vs.py | 3 +- conan/internal/api/export.py | 26 +- conan/internal/api/install/generators.py | 33 +- conan/internal/api/list/query_parse.py | 4 +- conan/internal/api/new/basic.py | 1 + conan/internal/api/new/bazel_7_exe.py | 2 +- conan/internal/api/new/workspace.py | 38 +- conan/internal/api/profile/profile_loader.py | 31 +- conan/internal/api/upload.py | 14 +- conan/internal/api/uploader.py | 215 ++-- conan/internal/cache/cache.py | 52 +- conan/internal/cache/db/cache_database.py | 23 +- conan/internal/cache/db/packages_table.py | 40 +- conan/internal/cache/db/recipes_table.py | 40 +- conan/internal/cache/db/table.py | 34 +- conan/internal/cache/integrity_check.py | 25 +- conan/internal/conan_app.py | 8 +- conan/internal/default_settings.py | 38 +- conan/internal/deploy.py | 9 +- conan/internal/graph/compatibility.py | 18 +- conan/internal/graph/compute_pid.py | 18 +- conan/internal/graph/graph.py | 14 +- conan/internal/graph/graph_binaries.py | 226 ++-- conan/internal/graph/graph_builder.py | 67 +- conan/internal/graph/graph_error.py | 19 +- conan/internal/graph/install_graph.py | 2 +- conan/internal/graph/installer.py | 43 +- conan/internal/graph/proxy.py | 35 +- conan/internal/graph/python_requires.py | 4 +- conan/internal/graph/range_resolver.py | 8 + conan/internal/hook_manager.py | 9 +- conan/internal/internal_tools.py | 3 +- conan/internal/loader.py | 3 +- conan/internal/methods.py | 3 + conan/internal/model/conan_file.py | 8 +- conan/internal/model/conanconfig.py | 36 + conan/internal/model/conanfile_interface.py | 10 + conan/internal/model/conf.py | 25 +- conan/internal/model/cpp_info.py | 2 +- conan/internal/model/dependencies.py | 10 +- conan/internal/model/info.py | 8 + conan/internal/model/layout.py | 6 +- conan/internal/model/lockfile.py | 4 +- conan/internal/model/manifest.py | 11 +- conan/internal/model/options.py | 5 + conan/internal/model/profile.py | 5 +- conan/internal/model/requires.py | 4 +- conan/internal/model/workspace.py | 45 +- conan/internal/paths.py | 8 +- .../internal/rest/caching_file_downloader.py | 1 - conan/internal/rest/client_routes.py | 7 +- conan/internal/rest/conan_requester.py | 21 +- conan/internal/rest/download_cache.py | 16 +- conan/internal/rest/file_downloader.py | 4 +- conan/internal/rest/file_uploader.py | 2 +- conan/internal/rest/pkg_sign.py | 6 +- conan/internal/rest/remote_manager.py | 108 +- conan/internal/rest/rest_client.py | 8 +- .../rest/rest_client_local_recipe_index.py | 11 +- conan/internal/rest/rest_client_v2.py | 55 +- conan/internal/rest/rest_routes.py | 2 +- conan/internal/runner/docker.py | 6 +- conan/internal/runner/output.py | 1 + conan/internal/util/__init__.py | 31 + conan/internal/util/files.py | 3 + conan/test/assets/genconanfile.py | 2 +- conan/test/assets/premake.py | 7 +- conan/test/assets/visual_project_files.py | 2 - conan/test/utils/artifactory.py | 4 +- conan/test/utils/mocks.py | 2 +- conan/test/utils/server_launcher.py | 4 +- conan/test/utils/test_files.py | 5 +- conan/test/utils/tools.py | 24 +- conan/tools/android/utils.py | 3 +- conan/tools/apple/apple.py | 53 +- conan/tools/apple/xcodebuild.py | 17 +- conan/tools/apple/xcodedeps.py | 15 +- conan/tools/apple/xcodetoolchain.py | 11 +- conan/tools/build/__init__.py | 1 + conan/tools/build/compiler.py | 44 + conan/tools/build/cpu.py | 37 +- conan/tools/build/cstd.py | 3 + conan/tools/build/flags.py | 70 +- conan/tools/cmake/__init__.py | 28 +- conan/tools/cmake/cmake.py | 71 +- .../__init__.py | 0 .../cmakeconfigdeps.py} | 51 +- .../{cmakedeps2 => cmakeconfigdeps}/config.py | 94 +- .../config_version.py | 2 +- .../target_configuration.py | 105 +- .../targets.py | 0 conan/tools/cmake/cmakedeps/cmakedeps.py | 16 +- .../cmake/cmakedeps/templates/__init__.py | 2 +- .../tools/cmake/cmakedeps/templates/config.py | 24 + .../cmake/cmakedeps/templates/target_data.py | 10 +- conan/tools/cmake/layout.py | 7 +- conan/tools/cmake/presets.py | 21 +- conan/tools/cmake/toolchain/blocks.py | 62 +- conan/tools/cmake/toolchain/toolchain.py | 30 +- conan/tools/cmake/utils.py | 40 + conan/tools/env/environment.py | 197 ++- conan/tools/files/__init__.py | 1 - conan/tools/files/conandata.py | 3 +- conan/tools/files/copy_pattern.py | 7 +- conan/tools/files/files.py | 30 +- conan/tools/files/packager.py | 99 -- conan/tools/gnu/autotools.py | 11 +- conan/tools/gnu/autotoolstoolchain.py | 31 +- conan/tools/gnu/gnudeps_flags.py | 2 +- conan/tools/gnu/gnutoolchain.py | 33 +- conan/tools/google/bazel.py | 2 +- conan/tools/google/bazeldeps.py | 31 +- conan/tools/intel/intel_cc.py | 6 +- conan/tools/layout/__init__.py | 3 + conan/tools/meson/helpers.py | 15 +- conan/tools/meson/meson.py | 11 +- conan/tools/meson/toolchain.py | 14 +- conan/tools/microsoft/msbuild.py | 6 +- conan/tools/microsoft/nmakedeps.py | 2 +- conan/tools/microsoft/nmaketoolchain.py | 25 +- conan/tools/microsoft/toolchain.py | 52 +- conan/tools/microsoft/visual.py | 28 +- conan/tools/premake/premake.py | 11 +- conan/tools/premake/premakedeps.py | 2 +- conan/tools/qbs/qbs.py | 2 +- conan/tools/ros/rosenv.py | 53 +- conan/tools/sbom/cyclonedx.py | 53 +- conan/tools/sbom/spdx_licenses.py | 1 - conan/tools/system/__init__.py | 1 + conan/tools/system/package_manager.py | 90 +- conan/tools/system/pip_manager.py | 73 ++ conans/migrations.py | 2 +- conans/model/package_ref.py | 31 - conans/model/recipe_ref.py | 31 - conans/requirements.txt | 4 +- conans/requirements_dev.txt | 2 - conans/server/rest/controller/v2/search.py | 5 +- conans/server/service/authorize.py | 6 +- conans/server/service/v2/search.py | 13 +- setup.py | 10 +- setup_server.py | 9 +- test/README.md | 16 +- test/conftest.py | 32 +- test/functional/command/report_test.py | 112 +- ...install_test.py => test_config_install.py} | 807 +++++++------ .../functional/command/test_install_deploy.py | 67 +- test/functional/conftest.py | 2 +- .../test_build_system_layout_helpers.py | 5 +- .../layout/test_layout_autopackage.py | 393 ------ test/functional/layout/test_source_folder.py | 2 +- test/functional/only_source_test.py | 151 --- test/functional/revisions_test.py | 209 ++-- test/functional/sbom/test_cyclonedx.py | 21 +- test/functional/subsystems_build_test.py | 3 +- .../toolchains/apple/test_xcodebuild.py | 66 + .../apple/test_xcodedeps_components.py | 14 +- .../toolchains/apple/test_xcodetoolchain.py | 2 +- .../toolchains/autotools}/__init__.py | 0 .../autotools/test_universal_binaries.py | 67 ++ .../cmake/cmakedeps/test_apple_frameworks.py | 2 +- .../cmakedeps/test_cmakedeps_components.py | 33 +- .../test_cmakedeps_components_names.py | 53 +- .../test_cmakedeps_custom_configs.py | 194 ++- .../test_cmakedeps_find_module_and_config.py | 10 +- .../cmakedeps2/test_cmakeconfigdeps_new.py | 48 +- .../cmake/test_cmake_and_no_soname_flag.py | 2 +- .../cmake/test_cmake_extra_variables.py | 54 + .../toolchains/cmake/test_cmake_find_none.py | 159 ++- .../toolchains/cmake/test_cmake_multi.py | 103 ++ .../toolchains/cmake/test_cmake_toolchain.py | 86 +- .../test_cmake_toolchain_vxworks_clang.py | 127 -- .../cmake/test_cmake_toolchain_win_clang.py | 38 +- .../cmake/test_cmake_toolchain_xcode_flags.py | 13 +- .../cmake/test_cmaketoolchain_paths.py | 2 +- test/functional/toolchains/cmake/test_cps.py | 559 +++++++++ .../functional/toolchains/cmake/test_ninja.py | 3 +- .../cmake/test_universal_binaries.py | 8 +- .../toolchains/emscripten}/__init__.py | 0 .../gnu/autotools/test_apple_toolchain.py | 2 - .../toolchains/gnu/autotools/test_win_bash.py | 13 +- .../toolchains/gnu/test_gnutoolchain_apple.py | 2 - .../toolchains/gnu/test_pkgconfigdeps.py | 2 - .../toolchains/gnu/test_universal_binaries.py | 66 + .../toolchains/google/test_bazel.py | 57 +- .../test_bazeltoolchain_cross_compilation.py | 2 +- .../toolchains/ios/test_using_cmake.py | 110 +- test/functional/toolchains/meson/_base.py | 54 +- .../toolchains/meson/test_backend.py | 2 - .../meson/test_cross_compilation.py | 49 +- .../toolchains/meson/test_install.py | 29 +- .../functional/toolchains/meson/test_meson.py | 95 +- .../meson/test_meson_and_gnu_deps_flags.py | 4 +- .../toolchains/meson/test_meson_and_objc.py | 5 - .../toolchains/meson/test_pkg_config_reuse.py | 28 +- .../meson/test_preprocessor_definitions.py | 40 +- .../toolchains/meson/test_subproject.py | 18 +- test/functional/toolchains/meson/test_test.py | 22 +- .../meson/test_v2_meson_template.py | 2 + .../toolchains/microsoft/test_msbuilddeps.py | 505 ++++---- .../microsoft/test_msbuildtoolchain.py | 40 + test/functional/toolchains/test_basic.py | 36 +- .../toolchains/test_nmake_toolchain.py | 40 +- test/functional/toolchains/test_premake.py | 77 +- .../functional/toolchains/test_txt_cmdline.py | 61 - .../tools/system/package_manager_test.py | 8 - .../tools/system/pip_manager_test.py | 141 +++ test/functional/tools/test_apple_tools.py | 8 +- test/functional/workspace/test_workspace.py | 55 +- .../build_requires/build_requires_test.py | 143 ++- .../profile_build_requires_test.py | 3 +- .../test_install_test_build_require.py | 6 +- .../build_requires/test_toolchain_packages.py | 3 +- test/integration/cache/backup_sources_test.py | 13 +- test/integration/cache/cache2_update_test.py | 78 +- test/integration/cache/storage_path_test.py | 17 + .../cache/test_home_special_char.py | 3 - .../cache/test_package_revisions.py | 2 +- .../command/cache/test_cache_clean.py | 17 +- .../command/cache/test_cache_integrity.py | 51 +- .../command/cache/test_cache_save_restore.py | 32 +- test/integration/command/create_test.py | 46 +- .../command/custom_commands_test.py | 33 + .../download_selected_packages_test.py | 2 +- .../command/export/export_path_test.py | 2 +- .../export}/export_sources_test.py | 0 .../integration/command/export/export_test.py | 4 +- .../integration/command/export/test_export.py | 18 + test/integration/command/export_pkg_test.py | 2 +- test/integration/command/help_test.py | 5 +- test/integration/command/info/info_test.py | 68 +- .../command/info/test_graph_info_graphical.py | 11 + .../command/info/test_info_build_order.py | 16 +- .../command/install/install_test.py | 8 +- .../command/install/install_update_test.py | 6 +- .../command/install/test_graph_build_mode.py | 31 + test/integration/command/list/list_test.py | 29 +- .../list/test_combined_pkglist_flows.py | 170 ++- .../integration/command/list/test_list_lru.py | 17 + .../command/remote}/__init__.py | 0 .../command/{ => remote}/remote_test.py | 48 +- .../{ => remote}/remote_verify_ssl_test.py | 0 .../command/{ => remote}/test_remote_users.py | 2 +- test/integration/command/remove_test.py | 2 +- test/integration/command/source_test.py | 2 +- test/integration/command/test_audit.py | 33 +- test/integration/command/test_build.py | 2 +- .../command/test_graph_find_binaries.py | 36 +- test/integration/command/test_inspect.py | 2 +- test/integration/command/test_new.py | 10 +- test/integration/command/test_outdated.py | 2 +- test/integration/command/test_output.py | 4 +- test/integration/command/test_package_test.py | 14 +- test/integration/command/test_run.py | 105 ++ .../command/upload/test_upload_bundle.py | 22 +- .../command/upload/upload_compression_test.py | 2 +- .../integration/command/upload/upload_test.py | 9 +- test/integration/conanfile/init_test.py | 33 + .../conanfile/required_conan_version_test.py | 2 +- test/integration/conanfile/test_deprecated.py | 17 - .../conf/test_auth_source_plugin.py | 86 +- .../configuration/conf/test_conf.py | 2 +- .../custom_setting_test_package_test.py | 2 +- .../configuration/default_profile_test.py | 13 +- .../integration/configuration/profile_test.py | 32 +- .../configuration/requester_test.py | 8 +- .../configuration/required_version_test.py | 2 +- .../configuration/test_auth_remote_plugin.py | 17 +- .../configuration/test_profile_jinja.py | 18 + test/integration/cps/test_cps.py | 98 +- .../integration/editable/editable_add_test.py | 2 +- .../editable/editable_remove_test.py | 2 +- .../editable/test_editable_envvars.py | 108 +- .../environment/test_buildenv_profile.py | 17 + test/integration/environment/test_env.py | 141 ++- .../integration/extensions/hooks/hook_test.py | 11 + .../graph/core/graph_manager_base.py | 3 +- .../graph/core/graph_manager_test.py | 39 +- test/integration/graph/core/test_alias.py | 2 +- .../graph/core/test_auto_package_type.py | 1 + .../graph/core/test_build_requires.py | 53 +- test/integration/graph/core/test_provides.py | 23 +- .../graph/core/test_version_ranges.py | 14 +- .../graph/test_dependencies_visit.py | 23 +- .../graph/test_divergent_cppstd_build_host.py | 11 +- .../graph/test_platform_requires.py | 2 +- .../graph/test_replace_requires.py | 30 + .../graph/test_require_same_pkg_versions.py | 142 +++ test/integration/graph/test_system_tools.py | 1 - test/integration/graph/test_test_requires.py | 11 +- .../version_ranges_cached_test.py | 2 +- test/integration/layout/devflow_test.py | 43 +- .../lockfile/test_lock_pyrequires.py | 80 +- .../lockfile/test_lock_requires.py | 2 +- .../lockfile/test_user_overrides.py | 54 +- .../metadata/test_metadata_commands.py | 89 +- .../metadata/test_metadata_deploy.py | 2 +- test/integration/options/options_test.py | 79 +- .../options/test_configure_options.py | 10 +- .../integration/package_id/compatible_test.py | 249 +++- .../package_id_requires_modes_test.py | 18 +- .../python_requires_package_id_test.py | 2 +- .../package_id/test_cache_compatibles.py | 20 + .../package_id/test_config_package_id.py | 14 +- test/integration/package_id/test_validate.py | 121 +- .../package_id/transitive_header_only_test.py | 1 - .../transitive_options_affect_id_test.py | 42 + .../py_requires/python_requires_test.py | 2 +- .../remote/broken_download_test.py | 6 +- test/integration/remote/rest_api_test.py | 3 +- test/integration/remote/retry_test.py | 6 +- .../remote/test_local_recipes_index.py | 45 + .../remote/test_remote_recipes_only.py | 84 ++ .../settings/settings_override_test.py | 4 +- test/integration/symlinks/symlinks_test.py | 55 +- test/integration/test_components.py | 1 + test/integration/test_components_error.py | 25 + test/integration/test_compressions.py | 143 +++ test/integration/test_migrations.py | 37 +- test/integration/test_recipe_policies.py | 53 + test/integration/tgz_macos_dot_files_test.py | 6 +- .../toolchains/apple/test_xcodedeps.py | 59 + .../cmake/cmakedeps/test_cmakedeps.py | 31 + .../test_cmakeconfigdeps_frameworks.py | 24 + .../cmake/cmakedeps2/test_cmakedeps.py | 355 ++++-- .../toolchains/cmake/test_cmaketoolchain.py | 32 +- .../toolchains/env/test_buildenv.py | 40 + .../toolchains/gnu/test_autotoolstoolchain.py | 7 +- .../toolchains/gnu/test_basic_layout.py | 57 + .../toolchains/gnu/test_gnutoolchain.py | 6 +- .../toolchains/gnu/test_makedeps.py | 1 - .../toolchains/google/test_bazeldeps.py | 181 ++- .../toolchains/intel/test_intel_cc.py | 14 + .../toolchains/meson/test_mesontoolchain.py | 57 +- .../toolchains/microsoft/vcvars_test.py | 75 +- .../toolchains/premake/test_premake.py | 32 +- .../toolchains/premake/test_premakedeps.py | 4 +- .../test_raise_on_universal_binaries.py | 96 +- test/integration/tools/ros/test_rosenv.py | 37 +- .../tools/system/package_manager_test.py | 186 ++- test/integration/workspace/test_workspace.py | 448 ++++++- .../test_compatibility_performance.py | 129 ++ test/performance/test_db_performance.py | 74 ++ test/unittests/cli/test_cli_ref_matching.py | 7 + .../client/build/c_std_flags_test.py | 77 +- .../client/build/compiler_flags_test.py | 177 +-- .../client/build/cpp_std_flags_test.py | 106 +- .../client/command/parse_arguments_test.py | 1 + .../unittests/client/conanfile_loader_test.py | 1 - .../client/conf/detect/test_gcc_compiler.py | 2 +- .../unittests/client/graph/deps_graph_test.py | 2 +- test/unittests/client/remote_manager_test.py | 65 +- test/unittests/client/rest/requester_test.py | 2 +- test/unittests/client/rest/response_test.py | 2 +- .../rest_client_v2/rest_client_v2_test.py | 22 +- .../toolchain/autotools/autotools_test.py | 77 -- .../client/tools/cppstd_required_test.py | 2 +- .../client/tools/files/rename_test.py | 64 +- test/unittests/client/tools/test_env.py | 2 +- test/unittests/client/userio_test.py | 1 - .../model/build_info/components_test.py | 2 +- .../model/build_info/new_build_info_test.py | 4 +- test/unittests/model/options_test.py | 38 +- test/unittests/model/other_settings_test.py | 133 +-- test/unittests/model/settings_test.py | 31 +- test/unittests/model/test_list.py | 49 + .../model/version/test_version_comparison.py | 5 +- .../model/version/test_version_range.py | 17 + test/unittests/model/versionrepr_test.py | 2 - .../search/search_query_parse_test.py | 2 +- .../server/conan_server_config_parser_test.py | 18 +- test/unittests/server/service/service_test.py | 6 + .../unittests/tools/apple/test_apple_tools.py | 2 +- .../tools/apple/test_xcodebuild.py | 0 test/unittests/tools/build/test_can_run.py | 14 +- test/unittests/tools/build/test_compiler.py | 34 + test/unittests/tools/build/test_cppstd.py | 17 + test/unittests/tools/build/test_cstd.py | 13 + .../tools/cmake/test_cmake_install.py | 2 +- .../cmake/test_cmake_presets_definitions.py | 2 +- test/unittests/tools/cmake/test_cmake_test.py | 24 + .../tools/cmake/test_cmaketoolchain.py | 1061 +++++++---------- test/unittests/tools/env/test_env.py | 7 +- test/unittests/tools/env/test_env_files.py | 49 +- test/unittests/tools/files/test_downloads.py | 1 + test/unittests/tools/files/test_patches.py | 2 +- test/unittests/tools/files/test_rm.py | 48 +- test/unittests/tools/files/test_tool_copy.py | 6 +- test/unittests/tools/files/test_zipping.py | 29 +- test/unittests/tools/gnu/autotools_test.py | 109 +- .../gnu}/autotools_toolchain_test.py | 67 +- test/unittests/tools/gnu/gnudepsflags_test.py | 2 +- test/unittests/tools/gnu/test_gnutoolchain.py | 21 +- test/unittests/tools/intel/test_intel_cc.py | 2 +- test/unittests/tools/meson/test_meson.py | 26 +- .../unittests/tools/microsoft/test_msbuild.py | 8 +- .../tools/microsoft/test_msvs_toolset.py | 1 + test/unittests/tools/system/__init__.py | 0 .../tools/system/pip_manager_test.py | 49 + test/unittests/util/conanfile_tools_test.py | 35 +- test/unittests/util/detect_libc_test.py | 6 +- test/unittests/util/detect_test.py | 25 +- .../util/detected_architecture_test.py | 2 +- .../util/files/strip_root_extract_test.py | 6 +- test/unittests/util/files/tar_extract_test.py | 6 +- test/unittests/util/test_encrypt.py | 13 +- 468 files changed, 13969 insertions(+), 5942 deletions(-) create mode 100644 .github/workflows/build-binaries.yml create mode 100644 conan/cli/commands/run.py create mode 100644 conan/internal/model/conanconfig.py create mode 100644 conan/tools/build/compiler.py rename conan/tools/cmake/{cmakedeps2 => cmakeconfigdeps}/__init__.py (100%) rename conan/tools/cmake/{cmakedeps2/cmakedeps.py => cmakeconfigdeps/cmakeconfigdeps.py} (86%) rename conan/tools/cmake/{cmakedeps2 => cmakeconfigdeps}/config.py (52%) rename conan/tools/cmake/{cmakedeps2 => cmakeconfigdeps}/config_version.py (97%) rename conan/tools/cmake/{cmakedeps2 => cmakeconfigdeps}/target_configuration.py (86%) rename conan/tools/cmake/{cmakedeps2 => cmakeconfigdeps}/targets.py (100%) delete mode 100644 conan/tools/files/packager.py create mode 100644 conan/tools/system/pip_manager.py delete mode 100644 conans/model/package_ref.py delete mode 100644 conans/model/recipe_ref.py rename test/functional/command/{config_install_test.py => test_config_install.py} (50%) delete mode 100644 test/functional/layout/test_layout_autopackage.py delete mode 100644 test/functional/only_source_test.py rename {conans/model => test/functional/toolchains/autotools}/__init__.py (100%) create mode 100644 test/functional/toolchains/autotools/test_universal_binaries.py create mode 100644 test/functional/toolchains/cmake/test_cmake_extra_variables.py create mode 100644 test/functional/toolchains/cmake/test_cmake_multi.py delete mode 100644 test/functional/toolchains/cmake/test_cmake_toolchain_vxworks_clang.py create mode 100644 test/functional/toolchains/cmake/test_cps.py rename test/{unittests/client/toolchain => functional/toolchains/emscripten}/__init__.py (100%) create mode 100644 test/functional/toolchains/gnu/test_universal_binaries.py delete mode 100644 test/functional/toolchains/test_txt_cmdline.py create mode 100644 test/functional/tools/system/pip_manager_test.py rename test/integration/{ => command/export}/export_sources_test.py (100%) rename test/{unittests/client/toolchain/autotools => integration/command/remote}/__init__.py (100%) rename test/integration/command/{ => remote}/remote_test.py (90%) rename test/integration/command/{ => remote}/remote_verify_ssl_test.py (100%) rename test/integration/command/{ => remote}/test_remote_users.py (99%) create mode 100644 test/integration/command/test_run.py create mode 100644 test/integration/remote/test_remote_recipes_only.py create mode 100644 test/integration/test_compressions.py create mode 100644 test/integration/test_recipe_policies.py create mode 100644 test/performance/test_compatibility_performance.py create mode 100644 test/performance/test_db_performance.py delete mode 100644 test/unittests/client/toolchain/autotools/autotools_test.py create mode 100644 test/unittests/model/test_list.py rename test/unittests/{client => }/tools/apple/test_xcodebuild.py (100%) create mode 100644 test/unittests/tools/build/test_compiler.py rename test/unittests/{client/toolchain/autotools => tools/gnu}/autotools_toolchain_test.py (90%) create mode 100644 test/unittests/tools/system/__init__.py create mode 100644 test/unittests/tools/system/pip_manager_test.py diff --git a/.ci/docker/conan-tests b/.ci/docker/conan-tests index 5f7ee6f2931..5c9bc564c03 100644 --- a/.ci/docker/conan-tests +++ b/.ci/docker/conan-tests @@ -4,19 +4,20 @@ LABEL maintainer="Conan.io " ENV DEBIAN_FRONTEND=noninteractive -ENV PY36=3.6.15 \ +ENV PY37=3.7.9 \ PY38=3.8.6 \ PY39=3.9.2 \ PY310=3.10.16 \ PY312=3.12.3 \ PY313=3.13.0 \ + PY314=3.14.0 \ CMAKE_3_15=/usr/share/cmake-3.15.7/bin/cmake \ CMAKE_3_16=/usr/share/cmake-3.16.9/bin/cmake \ CMAKE_3_17=/usr/share/cmake-3.17.5/bin/cmake \ CMAKE_3_19=/usr/share/cmake-3.19.7/bin/cmake \ CMAKE_3_23=/usr/share/cmake-3.23.5/bin/cmake \ CMAKE_3_27=/usr/share/cmake-3.27.9/bin/cmake \ - CMAKE_4_0=/usr/share/cmake-4.0.0-rc3/bin/cmake \ + CMAKE_4_2=/usr/share/cmake-4.2.1/bin/cmake \ GCC_9=/usr/bin/gcc-9 \ GXX_9=/usr/bin/g++-9 \ GCC_11=/usr/bin/gcc-11 \ @@ -24,9 +25,9 @@ ENV PY36=3.6.15 \ CLANG_14=/usr/bin/clang-14 \ CLANGXX_14=/usr/bin/clang++-14 \ BAZEL_6=6.5.0 \ - BAZEL_7=7.4.1 \ - BAZEL_8=8.0.0 \ - EMSDK=4.0.10 + BAZEL_7=7.6.2 \ + BAZEL_8=8.4.2 \ + EMSDK=4.0.22 RUN apt-get update && \ apt-get install -y --no-install-recommends \ @@ -95,12 +96,12 @@ ENV PYENV_ROOT $HOME/.pyenv ENV PATH $PYENV_ROOT/bin:$PYENV_ROOT/shims:/usr/bin:/bin:$PATH RUN curl https://pyenv.run | bash && \ - pyenv install $PY36 && \ + pyenv install $PY37 && \ pyenv install $PY38 && \ pyenv install $PY39 && \ pyenv install $PY310 && \ pyenv install $PY312 && \ - pyenv install $PY313 && \ + pyenv install $PY314 && \ pyenv global $PY39 && \ curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python get-pip.py && \ @@ -122,15 +123,15 @@ RUN wget https://github.com/Kitware/CMake/releases/download/v3.15.7/cmake-3.15.7 tar -xvzf cmake-3.23.5-Linux-x86_64.tar.gz && mv cmake-3.23.5-linux-x86_64/ /usr/share/cmake-3.23.5 && \ wget https://github.com/Kitware/CMake/releases/download/v3.27.9/cmake-3.27.9-Linux-x86_64.tar.gz && \ tar -xvzf cmake-3.27.9-Linux-x86_64.tar.gz && mv cmake-3.27.9-linux-x86_64/ /usr/share/cmake-3.27.9 && \ - wget https://cmake.org/files/v4.0/cmake-4.0.0-rc3-linux-x86_64.tar.gz && \ - tar -xvzf cmake-4.0.0-rc3-linux-x86_64.tar.gz && mv cmake-4.0.0-rc3-linux-x86_64/ /usr/share/cmake-4.0.0-rc3 && \ + wget https://cmake.org/files/v4.2/cmake-4.2.1-linux-x86_64.tar.gz && \ + tar -xvzf cmake-4.2.1-linux-x86_64.tar.gz && mv cmake-4.2.1-linux-x86_64/ /usr/share/cmake-4.2.1 && \ update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_15 10 && \ update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_16 20 && \ update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_17 30 && \ update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_19 40 && \ update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_23 50 && \ update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_27 60 && \ - update-alternatives --install /usr/bin/cmake cmake $CMAKE_4_0 70 && \ + update-alternatives --install /usr/bin/cmake cmake $CMAKE_4_2 70 && \ # set CMake 3.15 as default update-alternatives --set cmake $CMAKE_3_15 @@ -174,6 +175,7 @@ RUN cd /tmp && \ wget https://github.com/emscripten-core/emsdk/archive/refs/tags/${EMSDK}.tar.gz && \ tar xzf ${EMSDK}.tar.gz --directory /usr/share && \ cd /usr/share/emsdk-${EMSDK} && \ + pyenv local 3.12 && \ ./emsdk update && \ ./emsdk install latest && \ ./emsdk activate latest --permanent && \ diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index e67ca7129fd..631791eb678 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -25,9 +25,10 @@ To contribute follow the next steps: 3. Fork the [Conan main repository](https://github.com/conan-io/conan) and create a `feature/xxx` branch from the `develop2` branch and develop your fix/feature as discussed in previous step. 4. Try to keep your branch updated with the `develop2` branch to avoid conflicts. -5. Open a pull request, and select `develop2` as the base branch. Never open a pull request to ``release/xxx`` branches, unless the branch is to be part of the next 2.X.Y patch. In that case, the PR should be targeted to release/2.0. -6. Add the text (besides other comments): "fixes #IssueNumber" in the body of the PR, referring to the issue of step 1. -7. Submit a PR to the Conan documentation about the changes done providing examples if needed. +5. Run the ``test/unittest`` and ``test/integration`` test suite locally, as described in [Conan tests guidelines section](https://github.com/conan-io/conan/blob/develop2/test/README.md). If you are doing changes to a build system integration, locate the respective folder for that integration in ``test/functional`` and run the tests of that folder. It is not expected that contributors have to run the full test suite locally, as it requires too many external tools. +6. Open a pull request, and select `develop2` as the base branch. Never open a pull request to ``release/xxx`` branches, unless the branch is to be part of the next 2.X.Y patch. In that case, the PR should be targeted to ``release/2.X``. +7. Add the text (besides other comments): "fixes #IssueNumber" in the body of the PR, referring to the issue of step 1. +8. Submit a PR to the Conan documentation about the changes done providing examples if needed. The ``conan-io`` organization maintainers will review and help with the coding of tests. Finally it will be assigned to milestone. @@ -102,8 +103,7 @@ import shutil from tqdm import tqdm -from conans.client.tools import which -from conans.errors import ConanException -from conans.model.version import Version +from conan.tools.cmake import CMakeToolchain +from conan.tools.files import save, load ``` -- Write unit tests, if possible +- Write tests, if possible diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml new file mode 100644 index 00000000000..a781d448c7e --- /dev/null +++ b/.github/workflows/build-binaries.yml @@ -0,0 +1,68 @@ +name: Build Conan Binaries +run-name: Build Conan Binaries - v${{ inputs.conan_version }} - ${{ inputs.target_sha }} - ${{ inputs.request_id }} + +on: + workflow_dispatch: + inputs: + conan_version: { description: Conan version to package, required: true, type: string } + target_sha: { description: Git SHA to package, required: true, type: string } + request_id: { description: ID to identify run, required: true, type: string } + +permissions: + contents: read + +jobs: + package: + name: Package for ${{ matrix.platform }}/${{ matrix.arch }} + strategy: + fail-fast: false + matrix: + include: + - { platform: Linux, arch: x86_64, runner: ubuntu-24.04 } + - { platform: Windows, arch: x86_64, runner: windows-2022 } + - { platform: Windows, arch: i686, runner: windows-2022 } + - { platform: Windows, arch: arm64, runner: windows-11-arm } + - { platform: Macos, arch: arm64, runner: macos-14 } + - { platform: Macos, arch: x86_64, runner: macos-14 } + - { platform: Linux, arch: arm64, runner: ubuntu-24.04-arm } + runs-on: ${{ matrix.runner }} + + steps: + - name: Show inputs + shell: bash + run: | + echo "conan_version=${{ inputs.conan_version }}" + echo "target_sha=${{ inputs.target_sha }}" + echo "request_id=${{ inputs.request_id }}" + + - name: Generate Read-Only App Token + id: generate_token + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ secrets.GH_APP_RELEASE_ID }} + private-key: ${{ secrets.GH_APP_RELEASE_PRIVATE_KEY }} + permission-contents: read + owner: conan-io + repositories: | + conan + release-tools + + - name: Checkout release-tools + uses: actions/checkout@v4 + with: + repository: conan-io/release-tools + token: ${{ steps.generate_token.outputs.token }} + + - name: Build packages + shell: bash + env: + PLATFORM: ${{ matrix.platform }} + ARCH: ${{ matrix.arch }} + run: | + ./jenkins/build_packages.sh "${{ inputs.conan_version }}" "${{ inputs.target_sha }}" + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: conan-${{ inputs.conan_version }}-${{ inputs.target_sha }}-${{ matrix.platform }}-${{ matrix.arch }}-${{ inputs.request_id }} + path: dist/* diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6297ae0f1fc..1e8abfe2e16 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,6 +16,20 @@ concurrency: cancel-in-progress: true jobs: + ensure_latest_tag_merged: + runs-on: ubuntu-latest + name: Ensure latest release is merged in develop2 branch + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - shell: bash + run: | + git fetch --tags --prune + latest_tag=$(git tag -l --sort=-v:refname | head -n1) + echo "Checking that branch 'develop2' contains the latest release tag: $latest_tag" + git merge-base --is-ancestor "$latest_tag" origin/develop2 + set_python_versions: runs-on: ubuntu-latest outputs: @@ -27,29 +41,29 @@ jobs: id: set_versions run: | if [[ "${{ github.ref }}" == "refs/heads/develop2" || "${{ github.ref }}" == refs/heads/release/* ]]; then - echo "python_versions_linux_windows=['3.13', '3.6']" >> $GITHUB_OUTPUT - echo "python_versions_macos=['3.13', '3.8']" >> $GITHUB_OUTPUT + echo "python_versions_linux_windows=['3.14', '3.7']" >> $GITHUB_OUTPUT + echo "python_versions_macos=['3.14', '3.8']" >> $GITHUB_OUTPUT else echo "python_versions_linux_windows=['3.10']" >> $GITHUB_OUTPUT echo "python_versions_macos=['3.10']" >> $GITHUB_OUTPUT fi linux_suite: - needs: set_python_versions + needs: [ensure_latest_tag_merged, set_python_versions] uses: ./.github/workflows/linux-tests.yml name: Linux test suite with: python-versions: ${{ needs.set_python_versions.outputs.python_versions_linux_windows }} osx_suite: - needs: set_python_versions + needs: [ensure_latest_tag_merged, set_python_versions] uses: ./.github/workflows/osx-tests.yml name: OSX test suite with: python-versions: ${{ needs.set_python_versions.outputs.python_versions_macos }} windows_suite: - needs: set_python_versions + needs: [ensure_latest_tag_merged, set_python_versions] uses: ./.github/workflows/win-tests.yml name: Windows test suite with: diff --git a/.github/workflows/osx-tests.yml b/.github/workflows/osx-tests.yml index 3547d0110b7..b98cb026f2b 100644 --- a/.github/workflows/osx-tests.yml +++ b/.github/workflows/osx-tests.yml @@ -20,13 +20,6 @@ jobs: with: python-version: '3.10' - - name: Cache pip packages - id: cache-pip - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('conans/requirements*.txt') }} - - name: Install Python requirements run: | pip install --upgrade pip @@ -58,10 +51,10 @@ jobs: ~/Applications/CMake/3.19.7 ~/Applications/CMake/3.23.5 ~/Applications/CMake/3.27.9 - ~/Applications/CMake/4.0.0-rc3 + ~/Applications/CMake/4.2.1 ~/Applications/bazel/6.5.0 - ~/Applications/bazel/7.4.1 - ~/Applications/bazel/8.0.0 + ~/Applications/bazel/7.6.2 + ~/Applications/bazel/8.4.2 key: ${{ runner.os }}-conan-tools-cache - name: Build CMake old versions not available for ARM @@ -87,7 +80,7 @@ jobs: if: steps.cache-tools.outputs.cache-hit != 'true' run: | set -e - CMAKE_PRECOMP_VERSIONS=("3.19.7" "3.23.5" "3.27.9" "4.0.0-rc3") + CMAKE_PRECOMP_VERSIONS=("3.19.7" "3.23.5" "3.27.9" "4.2.1") for version in "${CMAKE_PRECOMP_VERSIONS[@]}"; do echo "Downloading and installing precompiled universal CMake version ${version}..." wget -q --no-check-certificate https://cmake.org/files/v${version%.*}/cmake-${version}-macos-universal.tar.gz @@ -107,7 +100,7 @@ jobs: if: steps.cache-tools.outputs.cache-hit != 'true' run: | set -e - for version in 6.5.0 7.4.1 8.0.0; do + for version in 6.5.0 7.6.2 8.4.2; do mkdir -p ${HOME}/Applications/bazel/${version} wget -q -O ${HOME}/Applications/bazel/${version}/bazel https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-darwin-arm64 chmod +x ${HOME}/Applications/bazel/${version}/bazel @@ -133,12 +126,6 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Restore pip cache - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('conans/requirements*.txt') }} - - name: Restore tools cache uses: actions/cache@v4 with: @@ -147,12 +134,31 @@ jobs: ~/Applications/CMake/3.19.7 ~/Applications/CMake/3.23.5 ~/Applications/CMake/3.27.9 - ~/Applications/CMake/4.0.0-rc3 + ~/Applications/CMake/4.2.1 ~/Applications/bazel/6.5.0 - ~/Applications/bazel/7.4.1 - ~/Applications/bazel/8.0.0 + ~/Applications/bazel/7.6.2 + ~/Applications/bazel/8.4.2 key: ${{ runner.os }}-conan-tools-cache + - name: Select Xcode 16.4 + run: | + sudo xcode-select -s /Applications/Xcode_16.4.app + xcodebuild -version + xcrun --sdk macosx --show-sdk-version + clang --version + + # Install system dependencies BEFORE setting up the matrix Python. + # This prevents Emscripten (which requires Python 3.10+) from crashing + # when trying to use Python 3.8 from the matrix. + - name: Install homebrew dependencies + run: | + brew update + brew install xcodegen make libtool zlib autoconf automake ninja emscripten + export PATH=${HOME}/Applications/CMake/3.15.7/bin:$PATH:/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin + emcc --version + cmake --version + bazel --version + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -166,13 +172,6 @@ jobs: pip install -r conans/requirements_dev.txt pip install meson - - name: Install homebrew dependencies - run: | - brew install xcodegen make libtool zlib autoconf automake ninja emscripten - export PATH=${HOME}/Applications/CMake/3.15.7/bin:$PATH:/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin - cmake --version - bazel --version - - name: Run tests uses: ./.github/actions/test-coverage with: diff --git a/.github/workflows/win-tests.yml b/.github/workflows/win-tests.yml index bfc3bcff231..dc51e9329f2 100644 --- a/.github/workflows/win-tests.yml +++ b/.github/workflows/win-tests.yml @@ -162,16 +162,16 @@ jobs: C:\tools\cmake\3.19.7 C:\tools\cmake\3.23.5 C:\tools\cmake\3.27.9 - C:\tools\cmake\4.0.0-rc3 + C:\tools\cmake\4.2.1 C:\tools\bazel\6.5.0 - C:\tools\bazel\7.4.1 - C:\tools\bazel\8.0.0 + C:\tools\bazel\7.6.2 + C:\tools\bazel\8.4.2 key: ${{ runner.os }}-conan-tools-cache - name: Install CMake versions if: steps.cache-tools.outputs.cache-hit != 'true' run: | - $CMAKE_BUILD_VERSIONS = "3.15.7", "3.19.7", "3.23.5", "3.27.9", "4.0.0-rc3" + $CMAKE_BUILD_VERSIONS = "3.15.7", "3.19.7", "3.23.5", "3.27.9", "4.2.1" foreach ($version in $CMAKE_BUILD_VERSIONS) { Write-Host "Downloading CMake version $version for Windows..." $destination = "C:\tools\cmake\$version" @@ -194,7 +194,7 @@ jobs: - name: Install Bazel versions if: steps.cache-tools.outputs.cache-hit != 'true' run: | - $BAZEL_BUILD_VERSIONS = "6.5.0", "7.4.1", "8.0.0" + $BAZEL_BUILD_VERSIONS = "6.5.0", "7.6.2", "8.4.2" foreach ($version in $BAZEL_BUILD_VERSIONS) { Write-Host "Downloading Bazel version $version for Windows..." $destination = "C:\tools\bazel\$version" diff --git a/README.md b/README.md index 70207a0bfed..d3318ced182 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Decentralized, open-source (MIT), C/C++ package manager. - Homepage: https://conan.io/ - Github: https://github.com/conan-io/conan - Docs: https://docs.conan.io -- Slack: https://cpplang.slack.com (#conan channel. Please, click [here](https://join.slack.com/t/cpplang/shared_invite/zt-1snzdn6rp-rOUxF3166oz1_11Tr5H~xg) to get an invitation) +- Slack: https://cpplang.slack.com (#conan channel. Please, click [here](https://cppalliance.org/slack/#cpp-slack) to get an invitation) - Twitter: https://twitter.com/conan_io - Blog: https://blog.conan.io - Security reports: https://jfrog.com/trust/report-vulnerability diff --git a/conan/__init__.py b/conan/__init__.py index 892f6ea47d9..febca134c4b 100644 --- a/conan/__init__.py +++ b/conan/__init__.py @@ -2,5 +2,5 @@ from conan.internal.model.workspace import Workspace from conan.internal.model.version import Version -__version__ = '2.20.0-dev' +__version__ = '2.25.0-dev' conan_version = Version(__version__) diff --git a/conan/api/conan_api.py b/conan/api/conan_api.py index 17edb639207..a3d9225f751 100644 --- a/conan/api/conan_api.py +++ b/conan/api/conan_api.py @@ -23,6 +23,7 @@ from conan.errors import ConanException from conan.internal.cache.home_paths import HomePaths from conan.internal.hook_manager import HookManager +from conan.internal.loader import load_python_file from conan.internal.model.conf import load_global_conf, ConfDefinition, CORE_CONF_PATTERN from conan.internal.model.settings import load_settings_yml from conan.internal.paths import get_conan_user_home @@ -40,12 +41,12 @@ class ConanAPI: def __init__(self, cache_folder=None): """ :param cache_folder: Conan cache/home folder. It will have less priority than the - "home_folder" defined in a Workspace. + ``"home_folder"`` defined in a Workspace. """ version = sys.version_info - if version.major == 2 or version.minor < 6: - raise ConanException("Conan needs Python >= 3.6") + if version.major == 2 or version.minor < 7: + raise ConanException("Conan needs Python >= 3.7") if cache_folder is not None and not os.path.isabs(cache_folder): raise ConanException("cache_folder has to be an absolute path") @@ -66,21 +67,27 @@ def __init__(self, cache_folder=None): self.profiles = ProfilesAPI(self, self._api_helpers) self.install = InstallAPI(self, self._api_helpers) self.graph = GraphAPI(self, self._api_helpers) - self.export = ExportAPI(self, self._api_helpers) + #: Used to export recipes and pre-compiled package binaries to the Conan cache + self.export: ExportAPI = ExportAPI(self, self._api_helpers) self.remove = RemoveAPI(self) self.new = NewAPI(self) - self.upload = UploadAPI(self, self._api_helpers) - self.download = DownloadAPI(self) - self.cache = CacheAPI(self, self._api_helpers) - self.lockfile = LockfileAPI(self) + #: Used to upload recipes and packages to remotes + self.upload: UploadAPI = UploadAPI(self, self._api_helpers) + #: Used to download recipes and packages from remotes + self.download: DownloadAPI = DownloadAPI(self) + #: Used to interact wit the packages storage cache + self.cache: CacheAPI = CacheAPI(self, self._api_helpers) + #: Used to read and manage lockfile files + self.lockfile: LockfileAPI = LockfileAPI(self) self.local = LocalAPI(self, self._api_helpers) - self.audit = AuditAPI(self) + #: Used to check vulnerabilities of dependencies + self.audit: AuditAPI = AuditAPI(self) # Now, lazy loading of editables self.workspace = WorkspaceAPI(self) - self.report = ReportAPI(self, self._api_helpers) + self.report: ReportAPI = ReportAPI(self, self._api_helpers) @property - def home_folder(self): + def home_folder(self) -> str: """ Where the Conan user home is located. Read only. Can be modified by the ``CONAN_HOME`` environment variable or by the ``.conanrc`` file in the current directory or any parent directory @@ -93,7 +100,6 @@ def reinit(self): Reinitialize the Conan API. This is useful when the configuration changes. """ self._api_helpers.reinit() - self.remotes.reinit() self.local.reinit() def migrate(self): @@ -109,6 +115,26 @@ def __init__(self, conan_api): self._cli_core_confs = None self._init_global_conf() self.hook_manager = HookManager(HomePaths(self._conan_api.home_folder).hooks_path) + # Wraps an http_requester to inject proxies, certs, etc + self._requester = ConanRequester(self.global_conf, self._conan_api.home_folder) + self._settings_yml = None + self._compression_plugin = None + + @property + def compression_plugin(self): + if self._compression_plugin is None: + compression_plugin_path = HomePaths( + self._conan_api.home_folder).compression_plugin_path + if not os.path.exists(compression_plugin_path): + self._compression_plugin = False # Avoid FS re-check + return None + mod, _ = load_python_file(compression_plugin_path) + # A plugin can provide just 1 of them + if not hasattr(mod, "tar_extract") and not hasattr(mod, "tar_compress"): + raise ConanException("The 'compression.py' plugin does not contain " + "required `tar_extract` or `tar_compress` functions") + self._compression_plugin = mod + return self._compression_plugin def set_core_confs(self, core_confs): confs = ConfDefinition() @@ -133,7 +159,16 @@ def _init_global_conf(self): def reinit(self): self._init_global_conf() self.hook_manager.reinit() + self._requester = ConanRequester(self.global_conf, self._conan_api.home_folder) + self._settings_yml = None + self._compression_plugin = None @property def settings_yml(self): - return load_settings_yml(self._conan_api.home_folder) + if self._settings_yml is None: + self._settings_yml = load_settings_yml(self._conan_api.home_folder) + return self._settings_yml + + @property + def requester(self): + return self._requester diff --git a/conan/api/input.py b/conan/api/input.py index 258ded10dfa..6636697a9ab 100644 --- a/conan/api/input.py +++ b/conan/api/input.py @@ -78,7 +78,7 @@ def request_boolean(self, msg, default_option=None): """Request user to input a boolean""" ret = None while ret is None: - if default_option is True: + if default_option: s = self.request_string("%s (YES/no)" % msg) elif default_option is False: s = self.request_string("%s (NO/yes)" % msg) diff --git a/conan/api/model/list.py b/conan/api/model/list.py index 8faedec405d..637330afa26 100644 --- a/conan/api/model/list.py +++ b/conan/api/model/list.py @@ -1,9 +1,12 @@ +import copy import fnmatch import json import os from json import JSONDecodeError +from typing import Iterable, Tuple, Dict from conan.api.model import RecipeReference, PkgReference +from conan.api.output import ConanOutput from conan.errors import ConanException from conan.internal.errors import NotFoundException from conan.internal.model.version_range import VersionRange @@ -104,7 +107,7 @@ def load_graph(graphfile, graph_recipes=None, graph_binaries=None, context=None) ) mpkglist = MultiPackagesList._define_graph(graph, graph_recipes, graph_binaries, - context=base_context) + context=base_context) if context == "build-only": host = MultiPackagesList._define_graph(graph, graph_recipes, graph_binaries, context="host") @@ -150,11 +153,11 @@ def _define_graph(graph, graph_recipes=None, graph_binaries=None, context=None): continue pyref = RecipeReference.loads(pyref) if any(r == "*" or r == pyrecipe for r in recipes): - cache_list.add_refs([pyref]) + cache_list.add_ref(pyref) pyremote = pyreq["remote"] if pyremote: remote_list = pkglist.lists.setdefault(pyremote, PackagesList()) - remote_list.add_refs([pyref]) + remote_list.add_ref(pyref) recipe = node["recipe"] if recipe in (RECIPE_EDITABLE, RECIPE_CONSUMER, RECIPE_VIRTUAL, RECIPE_PLATFORM): @@ -165,18 +168,18 @@ def _define_graph(graph, graph_recipes=None, graph_binaries=None, context=None): ref.timestamp = node["rrev_timestamp"] recipe = recipe.lower() if any(r == "*" or r == recipe for r in recipes): - cache_list.add_refs([ref]) + cache_list.add_ref(ref) remote = node["remote"] if remote: remote_list = pkglist.lists.setdefault(remote, PackagesList()) - remote_list.add_refs([ref]) + remote_list.add_ref(ref) pref = PkgReference(ref, node["package_id"], node["prev"], node["prev_timestamp"]) binary_remote = node["binary_remote"] if binary_remote: remote_list = pkglist.lists.setdefault(binary_remote, PackagesList()) - remote_list.add_refs([ref]) # Binary listed forces recipe listed - remote_list.add_prefs(ref, [pref]) + remote_list.add_ref(ref) # Binary listed forces recipe listed + remote_list.add_pref(pref) binary = node["binary"] if binary in (BINARY_SKIP, BINARY_INVALID, BINARY_MISSING): @@ -184,18 +187,23 @@ def _define_graph(graph, graph_recipes=None, graph_binaries=None, context=None): binary = binary.lower() if any(b == "*" or b == binary for b in binaries): - cache_list.add_refs([ref]) # Binary listed forces recipe listed - cache_list.add_prefs(ref, [pref]) - cache_list.add_configurations({pref: node["info"]}) + cache_list.add_ref(ref) # Binary listed forces recipe listed + cache_list.add_pref(pref, node["info"]) return pkglist class PackagesList: """ A collection of recipes, revisions and packages.""" def __init__(self): - self.recipes = {} + self._data = {} + + def __bool__(self): + """ Whether the package list contains any recipe""" + return bool(self._data) def merge(self, other): + assert isinstance(other, PackagesList) + def recursive_dict_update(d, u): # TODO: repeated from conandata.py for k, v in u.items(): if isinstance(v, dict): @@ -203,70 +211,94 @@ def recursive_dict_update(d, u): # TODO: repeated from conandata.py else: d[k] = v return d - recursive_dict_update(self.recipes, other.recipes) + recursive_dict_update(self._data, other._data) def keep_outer(self, other): - if not self.recipes: + assert isinstance(other, PackagesList) + if not self._data: return - for ref, info in other.recipes.items(): - if self.recipes.get(ref, {}) == info: - self.recipes.pop(ref) + for ref, info in other._data.items(): + if self._data.get(ref, {}) == info: + self._data.pop(ref) def split(self): """ - Returns a list of PackageList, splitted one per reference. + Returns a list of PackageList, split one per reference. This can be useful to parallelize things like upload, parallelizing per-reference """ result = [] - for r, content in self.recipes.items(): + for r, content in self._data.items(): subpkglist = PackagesList() - subpkglist.recipes[r] = content + subpkglist._data[r] = content result.append(subpkglist) return result - def only_recipes(self): - result = {} - for ref, ref_dict in self.recipes.items(): + def only_recipes(self) -> None: + """ Filter out all the packages and package revisions, keep only the recipes and + recipe revisions in self._data. + """ + for ref, ref_dict in self._data.items(): for rrev_dict in ref_dict.get("revisions", {}).values(): rrev_dict.pop("packages", None) - return result def add_refs(self, refs): + ConanOutput().warning("PackagesLists.add_refs() non-public, non-documented method will be " + "removed, use .add_ref() instead", warn_tag="deprecated") # RREVS alreday come in ASCENDING order, so upload does older revisions first for ref in refs: - ref_dict = self.recipes.setdefault(str(ref), {}) - if ref.revision: - revs_dict = ref_dict.setdefault("revisions", {}) - rev_dict = revs_dict.setdefault(ref.revision, {}) - if ref.timestamp: - rev_dict["timestamp"] = ref.timestamp + self.add_ref(ref) + + def add_ref(self, ref: RecipeReference) -> None: + """ + Adds a new RecipeReference to a package list + """ + ref_dict = self._data.setdefault(str(ref), {}) + if ref.revision: + revs_dict = ref_dict.setdefault("revisions", {}) + rev_dict = revs_dict.setdefault(ref.revision, {}) + if ref.timestamp: + rev_dict["timestamp"] = ref.timestamp def add_prefs(self, rrev, prefs): + ConanOutput().warning("PackageLists.add_prefs() non-public, non-documented method will be " + "removed, use .add_pref() instead", warn_tag="deprecated") # Prevs already come in ASCENDING order, so upload does older revisions first - revs_dict = self.recipes[str(rrev)]["revisions"] - rev_dict = revs_dict[rrev.revision] - packages_dict = rev_dict.setdefault("packages", {}) + for p in prefs: + self.add_pref(p) - for pref in prefs: - package_dict = packages_dict.setdefault(pref.package_id, {}) - if pref.revision: - prevs_dict = package_dict.setdefault("revisions", {}) - prev_dict = prevs_dict.setdefault(pref.revision, {}) - if pref.timestamp: - prev_dict["timestamp"] = pref.timestamp + def add_pref(self, pref: PkgReference, pkg_info: dict = None) -> None: + """ + Add a PkgReference to an already existing RecipeReference inside a package list + """ + # Prevs already come in ASCENDING order, so upload does older revisions first + rev_dict = self.recipe_dict(pref.ref) + packages_dict = rev_dict.setdefault("packages", {}) + package_dict = packages_dict.setdefault(pref.package_id, {}) + if pref.revision: + prevs_dict = package_dict.setdefault("revisions", {}) + prev_dict = prevs_dict.setdefault(pref.revision, {}) + if pref.timestamp: + prev_dict["timestamp"] = pref.timestamp + if pkg_info is not None: + package_dict["info"] = pkg_info def add_configurations(self, confs): + ConanOutput().warning("PackageLists.add_configurations() non-public, non-documented method " + "will be removed, use .add_pref() instead", + warn_tag="deprecated") for pref, conf in confs.items(): - rev_dict = self.recipes[str(pref.ref)]["revisions"][pref.ref.revision] + rev_dict = self.recipe_dict(pref.ref) try: rev_dict["packages"][pref.package_id]["info"] = conf except KeyError: # If package_id does not exist, do nothing, only add to existing prefs pass def refs(self): + ConanOutput().warning("PackageLists.refs() non-public, non-documented method will be " + "removed, use .items() instead", warn_tag="deprecated") result = {} - for ref, ref_dict in self.recipes.items(): + for ref, ref_dict in self._data.items(): for rrev, rrev_dict in ref_dict.get("revisions", {}).items(): t = rrev_dict.get("timestamp") recipe = RecipeReference.loads(f"{ref}#{rrev}") # TODO: optimize this @@ -275,8 +307,45 @@ def refs(self): result[recipe] = rrev_dict return result + def items(self) -> Iterable[Tuple[RecipeReference, Dict[PkgReference, Dict]]]: + """ Iterate the contents of the package list. + + The first dictionary is the information directly belonging to the recipe-revision. + The second dictionary contains PkgReference as keys, and a dictionary with the values + belonging to that specific package reference (settings, options, etc.). + """ + for ref, ref_dict in self._data.items(): + for rrev, rrev_dict in ref_dict.get("revisions", {}).items(): + recipe = RecipeReference.loads(f"{ref}#{rrev}") # TODO: optimize this + t = rrev_dict.get("timestamp") + if t is not None: + recipe.timestamp = t + packages = {} + for package_id, pkg_info in rrev_dict.get("packages", {}).items(): + prevs = pkg_info.get("revisions", {}) + for prev, prev_info in prevs.items(): + t = prev_info.get("timestamp") + pref = PkgReference(recipe, package_id, prev, t) + packages[pref] = prev_info + yield recipe, packages + + def recipe_dict(self, ref: RecipeReference): + """ Gives read/write access to the dictionary containing a specific RecipeReference + information. + """ + return self._data[str(ref)]["revisions"][ref.revision] + + def package_dict(self, pref: PkgReference): + """ Gives read/write access to the dictionary containing a specific PkgReference + information + """ + ref_dict = self.recipe_dict(pref.ref) + return ref_dict["packages"][pref.package_id]["revisions"][pref.revision] + @staticmethod def prefs(ref, recipe_bundle): + ConanOutput().warning("PackageLists.prefs() non-public, non-documented method will be " + "removed, use .items() instead", warn_tag="deprecated") result = {} for package_id, pkg_bundle in recipe_bundle.get("packages", {}).items(): prevs = pkg_bundle.get("revisions", {}) @@ -288,13 +357,13 @@ def prefs(ref, recipe_bundle): def serialize(self): """ Serialize the instance to a dictionary.""" - return self.recipes.copy() + return copy.deepcopy(self._data) @staticmethod def deserialize(data): """ Loads the data from a serialized dictionary.""" result = PackagesList() - result.recipes = data + result._data = copy.deepcopy(data) return result @@ -357,10 +426,10 @@ def _version_range(self): if self.version and self.version.startswith("[") and self.version.endswith("]"): return VersionRange(self.version[1:-1]) - def filter_versions(self, refs): + def filter_versions(self, refs, resolve_prereleases=None): vrange = self._version_range if vrange: - refs = [r for r in refs if vrange.contains(r.version, None)] + refs = [r for r in refs if vrange.contains(r.version, resolve_prereleases)] return refs @property diff --git a/conan/api/model/remote.py b/conan/api/model/remote.py index d8ab67f2bb0..728957f41dc 100644 --- a/conan/api/model/remote.py +++ b/conan/api/model/remote.py @@ -3,16 +3,29 @@ class Remote: """ - The ``Remote`` class represents a remote registry of packages. It's a read-only opaque object that - should not be created directly, but obtained from the relevant ``RemotesAPI`` subapi methods. + The ``Remote`` class represents a remote registry of packages. """ def __init__(self, name, url, verify_ssl=True, disabled=False, allowed_packages=None, - remote_type=None): + remote_type=None, recipes_only=False): + """ A Remote object can be constructed to be passed as an argument to + RemotesAPI methods. When possible, it is better to use Remote objects returned by the API, + but for the ``RemotesAPI.add()`` method, for which a new constructed object is necessary. + It is recommended to use named arguments like ``Remote(..., verify_ssl=False)`` in + the constructor. + :param name: The name of the remote. + :param url: The URL of the remote repository (or local folder for "local-recipes-index"). + :param verify_ssl: Enable SSL Certificate validation. + :param disabled: Disable the remote repository. + :param allowed_packages: List of patterns of allowed packages from this remote + :param remote_type: Type of the remote repository, use "local-recipes-index" or ``None`` + :param recipes_only: If True, binaries form this remote will be ignored and never used + """ self.name = name # Read only, is the key self.url = url self.verify_ssl = verify_ssl self.disabled = disabled self.allowed_packages = allowed_packages + self.recipes_only = recipes_only self.remote_type = remote_type self._caching = {} @@ -26,6 +39,8 @@ def __str__(self): allowed_msg = "" if self.allowed_packages: allowed_msg = ", Allowed packages: {}".format(", ".join(self.allowed_packages)) + if self.recipes_only: + allowed_msg += ", Recipes only" if self.remote_type == LOCAL_RECIPES_INDEX: return "{}: {} [{}, Enabled: {}{}]".format(self.name, self.url, LOCAL_RECIPES_INDEX, not self.disabled, allowed_msg) diff --git a/conan/api/output.py b/conan/api/output.py index 30391cba97a..520e5d560cb 100644 --- a/conan/api/output.py +++ b/conan/api/output.py @@ -124,22 +124,34 @@ def define_silence_warnings(cls, warnings): def set_warnings_as_errors(cls, value): cls._warnings_as_errors = value + @classmethod + def get_output_level(cls): + return cls._conan_output_level + + @classmethod + def set_output_level(cls, level): + cls._conan_output_level = level + + @classmethod + def valid_log_levels(cls): + return {"quiet": LEVEL_QUIET, # -vquiet 80 + "error": LEVEL_ERROR, # -verror 70 + "warning": LEVEL_WARNING, # -vwaring 60 + "notice": LEVEL_NOTICE, # -vnotice 50 + "status": LEVEL_STATUS, # -vstatus 40 + None: LEVEL_VERBOSE, # -v 30 + "verbose": LEVEL_VERBOSE, # -vverbose 30 + "debug": LEVEL_DEBUG, # -vdebug 20 + "v": LEVEL_DEBUG, # -vv 20 + "trace": LEVEL_TRACE, # -vtrace 10 + "vv": LEVEL_TRACE # -vvv 10 + } + @classmethod def define_log_level(cls, v): env_level = os.getenv("CONAN_LOG_LEVEL") v = env_level or v - levels = {"quiet": LEVEL_QUIET, # -vquiet 80 - "error": LEVEL_ERROR, # -verror 70 - "warning": LEVEL_WARNING, # -vwaring 60 - "notice": LEVEL_NOTICE, # -vnotice 50 - "status": LEVEL_STATUS, # -vstatus 40 - None: LEVEL_VERBOSE, # -v 30 - "verbose": LEVEL_VERBOSE, # -vverbose 30 - "debug": LEVEL_DEBUG, # -vdebug 20 - "v": LEVEL_DEBUG, # -vv 20 - "trace": LEVEL_TRACE, # -vtrace 10 - "vv": LEVEL_TRACE # -vvv 10 - } + levels = cls.valid_log_levels() try: level = levels[v] except KeyError: @@ -147,7 +159,7 @@ def define_log_level(cls, v): vals = "quiet, error, warning, notice, status, verbose, debug(v), trace(vv)" raise ConanException(f"Invalid argument '-v{v}'{msg}.\nAllowed values: {vals}") else: - cls._conan_output_level = level + cls.set_output_level(level) @classmethod def level_allowed(cls, level): @@ -266,13 +278,13 @@ def verbose(self, msg: str, fg: str = None, bg: str = None): self._write_message(msg, fg=fg, bg=bg) return self - def status(self, msg: str, fg: str = None, bg: str = None): + def status(self, msg: str, fg: str = None, bg: str = None, newline: bool = True): """ Provides general information about the system or ongoing operations. Info messages are basic and used to inform about common events, like the start or completion of processes, without implying specific problems or achievements.""" if self._conan_output_level <= LEVEL_STATUS: - self._write_message(msg, fg=fg, bg=bg) + self._write_message(msg, fg=fg, bg=bg, newline=newline) return self info = status diff --git a/conan/api/subapi/audit.py b/conan/api/subapi/audit.py index bc7afa26f1b..9545490f5aa 100644 --- a/conan/api/subapi/audit.py +++ b/conan/api/subapi/audit.py @@ -28,12 +28,14 @@ def __init__(self, conan_api): } @staticmethod - def scan(deps_graph, provider): + def scan(deps_graph, provider, context=None): """ Scan a given recipe for vulnerabilities in its dependencies. """ refs = sorted(set(RecipeReference.loads(f"{node.ref.name}/{node.ref.version}") - for node in deps_graph.nodes[1:]), key=lambda ref: ref.name) + for node in deps_graph.nodes[1:] + if context is None or node.context == context), + key=lambda ref: ref.name) return provider.get_cves(refs) @staticmethod @@ -66,9 +68,9 @@ def get_provider(self, provider_name): ) raise ConanException( - f"Provider '{provider_name}' not found. Please specify a valid provider name or add it using: " - f"'conan audit provider add {provider_name} {add_arguments} --token='\n" - f"{register_message}" + f"Provider '{provider_name}' not found. Please specify a valid provider name or add " + f"it using: 'conan audit provider add {provider_name} {add_arguments} " + f"--token='\n{register_message}" ) provider_data = providers[provider_name] @@ -80,9 +82,11 @@ def get_provider(self, provider_name): provider_data["token"] = env_token elif "token" in provider_data: try: - provider_data["token"] = decode(base64.standard_b64decode(provider_data["token"]).decode(), CYPHER_KEY) - except binascii.Error as e: - raise ConanException(f"Invalid token format for provider '{provider_name}'. The token might be corrupt.") + enc_token = base64.standard_b64decode(provider_data["token"]).decode() + provider_data["token"] = decode(enc_token, CYPHER_KEY) + except binascii.Error: + raise ConanException(f"Invalid token format for provider '{provider_name}'. " + f"The token might be corrupt.") provider_cls = self._provider_cls.get(provider_data["type"]) @@ -140,7 +144,8 @@ def auth_provider(self, provider, token): providers = _load_providers(self._providers_path) assert provider.name in providers - providers[provider.name]["token"] = base64.standard_b64encode(encode(token, CYPHER_KEY).encode()).decode() + encode_token = encode(token, CYPHER_KEY).encode() + providers[provider.name]["token"] = base64.standard_b64encode(encode_token).decode() setattr(provider, "token", token) _save_providers(self._providers_path, providers) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index 5855f72eeb8..da86417e0e1 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -5,13 +5,15 @@ from conan.api.model import PackagesList from conan.api.output import ConanOutput -from conan.internal.api.uploader import compress_files +from conan.internal.api.uploader import compress_files, get_compress_level from conan.internal.cache.cache import PkgCache -from conan.internal.cache.conan_reference_layout import EXPORT_SRC_FOLDER, EXPORT_FOLDER, SRC_FOLDER, \ - METADATA, DOWNLOAD_EXPORT_FOLDER +from conan.internal.cache.conan_reference_layout import (EXPORT_SRC_FOLDER, EXPORT_FOLDER, + SRC_FOLDER, METADATA, + DOWNLOAD_EXPORT_FOLDER) from conan.internal.cache.home_paths import HomePaths from conan.internal.cache.integrity_check import IntegrityChecker from conan.internal.loader import load_python_file +from conan.internal.paths import COMPRESSIONS from conan.internal.rest.download_cache import DownloadCache from conan.errors import ConanException from conan.api.model import PkgReference @@ -21,6 +23,8 @@ class CacheAPI: + """ This CacheAPI is used to interact with the packages storage cache + """ def __init__(self, conan_api, api_helpers): self._conan_api = conan_api @@ -71,17 +75,29 @@ def package_path(self, pref: PkgReference): return ref_layout.finalize() return _check_folder_existence(pref, "package", ref_layout.package()) - def check_integrity(self, package_list): - """Check if the recipes and packages are corrupted (it will raise a ConanExcepcion)""" + def check_integrity(self, package_list, return_pkg_list=False): + """ + Check if the recipes and packages are corrupted + + :param package_list: PackagesList to check + :param return_pkg_list: If True, return a PackagesList with corrupted artifacts + :return: PackagesList with corrupted artifacts if return_pkg_list is True + :raises: ConanExcepcion if there are corrupted artifacts and return_pkg_list is False + """ cache = PkgCache(self._conan_api.cache_folder, self._api_helpers.global_conf) checker = IntegrityChecker(cache) - checker.check(package_list) + corrupted_pkg_list = checker.check(package_list) + if return_pkg_list: + return corrupted_pkg_list + if corrupted_pkg_list: + raise ConanException("There are corrupted artifacts, check the error logs") def clean(self, package_list, source=True, build=True, download=True, temp=True, backup_sources=False): """ Remove non critical folders from the cache, like source, build and download (.tgz store) folders. + :param package_list: the package lists that should be cleaned :param source: boolean, remove the "source" folder if True :param build: boolean, remove the "build" folder if True @@ -105,19 +121,20 @@ def clean(self, package_list, source=True, build=True, download=True, temp=True, if not os.path.exists(manifest) or not os.path.exists(info): rmdir(folder) if backup_sources: - backup_files = self._conan_api.cache.get_backup_sources(package_list, exclude=False, only_upload=False) + backup_files = self._conan_api.cache.get_backup_sources(package_list, exclude=False, + only_upload=False) ConanOutput().verbose(f"Cleaning {len(backup_files)} backup sources") for f in backup_files: remove(f) - for ref, ref_bundle in package_list.refs().items(): + for ref, packages in package_list.items(): ConanOutput(ref.repr_notime()).verbose("Cleaning recipe cache contents") ref_layout = cache.recipe_layout(ref) if source: rmdir(ref_layout.source()) if download: rmdir(ref_layout.download_export()) - for pref, _ in package_list.prefs(ref, ref_bundle).items(): + for pref in packages: ConanOutput(pref).verbose("Cleaning package cache contents") pref_layout = cache.pkg_layout(pref) if build: @@ -127,18 +144,24 @@ def clean(self, package_list, source=True, build=True, download=True, temp=True, if download: rmdir(pref_layout.download_package()) - def save(self, package_list, tgz_path, no_source=False): + def save(self, package_list: PackagesList, tgz_path, no_source=False) -> None: global_conf = self._api_helpers.global_conf cache = PkgCache(self._conan_api.cache_folder, global_conf) cache_folder = cache.store # Note, this is not the home, but the actual package cache out = ConanOutput() mkdir(os.path.dirname(tgz_path)) + tgz_name = os.path.basename(tgz_path) + compressformat = next((e for e in COMPRESSIONS if tgz_name.endswith(e)), None) + if not compressformat: + raise ConanException(f"Unsupported compression format for {tgz_name}") + compresslevel = get_compress_level(compressformat, global_conf) tar_files: dict[str, str] = {} # {path_in_tar: abs_path} - for ref, ref_bundle in package_list.refs().items(): + for ref, packages in package_list.items(): ref_layout = cache.recipe_layout(ref) recipe_folder = os.path.relpath(ref_layout.base_folder, cache_folder) recipe_folder = recipe_folder.replace("\\", "/") # make win paths portable + ref_bundle = package_list.recipe_dict(ref) ref_bundle["recipe_folder"] = recipe_folder out.info(f"Saving {ref}: {recipe_folder}") # Package only selected folders, not DOWNLOAD one @@ -152,19 +175,20 @@ def save(self, package_list, tgz_path, no_source=False): if os.path.exists(path): tar_files[f"{recipe_folder}/{DOWNLOAD_EXPORT_FOLDER}/{METADATA}"] = path - for pref, pref_bundle in package_list.prefs(ref, ref_bundle).items(): + for pref in packages: pref_layout = cache.pkg_layout(pref) pkg_folder = pref_layout.package() folder = os.path.relpath(pkg_folder, cache_folder) folder = folder.replace("\\", "/") # make win paths portable - pref_bundle["package_folder"] = folder + pkg_dict = package_list.package_dict(pref) + pkg_dict["package_folder"] = folder out.info(f"Saving {pref}: {folder}") tar_files[folder] = os.path.join(cache_folder, folder) if os.path.exists(pref_layout.metadata()): metadata_folder = os.path.relpath(pref_layout.metadata(), cache_folder) metadata_folder = metadata_folder.replace("\\", "/") # make paths portable - pref_bundle["metadata_folder"] = metadata_folder + pkg_dict["metadata_folder"] = metadata_folder out.info(f"Saving {pref} metadata: {metadata_folder}") tar_files[metadata_folder] = os.path.join(cache_folder, metadata_folder) @@ -173,11 +197,12 @@ def save(self, package_list, tgz_path, no_source=False): pkglist_path = os.path.join(tempfile.gettempdir(), "pkglist.json") save(pkglist_path, serialized) tar_files["pkglist.json"] = pkglist_path - compress_files(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), conf=self._conan_api.config, - recursive=True, ref=None, compression_plugin=self.compression_plugin) + compress_files(tar_files, tgz_name, os.path.dirname(tgz_path), compresslevel, + recursive=True, compression_plugin=self.compression_plugin) remove(pkglist_path) + ConanOutput().success(f"Created cache save file: {tgz_path}") - def restore(self, path): + def restore(self, path) -> PackagesList: if not os.path.isfile(path): raise ConanException(f"Restore archive doesn't exist in {path}") @@ -202,7 +227,8 @@ def restore(self, path): # After unzipping the files, we need to update the DB that references these files out = ConanOutput() package_list = PackagesList.deserialize(json.loads(pkglist)) - for ref, ref_bundle in package_list.refs().items(): + for ref, packages in package_list.items(): + ref_bundle = package_list.recipe_dict(ref) ref.timestamp = revision_timestamp_now() ref_bundle["timestamp"] = ref.timestamp try: @@ -215,8 +241,9 @@ def restore(self, path): # In the case of recipes, they are always "in place", so just checking it assert rel_path == recipe_folder, f"{rel_path}!={recipe_folder}" out.info(f"Restore: {ref} in {recipe_folder}") - for pref, pref_bundle in package_list.prefs(ref, ref_bundle).items(): + for pref in packages: pref.timestamp = revision_timestamp_now() + pref_bundle = package_list.package_dict(pref) pref_bundle["timestamp"] = pref.timestamp try: pkg_layout = cache.pkg_layout(pref) @@ -254,17 +281,21 @@ def restore(self, path): def get_backup_sources(self, package_list=None, exclude=True, only_upload=True): """Get list of backup source files currently present in the cache, - either all of them if no argument, or filtered by those belonging to the references in the package_list - - @param package_list: a PackagesList object to filter backup files from (The files should have been downloaded form any of the references in the package_list) - @param exclude: if True, exclude the sources that come from URLs present the core.sources:exclude_urls global conf - @param only_upload: if True, only return the files for packages that are set to be uploaded + either all of them if no argument, or filtered by those belonging to the references + in the package_list + + :param package_list: a PackagesList object to filter backup files from (The files should + have been downloaded form any of the references in the package_list) + :param exclude: if True, exclude the sources that come from URLs present the + core.sources:exclude_urls global conf + :param only_upload: if True, only return the files for packages that are set to be uploaded """ config = self._api_helpers.global_conf download_cache_path = config.get("core.sources:download_cache") download_cache_path = download_cache_path or HomePaths( self._conan_api.cache_folder).default_sources_backup_folder - excluded_urls = config.get("core.sources:exclude_urls", check_type=list, default=[]) if exclude else [] + excluded_urls = config.get("core.sources:exclude_urls", + check_type=list, default=[]) if exclude else [] download_cache = DownloadCache(download_cache_path) return download_cache.get_backup_sources_files(excluded_urls, package_list, only_upload) @@ -276,24 +307,11 @@ def path_to_ref(self, path): result = cache.path_to_ref(base) return result - @property - def compression_plugin(self): - if self._compression_plugin is None: - compression_plugin_path = HomePaths(self._conan_api.home_folder).compression_plugin_path - if not os.path.exists(compression_plugin_path): - self._compression_plugin = False # Avoid FS re-check - return None - mod, _ = load_python_file(compression_plugin_path) - if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"): - raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions") - self._compression_plugin = mod - return self._compression_plugin or None - def _resolve_latest_ref(cache, ref): if ref.revision is None or ref.revision == "latest": ref.revision = None - result = cache.get_latest_recipe_reference(ref) + result = cache.get_latest_recipe_revision(ref) if result is None: raise ConanException(f"'{ref}' not found in cache") ref = result @@ -304,7 +322,7 @@ def _resolve_latest_pref(cache, pref): pref.ref = _resolve_latest_ref(cache, pref.ref) if pref.revision is None or pref.revision == "latest": pref.revision = None - result = cache.get_latest_package_reference(pref) + result = cache.get_latest_package_revision(pref) if result is None: raise ConanException(f"'{pref.repr_notime()}' not found in cache") pref = result diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index 4fdb2b91e54..bf3c212b6de 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -1,4 +1,3 @@ -import json import os from conan.api.output import ConanOutput @@ -9,10 +8,12 @@ from conan.internal.graph.graph_builder import DepsGraphBuilder from conan.internal.graph.profile_node_definer import consumer_definer from conan.errors import ConanException + +from conan.internal.model.conanconfig import loadconanconfig, saveconanconfig, loadconanconfig_yml from conan.internal.model.conf import BUILT_IN_CONFS from conan.internal.model.pkg_type import PackageType -from conan.api.model import RecipeReference, PkgReference -from conan.internal.util.files import load, save, rmdir, remove +from conan.api.model import RecipeReference, Remote +from conan.internal.util.files import rmdir, remove class ConfigAPI: @@ -42,78 +43,152 @@ def install(self, path_or_url, verify_ssl, config_type=None, args=None, """ from conan.internal.api.config.config_installer import configuration_install cache_folder = self._conan_api.cache_folder - requester = self._conan_api.remotes.requester + requester = self._helpers.requester configuration_install(cache_folder, requester, path_or_url, verify_ssl, config_type=config_type, args=args, source_folder=source_folder, target_folder=target_folder) self._conan_api.reinit() - def install_pkg(self, ref, lockfile=None, force=False, remotes=None, - profile=None) -> PkgReference: + def install_package(self, require, lockfile=None, force=False, remotes=None, profile=None): + ConanOutput().warning("The 'conan config install-pkg' is experimental", + warn_tag="experimental") + require = RecipeReference.loads(require) + required_pkgs = self.fetch_packages([require], lockfile, remotes, profile) + installed_refs = self._install_pkgs(required_pkgs, force) + self._conan_api.reinit() + return installed_refs + + @staticmethod + def load_conanconfig(path, remotes): + if os.path.isdir(path): + path = os.path.join(path, "conanconfig.yml") + requested_requires, urls = loadconanconfig_yml(path) + if urls: + new_remotes = [Remote(f"config_install_url{'_' + str(i)}", url=url) + for i, url in enumerate(urls)] + remotes = remotes or [] + remotes += new_remotes + return requested_requires, remotes + + def install_conanconfig(self, path, lockfile=None, force=False, remotes=None, profile=None): + ConanOutput().warning("The 'conan config install-pkg' is experimental", + warn_tag="experimental") + requested_requires, remotes = self.load_conanconfig(path, remotes) + required_pkgs = self.fetch_packages(requested_requires, lockfile, remotes, profile) + installed_refs = self._install_pkgs(required_pkgs, force) + self._conan_api.reinit() + return installed_refs + + def _install_pkgs(self, required_pkgs, force): + out = ConanOutput() + out.title("Configuration packages to install") + config_version_file = HomePaths(self._conan_api.home_folder).config_version_path + if not os.path.exists(config_version_file): + config_versions = [] + else: + ConanOutput().info(f"Reading existing config-versions file: {config_version_file}") + config_versions = loadconanconfig(config_version_file) + config_versions_dict = {r.name: r for r in config_versions} + if len(config_versions_dict) < len(config_versions): + raise ConanException("There are multiple requirements for the same package " + f"with different versions: {config_version_file}") + + new_config = config_versions_dict.copy() + for required_pkg in required_pkgs: + new_config.pop(required_pkg.ref.name, None) # To ensure new order + new_config[required_pkg.ref.name] = required_pkg.ref + final_config_refs = [r for r in new_config.values()] + + prev_refs = "\n\t".join(repr(r) for r in config_versions) + out.info(f"Previously installed configuration packages:\n\t{prev_refs}") + + new_refs = "\n\t".join(r.repr_notime() for r in final_config_refs) + out.info(f"New configuration packages to install:\n\t{new_refs}") + + if list(config_versions_dict) == list(new_config)[:len(config_versions_dict)]: + # There is no conflict in order, can be done safely + if final_config_refs == config_versions: + if force: + out.warning("The requested configurations are identical to the already " + "installed ones, but forcing re-installation because --force") + to_install = required_pkgs + else: + out.info("The requested configurations are identical to the already " + "installed ones, skipping re-installation") + to_install = [] + else: + out.info("Installing new or updating configuration packages") + to_install = required_pkgs + else: + # Change in order of existing configuration + if force: + out.warning("Installing these configuration packages will break the " + "existing order, with possible side effects. " + "Forcing the installation because --force was defined", warn_tag="risk") + to_install = required_pkgs + else: + msg = ("Installing these configuration packages will break the " + "existing order, with possible side effects, like breaking 'package_ids'.\n" + "If you still want to enforce this configuration you can:\n" + " Use 'conan config clean' first to fully reset your configuration.\n" + " Or use 'conan config install-pkg --force' to force installation.") + raise ConanException(msg) + + out.title("Installing configuration from packages") + # install things and update the Conan cache "config_versions.json" file + from conan.internal.api.config.config_installer import configuration_install + cache_folder = self._conan_api.cache_folder + requester = self._helpers.requester + for pkg in to_install: + out.info(f"Installing configuration from {pkg.ref}") + configuration_install(cache_folder, requester, uri=pkg.conanfile.package_folder, + verify_ssl=False, config_type="dir", + ignore=["conaninfo.txt", "conanmanifest.txt"]) + + saveconanconfig(config_version_file, final_config_refs) + return final_config_refs + + def fetch_packages(self, refs, lockfile=None, remotes=None, profile=None): """ install configuration stored inside a Conan package The installation of configuration will reinitialize the full ConanAPI """ - ConanOutput().warning("The 'conan config install-pkg' is experimental", - warn_tag="experimental") conan_api = self._conan_api remotes = conan_api.remotes.list() if remotes is None else remotes profile_host = profile_build = profile or conan_api.profiles.get_profile([]) app = ConanApp(self._conan_api) - # Computation of a very simple graph that requires "ref" - conanfile = app.loader.load_virtual(requires=[RecipeReference.loads(ref)]) - consumer_definer(conanfile, profile_host, profile_build) - root_node = Node(ref=None, conanfile=conanfile, context=CONTEXT_HOST, recipe=RECIPE_VIRTUAL) - root_node.is_conf = True - update = ["*"] - builder = DepsGraphBuilder(app.proxy, app.loader, app.range_resolver, app.cache, remotes, - update, update, self._helpers.global_conf) - deps_graph = builder.load_graph(root_node, profile_host, profile_build, lockfile) - - # Basic checks of the package: correct package_type and no-dependencies - deps_graph.report_graph_error() - pkg = deps_graph.root.edges[0].dst - ConanOutput().info(f"Configuration from package: {pkg}") - if pkg.conanfile.package_type is not PackageType.CONF: - raise ConanException(f'{pkg.conanfile} is not of package_type="configuration"') - if pkg.edges: - raise ConanException(f"Configuration package {pkg.ref} cannot have dependencies") - - # The computation of the "package_id" and the download of the package is done as usual - # By default we allow all remotes, and build_mode=None, always updating - conan_api.graph.analyze_binaries(deps_graph, None, remotes, update=update, lockfile=lockfile) - conan_api.install.install_binaries(deps_graph=deps_graph, remotes=remotes) - - # We check if this specific version is already installed - config_pref = pkg.pref.repr_notime() - config_versions = [] - config_version_file = HomePaths(conan_api.home_folder).config_version_path - if os.path.exists(config_version_file): - config_versions = json.loads(load(config_version_file)) - config_versions = config_versions["config_version"] - if config_pref in config_versions: - if force: - ConanOutput().info(f"Package '{pkg}' already configured, " - "but re-installation forced") - else: - ConanOutput().info(f"Package '{pkg}' already configured, " - "skipping configuration install") - return pkg.pref # Already installed, we can skip repeating the install + ConanOutput().title("Fetching requested configuration packages") + result = [] + for ref in refs: + # Computation of a very simple graph that requires "ref" + # Need to convert input requires to RecipeReference + conanfile = app.loader.load_virtual(requires=[ref]) + consumer_definer(conanfile, profile_host, profile_build) + root_node = Node(ref=None, conanfile=conanfile, context=CONTEXT_HOST, + recipe=RECIPE_VIRTUAL) + root_node.is_conf = True + update = ["*"] + builder = DepsGraphBuilder(app.proxy, app.loader, app.range_resolver, app.cache, remotes, + update, update, self._helpers.global_conf) + deps_graph = builder.load_graph(root_node, profile_host, profile_build, lockfile) - from conan.internal.api.config.config_installer import configuration_install - cache_folder = self._conan_api.cache_folder - requester = self._conan_api.remotes.requester - configuration_install(cache_folder, requester, uri=pkg.conanfile.package_folder, - verify_ssl=False, config_type="dir", - ignore=["conaninfo.txt", "conanmanifest.txt"]) - # We save the current package full reference in the file for future - # And for ``package_id`` computation - config_versions = {ref.split("/", 1)[0]: ref for ref in config_versions} - config_versions[pkg.pref.ref.name] = pkg.pref.repr_notime() - save(config_version_file, json.dumps({"config_version": list(config_versions.values())})) - self._conan_api.reinit() - return pkg.pref + # Basic checks of the package: correct package_type and no-dependencies + deps_graph.report_graph_error() + pkg = deps_graph.root.edges[0].dst + ConanOutput().info(f"Configuration from package: {pkg}") + if pkg.conanfile.package_type is not PackageType.CONF: + raise ConanException(f'{pkg.conanfile} is not of package_type="configuration"') + if pkg.edges: + raise ConanException(f"Configuration package {pkg.ref} cannot have dependencies") + + # The computation of the "package_id" and the download of the package is done as usual + # By default we allow all remotes, and build_mode=None, always updating + conan_api.graph.analyze_binaries(deps_graph, None, remotes, update=update, + lockfile=lockfile) + conan_api.install.install_binaries(deps_graph=deps_graph, remotes=remotes) + result.append(pkg) + return result def get(self, name, default=None, check_type=None): """ get the value of a global.conf item @@ -150,3 +225,40 @@ def clean(self): self._conan_api.reinit() # CHECK: This also generates a remotes.json that is not there after a conan profile show? self._conan_api.migrate() + + @property + def settings_yml(self): + """ Get the contents of the settings.yml and user_settings.yml files, + which define the possible values for settings. + + Note that this is different from the settings present in a conanfile, + which represent the actual values for a specific package, while this + property represents the possible values for each setting. + + :returns: A read-only object representing the settings scheme, with a + ``possible_values()`` method that returns a dictionary with the possible values for each setting, + and a ``fields`` property that returns an ordered list with the fields of each setting. + Note that it's possible to access nested settings using attribute access, + such as ``settings_yml.compiler.possible_values()``. + """ + + class SettingsYmlInterface: + def __init__(self, settings): + self._settings = settings + + def possible_values(self): + """ returns a dict with the possible values for each setting """ + return self._settings.possible_values() + + @property + def fields(self): + """ returns a dict with the fields of each setting """ + return self._settings.fields + + def __getattr__(self, item): + return SettingsYmlInterface(getattr(self._settings, item)) + + def __str__(self): + return str(self._settings) + + return SettingsYmlInterface(self._helpers.settings_yml) diff --git a/conan/api/subapi/download.py b/conan/api/subapi/download.py index 145e2430d51..a4421a2b4d1 100644 --- a/conan/api/subapi/download.py +++ b/conan/api/subapi/download.py @@ -11,6 +11,7 @@ class DownloadAPI: + """ This API is used to download recipes and packages from a remote server.""" def __init__(self, conan_api): self._conan_api = conan_api @@ -36,7 +37,7 @@ def recipe(self, ref: RecipeReference, remote: Remote, metadata: Optional[List[s if ref.timestamp is None: # we didnt obtain the timestamp before (in general it should be) # Respect the timestamp of the server, the ``get_recipe()`` doesn't do it internally # Best would be that ``get_recipe()`` returns the timestamp in the same call - server_ref = app.remote_manager.get_recipe_revision_reference(ref, remote) + server_ref = app.remote_manager.get_recipe_revision(ref, remote) assert server_ref == ref ref.timestamp = server_ref.timestamp app.remote_manager.get_recipe(ref, remote, metadata) @@ -69,7 +70,7 @@ def package(self, pref: PkgReference, remote: Remote, metadata: Optional[List[st if pref.timestamp is None: # we didn't obtain the timestamp before (in general it should be) # Respect the timestamp of the server - server_pref = app.remote_manager.get_package_revision_reference(pref, remote) + server_pref = app.remote_manager.get_package_revision(pref, remote) assert server_pref == pref pref.timestamp = server_pref.timestamp @@ -79,18 +80,24 @@ def package(self, pref: PkgReference, remote: Remote, metadata: Optional[List[st def download_full(self, package_list: PackagesList, remote: Remote, metadata: Optional[List[str]] = None): - """Download the recipes and packages specified in the package_list from the remote, - parallelized based on `core.download:parallel`""" + """Download the recipes and packages specified in the ``package_list`` from the remote, + parallelized based on ``core.download:parallel``""" def _download_pkglist(pkglist): - for ref, recipe_bundle in pkglist.refs().items(): + for ref, packages in pkglist.items(): self.recipe(ref, remote, metadata) - for pref, _ in pkglist.prefs(ref, recipe_bundle).items(): + ref_dict = pkglist.recipe_dict(ref) + ref_dict.pop("files", None) + ref_dict.pop("upload-urls", None) + for pref in packages: self.package(pref, remote, metadata) + pkg_dict = pkglist.package_dict(pref) + pkg_dict.pop("files", None) + pkg_dict.pop("upload-urls", None) t = time.time() parallel = self._conan_api.config.get("core.download:parallel", default=1, check_type=int) thread_pool = ThreadPool(parallel) if parallel > 1 else None - if not thread_pool or len(package_list.refs()) <= 1: + if not thread_pool or len(package_list._data) <= 1: # FIXME: Iteration when multiple rrevs _download_pkglist(package_list) else: ConanOutput().subtitle(f"Downloading with {parallel} parallel threads") diff --git a/conan/api/subapi/export.py b/conan/api/subapi/export.py index 49557a17947..797364c1574 100644 --- a/conan/api/subapi/export.py +++ b/conan/api/subapi/export.py @@ -1,27 +1,115 @@ +import os +from typing import List, Tuple + +from conan import ConanFile from conan.api.output import ConanOutput +from conan.cli.printers.graph import print_graph_basic from conan.internal.cache.cache import PkgCache from conan.internal.conan_app import ConanApp from conan.internal.api.export import cmd_export from conan.internal.methods import run_package_method from conan.internal.graph.graph import BINARY_BUILD, RECIPE_INCACHE -from conan.api.model import PkgReference +from conan.api.model import PkgReference, Remote, RecipeReference from conan.internal.util.files import mkdir class ExportAPI: + """ This API provides methods to export artifacts, both recipes and pre-compiled package + binaries from user folders to the Conan cache, as Conan recipes and Conan package binaries + """ def __init__(self, conan_api, helpers): self._conan_api = conan_api self._helpers = helpers - def export(self, path, name, version, user, channel, lockfile=None, remotes=None): + def export(self, path, name: str = None, version: str = None, user: str = None, + channel: str = None, lockfile=None, + remotes: List[Remote] = None) -> Tuple[RecipeReference, ConanFile]: + """ Exports a ``conanfile.py`` recipe, together with its associated files to the Conan cache. + A "recipe-revision" will be computed and assigned. + + :param path: Path to the conanfile to be exported + :param name: Optional package name. Typically not necessary as it is defined by the recipe + attribute or dynamically with the ``set_name()`` method. + If it is defined in recipe and as an argument, but they don't match, an error will be raised. + :param version: Optional version. It can be defined in the recipe with the version + attribute or dynamically with the 'set_version()' method. + If it is defined in recipe and as an argument, but they don't match, an error will be raised. + :param user: Optional user. Can be defined by recipe attribute. + If it is defined in recipe and as an argument, but they don't match, an error will be raised. + :param channel: Optional channel. Can be defined by recipe attribute. + If it is defined in recipe and as an argument, but they don't match, an error will be raised. + :param lockfile: Optional, only relevant if the recipe has 'python-requires' to be locked + :param remotes: Optional, only relevant to resolve 'python-requires' in remotes + :return: A tuple of the exported RecipeReference and a ConanFile object + """ ConanOutput().title("Exporting recipe to the cache") app = ConanApp(self._conan_api) hook_manager = self._helpers.hook_manager - return cmd_export(app, hook_manager, self._helpers.global_conf, path, name, version, - user, channel, graph_lock=lockfile, remotes=remotes) + return cmd_export(app.loader, app.cache, hook_manager, self._helpers.global_conf, path, + name, version, user, channel, graph_lock=lockfile, remotes=remotes) + + def export_pkg_graph(self, path, ref: RecipeReference, profile_host, profile_build, + remotes: List[Remote], lockfile=None, is_build_require=False, + skip_binaries=False, output_folder=None): + """Computes a dependency graph for a given configuration, for an already existing (previously + exported) recipe in the Conan cache. This method computes the full dependency graph, using + the profiles, lockfile and remotes information as any other install/graph/create command. + This is necessary in order to compute the "package_id" of the binary being exported + into the Conan cache. + The resulting dependency graph can be passed to ``export_pkg()`` method + + :param path: Path to the conanfile.py in the user folder + :param ref: full RecipeReference, including recipe-revision + :param profile_host: Profile for the host context + :param profile_build: Profile for the build context + :param lockfile: Optional lockfile + :param remotes: List of Remotes + :param is_build_require: In case a package intended to be used as a tool-requires + :param skip_binaries: + :param output_folder: The folder containing output files, like potential environment scripts + :return: A Graph object that can be passed to ``export_pkg()`` method + """ + assert ref.revision, "ref argument must have recipe-revision defined" + conan_api = self._conan_api + deps_graph = conan_api.graph.load_graph_consumer(path, + ref.name, str(ref.version), ref.user, + ref.channel, + profile_host=profile_host, + profile_build=profile_build, + lockfile=lockfile, remotes=remotes, + update=None, + is_build_require=is_build_require) + + print_graph_basic(deps_graph) + deps_graph.report_graph_error() + conan_api.graph.analyze_binaries(deps_graph, build_mode=[ref.name], lockfile=lockfile, + remotes=remotes) + deps_graph.report_graph_error() + + root_node = deps_graph.root + root_node.ref = ref # Make sure the root node revision is well defined + + if not skip_binaries: + # unless the user explicitly opts-out with --skip-binaries, it is necessary to install + # binaries, in case there are build_requires necessary like tool-requires=cmake + # and package() method doing ``cmake.install()`` + # for most cases, deps will be in cache already because of a previous "conan install" + # but if it is not the case, the binaries from remotes will be downloaded + conan_api.install.install_binaries(deps_graph=deps_graph, remotes=remotes) + source_folder = os.path.dirname(path) + conan_api.install.install_consumer(deps_graph=deps_graph, source_folder=source_folder, + output_folder=output_folder) + return deps_graph + + def export_pkg(self, graph, output_folder=None) -> None: + """Executes the ``package()`` method of the exported recipe in order to copy the artifacts + from user folder to the Conan cache package folder - def export_pkg(self, deps_graph, source_folder, output_folder): + :param graph: A Graph object + :param output_folder: Optional folder where generated files like environment scripts + of dependencies have been installed + """ cache = PkgCache(self._conan_api.cache_folder, self._helpers.global_conf) hook_manager = self._helpers.hook_manager @@ -29,8 +117,9 @@ def export_pkg(self, deps_graph, source_folder, output_folder): # to be downloaded from remotes # passing here the create_reference=ref argument is useful so the recipe is in "develop", # because the "package()" method is in develop=True already - pkg_node = deps_graph.root + pkg_node = graph.root ref = pkg_node.ref + source_folder = os.path.dirname(pkg_node.path) out = ConanOutput(scope=pkg_node.conanfile.display_name) out.info("Exporting binary from user folder to Conan cache") conanfile = pkg_node.conanfile diff --git a/conan/api/subapi/graph.py b/conan/api/subapi/graph.py index 1a00b88a613..3fb81c3621a 100644 --- a/conan/api/subapi/graph.py +++ b/conan/api/subapi/graph.py @@ -88,7 +88,8 @@ def load_root_test_conanfile(self, path, tested_reference, profile_host, profile return root_node def _load_root_virtual_conanfile(self, profile_host, profile_build, requires, tool_requires, - lockfile, remotes, update, check_updates=False, python_requires=None): + lockfile, remotes, update, check_updates=False, + python_requires=None): if not python_requires and not requires and not tool_requires: raise ConanException("Provide requires or tool_requires") app = ConanApp(self._conan_api) @@ -108,12 +109,15 @@ def _scope_options(profile, requires, tool_requires): Command line helper to scope options when ``command -o myoption=myvalue`` is used, that needs to be converted to "-o pkg:myoption=myvalue". The "pkg" value will be computed from the given requires/tool_requires + + This is legacy, as options should always be scoped now """ - # FIXME: This helper function here is not great, find a better place if requires and len(requires) == 1 and not tool_requires: - profile.options.scope(requires[0]) - if tool_requires and len(tool_requires) == 1 and not requires: - profile.options.scope(tool_requires[0]) + ref = requires[0] + if str(ref.version).startswith("["): + ref = ref.copy() + ref.version = "*" + profile.options.scope(ref) def load_graph_requires(self, requires, tool_requires, profile_host, profile_build, lockfile, remotes, update, check_updates=False, python_requires=None): @@ -164,8 +168,8 @@ def load_graph(self, root_node, profile_host, profile_build, lockfile=None, remo """ Compute the dependency graph, starting from a root package, evaluation the graph with the provided configuration in profile_build, and profile_host. The resulting graph is a graph of recipes, but packages are not computed yet (package_ids) will be empty in the - result. The result might have errors, like version or configuration conflicts, but it is still - possible to inspect it. Only trying to install such graph will fail + result. The result might have errors, like version or configuration conflicts, but it is + still possible to inspect it. Only trying to install such graph will fail :param root_node: the starting point, an already initialized Node structure, as returned by the "load_root_node" api @@ -189,8 +193,8 @@ def load_graph(self, root_node, profile_host, profile_build, lockfile=None, remo deps_graph = builder.load_graph(root_node, profile_host, profile_build, lockfile) return deps_graph - def analyze_binaries(self, graph, build_mode=None, remotes=None, update=None, lockfile=None, - build_modes_test=None, tested_graph=None): + def analyze_binaries(self, graph, build_mode=None, remotes=None, update=None, + lockfile=None, build_modes_test=None, tested_graph=None): """ Given a dependency graph, will compute the package_ids of all recipes in the graph, and evaluate if they should be built from sources, downloaded from a remote server, of if the packages are already in the local Conan cache @@ -199,8 +203,12 @@ def analyze_binaries(self, graph, build_mode=None, remotes=None, update=None, lo :param graph: a Conan dependency graph, as returned by "load_graph()" :param build_mode: TODO: Discuss if this should be a BuildMode object or list of arguments :param remotes: list of remotes - :param update: (False by default), if Conan should look for newer versions or - revisions for already existing recipes in the Conan cache + :param update: (``False`` by default), if Conan should look for newer versions or + revisions for already existing recipes in the Conan cache. It also accepts an array of + reference patterns to limit the update to those references if any of the items match. + Eg. ``False``, ``None`` or ``[]`` *means no update*, + ``True`` or ``["*"]`` *means update all*, + and ``["pkgA/*", "pkgB/1.0@user/channel"]`` *means to update only specific packages*. :param build_modes_test: the --build-test argument :param tested_graph: In case of a "test_package", the graph being tested """ diff --git a/conan/api/subapi/install.py b/conan/api/subapi/install.py index ff4a141cf67..fa60e903028 100644 --- a/conan/api/subapi/install.py +++ b/conan/api/subapi/install.py @@ -6,7 +6,7 @@ from conan.internal.graph.install_graph import InstallGraph from conan.internal.graph.installer import BinaryInstaller -from conan.errors import ConanInvalidConfiguration +from conan.errors import ConanInvalidConfiguration, ConanException class InstallAPI: @@ -15,10 +15,11 @@ def __init__(self, conan_api, helpers): self._conan_api = conan_api self._helpers = helpers - def install_binaries(self, deps_graph, remotes=None): + def install_binaries(self, deps_graph, remotes=None, return_install_error=False): """ Install binaries for dependency graph :param deps_graph: Dependency graph to intall packages for :param remotes: + :param return_install_error: If True, do not raise an exception, but return it """ app = ConanBasicApp(self._conan_api) installer = BinaryInstaller(app, self._helpers.global_conf, app.editable_packages, @@ -27,7 +28,14 @@ def install_binaries(self, deps_graph, remotes=None): install_graph.raise_errors() install_order = install_graph.install_order() installer.install_system_requires(deps_graph, install_order=install_order) - installer.install(deps_graph, remotes, install_order=install_order) + try: # To be able to capture the output, report or save graph.json, then raise later + installer.install(deps_graph, remotes, install_order=install_order) + except ConanException as e: + # If true, allows to return the exception, so progress can be reported like the + # already built binaries to upload them + if not return_install_error: + raise + return e def install_system_requires(self, graph, only_info=False): """ Install binaries for dependency graph @@ -52,7 +60,8 @@ def install_sources(self, graph, remotes): # TODO: Look for a better name def install_consumer(self, deps_graph, generators=None, source_folder=None, output_folder=None, - deploy=False, deploy_package=None, deploy_folder=None, envs_generation=None): + deploy=False, deploy_package=None, deploy_folder=None, + envs_generation=None): """ Once a dependency graph has been installed, there are things to be done, like invoking generators for the root consumer. This is necessary for example for conanfile.txt/py, or for "conan install -g @@ -93,5 +102,5 @@ def install_consumer(self, deps_graph, generators=None, source_folder=None, outp envs_generation=envs_generation) def deploy(self, graph, deployer, deploy_package=None, deploy_folder=None): - return do_deploys(self._conan_api.home_folder, graph, deployer, deploy_package=deploy_package, - deploy_folder=deploy_folder) + return do_deploys(self._conan_api.home_folder, graph, deployer, + deploy_package=deploy_package, deploy_folder=deploy_folder) diff --git a/conan/api/subapi/list.py b/conan/api/subapi/list.py index 56b6f3461aa..1d3a0ddf8bb 100644 --- a/conan/api/subapi/list.py +++ b/conan/api/subapi/list.py @@ -1,4 +1,3 @@ -import copy import os from collections import OrderedDict from typing import Dict @@ -57,9 +56,9 @@ def latest_recipe_revision(self, ref: RecipeReference, remote: Remote = None): assert ref.revision is None, "latest_recipe_revision: ref already have a revision" app = ConanBasicApp(self._conan_api) if remote: - ret = app.remote_manager.get_latest_recipe_reference(ref, remote=remote) + ret = app.remote_manager.get_latest_recipe_revision(ref, remote=remote) else: - ret = app.cache.get_latest_recipe_reference(ref) + ret = app.cache.get_latest_recipe_revision(ref) return ret @@ -69,9 +68,9 @@ def recipe_revisions(self, ref: RecipeReference, remote: Remote = None): assert ref.revision is None, "recipe_revisions: ref already have a revision" app = ConanBasicApp(self._conan_api) if remote: - results = app.remote_manager.get_recipe_revisions_references(ref, remote=remote) + results = app.remote_manager.get_recipe_revisions(ref, remote=remote) else: - results = app.cache.get_recipe_revisions_references(ref) + results = app.cache.get_recipe_revisions(ref) return results @@ -83,9 +82,9 @@ def latest_package_revision(self, pref: PkgReference, remote=None): assert pref.package_id is not None, "package_id must be defined" app = ConanBasicApp(self._conan_api) if remote: - ret = app.remote_manager.get_latest_package_reference(pref, remote=remote) + ret = app.remote_manager.get_latest_package_revision(pref, remote=remote) else: - ret = app.cache.get_latest_package_reference(pref) + ret = app.cache.get_latest_package_revision(pref) return ret def package_revisions(self, pref: PkgReference, remote=None): @@ -93,23 +92,20 @@ def package_revisions(self, pref: PkgReference, remote=None): "check latest first if needed" app = ConanBasicApp(self._conan_api) if remote: - results = app.remote_manager.get_package_revisions_references(pref, remote=remote) + results = app.remote_manager.get_package_revisions(pref, remote=remote) else: - results = app.cache.get_package_revisions_references(pref, only_latest_prev=False) + results = app.cache.get_package_revisions(pref) return results def _packages_configurations(self, ref: RecipeReference, remote=None) -> Dict[PkgReference, dict]: - assert ref.revision is not None, "packages: ref should have a revision. " \ - "Check latest if needed." + assert ref.revision is not None and ref.revision != "latest", \ + "packages: ref should have a revision. Check latest if needed." app = ConanBasicApp(self._conan_api) if not remote: prefs = app.cache.get_package_references(ref) packages = _get_cache_packages_binary_info(app.cache, prefs) else: - if ref.revision == "latest": - ref.revision = None - ref = app.remote_manager.get_latest_recipe_reference(ref, remote=remote) packages = app.remote_manager.search_packages(remote, ref) return packages @@ -120,8 +116,6 @@ def _filter_packages_configurations(pkg_configurations, query): :param query: str like "os=Windows AND (arch=x86 OR compiler=gcc)" :return: Dict[PkgReference, PkgConfiguration] """ - if query is None: - return pkg_configurations try: if "!" in query: raise ConanException("'!' character is not allowed") @@ -163,7 +157,8 @@ def _filter_packages_profile(packages, profile, ref): return result - def select(self, pattern: ListPattern, package_query=None, remote: Remote = None, lru=None, profile=None) -> PackagesList: + def select(self, pattern: ListPattern, package_query=None, remote: Remote = None, lru=None, + profile=None) -> PackagesList: """For a given pattern, return a list of recipes and packages matching the provided filters. :parameter ListPattern pattern: Search criteria @@ -177,6 +172,7 @@ def select(self, pattern: ListPattern, package_query=None, remote: Remote = None It can be a string like ``"2d"`` (2 days) or ``"3h"`` (3 hours). :parameter Profile profile: Profile to filter the packages by settings and options. """ + # TODO: Implement better error forwarding for "list" command that captures Exceptions if package_query and pattern.package_id and "*" not in pattern.package_id: raise ConanException("Cannot specify '-p' package queries, " "if 'package_id' is not a pattern") @@ -192,7 +188,9 @@ def select(self, pattern: ListPattern, package_query=None, remote: Remote = None remote_name = "local cache" if not remote else remote.name if search_ref: refs = _search_recipes(app, search_ref, remote=remote) - refs = pattern.filter_versions(refs) + global_conf = self._conan_api._api_helpers.global_conf # noqa + resolve_prereleases = global_conf.get("core.version_ranges:resolve_prereleases") + refs = pattern.filter_versions(refs, resolve_prereleases) pattern.check_refs(refs) out.info(f"Found {len(refs)} pkg/version recipes matching {search_ref} in {remote_name}") else: @@ -200,7 +198,8 @@ def select(self, pattern: ListPattern, package_query=None, remote: Remote = None # Show only the recipe references if pattern.package_id is None and pattern.rrev is None: - select_bundle.add_refs(refs) + for r in refs: + select_bundle.add_ref(r) return select_bundle def msg_format(msg, item, total): @@ -222,7 +221,8 @@ def msg_format(msg, item, total): if lru and pattern.package_id is None: # Filter LRUs rrevs = [r for r in rrevs if app.cache.get_recipe_lru(r) < limit_time] - select_bundle.add_refs(rrevs) + for rr in rrevs: + select_bundle.add_ref(rr) if pattern.package_id is None: # Stop if not displaying binaries continue @@ -263,8 +263,12 @@ def msg_format(msg, item, total): if lru: # Filter LRUs prefs = [r for r in prefs if app.cache.get_package_lru(r) < limit_time] - select_bundle.add_prefs(rrev, prefs) - select_bundle.add_configurations(packages) + # Packages dict has been listed, even if empty + select_bundle.recipe_dict(rrev)["packages"] = {} + for p in prefs: + # the "packages" dict is not using the package-revision + pkg_info = packages.get(PkgReference(p.ref, p.package_id)) + select_bundle.add_pref(p, pkg_info) return select_bundle def explain_missing_binaries(self, ref, conaninfo, remotes): @@ -292,7 +296,7 @@ def explain_missing_binaries(self, ref, conaninfo, remotes): candidates.sort() pkglist = PackagesList() - pkglist.add_refs([ref]) + pkglist.add_ref(ref) # Return the closest matches, stop adding when distance is increased candidate_distance = None for candidate in candidates: @@ -300,10 +304,9 @@ def explain_missing_binaries(self, ref, conaninfo, remotes): break candidate_distance = candidate.distance pref = candidate.pref - pkglist.add_prefs(ref, [pref]) - pkglist.add_configurations({pref: candidate.binary_config}) + pkglist.add_pref(pref, candidate.binary_config) # Add the diff data - rev_dict = pkglist.recipes[str(pref.ref)]["revisions"][pref.ref.revision] + rev_dict = pkglist.recipe_dict(ref) rev_dict["packages"][pref.package_id]["diff"] = candidate.serialize() remote = candidate.remote.name if candidate.remote else "Local Cache" rev_dict["packages"][pref.package_id]["remote"] = remote @@ -314,30 +317,51 @@ def find_remotes(self, package_list, remotes): (Experimental) Find the remotes where the current package lists can be found """ result = MultiPackagesList() + app = ConanBasicApp(self._conan_api) for r in remotes: result_pkg_list = PackagesList() - for ref, recipe_bundle in package_list.refs().items(): - ref_no_rev = copy.copy(ref) # TODO: Improve ugly API - ref_no_rev.revision = None + for ref, ref_contents in package_list.serialize().items(): + ref = RecipeReference.loads(ref) try: - revs = self.recipe_revisions(ref_no_rev, remote=r) + remote_rrevs = app.remote_manager.get_recipe_revisions(ref, remote=r) except NotFoundException: continue - if ref not in revs: # not found + revisions = ref_contents.get("revisions") + if revisions is None: # This is a package list just with name/version + if remote_rrevs: + result_pkg_list.add_ref(ref) continue - result_pkg_list.add_refs([ref]) - for pref, pref_bundle in package_list.prefs(ref, recipe_bundle).items(): - pref_no_rev = copy.copy(pref) # TODO: Improve ugly API - pref_no_rev.revision = None - try: - prevs = self.package_revisions(pref_no_rev, remote=r) - except NotFoundException: + + for revision, rev_content in revisions.items(): + ref.revision = revision + # We look for the value of revision in server, to return timestamp too + found = next((r for r in remote_rrevs if r == ref), None) + if not found: + continue + result_pkg_list.add_ref(found) + packages = rev_content.get("packages") + if packages is None: continue - if pref in prevs: - result_pkg_list.add_prefs(ref, [pref]) - info = recipe_bundle["packages"][pref.package_id]["info"] - result_pkg_list.add_configurations({pref: info}) - if result_pkg_list.recipes: + for pkgid, pkgcontent in packages.items(): + pref = PkgReference(ref, pkgid) + try: + remote_prefs = app.remote_manager.get_package_revisions(pref, remote=r) + except NotFoundException: + continue + pkg_revisions = pkgcontent.get("revisions") + if pkg_revisions is None: # This is a package_id, no prevs + if remote_prefs: + result_pkg_list.add_pref(pref, pkgcontent.get("info")) + continue + for pkg_revision, prev_content in pkg_revisions.items(): + pref.revision = pkg_revision + # We look for the value of revision in server, to return timestamp too + pfound = next((r for r in remote_prefs if r == pref), None) + if not pfound: + continue + result_pkg_list.add_pref(pfound, pkgcontent.get("info")) + + if result_pkg_list: result.add(r.name, result_pkg_list) return result @@ -368,9 +392,9 @@ def outdated(self, deps_graph, remotes): remote_ref_list = self.select(ref_pattern, package_query=None, remote=remote) except NotFoundException: continue - if not remote_ref_list.recipes: + if not remote_ref_list: continue - str_latest_ref = list(remote_ref_list.recipes.keys())[-1] + str_latest_ref = list(remote_ref_list.serialize().keys())[-1] recipe_ref = RecipeReference.loads(str_latest_ref) if (node_info["latest_remote"] is None or node_info["latest_remote"]["ref"] < recipe_ref): @@ -507,7 +531,7 @@ def _get_cache_packages_binary_info(cache, prefs) -> Dict[PkgReference, dict]: result = OrderedDict() for pref in prefs: - latest_prev = cache.get_latest_package_reference(pref) + latest_prev = cache.get_latest_package_revision(pref) pkg_layout = cache.pkg_layout(latest_prev) # Read conaninfo diff --git a/conan/api/subapi/lockfile.py b/conan/api/subapi/lockfile.py index bb5ae08c3eb..dc056a3bb46 100644 --- a/conan/api/subapi/lockfile.py +++ b/conan/api/subapi/lockfile.py @@ -13,15 +13,20 @@ def __init__(self, conan_api): self._conan_api = conan_api @staticmethod - def get_lockfile(lockfile=None, conanfile_path=None, cwd=None, partial=False, overrides=None): + def get_lockfile(lockfile=None, conanfile_path=None, cwd=None, partial=False, + overrides=None) -> Lockfile: """ obtain a lockfile, following this logic: - - If lockfile is explicitly defined, it would be either absolute or relative to cwd and - the lockfile file must exist. If lockfile="" (empty string) the default "conan.lock" - lockfile will not be automatically used even if it is present. - - If lockfile is not defined, it will still look for a default conan.lock: - - if conanfile_path is defined, it will be besides it - - if conanfile_path is not defined, the default conan.lock should be in cwd - - if the default conan.lock cannot be found, it is not an error + + If lockfile is explicitly defined, it would be either absolute or relative to cwd and + the lockfile file must exist. If lockfile="" (empty string) the default "conan.lock" + lockfile will not be automatically used even if it is present. + + If lockfile is not defined, it will still look for a default conan.lock: + + - if conanfile_path is defined, it will be besides it + - if conanfile_path is not defined, the default conan.lock should be in cwd + - if the default conan.lock cannot be found, it is not an error + :param partial: If the obtained lockfile will allow partial resolving :param cwd: the current working dir, if None, os.getcwd() will be used diff --git a/conan/api/subapi/new.py b/conan/api/subapi/new.py index 50a73f07fd6..2837fdcaeb7 100644 --- a/conan/api/subapi/new.py +++ b/conan/api/subapi/new.py @@ -166,7 +166,7 @@ def _read_files(self, template_folder): @staticmethod def render(template_files, definitions): result = {} - name = definitions.get("name", "pkg") + name = definitions.get("name", "mypkg") if isinstance(name, list): raise ConanException(f"name argument can't be multiple: {name}") if name != name.lower(): diff --git a/conan/api/subapi/profiles.py b/conan/api/subapi/profiles.py index a254977110f..10b4d4f666a 100644 --- a/conan/api/subapi/profiles.py +++ b/conan/api/subapi/profiles.py @@ -43,8 +43,11 @@ def get_default_build(self): :return: the path to the default "build" profile, either in the cache or as defined by the user in configuration """ - global_conf = self._api_helpers.global_conf - default_profile = global_conf.get("core:default_build_profile", default=DEFAULT_PROFILE_NAME) + default_profile = os.environ.get("CONAN_DEFAULT_BUILD_PROFILE") + if default_profile is None: + global_conf = self._api_helpers.global_conf + default_profile = global_conf.get("core:default_build_profile", + default=DEFAULT_PROFILE_NAME) default_profile = os.path.join(self._home_paths.profiles_path, default_profile) if not os.path.exists(default_profile): msg = ("The default build profile '{}' doesn't exist.\n" diff --git a/conan/api/subapi/remotes.py b/conan/api/subapi/remotes.py index 54191cac7a2..30e754b0694 100644 --- a/conan/api/subapi/remotes.py +++ b/conan/api/subapi/remotes.py @@ -8,7 +8,6 @@ from conan.api.output import ConanOutput from conan.internal.cache.home_paths import HomePaths from conan.internal.conan_app import ConanBasicApp -from conan.internal.rest.conan_requester import ConanRequester from conan.internal.rest.remote_credentials import RemoteCredentials from conan.internal.rest.rest_client_local_recipe_index import add_local_recipes_index_remote, \ remove_local_recipes_index_remote @@ -35,11 +34,6 @@ def __init__(self, conan_api, api_helpers): self._api_helpers = api_helpers self._home_folder = conan_api.home_folder self._remotes_file = HomePaths(self._home_folder).remotes_path - # Wraps an http_requester to inject proxies, certs, etc - self._requester = ConanRequester(api_helpers.global_conf, self._home_folder) - - def reinit(self): - self._requester = ConanRequester(self._api_helpers.global_conf, self._home_folder) def list(self, pattern=None, only_enabled=True): """ @@ -64,7 +58,8 @@ def disable(self, pattern): :param pattern: single ``str`` or list of ``str``. If the pattern is an exact name without wildcards like "*" and no remote is found matching that exact name, it will raise an error. - :return: the list of disabled :ref:`Remote ` objects (even if they were already disabled) + :return: the list of disabled :ref:`Remote ` objects (even if they + were already disabled) """ remotes = _load(self._remotes_file) disabled = _filter(remotes, pattern, only_enabled=False) @@ -82,7 +77,8 @@ def enable(self, pattern): :param pattern: single ``str`` or list of ``str``. If the pattern is an exact name without wildcards like "*" and no remote is found matching that exact name, it will raise an error. - :return: the list of enabled :ref:`Remote ` objects (even if they were already enabled) + :return: the list of enabled :ref:`Remote ` objects (even if they + were already enabled) """ remotes = _load(self._remotes_file) enabled = _filter(remotes, pattern, only_enabled=False) @@ -99,7 +95,8 @@ def get(self, remote_name): Obtain a :ref:`Remote ` object :param remote_name: the exact name of the remote to be returned - :return: the :ref:`Remote ` object, or raise an Exception if the remote does not exist. + :return: the :ref:`Remote ` object, or raise an Exception if the + remote does not exist. """ remotes = _load(self._remotes_file) try: @@ -159,7 +156,7 @@ def remove(self, pattern): return removed def update(self, remote_name: str, url=None, secure=None, disabled=None, index=None, - allowed_packages=None): + allowed_packages=None, recipes_only=None): """ Update an existing remote @@ -169,6 +166,8 @@ def update(self, remote_name: str, url=None, secure=None, disabled=None, index=N :param disabled: optional disabled state :param index: optional integer to change the order of the remote :param allowed_packages: optional list of packages allowed from this remote + :param recipes_only: optional boolean to only allow recipe downloads from this remote, + never package binaries """ remotes = _load(self._remotes_file) try: @@ -186,6 +185,8 @@ def update(self, remote_name: str, url=None, secure=None, disabled=None, index=N remote.disabled = disabled if allowed_packages is not None: remote.allowed_packages = allowed_packages + if recipes_only is not None: + remote.recipes_only = recipes_only if index is not None: remotes = [r for r in remotes if r.name != remote.name] @@ -282,10 +283,6 @@ def user_auth(self, remote: Remote, with_user=False, force=False): user, token, _ = localdb.get_login(remote.url) return user - @property - def requester(self): - return self._requester - def _load(remotes_file): if not os.path.exists(remotes_file): @@ -300,7 +297,8 @@ def _load(remotes_file): result = [] for r in data.get("remotes", []): remote = Remote(r["name"], r["url"], r["verify_ssl"], r.get("disabled", False), - r.get("allowed_packages"), r.get("remote_type")) + r.get("allowed_packages"), r.get("remote_type"), + r.get("recipes_only", False)) result.append(remote) return result @@ -315,6 +313,8 @@ def _save(remotes_file, remotes): remote["allowed_packages"] = r.allowed_packages if r.remote_type: remote["remote_type"] = r.remote_type + if r.recipes_only: + remote["recipes_only"] = r.recipes_only remote_list.append(remote) # This atomic replace avoids a corrupted remotes.json file if this is killed during the process save(remotes_file + ".tmp", json.dumps({"remotes": remote_list}, indent=True)) diff --git a/conan/api/subapi/report.py b/conan/api/subapi/report.py index ad1a5b3f758..134a3f479dc 100644 --- a/conan/api/subapi/report.py +++ b/conan/api/subapi/report.py @@ -14,6 +14,9 @@ class ReportAPI: + """ Used to compute the differences (the "diff") between two versions or revisions, for + both the recipe and source code. + """ def __init__(self, conan_api, helpers): self._conan_api = conan_api self._helpers = helpers @@ -21,6 +24,7 @@ def __init__(self, conan_api, helpers): def diff(self, old_reference, new_reference, remotes, old_path=None, new_path=None, cwd=None): """ Compare two recipes and return the differences. + :param old_reference: The reference of the old recipe. :param new_reference: The reference of the new recipe. :param remotes: List of remotes to search for the recipes. @@ -32,7 +36,8 @@ def diff(self, old_reference, new_reference, remotes, old_path=None, new_path=No def _source(path_to_conanfile, reference): if path_to_conanfile is None: - export_ref, cache_path = _get_ref_from_cache_or_remote(self._conan_api, reference, remotes) + export_ref, cache_path = _get_ref_from_cache_or_remote(self._conan_api, reference, + remotes) else: export_ref, cache_path = _export_recipe_from_path(self._conan_api, path_to_conanfile, reference, remotes, cwd) @@ -41,7 +46,6 @@ def _source(path_to_conanfile, reference): remotes) return export_ref, cache_path - old_export_ref, old_cache_path = _source(old_path, old_reference) new_export_ref, new_cache_path = _source(new_path, new_reference) @@ -57,7 +61,8 @@ def _source(path_to_conanfile, reference): f'"{old_diff_path}" "{new_diff_path}"') ConanOutput().info( - f"Generating diff from {old_export_ref.repr_notime()} to {new_export_ref.repr_notime()} (this might take a while)") + f"Generating diff from {old_export_ref.repr_notime()} to {new_export_ref.repr_notime()} " + f"(this might take a while)") ConanOutput().info(command) stdout, stderr = StringIO(), StringIO() @@ -79,6 +84,7 @@ def _source(path_to_conanfile, reference): "dst_prefix": dst_prefix, } + def _configure_source(conan_api, hook_manager, conanfile_path, ref, remotes): app = ConanApp(conan_api) conanfile = app.loader.load_consumer(conanfile_path, name=ref.name, version=str(ref.version), @@ -101,6 +107,7 @@ def _configure_source(conan_api, hook_manager, conanfile_path, ref, remotes): conanfile.folders.set_base_recipe_metadata(recipe_layout.metadata()) config_source(export_source_folder, conanfile, hook_manager) + def _get_ref_from_cache_or_remote(conan_api, reference, enabled_remotes): ref = RecipeReference.loads(reference) full_ref, matching_remote = None, False @@ -115,19 +122,20 @@ def _get_ref_from_cache_or_remote(conan_api, reference, enabled_remotes): full_ref = ref matching_remote = remote break - except: + except (Exception,): continue else: try: latest_recipe_revision = conan_api.list.latest_recipe_revision(ref, remote) - except: + except (Exception,): continue if full_ref is None or (latest_recipe_revision.timestamp > full_ref.timestamp): full_ref = latest_recipe_revision matching_remote = remote if full_ref is None or matching_remote is False: raise ConanException(f"No matching reference for {reference} in remotes.\n" - "If you want to check against a local recipe, add an additional --{old,new}-path arg.\n") + "If you want to check against a local recipe, add an " + "additional --{old,new}-path arg.\n") if matching_remote is not None: conan_api.download.recipe(full_ref, matching_remote) cache_path = conan_api.cache.export_path(full_ref) diff --git a/conan/api/subapi/upload.py b/conan/api/subapi/upload.py index ca46e0a81b1..944a24750d2 100644 --- a/conan/api/subapi/upload.py +++ b/conan/api/subapi/upload.py @@ -1,7 +1,9 @@ import os import time from multiprocessing.pool import ThreadPool +from typing import List +from conan.api.model import PackagesList, Remote from conan.api.output import ConanOutput from conan.internal.api.upload import add_urls from conan.internal.conan_app import ConanApp @@ -14,35 +16,52 @@ class UploadAPI: + """ This API is used to upload recipes and packages to a remote server.""" def __init__(self, conan_api, api_helpers): self._conan_api = conan_api self._api_helpers = api_helpers - def check_upstream(self, package_list, remote, enabled_remotes, force=False): - """Check if the artifacts are already in the specified remote, skipping them from - the package_list in that case""" + def check_upstream(self, package_list: PackagesList, remote: Remote, + enabled_remotes: List[Remote], force=False): + """ Checks ``remote`` for the existence of the recipes and packages in ``package_list``. + Items that are not present in the remote will add an ``upload`` key to the entry + with the value ``True``. + + If the recipe has an upload policy of ``skip``, it will be discarded from the upload list. + + :parameter package_list: A ``PackagesList`` object with the recipes and packages to check. + :parameter remote: Remote to check. + :parameter enabled_remotes: List of enabled remotes. This is used to possibly load + python_requires from the listed recipes if necessary. + :parameter force: If ``True``, it will skip the check and mark that all items need to be + uploaded. A ``force_upload`` key will be added to the entries that will be uploaded. + """ app = ConanApp(self._conan_api) - for ref, bundle in package_list.refs().items(): + for ref, _ in package_list.items(): layout = app.cache.recipe_layout(ref) conanfile_path = layout.conanfile() conanfile = app.loader.load_basic(conanfile_path, remotes=enabled_remotes) if conanfile.upload_policy == "skip": ConanOutput().info(f"{ref}: Skipping upload of binaries, " "because upload_policy='skip'") - bundle["packages"] = {} + package_list.recipe_dict(ref)["packages"] = {} UploadUpstreamChecker(app).check(package_list, remote, force) - def prepare(self, package_list, enabled_remotes, metadata=None): + def prepare(self, package_list: PackagesList, enabled_remotes: List[Remote], + metadata: List[str] = None): """Compress the recipes and packages and fill the upload_data objects with the complete information. It doesn't perform the upload nor checks upstream to see if the recipe is still there - :param package_list: - :param enabled_remotes: - :param metadata: A list of patterns of metadata that should be uploaded. Default None - means all metadata will be uploaded together with the pkg artifacts. If metadata is empty - string (""), it means that no metadata files should be uploaded.""" + + :param package_list: A PackagesList object with the recipes and packages to upload. + :param enabled_remotes: A list of remotes that are enabled in the client. + Recipe sources will attempt to be fetched from these remotes. + :param metadata: A list of patterns of metadata that should be uploaded. + Default ``None`` means all metadata will be uploaded together with the package artifacts. + If metadata contains an empty string (``""``), + it means that no metadata files should be uploaded.""" if metadata and metadata != [''] and '' in metadata: raise ConanException("Empty string and patterns can not be mixed for metadata.") app = ConanApp(self._conan_api) @@ -54,21 +73,42 @@ def prepare(self, package_list, enabled_remotes, metadata=None): # This might add files entries to package_list with signatures signer.sign(package_list) - def upload(self, package_list, remote): + def _upload(self, package_list, remote): app = ConanApp(self._conan_api) app.remote_manager.check_credentials(remote) executor = UploadExecutor(app) executor.upload(package_list, remote) - def upload_full(self, package_list, remote, enabled_remotes, check_integrity=False, force=False, - metadata=None, dry_run=False): + def upload_full(self, package_list: PackagesList, remote: Remote, enabled_remotes: List[Remote], + check_integrity=False, force=False, metadata: List[str] = None, dry_run=False): """ Does the whole process of uploading, including the possibility of parallelizing - per recipe based on `core.upload:parallel`: - - calls check_integrity - - checks which revision already exist in the server (not necessary to upload) - - prepare the artifacts to upload (compress .tgz) - - execute the actual upload - - upload potential sources backups + per recipe based on the ``core.upload:parallel`` conf. + + The steps that this method performs are: + - calls ``conan_api.cache.check_integrity`` to ensure the packages are not corrupted + - checks the upload policy of the recipes + - (if it is ``"skip"``, it will not upload the binaries, but will still upload + the metadata) + - checks which revisions already exist in the server so that it can skip the upload + - prepares the artifacts to upload (compresses the conan_package.tgz) + - executes the actual upload + - uploads associated sources backups if any + + :param package_list: A PackagesList object with the recipes and packages to upload. + :param remote: The remote to upload the packages to. + :param enabled_remotes: A list of remotes that are enabled in the client. + Recipe sources will attempt to be fetched from these remotes, + and to possibly load python_requires from the listed recipes if necessary. + :param check_integrity: If ``True``, it will check the integrity of the cache packages + before uploading them. This is useful to ensure that the packages are not corrupted. + :param force: If ``True``, it will force the upload of the recipes and packages, + even if they already exist in the remote. Note that this might update the timestamps + :param metadata: A list of patterns of metadata that should be uploaded. + Default ``None`` means all metadata will be uploaded together with the package artifacts. + If metadata contains an empty string (``""``), + it means that no metadata files should be uploaded. + :param dry_run: If ``True``, it will not perform the actual upload, + but will still prepare the artifacts and check the upstream. """ def _upload_pkglist(pkglist, subtitle=lambda _: None): @@ -76,14 +116,14 @@ def _upload_pkglist(pkglist, subtitle=lambda _: None): subtitle("Checking integrity of cache packages") self._conan_api.cache.check_integrity(pkglist) # Check if the recipes/packages are in the remote - subtitle("Checking server existing packages") + subtitle("Checking server for existing packages") self.check_upstream(pkglist, remote, enabled_remotes, force) subtitle("Preparing artifacts for upload") self.prepare(pkglist, enabled_remotes, metadata) if not dry_run: subtitle("Uploading artifacts") - self.upload(pkglist, remote) + self._upload(pkglist, remote) backup_files = self._conan_api.cache.get_backup_sources(pkglist) self.upload_backup_sources(backup_files) @@ -91,7 +131,7 @@ def _upload_pkglist(pkglist, subtitle=lambda _: None): ConanOutput().title(f"Uploading to remote {remote.name}") parallel = self._conan_api.config.get("core.upload:parallel", default=1, check_type=int) thread_pool = ThreadPool(parallel) if parallel > 1 else None - if not thread_pool or len(package_list.recipes) <= 1: + if not thread_pool or len(package_list._data) <= 1: # FIXME: Iteration when multiple rrevs _upload_pkglist(package_list, subtitle=ConanOutput().subtitle) else: ConanOutput().subtitle(f"Uploading with {parallel} parallel threads") @@ -103,7 +143,13 @@ def _upload_pkglist(pkglist, subtitle=lambda _: None): ConanOutput().success(f"Upload completed in {int(elapsed)}s\n") add_urls(package_list, remote) - def upload_backup_sources(self, files): + def upload_backup_sources(self, files: List) -> None: + """ + Upload to the server the backup sources files, that have been typically gathered by + CacheAPI.get_backup_sources() + + :param files: The list of files that must be uploaded + """ config = self._api_helpers.global_conf url = config.get("core.sources:upload_url", check_type=str) if url is None: @@ -114,9 +160,9 @@ def upload_backup_sources(self, files): output.subtitle("Uploading backup sources") if not files: output.info("No backup sources files to upload") - return files + return - requester = self._conan_api.remotes.requester + requester = self._api_helpers.requester uploader = FileUploader(requester, verify=True, config=config, source_credentials=True) # TODO: For Artifactory, we can list all files once and check from there instead # of 1 request per file, but this is more general @@ -142,4 +188,3 @@ def upload_backup_sources(self, files): f"/permissions, please provide 'source_credentials.json': {e}") output.success("Upload backup sources complete\n") - return files diff --git a/conan/api/subapi/workspace.py b/conan/api/subapi/workspace.py index ddfec264a8c..d2f56bac545 100644 --- a/conan/api/subapi/workspace.py +++ b/conan/api/subapi/workspace.py @@ -11,13 +11,17 @@ from conan.cli.printers.graph import print_graph_basic, print_graph_packages from conan.errors import ConanException from conan.internal.conan_app import ConanApp +from conan.internal.errors import conanfile_exception_formatter from conan.internal.graph.install_graph import ProfileArgs +from conan.internal.methods import auto_language, auto_shared_fpic_config_options, \ + auto_shared_fpic_configure +from conan.internal.model.options import Options from conan.internal.model.workspace import Workspace, WORKSPACE_YML, WORKSPACE_PY, WORKSPACE_FOLDER from conan.tools.scm import Git -from conan.internal.graph.graph import RECIPE_EDITABLE, DepsGraph, CONTEXT_HOST, RECIPE_VIRTUAL, Node, \ - RECIPE_CONSUMER +from conan.internal.graph.graph import (RECIPE_EDITABLE, DepsGraph, CONTEXT_HOST, RECIPE_VIRTUAL, + Node, RECIPE_CONSUMER) from conan.internal.graph.graph import TransitiveRequirement -from conan.internal.graph.profile_node_definer import consumer_definer +from conan.internal.graph.profile_node_definer import consumer_definer, initialize_conanfile_profile from conan.internal.loader import load_python_file from conan.internal.source import retrieve_exports_sources from conan.internal.util.files import merge_directories, save @@ -91,7 +95,6 @@ def __init__(self, conan_api): def enable(self, value): self._enabled = value - @property def name(self): self._check_ws() return self._ws.name() @@ -114,7 +117,7 @@ def packages(self): rel_path = editable_info["path"] path = os.path.normpath(os.path.join(self._folder, rel_path, "conanfile.py")) if not os.path.isfile(path): - raise ConanException(f"Workspace editable not found: {path}") + raise ConanException(f"Workspace package not found: {path}") ref = editable_info.get("ref") try: if ref is None: @@ -124,12 +127,12 @@ def packages(self): else: reference = RecipeReference.loads(ref) reference.validate_ref(reference) - except: - raise ConanException(f"Workspace editable reference could not be deduced by" + except Exception as e: + raise ConanException(f"Workspace package reference could not be deduced by" f" {rel_path}/conanfile.py or it is not" - f" correctly defined in the conanws.yml file.") + f" correctly defined in the conanws.yml file: {e}") if reference in packages: - raise ConanException(f"Workspace editable reference '{str(reference)}' already exists.") + raise ConanException(f"Workspace package '{str(reference)}' already exists.") packages[reference] = {"path": path} if editable_info.get("output_folder"): packages[reference]["output_folder"] = ( @@ -137,9 +140,10 @@ def packages(self): ) return packages - def open(self, require, remotes, cwd=None): + def open(self, ref, remotes, cwd=None): + cwd = cwd or os.getcwd() app = ConanApp(self._conan_api) - ref = RecipeReference.loads(require) + ref = RecipeReference.loads(ref) if isinstance(ref, str) else ref recipe = app.proxy.get_recipe(ref, remotes, update=False, check_update=False) layout, recipe_status, remote = recipe @@ -150,7 +154,7 @@ def open(self, require, remotes, cwd=None): conanfile, module = app.loader.load_basic_module(conanfile_path, remotes=remotes) scm = conanfile.conan_data.get("scm") if conanfile.conan_data else None - dst_path = os.path.join(cwd or os.getcwd(), ref.name) + dst_path = os.path.join(cwd, ref.name) if scm is None: conanfile.output.warning("conandata doesn't contain 'scm' information\n" "doing a local copy!!!") @@ -200,6 +204,45 @@ def add(self, path, name=None, version=None, user=None, channel=None, cwd=None, self._ws.add(ref, full_path, output_folder) return ref + def complete(self, profile_host, profile_build, lockfile, remotes, update): + packages = self.packages() + if not packages: + ConanOutput().info("There are no packages in this workspace, nothing to complete") + return + + for ref, info in packages.items(): + ConanOutput().title(f"Computing the dependency graph for package: {ref}") + gapi = self._conan_api.graph + deps_graph = gapi.load_graph_requires([ref], None, profile_host, profile_build, + lockfile, remotes, update) + deps_graph.report_graph_error() + print_graph_basic(deps_graph) + + nodes_to_complete = [] + for node in deps_graph.nodes[1:]: # Exclude the current virtual root + if node.recipe != RECIPE_EDITABLE: + # sanity check, a pacakge in the cache cannot have dependencies to the workspace + if any(d.node.recipe == RECIPE_EDITABLE for d in node.transitive_deps.values()): + nodes_to_complete.append(node) + + if not nodes_to_complete: + ConanOutput().info("There are no intermediate packages to add to the workspace") + return + + for node in nodes_to_complete: + full_path = os.path.join(self._folder, node.name, "conanfile.py") + dep_ref = node.ref + ConanOutput().info(f"Adding to workspace {dep_ref}") + try: + self._ws.add(dep_ref, full_path, output_folder=None) + except ConanException: + if os.path.isfile(full_path): + raise + ConanOutput().info(f"Conanfile in {node.name} not found, trying " + "to open it first") + self.open(dep_ref, remotes, cwd=self._folder) + self._ws.add(dep_ref, full_path, output_folder=None) + @staticmethod def init(path): abs_path = make_abs_path(path) @@ -234,24 +277,72 @@ def clean(self): def info(self): self._check_ws() - return {"name": self.name, + return {"name": self._ws.name(), "folder": self._folder, "packages": self._ws.packages()} + @staticmethod + def _init_options(conanfile, options): + if hasattr(conanfile, "config_options"): + with conanfile_exception_formatter(conanfile, "config_options"): + conanfile.config_options() + elif "auto_shared_fpic" in conanfile.implements: + auto_shared_fpic_config_options(conanfile) + + auto_language(conanfile) # default implementation removes `compiler.cstd` + + # Assign only the current package options values, but none of the dependencies + conanfile.options.apply_downstream(Options(), options, None, True) + + if hasattr(conanfile, "configure"): + with conanfile_exception_formatter(conanfile, "configure"): + conanfile.configure() + elif "auto_shared_fpic" in conanfile.implements: + auto_shared_fpic_configure(conanfile) + def super_build_graph(self, deps_graph, profile_host, profile_build): + order = [] + packages = self.packages() + + def find_folder(ref): + return next(os.path.dirname(os.path.relpath(p["path"], self._folder)) for p_ref, p in + packages.items() if p_ref == ref) + + for level in deps_graph.by_levels(): + items = [item for item in level if item.recipe == "Editable"] + level_order = [] + for node in items: + conanfile = node.conanfile + if hasattr(conanfile, "layout"): + with conanfile_exception_formatter(conanfile, "layout"): + conanfile.layout() + base_folder = find_folder(node.ref) + src_folder = os.path.normpath(os.path.join(base_folder, conanfile.folders.source)) + level_order.append({"ref": node.ref, "folder": src_folder.replace("\\", "/")}) + order.append(level_order) + + self._ws.build_order(order) + ConanOutput().title("Collapsing workspace packages") root_class = self._ws.root_conanfile() if root_class is not None: conanfile = root_class(f"{WORKSPACE_PY} base project Conanfile") - consumer_definer(conanfile, profile_host, profile_build) - root = Node(None, conanfile, context=CONTEXT_HOST, recipe=RECIPE_CONSUMER, - path=self._folder) # path lets use the conanws.py folder - root.should_build = True # It is a consumer, this is something we are building + # To inject things like cmd_wrapper to the consumer conanfile, so self.run() works + helpers = ConanApp(self._conan_api).loader._conanfile_helpers # noqa + conanfile._conan_helpers = helpers + conanfile._conan_is_consumer = True + initialize_conanfile_profile(conanfile, profile_build, profile_host, CONTEXT_HOST, + is_build_require=False) + # consumer_definer(conanfile, profile_host, profile_build) + self._init_options(conanfile, profile_host.options) for field in ("requires", "build_requires", "test_requires", "requirements", "build", "source", "package"): if getattr(conanfile, field, None): raise ConanException(f"Conanfile in conanws.py shouldn't have '{field}'") + root = Node(None, conanfile, context=CONTEXT_HOST, recipe=RECIPE_CONSUMER, + path=self._folder) # path lets use the conanws.py folder + root.should_build = True # It is a consumer, this is something we are building else: ConanOutput().info(f"Workspace {WORKSPACE_PY} not found in the workspace folder, " "using default behavior") @@ -261,10 +352,16 @@ def super_build_graph(self, deps_graph, profile_host, profile_build): result = DepsGraph() # TODO: We might need to copy more information from the original graph result.add_node(root) + conanfile.workspace_packages = {} + + self._check_graph(deps_graph) for node in deps_graph.nodes[1:]: # Exclude the current root if node.recipe != RECIPE_EDITABLE: result.add_node(node) continue + # At the moment we are exposing the full conanfile, docs will warn against usage of + # non pure functions + conanfile.workspace_packages[node.ref] = node.conanfile for r, t in node.transitive_deps.items(): if t.node.recipe == RECIPE_EDITABLE: continue @@ -282,6 +379,20 @@ def super_build_graph(self, deps_graph, profile_host, profile_build): return result + @staticmethod + def _check_graph(graph): + for node in graph.nodes[1:]: # Exclude the current root + if node.recipe != RECIPE_EDITABLE: + # sanity check, a pacakge in the cache cannot have dependencies to the workspace + deps_edit = [d.node for d in node.transitive_deps.values() + if d.node.recipe == RECIPE_EDITABLE] + if deps_edit: + raise ConanException(f"Workspace definition error. Package {node} in the " + f"Conan cache has dependencies to packages " + f"in the workspace: {deps_edit}\n" + "Try the 'conan workspace complete' to open/add " + "intermediate packages") + def export(self, lockfile=None, remotes=None): self._check_ws() exported = [] @@ -293,7 +404,6 @@ def export(self, lockfile=None, remotes=None): exported.append(ref) return exported - def select_packages(self, packages): self._check_ws() editable = self.packages() @@ -323,6 +433,9 @@ def build_order(self, packages, profile_host, profile_build, build_mode, lockfil remotes, update) deps_graph.report_graph_error() print_graph_basic(deps_graph) + + self._check_graph(deps_graph) + conan_api.graph.analyze_binaries(deps_graph, build_mode, remotes=remotes, update=update, lockfile=lockfile) print_graph_packages(deps_graph) diff --git a/conan/cli/args.py b/conan/cli/args.py index 95b889c45db..4e4b148a506 100644 --- a/conan/cli/args.py +++ b/conan/cli/args.py @@ -27,29 +27,31 @@ def add_lockfile_args(parser): - parser.add_argument("-l", "--lockfile", action=OnceArgument, - help="Path to a lockfile. Use --lockfile=\"\" to avoid automatic use of " - "existing 'conan.lock' file") - parser.add_argument("--lockfile-partial", action="store_true", - help="Do not raise an error if some dependency is not found in lockfile") - parser.add_argument("--lockfile-out", action=OnceArgument, - help="Filename of the updated lockfile") - parser.add_argument("--lockfile-packages", action="store_true", - help=argparse.SUPPRESS) - parser.add_argument("--lockfile-clean", action="store_true", - help="Remove unused entries from the lockfile") - parser.add_argument("--lockfile-overrides", - help="Overwrite lockfile overrides") + group = parser.add_argument_group("lockfile arguments") + group.add_argument("-l", "--lockfile", action=OnceArgument, + help="Path to a lockfile. Use --lockfile=\"\" to avoid automatic use of " + "existing 'conan.lock' file") + group.add_argument("--lockfile-partial", action="store_true", + help="Do not raise an error if some dependency is not found in lockfile") + group.add_argument("--lockfile-out", action=OnceArgument, + help="Filename of the updated lockfile") + group.add_argument("--lockfile-packages", action="store_true", + help=argparse.SUPPRESS) + group.add_argument("--lockfile-clean", action="store_true", + help="Remove unused entries from the lockfile") + group.add_argument("--lockfile-overrides", + help="Overwrite lockfile overrides") def add_common_install_arguments(parser): parser.add_argument("-b", "--build", action="append", help=_help_build_policies) - group = parser.add_mutually_exclusive_group() - group.add_argument("-r", "--remote", action="append", default=None, - help='Look in the specified remote or remotes server') - group.add_argument("-nr", "--no-remote", action="store_true", - help='Do not use remote, resolve exclusively in the cache') + group = parser.add_argument_group("remote arguments") + exclusive_group = group.add_mutually_exclusive_group() + exclusive_group.add_argument("-r", "--remote", action="append", default=None, + help='Look in the specified remote or remotes server') + exclusive_group.add_argument("-nr", "--no-remote", action="store_true", + help='Do not use remote, resolve exclusively in the cache') update_help = ("Will install newer versions and/or revisions in the local cache " "for the given reference name, or all references in the graph if no argument is supplied. " @@ -57,13 +59,15 @@ def add_common_install_arguments(parser): "satisfies the range. It will update to the " "latest revision for the resolved version range.") - parser.add_argument("-u", "--update", action="append", nargs="?", help=update_help, const="*") + group.add_argument("-u", "--update", action="append", nargs="?", help=update_help, const="*") add_profiles_args(parser) def add_profiles_args(parser): contexts = ["build", "host"] + group = parser.add_argument_group("profile arguments") + # This comes from the _AppendAction code but modified to add to the contexts class ContextAllAction(argparse.Action): @@ -75,29 +79,29 @@ def __call__(self, action_parser, namespace, values, option_string=None): setattr(namespace, self.dest + "_" + context, items) def create_config(short, long, example=None): - parser.add_argument(f"-{short}", f"--{long}", - default=None, - action="append", - dest=f"{long}_host", - metavar=long.upper(), - help=f'Apply the specified {long}. ' - f'By default, or if specifying -{short}:h (--{long}:host), it applies to the host context. ' - f'Use -{short}:b (--{long}:build) to specify the build context, ' - f'or -{short}:a (--{long}:all) to specify both contexts at once' - + ('' if not example else f". Example: {example}")) + group.add_argument(f"-{short}", f"--{long}", + default=None, + action="append", + dest=f"{long}_host", + metavar=long.upper(), + help=f'Apply the specified {long}. ' + f'By default, or if specifying -{short}:h (--{long}:host), it applies to the host context. ' + f'Use -{short}:b (--{long}:build) to specify the build context, ' + f'or -{short}:a (--{long}:all) to specify both contexts at once' + + ('' if not example else f". Example: {example}")) for context in contexts: - parser.add_argument(f"-{short}:{context[0]}", f"--{long}:{context}", - default=None, - action="append", - dest=f"{long}_{context}", - help="") - - parser.add_argument(f"-{short}:a", f"--{long}:all", - default=None, - action=ContextAllAction, - dest=long, - metavar=f"{long.upper()}_ALL", - help="") + group.add_argument(f"-{short}:{context[0]}", f"--{long}:{context}", + default=None, + action="append", + dest=f"{long}_{context}", + help="") + + group.add_argument(f"-{short}:a", f"--{long}:all", + default=None, + action=ContextAllAction, + dest=long, + metavar=f"{long.upper()}_ALL", + help="") create_config("pr", "profile") create_config("o", "options", '-o="pkg/*:with_qt=True"') @@ -106,27 +110,31 @@ def create_config(short, long, example=None): def add_reference_args(parser): - parser.add_argument("--name", action=OnceArgument, - help='Provide a package name if not specified in conanfile') - parser.add_argument("--version", action=OnceArgument, - help='Provide a package version if not specified in conanfile') - parser.add_argument("--user", action=OnceArgument, - help='Provide a user if not specified in conanfile') - parser.add_argument("--channel", action=OnceArgument, - help='Provide a channel if not specified in conanfile') + group = parser.add_argument_group("reference arguments") + group.add_argument("--name", action=OnceArgument, + help='Provide a package name if not specified in conanfile') + group.add_argument("--version", action=OnceArgument, + help='Provide a package version if not specified in conanfile') + group.add_argument("--user", action=OnceArgument, + help='Provide a user if not specified in conanfile') + group.add_argument("--channel", action=OnceArgument, + help='Provide a channel if not specified in conanfile') def common_graph_args(subparser): subparser.add_argument("path", nargs="?", help="Path to a folder containing a recipe (conanfile.py " "or conanfile.txt) or to a recipe file. e.g., " - "./my_project/conanfile.txt.") - add_reference_args(subparser) + "./my_project/conanfile.txt. Defaults to the current " + "directory when no --requires or --tool-requires is " + "given", + default=None) + add_common_install_arguments(subparser) subparser.add_argument("--requires", action="append", help='Directly provide requires instead of a conanfile') subparser.add_argument("--tool-requires", action='append', help='Directly provide tool-requires instead of a conanfile') - add_common_install_arguments(subparser) + add_reference_args(subparser) add_lockfile_args(subparser) @@ -134,10 +142,13 @@ def validate_common_graph_args(args): if args.requires and (args.name or args.version or args.user or args.channel): raise ConanException("Can't use --name, --version, --user or --channel arguments with " "--requires") - if not args.path and not args.requires and not args.tool_requires: - raise ConanException("Please specify a path to a conanfile or a '--requires='") if args.path and (args.requires or args.tool_requires): raise ConanException("--requires and --tool-requires arguments are incompatible with " f"[path] '{args.path}' argument") - if not args.path and args.build_require: + + if not args.requires and not args.tool_requires and args.path is None: + args.path = "." + + # graph build-order command does not define a build-require argument + if not args.path and getattr(args, "build_require", False): raise ConanException("--build-require should only be used with argument") diff --git a/conan/cli/cli.py b/conan/cli/cli.py index 4a9ed3ecec8..889c2ae9ce6 100644 --- a/conan/cli/cli.py +++ b/conan/cli/cli.py @@ -95,6 +95,7 @@ def _add_command(self, import_path, method_name, package=None): if command_wrapper.doc: name = f"{package}:{command_wrapper.name}" if package else command_wrapper.name self._commands[name] = command_wrapper + command_wrapper._prog = name # set the program name with possible package, if any # Avoiding duplicated command help messages if name not in self._groups[command_wrapper.group]: self._groups[command_wrapper.group].append(name) @@ -236,17 +237,6 @@ def exception_exit_error(exception): return ERROR_UNEXPECTED -def _warn_python_version(): - version = sys.version_info - if version.minor == 6: - ConanOutput().writeln("") - ConanOutput().warning("*"*80, warn_tag="deprecated") - ConanOutput().warning("Python 3.6 is end-of-life since 2021. " - "Conan future versions will drop support for it, " - "please upgrade Python", warn_tag="deprecated") - ConanOutput().warning("*" * 80, warn_tag="deprecated") - - def _warn_frozen_center(conan_api): remotes = conan_api.remotes.list() for r in remotes: @@ -306,7 +296,6 @@ def ctrl_break_handler(_, __): error = SUCCESS try: cli.run(args) - _warn_python_version() except BaseException as e: error = cli.exception_exit_error(e) sys.exit(error) diff --git a/conan/cli/command.py b/conan/cli/command.py index 4958f3ec5bf..e364aced2bf 100644 --- a/conan/cli/command.py +++ b/conan/cli/command.py @@ -46,11 +46,16 @@ def __init__(self, method, formatters=None): "its use briefly.".format(self._name)) @staticmethod - def _init_log_levels(parser): + def _init_core_options(parser): + # Define possible levels, including "" for verbose + possible_levels = list(ConanOutput.valid_log_levels().keys()) + possible_levels.pop(possible_levels.index(None)) parser.add_argument("-v", default="status", nargs='?', help="Level of detail of the output. Valid options from less verbose " "to more verbose: -vquiet, -verror, -vwarning, -vnotice, -vstatus, " - "-v or -vverbose, -vv or -vdebug, -vvv or -vtrace") + "-v or -vverbose, -vv or -vdebug, -vvv or -vtrace", + choices=possible_levels, + ) parser.add_argument("-cc", "--core-conf", action="append", help="Define core configuration, overwriting global.conf " "values. E.g.: -cc core:non_interactive=True") @@ -70,7 +75,8 @@ def _init_formatters(self, parser): parser.add_argument('-f', '--format', action=OnceArgument, help=help_message) parser.add_argument("--out-file", action=OnceArgument, - help="Write the output of the command to the specified file instead of stdout.") + help="Write the output of the command to the specified file instead of " + "stdout.") @property def name(self): @@ -110,7 +116,12 @@ def _format(self, parser, info, *args): def _dispatch_errors(info): if info and isinstance(info, dict): if info.get("conan_error"): - raise ConanException(info["conan_error"]) + e = info["conan_error"] + # Storing and launching an exception is better than the string, as it keeps + # the correct backtrace for debugging. + if isinstance(e, Exception): + raise e + raise ConanException(e) if info.get("conan_warning"): ConanOutput().warning(info["conan_warning"]) @@ -146,6 +157,7 @@ def __init__(self, method, group=None, formatters=None): self._subcommands = {} self._group = group or "Other" self._name = method.__name__.replace("_", "-") + self._prog = self._name def add_subcommand(self, subcommand): subcommand.set_name(self.name) @@ -153,10 +165,11 @@ def add_subcommand(self, subcommand): def run_cli(self, conan_api, *args): parser = ConanArgumentParser(conan_api, description=self._doc, - prog="conan {}".format(self._name), + prog="conan {}".format(self._prog), formatter_class=SmartFormatter) - self._init_log_levels(parser) self._init_formatters(parser) + self._init_core_options(parser) + parser.suggest_on_error = True info = self._method(conan_api, parser, *args) if not self._subcommands: return info @@ -175,10 +188,11 @@ def run_cli(self, conan_api, *args): def run(self, conan_api, *args): parser = ConanArgumentParser(conan_api, description=self._doc, - prog="conan {}".format(self._name), + prog="conan {}".format(self._prog), formatter_class=SmartFormatter) - self._init_log_levels(parser) self._init_formatters(parser) + self._init_core_options(parser) + parser.suggest_on_error = True info = self._method(conan_api, parser, *args) @@ -226,7 +240,8 @@ def set_parser(self, subcommand_parser, conan_api): self._parser = subcommand_parser.add_parser(self._name, conan_api=conan_api, help=self._doc) self._parser.description = self._doc self._init_formatters(self._parser) - self._init_log_levels(self._parser) + self._init_core_options(self._parser) + self._parser.suggest_on_error = True def conan_command(group=None, formatters=None): diff --git a/conan/cli/commands/audit.py b/conan/cli/commands/audit.py index 9bbe1fff558..c5f08802d99 100644 --- a/conan/cli/commands/audit.py +++ b/conan/cli/commands/audit.py @@ -60,6 +60,9 @@ def audit_scan(conan_api: ConanAPI, parser, subparser, *args) -> dict: help="Set threshold for severity level to raise an error. " "By default raises an error for any critical CVSS (9.0 or higher). " " Use 100.0 to disable it.") + subparser.add_argument("--context", help="Context to scan, by default both contexts are scanned " + "if not specified", + choices=["host", "build"], default=None) _add_provider_arg(subparser) args = parser.parse_args(*args) @@ -97,7 +100,7 @@ def audit_scan(conan_api: ConanAPI, parser, subparser, *args) -> dict: provider = conan_api.audit.get_provider(args.provider or CONAN_CENTER_AUDIT_PROVIDER_NAME) - scan_result = conan_api.audit.scan(deps_graph, provider) + scan_result = conan_api.audit.scan(deps_graph, provider, args.context) _parse_error_threshold(scan_result, args.severity_level) return scan_result @@ -111,9 +114,9 @@ def audit_list(conan_api: ConanAPI, parser, subparser, *args): """ input_group = subparser.add_mutually_exclusive_group(required=True) input_group.add_argument("reference", help="Reference to list vulnerabilities for", nargs="?") - input_group.add_argument("-l", "--list", help="pkglist file to list vulnerabilities for") - input_group.add_argument("-s", "--sbom", help="sbom file to list vulnerabilities for") - input_group.add_argument("-lock", "--lockfile", help="lockfile file to list vulnerabilities for") + input_group.add_argument("-l", "--list", help="Package list file to list vulnerabilities for") + input_group.add_argument("-s", "--sbom", help="SBOM file to list vulnerabilities for") + input_group.add_argument("-lock", "--lockfile", help="Path to the lockfile to check for vulnerabilities") subparser.add_argument("-r", "--remote", help="Remote to use for listing") _add_provider_arg(subparser) args = parser.parse_args(*args) diff --git a/conan/cli/commands/build.py b/conan/cli/commands/build.py index 4720e6259a1..d1a79cf9aaa 100644 --- a/conan/cli/commands/build.py +++ b/conan/cli/commands/build.py @@ -17,7 +17,9 @@ def build(conan_api, parser, *args): parser.add_argument("path", help='Path to a python-based recipe file or a folder ' 'containing a conanfile.py recipe. conanfile.txt ' - 'cannot be used with conan build.') + 'cannot be used with conan build. ' + 'Defaults to current directory', + default=".", nargs='?') add_reference_args(parser) parser.add_argument("-g", "--generator", action="append", help='Generators to use') parser.add_argument("-of", "--output-folder", diff --git a/conan/cli/commands/cache.py b/conan/cli/commands/cache.py index 975ff02b3c3..36cbda6fde1 100644 --- a/conan/cli/commands/cache.py +++ b/conan/cli/commands/cache.py @@ -2,7 +2,7 @@ from conan.api.conan_api import ConanAPI from conan.api.model import ListPattern, MultiPackagesList -from conan.api.output import cli_out_write, ConanOutput +from conan.api.output import cli_out_write from conan.cli import make_abs_path from conan.cli.command import conan_command, conan_subcommand, OnceArgument from conan.cli.commands.list import print_list_text, print_list_json @@ -120,7 +120,14 @@ def cache_clean(conan_api: ConanAPI, parser, subparser, *args): conan_api.cache.clean(package_list) -@conan_subcommand() +def print_list_check_integrity_json(data): + results = data["results"] + myjson = json.dumps(results, indent=4) + cli_out_write(myjson) + + +@conan_subcommand(formatters={"text": lambda _: (), + "json": print_list_check_integrity_json}) def cache_check_integrity(conan_api: ConanAPI, parser, subparser, *args): """ Check the integrity of the local cache for the given references @@ -146,8 +153,10 @@ def cache_check_integrity(conan_api: ConanAPI, parser, subparser, *args): else: ref_pattern = ListPattern(args.pattern, rrev="*", package_id="*", prev="*") package_list = conan_api.list.select(ref_pattern, package_query=args.package_query) - conan_api.cache.check_integrity(package_list) - ConanOutput().success("Integrity check: ok") + + corrupted_artifacts = conan_api.cache.check_integrity(package_list, return_pkg_list=True) + return {"results": {"Local Cache": corrupted_artifacts.serialize()}, + "conan_error": "There are corrupted artifacts, check the error logs" if corrupted_artifacts else ""} @conan_subcommand(formatters={"text": print_list_text, @@ -200,6 +209,6 @@ def cache_backup_upload(conan_api: ConanAPI, parser, subparser, *args): """ Upload all the source backups present in the cache """ - args = parser.parse_args(*args) + parser.parse_args(*args) files = conan_api.cache.get_backup_sources() conan_api.upload.upload_backup_sources(files) diff --git a/conan/cli/commands/config.py b/conan/cli/commands/config.py index c2691e89c01..0bcd1719700 100644 --- a/conan/cli/commands/config.py +++ b/conan/cli/commands/config.py @@ -1,5 +1,8 @@ +import os + from conan.api.model import Remote from conan.api.output import cli_out_write +from conan.cli import make_abs_path from conan.cli.command import conan_command, conan_subcommand, OnceArgument from conan.cli.formatters import default_json_formatter from conan.errors import ConanException @@ -59,8 +62,11 @@ def get_bool_from_text(value): # TODO: deprecate this def config_install_pkg(conan_api, parser, subparser, *args): """ (Experimental) Install the configuration (remotes, profiles, conf), from a Conan package + or from a conanconfig.yml file """ - subparser.add_argument("item", help="Conan require") + subparser.add_argument("reference", nargs="?", + help="Package reference 'pkg/version' to install configuration from " + "or path to 'conanconfig.yml' file") subparser.add_argument("-l", "--lockfile", action=OnceArgument, help="Path to a lockfile. Use --lockfile=\"\" to avoid automatic use of " "existing 'conan.lock' file") @@ -78,6 +84,13 @@ def config_install_pkg(conan_api, parser, subparser, *args): subparser.add_argument("-o", "--options", action="append", help="Options to install config") args = parser.parse_args(*args) + path = make_abs_path(args.reference or ".") + if os.path.isdir(path): + path = os.path.join(path, "conanconfig.yml") + path = path if os.path.exists(path) else None + if path is None and args.reference is None: + raise ConanException("Must provide a package reference or a path to a conanconfig.yml file") + lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, partial=args.lockfile_partial) @@ -87,10 +100,16 @@ def config_install_pkg(conan_api, parser, subparser, *args): default_profile = None profiles = [default_profile] if default_profile else [] profile = conan_api.profiles.get_profile(profiles, args.settings, args.options) - remotes = [Remote("_tmp_conan_config", url=args.url)] if args.url else None - config_pref = conan_api.config.install_pkg(args.item, lockfile=lockfile, force=args.force, - remotes=remotes, profile=profile) - lockfile = conan_api.lockfile.add_lockfile(lockfile, config_requires=[config_pref.ref]) + remotes = [Remote("config_install_url", url=args.url)] if args.url else None + + if path: + conanconfig = path + refs = conan_api.config.install_conanconfig(conanconfig, lockfile=lockfile, force=args.force, + remotes=remotes, profile=profile) + else: + refs = conan_api.config.install_package(args.reference, lockfile=lockfile, force=args.force, + remotes=remotes, profile=profile) + lockfile = conan_api.lockfile.add_lockfile(lockfile, config_requires=refs) conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out) diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index f4ea4ff510f..25d5c3930af 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -68,6 +68,7 @@ def create(conan_api, parser, *args): if args.build is not None and args.build_test is None: args.build_test = args.build + install_error = None if is_python_require: deps_graph = conan_api.graph.load_graph_requires([], [], profile_host=profile_host, @@ -98,7 +99,9 @@ def create(conan_api, parser, *args): update=args.update, lockfile=lockfile) print_graph_packages(deps_graph) - conan_api.install.install_binaries(deps_graph=deps_graph, remotes=remotes) + install_error = conan_api.install.install_binaries(deps_graph=deps_graph, remotes=remotes, + return_install_error=True) + # We update the lockfile, so it will be updated for later ``test_package`` lockfile = conan_api.lockfile.update_lockfile(lockfile, deps_graph, args.lockfile_packages, clean=args.lockfile_clean) @@ -111,7 +114,7 @@ def create(conan_api, parser, *args): and deps_graph.root.edges[0].dst.binary != "Build": test_conanfile_path = None # disable it - if test_conanfile_path: + if test_conanfile_path and not install_error: # TODO: We need arguments for: # - decide update policy "--test_package_update" # If it is a string, it will be injected always, if it is a RecipeReference, then it will @@ -126,7 +129,8 @@ def create(conan_api, parser, *args): conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out, cwd) return {"graph": deps_graph, - "conan_api": conan_api} + "conan_api": conan_api, + "conan_error": install_error} def _check_tested_reference_matches(deps_graph, tested_ref, out): diff --git a/conan/cli/commands/download.py b/conan/cli/commands/download.py index a7c72a0ccbd..386d9f25d37 100644 --- a/conan/cli/commands/download.py +++ b/conan/cli/commands/download.py @@ -56,7 +56,7 @@ def download(conan_api: ConanAPI, parser, *args): ref_pattern = ListPattern(args.pattern, package_id="*", only_recipe=args.only_recipe) package_list = conan_api.list.select(ref_pattern, args.package_query, remote) - if package_list.recipes: + if package_list: conan_api.download.download_full(package_list, remote, args.metadata) else: ConanOutput().warning(f"No packages were downloaded because the package list is empty.") diff --git a/conan/cli/commands/editable.py b/conan/cli/commands/editable.py index 23679fc65a8..20319823a66 100644 --- a/conan/cli/commands/editable.py +++ b/conan/cli/commands/editable.py @@ -19,7 +19,8 @@ def editable_add(conan_api, parser, subparser, *args): Define the given location as the package , so when this package is required, it is used from this location instead of the cache. """ - subparser.add_argument('path', help='Path to the package folder in the user workspace') + subparser.add_argument('path', help='Path to the package folder in the user workspace', + default=".", nargs='?') add_reference_args(subparser) subparser.add_argument("-of", "--output-folder", help='The root output folder for generated and build files') @@ -43,12 +44,16 @@ def editable_remove(conan_api, parser, subparser, *args): Remove the "editable" mode for this reference. """ subparser.add_argument("path", nargs="?", - help="Path to a folder containing a recipe (conanfile.py " - "or conanfile.txt) or to a recipe file. e.g., " - "./my_project/conanfile.txt.") + help="Path to a folder containing a recipe conanfile.py " + "or to a recipe file. e.g., " + "./my_project/conanfile.py.", + default=None) subparser.add_argument("-r", "--refs", action="append", help='Directly provide reference patterns') args = parser.parse_args(*args) + if not args.refs and args.path is None: + args.path = "." + # TODO: Fix this API to use get_conanfile_path editables = conan_api.local.editable_remove(args.path, args.refs) out = ConanOutput() if editables: diff --git a/conan/cli/commands/export.py b/conan/cli/commands/export.py index c052fe1cd4a..cc6c3e53d00 100644 --- a/conan/cli/commands/export.py +++ b/conan/cli/commands/export.py @@ -8,7 +8,9 @@ def common_args_export(parser): - parser.add_argument("path", help="Path to a folder containing a recipe (conanfile.py)") + parser.add_argument("path", help="Path to a folder containing a recipe (conanfile.py). " + "Defaults to current directory", + default=".", nargs="?") add_reference_args(parser) @@ -58,7 +60,7 @@ def export(conan_api, parser, *args): conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out, cwd) exported_list = PackagesList() - exported_list.add_refs([ref]) + exported_list.add_ref(ref) pkglist = MultiPackagesList() pkglist.add("Local Cache", exported_list) diff --git a/conan/cli/commands/export_pkg.py b/conan/cli/commands/export_pkg.py index 06b7936715b..f17b984cfae 100644 --- a/conan/cli/commands/export_pkg.py +++ b/conan/cli/commands/export_pkg.py @@ -6,7 +6,6 @@ from conan.cli.command import conan_command, OnceArgument from conan.cli.commands.create import _get_test_conanfile_path from conan.cli.formatters.graph import format_graph_json -from conan.cli.printers.graph import print_graph_basic from conan.errors import ConanException @@ -15,7 +14,9 @@ def export_pkg(conan_api, parser, *args): """ Create a package directly from pre-compiled binaries. """ - parser.add_argument("path", help="Path to a folder containing a recipe (conanfile.py)") + parser.add_argument("path", help="Path to a folder containing a recipe (conanfile.py). " + "Defaults to current directory", + default=".", nargs="?") parser.add_argument("-of", "--output-folder", help='The root output folder for generated and build files') parser.add_argument("--build-require", action='store_true', default=False, @@ -44,51 +45,33 @@ def export_pkg(conan_api, parser, *args): overrides=overrides) profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) remotes = conan_api.remotes.list(args.remote) if not args.no_remote else [] + output_folder = make_abs_path(args.output_folder, cwd) if args.output_folder else None + # UX error check, for python-requires use "conan export" or "conan create" conanfile = conan_api.local.inspect(path, remotes, lockfile, name=args.name, version=args.version, user=args.user, channel=args.channel) # The package_type is not fully processed at export if conanfile.package_type == "python-require": raise ConanException("export-pkg can only be used for binaries, not for 'python-require'") + + # First, ensure recipe is exported ref, conanfile = conan_api.export.export(path=path, name=args.name, version=args.version, user=args.user, channel=args.channel, lockfile=lockfile, remotes=remotes) lockfile = conan_api.lockfile.update_lockfile_export(lockfile, conanfile, ref, args.build_require) - # TODO: Maybe we want to be able to export-pkg it as --build-require - deps_graph = conan_api.graph.load_graph_consumer(path, - ref.name, str(ref.version), ref.user, ref.channel, - profile_host=profile_host, - profile_build=profile_build, - lockfile=lockfile, remotes=remotes, update=None, - is_build_require=args.build_require) - - print_graph_basic(deps_graph) - deps_graph.report_graph_error() - conan_api.graph.analyze_binaries(deps_graph, build_mode=[ref.name], lockfile=lockfile, - remotes=remotes) - deps_graph.report_graph_error() - - root_node = deps_graph.root - root_node.ref = ref - - if not args.skip_binaries: - # unless the user explicitly opts-out with --skip-binaries, it is necessary to install - # binaries, in case there are build_requires necessary to export, like tool-requires=cmake - # and package() method doing ``cmake.install()`` - # for most cases, deps would be in local cache already because of a previous "conan install" - # but if it is not the case, the binaries from remotes will be downloaded - conan_api.install.install_binaries(deps_graph=deps_graph, remotes=remotes) - source_folder = os.path.dirname(path) - output_folder = make_abs_path(args.output_folder, cwd) if args.output_folder else None - conan_api.install.install_consumer(deps_graph=deps_graph, source_folder=source_folder, - output_folder=output_folder) + # Compute the dependency graph to prepare the exporting of the package biary + graph = conan_api.export.export_pkg_graph(path=path, ref=ref, profile_host=profile_host, + profile_build=profile_build, lockfile=lockfile, + remotes=remotes, is_build_require=args.build_require, + skip_binaries=args.skip_binaries, + output_folder=output_folder) + # Now export the final binary ConanOutput().title("Exporting recipe and package to the cache") - conan_api.export.export_pkg(deps_graph, source_folder, output_folder) - - lockfile = conan_api.lockfile.update_lockfile(lockfile, deps_graph, args.lockfile_packages, + conan_api.export.export_pkg(graph, output_folder) + lockfile = conan_api.lockfile.update_lockfile(lockfile, graph, args.lockfile_packages, clean=args.lockfile_clean) test_package_folder = getattr(conanfile, "test_package_folder", None) \ @@ -102,5 +85,5 @@ def export_pkg(conan_api, parser, *args): remotes=remotes, lockfile=lockfile, update=None, build_modes=None) conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out, cwd) - return {"graph": deps_graph, + return {"graph": graph, "conan_api": conan_api} diff --git a/conan/cli/commands/graph.py b/conan/cli/commands/graph.py index 290bac5bda8..0e94e186c33 100644 --- a/conan/cli/commands/graph.py +++ b/conan/cli/commands/graph.py @@ -72,11 +72,7 @@ def graph_build_order(conan_api, parser, subparser, *args): help='Reduce the build order, output only those to build. Use this ' 'only if the result will not be merged later with other build-order') args = parser.parse_args(*args) - - # parameter validation - if args.requires and (args.name or args.version or args.user or args.channel): - raise ConanException("Can't use --name, --version, --user or --channel arguments with " - "--requires") + validate_common_graph_args(args) if args.order_by is None: ConanOutput().warning("Please specify --order-by argument", warn_tag="deprecated") @@ -173,7 +169,6 @@ def graph_info(conan_api, parser, subparser, *args): subparser.add_argument("--build-require", action='store_true', default=False, help='Whether the provided reference is a build-require') args = parser.parse_args(*args) - # parameter validation validate_common_graph_args(args) if args.format in ("html", "dot") and args.filter: diff --git a/conan/cli/commands/inspect.py b/conan/cli/commands/inspect.py index a7268ef0d30..e129d132727 100644 --- a/conan/cli/commands/inspect.py +++ b/conan/cli/commands/inspect.py @@ -23,7 +23,9 @@ def inspect(conan_api, parser, *args): """ Inspect a conanfile.py to return its public fields. """ - parser.add_argument("path", help="Path to a folder containing a recipe (conanfile.py)") + parser.add_argument("path", help="Path to a folder containing a recipe (conanfile.py). " + "Defaults to current directory", + default=".", nargs="?") group = parser.add_mutually_exclusive_group() group.add_argument("-r", "--remote", default=None, action="append", help="Remote names. Accepts wildcards ('*' means all the remotes available)") diff --git a/conan/cli/commands/install.py b/conan/cli/commands/install.py index faff8ed2a96..61643aeeab7 100644 --- a/conan/cli/commands/install.py +++ b/conan/cli/commands/install.py @@ -43,8 +43,29 @@ def install(conan_api, parser, *args): help="Generation strategy for virtual environment files for the root") args = parser.parse_args(*args) validate_common_graph_args(args) - # basic paths cwd = os.getcwd() + + deps_graph, lockfile, install_error = _run_install_command(conan_api, args, cwd) + + # Update lockfile if necessary + lockfile = conan_api.lockfile.update_lockfile(lockfile, deps_graph, args.lockfile_packages, + clean=args.lockfile_clean) + conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out, cwd) + return {"graph": deps_graph, + "conan_api": conan_api, + "conan_error": install_error} + + +def _run_install_command(conan_api, args, cwd, return_install_error=True): + """ + This method should not be imported as-is, it is internal to the installation process and + its signature might change without warning. + + Users are however free to copy its code and adapt it to their needs, as an example of + using the Conan API to perform an installation + """ + # basic paths + path = conan_api.local.get_conanfile_path(args.path, cwd, py=None) if args.path else None source_folder = os.path.dirname(path) if args.path else cwd output_folder = make_abs_path(args.output_folder, cwd) if args.output_folder else None @@ -72,17 +93,15 @@ def install(conan_api, parser, *args): print_graph_packages(deps_graph) # Installation of binaries and consumer generators - conan_api.install.install_binaries(deps_graph=deps_graph, remotes=remotes) - ConanOutput().title("Finalizing install (deploy, generators)") - conan_api.install.install_consumer(deps_graph, args.generator, source_folder, output_folder, - deploy=args.deployer, deploy_package=args.deployer_package, - deploy_folder=args.deployer_folder, - envs_generation=args.envs_generation) - ConanOutput().success("Install finished successfully") + install_error = conan_api.install.install_binaries(deps_graph=deps_graph, remotes=remotes, + return_install_error=return_install_error) + if not install_error: + ConanOutput().title("Finalizing install (deploy, generators)") + conan_api.install.install_consumer(deps_graph, args.generator, source_folder, output_folder, + deploy=getattr(args, "deployer", None), + deploy_package=getattr(args, "deployer_package", None), + deploy_folder=getattr(args, "deployer_folder", None), + envs_generation=getattr(args, "envs_generation", None)) + ConanOutput().success("Install finished successfully") - # Update lockfile if necessary - lockfile = conan_api.lockfile.update_lockfile(lockfile, deps_graph, args.lockfile_packages, - clean=args.lockfile_clean) - conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out, cwd) - return {"graph": deps_graph, - "conan_api": conan_api} + return deps_graph, lockfile, install_error diff --git a/conan/cli/commands/list.py b/conan/cli/commands/list.py index 6b5100b2ae1..3f8b6f7da70 100644 --- a/conan/cli/commands/list.py +++ b/conan/cli/commands/list.py @@ -57,7 +57,7 @@ def print_serial(item, indent=None, color_index=None): def print_list_text(results): """ Do a little format modification to serialized - list bundle, so it looks prettier on text output + package list, so it looks prettier on text output """ info = results["results"] @@ -264,6 +264,13 @@ def list(conan_api: ConanAPI, parser, *args): args.filter_settings or args.filter_options): raise ConanException("--package-query and --filter-xxx can only be done for binaries, " "a 'pkgname/version:*' pattern is necessary") + if args.lru: + if not ref_pattern.rrev and not ref_pattern.package_id: # If package_id => #latest + raise ConanException("'--lru' must be used with recipe revision pattern, " + "use 'pkg/version#*' argument") + if ref_pattern.package_id and not ref_pattern.prev: + raise ConanException("'--lru' must be used with package revision pattern, " + "use 'pkg/version:*#*' argument") # If neither remote nor cache are defined, show results only from cache pkglist = MultiPackagesList() profile = conan_api.profiles.get_profile(args.filter_profile or [], diff --git a/conan/cli/commands/lock.py b/conan/cli/commands/lock.py index 76ec4cc78a6..83e3fc7bba1 100644 --- a/conan/cli/commands/lock.py +++ b/conan/cli/commands/lock.py @@ -1,4 +1,3 @@ -from collections import defaultdict import os from conan.api.output import ConanOutput @@ -27,7 +26,6 @@ def lock_create(conan_api, parser, subparser, *args): subparser.add_argument("--build-require", action='store_true', default=False, help='Whether the provided reference is a build-require') args = parser.parse_args(*args) - # parameter validation validate_common_graph_args(args) @@ -175,29 +173,28 @@ def lock_update(conan_api, parser, subparser, *args): conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out) - - @conan_subcommand() def lock_upgrade(conan_api, parser, subparser, *args): """ - (Experimental) Upgrade requires, build-requires or python-requires from an existing lockfile given a conanfile - or a reference. + (Experimental) Upgrade requires, build-requires or python-requires from an existing lockfile + given a conanfile or a reference. """ - common_graph_args(subparser) - subparser.add_argument('--update-requires', action="append", help='Update requires from lockfile') - subparser.add_argument('--update-build-requires', action="append", help='Update build-requires from lockfile') - subparser.add_argument('--update-python-requires', action="append", help='Update python-requires from lockfile') - subparser.add_argument('--update-config-requires', action="append", help='Update config-requires from lockfile') - subparser.add_argument('--build-require', action='store_true', default=False, help='Whether the provided reference is a build-require') + subparser.add_argument('--update-requires', action="append", + help='Update requires from lockfile') + subparser.add_argument('--update-build-requires', action="append", + help='Update build-requires from lockfile') + subparser.add_argument('--update-python-requires', action="append", + help='Update python-requires from lockfile') + subparser.add_argument('--build-require', action='store_true', default=False, + help='Whether the provided reference is a build-require') args = parser.parse_args(*args) - # parameter validation validate_common_graph_args(args) - if not any([args.update_requires, args.update_build_requires, args.update_python_requires, args.update_config_requires]): + if not any([args.update_requires, args.update_build_requires, args.update_python_requires]): raise ConanException("At least one of --update-requires, --update-build-requires, " - "--update-python-requires or --update-config-requires should be specified") + "--update-python-requires should be specified") cwd = os.getcwd() path = conan_api.local.get_conanfile_path(args.path, cwd, py=None) if args.path else None @@ -205,14 +202,15 @@ def lock_upgrade(conan_api, parser, subparser, *args): overrides = eval(args.lockfile_overrides) if args.lockfile_overrides else None lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=path, cwd=cwd, partial=True, overrides=overrides) + if lockfile is None: + raise ConanException("No lockfile specified and default conan.lock not found") profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) # Remove the lockfile entries that will be updated lockfile = conan_api.lockfile.remove_lockfile(lockfile, requires=args.update_requires, python_requires=args.update_python_requires, - build_requires=args.update_build_requires, - config_requires=args.update_config_requires) + build_requires=args.update_build_requires) # Resolve new graph if path: graph = conan_api.graph.load_graph_consumer(path, args.name, args.version, @@ -230,6 +228,44 @@ def lock_upgrade(conan_api, parser, subparser, *args): lockfile=lockfile) print_graph_packages(graph) - lockfile = conan_api.lockfile.update_lockfile(lockfile, graph, args.lockfile_packages, - clean=args.lockfile_clean) + lockfile = conan_api.lockfile.update_lockfile(lockfile, graph, clean=args.lockfile_clean) + conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out or "conan.lock") + + +@conan_subcommand() +def lock_upgrade_config(conan_api, parser, subparser, *args): + """ + (Experimental) Upgrade config requires in a lockfile + """ + common_graph_args(subparser) + subparser.add_argument('--update-config-requires', action="append", + help='Update config-requires from lockfile') + args = parser.parse_args(*args) + validate_common_graph_args(args) + + if not args.update_config_requires: + raise ConanException("At least one --update-config-requires should be specified") + + cwd = os.getcwd() + path = conan_api.local.get_conanfile_path(args.path, cwd, py=None) if args.path else None + remotes = conan_api.remotes.list(args.remote) if not args.no_remote else [] + lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=path, + cwd=cwd, partial=True) + if lockfile is None: + raise ConanException("No lockfile specified and default conan.lock not found") + profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) + + # Remove the lockfile entries that will be updated + lockfile = conan_api.lockfile.remove_lockfile(lockfile, + config_requires=args.update_config_requires) + + if args.path: + path = make_abs_path(args.path) + reqs, remotes = conan_api.config.load_conanconfig(path, remotes) + else: + reqs = [RecipeReference.loads(r) for r in args.requires] + + pkgs = conan_api.config.fetch_packages(reqs, lockfile, remotes, profile_host) + refs = [p.ref for p in pkgs] + lockfile = conan_api.lockfile.add_lockfile(lockfile, config_requires=refs) conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out or "conan.lock") diff --git a/conan/cli/commands/remote.py b/conan/cli/commands/remote.py index 671ecf75bd3..43cd7d2d813 100644 --- a/conan/cli/commands/remote.py +++ b/conan/cli/commands/remote.py @@ -17,7 +17,8 @@ def _print_remotes_json(remotes): "url": r.url, "verify_ssl": r.verify_ssl, "enabled": not r.disabled, - "allowed_packages": r.allowed_packages,} + "allowed_packages": r.allowed_packages, + "recipes_only": r.recipes_only} for r in remotes] cli_out_write(json.dumps(info, indent=4)) @@ -83,6 +84,9 @@ def remote_add(conan_api, parser, subparser, *args): "this remote") subparser.add_argument("-t", "--type", choices=[LOCAL_RECIPES_INDEX], help="Define the remote type") + subparser.add_argument("--recipes-only", action="store_true", default=False, + help="Disallow binary downloads from this remote, only recipes " + "will be downloaded") subparser.set_defaults(secure=True) args = parser.parse_args(*args) @@ -91,7 +95,8 @@ def remote_add(conan_api, parser, subparser, *args): remote_type = args.type or (LOCAL_RECIPES_INDEX if os.path.isdir(url_folder) else None) url = url_folder if remote_type == LOCAL_RECIPES_INDEX else args.url r = Remote(args.name, url, args.secure, disabled=False, remote_type=remote_type, - allowed_packages=args.allowed_packages) + allowed_packages=args.allowed_packages, + recipes_only=args.recipes_only) conan_api.remotes.add(r, force=args.force, index=args.index) @@ -122,12 +127,22 @@ def remote_update(conan_api, parser, subparser, *args): subparser.add_argument("--index", action=OnceArgument, type=int, help="Insert the remote at a specific position in the remote list") subparser.add_argument("-ap", "--allowed-packages", action="append", default=None, - help="Add recipe reference pattern to the list of allowed packages for this remote") + help="Add recipe reference pattern to the list of allowed packages " + "for this remote") + subparser.add_argument("--recipes-only", default=None, const="True", nargs="?", + choices=["True", "False"], + help="Disallow binary downloads from this remote, only recipes will " + "be downloaded") + subparser.set_defaults(secure=None) args = parser.parse_args(*args) - if args.url is None and args.secure is None and args.index is None and args.allowed_packages is None: + if (args.url is None and args.secure is None and args.index is None and + args.allowed_packages is None and args.recipes_only is None): subparser.error("Please add at least one argument to update") - conan_api.remotes.update(args.remote, args.url, args.secure, index=args.index, allowed_packages=args.allowed_packages) + args.recipes_only = None if args.recipes_only is None else args.recipes_only == "True" + conan_api.remotes.update(args.remote, args.url, args.secure, index=args.index, + allowed_packages=args.allowed_packages, + recipes_only=args.recipes_only) @conan_subcommand() @@ -263,7 +278,8 @@ def _print_auth_json(results): @conan_subcommand(formatters={"text": _print_auth, "json": _print_auth_json}) def remote_auth(conan_api, parser, subparser, *args): """ - Authenticate in the defined remotes. Use CONAN_LOGIN_USERNAME* and CONAN_PASSWORD* variables if available. + Authenticate in the defined remotes. Use CONAN_LOGIN_USERNAME* and CONAN_PASSWORD* variables + if available. Ask for username and password interactively in case (re-)authentication is required and there are no CONAN_LOGIN* and CONAN_PASSWORD* variables available which could be used. Usually you'd use this method over conan remote login for scripting which needs to run in CI diff --git a/conan/cli/commands/remove.py b/conan/cli/commands/remove.py index 5e488b9b030..f57e180d337 100644 --- a/conan/cli/commands/remove.py +++ b/conan/cli/commands/remove.py @@ -1,5 +1,5 @@ from conan.api.conan_api import ConanAPI -from conan.api.model import ListPattern, MultiPackagesList +from conan.api.model import ListPattern, MultiPackagesList, PackagesList from conan.api.output import cli_out_write, ConanOutput from conan.api.input import UserInput from conan.cli import make_abs_path @@ -10,7 +10,7 @@ def summary_remove_list(results): """ Do a little format modification to serialized - list bundle, so it looks prettier on text output + package list so it looks prettier on text output """ cli_out_write("Remove summary:") info = results["results"] @@ -81,7 +81,7 @@ def confirmation(message): listfile = make_abs_path(args.list) multi_package_list = MultiPackagesList.load(listfile) package_list = multi_package_list[cache_name] - refs_to_remove = package_list.refs() + refs_to_remove = list(package_list.items()) if not refs_to_remove: # the package list might contain only refs, no revs ConanOutput().warning("Nothing to remove, package list do not contain recipe revisions") else: @@ -92,36 +92,30 @@ def confirmation(message): multi_package_list = MultiPackagesList() multi_package_list.add(cache_name, package_list) - # TODO: This iteration and removal of not-confirmed is ugly and complicated, improve it - for ref, ref_bundle in package_list.refs().items(): - ref_dict = package_list.recipes[str(ref)]["revisions"] - packages = ref_bundle.get("packages") - if packages is None: + result = PackagesList() + for ref, packages in package_list.items(): + ref_dict = package_list.recipe_dict(ref).copy() + packages_dict = ref_dict.pop("packages", None) + if packages_dict is None: if confirmation(f"Remove the recipe and all the packages of '{ref.repr_notime()}'?"): if not args.dry_run: conan_api.remove.recipe(ref, remote=remote) - else: - ref_dict.pop(ref.revision) - if not ref_dict: - package_list.recipes.pop(str(ref)) - continue - prefs = package_list.prefs(ref, ref_bundle) - if not prefs: - ConanOutput().info(f"No binaries to remove for '{ref.repr_notime()}'") - ref_dict.pop(ref.revision) - if not ref_dict: - package_list.recipes.pop(str(ref)) - continue - - for pref, _ in prefs.items(): - if confirmation(f"Remove the package '{pref.repr_notime()}'?"): - if not args.dry_run: - conan_api.remove.package(pref, remote=remote) - else: - pref_dict = packages[pref.package_id]["revisions"] - pref_dict.pop(pref.revision) - if not pref_dict: - packages.pop(pref.package_id) + result.add_ref(ref) + result.recipe_dict(ref).update(ref_dict) # it doesn't contain "packages" + else: + if not packages: # weird, there is inner package-ids but without prevs + ConanOutput().info(f"No binaries to remove for '{ref.repr_notime()}'") + continue + for pref, pkg_id_info in packages.items(): + if confirmation(f"Remove the package '{pref.repr_notime()}'?"): + if not args.dry_run: + conan_api.remove.package(pref, remote=remote) + result.add_ref(ref) + result.recipe_dict(ref).update(ref_dict) # it doesn't contain "packages" + result.add_pref(pref, pkg_id_info) + pkg_dict = package_list.package_dict(pref) + result.package_dict(pref).update(pkg_dict) + multi_package_list.add(cache_name, result) return { "results": multi_package_list.serialize(), diff --git a/conan/cli/commands/report.py b/conan/cli/commands/report.py index 84b8fc55214..1fd1958f3a7 100644 --- a/conan/cli/commands/report.py +++ b/conan/cli/commands/report.py @@ -5,7 +5,6 @@ from conan.cli.command import conan_command, conan_subcommand - @conan_command(group="Security") def report(conan_api: ConanAPI, parser, *args): """ @@ -40,7 +39,7 @@ def report_diff(conan_api, parser, subparser, *args): subparser.add_argument("-nr", "--new-reference", help=ref_help.format(type="New"), required=True) subparser.add_argument("-r", "--remote", action="append", default=None, - help='Look in the specified remote or remotes server') + help='Look in the specified remote or remotes server') args = parser.parse_args(*args) diff --git a/conan/cli/commands/run.py b/conan/cli/commands/run.py new file mode 100644 index 00000000000..70007250d6f --- /dev/null +++ b/conan/cli/commands/run.py @@ -0,0 +1,55 @@ +import os +import tempfile + +from conan.api.output import ConanOutput, LEVEL_STATUS, Color, LEVEL_ERROR, LEVEL_QUIET +from conan.cli.args import common_graph_args, validate_common_graph_args +from conan.cli.command import conan_command +from conan.cli.commands.install import _run_install_command +from conan.errors import ConanException + + +@conan_command(group="Consumer") +def run(conan_api, parser, *args): + """ + (Experimental) Run a command given a set of requirements from a recipe or from command line. + """ + common_graph_args(parser) + parser.add_argument("command", help="Command to run") + parser.add_argument("--context", help="Context to use, by default both contexts are activated " + "if not specified", + choices=["host", "build"], default=None) + parser.add_argument("--build-require", action='store_true', default=False, + help='Whether the provided path is a build-require') + args = parser.parse_args(*args) + validate_common_graph_args(args) + cwd = os.getcwd() + + ConanOutput().info("Installing and building dependencies, this might take a while...", + fg=Color.BRIGHT_MAGENTA) + previous_log_level = ConanOutput.get_output_level() + if previous_log_level == LEVEL_STATUS: + ConanOutput.set_output_level(LEVEL_QUIET) + + with tempfile.TemporaryDirectory("conanrun") as tmpdir: + # Default values for install + setattr(args, "output_folder", tmpdir) + setattr(args, "generator", []) + try: + deps_graph, lockfile, _ = _run_install_command(conan_api, args, cwd, + return_install_error=False) + except ConanException as e: + ConanOutput.set_output_level(previous_log_level) + ConanOutput().error("Error installing the dependencies. To debug this, you can either:\n" + " - Re-run the command with increased verbosity (-v, -vv)\n" + " - Run 'conan install' first to ensure dependencies are installed, " + "or to see errors during installation\n") + raise e + + context_env_map = { + "build": "conanbuild", + "host": "conanrun", + } + envfiles = list(context_env_map.values()) if args.context is None \ + else [context_env_map.get(args.context)] + ConanOutput.set_output_level(LEVEL_ERROR) + deps_graph.root.conanfile.run(args.command, cwd=cwd, env=envfiles) diff --git a/conan/cli/commands/search.py b/conan/cli/commands/search.py index 7a34033502c..08ca30c1660 100644 --- a/conan/cli/commands/search.py +++ b/conan/cli/commands/search.py @@ -7,7 +7,6 @@ from conan.errors import ConanException -# FIXME: "conan search" == "conan list (*) -r="*"" --> implement @conan_alias_command?? @conan_command(group="Consumer", formatters={"text": print_list_text, "json": print_list_json}) def search(conan_api: ConanAPI, parser, *args): @@ -31,11 +30,11 @@ def search(conan_api: ConanAPI, parser, *args): results = OrderedDict() for remote in remotes: try: - list_bundle = conan_api.list.select(ref_pattern, package_query=None, remote=remote) + pkglist = conan_api.list.select(ref_pattern, package_query=None, remote=remote) except Exception as e: results[remote.name] = {"error": str(e)} else: - results[remote.name] = list_bundle.serialize() + results[remote.name] = pkglist.serialize() return { "results": results } diff --git a/conan/cli/commands/source.py b/conan/cli/commands/source.py index 213d4ff08ce..4a694964b74 100644 --- a/conan/cli/commands/source.py +++ b/conan/cli/commands/source.py @@ -9,7 +9,9 @@ def source(conan_api, parser, *args): """ Call the source() method. """ - parser.add_argument("path", help="Path to a folder containing a conanfile.py") + parser.add_argument("path", help="Path to a folder containing a conanfile.py. " + "Defaults to current directory", + default=".", nargs="?") add_reference_args(parser) args = parser.parse_args(*args) diff --git a/conan/cli/commands/test.py b/conan/cli/commands/test.py index b030a8eb645..ca8d461832b 100644 --- a/conan/cli/commands/test.py +++ b/conan/cli/commands/test.py @@ -16,7 +16,9 @@ def test(conan_api, parser, *args): Test a package from a test_package folder. """ parser.add_argument("path", action=OnceArgument, - help="Path to a test_package folder containing a conanfile.py") + help="Path to a test_package folder containing a conanfile.py. " + "Defaults to a 'test_package' folder in the current directory", + default="test_package", nargs='?') parser.add_argument("reference", action=OnceArgument, help='Provide a package reference to test') add_common_install_arguments(parser) diff --git a/conan/cli/commands/upload.py b/conan/cli/commands/upload.py index 0338002c9dd..8211d9bc53f 100644 --- a/conan/cli/commands/upload.py +++ b/conan/cli/commands/upload.py @@ -1,5 +1,5 @@ from conan.api.conan_api import ConanAPI -from conan.api.model import ListPattern, MultiPackagesList +from conan.api.model import ListPattern, MultiPackagesList, PackagesList from conan.api.output import ConanOutput from conan.cli import make_abs_path from conan.cli.command import conan_command, OnceArgument @@ -10,7 +10,7 @@ def summary_upload_list(results): """ Do a little format modification to serialized - list bundle, so it looks prettier on text output + package list, so it looks prettier on text output """ ConanOutput().subtitle("Upload summary") info = results["results"] @@ -97,10 +97,10 @@ def upload(conan_api: ConanAPI, parser, *args): ref_pattern = ListPattern(args.pattern, package_id="*", only_recipe=args.only_recipe) package_list = conan_api.list.select(ref_pattern, package_query=args.package_query) - if package_list.recipes: + if package_list: # If only if search with "*" we ask for confirmation if not args.list and not args.confirm and "*" in args.pattern: - _ask_confirm_upload(conan_api, package_list) + package_list = _ask_confirm_upload(conan_api, package_list) conan_api.upload.upload_full(package_list, remote, enabled_remotes, args.check, args.force, args.metadata, args.dry_run) @@ -121,20 +121,17 @@ def upload(conan_api: ConanAPI, parser, *args): def _ask_confirm_upload(conan_api, package_list): ui = UserInput(conan_api.config.get("core:non_interactive")) - for ref, bundle in package_list.refs().items(): - msg = "Are you sure you want to upload recipe '%s'?" % ref.repr_notime() - ref_dict = package_list.recipes[str(ref)]["revisions"] - if not ui.request_boolean(msg): - ref_dict.pop(ref.revision) - # clean up empy refs - if not ref_dict: - package_list.recipes.pop(str(ref)) - else: - for pref, prev_bundle in package_list.prefs(ref, bundle).items(): - msg = "Are you sure you want to upload package '%s'?" % pref.repr_notime() - pkgs_dict = ref_dict[ref.revision]["packages"] - if not ui.request_boolean(msg): - pref_dict = pkgs_dict[pref.package_id]["revisions"] - pref_dict.pop(pref.revision) - if not pref_dict: - pkgs_dict.pop(pref.package_id) + result = PackagesList() + for ref, packages in package_list.items(): + msg = f"Are you sure you want to upload recipe '{ref.repr_notime()}'?" + if ui.request_boolean(msg): + result.add_ref(ref) + ref_dict = package_list.recipe_dict(ref).copy() + ref_dict.pop("packages", None) + result.recipe_dict(ref).update(ref_dict) + for pref, pkg_id_info in packages.items(): + msg = f"Are you sure you want to upload package '{pref.repr_notime()}'?" + if ui.request_boolean(msg): + result.add_pref(pref, pkg_id_info) + result.package_dict(pref).update(package_list.package_dict(pref)) + return result diff --git a/conan/cli/commands/workspace.py b/conan/cli/commands/workspace.py index 8f85e1fc7bb..e21593262c4 100644 --- a/conan/cli/commands/workspace.py +++ b/conan/cli/commands/workspace.py @@ -9,6 +9,7 @@ from conan.cli.args import add_reference_args, add_common_install_arguments, add_lockfile_args from conan.cli.command import conan_command, conan_subcommand from conan.cli.commands.list import print_serial +from conan.cli.formatters.graph import format_graph_json from conan.cli.printers import print_profiles from conan.cli.printers.graph import print_graph_packages, print_graph_basic from conan.errors import ConanException @@ -74,6 +75,29 @@ def workspace_add(conan_api: ConanAPI, parser, subparser, *args): ConanOutput().success("Reference '{}' added to workspace".format(ref)) +@conan_subcommand() +def workspace_complete(conan_api: ConanAPI, parser, subparser, *args): + """ + Complete the workspace, opening or adding intermediate packages to it that have + requirements to other packages in the workspace. + """ + add_common_install_arguments(subparser) + add_lockfile_args(subparser) + args = parser.parse_args(*args) + remotes = conan_api.remotes.list(args.remote) if not args.no_remote else [] + overrides = eval(args.lockfile_overrides) if args.lockfile_overrides else None + # The lockfile by default if not defined will be read from the root workspace folder + ws_folder = conan_api.workspace.folder() + lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=ws_folder, + cwd=None, partial=args.lockfile_partial, + overrides=overrides) + profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) + print_profiles(profile_host, profile_build) + + ConanOutput().box("Workspace completing intermediate packages") + conan_api.workspace.complete(profile_host, profile_build, lockfile, remotes, update=args.update) + + @conan_subcommand() def workspace_remove(conan_api: ConanAPI, parser, subparser, *args): """ @@ -86,7 +110,7 @@ def workspace_remove(conan_api: ConanAPI, parser, subparser, *args): ConanOutput().info(f"Removed from workspace: {removed}") -def print_json(data): +def _print_json(data): results = data["info"] myjson = json.dumps(results, indent=4) cli_out_write(myjson) @@ -103,7 +127,7 @@ def _print_workspace_info(data): print_serial(data["info"]) -@conan_subcommand(formatters={"text": _print_workspace_info, "json": print_json}) +@conan_subcommand(formatters={"text": _print_workspace_info, "json": _print_json}) def workspace_info(conan_api: ConanAPI, parser, subparser, *args): # noqa """ Display info for current workspace @@ -166,14 +190,17 @@ def _install_build(conan_api: ConanAPI, parser, subparser, build, *args): ref = RecipeReference.loads(elem["ref"]) for package_level in elem["packages"]: for package in package_level: - if package["binary"] == "Build": # Build external to Workspace - cmd = f'install {package["build_args"]} {profile_args}' - ConanOutput().box(f"Workspace building external {ref}") - ConanOutput().info(f"Command: {cmd}\n") - conan_api.command.run(cmd) - elif package["binary"] in ("Editable", "EditableBuild"): - path = all_editables[ref]["path"] - output_folder = all_editables[ref].get("output_folder") + ws_pkg = all_editables.get(ref) + is_editable = package["binary"] in ("Editable", "EditableBuild") + if ws_pkg is None: + if is_editable or package["binary"] == "Build": # Build extern to Workspace + cmd = f'install {package["build_args"]} {profile_args}' + ConanOutput().box(f"Workspace building external {ref}") + ConanOutput().info(f"Command: {cmd}\n") + conan_api.command.run(cmd) + else: + path = ws_pkg["path"] + output_folder = ws_pkg.get("output_folder") build_arg = "--build-require" if package["context"] == "build" else "" ref_args = " ".join(f"--{k}={getattr(ref, k)}" for k in ("name", "version", "user", "channel") @@ -187,7 +214,7 @@ def _install_build(conan_api: ConanAPI, parser, subparser, build, *args): conan_api.command.run(cmd) -@conan_subcommand() +@conan_subcommand(formatters={"json": format_graph_json}) def workspace_super_install(conan_api: ConanAPI, parser, subparser, *args): """ Install the workspace as a monolith, installing only external dependencies to the workspace, @@ -197,6 +224,15 @@ def workspace_super_install(conan_api: ConanAPI, parser, subparser, *args): subparser.add_argument("-g", "--generator", action="append", help='Generators to use') subparser.add_argument("-of", "--output-folder", help='The root output folder for generated and build files') + subparser.add_argument("-d", "--deployer", action="append", + help="Deploy using the provided deployer to the output folder. " + "Built-in deployers: 'full_deploy', 'direct_deploy', " + "'runtime_deploy'") + subparser.add_argument("--deployer-folder", + help="Deployer output folder, base build folder by default if not set") + subparser.add_argument("--deployer-package", action="append", + help="Execute the deploy() method of the packages matching " + "the provided patterns") subparser.add_argument("--envs-generation", default=None, choices=["false"], help="Generation strategy for virtual environment files for the root") add_common_install_arguments(subparser) @@ -217,7 +253,7 @@ def workspace_super_install(conan_api: ConanAPI, parser, subparser, *args): requires = conan_api.workspace.select_packages(args.pkg) deps_graph = conan_api.graph.load_graph_requires(requires, [], profile_host, profile_build, lockfile, - remotes, args.build, args.update) + remotes, update=args.update) deps_graph.report_graph_error() print_graph_basic(deps_graph) @@ -225,14 +261,20 @@ def workspace_super_install(conan_api: ConanAPI, parser, subparser, *args): ws_graph = conan_api.workspace.super_build_graph(deps_graph, profile_host, profile_build) ConanOutput().subtitle("Collapsed graph") print_graph_basic(ws_graph) - conan_api.graph.analyze_binaries(ws_graph, args.build, remotes=remotes, update=args.update, lockfile=lockfile) print_graph_packages(ws_graph) conan_api.install.install_binaries(deps_graph=ws_graph, remotes=remotes) + ConanOutput().title("Finalizing install (deploy, generators)") output_folder = make_abs_path(args.output_folder) if args.output_folder else None conan_api.install.install_consumer(ws_graph, args.generator, ws_folder, output_folder, + deploy=args.deployer, deploy_package=args.deployer_package, + deploy_folder=args.deployer_folder, envs_generation=args.envs_generation) + ConanOutput().success("Install finished successfully") + + return {"graph": ws_graph, + "conan_api": conan_api} @conan_subcommand() @@ -250,8 +292,8 @@ def workspace_init(conan_api: ConanAPI, parser, subparser, *args): Clean the temporary build folders when possible """ subparser.add_argument("path", nargs="?", default=os.getcwd(), - help="Path to a folder where the workspace will be initialized. " - "If does not exist") + help="Path to a folder where the workspace will be initialized. " + "Defaults to the current directory") args = parser.parse_args(*args) conan_api.workspace.init(args.path) diff --git a/conan/cli/formatters/audit/vulnerabilities.py b/conan/cli/formatters/audit/vulnerabilities.py index 70a10c4e485..468f5d8f21d 100644 --- a/conan/cli/formatters/audit/vulnerabilities.py +++ b/conan/cli/formatters/audit/vulnerabilities.py @@ -3,7 +3,7 @@ from jinja2 import select_autoescape, Template from conan.api.output import cli_out_write, Color -from conan.errors import ConanException + severity_order = { "Critical": 4, @@ -77,14 +77,56 @@ def wrap_and_indent(txt, limit=80, indent=2): desc = node.get("description", "") desc = (desc[:240] + "...") if len(desc) > 240 else desc desc_wrapped = wrap_and_indent(desc) + isWithdrawn = node.get("withdrawn", False) + publishedAt = node.get("publishedAt") cli_out_write(f"- {name}", fg=Color.BRIGHT_WHITE, endline="") + if isWithdrawn: + cli_out_write(" [WITHDRAWN]", fg=Color.BRIGHT_CYAN, endline="") cli_out_write(f" (Severity: {sev}{score_txt})", fg=sev_color) + advisories = node.get("advisories", {}) + jfrog_advisories = [adv for adv in advisories + if adv.get("name", "").startswith("JFSA-")] + for adv in jfrog_advisories: + if adv.get("shortDescription"): + cli_out_write(f" Summary provided by JFrog Research ({adv['name']})", + fg=Color.BRIGHT_GREEN) + cli_out_write(wrap_and_indent(f"Short description: {adv['shortDescription']}", + indent=4)) + if adv.get("severity"): + cli_out_write(f" Severity: ", endline="") + cli_out_write(adv['severity'], fg=severity_colors.get(adv['severity'])) + reasons = adv.get("impactReasons", []) + if reasons: + cli_out_write(f" Impact reasons:") + for reason in reasons: + cli_out_write(wrap_and_indent(f"* {reason['name']}", indent=8), + fg=Color.GREEN if reason['isPositive'] else Color.RED) + if result["provider_url"]: + expected_url = (result["provider_url"].rstrip("/") + + f"/ui/catalog/vulnerabilities/details/{adv['name']}") + cli_out_write(f" Url: {expected_url}") + cli_out_write("") + cli_out_write("\n" + desc_wrapped) + if publishedAt: + cli_out_write(f" Published at: ", endline="", fg=Color.BRIGHT_BLUE) + cli_out_write(publishedAt) + references = node.get("references") if references: - cli_out_write(f" url: {references[0]}", fg=Color.BRIGHT_BLUE) + cli_out_write(f" url: ", endline="", fg=Color.BRIGHT_BLUE) + cli_out_write(references[0]) + + vulnerablePackages = node.get("vulnerablePackages") + if vulnerablePackages: + fixVersions = [fix['version'] + for fix_edge in vulnerablePackages.get("edges", []) + for fix in fix_edge['node'].get("fixVersions", [])] + if fixVersions: + cli_out_write(f" fixed in version(s): ", endline="", fg=Color.BRIGHT_BLUE) + cli_out_write(', '.join(fixVersions)) cli_out_write("") color_for_total = Color.BRIGHT_RED if total_vulns else Color.BRIGHT_GREEN @@ -100,7 +142,7 @@ def wrap_and_indent(txt, limit=80, indent=2): "through patches applied in the recipe.\nTo verify if a patch has been applied, check the recipe in Conan Center.\n", fg=Color.BRIGHT_YELLOW) - if total_vulns > 0 or not "error" in result: + if total_vulns > 0 or "error" not in result: cli_out_write("\nVulnerability information provided by JFrog Catalog. Check " "https://audit.conan.io/jfrogcuration for more information.\n", fg=Color.BRIGHT_GREEN) @@ -118,29 +160,28 @@ def _render_vulns(vulns, template): template = Template(template, autoescape=select_autoescape(['html', 'xml'])) return template.render(vulns=vulns, version=__version__) + vuln_html = """ Conan Audit Vulnerabilities Report - + - - + + @@ -167,20 +214,16 @@ def _render_vulns(vulns, template):

Conan Audit Vulnerabilities Report

- +
- - - + - - - + @@ -190,21 +233,69 @@ def _render_vulns(vulns, template): {% set severity_id = parts[0] %} {% set severity_label = parts[1] if parts|length > 1 else parts[0] %} - - + - @@ -229,12 +320,14 @@ def _render_vulns(vulns, template): """ + def html_vuln_formatter(result): vulns = [] for ref, pkg_info in result["data"].items(): edges = pkg_info.get("vulnerabilities", {}).get("edges", []) if not edges: - description = "No vulnerabilities found." if not "error" in pkg_info else pkg_info["error"].get("details", "") + description = "No vulnerabilities found." if "error" not in pkg_info \ + else pkg_info["error"].get("details", "") vulns.append({ "package": ref, "vuln_id": "-", @@ -242,7 +335,12 @@ def html_vuln_formatter(result): "severity": "N/A", "score": "-", "description": description, - "references": [] + "references": [], + "withdrawn": False, + "advisories": [], + "provider_url": result.get("provider_url"), + "fixVersions": [], + "publishedAt": None }) else: sorted_vulns = sorted(edges, key=lambda v: -severity_order.get(v["node"].get("severity", "Medium"), 2)) @@ -256,6 +354,13 @@ def html_vuln_formatter(result): aliases = node.get("aliases", []) references = node.get("references", []) desc = node.get("description", "") + withdrawn = node.get("withdrawn", False) + advisories = node.get("advisories", []) + jfrogAdvisories = [adv for adv in advisories + if adv.get("name", "").startswith("JFSA-")] + fixVersions = [fix['version'] + for fix_edge in node.get("vulnerablePackages", {}).get("edges", []) + for fix in fix_edge['node'].get("fixVersions", [])] vulns.append({ "package": ref, "vuln_id": name, @@ -264,6 +369,11 @@ def html_vuln_formatter(result): "score": score_txt, "description": desc, "references": references, + "withdrawn": withdrawn, + "advisories": jfrogAdvisories, + "provider_url": result.get("provider_url"), + "fixVersions": fixVersions, + "publishedAt": node.get("publishedAt") }) cli_out_write(_render_vulns(vulns, vuln_html)) diff --git a/conan/cli/formatters/graph/graph_info_text.py b/conan/cli/formatters/graph/graph_info_text.py index aa431cd8573..ac04a056e60 100644 --- a/conan/cli/formatters/graph/graph_info_text.py +++ b/conan/cli/formatters/graph/graph_info_text.py @@ -1,13 +1,29 @@ import fnmatch from collections import OrderedDict +from conan.api.model import RecipeReference from conan.api.output import ConanOutput, cli_out_write def filter_graph(graph, package_filter=None, field_filter=None): if package_filter is not None: + def _matching(node, pattern): + if fnmatch.fnmatch(node["ref"] or "", pattern): + return True + if pattern == "&": # Handle the consumer pattern + if node["recipe"] == "Consumer": + return True + # How to deal with --requires=xxx --package-filter=& "consumers" + root = graph["nodes"]["0"] + if root["recipe"] == "Cli" and node is not root: + # We look if the current node is a direct dependency of the root node + node_ref = RecipeReference.loads(node["ref"]) + for dep in root["dependencies"].values(): + if dep["direct"] and node_ref == RecipeReference.loads(dep["ref"]): + return True + graph["nodes"] = {id_: n for id_, n in graph["nodes"].items() - if any(fnmatch.fnmatch(n["ref"] or "", p) for p in package_filter)} + if any(_matching(n, p) for p in package_filter)} if field_filter is not None: if "ref" not in field_filter: field_filter.append("ref") diff --git a/conan/cli/formatters/graph/info_graph_html.py b/conan/cli/formatters/graph/info_graph_html.py index a38edcf23cb..1d0160f71d0 100644 --- a/conan/cli/formatters/graph/info_graph_html.py +++ b/conan/cli/formatters/graph/info_graph_html.py @@ -87,8 +87,14 @@ global_edges = {}; let edge_counter = 0; let conflict=null; + let provide_conflict=null; + let missing_error=null; if (graph_data["error"] && graph_data["error"]["type"] == "conflict") conflict = graph_data["error"]; + else if (graph_data["error"] && graph_data["error"]["type"] == "provide_conflict") + provide_conflict = graph_data["error"]; + else if (graph_data["error"] && graph_data["error"]["type"] == "missing") + missing_error = graph_data["error"]; for (const [node_id, node] of Object.entries(graph_data["nodes"])) { if (node.context == "build" && hide_build) continue; if (node.test && hide_test) continue; @@ -128,6 +134,11 @@ color = "Black"; shape = "circle"; } + if (provide_conflict && provide_conflict.node.id == node_id){ + font.color = "white"; + color = "Black"; + shape = "circle"; + } if (search_pkgs) { let patterns = search_pkgs.split(',') .map(pattern => pattern.trim()) @@ -199,6 +210,34 @@ label: conflict.branch2.require.ref}); global_edges[edge_counter++] = conflict.branch2.require; } + if (provide_conflict) { + // The nodes are already there, we'll just add an edge to the conflict node + edges.push({id: edge_counter, + from: provide_conflict.conflicting_node.id, + to: provide_conflict.node.id, + color: {color: "Red", highlight: "Red"}, + label: provide_conflict.provided, + title: "Both nodes provide the same requirement: " + provide_conflict.provided.join(", "), + dashes: true}); + global_edges[edge_counter++] = {"provided": provide_conflict.provided}; + } + if(missing_error) { + nodes.push({ + id: "missing_node", + font: {multi: 'html', color: "white"}, + label: missing_error["require"]["ref"], + shape: "Circle", + color: {background: "Black"}, + }); + edges.push({id: edge_counter, + from: missing_error["node"]["id"], + to: "missing_node", + color: {color: "Red", highlight: "Red"}, + label: "missing", + title: "missing", + dashes: true}); + global_edges[edge_counter++] = {"missing": missing_error["error"]}; + } return {nodes: new vis.DataSet(nodes), edges: new vis.DataSet(edges)}; }; function define_legend() { diff --git a/conan/cli/formatters/report/diff.py b/conan/cli/formatters/report/diff.py index b64259d7a3a..547d1bbba0a 100644 --- a/conan/cli/formatters/report/diff.py +++ b/conan/cli/formatters/report/diff.py @@ -7,6 +7,7 @@ from conan.api.output import cli_out_write from conan.cli.formatters.report.diff_html import diff_html + def _generate_json(result): diff_text = result["diff"] src_prefix = result["src_prefix"] @@ -22,6 +23,7 @@ def _generate_json(result): ret[current_filename].append(line) return ret + def _get_filenames(line, src_prefix, dst_prefix): """ Extracts the source and destination filenames from a diff line. @@ -37,16 +39,15 @@ def _get_filenames(line, src_prefix, dst_prefix): return src_filename, dst_filename + def _render_diff(content, template, template_folder, **kwargs): from conan import __version__ template = Template(template, autoescape=True) + def _safe_filename(filename): # Calculate base64 of the filename return base64.b64encode(filename.encode(), altchars=b'-_').decode() - def _get_diff_filename(line): - return _get_filenames(line, kwargs["src_prefix"], kwargs["dst_prefix"])[0] - def _remove_prefixes(line): return line.replace(kwargs["src_prefix"][:-1], "").replace(kwargs["dst_prefix"][:-1], "") @@ -56,15 +57,80 @@ def _replace_cache_paths(line): def _replace_paths(line): return _remove_prefixes(_replace_cache_paths(line)) + def _extract_header(diff_lines): + # Header ends at the first occurrence of +++ line, + # and it can be at most 10 lines long + for i, line in enumerate(diff_lines[:10]): + if line.startswith("+++ "): + return diff_lines[:i + 1] + return diff_lines[:10] + + def _parse_header_is_deleted(header_contents): + return ("+++ /dev/null" in header_contents + or any("deleted file mode" in line for line in header_contents)) + + def _parse_header_rename_to(header_contents): + if not any("similarity index" in line for line in header_contents): + return None + for line in header_contents: + if line.startswith("rename to "): + return line[len("rename to "):] + return None + + per_folder = {"folders": {}, "files": {}} + for file in content: + header = _extract_header(content[file]) + renamed_to = _parse_header_rename_to(header) + replaced_path = _replace_paths(renamed_to or file) + replaced_file = replaced_path.replace("(old)", "").replace("(new)", "").replace("\\", "/") + bits = replaced_file.split("/")[1:] + cur = per_folder + for folder in bits[:-1]: + cur = cur["folders"].setdefault(folder, {"folders": {}, "files": {}}) + filename = bits[-1] + cur["files"][filename] = {"filename": file, # This is file so renamed use old name + "is_new": "(new)" in replaced_path, + "is_deleted": _parse_header_is_deleted(header), + "renamed_to": renamed_to, + "relative_path": replaced_path} + + def flatten_empty_folders(current_node): + for folder_data in current_node["folders"].values(): + flatten_empty_folders(folder_data) + + promoted_folders = {} + + # The list here is important to avoid modifying the dict while iterating + for folder_name, folder_data in list(current_node["folders"].items()): + if not folder_data["files"]: + for sub_folder_name, sub_folder_data in folder_data['folders'].items(): + new_key = os.path.join(folder_name, sub_folder_name) + promoted_folders[new_key] = sub_folder_data + + del current_node["folders"][folder_name] + + current_node["folders"].update(promoted_folders) + + flatten_empty_folders(per_folder) + + # Now sort each folder and file recursively + def sort_folders_and_files(node): + node["folders"] = dict(sorted(node["folders"].items())) + node["files"] = dict(sorted(node["files"].items(), key=lambda x: x[0].lower())) + for folder_data in node["folders"].values(): + sort_folders_and_files(folder_data) + sort_folders_and_files(per_folder) + return template.render(content=content, + per_folder=per_folder, base_template_path=template_folder, version=__version__, safe_filename=_safe_filename, replace_paths=_replace_paths, replace_cache_paths=_replace_cache_paths, remove_prefixes=_remove_prefixes, - get_diff_filename=_get_diff_filename, **kwargs) + def format_diff_html(result): conan_api = result["conan_api"] diff --git a/conan/cli/formatters/report/diff_html.py b/conan/cli/formatters/report/diff_html.py index b470fef1d4d..27a08057844 100644 --- a/conan/cli/formatters/report/diff_html.py +++ b/conan/cli/formatters/report/diff_html.py @@ -1,37 +1,674 @@ diff_html = r""" +{% macro render_sidebar_folder(folder, folder_info) %} + {%- for name, sub_folder_info in folder_info["folders"].items() %} + {% set folder_name = folder + "/" + name %} +
  • +
    + {{ name }} +
      + {{ render_sidebar_folder(folder_name, sub_folder_info) }} +
    +
    +
    • + {%- endfor %} + {%- for name, file_info in folder_info["files"].items() %} + {% set file_type = "renamed" if file_info["renamed_to"] else ( + "deleted" if file_info["is_deleted"] else ( + "new" if file_info["is_new"] else "old")) %} +
  • + + {% if file_info["renamed_to"] %} + {{ file_info["renamed_to"].split("/")[1:][-1] }} + {% else %} + {{ name }} + {% endif %} + +
    • + {%- endfor %} +{% endmacro %} + +{% macro render_diff_folder(folder_info) %} + {%- for name, sub_folder_info in folder_info["folders"].items() %} + {{ render_diff_folder(sub_folder_info) }} + {%- endfor %} + {%- for name, file_info in folder_info["files"].items() %} + {% set filename = file_info["filename"] %} + +
    +
    +
    + + + {{ replace_cache_paths(filename) | replace("(old)/", "") | replace("(new)/", "") }} + {% if file_info["renamed_to"] %} +  →  + {{ replace_cache_paths(file_info["renamed_to"]) | replace("(old)/", "") | replace("(new)/", "") }} + {% endif %} + +
    +     +
    +
    +
    +
    +
    + {%- endfor %} +{% endmacro %} - {{ old_reference }} - {{ new_reference }} + Diff report for {{ old_reference }} - {{ new_reference }}
    -
    - --- (old): {{ old_reference.repr_notime() }} -
    - +++ (new): {{ new_reference.repr_notime() }} + -

    File list:

    - - - - + +
    +
    +
    -
    - {%- for filename, lines in content.items() -%} -
    -
    - {%- for line in lines -%} - {%- if loop.first -%} -
    -

    {{ remove_prefixes(line) }}

    - {%- elif line.startswith('+++') %} - {{ replace_paths(line) }} -
    - {%- elif line.startswith('---') %} - {{ replace_paths(line) }} -
    - {%- elif line.startswith('+') %} - {{ line }} -
    - {%- elif line.startswith('-') %} - {{ line }} -
    - {%- else %} - {{ line }} -
    - {%- endif %} - {%- endfor -%} - {%- endfor -%} -
    +
    +

    Diff Report Between {{ old_reference.repr_notime() }} And {{ new_reference.repr_notime() }}

    + + {{ render_diff_folder(per_folder) }}
    diff --git a/conan/cps/cps.py b/conan/cps/cps.py index 6e04b7b8bbb..e3692293d31 100644 --- a/conan/cps/cps.py +++ b/conan/cps/cps.py @@ -38,14 +38,18 @@ def __init__(self, component_type=None): self.type = component_type or "unknown" self.definitions = [] self.requires = [] + self.link_requires = [] self.location = None self.link_location = None - self.link_libraries = None # system libraries + self.link_languages = [] + self.link_libraries = [] # system libraries def serialize(self): component = {"type": str(self.type)} if self.requires: component["requires"] = self.requires + if self.link_requires: + component["link_requires"] = self.link_requires if self.includes: component["includes"] = [x.replace("\\", "/") for x in self.includes] if self.definitions: @@ -56,18 +60,22 @@ def serialize(self): component["link_location"] = self.link_location if self.link_libraries: component["link_libraries"] = self.link_libraries + if self.link_languages: + component["link_languages"] = self.link_languages return component @staticmethod def deserialize(data): comp = CPSComponent() comp.type = CPSComponentType(data.get("type")) - comp.requires = data.get("requires") - comp.includes = data.get("includes") - comp.definitions = data.get("definitions") + comp.requires = data.get("requires", []) + comp.link_requires = data.get("link_requires", []) + comp.includes = data.get("includes", []) + comp.definitions = data.get("definitions", []) comp.location = data.get("location") comp.link_location = data.get("link_location") - comp.link_libraries = data.get("link_libraries") + comp.link_libraries = data.get("link_libraries", []) + comp.link_languages = data.get("link_languages", []) return comp @staticmethod @@ -90,12 +98,15 @@ def from_cpp_info(cpp_info, conanfile, libname=None): cps_comp.location = cpp_info.location cps_comp.link_location = cpp_info.link_location cps_comp.link_libraries = cpp_info.system_libs + langs = {"C": "c", "C++": "cpp"} + cps_comp.link_languages = [langs[lang] for lang in cpp_info.languages or []] required = cpp_info.requires cps_comp.requires = [f":{c}" if "::" not in c else c.replace("::", ":") for c in required] return cps_comp def update(self, conf, conf_def): # TODO: conf not used at the moent + self.link_languages = self.link_languages or conf_def.get("link_languages") self.location = self.location or conf_def.get("location") self.link_location = self.link_location or conf_def.get("link_location") self.link_libraries = self.link_libraries or conf_def.get("link_libraries") @@ -208,37 +219,48 @@ def to_conan(self): def strip_prefix(dirs): return [d.replace("@prefix@/", "") for d in dirs] + def lib_location(loc, info): + loc = loc.replace("@prefix@/", "") + info.libdirs = [os.path.dirname(loc)] + filename = os.path.basename(loc) + basefile, ext = os.path.splitext(filename) + if basefile.startswith("lib") and ext != ".lib": + basefile = basefile[3:] + info.libs = [basefile] + cpp_info = CppInfo() if len(self.components) == 1: comp = next(iter(self.components.values())) cpp_info.includedirs = strip_prefix(comp.includes) cpp_info.defines = comp.definitions - if comp.type is CPSComponentType.ARCHIVE: + cpp_info.system_libs = comp.link_libraries + # requires for 1 component packages are automatic in Conan with self.requires() + if comp.link_location: + link_location = comp.link_location + lib_location(link_location, cpp_info) location = comp.location location = location.replace("@prefix@/", "") - cpp_info.libdirs = [os.path.dirname(location)] - filename = os.path.basename(location) - basefile, ext = os.path.splitext(filename) - if basefile.startswith("lib") and ext != ".lib": - basefile = basefile[3:] - cpp_info.libs = [basefile] - # FIXME: Missing requires - cpp_info.system_libs = comp.link_libraries + cpp_info.bindirs = [os.path.dirname(location)] + elif comp.location: + location = comp.location + lib_location(location, cpp_info) else: for comp_name, comp in self.components.items(): cpp_comp = cpp_info.components[comp_name] cpp_comp.includedirs = strip_prefix(comp.includes) cpp_comp.defines = comp.definitions - if comp.type is CPSComponentType.ARCHIVE: + if comp.link_location: + link_location = comp.link_location + lib_location(link_location, cpp_comp) location = comp.location location = location.replace("@prefix@/", "") - cpp_comp.libdirs = [os.path.dirname(location)] - filename = os.path.basename(location) - basefile, ext = os.path.splitext(filename) - if basefile.startswith("lib") and ext != ".lib": - basefile = basefile[3:] - cpp_comp.libs = [basefile] - for r in comp.requires: + cpp_comp.bindirs = [os.path.dirname(location)] + elif comp.location: + location = comp.location + lib_location(location, cpp_comp) + # all requires are the same for Conan, it uses its traits information + requires = comp.link_requires + comp.requires + for r in requires: cpp_comp.requires.append(r[1:] if r.startswith(":") else r.replace(":", "::")) cpp_comp.system_libs = comp.link_libraries diff --git a/conan/internal/api/audit/providers.py b/conan/internal/api/audit/providers.py index ab6ed523d93..ebf5017718e 100644 --- a/conan/internal/api/audit/providers.py +++ b/conan/internal/api/audit/providers.py @@ -16,7 +16,7 @@ def __init__(self, conan_api, name, provider_data): self.url = provider_data["url"] self.type = provider_data["type"] self._token = provider_data.get("token") - self._session = conan_api.remotes.requester + self._session = conan_api._api_helpers.requester # noqa self._query_url = urljoin(self.url, "api/v1/query") def get_cves(self, refs): @@ -40,7 +40,7 @@ def get_cves(self, refs): raise ConanException("Missing authentication token. Please authenticate and retry.") - result = {"data": {}, "error": None} + result = {"data": {}, "error": None, "provider_url": None} for ref in refs: ConanOutput().info(f"Requesting vulnerability info for: {ref}") @@ -111,7 +111,7 @@ def __init__(self, conan_api, name, provider_data): self.url = provider_data["url"] self.type = provider_data["type"] self._token = provider_data.get("token") - self._session = conan_api.remotes.requester + self._session = conan_api._api_helpers.requester # noqa self._query_url = urljoin(self.url, "catalog/api/v0/public/graphql") def get_cves(self, refs): @@ -119,7 +119,7 @@ def get_cves(self, refs): raise ConanException(f"Missing authentication token for '{self.name}' provider.\n" f"Please authenticate using 'conan audit provider auth' and retry.") - result = {"data": {}, "error": None} + result = {"data": {}, "error": None, "provider_url": self.url} for ref in refs: try: @@ -127,7 +127,7 @@ def get_cves(self, refs): if "error" in response: result["data"][str(ref)] = {"error": {"details": response["error"]}} else: - result["data"][str(ref)] = response.get("data",{}).get("query",{}) + result["data"][str(ref)] = response.get("data", {}).get("query", {}) except Exception as e: result["conan_error"] = str(e) break @@ -150,17 +150,31 @@ def _build_query(ref): preferredBaseScore }} aliases + withdrawn + publishedAt advisories {{ name ...on JfrogAdvisory {{ name shortDescription - fullDescription - url severity - }} + impactReasons {{ + name + isPositive + }} }} + }} references + vulnerablePackages(first: 100) {{ + totalCount + edges {{ + node {{ + fixVersions {{ + version + }} + }} + }} + }} }} }} }} diff --git a/conan/internal/api/config/config_installer.py b/conan/internal/api/config/config_installer.py index e411897c941..359673ecb0f 100644 --- a/conan/internal/api/config/config_installer.py +++ b/conan/internal/api/config/config_installer.py @@ -19,6 +19,8 @@ class _ConanIgnoreMatcher: def __init__(self, conanignore_path, ignore=None): self._ignored_entries = {".conanignore"} self._included_entries = set() + if ignore: + self._ignored_entries.update(ignore) if conanignore_path is None or not os.path.exists(conanignore_path): return with open(conanignore_path, 'r') as conanignore: @@ -29,8 +31,6 @@ def __init__(self, conanignore_path, ignore=None): self._included_entries.add(line_content[1:]) else: self._ignored_entries.add(line_content) - if ignore: - self._ignored_entries.update(ignore) def matches(self, path): """Returns whether the path should be ignored @@ -143,6 +143,7 @@ def _process_file(directory, filename, config, cache_folder, folder): output.info("Copying file %s to %s" % (filename, target_folder)) _filecopy(directory, filename, target_folder) + def _process_folder(config, folder, cache_folder, ignore=None): if not os.path.isdir(folder): raise ConanException("No such directory: '%s'" % str(folder)) @@ -176,7 +177,7 @@ def _process_download(config, cache_folder, requester): raise ConanException("Error while installing config from %s\n%s" % (config.uri, str(e))) -class _ConfigOrigin(object): +class _ConfigOrigin: def __init__(self, uri, config_type, verify_ssl, args, source_folder, target_folder): if config_type: self.type = config_type diff --git a/conan/internal/api/detect/detect_api.py b/conan/internal/api/detect/detect_api.py index aefa2967a71..bee789fdbbd 100644 --- a/conan/internal/api/detect/detect_api.py +++ b/conan/internal/api/detect/detect_api.py @@ -462,14 +462,14 @@ def detect_default_compiler(): def default_msvc_ide_version(version): - version = {"194": "17", "193": "17", "192": "16", "191": "15"}.get(str(version)) + version = {"195": "18", "194": "17", "193": "17", "192": "16", "191": "15"}.get(str(version)) if version: return Version(version) def _detect_vs_ide_version(): from conan.internal.api.detect.detect_vs import vs_installation_path - msvc_versions = "17", "16", "15" + msvc_versions = "18", "17", "16", "15" for version in msvc_versions: vs_path = os.getenv('vs%s0comntools' % version) path = vs_path or vs_installation_path(version) @@ -488,7 +488,8 @@ def _cc_compiler(compiler_exe="cc"): compiler = "clang" if "clang" in out else "gcc" # clang and gcc have version after a space, first try to find that to skip extra numbers # that might appear in the first line of the output before the version - # There might also be a leading parenthesis that contains build information, so we try to skip it + # There might also be a leading parenthesis that contains build information, + # so we try to skip it installed_version = re.search(r"(?:\(.*\))? ([0-9]+(\.[0-9]+)*)", out) # Fallback to the first number we find optionally followed by other version fields installed_version = installed_version or re.search(r"([0-9]+(\.[0-9]+)*)", out) @@ -532,7 +533,7 @@ def detect_intel_compiler(compiler_exe="icx"): try: ret, out = detect_runner(f'"{compiler_exe}" --version') if ret != 0: - return None, None + return None, None, None compiler = "intel-cc" installed_version = re.search(r"(202[0-9]+(\.[0-9])?)", out).group() if installed_version: @@ -579,7 +580,8 @@ def detect_clang_compiler(compiler_exe="clang"): def detect_msvc_compiler(): ide_version = _detect_vs_ide_version() - version = {"17": "193", "16": "192", "15": "191"}.get(str(ide_version)) # Map to compiler + # Map to compiler + version = {"18": "195", "17": "193", "16": "192", "15": "191"}.get(str(ide_version)) if ide_version == "17": update = detect_msvc_update(version) # FIXME weird passing here the 193 compiler version if update and int(update) >= 10: diff --git a/conan/internal/api/detect/detect_vs.py b/conan/internal/api/detect/detect_vs.py index 50b66b793ac..b9659b9d234 100644 --- a/conan/internal/api/detect/detect_vs.py +++ b/conan/internal/api/detect/detect_vs.py @@ -11,12 +11,13 @@ def vs_installation_path(version): def vs_detect_update(version): - version = {"194": "17", "193": "17", "192": "16", "191": "15"}.get(str(version)) + version = {"195": "18", "194": "17", "193": "17", "192": "16", "191": "15"}.get(str(version)) full_version = _vs_installation_path(version)[1] components = full_version.split(".") if len(components) > 1: return components[1] + def _vs_installation_path(version): # TODO: Preference hardcoded, [conf] must be defined preference = ["Enterprise", "Professional", "Community", "BuildTools"] diff --git a/conan/internal/api/export.py b/conan/internal/api/export.py index 04fbbebcc40..3878e4671bd 100644 --- a/conan/internal/api/export.py +++ b/conan/internal/api/export.py @@ -12,13 +12,13 @@ from conan.internal.util.files import is_dirty, rmdir, set_dirty, mkdir, clean_dirty, chdir -def cmd_export(app, hook_manager, global_conf, conanfile_path, name, version, user, channel, +def cmd_export(loader, cache, hook_manager, global_conf, conanfile_path, + name, version, user, channel, graph_lock=None, remotes=None): """ Export the recipe param conanfile_path: the original source directory of the user containing a conanfile.py """ - loader, cache = app.loader, app.cache conanfile = loader.load_export(conanfile_path, name, version, user, channel, graph_lock, remotes=remotes) @@ -30,6 +30,24 @@ def cmd_export(app, hook_manager, global_conf, conanfile_path, name, version, us conanfile.display_name = str(ref) conanfile.output.scope = conanfile.display_name scoped_output = conanfile.output + # Even though the package_id_non_embed_mode is minor_mode by default, + # and package_id_unknown_mode is semver_mode by default, + # recipes with buggy versions that do not define the attribute will have + # the same problem regardless + if not isinstance(ref.version.major.value, int) and ref.version.minor is not None: + modes = [getattr(conanfile, f"package_id_{m}_mode", None) + for m in ("embed", "non_embed", "unknown")] + if (any(m in ("semver", "major", "minor", "patch") for m in modes) or + all(m is None for m in modes)): + msg = (f"Version '{ref.version}' contains an alphanumeric major alongside a minor " + f"version, without correct 'package_id_xxx_mode' attributes.\n" + f"This is highly discouraged due to unexpected package ID calculation " + f"risks. Either a different version scheme should be used " + f"(e.g., semantic versioning), or the 'package_id_xxx_mode' attributes " + f"should be set (to something other than major, minor, patch or semver modes).\n" + f"Refer to the documentation for more details: " + f"https://docs.conan.io/2/knowledge/guidelines.html#guidelines-bad-alphanumeric-majors") + scoped_output.warning(msg, warn_tag="risk") recipe_layout = cache.create_export_recipe_layout(ref) @@ -110,7 +128,9 @@ def _calc_revision(scoped_output, path, manifest, revision_mode, conanfile): if git.is_dirty(): raise ConanException("Can't have a dirty repository using revision_mode='scm' and doing" " 'conan export', please commit the changes and run again, or " - "use 'git_excluded = []' attribute") + "use 'revision_mode_excluded' recipe attribute or " + "'core.scm:excluded' global configuration to define the list of " + "excluded file patterns") scoped_output.info("Using git commit as the recipe revision: %s" % revision) diff --git a/conan/internal/api/install/generators.py b/conan/internal/api/install/generators.py index 0ee676a8133..87865d8f9fd 100644 --- a/conan/internal/api/install/generators.py +++ b/conan/internal/api/install/generators.py @@ -182,6 +182,11 @@ def deactivates(filenames): result.append(os.path.join(folder, "deactivate_{}".format(f))) return result + def deactivate_function_names(filenames): + return [os.path.splitext(os.path.basename(s))[0].replace("-", "_") + for s in reversed(filenames)] + + deactivation_mode = conanfile.conf.get("tools.env:deactivation_mode", default=None, check_type=str) generated = [] for group, env_scripts in conanfile.env_scripts.items(): subsystem = deduce_subsystem(conanfile, group) @@ -202,12 +207,19 @@ def deactivates(filenames): ps1s.append("$PSScriptRoot/"+path) if shs: def sh_content(files): - return ". " + " && . ".join('"{}"'.format(s) for s in files) + content = ". " + " && . ".join('"{}"'.format(s) for s in files) + if deactivation_mode == "function": + content += f"\n\ndeactivate_conan{group}() {{\n" + for deactivate_name in deactivate_function_names(shs): + content += f" deactivate_{deactivate_name}\n" + content += f" unset -f deactivate_conan{group}\n}}\n" + return content filename = "conan{}.sh".format(group) generated.append(filename) save(os.path.join(conanfile.generators_folder, filename), sh_content(shs)) - save(os.path.join(conanfile.generators_folder, "deactivate_{}".format(filename)), - sh_content(deactivates(shs))) + if not deactivation_mode: + save(os.path.join(conanfile.generators_folder, "deactivate_{}".format(filename)), + sh_content(deactivates(shs))) if bats: def bat_content(files): return "\r\n".join(["@echo off"] + ['call "{}"'.format(b) for b in files]) @@ -218,12 +230,21 @@ def bat_content(files): bat_content(deactivates(bats))) if ps1s: def ps1_content(files): - return "\r\n".join(['& "{}"'.format(b) for b in files]) + content = "\r\n".join(['& "{}"'.format(b) for b in files]) + if deactivation_mode == "function": + content += f"\n\nfunction global:deactivate_conan{group} {{\n" + for deactivate_name in deactivate_function_names(ps1s): + content += f" deactivate_{deactivate_name}\n" + content += (f" Remove-Item -Path function:deactivate_conan{group} " + "-ErrorAction SilentlyContinue" + "\n}\n") + return content filename = "conan{}.ps1".format(group) generated.append(filename) save(os.path.join(conanfile.generators_folder, filename), ps1_content(ps1s)) - save(os.path.join(conanfile.generators_folder, "deactivate_{}".format(filename)), - ps1_content(deactivates(ps1s))) + if not deactivation_mode: + save(os.path.join(conanfile.generators_folder, "deactivate_{}".format(filename)), + ps1_content(deactivates(ps1s))) if generated: conanfile.output.highlight("Generating aggregated env files") conanfile.output.info(f"Generated aggregated env files: {generated}") diff --git a/conan/internal/api/list/query_parse.py b/conan/internal/api/list/query_parse.py index 8e33e078d56..e3633b27b16 100644 --- a/conan/internal/api/list/query_parse.py +++ b/conan/internal/api/list/query_parse.py @@ -130,14 +130,14 @@ def _infix_to_postfix(exp): stack = [] i = -1 - while(i < len(exp) - 1): + while i < len(exp) - 1: i += 1 char = exp[i] if char == " ": # Ignore spaces between expressions and operators continue if char == ")": # Pop the stack until "(" and send them to output popped = None - while(popped != "(" and stack): + while popped != "(" and stack: popped = stack.pop() if popped != "(": output.append(popped) diff --git a/conan/internal/api/new/basic.py b/conan/internal/api/new/basic.py index e726e83f606..6ff7b19fb8d 100644 --- a/conan/internal/api/new/basic.py +++ b/conan/internal/api/new/basic.py @@ -1,6 +1,7 @@ def inject_get_or_else(variable, default): return variable + ' = "{% if ' + variable + " is defined %}{{ " + variable + " }}{% else %}" + default + '{% endif %}"' + _conanfile_header = f'''\ {inject_get_or_else("name", "pkg")} {inject_get_or_else("version", "1.0")} diff --git a/conan/internal/api/new/bazel_7_exe.py b/conan/internal/api/new/bazel_7_exe.py index f949088ca48..6418b01bbd0 100644 --- a/conan/internal/api/new/bazel_7_exe.py +++ b/conan/internal/api/new/bazel_7_exe.py @@ -70,4 +70,4 @@ def test(self): "MODULE.bazel": _bazel_workspace, ".bazelrc": _bazel_rc, "test_package/conanfile.py": test_conanfile_exe_v2 - } + } diff --git a/conan/internal/api/new/workspace.py b/conan/internal/api/new/workspace.py index 505751af839..0a400283495 100644 --- a/conan/internal/api/new/workspace.py +++ b/conan/internal/api/new/workspace.py @@ -19,28 +19,39 @@ include(FetchContent) -function(add_project SUBFOLDER) - message(STATUS "Adding project ${SUBFOLDER}") +function(add_project PACKAGE_NAME SUBFOLDER) + message(STATUS "Adding project: ${PACKAGE_NAME}. Folder: ${SUBFOLDER}") FetchContent_Declare( - ${SUBFOLDER} + ${PACKAGE_NAME} SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/${SUBFOLDER} SYSTEM OVERRIDE_FIND_PACKAGE ) - FetchContent_MakeAvailable(${SUBFOLDER}) + FetchContent_MakeAvailable(${PACKAGE_NAME}) endfunction() -add_project(liba) -# They should be defined in the liba/CMakeLists.txt, but we can fix it here -add_library(liba::liba ALIAS liba) -add_project(libb) -add_library(libb::libb ALIAS libb) -add_project(app1) +include(build/conanws_build_order.cmake) + +foreach(pair ${CONAN_WS_BUILD_ORDER}) + string(FIND "${pair}" ":" pos) + string(SUBSTRING "${pair}" 0 "${pos}" pkg) + math(EXPR pos "${pos} + 1") # Skip the separator + string(SUBSTRING "${pair}" "${pos}" -1 folder) + + add_project(${pkg} ${folder}) + # This target should be defined in the liba/CMakeLists.txt, but we can fix it here + get_target_property(target_type ${pkg} TYPE) + if (NOT target_type STREQUAL "EXECUTABLE") + add_library(${pkg}::${pkg} ALIAS ${pkg}) + endif() +endforeach() """ -conanfile = '''\ +conanfile = r'''\ +import json from conan import Workspace from conan import ConanFile +from conan.tools.files import save from conan.tools.cmake import CMakeDeps, CMakeToolchain, cmake_layout @@ -64,6 +75,11 @@ def layout(self): class Ws(Workspace): def root_conanfile(self): return MyWs + + def build_order(self, order): + super().build_order(order) # default behavior prints the build order + pkglist = " ".join([f'{it["ref"].name}:{it["folder"]}' for level in order for it in level]) + save(self, "build/conanws_build_order.cmake", f"set(CONAN_WS_BUILD_ORDER {pkglist})") ''' workspace_files = {"conanws.yml": conanws_yml, diff --git a/conan/internal/api/profile/profile_loader.py b/conan/internal/api/profile/profile_loader.py index 097092b760c..61342c293e2 100644 --- a/conan/internal/api/profile/profile_loader.py +++ b/conan/internal/api/profile/profile_loader.py @@ -99,10 +99,10 @@ def from_cli_args(self, profiles, settings, options, conf, cwd, context=None): def load_profile(self, profile_name, cwd=None, context=None): # TODO: This can be made private, only used in testing now cwd = cwd or os.getcwd() - profile = self._load_profile(profile_name, cwd, context) + profile = self._load_profile(profile_name, cwd, context, root_profile_name=profile_name) return profile - def _load_profile(self, profile_name, cwd, context): + def _load_profile(self, profile_name, cwd, context, root_profile_name): """ Will look for "profile_name" in disk if profile_name is absolute path, in current folder if path is relative or in the default folder otherwise. return: a Profile object @@ -122,6 +122,7 @@ def _load_profile(self, profile_name, cwd, context): "subprocess": subprocess, "profile_dir": base_path, "profile_name": file_path, + "root_profile_name": root_profile_name, "conan_version": conan_version, "detect_api": detect_api, "context": context} @@ -138,11 +139,11 @@ def _load_profile(self, profile_name, cwd, context): f"Check your Jinja2 syntax: {str(e)}") try: - return self._recurse_load_profile(text, profile_path, context) + return self._recurse_load_profile(text, profile_path, context, profile_name) except ConanException as exc: raise ConanException("Error reading '%s' profile: %s" % (profile_name, exc)) - def _recurse_load_profile(self, text, profile_path, context): + def _recurse_load_profile(self, text, profile_path, context, root_profile_name): """ Parse and return a Profile object from a text config like representation. cwd is needed to be able to load the includes """ @@ -154,7 +155,7 @@ def _recurse_load_profile(self, text, profile_path, context): # from parent profiles for include in profile_parser.includes: # Recursion !! - profile = self._load_profile(include, cwd, context) + profile = self._load_profile(include, cwd, context, root_profile_name) inherited_profile.compose_profile(profile) # Current profile before update with parents (but parent variables already applied) @@ -220,7 +221,7 @@ def __init__(self, text): raise ConanException("Error while parsing line %i: '%s'" % (counter, line)) -class _ProfileValueParser(object): +class _ProfileValueParser: """ parses a "pure" or "effective" profile, with no includes, no variables, as the one in the lockfiles, or once these things have been processed by ProfileParser """ @@ -332,6 +333,11 @@ def _parse_tool_requires(doc): pattern, req_list = "*", br_line else: pattern, req_list = tokens + if "[" in pattern: + ConanOutput().warning( + f"Tool requires pattern {pattern} contains a version range, which has no effect. " + f"Only '&' for consumer and '*' as wildcard are supported in this context.", + warn_tag="risk") refs = [RecipeReference.loads(r.strip()) for r in req_list.split(",")] result.setdefault(pattern, []).extend(refs) return result @@ -344,10 +350,16 @@ def get_package_name_value(item): if ":" in item: tmp = item.split(":", 1) packagename, item = tmp + if "[" in packagename: + ConanOutput().warning(f"Settings pattern {packagename} contains a version range, which has no effect. " + f"Only '&' for consumer and '*' as wildcard are supported in this context.", + warn_tag="risk") result_name, result_value = item.split("=", 1) result_name = result_name.strip() result_value = _unquote(result_value) + if result_value == "~": # Can be used to undefine an already defined setting + result_value = None return packagename, result_name, result_value package_settings = OrderedDict() @@ -388,10 +400,17 @@ def _get_simple_and_package_tuples(items): package_items = defaultdict(list) tuples = _get_tuples_list_from_extender_arg(items) for name, value in tuples: + if value == "~": # Can be used to undefine an already defined setting + value = None if ":" in name: # Scoped items tmp = name.split(":", 1) ref_name = tmp[0] name = tmp[1] + if "[" in ref_name: + ConanOutput().warning( + f"Pattern {ref_name} contains a version range, which has no effect. " + f"Only '&' for consumer and '*' as wildcard are supported in this context.", + warn_tag="risk") package_items[ref_name].append((name, value)) else: simple_items.append((name, value)) diff --git a/conan/internal/api/upload.py b/conan/internal/api/upload.py index 0093c083a12..c127204cb1d 100644 --- a/conan/internal/api/upload.py +++ b/conan/internal/api/upload.py @@ -4,13 +4,15 @@ def add_urls(package_list, remote): router = ClientV2Router(remote.url.rstrip("/")) - for ref, bundle in package_list.refs().items(): - for f, fp in bundle.get("files", {}).items(): - bundle.setdefault("upload-urls", {})[f] = { + for ref, packages in package_list.items(): + ref_info = package_list.recipe_dict(ref) + for f, fp in ref_info.get("files", {}).items(): + ref_info.setdefault("upload-urls", {})[f] = { 'url': router.recipe_file(ref, f), 'checksum': sha1sum(fp) } - for pref, prev_bundle in package_list.prefs(ref, bundle).items(): - for f, fp in prev_bundle.get("files", {}).items(): - prev_bundle.setdefault("upload-urls", {})[f] = { + for pref in packages: + pref_info = package_list.package_dict(pref) + for f, fp in pref_info.get("files", {}).items(): + pref_info.setdefault("upload-urls", {})[f] = { 'url': router.package_file(pref, f), 'checksum': sha1sum(fp) } diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index ee0ea15f769..3c93efc1b93 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -2,6 +2,7 @@ import gzip import os import shutil +import sys import tarfile import time @@ -10,9 +11,9 @@ from conan.internal.source import retrieve_exports_sources from conan.internal.errors import NotFoundException from conan.errors import ConanException -from conan.internal.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME, - EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO) -from conan.internal.util.files import (COMPRESSED_PLUGIN_TAR_NAME, clean_dirty, is_dirty, gather_files, remove, +from conan.internal.paths import CONAN_MANIFEST, CONANFILE, CONANINFO, COMPRESSIONS, \ + EXPORT_SOURCES_FILE_NAME, EXPORT_FILE_NAME, PACKAGE_FILE_NAME +from conan.internal.util.files import (clean_dirty, is_dirty, gather_files, set_dirty_context_manager, mkdir, human_size) UPLOAD_POLICY_FORCE = "force-upload" @@ -28,11 +29,13 @@ class UploadUpstreamChecker: def __init__(self, app: ConanApp): self._app = app - def check(self, upload_bundle, remote, force): - for ref, recipe_bundle in upload_bundle.refs().items(): - self._check_upstream_recipe(ref, recipe_bundle, remote, force) - for pref, prev_bundle in upload_bundle.prefs(ref, recipe_bundle).items(): - self._check_upstream_package(pref, prev_bundle, remote, force) + def check(self, package_list, remote, force): + for ref, packages in package_list.items(): + recipe_info = package_list.recipe_dict(ref) + self._check_upstream_recipe(ref, recipe_info, remote, force) + for pref in packages: + pkg_dict = package_list.package_dict(pref) + self._check_upstream_package(pref, pkg_dict, remote, force) def _check_upstream_recipe(self, ref, ref_bundle, remote, force): output = ConanOutput(scope=str(ref)) @@ -40,7 +43,7 @@ def _check_upstream_recipe(self, ref, ref_bundle, remote, force): try: assert ref.revision # TODO: It is a bit ugly, interface-wise to ask for revisions to check existence - server_ref = self._app.remote_manager.get_recipe_revision_reference(ref, remote) + server_ref = self._app.remote_manager.get_recipe_revision(ref, remote) assert server_ref # If successful (not raising NotFoundException), this will exist except NotFoundException: ref_bundle["force_upload"] = False @@ -61,7 +64,7 @@ def _check_upstream_package(self, pref, prev_bundle, remote, force): try: # TODO: It is a bit ugly, interface-wise to ask for revisions to check existence - server_revisions = self._app.remote_manager.get_package_revision_reference(pref, remote) + server_revisions = self._app.remote_manager.get_package_revision(pref, remote) assert server_revisions except NotFoundException: prev_bundle["force_upload"] = False @@ -78,14 +81,41 @@ def _check_upstream_package(self, pref, prev_bundle, remote, force): prev_bundle["upload"] = False +def get_compress_level(compressformat, global_conf): + if compressformat == "xz": + msg = ("The 'xz' compression is experimental. " + "Consumers using older Conan versions will not be able to install these packages. " + "Feedback is welcome, please report any issues as GitHub tickets.") + ConanOutput().warning(msg, warn_tag="experimental") + elif compressformat == "zst": + msg = ("The 'zst' compression is experimental. " + "Consumers installing packages created with this format must use Python >= 3.14. " + "Consumers using older Conan or Python versions will not be able to install these " + "packages. Feedback is welcome, please report any issues as GitHub tickets.") + ConanOutput().warning(msg, warn_tag="experimental") + + if compressformat == "zst" and sys.version_info.minor < 14: + raise ConanException("The 'core.upload:compression_format=zst' is only for Python>=3.14") + compresslevel = global_conf.get("core:compresslevel", check_type=int) + if compresslevel is None and compressformat == "gz": + compresslevel = global_conf.get("core.gzip:compresslevel", check_type=int) + # do not deprecate yet core.gzip:compresslevel, wait a bit to stabilize core:compresslevel + return compresslevel + + class PackagePreparator: def __init__(self, app: ConanApp, global_conf): self._app = app self._global_conf = global_conf + compressformat = self._global_conf.get("core.upload:compression_format", default="gz", + choices=COMPRESSIONS) + compresslevel = get_compress_level(compressformat, global_conf) + self._compressformat = compressformat + self._compresslevel = compresslevel def prepare(self, pkg_list, enabled_remotes): local_url = self._global_conf.get("core.scm:local_url", choices=["allow", "block"]) - for ref, bundle in pkg_list.refs().items(): + for ref, packages in pkg_list.items(): layout = self._app.cache.recipe_layout(ref) conanfile_path = layout.conanfile() conanfile = self._app.loader.load_basic(conanfile_path) @@ -96,9 +126,16 @@ def prepare(self, pkg_list, enabled_remotes): "This isn't a remote URL, the build won't be reproducible\n" "Failing because conf 'core.scm:local_url!=allow'") + # Just in case it was defined from a previous run + bundle = pkg_list.recipe_dict(ref) + bundle.pop("files", None) + bundle.pop("upload-urls", None) if bundle.get("upload"): self._prepare_recipe(ref, bundle, conanfile, enabled_remotes) - for pref, prev_bundle in pkg_list.prefs(ref, bundle).items(): + for pref in packages: + prev_bundle = pkg_list.package_dict(pref) + prev_bundle.pop("files", None) # If defined from a previous upload + prev_bundle.pop("upload-urls", None) if prev_bundle.get("upload"): self._prepare_package(pref, prev_bundle) @@ -119,14 +156,6 @@ def _prepare_recipe(self, ref, ref_bundle, conanfile, remotes): def _compress_recipe_files(self, layout, ref): download_export_folder = layout.download_export() - output = ConanOutput(scope=str(ref)) - for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME): - tgz_path = os.path.join(download_export_folder, f) - if is_dirty(tgz_path): - output.warning("Removing %s, marked as dirty" % f) - os.remove(tgz_path) - clean_dirty(tgz_path) - export_folder = layout.export() files, symlinked_folders = gather_files(export_folder) files.update(symlinked_folders) @@ -150,18 +179,13 @@ def _compress_recipe_files(self, layout, ref): files.pop(CONANFILE) files.pop(CONAN_MANIFEST) - def add_tgz(tgz_name, tgz_files): - tgz = os.path.join(download_export_folder, tgz_name) - if os.path.isfile(tgz): - result[tgz_name] = tgz - elif tgz_files: - tgz = compress_files(tgz_files, tgz_name, download_export_folder, - conf=self._global_conf, ref=ref, - compression_plugin=self._app.conan_api.cache.compression_plugin) - result[tgz_name] = tgz - - add_tgz(EXPORT_TGZ_NAME, files) - add_tgz(EXPORT_SOURCES_TGZ_NAME, src_files) + if files: + comp = self._compressed_file(EXPORT_FILE_NAME, files, download_export_folder, ref) + result[comp] = os.path.join(download_export_folder, comp) + if src_files: + comp = self._compressed_file(EXPORT_SOURCES_FILE_NAME, src_files, + download_export_folder, ref) + result[comp] = os.path.join(download_export_folder, comp) return result def _prepare_package(self, pref, prev_bundle): @@ -172,14 +196,40 @@ def _prepare_package(self, pref, prev_bundle): cache_files = self._compress_package_files(pkg_layout, pref) prev_bundle["files"] = cache_files + def _compressed_file(self, filename, files, download_folder, ref): + output = ConanOutput(scope=str(ref)) + + # Check if there is some existing compressed file already + matches = [] + for extension in COMPRESSIONS: + file_name = filename + extension + package_file = os.path.join(download_folder, file_name) + if is_dirty(package_file): + output.warning(f"Removing {file_name}, marked as dirty") + os.remove(package_file) + clean_dirty(package_file) + if os.path.isfile(package_file): + matches.append(file_name) + if len(matches) > 1: + raise ConanException(f"{ref}: Multiple package files found for {filename}: {matches}") + if len(matches) == 1: + existing = matches[0] + if not existing.endswith(self._compressformat): + output.info(f"Existing {existing} compressed file, " + f"keeping it, not using '{self._compressformat}' format") + return existing + + file_name = filename + self._compressformat + package_file = os.path.join(download_folder, file_name) + compression_plugin = self._app.conan_api.cache.compression_plugin + compressed_path = compress_files(files, file_name, download_folder, + compresslevel=self._compresslevel, ref=ref, plugin=compression_plugin) + assert compressed_path == package_file + assert os.path.exists(package_file) + return file_name + def _compress_package_files(self, layout, pref): - output = ConanOutput(scope=str(pref)) download_pkg_folder = layout.download_package() - package_tgz = os.path.join(download_pkg_folder, PACKAGE_TGZ_NAME) - if is_dirty(package_tgz): - output.warning("Removing %s, marked as dirty" % PACKAGE_TGZ_NAME) - os.remove(package_tgz) - clean_dirty(package_tgz) # Get all the files in that directory # existing package @@ -200,15 +250,8 @@ def _compress_package_files(self, layout, pref): files.pop(CONANINFO) files.pop(CONAN_MANIFEST) - if not os.path.isfile(package_tgz): - tgz_files = {f: path for f, path in files.items()} - tgz_path = compress_files(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, - conf=self._global_conf, ref=pref, - compression_plugin=self._app.conan_api.cache.compression_plugin) - assert tgz_path == package_tgz - assert os.path.exists(package_tgz) - - return {PACKAGE_TGZ_NAME: package_tgz, + compressed_file = self._compressed_file(PACKAGE_FILE_NAME, files, download_pkg_folder, pref) + return {compressed_file: os.path.join(download_pkg_folder, compressed_file), CONANINFO: os.path.join(download_pkg_folder, CONANINFO), CONAN_MANIFEST: os.path.join(download_pkg_folder, CONAN_MANIFEST)} @@ -222,10 +265,12 @@ def __init__(self, app: ConanApp): self._app = app def upload(self, upload_data, remote): - for ref, bundle in upload_data.refs().items(): + for ref, packages in upload_data.items(): + bundle = upload_data.recipe_dict(ref) if bundle.get("upload"): self.upload_recipe(ref, bundle, remote) - for pref, prev_bundle in upload_data.prefs(ref, bundle).items(): + for pref in packages: + prev_bundle = upload_data.package_dict(pref) if prev_bundle.get("upload"): self.upload_package(pref, prev_bundle, remote) @@ -271,14 +316,42 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): return t -def compress_files(files, name, dest_dir, conf=None, ref=None, recursive=False, compression_plugin=None): - if compression_plugin: - return _compress_files_with_plugin(files, name, dest_dir, conf, ref, recursive, compression_plugin) - +def compress_files(files, name, dest_dir, conf=None, ref=None, recursive=False, + compression_plugin=None): tgz_path = os.path.join(dest_dir, name) t1 = time.time() - # FIXME, better write to disk sequentially and not keep tgz contents in memory + ConanOutput(scope=str(ref or "")).info(f"Compressing {name}") + + if ref: + ConanOutput(scope=str(ref) if ref else None).info(f"Compressing {name}") + + if compression_plugin: + ConanOutput(scope=str(ref or "")).info(f"Compressing {name} using compression plugin") + compressed_extension = compression_plugin.tar_compress( + archive_path=abs_path_without_extension, + files=files, + recursive=recursive, + conf=conf, + ref=ref, + ) + return tgz_path + + if name.endswith("zst"): + with tarfile.open(tgz_path, "w:zst", level=compresslevel) as tar: # noqa Py314 only + for filename, abs_path in sorted(files.items()): + tar.add(abs_path, filename, recursive=recursive) + ConanOutput().debug(f"{name} compressed in {time.time() - t1} time") + return tgz_path + + if name.endswith("xz"): + # The default to PAX_FORMAT in case of Python 3.7 + with tarfile.open(tgz_path, "w:xz", preset=compresslevel, format=tarfile.PAX_FORMAT) as tar: + for filename, abs_path in sorted(files.items()): + tar.add(abs_path, filename, recursive=recursive) + ConanOutput().debug(f"{name} compressed in {time.time() - t1} time") + return tgz_path + with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) @@ -287,36 +360,14 @@ def compress_files(files, name, dest_dir, conf=None, ref=None, recursive=False, tgz.add(abs_path, filename, recursive=recursive) tgz.close() - duration = time.time() - t1 - ConanOutput().debug(f"{name} compressed in {duration} time") + ConanOutput().debug(f"{name} compressed in {time.time() - t1} time") return tgz_path def _compress_files_with_plugin(files, name, dest_dir, conf, ref, recursive, compression_plugin): t1 = time.time() abs_path_without_extension = os.path.join(dest_dir, COMPRESSED_PLUGIN_TAR_NAME) - ConanOutput(scope=str(ref or "")).info(f"Compressing {name} using compression plugin") - compressed_extension = compression_plugin.tar_compress( - archive_path=abs_path_without_extension, - files=files, - recursive=recursive, - conf=conf, - ref=ref, - ) - ConanOutput().debug(f"Compressed in {time.time() - t1} time") - ConanOutput().success(f"{time.time() - t1}") - if not compressed_extension or not compressed_extension.startswith("."): - raise ConanException("The 'compression.py' did not return the compressed extension.") - - compressed_path = abs_path_without_extension + compressed_extension - t1 = time.time() - tgz_path = os.path.join(dest_dir, name) - with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: - tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=0) - tgz.add(compressed_path, arcname=os.path.basename(compressed_path), recursive=recursive) - tgz.close() - ConanOutput().debug(f"{name} wrapped in {time.time() - t1} time") - remove(compressed_path) - return tgz_path + + def _total_size(cache_files): total_size = 0 @@ -341,7 +392,8 @@ def _metadata_files(folder, metadata): def gather_metadata(package_list, cache, metadata): - for rref, recipe_bundle in package_list.refs().items(): + for rref, packages in package_list.items(): + recipe_bundle = package_list.recipe_dict(rref) if metadata or recipe_bundle["upload"]: metadata_folder = cache.recipe_layout(rref).metadata() files = _metadata_files(metadata_folder, metadata) @@ -350,7 +402,8 @@ def gather_metadata(package_list, cache, metadata): recipe_bundle.setdefault("files", {}).update(files) recipe_bundle["upload"] = True - for pref, pkg_bundle in package_list.prefs(rref, recipe_bundle).items(): + for pref in packages: + pkg_bundle = package_list.package_dict(pref) if metadata or pkg_bundle["upload"]: metadata_folder = cache.pkg_layout(pref).metadata() files = _metadata_files(metadata_folder, metadata) diff --git a/conan/internal/cache/cache.py b/conan/internal/cache/cache.py index c21d4db4134..3be2832c01f 100644 --- a/conan/internal/cache/cache.py +++ b/conan/internal/cache/cache.py @@ -110,6 +110,9 @@ def create_build_pkg_layout(self, pref: PkgReference): def recipe_layout(self, ref: RecipeReference): """ the revision must exists, the folder must exist + The regular graph building will use this method if the revision is defined, like + when using lockfiles or explicit, or recipe_layout_latest() if not, to do one single DB + query """ assert ref.revision is not None ref_data = self._db.get_recipe(ref) @@ -117,18 +120,30 @@ def recipe_layout(self, ref: RecipeReference): ref = ref_data.get("ref") # new revision with timestamp return RecipeLayout(ref, os.path.join(self._base_folder, ref_path)) - def get_latest_recipe_reference(self, ref: RecipeReference): + def recipe_layout_latest(self, ref: RecipeReference): + """ the revision must be None, the folder must exist + This method was added so the ConanProxy used to resolve the dependency graph + avoid doing 2 DB calls when the revision is not defined + """ + assert ref.revision is None + ref_data = self._db.get_latest_recipe(ref) + ref_path = ref_data.get("path") + ref = ref_data.get("ref") # new revision with timestamp + return RecipeLayout(ref, os.path.join(self._base_folder, ref_path)) + + def get_latest_recipe_revision(self, ref: RecipeReference) -> RecipeReference: assert ref.revision is None ref_data = self._db.get_latest_recipe(ref) return ref_data.get("ref") - def get_recipe_revisions_references(self, ref: RecipeReference): + def get_recipe_revisions(self, ref: RecipeReference): # For listing multiple revisions only assert ref.revision is None return self._db.get_recipe_revisions_references(ref) def pkg_layout(self, pref: PkgReference): """ the revision must exists, the folder must exist + No longer used by GraphBinariesAnalyzer """ assert pref.ref.revision, "Recipe revision must be known to get the package layout" assert pref.package_id, "Package id must be known to get the package layout" @@ -138,6 +153,23 @@ def pkg_layout(self, pref: PkgReference): # we use abspath to convert cache forward slash in Windows to backslash return PackageLayout(pref, os.path.abspath(os.path.join(self._base_folder, pref_path))) + def pkg_layout_latest(self, pref: PkgReference): + """ + GraphBinariesAnalyzer will call this method to avoid doing 2 DB calls, previously + it was using pkg_layout() after a get_latest_package_revision() + """ + assert pref.ref.revision, "Recipe revision must be known to get the package layout" + assert pref.package_id, "Package id must be known to get the package layout" + assert pref.revision is None + + pref_data = self._db.get_latest_package_reference_data(pref) + if pref_data is None: + return None + pref_path = pref_data.get("path") + pref = pref_data.get("pref") # new revision with timestamp + # we use abspath to convert cache forward slash in Windows to backslash + return PackageLayout(pref, os.path.abspath(os.path.join(self._base_folder, pref_path))) + def create_ref_layout(self, ref: RecipeReference): """ called exclusively by: - RemoteManager.get_recipe() @@ -183,7 +215,9 @@ def exists_prev(self, pref): # Used just by download to skip downloads if prev already exists in cache return self._db.exists_prev(pref) - def get_latest_package_reference(self, pref): + def get_latest_package_revision(self, pref: PkgReference) -> PkgReference: + # This is no longer needed by the Graph resolution functionality, only by ListAPI + # its usage in graph resolution has been replaced by a single call to pkg_layout_latest() return self._db.get_latest_package_reference(pref) def get_package_references(self, ref: RecipeReference, @@ -191,8 +225,8 @@ def get_package_references(self, ref: RecipeReference, """Get the latest package references""" return self._db.get_package_references(ref, only_latest_prev) - def get_package_revisions_references(self, pref: PkgReference, only_latest_prev=False): - return self._db.get_package_revisions_references(pref, only_latest_prev) + def get_package_revisions(self, pref: PkgReference) -> List[PkgReference]: + return self._db.get_package_revisions_references(pref) def get_matching_build_id(self, ref, build_id): return self._db.get_matching_build_id(ref, build_id) @@ -269,14 +303,14 @@ def assign_rrev(self, layout: RecipeLayout): def get_recipe_lru(self, ref): return self._db.get_recipe_lru(ref) - def update_recipe_lru(self, ref): - self._db.update_recipe_lru(ref) + def update_recipes_lru(self, refs): + self._db.update_recipes_lru(refs) def get_package_lru(self, pref): return self._db.get_package_lru(pref) - def update_package_lru(self, pref): - self._db.update_package_lru(pref) + def update_packages_lru(self, prefs): + self._db.update_packages_lru(prefs) def path_to_ref(self, path): try: diff --git a/conan/internal/cache/db/cache_database.py b/conan/internal/cache/db/cache_database.py index e18468950c5..56c68e685e2 100644 --- a/conan/internal/cache/db/cache_database.py +++ b/conan/internal/cache/db/cache_database.py @@ -24,8 +24,14 @@ def __init__(self, filename): def exists_prev(self, ref): return self._packages.get_package_revisions_reference_exists(ref) - def get_latest_package_reference(self, ref): - prevs = self.get_package_revisions_references(ref, True) + def get_latest_package_reference(self, pref): + prevs = list(self._packages.get_package_revisions_references(pref, only_latest_prev=True)) + return prevs[0]["pref"] if prevs else None + + def get_latest_package_reference_data(self, pref): + # Used just for PkgCache.pkg_layout_latest() + # TODO: This can be refactored, unified with get_latest_package_reference() + prevs = list(self._packages.get_package_revisions_references(pref, only_latest_prev=True)) return prevs[0] if prevs else None def update_recipe_timestamp(self, ref): @@ -40,11 +46,11 @@ def get_recipe_lru(self, ref): def get_package_lru(self, pref: PkgReference): return self._packages.get(pref)["lru"] - def update_recipe_lru(self, ref): - self._recipes.update_lru(ref) + def update_recipes_lru(self, refs): + self._recipes.update_lru(refs) - def update_package_lru(self, pref): - self._packages.update_lru(pref) + def update_packages_lru(self, prefs): + self._packages.update_lru(prefs) def remove_recipe(self, ref: RecipeReference): # Removing the recipe must remove all the package binaries too from DB @@ -92,9 +98,10 @@ def list_references(self, pattern=None): return [ref for ref in self._recipes.all_references() if pattern is None or ref.partial_match(pattern)] - def get_package_revisions_references(self, pref: PkgReference, only_latest_prev=False): + def get_package_revisions_references(self, pref: PkgReference): return [d["pref"] - for d in self._packages.get_package_revisions_references(pref, only_latest_prev)] + for d in self._packages.get_package_revisions_references(pref, + only_latest_prev=False)] def get_package_references(self, ref: RecipeReference, only_latest_prev=True): return [d["pref"] diff --git a/conan/internal/cache/db/packages_table.py b/conan/internal/cache/db/packages_table.py index 5523c65bc30..1f53f08f07a 100644 --- a/conan/internal/cache/db/packages_table.py +++ b/conan/internal/cache/db/packages_table.py @@ -13,7 +13,7 @@ class PackagesDBTable(BaseDbTable): ('rrev', str), ('pkgid', str, True), ('prev', str, True), - ('path', str, False, None, True), + ('path', str, False, True), ('timestamp', float), ('build_id', str, True), ('lru', int)] @@ -100,17 +100,19 @@ def update_timestamp(self, pref: PkgReference, path: str, build_id: str): except sqlite3.IntegrityError: raise ConanReferenceAlreadyExistsInDB(f"Reference '{repr(pref)}' already exists") - def update_lru(self, pref): - assert pref.revision is not None + def update_lru(self, prefs): # TODO: InstallGraph is dropping the pref.timestamp, cannot be checked here yet # assert pref.timestamp is not None, f"PREF _TIMESSTAMP IS NONE {repr(pref)}" - where_clause = self._where_clause(pref) + params = [(str(pref.ref), pref.ref.revision, pref.package_id, pref.revision) + for pref in prefs] + where_clause = (f"{self.columns.reference} = ? AND {self.columns.rrev} = ? " + f"AND {self.columns.pkgid} = ? AND {self.columns.prev} = ?") lru = timestamp_now() query = f"UPDATE {self.table_name} " \ f"SET {self.columns.lru} = '{lru}' " \ f"WHERE {where_clause};" with self.db_connection() as conn: - conn.execute(query) + conn.executemany(query, params) def remove_build_id(self, pref): where_clause = self._where_clause(pref) @@ -168,8 +170,8 @@ def get_package_revisions_references(self, pref: PkgReference, only_latest_prev= f'ORDER BY {self.columns.timestamp} DESC' with self.db_connection() as conn: r = conn.execute(query) - for row in r.fetchall(): - yield self._as_dict(self.row_type(*row)) + rows = r.fetchall() + return [self._as_dict(self.row_type(*row)) for row in rows] def get_package_revisions_reference_exists(self, pref: PkgReference): assert pref.ref.revision, "To check package revision existence you must provide a recipe revision." @@ -185,7 +187,7 @@ def get_package_revisions_reference_exists(self, pref: PkgReference): with self.db_connection() as conn: r = conn.execute(query) row = r.fetchone() - return bool(row) + return bool(row) def get_package_references(self, ref: RecipeReference, only_latest_prev=True): # Return the latest revisions @@ -212,8 +214,8 @@ def get_package_references(self, ref: RecipeReference, only_latest_prev=True): f'ORDER BY {self.columns.timestamp} DESC' with self.db_connection() as conn: r = conn.execute(query) - for row in r.fetchall(): - yield self._as_dict(self.row_type(*row)) + rows = r.fetchall() + return [self._as_dict(self.row_type(*row)) for row in rows] def get_package_references_with_build_id_match(self, ref: RecipeReference, build_id): # Return the latest revisions @@ -237,9 +239,9 @@ def get_package_references_with_build_id_match(self, ref: RecipeReference, build with self.db_connection() as conn: r = conn.execute(query) row = r.fetchone() - if row: - return self._as_dict(self.row_type(*row)) - return None + if row: + return self._as_dict(self.row_type(*row)) + return None def path_to_ref(self, path): query = f'SELECT * FROM {self.table_name} ' \ @@ -247,9 +249,9 @@ def path_to_ref(self, path): with self.db_connection() as conn: r = conn.execute(query) row = r.fetchone() - if not row: - return None - ref = RecipeReference.loads(row[0]) - ref.revision = row[1] - pref = PkgReference(ref, row[2], row[3], row[5]) - return pref + if not row: + return None + ref = RecipeReference.loads(row[0]) + ref.revision = row[1] + pref = PkgReference(ref, row[2], row[3], row[5]) + return pref diff --git a/conan/internal/cache/db/recipes_table.py b/conan/internal/cache/db/recipes_table.py index c9030c96024..6ed33d7942b 100644 --- a/conan/internal/cache/db/recipes_table.py +++ b/conan/internal/cache/db/recipes_table.py @@ -10,7 +10,7 @@ class RecipesDBTable(BaseDbTable): table_name = 'recipes' columns_description = [('reference', str), ('rrev', str), - ('path', str, False, None, True), + ('path', str, False, True), ('timestamp', float), ('lru', int)] unique_together = ('reference', 'rrev') @@ -60,16 +60,15 @@ def update_timestamp(self, ref: RecipeReference): with self.db_connection() as conn: conn.execute(query) - def update_lru(self, ref): - assert ref.revision is not None - assert ref.timestamp is not None - where_clause = self._where_clause(ref) + def update_lru(self, refs): + params = [(str(ref), ref.revision) for ref in refs] + where_clause = f"{self.columns.reference} = ? AND {self.columns.rrev} = ?" lru = timestamp_now() query = f"UPDATE {self.table_name} " \ f"SET {self.columns.lru} = '{lru}' " \ f"WHERE {where_clause};" with self.db_connection() as conn: - conn.execute(query) + conn.executemany(query, params) def remove(self, ref: RecipeReference): where_clause = self._where_clause(ref) @@ -85,7 +84,7 @@ def all_references(self): with self.db_connection() as conn: r = conn.execute(query) rows = r.fetchall() - return [RecipeReference.loads(row[0]) for row in rows] + return [RecipeReference.loads(row[0]) for row in rows] def get_recipe(self, ref: RecipeReference): query = f'SELECT * FROM {self.table_name} ' \ @@ -94,9 +93,9 @@ def get_recipe(self, ref: RecipeReference): with self.db_connection() as conn: r = conn.execute(query) row = r.fetchone() - if not row: - raise ConanReferenceDoesNotExistInDB(f"Recipe '{ref.repr_notime()}' not found") - ret = self._as_dict(self.row_type(*row)) + if not row: + raise ConanReferenceDoesNotExistInDB(f"Recipe '{ref.repr_notime()}' not found") + ret = self._as_dict(self.row_type(*row)) return ret def get_latest_recipe(self, ref: RecipeReference): @@ -112,9 +111,9 @@ def get_latest_recipe(self, ref: RecipeReference): with self.db_connection() as conn: r = conn.execute(query) row = r.fetchone() - if row is None: - raise ConanReferenceDoesNotExistInDB(f"Recipe '{ref}' not found") - ret = self._as_dict(self.row_type(*row)) + if row is None: + raise ConanReferenceDoesNotExistInDB(f"Recipe '{ref}' not found") + ret = self._as_dict(self.row_type(*row)) return ret def get_recipe_revisions_references(self, ref: RecipeReference): @@ -125,7 +124,8 @@ def get_recipe_revisions_references(self, ref: RecipeReference): with self.db_connection() as conn: r = conn.execute(query) - ret = [self._as_dict(self.row_type(*row))["ref"] for row in r.fetchall()] + rows = r.fetchall() + ret = [self._as_dict(self.row_type(*row))["ref"] for row in rows] return ret def path_to_ref(self, path): @@ -134,9 +134,9 @@ def path_to_ref(self, path): with self.db_connection() as conn: r = conn.execute(query) row = r.fetchone() - if not row: - return None - ref = RecipeReference.loads(row[0]) - ref.revision = row[1] - ref.timestamp = row[3] - return ref + if not row: + return None + ref = RecipeReference.loads(row[0]) + ref.revision = row[1] + ref.timestamp = row[3] + return ref diff --git a/conan/internal/cache/db/table.py b/conan/internal/cache/db/table.py index e4e150d26e4..241b55550e1 100644 --- a/conan/internal/cache/db/table.py +++ b/conan/internal/cache/db/table.py @@ -2,7 +2,9 @@ import threading from collections import defaultdict, namedtuple from contextlib import contextmanager -from typing import Tuple, List, Optional +from typing import Tuple, List + +from conan.errors import ConanException class BaseDbTable: @@ -23,7 +25,11 @@ def __init__(self, filename): @contextmanager def db_connection(self): - assert self._lock.acquire(timeout=20), "Conan failed to acquire database lock" + if not self._lock.acquire(timeout=20): + raise ConanException("Conan failed to acquire database lock in 20s. Maybe the system is" + "under very heavy load. Please report it to Github tickets") + # isolation_level=None, puts it in regular SQLITE autocommit mode, every + # connection.execute() will autocommit connection = sqlite3.connect(self.filename, isolation_level=None, timeout=20) try: yield connection @@ -32,25 +38,19 @@ def db_connection(self): self._lock.release() def create_table(self): - def field(name, typename, nullable=False, check_constraints: Optional[List] = None, - unique=False): + def field(name, typename, nullable=False, unique=False): field_str = name - if typename in [str, ]: + if typename is str: field_str += ' text' - elif typename in [int, ]: + elif typename is int: field_str += ' integer' - elif typename in [float, ]: - field_str += ' real' else: - assert False, f"sqlite3 type not mapped for type '{typename}'" + assert typename is float, f"sqlite3 type not mapped for type '{typename}'" + field_str += ' real' if not nullable: field_str += ' NOT NULL' - if check_constraints: - constraints = ', '.join([str(it) for it in check_constraints]) - field_str += f' CHECK ({name} IN ({constraints}))' - if unique: field_str += ' UNIQUE' @@ -61,11 +61,3 @@ def field(name, typename, nullable=False, check_constraints: Optional[List] = No table_checks = f", UNIQUE({', '.join(self.unique_together)})" if self.unique_together else '' with self.db_connection() as conn: conn.execute(f"CREATE TABLE {guard} {self.table_name} ({fields} {table_checks});") - - def dump(self): - print(f"********* BEGINTABLE {self.table_name}*************") - with self.db_connection() as conn: - r = conn.execute(f'SELECT rowid, * FROM {self.table_name}') - for it in r.fetchall(): - print(str(it)) - print(f"********* ENDTABLE {self.table_name}*************") diff --git a/conan/internal/cache/integrity_check.py b/conan/internal/cache/integrity_check.py index 43b526d11f6..f5b22690c13 100644 --- a/conan/internal/cache/integrity_check.py +++ b/conan/internal/cache/integrity_check.py @@ -1,7 +1,7 @@ import os +from conan.api.model.list import PackagesList from conan.api.output import ConanOutput -from conan.errors import ConanException from conan.api.model import PkgReference from conan.api.model import RecipeReference @@ -20,14 +20,21 @@ class IntegrityChecker: def __init__(self, cache): self._cache = cache - def check(self, pkg_list): - corrupted = False - for ref, recipe_bundle in pkg_list.refs().items(): - corrupted = self._recipe_corrupted(ref) or corrupted - for pref, prev_bundle in pkg_list.prefs(ref, recipe_bundle).items(): - corrupted = self._package_corrupted(pref) or corrupted - if corrupted: - raise ConanException("There are corrupted artifacts, check the error logs") + def check(self, pkg_list) -> PackagesList: + corrupted_pkglist = PackagesList() + for ref, packages in pkg_list.items(): + # Check if any of the packages are corrupted + if self._recipe_corrupted(ref): + # If the recipe is corrupted, all its packages are considered corrupted + corrupted_pkglist.add_ref(ref) + else: + # Do not check any binary if the recipe is corrupted + for pref in packages: + if self._package_corrupted(pref): + corrupted_pkglist.add_ref(ref) + # Cannot add package reference without having the recipe reference already added + corrupted_pkglist.add_pref(pref) + return corrupted_pkglist def _recipe_corrupted(self, ref: RecipeReference): layout = self._cache.recipe_layout(ref) diff --git a/conan/internal/conan_app.py b/conan/internal/conan_app.py index 48b514babb0..8cb9fa2a4cb 100644 --- a/conan/internal/conan_app.py +++ b/conan/internal/conan_app.py @@ -51,8 +51,8 @@ def __init__(self, conan_api): self.cache = PkgCache(self.cache_folder, global_conf) # Wraps RestApiClient to add authentication support (same interface) localdb = LocalDB(cache_folder) - auth_manager = ConanApiAuthManager(conan_api.remotes.requester, cache_folder, localdb, - global_conf) + requester = conan_api._api_helpers.requester # noqa + auth_manager = ConanApiAuthManager(requester, cache_folder, localdb, global_conf) # Handle remote connections self.remote_manager = RemoteManager(self.cache, auth_manager, cache_folder, self.conan_api) global_editables = conan_api.local.editable_packages @@ -72,8 +72,8 @@ def __init__(self, conan_api): self.pyreq_loader = PyRequireLoader(self, self._global_conf) cmd_wrap = CmdWrapper(HomePaths(self.cache_folder).wrapper_path) - conanfile_helpers = ConanFileHelpers(conan_api.remotes.requester, cmd_wrap, - self._global_conf, self.cache, self.cache_folder) + requester = conan_api._api_helpers.requester # noqa + conanfile_helpers = ConanFileHelpers(requester, cmd_wrap, self._global_conf, self.cache, self.cache_folder) self.loader = ConanFileLoader(self.pyreq_loader, conanfile_helpers) diff --git a/conan/internal/default_settings.py b/conan/internal/default_settings.py index 6b260e7b26f..16ee28d3c83 100644 --- a/conan/internal/default_settings.py +++ b/conan/internal/default_settings.py @@ -22,26 +22,30 @@ "14.0", "14.1", "14.2", "14.3", "14.4", "14.5", "14.6", "14.7", "14.8", "15.0", "15.1", "15.2", "15.3", "15.4", "15.5", "15.6", "15.7", "15.8", "16.0", "16.1", "16.2", "16.3", "16.4", "16.5", "16.6", "16.7", - "17.0", "17.1", "17.2", "17.3", "17.4", "17.5", "17.6", "17.8", - "18.0", "18.1", "18.2", "18.3", "18.4", "18.5", "18.6"] + "17.0", "17.1", "17.2", "17.3", "17.4", "17.5", "17.6", "17.7", "17.8", + "18.0", "18.1", "18.2", "18.3", "18.4", "18.5", "18.6", "18.7", + "26.0", "26.1", "26.2"] sdk: ["iphoneos", "iphonesimulator"] sdk_version: [null, "11.3", "11.4", "12.0", "12.1", "12.2", "12.4", "13.0", "13.1", "13.2", "13.3", "13.4", "13.5", "13.6", "13.7", "14.0", "14.1", "14.2", "14.3", "14.4", "14.5", "15.0", "15.2", "15.4", "15.5", "16.0", "16.1", "16.2", "16.4", "17.0", "17.1", "17.2", "17.4", "17.5", - "18.0", "18.1", "18.2", "18.4", "18.5", "18.6"] + "18.0", "18.1", "18.2", "18.4", "18.5", + "26.0", "26.1", "26.2"] watchOS: version: ["4.0", "4.1", "4.2", "4.3", "5.0", "5.1", "5.2", "5.3", "6.0", "6.1", "6.2", "6.3", "7.0", "7.1", "7.2", "7.3", "7.4", "7.5", "7.6", "8.0", "8.1", "8.3", "8.4", "8.5", "8.6", "8.7", "9.0","9.1", "9.2", "9.3", "9.4", "9.5", "9.6", "10.0", "10.1", "10.2", "10.3", "10.4", "10.5", "10.6", - "11.0", "11.1", "11.2", "11.3", "11.4"] + "11.0", "11.1", "11.2", "11.3", "11.4", "11.5", "11.6", + "26.0", "26.1", "26.2"] sdk: ["watchos", "watchsimulator"] sdk_version: [null, "4.3", "5.0", "5.1", "5.2", "5.3", "6.0", "6.1", "6.2", "7.0", "7.1", "7.2", "7.3", "7.4", "8.0", "8.0.1", "8.3", "8.5", "9.0", "9.1", "9.4", "10.0", "10.1", "10.2", "10.4", "10.5", - "11.0", "11.1", "11.2", "11.4"] + "11.0", "11.1", "11.2", "11.4", "11.5", + "26.0", "26.1", "26.2"] tvOS: version: ["11.0", "11.1", "11.2", "11.3", "11.4", "12.0", "12.1", "12.2", "12.3", "12.4", @@ -50,26 +54,32 @@ "15.0", "15.1", "15.2", "15.3", "15.4", "15.5", "15.6", "16.0", "16.1", "16.2", "16.3", "16.4", "16.5", "16.6", "17.0", "17.1", "17.2", "17.3", "17.4", "17.5", "17.6", - "18.0", "18.1", "18.2", "18.3", "18.4", "18.5", "18.6"] + "18.0", "18.1", "18.2", "18.3", "18.4", "18.5", "18.6", + "26.0", "26.1", "26.2"] sdk: ["appletvos", "appletvsimulator"] sdk_version: [null, "11.3", "11.4", "12.0", "12.1", "12.2", "12.4", "13.0", "13.2", "13.3", "13.4", "14.0", "14.2", "14.3", "14.4", "14.5", "15.0", "15.2", "15.4", "15.5", "16.0", "16.1", "16.4", "17.0", "17.1", "17.2", "17.4", "17.5", - "18.0", "18.1", "18.2", "18.4", "18.5", "18.6"] + "18.0", "18.1", "18.2", "18.4", "18.5", + "26.0", "26.1", "26.2"] visionOS: - version: ["1.0", "1.1", "1.2", "1.3", "2.0", "2.1", "2.2", "2.3", "2.4"] + version: ["1.0", "1.1", "1.2", "1.3", "2.0", "2.1", "2.2", "2.3", "2.4", "2.5", "2.6", + "26.0", "26.1", "26.2"] sdk: ["xros", "xrsimulator"] - sdk_version: [null, "1.0", "1.1", "1.2", "1.3", "2.0", "2.1", "2.2", "2.4"] + sdk_version: [null, "1.0", "1.1", "1.2", "1.3", "2.0", "2.1", "2.2", "2.4", "2.5", + "26.0", "26.1", "26.2"] Macos: version: [null, "10.6", "10.7", "10.8", "10.9", "10.10", "10.11", "10.12", "10.13", "10.14", "10.15", "11.0", "11.1", "11.2", "11.3", "11.4", "11.5", "11.6", "11.7", "12.0", "12.1", "12.2", "12.3", "12.4", "12.5", "12.6", "12.7", "13.0", "13.1", "13.2", "13.3", "13.4", "13.5", "13.6", "13.7", "14.0", "14.1", "14.2", "14.3", "14.4", "14.5", "14.6", "14.7", - "15.0", "15.1", "15.2", "15.3", "15.4"] + "15.0", "15.1", "15.2", "15.3", "15.4", "15.5", "15.6", "15.7", + "26.0", "26.1", "26.2"] sdk_version: [null, "10.13", "10.14", "10.15", "11.0", "11.1", "11.2", "11.3", "12.0", "12.1", "12.3", "12.4", "13.0", "13.1", "13.3", "14.0", "14.2", "14.4", "14.5", - "15.0", "15.1", "15.2", "15.4"] + "15.0", "15.1", "15.2", "15.4", "15.5", + "26.0", "26.1", "26.2"] subsystem: null: catalyst: @@ -120,7 +130,7 @@ cppstd: [null, 98, gnu98, 11, gnu11, 14, gnu14, 17, gnu17, 20, gnu20, 23, gnu23, 26, gnu26] cstd: [null, 99, gnu99, 11, gnu11, 17, gnu17, 23, gnu23] msvc: - version: [170, 180, 190, 191, 192, 193, 194] + version: [170, 180, 190, 191, 192, 193, 194, 195] update: [null, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9] runtime: [static, dynamic] runtime_type: [Debug, Release] @@ -131,12 +141,12 @@ version: ["3.3", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9", "4.0", "5.0", "6.0", "7.0", "7.1", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", - "18", "19", "20"] + "18", "19", "20", "21"] libcxx: [null, libstdc++, libstdc++11, libc++, c++_shared, c++_static] cppstd: [null, 98, gnu98, 11, gnu11, 14, gnu14, 17, gnu17, 20, gnu20, 23, gnu23, 26, gnu26] runtime: [null, static, dynamic] runtime_type: [null, Debug, Release] - runtime_version: [null, v140, v141, v142, v143, v144] + runtime_version: [null, v140, v141, v142, v143, v144, v145] cstd: [null, 99, gnu99, 11, gnu11, 17, gnu17, 23, gnu23] apple-clang: version: ["5.0", "5.1", "6.0", "6.1", "7.0", "7.3", "8.0", "8.1", "9.0", "9.1", diff --git a/conan/internal/deploy.py b/conan/internal/deploy.py index e8c3950ce0a..1ce4b53f95e 100644 --- a/conan/internal/deploy.py +++ b/conan/internal/deploy.py @@ -102,6 +102,7 @@ def runtime_deploy(graph, output_folder): Deploy all the shared libraries and the executables of the dependencies in a flat directory. It preserves symlinks in case the configuration tools.deployer:symlinks is True. + It preserves the directory structure when having subfolders """ conanfile = graph.root.conanfile output = ConanOutput(scope="runtime_deploy") @@ -136,7 +137,7 @@ def runtime_deploy(graph, output_folder): def _flatten_directory(dep, src_dir, output_dir, symlinks, extension_filter=None): """ - Copy all the files from the source directory in a flat output directory. + Copy all the files from the source directory in a flat output directory, respecting subfolders. An optional string, named extension_filter, can be set to copy only the files with the listed extensions. """ @@ -144,16 +145,20 @@ def _flatten_directory(dep, src_dir, output_dir, symlinks, extension_filter=None output = ConanOutput(scope="runtime_deploy") for src_dirpath, _, src_filenames in os.walk(src_dir, followlinks=symlinks): + rel_path = os.path.relpath(src_dirpath, src_dir) for src_filename in src_filenames: if extension_filter and not any(fnmatch.fnmatch(src_filename, f'*{ext}') for ext in extension_filter): continue src_filepath = os.path.join(src_dirpath, src_filename) - dest_filepath = os.path.join(output_dir, src_filename) + dest_filepath = os.path.join(output_dir, rel_path, src_filename) if not symlinks and os.path.islink(src_filepath): continue + if not os.path.exists(os.path.dirname(dest_filepath)): + os.makedirs(os.path.dirname(dest_filepath)) + if os.path.exists(dest_filepath): if filecmp.cmp(src_filepath, dest_filepath): # Be efficient, do not copy output.verbose(f"{dest_filepath} exists with same contents, skipping copy") diff --git a/conan/internal/graph/compatibility.py b/conan/internal/graph/compatibility.py index e9131b81f0e..2de231caa46 100644 --- a/conan/internal/graph/compatibility.py +++ b/conan/internal/graph/compatibility.py @@ -89,24 +89,28 @@ def migrate_compatibility_files(cache_folder): compatibility_file = os.path.join(compatible_folder, "compatibility.py") cppstd_compat_file = os.path.join(compatible_folder, "cppstd_compat.py") - def _should_migrate_file(file_path): + def _is_migratable(file_path): if not os.path.exists(file_path): return True content = load(file_path) first_line = content.lstrip().split("\n", 1)[0] return CONAN_GENERATED_COMMENT in first_line - if _should_migrate_file(compatibility_file) and _should_migrate_file(cppstd_compat_file): - if os.path.exists(compatibility_file) and load(compatibility_file) != _default_compat: - ConanOutput().success("Migration: Successfully updated compatibility.py") - save(compatibility_file, _default_compat) + if _is_migratable(compatibility_file) and _is_migratable(cppstd_compat_file): + compatibility_exists = os.path.exists(compatibility_file) + needs_update = not compatibility_exists or load(compatibility_file) != _default_compat + if needs_update: + save(compatibility_file, _default_compat) + if compatibility_exists: + ConanOutput().success("Migration: Successfully updated compatibility.py") if os.path.exists(cppstd_compat_file): os.remove(cppstd_compat_file) class BinaryCompatibility: - def __init__(self, compatibility_plugin_folder): + def __init__(self, compatibility_plugin_folder, hook_manager): + self._hook_manager = hook_manager compatibility_file = os.path.join(compatibility_plugin_folder, "compatibility.py") if not os.path.exists(compatibility_file): raise ConanException("The 'compatibility.py' plugin file doesn't exist. If you want " @@ -143,7 +147,7 @@ def compatibles(self, conanfile): conanfile.settings = c.settings conanfile.settings_target = c.settings_target conanfile.options = c.options - run_validate_package_id(conanfile) + run_validate_package_id(conanfile, self._hook_manager) pid = c.package_id() if pid not in result and not c.invalid: result[pid] = c diff --git a/conan/internal/graph/compute_pid.py b/conan/internal/graph/compute_pid.py index 4c4c6abb47b..5c83bcb040e 100644 --- a/conan/internal/graph/compute_pid.py +++ b/conan/internal/graph/compute_pid.py @@ -3,7 +3,8 @@ from conan.internal.errors import conanfile_remove_attr, conanfile_exception_formatter from conan.errors import ConanException, ConanInvalidConfiguration from conan.internal.methods import auto_header_only_package_id -from conan.internal.model.info import ConanInfo, RequirementsInfo, RequirementInfo, PythonRequiresInfo +from conan.internal.model.info import (ConanInfo, RequirementsInfo, RequirementInfo, + PythonRequiresInfo) def compute_package_id(node, modes, config_version, hook_manager): @@ -60,7 +61,7 @@ def compute_package_id(node, modes, config_version, hook_manager): node.package_id = info.package_id() -def run_validate_package_id(conanfile, hook_manager=None): +def run_validate_package_id(conanfile, hook_manager): # IMPORTANT: This validation code must run before calling info.package_id(), to mark "invalid" if hasattr(conanfile, "validate_build"): with conanfile_exception_formatter(conanfile, "validate_build"): @@ -71,14 +72,15 @@ def run_validate_package_id(conanfile, hook_manager=None): # This 'cant_build' will be ignored if we don't have to build the node. conanfile.info.cant_build = str(e) - if hasattr(conanfile, "validate"): + if hasattr(conanfile, "validate") or hook_manager.validate_hook: with conanfile_exception_formatter(conanfile, "validate"): with conanfile_remove_attr(conanfile, ['cpp_info'], "validate"): try: - if hook_manager: + if hook_manager.validate_hook: hook_manager.execute("pre_validate", conanfile=conanfile) - conanfile.validate() - if hook_manager: + if hasattr(conanfile, "validate"): + conanfile.validate() + if hook_manager.validate_hook: hook_manager.execute("post_validate", conanfile=conanfile) except ConanInvalidConfiguration as e: conanfile.info.invalid = str(e) @@ -90,5 +92,9 @@ def run_validate_package_id(conanfile, hook_manager=None): conanfile.package_id() elif "auto_header_only" in conanfile.implements: auto_header_only_package_id(conanfile) + if hook_manager.post_package_id_hook: + with conanfile_exception_formatter(conanfile, "package_id"): + with conanfile_remove_attr(conanfile, ['cpp_info', 'settings', 'options'], "package_id"): + hook_manager.execute("post_package_id", conanfile=conanfile) conanfile.info.validate() diff --git a/conan/internal/graph/graph.py b/conan/internal/graph/graph.py index eea5bc007bc..924b7e2248b 100644 --- a/conan/internal/graph/graph.py +++ b/conan/internal/graph/graph.py @@ -12,7 +12,7 @@ RECIPE_NEWER = "Newer" # The local recipe is modified and newer timestamp than server RECIPE_NOT_IN_REMOTE = "Not in remote" RECIPE_UPDATEABLE = "Update available" # The update of recipe is available (only in conan info) -RECIPE_NO_REMOTE = "No remote" +# These recipes do not have a full reference, not in the cache RECIPE_EDITABLE = "Editable" RECIPE_CONSUMER = "Consumer" # A conanfile from the user RECIPE_VIRTUAL = "Cli" # A virtual conanfile (dynamic in memory conanfile) @@ -120,7 +120,7 @@ def propagate_downstream(self, require, node, visibility_conflicts, src_node=Non # Take into account that while propagating we can find RUNTIME shared conflicts we # didn't find at check_downstream_exist, because we didn't know the shared/static existing = self.transitive_deps.get(require) - ill_formed = False + if existing is not None and existing.require is not require: if existing.node is not None and existing.node.ref != node.ref: # print(" +++++Runtime conflict!", require, "with", node.ref) @@ -138,11 +138,13 @@ def propagate_downstream(self, require, node, visibility_conflicts, src_node=Non assert not require.version_range # No ranges slip into transitive_deps definitions # TODO: Might need to move to an update() for performance - self.transitive_deps.pop(require, None) + poped = self.transitive_deps.pop(require, None) self.transitive_deps[require] = TransitiveRequirement(require, node) - if ill_formed: # remove dead .edges, to avoid orphans - direct_nodes = set(t.node for t in self.transitive_deps.values() if t.require.direct) - self.edges = [e for e in self.edges if e.dst in direct_nodes] + if poped is not None: # adjust .edges, to avoid orphans + for e in self.edges: + if e.dst is poped.node: # check for identity, pointing to that node + e.dst = node + break if self.conanfile.vendor: return diff --git a/conan/internal/graph/graph_binaries.py b/conan/internal/graph/graph_binaries.py index 128e796464b..74081a890a4 100644 --- a/conan/internal/graph/graph_binaries.py +++ b/conan/internal/graph/graph_binaries.py @@ -1,8 +1,6 @@ -import json import os -from collections import OrderedDict -from conan.api.output import ConanOutput +from conan.api.output import ConanOutput, Color from conan.internal.cache.home_paths import HomePaths from conan.internal.graph.build_mode import BuildMode from conan.internal.graph.compatibility import BinaryCompatibility @@ -13,14 +11,12 @@ BINARY_INVALID, BINARY_EDITABLE_BUILD, RECIPE_PLATFORM, BINARY_PLATFORM) from conan.internal.graph.proxy import should_update_reference -from conan.internal.errors import conanfile_exception_formatter, ConanConnectionError, NotFoundException, \ - PackageNotFoundException +from conan.internal.errors import (conanfile_exception_formatter, ConanConnectionError, + NotFoundException, PackageNotFoundException) from conan.errors import ConanException +from conan.internal.model.conanconfig import loadconanconfig from conan.internal.model.info import RequirementInfo, RequirementsInfo -from conan.api.model import PkgReference -from conan.api.model import RecipeReference from conan.internal.model.pkg_type import PackageType -from conan.internal.util.files import load class GraphBinariesAnalyzer: @@ -34,7 +30,7 @@ def __init__(self, conan_app, global_conf, hook_manager): # These are the nodes with pref (not including PREV) that have been evaluated self._evaluated = {} # {pref: [nodes]} compat_folder = HomePaths(conan_app.cache_folder).compatibility_plugin_path - self._compatibility = BinaryCompatibility(compat_folder) + self._compatibility = BinaryCompatibility(compat_folder, hook_manager) unknown_mode = global_conf.get("core.package_id:default_unknown_mode", default="semver_mode") non_embed = global_conf.get("core.package_id:default_non_embed_mode", default="minor_mode") # recipe_revision_mode already takes into account the package_id @@ -42,6 +38,7 @@ def __init__(self, conan_app, global_conf, hook_manager): python_mode = global_conf.get("core.package_id:default_python_mode", default="minor_mode") build_mode = global_conf.get("core.package_id:default_build_mode", default=None) self._modes = unknown_mode, non_embed, embed_mode, python_mode, build_mode + self._warn_about_new_compatibility = False @staticmethod def _evaluate_build(node, build_mode): @@ -78,7 +75,7 @@ def _get_package_from_remotes(self, node, remotes, update): for r in remotes: try: info = node.conanfile.info - latest_pref = self._remote_manager.get_latest_package_reference(pref, r, info) + latest_pref = self._remote_manager.get_latest_package_revision(pref, r, info) results.append({'pref': latest_pref, 'remote': r}) if len(results) > 0 and not should_update_reference(node.ref, update): break @@ -124,7 +121,8 @@ def _evaluate_is_cached(self, node): return True self._evaluated[pref] = [node] - def _get_compatible_packages(self, node): + def _compatible_get_packages(self, node): + # Evaluate the ``compatibility() methods to compute the compatible hypothesis variants conanfile = node.conanfile original_package_id = node.package_id @@ -151,39 +149,149 @@ def _compatible_found(conanfile, pkg_id, compatible_pkg): conanfile.options = conanfile.options.copy_conaninfo_options() conanfile.options.update_options(compatible_pkg.options) - def _find_existing_compatible_binaries(self, node, compatibles, remotes, update): + def _compatible_find_existing_binaries(self, node, compatibles, remotes, update): + # Do the actual search in the cache and remotes for the compatible package-ids conanfile = node.conanfile original_binary = node.binary original_package_id = node.package_id conanfile.output.info(f"Main binary package '{original_package_id}' missing") conanfile.output.info(f"Checking {len(compatibles)} compatible configurations") - for package_id, compatible_package in compatibles.items(): - if should_update_reference(node.ref, update): - conanfile.output.info(f"'{package_id}': " - f"{conanfile.info.dump_diff(compatible_package)}") - node._package_id = package_id # Modifying package id under the hood, FIXME - node.binary = None # Invalidate it - self._process_compatible_node(node, remotes, update) # TODO: what if BINARY_BUILD - if node.binary in (BINARY_CACHE, BINARY_UPDATE, BINARY_DOWNLOAD): - self._compatible_found(conanfile, package_id, compatible_package) - return + compatibility_mode = self._global_conf.get("core.graph:compatibility_mode", + choices=("optimized",)) + use_compatibility_optimization = compatibility_mode == "optimized" + if not should_update_reference(conanfile.ref, update): - conanfile.output.info(f"Compatible configurations not found in cache, checking servers") + # First look all in the cache for package_id, compatible_package in compatibles.items(): + node._package_id = package_id # Modifying package id under the hood, FIXME + node.binary = None # Invalidate it + # Check that this same reference hasn't already been checked + if self._evaluate_is_cached(node): + # If we have already processed this compatible pref, + # mark it as usable based on previous evaluation + if node.binary in (BINARY_CACHE, BINARY_DOWNLOAD): + self._compatible_found(conanfile, package_id, compatible_package) + return + cache_latest_prev = self._compatible_cache_latest_prev(node) # not check remotes + if cache_latest_prev: + # If we have binary info, it means that the package was already processed, + # and we got a hit from the cache of compatibles + self._binary_in_cache(node, cache_latest_prev) + self._compatible_found(conanfile, package_id, compatible_package) + return + # If not found in the cache, then look for the first one in servers + conanfile.output.info(f"Compatible configurations not found in cache, checking servers") + if use_compatibility_optimization: + compatible_packages = self._compatible_get_packages_from_remotes(node.ref, remotes) + candidates = {pkg_id: pkg for pkg_id, pkg in compatibles.items() + if pkg_id in compatible_packages} + node.conanfile.output.info(f"Found {len(candidates)} compatible configurations " + f"in remotes") + else: + candidates = compatibles + compatible_packages = {} + self._warn_about_new_compatibility = True + for package_id, compatible_package in candidates.items(): conanfile.output.info(f"'{package_id}': " f"{conanfile.info.dump_diff(compatible_package)}") node._package_id = package_id # Modifying package id under the hood, FIXME node.binary = None # Invalidate it - self._evaluate_download(node, remotes, update) + # We already know which remotes have that package_id + available_remotes = compatible_packages.get(package_id, remotes) + self._evaluate_download(node, available_remotes, update=False) if node.binary == BINARY_DOWNLOAD: self._compatible_found(conanfile, package_id, compatible_package) return + else: # Need to check in servers too for the latest thing + if use_compatibility_optimization: + compatible_packages = self._compatible_get_packages_from_remotes(node.ref, remotes) + else: + compatible_packages = {} + self._warn_about_new_compatibility = True + for package_id, compatible_package in compatibles.items(): + conanfile.output.info(f"'{package_id}': " + f"{conanfile.info.dump_diff(compatible_package)}") + node._package_id = package_id # Modifying package id under the hood, FIXME + node.binary = None # Invalidate it + + if self._evaluate_is_cached(node): + # If we have already processed this compatible pref, + # mark it as usable based on previous evaluation + if node.binary in (BINARY_CACHE, BINARY_DOWNLOAD, BINARY_UPDATE): + self._compatible_found(conanfile, package_id, compatible_package) + return + cache_latest_prev = self._compatible_cache_latest_prev(node) # Not check remotes + available_remotes = compatible_packages.get(package_id, + [] if use_compatibility_optimization + else remotes) + if cache_latest_prev: + self._evaluate_cache_update(cache_latest_prev, node, available_remotes, update) + else: + if available_remotes: + self._evaluate_download(node, available_remotes, update) + else: + # If not in remotes, mark as missing, no need for further checks + node.binary = BINARY_MISSING + if node.binary in (BINARY_CACHE, BINARY_UPDATE, BINARY_DOWNLOAD): + self._compatible_found(conanfile, package_id, compatible_package) + return + node.conanfile.output.info("No compatible configuration found", fg=Color.BRIGHT_CYAN) # If no compatible is found, restore original state node.binary = original_binary node._package_id = original_package_id - def _find_build_compatible_binary(self, node, compatibles): + def _compatible_cache_latest_prev(self, node): + """ simplified checking of compatible_packages, that should be found existing, but + will never be built, for example. They cannot be editable either at this point. + """ + # TODO: Test that this works + if node.conanfile.info.invalid: + node.binary = BINARY_INVALID + return None + + # Obtain the cache_latest valid one, cleaning things if dirty + while True: + package_layout = self._cache.pkg_layout_latest(node.pref) + cache_latest_prev = package_layout.reference if package_layout else None + if cache_latest_prev is None: + break + if not self._evaluate_clean_pkg_folder_dirty(node, package_layout): + break + + return cache_latest_prev + + @staticmethod + def _binary_in_cache(node, cache_latest_prev): + assert cache_latest_prev.revision + assert node.binary is None + node.binary = BINARY_CACHE + node.binary_remote = None + node.prev = cache_latest_prev.revision + node.pref_timestamp = cache_latest_prev.timestamp + + def _compatible_get_packages_from_remotes(self, ref, remotes): + """ + Get available package ids in remotes for the given node reference + """ + results = {} + for remote in remotes: + try: + remote_prefs = self._remote_manager.search_packages(remote, ref, list_only=True) + if remote_prefs: + for remote_pref in remote_prefs: + results.setdefault(remote_pref.package_id, []).append(remote) + except NotFoundException: + # Not finding the reference in the remote is not an error, just continue + pass + except ConanConnectionError: + ConanOutput().error(f"Failed finding for package ids '{ref}' in " + f"remote '{remote.name}': remote not available") + raise + + return results + + def _compatible_find_build_binary(self, node, compatibles): original_binary = node.binary original_package_id = node.package_id output = node.conanfile.output @@ -209,9 +317,9 @@ def _evaluate_node(self, node, build_mode, remotes, update): if node.binary == BINARY_MISSING \ and not build_mode.should_build_missing(node.conanfile) and not node.should_build: - compatibles = self._get_compatible_packages(node) + compatibles = self._compatible_get_packages(node) if compatibles: - self._find_existing_compatible_binaries(node, compatibles, remotes, update) + self._compatible_find_existing_binaries(node, compatibles, remotes, update) if node.binary == BINARY_MISSING and build_mode.allowed(node.conanfile): node.should_build = True @@ -220,17 +328,17 @@ def _evaluate_node(self, node, build_mode, remotes, update): if node.binary == BINARY_INVALID and build_mode.allowed_compatible(node.conanfile): if compatibles is None: - compatibles = self._get_compatible_packages(node) + compatibles = self._compatible_get_packages(node) if compatibles: - self._find_build_compatible_binary(node, compatibles) + self._compatible_find_build_binary(node, compatibles) if node.binary == BINARY_BUILD: conanfile = node.conanfile if conanfile.vendor and not conanfile.conf.get("tools.graph:vendor", choices=("build",)): node.conanfile.info.invalid = f"The package '{conanfile.ref}' is a vendoring one, " \ f"needs to be built from source, but it " \ - "didn't enable 'tools.graph:vendor=build' to compute " \ - "its dependencies" + "didn't enable 'tools.graph:vendor=build' to compute" \ + " its dependencies" node.binary = BINARY_INVALID if any(n.node.binary in (BINARY_EDITABLE, BINARY_EDITABLE_BUILD) for n in node.transitive_deps.values()): @@ -265,10 +373,10 @@ def _process_node(self, node, build_mode, remotes, update): # Obtain the cache_latest valid one, cleaning things if dirty while True: - cache_latest_prev = self._cache.get_latest_package_reference(node.pref) + package_layout = self._cache.pkg_layout_latest(node.pref) + cache_latest_prev = package_layout.reference if package_layout else None if cache_latest_prev is None: break - package_layout = self._cache.pkg_layout(cache_latest_prev) if not self._evaluate_clean_pkg_folder_dirty(node, package_layout): break @@ -283,35 +391,7 @@ def _process_node(self, node, build_mode, remotes, update): elif cache_latest_prev is None: # This binary does NOT exist in the cache self._evaluate_download(node, remotes, update) else: # This binary already exists in the cache, maybe can be updated - self._evaluate_in_cache(cache_latest_prev, node, remotes, update) - - def _process_compatible_node(self, node, remotes, update): - """ simplified checking of compatible_packages, that should be found existing, but - will never be built, for example. They cannot be editable either at this point. - """ - # Check that this same reference hasn't already been checked - if self._evaluate_is_cached(node): - return - - # TODO: Test that this works - if node.conanfile.info.invalid: - node.binary = BINARY_INVALID - return - - # Obtain the cache_latest valid one, cleaning things if dirty - while True: - cache_latest_prev = self._cache.get_latest_package_reference(node.pref) - if cache_latest_prev is None: - break - package_layout = self._cache.pkg_layout(cache_latest_prev) - if not self._evaluate_clean_pkg_folder_dirty(node, package_layout): - break - - if cache_latest_prev is not None: - # This binary already exists in the cache, maybe can be updated - self._evaluate_in_cache(cache_latest_prev, node, remotes, update) - elif should_update_reference(node.ref, update): - self._evaluate_download(node, remotes, update) + self._evaluate_cache_update(cache_latest_prev, node, remotes, update) def _process_locked_node(self, node, build_mode, locked_prev): # Check that this same reference hasn't already been checked @@ -347,7 +427,7 @@ def _evaluate_download(self, node, remotes, update): else: node.binary = BINARY_DOWNLOAD - def _evaluate_in_cache(self, cache_latest_prev, node, remotes, update): + def _evaluate_cache_update(self, cache_latest_prev, node, remotes, update): assert cache_latest_prev.revision if should_update_reference(node.ref, update): output = node.conanfile.output @@ -382,16 +462,8 @@ def _config_version(self): return config_version_file = HomePaths(self._home_folder).config_version_path try: - config_refs = json.loads(load(config_version_file))["config_version"] - result = OrderedDict() - for r in config_refs: - try: - config_ref = PkgReference.loads(r) - req_info = RequirementInfo(config_ref.ref, config_ref.package_id, config_mode) - except ConanException: - config_ref = RecipeReference.loads(r) - req_info = RequirementInfo(config_ref, None, config_mode) - result[config_ref] = req_info + config_refs = loadconanconfig(config_version_file) + result = {r: RequirementInfo(r, None, config_mode) for r in config_refs} except Exception as e: raise ConanException(f"core.package_id:config_mode defined, but error while loading " f"'{os.path.basename(config_version_file)}'" @@ -456,6 +528,14 @@ def _evaluate_single(n): for n in pref_nodes[1:]: _evaluate_single(n) + if self._warn_about_new_compatibility: + (ConanOutput().info("\nA new experimental approach for binary compatibility detection " + "is available.\n" + " Enable it by setting the ", newline=False) + .info('core.graph:compatibility_mode=optimized', newline=False, fg=Color.BRIGHT_YELLOW) + .info(" conf\n and get improved performance when querying multiple " + "compatible binaries in remotes.\n")) + # Last level is always necessarily a consumer or a virtual assert len(levels[-1]) == 1 node = levels[-1][0] diff --git a/conan/internal/graph/graph_builder.py b/conan/internal/graph/graph_builder.py index aaf09e5c96a..e4f2532dc07 100644 --- a/conan/internal/graph/graph_builder.py +++ b/conan/internal/graph/graph_builder.py @@ -1,14 +1,13 @@ -import copy - from collections import deque from conan.internal.cache.conan_reference_layout import BasicLayout from conan.internal.methods import run_configure_method from conan.internal.model.recipe_ref import ref_matches from conan.internal.graph.graph import DepsGraph, Node, CONTEXT_HOST, \ - CONTEXT_BUILD, TransitiveRequirement, RECIPE_VIRTUAL, RECIPE_EDITABLE + CONTEXT_BUILD, TransitiveRequirement, RECIPE_VIRTUAL, RECIPE_EDITABLE, RECIPE_CONSUMER from conan.internal.graph.graph import RECIPE_PLATFORM -from conan.internal.graph.graph_error import GraphLoopError, GraphConflictError, GraphMissingError, GraphError +from conan.internal.graph.graph_error import (GraphLoopError, GraphConflictError, GraphMissingError, + GraphError) from conan.internal.graph.profile_node_definer import initialize_conanfile_profile from conan.internal.graph.provides import check_graph_provides from conan.errors import ConanException @@ -17,6 +16,7 @@ from conan.internal.model.pkg_type import PackageType from conan.api.model import RecipeReference from conan.internal.model.requires import Requirement +from conan.internal.model.version_range import VersionRange class DepsGraphBuilder: @@ -67,6 +67,10 @@ def load_graph(self, root_node, profile_host, profile_build, graph_lock=None): except GraphError as e: dep_graph.error = e dep_graph.resolved_ranges = self._resolver.resolved_ranges + refs = set(n.ref for n in dep_graph.nodes + if n.recipe not in (RECIPE_VIRTUAL, RECIPE_EDITABLE, RECIPE_CONSUMER, + RECIPE_PLATFORM)) + self._cache.update_recipes_lru(refs) return dep_graph def _expand_require(self, require, node, graph, profile_host, profile_build, graph_lock): @@ -75,6 +79,7 @@ def _expand_require(self, require, node, graph, profile_host, profile_build, gra # node -(require)-> previous (creates a diamond with a previously existing node) # TODO: allow bootstrapping, use references instead of names # print(" Expanding require ", node, "->", require) + self._deduce_host_version(require, node) previous = node.check_downstream_exists(require) prev_node = None if previous is not None: @@ -134,6 +139,7 @@ def _save_options_conflicts(self, node, require, prev_node, graph): @staticmethod def _conflicting_version(require, node, prev_require, prev_node, prev_ref, base_previous, resolve_prereleases): + # As we are closing a diamond, there can be conflicts. This will raise if so version_range = require.version_range prev_version_range = prev_require.version_range if prev_node is None else None if version_range: @@ -153,35 +159,24 @@ def _conflicting_version(require, node, raise GraphConflictError(node, require, prev_node, prev_require, base_previous) elif prev_version_range is not None: if require.ref.user != prev_require.ref.user or \ - require.ref.channel != prev_require.ref.channel: - raise GraphConflictError(node, require, prev_node, prev_require, base_previous) - if not prev_version_range.contains(require.ref.version, resolve_prereleases): + require.ref.channel != prev_require.ref.channel or \ + not prev_version_range.contains(require.ref.version, resolve_prereleases): raise GraphConflictError(node, require, prev_node, prev_require, base_previous) else: - def _conflicting_refs(ref1, ref2): - ref1_norev = copy.copy(ref1) - ref1_norev.revision = None - ref2_norev = copy.copy(ref2) - ref2_norev.revision = None - if ref2_norev != ref1_norev: - return True - # Computed node, if is Editable, has revision=None - # If new_ref.revision is None we cannot assume any conflict, user hasn't specified - # a revision, so it's ok any previous_ref - if ref1.revision and ref2.revision and ref1.revision != ref2.revision: - return True - - # As we are closing a diamond, there can be conflicts. This will raise if so - conflict = _conflicting_refs(prev_ref, require.ref) - if conflict: # It is possible to get conflict from alias, try to resolve it + if prev_ref != require.ref: raise GraphConflictError(node, require, prev_node, prev_require, base_previous) + # If there is no conflict, then the incomplete require without revision can be updated + # with the previous revision to avoid the later conflict + if prev_ref.revision is not None and require.ref.revision is None: + require.ref.revision = prev_ref.revision @staticmethod def _prepare_node(node, profile_host, profile_build, down_options, define_consumers=False): # basic node configuration: calling configure() and requirements() conanfile, ref = node.conanfile, node.ref - profile_options = profile_host.options if node.context == CONTEXT_HOST else profile_build.options + profile_options = profile_host.options if node.context == CONTEXT_HOST \ + else profile_build.options assert isinstance(profile_options, Options), type(profile_options) run_configure_method(conanfile, down_options, profile_options, ref) @@ -270,7 +265,10 @@ def _resolve_alias(self, node, require, alias, graph): graph.aliased[alias] = pointed_ref # Caching the alias new_req = Requirement(pointed_ref) # FIXME: Ugly temp creation just for alias check alias = new_req.alias - node.conanfile.output.warning("Requirement 'alias' is provided in Conan 2 mainly for compatibility and upgrade from Conan 1, but it is an undocumented and legacy feature. Please update to use standard versioning mechanisms", warn_tag="legacy") + node.conanfile.output.warning("Requirement 'alias' is provided in Conan 2 mainly for " + "compatibility and upgrade from Conan 1, but it is an " + "undocumented and legacy feature. Please update to use " + "standard versioning mechanisms", warn_tag="legacy") def _resolve_recipe(self, ref, graph_lock): result = self._proxy.get_recipe(ref, self._remotes, self._update, self._check_update) @@ -321,8 +319,19 @@ def _resolve_replace_requires(self, node, require, profile_build, profile_host, continue # no match in name if pattern.version != "*": # we need to check versions rrange = require.version_range - valid = rrange.contains(pattern.version, self._resolve_prereleases) if rrange else \ - require.ref.version == pattern.version + # Is the version pattern a range itself? + pversion = repr(pattern.version) + if pversion[0] == "[" and pversion[-1] == "]": + prange = VersionRange(pversion[1:-1]) + if rrange: + valid = prange.intersection(rrange) is not None + else: + valid = prange.contains(require.ref.version, self._resolve_prereleases) + else: + if rrange: + valid = rrange.contains(pattern.version, self._resolve_prereleases) + else: + valid = require.ref.version == pattern.version if not valid: continue if pattern.user != "*" and pattern.user != require.ref.user: @@ -346,7 +355,8 @@ def _resolve_replace_requires(self, node, require, profile_build, profile_host, node.replaced_requires[original_require] = require break # First match executes the alternative and finishes checking others - def _create_new_node(self, node, require, graph, profile_host, profile_build, graph_lock): + @staticmethod + def _deduce_host_version(require, node): require_version = str(require.ref.version) if require_version.startswith(""): if not require.build or require.visible: @@ -365,6 +375,7 @@ def _create_new_node(self, node, require, graph, profile_host, profile_build, gr "host dependency") require.ref.version = transitive.require.ref.version + def _create_new_node(self, node, require, graph, profile_host, profile_build, graph_lock): resolved = self._resolved_system(node, require, profile_build, profile_host, self._resolve_prereleases) if graph_lock is not None: diff --git a/conan/internal/graph/graph_error.py b/conan/internal/graph/graph_error.py index 0bb88f0b04a..e864cd240c0 100644 --- a/conan/internal/graph/graph_error.py +++ b/conan/internal/graph/graph_error.py @@ -57,8 +57,15 @@ def __init__(self, node, require, missing_error): self.require = require self.missing_error = missing_error + def serialize(self): + return {"type": "missing", + "node": {"id": self.node.id, "ref": str(self.node.ref)}, + "require": self.require.serialize(), + "error": self.missing_error} + def __str__(self): - return f"Package '{self.require.ref}' not resolved: {self.missing_error}." + return (f"Package '{self.require.ref}' not resolved: {self.missing_error}. " + f"Required by '{self.node.ref or 'cli'}'") class GraphProvidesError(GraphError): @@ -68,6 +75,14 @@ def __init__(self, node, conflicting_node): self.conflicting_node = conflicting_node node.error = conflicting_node.error + def serialize(self): + return {"type": "provide_conflict", + "node": {"id": self.node.id, "ref": str(self.node.ref)}, + "conflicting_node": {"id": self.conflicting_node.id, + "ref": str(self.conflicting_node.ref)}, + "provided": self.node.conanfile.provides or self.conflicting_node.conanfile.provides} + def __str__(self): + provides = self.node.conanfile.provides or self.conflicting_node.conanfile.provides return f"Provide Conflict: Both '{self.node.ref}' and '{self.conflicting_node.ref}' " \ - f"provide '{self.node.conanfile.provides or self.conflicting_node.conanfile.provides}'." + f"provide '{provides}'." diff --git a/conan/internal/graph/install_graph.py b/conan/internal/graph/install_graph.py index 41879aec86c..dd8d9fffbb7 100644 --- a/conan/internal/graph/install_graph.py +++ b/conan/internal/graph/install_graph.py @@ -397,7 +397,7 @@ def load(filename): def merge(self, other): """ - @type other: InstallGraph + :type other: InstallGraph """ if self.reduced or other.reduced: raise ConanException("Reduced build-order files cannot be merged") diff --git a/conan/internal/graph/installer.py b/conan/internal/graph/installer.py index a2b6d0773fe..4911784ff5c 100644 --- a/conan/internal/graph/installer.py +++ b/conan/internal/graph/installer.py @@ -5,8 +5,9 @@ from conan.api.output import ConanOutput, Color from conan.internal.methods import run_build_method, run_package_method from conan.internal.api.install.generators import write_generators -from conan.internal.graph.graph import BINARY_BUILD, BINARY_CACHE, BINARY_DOWNLOAD, BINARY_EDITABLE, \ - BINARY_UPDATE, BINARY_EDITABLE_BUILD, BINARY_SKIP +from conan.internal.graph.graph import (BINARY_BUILD, BINARY_CACHE, BINARY_DOWNLOAD, + BINARY_EDITABLE, BINARY_UPDATE, BINARY_EDITABLE_BUILD, + BINARY_SKIP) from conan.internal.graph.install_graph import InstallGraph from conan.internal.source import retrieve_exports_sources, config_source from conan.internal.errors import conanfile_remove_attr, conanfile_exception_formatter @@ -14,6 +15,7 @@ from conan.internal.model.cpp_info import CppInfo, MockInfoProperty from conan.api.model import PkgReference from conan.internal.paths import CONANINFO +from conan.internal.util import cpu_count from conan.internal.util.files import clean_dirty, is_dirty, mkdir, rmdir, save, set_dirty, chdir @@ -94,7 +96,8 @@ def _build(self, conanfile, pref): conanfile.output.success("Package '%s' built" % pref.package_id) conanfile.output.info("Build folder %s" % conanfile.build_folder) except Exception as exc: - conanfile.output.error(f"\nPackage '{pref.package_id}' build failed", error_type="exception") + conanfile.output.error(f"\nPackage '{pref.package_id}' build failed", + error_type="exception") conanfile.output.warning("Build folder %s" % conanfile.build_folder) if isinstance(exc, ConanException): raise exc @@ -250,12 +253,17 @@ def install(self, deps_graph, remotes, install_order=None): handled_count = 1 self._download_bulk(install_order) + prefs_lru = [] for level in install_order: for install_reference in level: for package in install_reference.packages.values(): self._install_source(package.nodes[0], remotes) self._handle_package(package, install_reference, handled_count, package_count) handled_count += 1 + if package.binary == BINARY_CACHE: + prefs_lru.append(package.nodes[0].pref) + + self._cache.update_packages_lru(prefs_lru) MockInfoProperty.message() @@ -275,8 +283,9 @@ def _download_bulk(self, install_order): download_count = len(downloads) plural = 's' if download_count != 1 else '' ConanOutput().subtitle(f"Downloading {download_count} package{plural}") - parallel = self._global_conf.get("core.download:parallel", check_type=int) - if parallel is not None: + parallel = self._global_conf.get("core.download:parallel", check_type=int, + default=cpu_count()) + if parallel: # User can define core.download:parallel=0 to deactivate parallelism ConanOutput().info("Downloading binary packages in %s parallel threads" % parallel) thread_pool = ThreadPool(parallel) thread_pool.map(self._download_pkg, downloads) @@ -325,9 +334,9 @@ def _handle_package(self, package, install_reference, handled_count, total_count if package.binary == BINARY_CACHE: node = package.nodes[0] pref = node.pref - self._cache.update_package_lru(pref) assert node.prev, "PREV for %s is None" % str(pref) - node.conanfile.output.success(f'Already installed! ({handled_count} of {total_count})') + msg = f'Already installed! ({handled_count} of {total_count})' + node.conanfile.output.success(msg) # Make sure that all nodes with same pref compute package_info() pkg_folder = package_layout.package() @@ -442,12 +451,20 @@ def _call_package_info(self, conanfile, package_folder, is_editable): # Paste the editable cpp_info but prioritizing it, only if a # variable is not declared at build/source, the package will keep the value - conanfile.buildenv_info.compose_env(conanfile.layouts.source.buildenv_info) - conanfile.buildenv_info.compose_env(conanfile.layouts.build.buildenv_info) - conanfile.runenv_info.compose_env(conanfile.layouts.source.runenv_info) - conanfile.runenv_info.compose_env(conanfile.layouts.build.runenv_info) - conanfile.conf_info.compose_conf(conanfile.layouts.source.conf_info) - conanfile.conf_info.compose_conf(conanfile.layouts.build.conf_info) + full_buildenv_info = conanfile.layouts.source.buildenv_info.copy() + full_buildenv_info.compose_env(conanfile.layouts.build.buildenv_info) + full_buildenv_info.compose_env(conanfile.buildenv_info) + conanfile.buildenv_info = full_buildenv_info + + full_runenv_info = conanfile.layouts.source.runenv_info.copy() + full_runenv_info.compose_env(conanfile.layouts.build.runenv_info) + full_runenv_info.compose_env(conanfile.runenv_info) + conanfile.runenv_info = full_runenv_info + + full_conf_info = conanfile.layouts.source.conf_info.copy() + full_conf_info.compose_conf(conanfile.layouts.build.conf_info) + full_conf_info.compose_conf(conanfile.conf_info) + conanfile.conf_info = full_conf_info else: conanfile.layouts.package.set_relative_base_folder(conanfile.package_folder) conanfile.buildenv_info.compose_env(conanfile.layouts.package.buildenv_info) diff --git a/conan/internal/graph/proxy.py b/conan/internal/graph/proxy.py index 708dc8ac851..f47c11f23e7 100644 --- a/conan/internal/graph/proxy.py +++ b/conan/internal/graph/proxy.py @@ -3,7 +3,7 @@ from conan.internal.graph.graph import (RECIPE_DOWNLOADED, RECIPE_INCACHE, RECIPE_NEWER, RECIPE_NOT_IN_REMOTE, RECIPE_UPDATED, RECIPE_EDITABLE, RECIPE_INCACHE_DATE_UPDATED, RECIPE_UPDATEABLE) -from conan.internal.errors import NotFoundException +from conan.internal.errors import NotFoundException, ConanReferenceAlreadyExistsInDB from conan.errors import ConanException @@ -38,9 +38,11 @@ def _get_recipe(self, reference, remotes, update, check_update): # check if it there's any revision of this recipe in the local cache try: - cache_ref = self._cache.get_latest_recipe_reference(reference) \ - if reference.revision is None else reference - recipe_layout = self._cache.recipe_layout(cache_ref) + # Just do 1 call to the DB, not 2 + if reference.revision is None: + recipe_layout = self._cache.recipe_layout_latest(reference) + else: + recipe_layout = self._cache.recipe_layout(reference) ref = recipe_layout.reference # latest revision if it was not defined except ConanException: # NOT in disk, must be retrieved from remotes @@ -49,8 +51,6 @@ def _get_recipe(self, reference, remotes, update, check_update): status = RECIPE_DOWNLOADED return layout, status, remote - self._cache.update_recipe_lru(ref) - # TODO: cache2.0: check with new --update flows # TODO: If the revision is given, then we don't need to check for updates? if not (check_update or should_update_reference(reference, update)): @@ -65,19 +65,22 @@ def _get_recipe(self, reference, remotes, update, check_update): return recipe_layout, status, None # Something found in remotes, check if we already have the latest in local cache - # TODO: cache2.0 here if we already have a revision in the cache but we add the - # --update argument and we find that same revision in server, we will not - # download anything but we will UPDATE the date of that revision in the - # local cache and WE ARE ALSO UPDATING THE REMOTE - # Check if this is the flow we want to follow assert ref.timestamp cache_time = ref.timestamp if remote_ref.revision != ref.revision: if cache_time < remote_ref.timestamp: # the remote one is newer if should_update_reference(remote_ref, update): - output.info("Retrieving from remote '%s'..." % remote.name) - new_recipe_layout = self._download(remote_ref, remote) + output.info(f"Updating to latest from remote '{remote.name}'...") + try: + new_recipe_layout = self._download(remote_ref, remote) + except ConanReferenceAlreadyExistsInDB: + # When updating to a newer revision in the server, but it already exists + # in the cache with an older timestamp + output.info(f"Latest from '{remote.name}' was found in " + "the cache, using it and updating its timestamp") + new_recipe_layout = self._cache.recipe_layout(remote_ref) + self._cache.update_recipe_timestamp(remote_ref) # make it latest status = RECIPE_UPDATED return new_recipe_layout, status, remote else: @@ -113,13 +116,13 @@ def _find_newest_recipe_in_remotes(self, reference, remotes, update, check_updat ConanProxy.update_policy_legacy_warning = True ConanOutput().warning("The 'core:update_policy' conf is deprecated and will " "be removed in future versions", warn_tag="deprecated") - refs = self._remote_manager.get_recipe_revisions_references(reference, remote) + refs = self._remote_manager.get_recipe_revisions(reference, remote) results.extend([{'remote': remote, 'ref': ref} for ref in refs]) continue if not reference.revision: - ref = self._remote_manager.get_latest_recipe_reference(reference, remote) + ref = self._remote_manager.get_latest_recipe_revision(reference, remote) else: - ref = self._remote_manager.get_recipe_revision_reference(reference, remote) + ref = self._remote_manager.get_recipe_revision(reference, remote) if not need_update: return remote, ref results.append({'remote': remote, 'ref': ref}) diff --git a/conan/internal/graph/python_requires.py b/conan/internal/graph/python_requires.py index 275ac845118..689bbb9f868 100644 --- a/conan/internal/graph/python_requires.py +++ b/conan/internal/graph/python_requires.py @@ -5,7 +5,7 @@ from conan.internal.model.requires import Requirement -class PyRequire(object): +class PyRequire: def __init__(self, module, conanfile, ref, path, recipe_status, remote): self.module = module self.conanfile = conanfile @@ -64,7 +64,7 @@ def add_pyrequire(self, py_require): self._pyrequires[name] = transitive_py_require -class PyRequireLoader(object): +class PyRequireLoader: def __init__(self, conan_app, global_conf): self._proxy = conan_app.proxy self._range_resolver = conan_app.range_resolver diff --git a/conan/internal/graph/range_resolver.py b/conan/internal/graph/range_resolver.py index 21fc6698c8f..f6d3f5795df 100644 --- a/conan/internal/graph/range_resolver.py +++ b/conan/internal/graph/range_resolver.py @@ -1,3 +1,4 @@ +from conan.api.output import ConanOutput from conan.internal.graph.proxy import should_update_reference from conan.errors import ConanException from conan.api.model import RecipeReference @@ -26,6 +27,13 @@ def resolve(self, require, base_conanref, remotes, update): return assert isinstance(version_range, VersionRange) + if require.ref.revision is not None: + ConanOutput(base_conanref).warning("Specifying a revision for requirement " + f"'{require.ref.repr_notime()}' together with " + "a version range has no effect. " + "The revision will be ignored.", + warn_tag="risk") + # Check if this ref with version range was already solved previous_ref = self.resolved_ranges.get(require.ref) if previous_ref is not None: diff --git a/conan/internal/hook_manager.py b/conan/internal/hook_manager.py index f790c5d2827..de833ee68ee 100644 --- a/conan/internal/hook_manager.py +++ b/conan/internal/hook_manager.py @@ -9,7 +9,8 @@ "pre_generate", "post_generate", "pre_build", "post_build", "post_build_fail", "pre_package", "post_package", - "pre_package_info", "post_package_info"] + "pre_package_info", "post_package_info", + "post_package_id"] # package_id is called a lot more than others, only post class HookManager: @@ -17,6 +18,8 @@ class HookManager: def __init__(self, hooks_folder): self._hooks_folder = hooks_folder self.hooks = {} + self.validate_hook = False # Quick check for performance + self.post_package_id_hook = False # Quick check for performance self._load_hooks() # A bit dirty, but avoid breaking tests def execute(self, method_name, conanfile): @@ -32,7 +35,7 @@ def execute(self, method_name, conanfile): conanfile.display_name = "%s: [HOOK - %s] %s()" % (conanfile.display_name, name, method_name) method(conanfile) - except ConanInvalidConfiguration as e: + except ConanInvalidConfiguration: raise except Exception as e: raise ConanException("[HOOK - %s] %s(): %s" % (name, method_name, str(e))) @@ -51,6 +54,8 @@ def _load_hooks(self): # This is difficult to test, apparently in most cases os.walk is alphabetical for name, hook_path in sorted(hooks.items()): self._load_hook(hook_path, name) + self.validate_hook = bool(self.hooks.get("pre_validate") or self.hooks.get("post_validate")) + self.post_package_id_hook = bool(self.hooks.get("post_package_id")) def _load_hook(self, hook_path, hook_name): try: diff --git a/conan/internal/internal_tools.py b/conan/internal/internal_tools.py index ccf594e57ef..2fda60249cb 100644 --- a/conan/internal/internal_tools.py +++ b/conan/internal/internal_tools.py @@ -4,7 +4,8 @@ def is_universal_arch(settings_value, valid_definitions): - if settings_value is None or valid_definitions is None or universal_arch_separator not in settings_value: + if (settings_value is None or valid_definitions is None + or universal_arch_separator not in settings_value): return False parts = settings_value.split(universal_arch_separator) diff --git a/conan/internal/loader.py b/conan/internal/loader.py index f3634ee9a6f..0dccf125a07 100644 --- a/conan/internal/loader.py +++ b/conan/internal/loader.py @@ -63,7 +63,7 @@ def load_basic_module(self, conanfile_path, graph_lock=None, display="", remotes elif tested_python_requires: if getattr(conanfile, "python_requires", None) != "tested_reference_str": ConanOutput().warning("test_package/conanfile.py should declare 'python_requires" - " = \"tested_reference_str\"'", warn_tag="deprecated") + " = \"tested_reference_str\"'", warn_tag="deprecated") conanfile.python_requires = tested_python_requires if self._pyreq_loader: @@ -254,6 +254,7 @@ def load_virtual(self, requires=None, tool_requires=None, python_requires=None, # If user don't specify namespace in options, assume that it is # for the reference (keep compatibility) conanfile = ConanFile(display_name="cli") + conanfile._conan_helpers = self._conanfile_helpers if tool_requires: for reference in tool_requires: diff --git a/conan/internal/methods.py b/conan/internal/methods.py index 841330e1713..82c392e6552 100644 --- a/conan/internal/methods.py +++ b/conan/internal/methods.py @@ -24,6 +24,9 @@ def run_source_method(conanfile, hook_manager): def run_build_method(conanfile, hook_manager): + if os.path.isfile(conanfile.build_folder): + raise ConanException(f"{conanfile}: Failed to create build folder, there is already a file " + f"named: {conanfile.build_folder}") mkdir(conanfile.build_folder) mkdir(conanfile.package_metadata_folder) with chdir(conanfile.build_folder): diff --git a/conan/internal/model/conan_file.py b/conan/internal/model/conan_file.py index 50688b7dde8..37bc95e9c7b 100644 --- a/conan/internal/model/conan_file.py +++ b/conan/internal/model/conan_file.py @@ -1,7 +1,8 @@ import os +import subprocess from pathlib import Path -from conan.api.output import ConanOutput, Color +from conan.api.output import ConanOutput, Color, LEVEL_QUIET from conan.internal.subsystems import command_env_wrapper from conan.errors import ConanException from conan.internal.model.cpp_info import MockInfoProperty @@ -386,8 +387,11 @@ def run(self, command: str, stdout=None, cwd=None, ignore_errors=False, env="", wrapped_cmd = command_env_wrapper(self, command, env, envfiles_folder=envfiles_folder) from conan.internal.util.runners import conan_run if not quiet: - ConanOutput().writeln(f"{self.display_name}: RUN: {command}", fg=Color.BRIGHT_BLUE) + ConanOutput().info(f"{self.display_name}: RUN: {command}", fg=Color.BRIGHT_BLUE) ConanOutput().debug(f"{self.display_name}: Full command: {wrapped_cmd}") + if quiet or ConanOutput.get_output_level() == LEVEL_QUIET: + stdout = subprocess.DEVNULL if stdout is None else stdout + stderr = subprocess.DEVNULL if stderr is None else stderr retcode = conan_run(wrapped_cmd, cwd=cwd, stdout=stdout, stderr=stderr, shell=shell) if not quiet: ConanOutput().writeln("") diff --git a/conan/internal/model/conanconfig.py b/conan/internal/model/conanconfig.py new file mode 100644 index 00000000000..c41950645c6 --- /dev/null +++ b/conan/internal/model/conanconfig.py @@ -0,0 +1,36 @@ +import json + +import yaml + +from conan.api.model import RecipeReference +from conan.errors import ConanException +from conan.internal.util.files import load, save + + +def loadconanconfig(filename): + try: + contents = json.loads(load(filename)) + config_versions = contents["config_version"] + config_versions = [RecipeReference.loads(r) for r in config_versions] + except Exception as e: + raise ConanException(f"Error while loading config file {filename}: {str(e)}") + return config_versions + + +def loadconanconfig_yml(filename): + try: + contents = yaml.safe_load(load(filename)) + config_versions = contents["packages"] + config_versions = [RecipeReference.loads(r) for r in config_versions] + urls = contents.get("urls") + except Exception as e: + raise ConanException(f"Error while loading config file {filename}: {str(e)}") + return config_versions, urls + + +def saveconanconfig(filename, config_versions): + try: + config_versions = [r.repr_notime() for r in config_versions] + save(filename, json.dumps({"config_version": config_versions}, indent=4)) + except Exception as e: + raise ConanException(f"Error while saving config file {filename}: {str(e)}") diff --git a/conan/internal/model/conanfile_interface.py b/conan/internal/model/conanfile_interface.py index 76aed619e1f..b84f0b9c5d3 100644 --- a/conan/internal/model/conanfile_interface.py +++ b/conan/internal/model/conanfile_interface.py @@ -1,4 +1,5 @@ from pathlib import Path + from conan.internal.graph.graph import CONTEXT_BUILD @@ -142,3 +143,12 @@ def url(self): @property def extension_properties(self): return getattr(self._conanfile, "extension_properties", {}) + + @property + def recipe(self) -> str: + # IMPORTANT: this should be used only for "informational" purposes, see GH#18996. + return self._conanfile._conan_node.recipe + + @property + def conf(self): + return self._conanfile.conf diff --git a/conan/internal/model/conf.py b/conan/internal/model/conf.py index 330d3366795..ab59e754d23 100644 --- a/conan/internal/model/conf.py +++ b/conan/internal/model/conf.py @@ -1,4 +1,5 @@ import copy +import hashlib import numbers import platform import re @@ -59,11 +60,16 @@ "core.net.http:cacert_path": "Path containing a custom Cacert file", "core.net.http:client_cert": "Path or tuple of files containing a client cert (and key)", "core.net.http:clean_system_proxy": "If defined, the proxies system env-vars will be discarded", - # Gzip compression + # Compression for `conan upload` + "core.upload:compression_format": "The compression format used when uploading Conan packages. " + "Possible values: 'zst', 'xz', 'gz' (default=gz)", "core.gzip:compresslevel": "The Gzip compression level for Conan artifacts (default=9)", + "core:compresslevel": "The compression level for Conan artifacts (default zstd=3, gz=9)", # Excluded from revision_mode = "scm" dirty and Git().is_dirty() checks "core.scm:excluded": "List of excluded patterns for builtin git dirty checks", "core.scm:local_url": "By default allows to store local folders as remote url, but not upload them. Use 'allow' for allowing upload and 'block' to completely forbid it", + # Compatibility opt-in, to be removed in future versions as optimized behavior becomes default + "core.graph:compatibility_mode": "(Experimental) Set this to 'optimized' to enable the improved compatibility behaviour when querying multiple compatible binaries in remotes", # Tools "tools.android:ndk_path": "Argument for the CMAKE_ANDROID_NDK", "tools.android:cmake_legacy_toolchain": "Define to explicitly pass ANDROID_USE_LEGACY_TOOLCHAIN_FILE in CMake toolchain", @@ -93,6 +99,7 @@ "tools.cmake.cmake_layout:test_folder": "(Experimental) Allow configuring the base folder of the build for test_package", "tools.cmake:cmake_program": "Path to CMake executable", "tools.cmake.cmakedeps:new": "Use the new CMakeDeps generator", + "tools.cmake:ctest_args": "To inject list of arguments to CMake.ctest() runner", "tools.cmake:install_strip": "(Deprecated) Add --strip to cmake.install(). Use tools.build:install_strip instead", "tools.deployer:symlinks": "Set to False to disable deployers copying symlinks", "tools.files.download:retry": "Number of retries in case of failure when downloading", @@ -104,6 +111,7 @@ "tools.graph:skip_build": "(Experimental) Do not expand build/tool_requires", "tools.graph:skip_test": "(Experimental) Do not expand test_requires. If building it might need 'tools.build:skip_test=True'", "tools.gnu:make_program": "Indicate path to make program", + "tools.gnu:disable_flags": "Disable the automatic addition of flags to some build systems. List of possible values: ['arch', 'arch_link', 'libcxx', 'build_type', 'build_type_link', 'threads','cppstd', 'cstd']", "tools.gnu:define_libcxx11_abi": "Force definition of GLIBCXX_USE_CXX11_ABI=1 for libstdc++11", "tools.gnu:pkg_config": "Path to pkg-config executable used by PkgConfig build helper", "tools.gnu:build_triplet": "Custom build triplet to pass to Autotools scripts", @@ -129,11 +137,14 @@ "tools.system.package_manager:mode": "Mode for package_manager tools: 'check', 'report', 'report-installed' or 'install'", "tools.system.package_manager:sudo": "Use 'sudo' when invoking the package manager tools in Linux (False by default)", "tools.system.package_manager:sudo_askpass": "Use the '-A' argument if using sudo in Linux to invoke the system package manager (False by default)", + "tools.system.pipenv:python_interpreter": "Path to the Python interpreter to be used to create the virtualenv", "tools.apple:sdk_path": "Path to the SDK to be used", "tools.apple:enable_bitcode": "(boolean) Enable/Disable Bitcode Apple Clang flags", "tools.apple:enable_arc": "(boolean) Enable/Disable ARC Apple Clang flags", "tools.apple:enable_visibility": "(boolean) Enable/Disable Visibility Apple Clang flags", "tools.env.virtualenv:powershell": "If specified, it generates PowerShell launchers (.ps1). Use this configuration setting the PowerShell executable you want to use (e.g., 'powershell.exe' or 'pwsh'). Setting it to True or False is deprecated as of Conan 2.11.0.", + "tools.env:dotenv": "(Experimental) Generate dotenv environment files", + "tools.env:deactivation_mode": "(Experimental) If 'function', generate a deactivate function instead of a script to unset the environment variables", # Compilers/Flags configurations "tools.build:compiler_executables": "Defines a Python dict-like with the compilers path to be used. Allowed keys {'c', 'cpp', 'cuda', 'objc', 'objcxx', 'rc', 'fortran', 'asm', 'hip', 'ispc'}", "tools.build:cxxflags": "List of extra CXX flags used by different toolchains like CMakeToolchain, AutotoolsToolchain and MesonToolchain", @@ -143,7 +154,7 @@ "tools.build:exelinkflags": "List of extra flags used by different toolchains like CMakeToolchain, AutotoolsToolchain and MesonToolchain", "tools.build:linker_scripts": "List of linker script files to pass to the linker used by different toolchains like CMakeToolchain, AutotoolsToolchain, and MesonToolchain", # Toolchain installation - "tools.build:install_strip": "(boolean) Strip the binaries when installing them with CMake and Meson", + "tools.build:install_strip": "(boolean) Strip the binaries when installing them with CMake, Meson and Autotools", # Package ID composition "tools.info.package_id:confs": "List of existing configuration to be part of the package ID", } @@ -500,13 +511,6 @@ def compose_conf(self, other): existing.compose_conf_value(v) return self - def filter_user_modules(self): - result = Conf() - for k, v in self._values.items(): - if _is_profile_module(k): - result._values[k] = v - return result - def copy_conaninfo_conf(self): """ Get a new `Conf()` object with all the configurations required by the consumer @@ -756,7 +760,8 @@ def load_global_conf(home_folder): content = template.render({"platform": platform, "os": os, "distro": distro, "conan_version": conan_version, "conan_home_folder": home_folder, - "detect_api": detect_api}) + "detect_api": detect_api, + "hashlib": hashlib}) new_config.loads(content) else: # creation of a blank global.conf file for user convenience default_global_conf = textwrap.dedent("""\ diff --git a/conan/internal/model/cpp_info.py b/conan/internal/model/cpp_info.py index 3d6fab03f9a..4a2fd191070 100644 --- a/conan/internal/model/cpp_info.py +++ b/conan/internal/model/cpp_info.py @@ -648,7 +648,7 @@ def deduce_locations(self, conanfile, component_name=""): "cannot deduce locations") # fully defined by user in conanfile, nothing to do. if self._location or self._link_location: - if self._type not in [PackageType.SHARED, PackageType.STATIC]: + if self._type is None or self._type not in [PackageType.SHARED, PackageType.STATIC]: raise ConanException(f"{name} location defined without defined library type") return diff --git a/conan/internal/model/dependencies.py b/conan/internal/model/dependencies.py index b830e04a0bb..f4f58acb05e 100644 --- a/conan/internal/model/dependencies.py +++ b/conan/internal/model/dependencies.py @@ -1,12 +1,12 @@ from collections import OrderedDict -from conan.internal.graph.graph import RECIPE_PLATFORM -from conan.errors import ConanException from conan.api.model import RecipeReference +from conan.errors import ConanException +from conan.internal.graph.graph import RECIPE_PLATFORM from conan.internal.model.conanfile_interface import ConanFileInterface -class UserRequirementsDict(object): +class UserRequirementsDict: """ user facing dict to allow access of dependencies by name """ def __init__(self, data, require_filter=None): @@ -117,9 +117,7 @@ def filter_fn(require): data = OrderedDict((k, v) for k, v in self._data.items() if filter_fn(k)) if remove_system: - data = OrderedDict((k, v) for k, v in data.items() - # TODO: Make "recipe" part of ConanFileInterface model - if v._conanfile._conan_node.recipe != RECIPE_PLATFORM) + data = OrderedDict((k, v) for k, v in data.items() if v.recipe != RECIPE_PLATFORM) return ConanFileDependencies(data, require_filter) def transitive_requires(self, other): diff --git a/conan/internal/model/info.py b/conan/internal/model/info.py index e5f59d8458d..b7d30660d47 100644 --- a/conan/internal/model/info.py +++ b/conan/internal/model/info.py @@ -20,11 +20,15 @@ def stable(self): return self.major() def major(self): + # This check is to avoid breaking non-integer major versions + # for legacy reasons. Users are warned against using them if not isinstance(self._version.major.value, int): return str(self._version.major) return ".".join([str(self._version.major), 'Y', 'Z']) def minor(self): + # This check is to avoid breaking non-integer major versions + # for legacy reasons. Users are warned against using them if not isinstance(self._version.major.value, int): return str(self._version.major) @@ -33,6 +37,8 @@ def minor(self): return ".".join([v0, v1, 'Z']) def patch(self): + # This check is to avoid breaking non-integer major versions + # for legacy reasons. Users are warned against using them if not isinstance(self._version.major.value, int): return str(self._version.major) @@ -42,6 +48,8 @@ def patch(self): return ".".join([v0, v1, v2]) def pre(self): + # This check is to avoid breaking non-integer major versions + # for legacy reasons. Users are warned against using them if not isinstance(self._version.major.value, int): return str(self._version.major) diff --git a/conan/internal/model/layout.py b/conan/internal/model/layout.py index f03c42efdb3..f2c642354f2 100644 --- a/conan/internal/model/layout.py +++ b/conan/internal/model/layout.py @@ -4,7 +4,7 @@ from conan.internal.model.conf import Conf -class Infos(object): +class Infos: def __init__(self): self.source = CppInfo() @@ -12,7 +12,7 @@ def __init__(self): self.package = CppInfo(set_defaults=True) -class PartialLayout(object): +class PartialLayout: def __init__(self): from conan.tools.env import Environment self.buildenv_info = Environment() @@ -25,7 +25,7 @@ def set_relative_base_folder(self, folder): self.conf_info.set_relative_base_folder(folder) -class Layouts(object): +class Layouts: def __init__(self): self.source = PartialLayout() self.build = PartialLayout() diff --git a/conan/internal/model/lockfile.py b/conan/internal/model/lockfile.py index 5a397907066..9bb93ac9c4f 100644 --- a/conan/internal/model/lockfile.py +++ b/conan/internal/model/lockfile.py @@ -115,7 +115,7 @@ def merge(self, other): self.sort() -class Lockfile(object): +class Lockfile: def __init__(self, deps_graph=None, lock_packages=False): self._requires = _LockRequires() @@ -176,7 +176,7 @@ def dumps(self): return json.dumps(self.serialize(), indent=4) def save(self, path): - save(path, self.dumps()) + save(path, self.dumps() + "\n") def merge(self, other): """ diff --git a/conan/internal/model/manifest.py b/conan/internal/model/manifest.py index 849f40339cc..60173f751c7 100644 --- a/conan/internal/model/manifest.py +++ b/conan/internal/model/manifest.py @@ -1,12 +1,13 @@ import os from collections import defaultdict -from conan.internal.paths import CONAN_MANIFEST, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME +from conan.internal.paths import CONAN_MANIFEST, COMPRESSIONS, PACKAGE_FILE_NAME, EXPORT_FILE_NAME, \ + EXPORT_SOURCES_FILE_NAME from conan.internal.util.dates import timestamp_now, timestamp_to_str from conan.internal.util.files import load, md5, md5sum, save, gather_files -class FileTreeManifest(object): +class FileTreeManifest: def __init__(self, the_time, file_sums): """file_sums is a dict with filepaths and md5's: {filepath/to/file.txt: md5}""" @@ -91,8 +92,10 @@ def create(cls, folder, exports_sources_folder=None): """ files, _ = gather_files(folder) # The folders symlinks are discarded for the manifest - for f in (PACKAGE_TGZ_NAME, EXPORT_TGZ_NAME, CONAN_MANIFEST, EXPORT_SOURCES_TGZ_NAME): - files.pop(f, None) + for f in (PACKAGE_FILE_NAME, EXPORT_FILE_NAME, EXPORT_SOURCES_FILE_NAME): + for e in COMPRESSIONS: + files.pop(f + e, None) + files.pop(CONAN_MANIFEST, None) file_dict = {} for name, filepath in files.items(): diff --git a/conan/internal/model/options.py b/conan/internal/model/options.py index 99aff017144..9d83e4d7dd8 100644 --- a/conan/internal/model/options.py +++ b/conan/internal/model/options.py @@ -242,6 +242,11 @@ def __init__(self, options=None, options_values=None): "deprecated, use a pattern like `{}/*:{}` " \ "instead".format(k, package, option) raise ConanException(msg) + if "[" in package: + msg = (f"Options pattern {package} contains a version range, which has no effect. " + f"Only '&' for consumer and '*' as wildcard are supported in this context.") + from conan.api.output import ConanOutput + ConanOutput().warning(msg, warn_tag="risk") self._deps_package_options.setdefault(package, _PackageOptions())[option] = v else: self._package_options[k] = v diff --git a/conan/internal/model/profile.py b/conan/internal/model/profile.py index 7f4e37c98c8..c9f95b4da97 100644 --- a/conan/internal/model/profile.py +++ b/conan/internal/model/profile.py @@ -154,7 +154,8 @@ def compose_profile(self, other): runner_type = self.runner.get("type") other_runner_type = other.runner.get("type") if runner_type and other_runner_type and runner_type != other_runner_type: - raise ConanException(f"Found different runner types in profile composition ({runner_type} and {other_runner_type})") + raise ConanException(f"Found different runner types in profile composition " + f"({runner_type} and {other_runner_type})") self.runner.update(other.runner) current_platform_tool_requires = {r.name: r for r in self.platform_tool_requires} @@ -172,7 +173,7 @@ def update_settings(self, new_settings): """Mix the specified settings with the current profile. Specified settings are prioritized to profile""" - assert(isinstance(new_settings, OrderedDict)) + assert isinstance(new_settings, OrderedDict) # apply the current profile res = copy.copy(self.settings) diff --git a/conan/internal/model/requires.py b/conan/internal/model/requires.py index 4a02d9e6847..9ca88535232 100644 --- a/conan/internal/model/requires.py +++ b/conan/internal/model/requires.py @@ -304,7 +304,9 @@ def transform_downstream(self, pkg_type, require, dep_pkg_type): elif pkg_type is PackageType.HEADER: downstream_require = Requirement(require.ref, headers=require.headers, libs=require.libs, run=require.run) else: - assert pkg_type == PackageType.UNKNOWN + if pkg_type != PackageType.UNKNOWN: + raise ConanException(f"Package '{self.ref}' with type '{pkg_type}' cannot have " + f"a '{dep_pkg_type}' dependency to '{require.ref}'") # TODO: This is undertested, changing it did not break tests downstream_require = require.copy_requirement() elif dep_pkg_type is PackageType.HEADER: diff --git a/conan/internal/model/workspace.py b/conan/internal/model/workspace.py index 38c12a34821..41c793ee4c0 100644 --- a/conan/internal/model/workspace.py +++ b/conan/internal/model/workspace.py @@ -5,6 +5,7 @@ from conan.api.output import ConanOutput from conan.errors import ConanException +from conan.internal.errors import scoped_traceback from conan.internal.util.files import load, save # Related folder @@ -23,7 +24,24 @@ def __init__(self, folder, conan_api): self.folder = folder self.conan_data = self._conan_load_data() self._conan_api = conan_api - self.output = ConanOutput(scope=self.name()) + self.output = ConanOutput(scope=f"Workspace '{self.name()}'") + + def __getattribute__(self, item): + # Return a protected wrapper around workspace overridable callables in order to + # be able to have clean errors if user errors in conanws.py code + myattr = object.__getattribute__(self, item) + if item not in ("name", "packages", "add", "remove", "clean", "build_order"): + return myattr + + def wrapper(*args, **kwargs): + try: + return myattr(*args, **kwargs) + except ConanException: + raise + except Exception as e: + m = scoped_traceback(f"Error in {item}() method", e, scope="conanws.py") + raise ConanException(f"Workspace conanws.py file: {m}") + return wrapper def name(self): return self.conan_data.get("name") or os.path.basename(self.folder) @@ -39,7 +57,8 @@ def _conan_load_data(self): return data or {} def add(self, ref, path, output_folder): - assert os.path.isfile(path) + if not path or not os.path.isfile(path): + raise ConanException(f"Cannot add to workspace. File not found: {path}") path = self._conan_rel_path(os.path.dirname(path)) editable = { "path": path, @@ -48,16 +67,19 @@ def add(self, ref, path, output_folder): if output_folder: editable["output_folder"] = self._conan_rel_path(output_folder) packages = self.conan_data.setdefault("packages", []) - if any(p["path"] == path for p in packages): - self.output.warning(f"Package {path} already exists, skipping") - return - packages.append(editable) + for p in packages: + if p["path"] == path: + self.output.warning(f"Package {path} already exists, updating its reference") + p["ref"] = editable["ref"] + break + else: + packages.append(editable) save(os.path.join(self.folder, WORKSPACE_YML), yaml.dump(self.conan_data)) def remove(self, path): path = self._conan_rel_path(path) - package_found = next(package_info for package_info in self.conan_data.get("packages", []) - if package_info["path"].replace("\\", "/") == path) + package_found = next((package_info for package_info in self.conan_data.get("packages", []) + if package_info["path"].replace("\\", "/") == path), None) if not package_found: raise ConanException(f"No editable package to remove from this path: {path}") self.conan_data["packages"].remove(package_found) @@ -104,3 +126,10 @@ def load_conanfile(self, conanfile_path): def root_conanfile(self): # noqa return None + + def build_order(self, order): # noqa + msg = ["Packages build order:"] + for level in order: + for item in level: + msg.append(f" {item['ref']}: {item['folder']}") + self.output.info("\n".join(msg)) diff --git a/conan/internal/paths.py b/conan/internal/paths.py index 2d929734694..adc7d0dfa6f 100644 --- a/conan/internal/paths.py +++ b/conan/internal/paths.py @@ -48,6 +48,7 @@ def find_file_walk_up(start, filename, end=None): path = path.parent return None + def get_conan_user_home(): def _user_home_from_conanrc_file(): @@ -85,7 +86,8 @@ def _user_home_from_conanrc_file(): CONANFILE_TXT = "conanfile.txt" CONAN_MANIFEST = "conanmanifest.txt" CONANINFO = "conaninfo.txt" -PACKAGE_TGZ_NAME = "conan_package.tgz" -EXPORT_TGZ_NAME = "conan_export.tgz" -EXPORT_SOURCES_TGZ_NAME = "conan_sources.tgz" +PACKAGE_FILE_NAME = "conan_package.t" +EXPORT_FILE_NAME = "conan_export.t" +EXPORT_SOURCES_FILE_NAME = "conan_sources.t" +COMPRESSIONS = "gz", "xz", "zst" DATA_YML = "conandata.yml" diff --git a/conan/internal/rest/caching_file_downloader.py b/conan/internal/rest/caching_file_downloader.py index 5061d27c565..b2b50a8522b 100644 --- a/conan/internal/rest/caching_file_downloader.py +++ b/conan/internal/rest/caching_file_downloader.py @@ -69,7 +69,6 @@ def _caching_download(self, urls, file_path, raise ConanException("Trying to download sources from None backup remote." f" Remotes were: {backups_urls}") for backup_url in backups_urls: - is_last = backup_url is backups_urls[-1] if backup_url == "origin": # recipe defined URLs if self._origin_download(urls, cached_path, retry, retry_wait, diff --git a/conan/internal/rest/client_routes.py b/conan/internal/rest/client_routes.py index e7de1ba8c28..8bfd8436668 100644 --- a/conan/internal/rest/client_routes.py +++ b/conan/internal/rest/client_routes.py @@ -42,12 +42,15 @@ def search(self, pattern, ignorecase): query = "?%s" % urlencode(params) return self.base_url + "%s%s" % (self.routes.common_search, query) - def search_packages(self, ref): + def search_packages(self, ref, list_only): """URL search packages for a recipe""" route = self.routes.common_search_packages_revision \ if ref.revision else self.routes.common_search_packages url = _format_ref(route, ref) - return self.base_url + url + + params = {"list_only": list_only} + query = "?%s" % urlencode(params) + return self.base_url + url + query def common_authenticate(self): return self.base_url + self.routes.common_authenticate diff --git a/conan/internal/rest/conan_requester.py b/conan/internal/rest/conan_requester.py index 4c9d3b24734..6d00e2ef254 100644 --- a/conan/internal/rest/conan_requester.py +++ b/conan/internal/rest/conan_requester.py @@ -45,19 +45,9 @@ def __init__(self, cache_folder): return def _get_auth(credentials): - result = {} - has_auth = False - if "token" in credentials: - result["token"] = credentials["token"] - has_auth = True - if "user" in credentials and "password" in credentials: - result["user"] = credentials["user"] - result["password"] = credentials["password"] - has_auth = True - if has_auth: - return result - else: - raise ConanException(f"Unknown credentials method for '{credentials['url']}'") + if "token" in credentials or ("user" in credentials and "password" in credentials): + return credentials + raise ConanException(f"Unknown credentials method for '{credentials['url']}'") try: template = Template(load(creds_path)) @@ -82,6 +72,8 @@ def add_auth(self, url, kwargs): kwargs["headers"]["Authorization"] = f"Bearer {c.get('token')}" if c.get("user") and c.get("password"): kwargs["auth"] = (c.get("user"), c.get("password")) + if c.get("headers"): + kwargs.setdefault("headers", {}).update(c["headers"]) return # Then, try to find the credentials in "_urls" @@ -94,6 +86,9 @@ def add_auth(self, url, kwargs): password = creds.get("password") if user and password: kwargs["auth"] = (user, password) + headers = creds.get("headers") + if headers: + kwargs.setdefault("headers", {}).update(headers) break diff --git a/conan/internal/rest/download_cache.py b/conan/internal/rest/download_cache.py index 9d9c916fbdc..5e366084470 100644 --- a/conan/internal/rest/download_cache.py +++ b/conan/internal/rest/download_cache.py @@ -70,15 +70,13 @@ def has_excluded_urls(backup_urls): for excluded_url in excluded_urls) for url in backup_urls) - def should_upload_sources(package): - return any(prev.get("upload") for prev in package["revisions"].values()) - all_refs = set() if package_list is not None: - for k, ref in package_list.refs().items(): - packages = ref.get("packages", {}).values() - if not only_upload or ref.get("upload") or any(should_upload_sources(p) for p in packages): - all_refs.add(str(k)) + for ref, packages in package_list.items(): + ref_info = package_list.recipe_dict(ref) + if (not only_upload or ref_info.get("upload") + or any(package_list.package_dict(p).get("upload") for p in packages)): + all_refs.add(str(ref)) path_backups_contents = [] @@ -87,7 +85,9 @@ def should_upload_sources(package): if remove_if_dirty(os.path.join(path_backups, path)): continue if path.endswith(dirty_ext): - # TODO: Clear the dirty file marker if it does not have a matching downloaded file + if not os.path.exists(os.path.join(path_backups, os.path.splitext(path)[0])): + if os.path.exists(os.path.join(path_backups, path)): + os.remove(os.path.join(path_backups, path)) continue if not path.endswith(".json"): path_backups_contents.append(path) diff --git a/conan/internal/rest/file_downloader.py b/conan/internal/rest/file_downloader.py index 9adccca81e2..a61b956b825 100644 --- a/conan/internal/rest/file_downloader.py +++ b/conan/internal/rest/file_downloader.py @@ -5,8 +5,8 @@ from conan.api.output import ConanOutput, TimedOutput from conan.internal.rest import response_to_str -from conan.internal.errors import ConanConnectionError, RequestErrorException, AuthenticationException, \ - ForbiddenException, NotFoundException +from conan.internal.errors import (ConanConnectionError, RequestErrorException, + AuthenticationException, ForbiddenException, NotFoundException) from conan.errors import ConanException from conan.internal.util.files import human_size, check_with_algorithm_sum diff --git a/conan/internal/rest/file_uploader.py b/conan/internal/rest/file_uploader.py index 90b2d8e27ce..3659937d3c7 100644 --- a/conan/internal/rest/file_uploader.py +++ b/conan/internal/rest/file_uploader.py @@ -10,7 +10,7 @@ from conan.internal.util.files import sha1sum -class FileUploader(object): +class FileUploader: def __init__(self, requester, verify, config, source_credentials=None): self._requester = requester diff --git a/conan/internal/rest/pkg_sign.py b/conan/internal/rest/pkg_sign.py index baa8e8c72cc..96d1eb1072a 100644 --- a/conan/internal/rest/pkg_sign.py +++ b/conan/internal/rest/pkg_sign.py @@ -29,10 +29,12 @@ def _sign(ref, files, folder): for f in os.listdir(metadata_sign): files[f"{METADATA}/sign/{f}"] = os.path.join(metadata_sign, f) - for rref, recipe_bundle in upload_data.refs().items(): + for rref, packages in upload_data.items(): + recipe_bundle = upload_data.recipe_dict(rref) if recipe_bundle["upload"]: _sign(rref, recipe_bundle["files"], self._cache.recipe_layout(rref).download_export()) - for pref, pkg_bundle in upload_data.prefs(rref, recipe_bundle).items(): + for pref in packages: + pkg_bundle = upload_data.package_dict(pref) if pkg_bundle["upload"]: _sign(pref, pkg_bundle["files"], self._cache.pkg_layout(pref).download_package()) diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index 418f6a2ddcc..78e8b47c06e 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -1,11 +1,14 @@ import os import shutil +import sys + from collections import namedtuple from typing import List from requests.exceptions import ConnectionError from conan.api.model import LOCAL_RECIPES_INDEX +from conan.internal.paths import CONANINFO, CONAN_MANIFEST, PACKAGE_FILE_NAME, EXPORT_FILE_NAME from conan.internal.rest.rest_client_local_recipe_index import RestApiClientLocalRecipesIndex from conan.api.model import Remote from conan.api.output import ConanOutput @@ -17,7 +20,6 @@ from conan.api.model import PkgReference from conan.api.model import RecipeReference from conan.internal.util.files import rmdir, human_size -from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME from conan.internal.util.files import mkdir, tar_extract @@ -62,6 +64,7 @@ def get_recipe(self, ref, remote, metadata=None): local_folder_remote = self._local_folder_remote(remote) if local_folder_remote is not None: local_folder_remote.get_recipe(ref, export_folder) + mkdir(layout.metadata()) return layout download_export = layout.download_export() @@ -81,11 +84,13 @@ def get_recipe(self, ref, remote, metadata=None): f"no conanmanifest.txt") self._signer.verify(ref, download_export, files=zipped_files) except BaseException: # So KeyboardInterrupt also cleans things - ConanOutput(scope=str(ref)).error(f"Error downloading from remote '{remote.name}'", error_type="exception") + ConanOutput(scope=str(ref)).error(f"Error downloading from remote '{remote.name}'", + error_type="exception") self._cache.remove_recipe_layout(layout) raise export_folder = layout.export() - tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None) + export_file = next((f for f in zipped_files if f.startswith(EXPORT_FILE_NAME)), None) + tgz_file = zipped_files.pop(export_file, None) if tgz_file: uncompress_file(tgz_file, export_folder, scope=str(ref), conan_api=self._conan_api) @@ -95,6 +100,7 @@ def get_recipe(self, ref, remote, metadata=None): # Make sure that the source dir is deleted rmdir(layout.source()) + mkdir(layout.metadata()) return layout def get_recipe_metadata(self, ref, remote, metadata): @@ -110,7 +116,8 @@ def get_recipe_metadata(self, ref, remote, metadata): self._call_remote(remote, "get_recipe", ref, download_export, metadata, only_metadata=True) except BaseException: # So KeyboardInterrupt also cleans things - output.error(f"Error downloading metadata from remote '{remote.name}'", error_type="exception") + output.error(f"Error downloading metadata from remote '{remote.name}'", + error_type="exception") raise def get_recipe_sources(self, ref, layout, remote): @@ -129,8 +136,14 @@ def get_recipe_sources(self, ref, layout, remote): return self._signer.verify(ref, download_folder, files=zipped_files) +<<<<<<< HEAD tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME] uncompress_file(tgz_file, export_sources_folder, scope=str(ref), conan_api=self._conan_api) +======= + # Only 1 file is guaranteed + tgz_file = next(iter(zipped_files.values())) + uncompress_file(tgz_file, export_sources_folder, scope=str(ref)) +>>>>>>> feature/builtin_compression def get_package(self, pref, remote, metadata=None): output = ConanOutput(scope=str(pref.ref)) @@ -140,6 +153,7 @@ def get_package(self, pref, remote, metadata=None): pkg_layout = self._cache.create_pkg_layout(pref) with pkg_layout.set_dirty_context_manager(): + mkdir(pkg_layout.metadata()) self._get_package(pkg_layout, pref, remote, output, metadata) def get_package_metadata(self, pref, remote, metadata): @@ -157,13 +171,16 @@ def get_package_metadata(self, pref, remote, metadata): self._call_remote(remote, "get_package", pref, download_pkg_folder, metadata, only_metadata=True) except BaseException as e: # So KeyboardInterrupt also cleans things - output.error(f"Exception while getting package metadata: {str(pref.package_id)}", error_type="exception") + output.error(f"Exception while getting package metadata: {str(pref.package_id)}", + error_type="exception") output.error(f"Exception: {type(e)} {str(e)}", error_type="exception") raise def _get_package(self, layout, pref, remote, scoped_output, metadata): try: assert pref.revision is not None + if remote.recipes_only: + raise NotFoundException(f"Remote '{remote.name}' doesn't allow binary downloads") download_pkg_folder = layout.download_package() # Download files to the pkg_tgz folder, not to the final one @@ -171,12 +188,15 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata): metadata, only_metadata=False) zipped_files = {k: v for k, v in zipped_files.items() if not k.startswith(METADATA)} # quick server package integrity check: - for f in ("conaninfo.txt", "conanmanifest.txt", "conan_package.tgz"): + for f in (CONANINFO, CONAN_MANIFEST): if f not in zipped_files: raise ConanException(f"Corrupted {pref} in '{remote.name}' remote: no {f}") + + # This is guaranteed to exists, otherwise RestClient would have raised already + package_file = next(f for f in zipped_files if PACKAGE_FILE_NAME in f) self._signer.verify(pref, download_pkg_folder, zipped_files) - tgz_file = zipped_files.pop(PACKAGE_TGZ_NAME, None) + tgz_file = zipped_files.pop(package_file) package_folder = layout.package() uncompress_file(tgz_file, package_folder, scope=str(pref.ref), conan_api=self._conan_api) mkdir(package_folder) # Just in case it doesn't exist, because uncompress did nothing @@ -189,11 +209,13 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata): raise PackageNotFoundException(pref) except BaseException as e: # So KeyboardInterrupt also cleans things self._cache.remove_package_layout(layout) - scoped_output.error(f"Exception while getting package: {str(pref.package_id)}", error_type="exception") + scoped_output.error(f"Exception while getting package: {str(pref.package_id)}", + error_type="exception") scoped_output.error(f"Exception: {type(e)} {str(e)}", error_type="exception") raise def search_recipes(self, remote, pattern): + # Used by ListAPI to "conan list *" recipes, and by RangeResolver to resolve version-ranges cached_method = remote._caching.setdefault("search_recipes", {}) try: return cached_method[pattern] @@ -202,12 +224,18 @@ def search_recipes(self, remote, pattern): cached_method[pattern] = result return result - def search_packages(self, remote, ref): - packages = self._call_remote(remote, "search_packages", ref) - # Avoid serializing conaninfo in server side - packages = {PkgReference(ref, pid): load_binary_info(data["content"]) - if "content" in data else data - for pid, data in packages.items() if not data.get("recipe_hash")} + def search_packages(self, remote, ref, list_only=False): + # Used only by ListAPI to list the different package_ids for a reference + if remote.recipes_only: + return {} + packages = self._call_remote(remote, "search_packages", ref, list_only) + if list_only: + packages = {PkgReference(ref, pid): None for pid, data in packages.items()} + else: + # Avoid serializing conaninfo in server side + packages = {PkgReference(ref, pid): load_binary_info(data["content"]) + if "content" in data else data + for pid, data in packages.items() if not data.get("recipe_hash")} return packages def remove_recipe(self, ref, remote): @@ -225,20 +253,45 @@ def remove_all_packages(self, ref, remote): def authenticate(self, remote, name, password): return self._call_remote(remote, 'authenticate', name, password, enforce_disabled=False) - def get_recipe_revisions_references(self, ref, remote): + def get_recipe_revisions(self, ref: RecipeReference, remote: Remote) -> List[RecipeReference]: + # Used by ListAPI to list recipe revisions for a ref without revision + # and by ConanProxy resolving legacy_update Conan 1 logic assert ref.revision is None, "get_recipe_revisions_references of a reference with revision" return self._call_remote(remote, "get_recipe_revisions_references", ref) - def get_package_revisions_references(self, pref, remote, headers=None) -> List[PkgReference]: - assert pref.revision is None, "get_package_revisions_references of a reference with revision" - return self._call_remote(remote, "get_package_revisions_references", pref, headers=headers) + def get_recipe_revision(self, ref: RecipeReference, remote: Remote) -> RecipeReference: + # Used by UploadUpstreamChecker to see if the revision exist in the server + # Used by Download, to get timestamp from server and respect it + # Used by ConanProxy to confirm existence of specific revision + assert ref.revision is not None, "recipe_exists needs a revision" + return self._call_remote(remote, "get_recipe_revision_reference", ref) - def get_latest_recipe_reference(self, ref, remote): + def get_latest_recipe_revision(self, ref: RecipeReference, remote: Remote) -> RecipeReference: + # Used by ListAPI to retrieve the latest revision + # Used by ConanProxy to resolve to the latest revision assert ref.revision is None, "get_latest_recipe_reference of a reference with revision" return self._call_remote(remote, "get_latest_recipe_reference", ref) - def get_latest_package_reference(self, pref, remote, info=None) -> PkgReference: + def get_package_revisions(self, pref: PkgReference, remote: Remote) -> List[PkgReference]: + # Used by ListAPI to retrieve multiple package revisions + assert pref.revision is None, "get_package_revisions_references of a reference with revision" + if remote.recipes_only: + return [] + return self._call_remote(remote, "get_package_revisions_references", pref) + + def get_package_revision(self, pref: PkgReference, remote: Remote) -> PkgReference: + # Used by UploadUpstreamChecker to see if the revision exist in the server + # Used by Download, to get timestamp from server and respect it + assert pref.revision is not None, "get_package_revision_reference needs a revision" + return self._call_remote(remote, "get_package_revision_reference", pref) + + def get_latest_package_revision(self, pref: PkgReference, remote: Remote, + info=None) -> PkgReference: + # Used by List to resolve the latest package revision + # Used by GraphBinariesAnalyzer to resolve to latest package revision assert pref.revision is None, "get_latest_package_reference of a reference with revision" + if remote.recipes_only: + raise NotFoundException(f"Remote '{remote.name}' doesn't allow binary downloads") # These headers are useful to know what configurations are being requested in the server headers = None if info: @@ -272,14 +325,6 @@ def get_latest_package_reference(self, pref, remote, info=None) -> PkgReference: raise NotFoundException(result.message) return result - def get_recipe_revision_reference(self, ref, remote) -> bool: - assert ref.revision is not None, "recipe_exists needs a revision" - return self._call_remote(remote, "get_recipe_revision_reference", ref) - - def get_package_revision_reference(self, pref, remote) -> bool: - assert pref.revision is not None, "get_package_revision_reference needs a revision" - return self._call_remote(remote, "get_package_revision_reference", pref) - def _call_remote(self, remote, method, *args, **kwargs): assert (isinstance(remote, Remote)) enforce_disabled = kwargs.pop("enforce_disabled", True) @@ -304,7 +349,14 @@ def _call_remote(self, remote, method, *args, **kwargs): raise ConanException(exc, remote=remote) +<<<<<<< HEAD def uncompress_file(src_path, dest_folder, scope="", conan_api=None): +======= +def uncompress_file(src_path, dest_folder, scope=None): + if sys.version_info.minor < 14 and src_path.endswith("zst"): + raise ConanException(f"File {os.path.basename(src_path)} compressed with 'zst', " + f"unsupported for Python<3.14 ") +>>>>>>> feature/builtin_compression try: filesize = os.path.getsize(src_path) big_file = filesize > 10000000 # 10 MB diff --git a/conan/internal/rest/rest_client.py b/conan/internal/rest/rest_client.py index 678190cfc7e..b8006572e05 100644 --- a/conan/internal/rest/rest_client.py +++ b/conan/internal/rest/rest_client.py @@ -68,8 +68,8 @@ def check_credentials(self, force_auth=False): def search(self, pattern=None, ignorecase=True): return self._get_api().search(pattern, ignorecase) - def search_packages(self, reference): - return self._get_api().search_packages(reference) + def search_packages(self, reference, list_only=False): + return self._get_api().search_packages(reference, list_only) def remove_recipe(self, ref): return self._get_api().remove_recipe(ref) @@ -83,8 +83,8 @@ def remove_packages(self, prefs): def get_recipe_revisions_references(self, ref): return self._get_api().get_recipe_revisions_references(ref) - def get_package_revisions_references(self, pref, headers=None): - return self._get_api().get_package_revisions_references(pref, headers=headers) + def get_package_revisions_references(self, pref): + return self._get_api().get_package_revisions_references(pref) def get_latest_recipe_reference(self, ref): return self._get_api().get_latest_recipe_reference(ref) diff --git a/conan/internal/rest/rest_client_local_recipe_index.py b/conan/internal/rest/rest_client_local_recipe_index.py index 5b37dceb81c..509378fefa8 100644 --- a/conan/internal/rest/rest_client_local_recipe_index.py +++ b/conan/internal/rest/rest_client_local_recipe_index.py @@ -12,7 +12,7 @@ from conan.internal.api.export import cmd_export from conan.internal.hook_manager import HookManager from conan.internal.loader import ConanFileLoader -from conan.internal.errors import ConanReferenceDoesNotExistInDB, NotFoundException, RecipeNotFoundException, \ +from conan.internal.errors import ConanReferenceDoesNotExistInDB, RecipeNotFoundException, \ PackageNotFoundException from conan.errors import ConanException from conan.internal.model.conf import ConfDefinition @@ -79,7 +79,7 @@ def get_recipe(self, ref, dest_folder): def get_recipe_sources(self, ref, dest_folder): try: export_sources = self._app.cache.recipe_layout(ref).export_sources() - except ConanReferenceDoesNotExistInDB as e: + except ConanReferenceDoesNotExistInDB: # This can happen when there a local-recipes-index is being queried for sources it # doesn't contain # If that is the case, check if they are in the repo, try to export @@ -107,7 +107,7 @@ def check_credentials(self, force_auth=False): def search(self, pattern=None): return self._layout.get_recipes_references(pattern) - def search_packages(self, reference): + def search_packages(self, reference, _=False): assert self and reference return {} @@ -124,7 +124,7 @@ def get_recipe_revisions_references(self, ref): ref = self._export_recipe(ref) return [ref] - def get_package_revisions_references(self, pref, headers=None): + def get_package_revisions_references(self, pref): raise PackageNotFoundException(pref) def get_latest_recipe_reference(self, ref): @@ -149,7 +149,8 @@ def _export_recipe(self, ref): sys.stderr = StringIO() try: global_conf = ConfDefinition() - new_ref, _ = cmd_export(self._app, self._hook_manager, global_conf, conanfile_path, + new_ref, _ = cmd_export(self._app.loader, self._app.cache, self._hook_manager, + global_conf, conanfile_path, ref.name, str(ref.version), None, None, remotes=[self._remote]) except Exception as e: raise ConanException(f"Error while exporting recipe from remote: {self._remote.name}\n" diff --git a/conan/internal/rest/rest_client_v2.py b/conan/internal/rest/rest_client_v2.py index 4b795fa70a7..1b3784a66ca 100644 --- a/conan/internal/rest/rest_client_v2.py +++ b/conan/internal/rest/rest_client_v2.py @@ -9,7 +9,8 @@ from uuid import getnode as get_mac from conan.api.output import ConanOutput - +from conan.internal.paths import EXPORT_SOURCES_FILE_NAME, CONANINFO, CONAN_MANIFEST, \ + EXPORT_FILE_NAME, PACKAGE_FILE_NAME from conan.internal.rest.caching_file_downloader import ConanInternalCacheDownloader from conan.internal.rest import response_to_str from conan.internal.rest.client_routes import ClientV2Router @@ -18,7 +19,6 @@ RecipeNotFoundException, PackageNotFoundException, EXCEPTION_CODE_MAPPING from conan.errors import ConanException from conan.api.model import PkgReference -from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME from conan.api.model import RecipeReference from conan.internal.util.dates import from_iso8601_to_timestamp @@ -219,9 +219,9 @@ def search(self, pattern=None, ignorecase=True): ret.append(ref) return ret - def search_packages(self, ref): + def search_packages(self, ref, list_only): """Client is filtering by the query""" - url = self.router.search_packages(ref) + url = self.router.search_packages(ref, list_only) package_infos = self._get_json(url) return package_infos @@ -239,9 +239,11 @@ def get_recipe(self, ref, dest_folder, metadata, only_metadata): result = {} if not only_metadata: - accepted_files = ["conanfile.py", "conan_export.tgz", "conanmanifest.txt", - "metadata/sign"] + accepted_files = ["conanfile.py", CONAN_MANIFEST, "metadata/sign"] files = [f for f in server_files if any(f.startswith(m) for m in accepted_files)] + export_file = self._find_compressed_file(ref, server_files, EXPORT_FILE_NAME) + if export_file is not None: + files.append(export_file) # If we didn't indicated reference, server got the latest, use absolute now, it's safer urls = {fn: self.router.recipe_file(ref, fn) for fn in files} self._download_and_save_files(urls, dest_folder, files, parallel=True) @@ -260,9 +262,10 @@ def get_recipe_sources(self, ref, dest_folder): url = self.router.recipe_snapshot(ref) data = self._get_file_list_json(url) files = data["files"] - if EXPORT_SOURCES_TGZ_NAME not in files: + src_file = self._find_compressed_file(ref, files, EXPORT_SOURCES_FILE_NAME) + if src_file is None: return None - files = [EXPORT_SOURCES_TGZ_NAME, ] + files = [src_file, ] # If we didn't indicated reference, server got the latest, use absolute now, it's safer urls = {fn: self.router.recipe_file(ref, fn) for fn in files} @@ -270,6 +273,19 @@ def get_recipe_sources(self, ref, dest_folder): ret = {fn: os.path.join(dest_folder, fn) for fn in files} return ret + @staticmethod + def _find_compressed_file(ref, server_files, artifact, exists=False): + pkg_files = [f for f in server_files if f.startswith(artifact)] + if len(pkg_files) > 1: + raise ConanException(f"{ref} is corrupted in the server, it contains " + f"more than one compressed file: {sorted(pkg_files)}") + if not pkg_files: + if not exists: + return None + raise ConanException(f"Recipe {ref} is corrupted in the server, it doesn't contain " + f"a {artifact} file") + return pkg_files[0] + def get_package(self, pref, dest_folder, metadata, only_metadata): url = self.router.package_snapshot(pref) data = self._get_file_list_json(url) @@ -277,8 +293,8 @@ def get_package(self, pref, dest_folder, metadata, only_metadata): result = {} # Download only known files, but not metadata (except sign) if not only_metadata: # Retrieve package first, then metadata - accepted_files = ["conaninfo.txt", "conan_package.tgz", "conanmanifest.txt", - "metadata/sign"] + pkg_file = self._find_compressed_file(pref, server_files, PACKAGE_FILE_NAME, exists=True) + accepted_files = [CONANINFO, pkg_file, CONAN_MANIFEST, "metadata/sign"] files = [f for f in server_files if any(f.startswith(m) for m in accepted_files)] # If we didn't indicated reference, server got the latest, use absolute now, it's safer urls = {fn: self.router.package_file(pref, fn) for fn in files} @@ -356,10 +372,10 @@ def remove_all_packages(self, ref): if response.status_code == 404: # Double check if it is a 404 because there are no packages try: - package_search_url = self.router.search_packages(ref) + package_search_url = self.router.search_packages(ref, list_only=True) if not self._get_json(package_search_url): return - except Exception as e: + except Exception: pass if response.status_code != 200: # Error message is text # To be able to access ret.text (ret.content are bytes) @@ -433,6 +449,7 @@ def get_package_revision_reference(self, pref): raise PackageNotFoundException(pref) def get_recipe_revisions_references(self, ref): + assert ref.revision is None url = self.router.recipe_revisions(ref) tmp = self._get_json(url)["revisions"] remote_refs = [] @@ -441,9 +458,6 @@ def get_recipe_revisions_references(self, ref): _tmp.revision = item.get("revision") _tmp.timestamp = from_iso8601_to_timestamp(item.get("time")) remote_refs.append(_tmp) - - if ref.revision: # FIXME: This is a bit messy, is it checking the existance? or getting the time? or both? - assert "This shoudln't be happening, get_recipe_revisions_references" return remote_refs def get_latest_recipe_reference(self, ref): @@ -454,17 +468,12 @@ def get_latest_recipe_reference(self, ref): remote_ref.timestamp = from_iso8601_to_timestamp(data.get("time")) return remote_ref - def get_package_revisions_references(self, pref, headers=None): + def get_package_revisions_references(self, pref): + assert pref.revision is None url = self.router.package_revisions(pref) - tmp = self._get_json(url, headers=headers)["revisions"] + tmp = self._get_json(url)["revisions"] remote_prefs = [PkgReference(pref.ref, pref.package_id, item.get("revision"), from_iso8601_to_timestamp(item.get("time"))) for item in tmp] - - if pref.revision: # FIXME: This is a bit messy, is it checking the existance? or getting the time? or both? - for _pref in remote_prefs: - if _pref.revision == pref.revision: - return [_pref] - raise PackageNotFoundException(pref) return remote_prefs def get_latest_package_reference(self, pref: PkgReference, headers): diff --git a/conan/internal/rest/rest_routes.py b/conan/internal/rest/rest_routes.py index b2059162deb..c1fe5b84e59 100644 --- a/conan/internal/rest/rest_routes.py +++ b/conan/internal/rest/rest_routes.py @@ -1,4 +1,4 @@ -class RestRoutes(object): +class RestRoutes: ping = "ping" common_search = "conans/search" common_authenticate = "users/authenticate" diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index b6d24f83275..4a62993c81d 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -18,6 +18,7 @@ from conan.internal.runner.output import RunnerOutput from conan.internal.conan_app import ConanApp + class _ContainerConfig(NamedTuple): class Build(NamedTuple): dockerfile: Optional[str] = None @@ -74,6 +75,7 @@ def _instans_or_error(value, obj): ) ) + class DockerRunner: def __init__(self, conan_api: ConanAPI, command: str, host_profile: Profile, build_profile: Profile, args: Namespace, raw_args: list[str]): self.logger = ConanOutput() @@ -118,7 +120,7 @@ def run(self) -> None: raise ConanException(f'"{e.command}" inside docker fail') finally: if self.container: - error = sys.exc_info()[0] is not None # Check if error has been raised + error = sys.exc_info()[0] is not None # Check if error has been raised log = self.logger.error if error else self.logger.status log('Stopping container') self.container.stop() @@ -242,7 +244,7 @@ def _run_command(self, command: str, workdir: Optional[str] = None, verbose: boo raise RunnerException(command=command, stdout_log=stdout_log, stderr_log=stderr_log) return stdout_log, stderr_log - def _get_volumes_and_docker_path(self) -> tuple[dict,str]: + def _get_volumes_and_docker_path(self) -> tuple[dict, str]: app = ConanApp(self.conan_api) remotes = self.conan_api.remotes.list(self.args.remote) if not self.args.no_remote else [] conanfile = app.loader.load_consumer(self.abs_host_path / "conanfile.py", remotes=remotes) diff --git a/conan/internal/runner/output.py b/conan/internal/runner/output.py index a1732aa9e5d..752263f99c7 100644 --- a/conan/internal/runner/output.py +++ b/conan/internal/runner/output.py @@ -1,5 +1,6 @@ from conan.api.output import Color, ConanOutput + class RunnerOutput(ConanOutput): def __init__(self, runner_info: str): super().__init__() diff --git a/conan/internal/util/__init__.py b/conan/internal/util/__init__.py index e69de29bb2d..1dd6437d9ca 100644 --- a/conan/internal/util/__init__.py +++ b/conan/internal/util/__init__.py @@ -0,0 +1,31 @@ +import math +import multiprocessing +import os + +from conan.internal.util.files import load + + +def cpu_count(): + try: + try: + # This is necessary to deduce docker cpu_count + cfs_quota_us = cfs_period_us = 0 + # cgroup2 + if os.path.exists("/sys/fs/cgroup/cgroup.controllers"): + cpu_max = load("/sys/fs/cgroup/cpu.max").split() + if cpu_max[0] != "max": + if len(cpu_max) == 1: + cfs_quota_us, cfs_period_us = int(cpu_max[0]), 100_000 + else: + cfs_quota_us, cfs_period_us = map(int, cpu_max) + else: # cgroup1 + cfs_quota_us = int(load("/sys/fs/cgroup/cpu/cpu.cfs_quota_us")) + cfs_period_us = int(load("/sys/fs/cgroup/cpu/cpu.cfs_period_us")) + if cfs_quota_us > 0 and cfs_period_us > 0: + return int(math.ceil(cfs_quota_us / cfs_period_us)) + except (EnvironmentError, TypeError): + pass + return multiprocessing.cpu_count() + except NotImplementedError: + # print("multiprocessing.cpu_count() not implemented. Defaulting to 1 cpu") + return 1 # Safe guess diff --git a/conan/internal/util/files.py b/conan/internal/util/files.py index 40a989cc5e7..76596ca055c 100644 --- a/conan/internal/util/files.py +++ b/conan/internal/util/files.py @@ -1,7 +1,10 @@ import errno +<<<<<<< HEAD from pathlib import Path import tempfile import gzip +======= +>>>>>>> feature/builtin_compression import hashlib import os import platform diff --git a/conan/test/assets/genconanfile.py b/conan/test/assets/genconanfile.py index b90c3951e4f..720abdfe9c0 100644 --- a/conan/test/assets/genconanfile.py +++ b/conan/test/assets/genconanfile.py @@ -1,7 +1,7 @@ from conan.api.model import RecipeReference -class GenConanfile(object): +class GenConanfile: """ USAGE: diff --git a/conan/test/assets/premake.py b/conan/test/assets/premake.py index eb290d1c074..64eccfc4ec8 100644 --- a/conan/test/assets/premake.py +++ b/conan/test/assets/premake.py @@ -3,11 +3,12 @@ from jinja2 import Template -def gen_premake5(workspace, projects, includedirs=["."], language="CXX", std="17"): +def gen_premake5(workspace, projects, includedirs=None, configurations=None): + includedirs = includedirs if includedirs is not None else ["."] premake5 = textwrap.dedent("""\ workspace "{{workspace}}" cppdialect "C++17" - configurations { "Debug", "Release" } + configurations {{premake_quote(configurations)}} fatalwarnings {"All"} floatingpoint "Fast" includedirs {{premake_quote(includedirs)}} @@ -37,8 +38,8 @@ def premake_quote(s): { "premake_quote": premake_quote, "workspace": workspace, + "configurations": configurations or ["Debug", "Release"], "includedirs": includedirs, "projects": projects, } ) - diff --git a/conan/test/assets/visual_project_files.py b/conan/test/assets/visual_project_files.py index 9b9f5ceb587..35e5516036b 100644 --- a/conan/test/assets/visual_project_files.py +++ b/conan/test/assets/visual_project_files.py @@ -34,8 +34,6 @@ HideSolutionNode = FALSE EndGlobalSection EndGlobal - - ''' vcxproj_file = r''' diff --git a/conan/test/utils/artifactory.py b/conan/test/utils/artifactory.py index c05aa2081c7..323de1f3278 100644 --- a/conan/test/utils/artifactory.py +++ b/conan/test/utils/artifactory.py @@ -12,7 +12,7 @@ ARTIFACTORY_DEFAULT_URL = os.getenv("ARTIFACTORY_DEFAULT_URL", "http://localhost:8090/artifactory") -class _ArtifactoryServerStore(object): +class _ArtifactoryServerStore: def __init__(self, repo_url, user, password): self._user = user or ARTIFACTORY_DEFAULT_USER @@ -67,7 +67,7 @@ def get_last_package_revision(self, ref): return revisions[0] -class ArtifactoryServer(object): +class ArtifactoryServer: def __init__(self, *args, **kwargs): self._user = ARTIFACTORY_DEFAULT_USER diff --git a/conan/test/utils/mocks.py b/conan/test/utils/mocks.py index cd4e9f45eab..0c16a7c4ecd 100644 --- a/conan/test/utils/mocks.py +++ b/conan/test/utils/mocks.py @@ -27,7 +27,7 @@ def readline(self): return ret -class MockSettings(object): +class MockSettings: def __init__(self, values): self.values = values diff --git a/conan/test/utils/server_launcher.py b/conan/test/utils/server_launcher.py index 93c08529316..78f2856d691 100644 --- a/conan/test/utils/server_launcher.py +++ b/conan/test/utils/server_launcher.py @@ -17,7 +17,7 @@ TESTING_REMOTE_PRIVATE_PASS = "private_pass" -class TestServerLauncher(object): +class TestServerLauncher: def __init__(self, base_path=None, read_permissions=None, write_permissions=None, users=None, base_url=None, plugins=None, @@ -85,7 +85,7 @@ def stop(self): self._stop.set() def stopped(self): - return self._stop.isSet() + return self._stop.is_set() self.t1 = StoppableThread(target=self.ra.run, kwargs={"host": "0.0.0.0", "quiet": True}) self.t1.daemon = daemon diff --git a/conan/test/utils/test_files.py b/conan/test/utils/test_files.py index 7b3f011116d..514ff5f8e51 100644 --- a/conan/test/utils/test_files.py +++ b/conan/test/utils/test_files.py @@ -12,7 +12,6 @@ from conan.tools.files.files import untargz from conan.internal.subsystems import get_cased_path from conan.errors import ConanException -from conan.internal.paths import PACKAGE_TGZ_NAME def wait_until_removed(folder): @@ -52,14 +51,14 @@ def temp_folder(path_with_spaces=True, create_dir=True): def uncompress_packaged_files(paths, pref): - rev = paths.get_last_revision(pref.ref).revision _tmp = copy.copy(pref) _tmp.revision = None prev = paths.get_last_package_revision(_tmp).revision pref.revision = prev package_path = paths.package(pref) - if not(os.path.exists(os.path.join(package_path, PACKAGE_TGZ_NAME))): + PACKAGE_TGZ_NAME = "conan_package.tgz" + if not (os.path.exists(os.path.join(package_path, PACKAGE_TGZ_NAME))): raise ConanException("%s not found in %s" % (PACKAGE_TGZ_NAME, package_path)) tmp = temp_folder() untargz(os.path.join(package_path, PACKAGE_TGZ_NAME), tmp) diff --git a/conan/test/utils/tools.py b/conan/test/utils/tools.py index 4d6a12db5bb..80dfd85a9c2 100644 --- a/conan/test/utils/tools.py +++ b/conan/test/utils/tools.py @@ -16,10 +16,10 @@ from inspect import getframeinfo, stack from urllib.parse import urlsplit, urlunsplit -import mock +from unittest import mock import pytest import requests -from mock import Mock +from unittest.mock import Mock from requests.exceptions import HTTPError from webtest.app import TestApp @@ -74,14 +74,14 @@ os=Macos arch={arch_setting} compiler=apple-clang - compiler.version=16 + compiler.version=17 compiler.libcxx=libc++ build_type=Release """) } -class TestingResponse(object): +class TestingResponse: """Wraps a response from TestApp external tool to guarantee the presence of response.ok, response.content and response.status_code, as it was a requests library object. @@ -469,7 +469,7 @@ def open_home(self, filename): return self.open(os.path.join(self.cache_folder, filename)) @property - def cache(self): + def cache(self) -> PkgCache: # Returns a temporary cache object intended for inspecting it return PkgCache(self.cache_folder, load_global_conf(self.cache_folder)) @@ -663,7 +663,7 @@ def export(self, ref, conanfile=GenConanfile(), args=None): self.run("export .") tmp = copy.copy(ref) tmp.revision = None - rrev = self.cache.get_latest_recipe_reference(tmp).revision + rrev = self.cache.get_latest_recipe_revision(tmp).revision tmp = copy.copy(ref) tmp.revision = rrev return tmp @@ -696,7 +696,7 @@ def init_git_repo(self, files=None, branch=None, submodules=None, folder=None, o def get_latest_package_reference(self, ref, package_id=None) -> PkgReference: """Get the latest PkgReference given a ConanReference""" ref_ = RecipeReference.loads(ref) if isinstance(ref, str) else ref - latest_rrev = self.cache.get_latest_recipe_reference(ref_) + latest_rrev = self.cache.get_latest_recipe_revision(ref_) if package_id: pref = PkgReference(latest_rrev, package_id) else: @@ -706,19 +706,19 @@ def get_latest_package_reference(self, ref, package_id=None) -> PkgReference: f"provide a single package_id instead" \ if len(package_ids) > 0 else "No binary packages found" pref = package_ids[0] - return self.cache.get_latest_package_reference(pref) + return self.cache.get_latest_package_revision(pref) def get_latest_pkg_layout(self, pref: PkgReference) -> PackageLayout: """Get the latest PackageLayout given a file reference""" # Let's make it easier for all the test clients - latest_prev = self.cache.get_latest_package_reference(pref) + latest_prev = self.cache.get_latest_package_revision(pref) pkg_layout = self.cache.pkg_layout(latest_prev) return pkg_layout def get_latest_ref_layout(self, ref) -> RecipeLayout: """Get the latest RecipeLayout given a file reference""" if not ref.revision: - ref = self.cache.get_latest_recipe_reference(ref) + ref = self.cache.get_latest_recipe_revision(ref) ref_layout = self.cache.recipe_layout(ref) return ref_layout @@ -731,11 +731,11 @@ def get_default_build_profile(self): return api.profiles.get_profile([api.profiles.get_default_build()]) def recipe_exists(self, ref): - rrev = self.cache.get_recipe_revisions_references(ref) + rrev = self.cache.get_recipe_revisions(ref) return True if rrev else False def package_exists(self, pref): - prev = self.cache.get_package_revisions_references(pref) + prev = self.cache.get_package_revisions(pref) return True if prev else False def assert_listed_require(self, requires, build=False, python=False, test=False, diff --git a/conan/tools/android/utils.py b/conan/tools/android/utils.py index 54bd6439330..d16c1738d41 100644 --- a/conan/tools/android/utils.py +++ b/conan/tools/android/utils.py @@ -10,7 +10,8 @@ def android_abi(conanfile, context="host"): :return: Android-NDK ABI """ if context not in ("host", "build", "target"): - raise ConanException(f"context argument must be either 'host', 'build' or 'target', was '{context}'") + raise ConanException(f"context argument must be either 'host', 'build' or 'target', " + f"was '{context}'") try: settings = getattr(conanfile, f"settings_{context}") diff --git a/conan/tools/apple/apple.py b/conan/tools/apple/apple.py index 844084387f4..3c507a6536a 100644 --- a/conan/tools/apple/apple.py +++ b/conan/tools/apple/apple.py @@ -1,6 +1,7 @@ import os from io import StringIO +from conan.internal.internal_tools import universal_arch_separator from conan.internal.util.runners import check_output_runner from conan.tools.build import cmd_args_to_string from conan.errors import ConanException @@ -85,27 +86,39 @@ def apple_min_version_flag(conanfile): }.get(os_sdk, "") -def resolve_apple_flags(conanfile, is_cross_building=False): +def resolve_apple_flags(conanfile, is_cross_building=False, is_universal=False): """ Gets the most common flags in Apple systems. If it's a cross-building context SDK path is mandatory so if it could raise an exception if SDK is not found. :param conanfile: instance. :param is_cross_building: boolean to indicate if it's a cross-building context. - :return: tuple of Apple flags (apple_min_version_flag, apple_arch, apple_isysroot_flag). + :param is_universal: boolean to indicate if it's a universal binary. + :return: tuple of Apple flags (apple_min_version_flag, apple_arch_flags, apple_isysroot_flag). """ if not is_apple_os(conanfile): # Keeping legacy defaults return "", None, None - apple_arch_flag = apple_isysroot_flag = None - if is_cross_building: + apple_arch_flags = apple_isysroot_flag = None + + if is_universal: + arch_ = conanfile.settings.get_safe("arch") + apple_arch_flags = " ".join([f"-arch {_to_apple_arch(arch, default=arch)}" for arch in + arch_.split(universal_arch_separator)]) + sdk_path = conanfile.conf.get("tools.apple:sdk_path") + if sdk_path: + # Ideally, -isysroot should be added whenever sdk_path is defined. + # For now, we only set it in this case to avoid changing existing behavior. + apple_isysroot_flag = f"-isysroot {sdk_path}" + elif is_cross_building: arch = to_apple_arch(conanfile) sdk_path = apple_sdk_path(conanfile, is_cross_building=is_cross_building) apple_isysroot_flag = f"-isysroot {sdk_path}" if sdk_path else "" - apple_arch_flag = f"-arch {arch}" if arch else "" + apple_arch_flags = f"-arch {arch}" if arch else "" + min_version_flag = apple_min_version_flag(conanfile) - return min_version_flag, apple_arch_flag, apple_isysroot_flag + return min_version_flag, apple_arch_flags, apple_isysroot_flag def xcodebuild_deployment_target_key(os_name): @@ -128,7 +141,8 @@ def __init__(self, conanfile, sdk=None, use_settings_target=False): :param conanfile: Conanfile instance. :param sdk: Will skip the flag when ``False`` is passed and will try to adjust the sdk it automatically if ``None`` is passed. - :param use_settings_target: Try to use ``settings_target`` in case they exist (``False`` by default) + :param use_settings_target: Try to use ``settings_target`` in case they exist + (``False`` by default) """ settings = conanfile.settings if use_settings_target and conanfile.settings_target is not None: @@ -218,7 +232,7 @@ def install_name_tool(self): def _get_dylib_install_name(otool, path_to_dylib): command = f"{otool} -D {path_to_dylib}" - output = iter(check_output_runner(command).splitlines()) + output = iter(check_output_runner(command).splitlines()) # Note: if otool return multiple entries for different architectures # assume they are the same and pick the first one. for line in output: @@ -248,7 +262,8 @@ def _darwin_is_binary(file, binary_type): return binary_type in check_output_runner(check_file) def _darwin_collect_binaries(folder, binary_type): - return [os.path.join(folder, f) for f in os.listdir(folder) if _darwin_is_binary(os.path.join(folder, f), binary_type)] + return [os.path.join(folder, f) for f in os.listdir(folder) + if _darwin_is_binary(os.path.join(folder, f), binary_type)] def _fix_install_name(dylib_path, new_name): command = f"{install_name_tool} {dylib_path} -id {new_name}" @@ -275,7 +290,7 @@ def _get_shared_dependencies(binary_file): ret = [s.split("(")[0].strip() for s in all_shared.splitlines()] return ret - def _fix_dylib_files(conanfile): + def _fix_dylib_files(): substitutions = {} libdirs = getattr(conanfile.cpp.package, "libdirs") for libdir in libdirs: @@ -287,7 +302,7 @@ def _fix_dylib_files(conanfile): # fix LC_ID_DYLIB in first pass for shared_lib in shared_libs: install_name = _get_dylib_install_name(otool, shared_lib) - #TODO: we probably only want to fix the install the name if + # TODO: we probably only want to fix the install the name if # it starts with `/`. rpath_name = f"@rpath/{os.path.basename(install_name)}" _fix_install_name(shared_lib, rpath_name) @@ -300,7 +315,7 @@ def _fix_dylib_files(conanfile): return substitutions - def _fix_executables(conanfile, substitutions): + def _fix_executables(substitutions): # Fix the install name for executables inside the package # that reference libraries we just patched bindirs = getattr(conanfile.cpp.package, "bindirs") @@ -316,7 +331,8 @@ def _fix_executables(conanfile, substitutions): # Fix install names of libraries from within the same package deps = _get_shared_dependencies(executable) for dep in deps: - dep_base = os.path.join(os.path.dirname(dep), os.path.basename(dep).split('.')[0]) + dep_base = os.path.join(os.path.dirname(dep), + os.path.basename(dep).split('.')[0]) match = [k for k in substitutions.keys() if k.startswith(dep_base)] if match: _fix_dep_name(executable, dep, substitutions[match[0]]) @@ -324,20 +340,21 @@ def _fix_executables(conanfile, substitutions): # Add relative rpath to library directories, avoiding possible # existing duplicates libdirs = getattr(conanfile.cpp.package, "libdirs") - libdirs = [os.path.join(conanfile.package_folder, dir) for dir in libdirs] - rel_paths = [f"@executable_path/{os.path.relpath(dir, full_folder)}" for dir in libdirs] + libdirs = [os.path.join(conanfile.package_folder, libdir) for libdir in libdirs] + rel_paths = [f"@executable_path/{os.path.relpath(libdir, full_folder)}" + for libdir in libdirs] existing_rpaths = _get_rpath_entries(executable) rpaths_to_add = list(set(rel_paths) - set(existing_rpaths)) for entry in rpaths_to_add: command = f"{install_name_tool} {executable} -add_rpath {entry}" conanfile.run(command) - substitutions = _fix_dylib_files(conanfile) + substitutes = _fix_dylib_files() # Only "fix" executables if dylib files were patched, otherwise # there is nothing to do. - if substitutions: - _fix_executables(conanfile, substitutions) + if substitutes: + _fix_executables(substitutes) def apple_extra_flags(conanfile): diff --git a/conan/tools/apple/xcodebuild.py b/conan/tools/apple/xcodebuild.py index 637c4739d17..be050523700 100644 --- a/conan/tools/apple/xcodebuild.py +++ b/conan/tools/apple/xcodebuild.py @@ -1,7 +1,8 @@ from conan.tools.apple.apple import to_apple_arch, xcodebuild_deployment_target_key +from conan.tools.build import cmd_args_to_string -class XcodeBuild(object): +class XcodeBuild: def __init__(self, conanfile): self._conanfile = conanfile self._build_type = conanfile.settings.get_safe("build_type") @@ -28,7 +29,7 @@ def _sdkroot(self): sdk = "{}{}".format(self._sdk, self._sdk_version) return "SDKROOT={}".format(sdk) if sdk else "" - def build(self, xcodeproj, target=None): + def build(self, xcodeproj, target=None, configuration=None, cli_args=None): """ Call to ``xcodebuild`` to build a Xcode project. @@ -36,15 +37,23 @@ def build(self, xcodeproj, target=None): :param target: the target to build, in case this argument is passed to the ``build()`` method it will add the ``-target`` argument to the build system call. If not passed, it will build all the targets passing the ``-alltargets`` argument instead. + :param configuration: Build configuration to use (e.g., ``Debug``, ``Release``). + Defaults to the recipe's ``settings.build_type``. + :param cli_args: Extra options to pass directly to ``xcodebuild`` (list of strings). + Examples: ``["-xcconfig", ""]`` or custom + Xcode build settings like ``["BUILD_LIBRARY_FOR_DISTRIBUTION=YES"]``. :return: the return code for the launched ``xcodebuild`` command. """ target = "-target '{}'".format(target) if target else "-alltargets" + build_config = configuration or self._build_type cmd = "xcodebuild -project '{}' -configuration {} -arch {} " \ - "{} {} {}".format(xcodeproj, self._build_type, self._arch, self._sdkroot, + "{} {} {}".format(xcodeproj, build_config, self._arch, self._sdkroot, self._verbosity, target) - deployment_target_key = xcodebuild_deployment_target_key(self._os) if deployment_target_key and self._os_version: cmd += f" {deployment_target_key}={self._os_version}" + if cli_args: + cmd += " " + cmd_args_to_string(cli_args) + self._conanfile.run(cmd) diff --git a/conan/tools/apple/xcodedeps.py b/conan/tools/apple/xcodedeps.py index 17dbf31da7c..f78ef8878f6 100644 --- a/conan/tools/apple/xcodedeps.py +++ b/conan/tools/apple/xcodedeps.py @@ -1,4 +1,5 @@ import os +import re import textwrap from collections import OrderedDict @@ -20,8 +21,7 @@ def _format_name(name): - name = name.replace(".", "_").replace("-", "_") - return name.lower() + return re.sub(r'[^A-Za-z0-9_]', '_', name).lower() def _xcconfig_settings_filename(settings, configuration): @@ -59,7 +59,7 @@ def _add_includes_to_file_or_create(filename, template, files_to_include): return content -class XcodeDeps(object): +class XcodeDeps: general_name = "conandeps.xcconfig" _conf_xconfig = textwrap.dedent("""\ @@ -93,7 +93,7 @@ class XcodeDeps(object): // Link options for {{pkg_name}}_{{comp_name}} LIBRARY_SEARCH_PATHS = $(inherited) $(LIBRARY_SEARCH_PATHS_{{pkg_name}}_{{comp_name}}) OTHER_LDFLAGS = $(inherited) $(OTHER_LDFLAGS_{{pkg_name}}_{{comp_name}}) - """) + """) _all_xconfig = textwrap.dedent("""\ // Conan XcodeDeps generated file @@ -256,7 +256,10 @@ def _content(self): dep_name = _format_name(dep.ref.name) include_components_names = [] + transitive_requires = [r for r, _ in + get_transitive_requires(self._conanfile, dep).items()] if dep.cpp_info.has_components: + transitive_dep_names = [_format_name(dep.ref.name) for dep in transitive_requires] sorted_components = dep.cpp_info.get_sorted_components().items() for comp_name, comp_cpp_info in sorted_components: @@ -266,7 +269,8 @@ def _content(self): # "list of names from required components from other packages") def _get_component_requires(component): requires_external = [(req.split("::")[0], req.split("::")[1]) for req in - component.requires if "::" in req] + component.requires if "::" in req + and req.split("::")[0] in transitive_dep_names] requires_internal = [dep.cpp_info.components.get(req) for req in component.requires if "::" not in req] return requires_internal, requires_external @@ -301,7 +305,6 @@ def _transitive_components(component): result.update(component_content) else: public_deps = [] - transitive_requires = [r for r, _ in get_transitive_requires(self._conanfile, dep).items()] for r, d in dep.dependencies.direct_host.items(): if r not in transitive_requires: continue diff --git a/conan/tools/apple/xcodetoolchain.py b/conan/tools/apple/xcodetoolchain.py index b6aca20e339..76d8c79180a 100644 --- a/conan/tools/apple/xcodetoolchain.py +++ b/conan/tools/apple/xcodetoolchain.py @@ -7,7 +7,7 @@ from conan.internal.util.files import save -class XcodeToolchain(object): +class XcodeToolchain: filename = "conantoolchain" extension = ".xcconfig" @@ -71,17 +71,20 @@ def _apple_deployment_target(self): @property def _clang_cxx_library(self): - return 'CLANG_CXX_LIBRARY{}={}'.format(_xcconfig_conditional(self._conanfile.settings, self.configuration), + return 'CLANG_CXX_LIBRARY{}={}'.format(_xcconfig_conditional(self._conanfile.settings, + self.configuration), self.libcxx) if self.libcxx else "" @property def _clang_cxx_language_standard(self): return 'CLANG_CXX_LANGUAGE_STANDARD{}={}'.format(_xcconfig_conditional(self._conanfile.settings, self.configuration), self._cppstd) if self._cppstd else "" + @property def _vars_xconfig_filename(self): - return "conantoolchain{}{}".format(_xcconfig_settings_filename(self._conanfile.settings, self.configuration), - self.extension) + return "conantoolchain{}{}".format(_xcconfig_settings_filename(self._conanfile.settings, + self.configuration), + self.extension) @property def _vars_xconfig_content(self): diff --git a/conan/tools/build/__init__.py b/conan/tools/build/__init__.py index f709bb2c878..2d5dddec2df 100644 --- a/conan/tools/build/__init__.py +++ b/conan/tools/build/__init__.py @@ -11,6 +11,7 @@ from conan.tools.build.cpu import build_jobs from conan.tools.build.cross_building import cross_building, can_run from conan.tools.build.stdcpp_library import stdcpp_library +from conan.tools.build.compiler import check_min_compiler_version from conan.errors import ConanException CONAN_TOOLCHAIN_ARGS_FILE = "conanbuild.conf" diff --git a/conan/tools/build/compiler.py b/conan/tools/build/compiler.py new file mode 100644 index 00000000000..3392e922016 --- /dev/null +++ b/conan/tools/build/compiler.py @@ -0,0 +1,44 @@ +from conan.errors import ConanInvalidConfiguration, ConanException +from conan.internal.model.version import Version + + +def check_min_compiler_version(conanfile, compiler_restrictions): + """(Experimental) Checks if the current compiler and its version meet the minimum requirements. + + :param conanfile: The current recipe object. Always use ``self``. + :param compiler_restrictions: + A list of tuples, where each tuple contains: + + - **compiler** (*str*): The name of the compiler (e.g., "gcc", "msvc"). + - **min_version** (*str*): The minimum required version as a string (e.g., "14", "19.0"). + - **reason** (*str*): A string explaining the reason for the version requirement. + :raises ConanException: + If the 'compiler' or 'compiler.version' settings are not defined. + :raises ConanInvalidConfiguration: + If the found compiler version is less than the specified minimum version for that compiler. + + :Example: + .. code-block:: python + + def validate(self): + compiler_restrictions = [ + ("clang", "14", "requires C++20 coroutines support"), + ("gcc", "12", "requires C++20 modules support") + ] + check_min_compiler_version(self, compiler_restrictions) + """ + compiler_value = conanfile.settings.get_safe("compiler") + if not compiler_value: + raise ConanException("Called check_min_compiler_version with no compiler defined") + compiler_version = conanfile.settings.get_safe("compiler.version") + if not compiler_version: + raise ConanException("Called check_min_compiler_version with no compiler.version defined") + + for compiler, min_version, reason in compiler_restrictions: + if compiler_value == compiler: + if Version(compiler_version) < Version(min_version): + ref = conanfile.ref if hasattr(conanfile, "ref") else conanfile.name + raise ConanInvalidConfiguration( + f"{ref} requires {compiler} >= {min_version}, but {compiler} {compiler_version} was found\n" + f"Reason: {reason}") + break diff --git a/conan/tools/build/cpu.py b/conan/tools/build/cpu.py index c11bb90bebc..0380850834d 100644 --- a/conan/tools/build/cpu.py +++ b/conan/tools/build/cpu.py @@ -1,8 +1,4 @@ -import math -import multiprocessing -import os - -from conan.internal.util.files import load +from conan.internal.util import cpu_count def build_jobs(conanfile): @@ -22,33 +18,4 @@ def build_jobs(conanfile): :param conanfile: The current recipe object. Always use ``self``. :return: ``int`` with the number of jobs """ - njobs = conanfile.conf.get("tools.build:jobs", - default=_cpu_count(), - check_type=int) - return njobs - - -def _cpu_count(): - try: - try: - # This is necessary to deduce docker cpu_count - cfs_quota_us = cfs_period_us = 0 - # cgroup2 - if os.path.exists("/sys/fs/cgroup/cgroup.controllers"): - cpu_max = load("/sys/fs/cgroup/cpu.max").split() - if cpu_max[0] != "max": - if len(cpu_max) == 1: - cfs_quota_us, cfs_period_us = int(cpu_max[0]), 100_000 - else: - cfs_quota_us, cfs_period_us = map(int, cpu_max) - else: # cgroup1 - cfs_quota_us = int(load("/sys/fs/cgroup/cpu/cpu.cfs_quota_us")) - cfs_period_us = int(load("/sys/fs/cgroup/cpu/cpu.cfs_period_us")) - if cfs_quota_us > 0 and cfs_period_us > 0: - return int(math.ceil(cfs_quota_us / cfs_period_us)) - except (EnvironmentError, TypeError): - pass - return multiprocessing.cpu_count() - except NotImplementedError: - # print("multiprocessing.cpu_count() not implemented. Defaulting to 1 cpu") - return 1 # Safe guess + return conanfile.conf.get("tools.build:jobs", default=cpu_count(), check_type=int) diff --git a/conan/tools/build/cstd.py b/conan/tools/build/cstd.py index 59e3f1145c7..e655e7efb33 100644 --- a/conan/tools/build/cstd.py +++ b/conan/tools/build/cstd.py @@ -74,6 +74,7 @@ def valid_max_cstd(conanfile, cstd, gnu_extensions=False): return False return True + def default_cstd(conanfile, compiler=None, compiler_version=None): """ Get the default ``compiler.cstd`` for the "conanfile.settings.compiler" and "conanfile @@ -90,6 +91,7 @@ def default_cstd(conanfile, compiler=None, compiler_version=None): raise ConanException("Called default_cppstd with no compiler or no compiler.version") return default_cstd_(compiler, Version(compiler_version)) + def supported_cstd(conanfile, compiler=None, compiler_version=None): """ Get a list of supported ``compiler.cstd`` for the "conanfile.settings.compiler" and @@ -192,6 +194,7 @@ def _clang_supported_cstd(version): return ["99", "gnu99", "11", "gnu11", "17", "gnu17"] return ["99", "gnu99", "11", "gnu11", "17", "gnu17", "23", "gnu23"] + def _emcc_supported_cstd(version): """ emcc is based on clang but follow different versioning scheme. diff --git a/conan/tools/build/flags.py b/conan/tools/build/flags.py index 2b3991fa833..cdaf6b4455b 100644 --- a/conan/tools/build/flags.py +++ b/conan/tools/build/flags.py @@ -1,11 +1,26 @@ +from conan.errors import ConanException from conan.internal.model.version import Version +def disable_flag(conanfile, flag): + disable_flags = conanfile.conf.get("tools.gnu:disable_flags", check_type=list) + if disable_flags is None: + return False + valid = ["arch", "arch_link", "libcxx", "build_type", "build_type_link", "threads", + "cppstd", "cstd"] + for v in disable_flags: + if v not in valid: + raise ConanException(f"tools.gnu:disable_flags value '{v}', must be one of: {valid}") + return flag in disable_flags + + def architecture_flag(conanfile): """ returns flags specific to the target architecture and compiler Used by CMakeToolchain and AutotoolsToolchain """ + if disable_flag(conanfile, "arch"): + return "" settings = conanfile.settings from conan.tools.apple.apple import _to_apple_arch compiler = settings.get_safe("compiler") @@ -76,6 +91,8 @@ def architecture_link_flag(conanfile): """ returns exclusively linker flags specific to the target architecture and compiler """ + if disable_flag(conanfile, "arch_link"): + return "" compiler = conanfile.settings.get_safe("compiler") arch = conanfile.settings.get_safe("arch") if compiler == "emcc": @@ -90,6 +107,8 @@ def libcxx_flags(conanfile): libcxx = conanfile.settings.get_safe("compiler.libcxx") if not libcxx: return None, None + if disable_flag(conanfile, "libcxx"): + return None, None compiler = conanfile.settings.get_safe("compiler") lib = stdlib11 = None if compiler == "apple-clang": @@ -144,6 +163,8 @@ def build_type_flags(conanfile): (-s, -g, /Zi, etc.) Used only by AutotoolsToolchain """ + if disable_flag(conanfile, "build_type"): + return [] settings = conanfile.settings compiler = settings.get_safe("compiler") build_type = settings.get_safe("build_type") @@ -195,10 +216,13 @@ def build_type_flags(conanfile): return flags return [] + def threads_flags(conanfile): """ returns flags specific to the threading model used by the compiler """ + if disable_flag(conanfile, "threads"): + return [] compiler = conanfile.settings.get_safe("compiler") threads = conanfile.settings.get_safe("compiler.threads") if compiler == "emcc": @@ -208,6 +232,7 @@ def threads_flags(conanfile): return ["-sWASM_WORKERS=1"] return [] + def llvm_clang_front(conanfile): # Only Windows clang with MSVC backend (LLVM/Clang, not MSYS2 clang) if (conanfile.settings.get_safe("os") != "Windows" or @@ -217,7 +242,7 @@ def llvm_clang_front(conanfile): compilers = conanfile.conf.get("tools.build:compiler_executables", default={}) if "clang-cl" in compilers.get("c", "") or "clang-cl" in compilers.get("cpp", ""): return "clang-cl" # The MSVC-compatible front - return "clang" # The GNU-compatible front + return "clang" # The GNU-compatible front def cppstd_flag(conanfile) -> str: @@ -241,6 +266,9 @@ def cppstd_flag(conanfile) -> str: if not compiler or not compiler_version or not cppstd: return "" + if disable_flag(conanfile, "cppstd"): + return "" + func = {"gcc": _cppstd_gcc, "clang": _cppstd_clang, "apple-clang": _cppstd_apple_clang, @@ -460,43 +488,6 @@ def _cppstd_gcc(gcc_version, cppstd): return f'-std={flag}' if flag else None -def _cppstd_intel_common(intel_version, cppstd, vgnu98, vgnu0x): - # https://software.intel.com/en-us/cpp-compiler-developer-guide-and-reference-std-qstd - # https://software.intel.com/en-us/articles/intel-cpp-compiler-release-notes - # NOTE: there are only gnu++98 and gnu++0x, and only for Linux/macOS - v98 = v11 = v14 = v17 = v20 = None - vgnu11 = vgnu14 = vgnu17 = vgnu20 = None - - if intel_version >= "12": - v11 = "c++0x" - vgnu11 = vgnu0x - if intel_version >= "14": - v11 = "c++11" - vgnu11 = vgnu0x - if intel_version >= "16": - v14 = "c++14" - if intel_version >= "18": - v17 = "c++17" - if intel_version >= "19.1": - v20 = "c++20" - - return {"98": v98, "gnu98": vgnu98, - "11": v11, "gnu11": vgnu11, - "14": v14, "gnu14": vgnu14, - "17": v17, "gnu17": vgnu17, - "20": v20, "gnu20": vgnu20}.get(cppstd) - - -def _cppstd_intel_gcc(intel_version, cppstd): - flag = _cppstd_intel_common(intel_version, cppstd, "gnu++98", "gnu++0x") - return f'-std={flag}' if flag else None - - -def _cppstd_intel_visualstudio(intel_version, cppstd): - flag = _cppstd_intel_common(intel_version, cppstd, None, None) - return f'/Qstd={flag}' if flag else None - - def _cppstd_mcst_lcc(mcst_lcc_version, cppstd): v11 = vgnu11 = v14 = vgnu14 = v17 = vgnu17 = v20 = vgnu20 = None @@ -578,6 +569,9 @@ def cstd_flag(conanfile) -> str: if not compiler or not compiler_version or not cstd: return "" + if disable_flag(conanfile, "cstd"): + return "" + func = {"gcc": _cstd_gcc, "clang": _cstd_clang, "apple-clang": _cstd_apple_clang, diff --git a/conan/tools/cmake/__init__.py b/conan/tools/cmake/__init__.py index d6a1c431feb..4fd251adbfc 100644 --- a/conan/tools/cmake/__init__.py +++ b/conan/tools/cmake/__init__.py @@ -1,31 +1,17 @@ -from conan.errors import ConanException from conan.tools.cmake.toolchain.toolchain import CMakeToolchain from conan.tools.cmake.cmake import CMake +from conan.tools.cmake.cmakeconfigdeps.cmakeconfigdeps import CMakeConfigDeps from conan.tools.cmake.layout import cmake_layout def CMakeDeps(conanfile): # noqa if conanfile.conf.get("tools.cmake.cmakedeps:new", choices=["will_break_next", "recipe_will_break"]) == "will_break_next": - from conan.tools.cmake.cmakedeps2.cmakedeps import CMakeDeps2 - conanfile.output.warning("Using the new CMakeConfigDeps generator, behind the " - "'tools.cmake.cmakedeps:new' gate conf. This conf will change " - "next release, breaking, so use it only for testing and dev") - return CMakeDeps2(conanfile) + conanfile.output.warning("On the fly replacement of CMakeDeps by CMakeConfigDeps generator, " + "because 'tools.cmake.cmakedeps:new' incubating conf activated. " + "This conf is incubating and will break in next releases. " + "CMakeConfigDeps is now experimental and can be used as such in " + "recipes.") + return CMakeConfigDeps(conanfile) from conan.tools.cmake.cmakedeps.cmakedeps import CMakeDeps as _CMakeDeps return _CMakeDeps(conanfile) - - - -def CMakeConfigDeps(conanfile): # noqa - if conanfile.conf.get("tools.cmake.cmakedeps:new", - choices=["will_break_next", "recipe_will_break"]): - from conan.tools.cmake.cmakedeps2.cmakedeps import CMakeDeps2 - conanfile.output.warning("Using the new CMakeConfigDeps generator, behind the " - "'tools.cmake.cmakedeps:new' gate conf. This conf will change " - "next release, breaking, so use it only for testing and dev") - return CMakeDeps2(conanfile) - else: - raise ConanException("CMakeConfigDeps is being used in conanfile, but the conf " - "'tools.cmake.cmakedeps:new' is not enabled. This is an incubating" - "feature, please define it to enable CMakeConfigDeps usage") diff --git a/conan/tools/cmake/cmake.py b/conan/tools/cmake/cmake.py index 56580eb1653..1d99223d068 100644 --- a/conan/tools/cmake/cmake.py +++ b/conan/tools/cmake/cmake.py @@ -61,15 +61,15 @@ def is_multi_configuration(self): return is_multi_configuration(self._generator) def configure(self, variables=None, build_script_folder=None, cli_args=None, - stdout=None, stderr=None): + stdout=None, stderr=None, subfolder=None): """ Reads the ``CMakePresets.json`` file generated by the :ref:`CMakeToolchain` to get: - The generator, to append ``-G="xxx"``. - - The path to the toolchain and append ``-DCMAKE_TOOLCHAIN_FILE=/path/conan_toolchain.cmake`` - - The declared ``cache variables`` and append ``-Dxxx``. + - Toolchain path to append ``-DCMAKE_TOOLCHAIN_FILE=/path/conan_toolchain.cmake`` + - The declared ``cache variables`` to append ``-Dxxx``. and call ``cmake``. @@ -83,6 +83,9 @@ def configure(self, variables=None, build_script_folder=None, cli_args=None, ``self.folders.source`` at the ``layout()`` method. :param cli_args: List of arguments ``[arg1, arg2, ...]`` that will be passed as extra CLI arguments to pass to cmake invocation + :param subfolder: (Experimental): subfolder to be created inside the ``build_folder`` and + the ``package_folder``. If not provided, files will be + placed in the ``build_folder`` and ``package_folder`` root. :param stdout: Use it to redirect stdout to this stream :param stderr: Use it to redirect stderr to this stream """ @@ -93,13 +96,18 @@ def configure(self, variables=None, build_script_folder=None, cli_args=None, cmakelist_folder = cmakelist_folder.replace("\\", "/") build_folder = self._conanfile.build_folder + if subfolder: + build_folder = os.path.join(self._conanfile.build_folder, subfolder) mkdir(self._conanfile, build_folder) arg_list = [self._cmake_program] if self._generator: arg_list.append('-G "{}"'.format(self._generator)) if self._toolchain_file: - toolpath = self._toolchain_file.replace("\\", "/") + toolpath = self._toolchain_file + if subfolder: + toolpath = os.path.relpath(self._toolchain_file, start=subfolder) + toolpath = toolpath.replace("\\", "/") arg_list.append('-DCMAKE_TOOLCHAIN_FILE="{}"'.format(toolpath)) if self._conanfile.package_folder: pkg_folder = self._conanfile.package_folder.replace("\\", "/") @@ -110,6 +118,7 @@ def configure(self, variables=None, build_script_folder=None, cli_args=None, self._cache_variables.update(variables) arg_list.extend(['-D{}="{}"'.format(k, v) for k, v in self._cache_variables.items()]) + arg_list.append('"{}"'.format(cmakelist_folder)) if not cli_args or ("--log-level" not in cli_args and "--loglevel" not in cli_args): @@ -141,8 +150,10 @@ def _config_arg(self, build_type): return build_config def _build(self, build_type=None, target=None, cli_args=None, build_tool_args=None, env="", - stdout=None, stderr=None): + stdout=None, stderr=None, subfolder=None): bf = self._conanfile.build_folder + if subfolder: + bf = os.path.join(self._conanfile.build_folder, subfolder) build_config = self._config_arg(build_type) args = [] @@ -168,27 +179,32 @@ def _build(self, build_type=None, target=None, cli_args=None, build_tool_args=No self._conanfile.run(command, env=env, stdout=stdout, stderr=stderr) def build(self, build_type=None, target=None, cli_args=None, build_tool_args=None, - stdout=None, stderr=None): + stdout=None, stderr=None, subfolder=None): """ :param build_type: Use it only to override the value defined in the ``settings.build_type`` for a multi-configuration generator (e.g. Visual Studio, XCode). This value will be ignored for single-configuration generators, they will use the one defined in the toolchain file during the install step. - :param target: The name of a single build target as a string, or names of multiple build targets in a - list of strings to be passed to the ``--target`` argument. + :param target: The name of a single build target as a string, or names of multiple build + targets in a list of strings to be passed to the ``--target`` argument. :param cli_args: A list of arguments ``[arg1, arg2, ...]`` that will be passed to the ``cmake --build ... arg1 arg2`` command directly. :param build_tool_args: A list of arguments ``[barg1, barg2, ...]`` for the underlying build system that will be passed to the command line after the ``--`` indicator: ``cmake --build ... -- barg1 barg2`` + :param subfolder: (Experimental): subfolder to be created inside the ``build_folder`` and + the ``package_folder``. If not provided, files will be + placed in the ``build_folder`` and ``package_folder`` root. :param stdout: Use it to redirect stdout to this stream :param stderr: Use it to redirect stderr to this stream """ self._conanfile.output.info("Running CMake.build()") - self._build(build_type, target, cli_args, build_tool_args, stdout=stdout, stderr=stderr) + self._build(build_type, target, cli_args, build_tool_args, subfolder=subfolder, + stdout=stdout, stderr=stderr) - def install(self, build_type=None, component=None, cli_args=None, stdout=None, stderr=None): + def install(self, build_type=None, component=None, cli_args=None, stdout=None, stderr=None, + subfolder=None): """ Equivalent to running ``cmake --install`` @@ -199,25 +215,35 @@ def install(self, build_type=None, component=None, cli_args=None, stdout=None, s not build type. :param cli_args: A list of arguments ``[arg1, arg2, ...]`` for the underlying build system that will be passed to the command line: ``cmake --install ... arg1 arg2`` + :param subfolder: (Experimental): subfolder to be created inside the ``build_folder`` and + the ``package_folder``. If not provided, files will be + placed in the ``build_folder`` and ``package_folder`` root. :param stdout: Use it to redirect stdout to this stream :param stderr: Use it to redirect stderr to this stream """ self._conanfile.output.info("Running CMake.install()") - mkdir(self._conanfile, self._conanfile.package_folder) + package_folder = self._conanfile.package_folder + build_folder = self._conanfile.build_folder + if subfolder: + package_folder = os.path.join(self._conanfile.package_folder, subfolder) + build_folder = os.path.join(self._conanfile.build_folder, subfolder) + mkdir(self._conanfile, package_folder) build_config = self._config_arg(build_type) - pkg_folder = '"{}"'.format(self._conanfile.package_folder.replace("\\", "/")) - build_folder = '"{}"'.format(self._conanfile.build_folder) + pkg_folder = '"{}"'.format(package_folder.replace("\\", "/")) + build_folder = '"{}"'.format(build_folder.replace("\\", "/")) arg_list = ["--install", build_folder, build_config, "--prefix", pkg_folder] if component: arg_list.extend(["--component", component]) arg_list.extend(self._compilation_verbosity_arg) - deprecated_install_strip = self._conanfile.conf.get("tools.cmake:install_strip", check_type=bool) + deprecated_install_strip = self._conanfile.conf.get("tools.cmake:install_strip", + check_type=bool) if deprecated_install_strip: - self._conanfile.output.warning("The 'tools.cmake:install_strip' configuration is deprecated, " - "use 'tools.build:install_strip' instead.", warn_tag="deprecated") + self._conanfile.output.warning("The 'tools.cmake:install_strip' configuration is " + "deprecated, use 'tools.build:install_strip' instead.", + warn_tag="deprecated") do_strip = self._conanfile.conf.get("tools.build:install_strip", check_type=bool) if do_strip or deprecated_install_strip: @@ -265,7 +291,8 @@ def ctest(self, cli_args=None, env="", stdout=None, stderr=None): """ Equivalent to running ctest ... - :param cli_args: List of arguments ``[arg1, arg2, ...]`` to be passed as extra ctest command line arguments + :param cli_args: List of arguments ``[arg1, arg2, ...]`` to be passed as extra ctest + command line arguments :param env: the environment files to activate, by default conanbuild + conanrun :param stdout: Use it to redirect stdout to this stream :param stderr: Use it to redirect stderr to this stream @@ -282,11 +309,19 @@ def ctest(self, cli_args=None, env="", stdout=None, stderr=None): if njobs: arg_list.append(f"--parallel {njobs}") + arg_list.extend(cli_args or []) + verbosity = self._conanfile.conf.get("tools.build:verbosity", choices=("quiet", "verbose")) if verbosity: + # https://cmake.org/cmake/help/latest/manual/ctest.1.html + # Options such as --verbose, --extra-verbose, and --debug are + # ignored if --quiet is specified. arg_list.append(f"--{verbosity}") - arg_list.extend(cli_args or []) + extra_args = self._conanfile.conf.get("tools.cmake:ctest_args", check_type=list) + if extra_args: + arg_list.extend(extra_args) + arg_list = " ".join(filter(None, arg_list)) command = f"ctest {arg_list}" diff --git a/conan/tools/cmake/cmakedeps2/__init__.py b/conan/tools/cmake/cmakeconfigdeps/__init__.py similarity index 100% rename from conan/tools/cmake/cmakedeps2/__init__.py rename to conan/tools/cmake/cmakeconfigdeps/__init__.py diff --git a/conan/tools/cmake/cmakedeps2/cmakedeps.py b/conan/tools/cmake/cmakeconfigdeps/cmakeconfigdeps.py similarity index 86% rename from conan/tools/cmake/cmakedeps2/cmakedeps.py rename to conan/tools/cmake/cmakeconfigdeps/cmakeconfigdeps.py index d6b33b8cc89..008ddc67e51 100644 --- a/conan/tools/cmake/cmakedeps2/cmakedeps.py +++ b/conan/tools/cmake/cmakeconfigdeps/cmakeconfigdeps.py @@ -1,3 +1,4 @@ +import glob import os import re import textwrap @@ -9,10 +10,10 @@ from conan.internal import check_duplicated_generator from conan.internal.api.install.generators import relativize_path from conan.internal.model.dependencies import get_transitive_requires -from conan.tools.cmake.cmakedeps2.config import ConfigTemplate2 -from conan.tools.cmake.cmakedeps2.config_version import ConfigVersionTemplate2 -from conan.tools.cmake.cmakedeps2.target_configuration import TargetConfigurationTemplate2 -from conan.tools.cmake.cmakedeps2.targets import TargetsTemplate2 +from conan.tools.cmake.cmakeconfigdeps.config import ConfigTemplate2 +from conan.tools.cmake.cmakeconfigdeps.config_version import ConfigVersionTemplate2 +from conan.tools.cmake.cmakeconfigdeps.target_configuration import TargetConfigurationTemplate2 +from conan.tools.cmake.cmakeconfigdeps.targets import TargetsTemplate2 from conan.tools.files import save from conan.internal.util.files import load @@ -22,7 +23,7 @@ FIND_MODE_BOTH = "both" -class CMakeDeps2: +class CMakeConfigDeps: def __init__(self, conanfile): self._conanfile = conanfile @@ -38,6 +39,9 @@ def __init__(self, conanfile): self._properties = {} def generate(self): + self._conanfile.output.warning("CMakeConfigDeps is experimental, and might get " + "breaking changes in future releases", + warn_tag="experimental") check_duplicated_generator(self, self._conanfile) # Current directory is the generators_folder generator_files = self._content() @@ -68,14 +72,15 @@ def _content(self): if require.direct: direct_deps.append((require, dep)) - config = ConfigTemplate2(self, dep) + full_cpp_info = dep.cpp_info.deduce_full_cpp_info(dep) + config = ConfigTemplate2(self, require, dep, full_cpp_info) ret[config.filename] = config.content() config_version = ConfigVersionTemplate2(self, dep) ret[config_version.filename] = config_version.content() targets = TargetsTemplate2(self, dep) ret[targets.filename] = targets.content() - target_configuration = TargetConfigurationTemplate2(self, dep, require) + target_configuration = TargetConfigurationTemplate2(self, dep, require, full_cpp_info) ret[target_configuration.filename] = target_configuration.content() self._print_help(direct_deps) @@ -197,9 +202,16 @@ def _get_cmake_paths(self, requirements, dirs_name): def generate(self): template = textwrap.dedent("""\ + set(CMAKE_FIND_PACKAGE_PREFER_CONFIG ON) + {% for pkg_name, folder in pkg_paths.items() %} set({{pkg_name}}_DIR "{{folder}}") {% endfor %} + {% for pkg_name, folders in pkg_paths_multi.items() %} + {% for folder in folders %} + list(APPEND CONAN_{{pkg_name}}_DIR_MULTI "{{folder}}") + {% endfor %} + {% endfor %} {% if host_runtime_dirs %} set(CONAN_RUNTIME_LIB_DIRS {{ host_runtime_dirs }} ) # Only for VS, needs CMake>=3.27 @@ -231,6 +243,17 @@ def generate(self): # if not, test_cmake_add_subdirectory test fails # content.append('set(CMAKE_FIND_PACKAGE_PREFER_CONFIG ON)') pkg_paths = {} + + pkg_paths_multi = {} + if os.path.exists(self._conan_cmakedeps_paths): + existing_toolchain = load(self._conan_cmakedeps_paths) + pattern_paths = r"list\(APPEND CONAN_([A-Za-z0-9-_]*)_DIR_MULTI \"([^)]*)\"\)" + variable_match = re.findall(pattern_paths, existing_toolchain) + for (captured_name, captured_path) in variable_match: + path_list = pkg_paths_multi.setdefault(captured_name, []) + if captured_path not in path_list: + path_list.append(captured_path) + for req, dep in all_reqs: cmake_find_mode = self._cmakedeps.get_property("cmake_find_mode", dep) cmake_find_mode = cmake_find_mode or FIND_MODE_CONFIG @@ -239,6 +262,15 @@ def generate(self): pkg_name = self._cmakedeps.get_cmake_filename(dep) # https://cmake.org/cmake/help/v3.22/guide/using-dependencies/index.html if cmake_find_mode == FIND_MODE_NONE: + cps = glob.glob(os.path.join(dep.package_folder, f"**/{pkg_name}.cps"), + recursive=True) + if cps: + loc = os.path.dirname(os.path.join(dep.package_folder, cps[0])) + loc = loc.replace("\\", "/") + pkg_paths[pkg_name] = relativize_path(loc, self._conanfile, + "${CMAKE_CURRENT_LIST_DIR}") + continue + try: # This is irrespective of the components, it should be in the root cpp_info # To define the location of the pkg-config.cmake file @@ -252,6 +284,10 @@ def generate(self): if os.path.isfile(os.path.join(pkg_folder, filename)): pkg_paths[pkg_name] = relativize_path(pkg_folder, self._conanfile, "${CMAKE_CURRENT_LIST_DIR}") + + existing_paths = pkg_paths_multi.setdefault(pkg_name, []) + if pkg_folder not in existing_paths: + existing_paths.append(pkg_folder) continue # If CMakeDeps generated, the folder is this one @@ -266,6 +302,7 @@ def generate(self): cmake_module_path = self._get_cmake_paths(all_reqs, "builddirs") context = {"host_runtime_dirs": self._get_host_runtime_dirs(), "pkg_paths": pkg_paths, + "pkg_paths_multi": pkg_paths_multi, "cmake_program_path": _join_paths(self._conanfile, cmake_program_path), "cmake_library_path": _join_paths(self._conanfile, cmake_library_path), "cmake_include_path": _join_paths(self._conanfile, cmake_include_path), diff --git a/conan/tools/cmake/cmakedeps2/config.py b/conan/tools/cmake/cmakeconfigdeps/config.py similarity index 52% rename from conan/tools/cmake/cmakedeps2/config.py rename to conan/tools/cmake/cmakeconfigdeps/config.py index fe7be4a00aa..0dd6ab11ecb 100644 --- a/conan/tools/cmake/cmakedeps2/config.py +++ b/conan/tools/cmake/cmakeconfigdeps/config.py @@ -2,7 +2,7 @@ import jinja2 from jinja2 import Template - +from conan.tools.cmake.utils import parse_extra_variable from conan.internal.api.install.generators import relativize_path @@ -11,9 +11,11 @@ class ConfigTemplate2: FooConfig.cmake foo-config.cmake """ - def __init__(self, cmakedeps, conanfile): + def __init__(self, cmakedeps, require, conanfile, full_cpp_info): self._cmakedeps = cmakedeps + self._require = require self._conanfile = conanfile + self._full_cpp_info = full_cpp_info def content(self): t = Template(self._template, trim_blocks=True, lstrip_blocks=True, @@ -57,11 +59,54 @@ def _context(self): cmakename = cmakename.split("::", 1)[1] components.append(cmakename or name) components = " ".join(components) if components else "" - return {"filename": f, - "components": components, - "pkg_name": pkg_name, - "targets_include_file": targets_include, - "build_modules_paths": build_modules_paths} + + result = {"filename": f, + "components": components, + "pkg_name": pkg_name, + "targets_include_file": targets_include, + "build_modules_paths": build_modules_paths} + + conf_extra_variables = self._conanfile.conf.get("tools.cmake.cmaketoolchain:extra_variables", + default={}, check_type=dict) + dep_extra_variables = self._cmakedeps.get_property("cmake_extra_variables", self._conanfile, + check_type=dict) or {} + # The configuration variables have precedence over the dependency ones + extra_variables = {dep: value for dep, value in dep_extra_variables.items() + if dep not in conf_extra_variables} + parsed_extra_variables = {} + for key, value in extra_variables.items(): + parsed_extra_variables[key] = parse_extra_variable("cmake_extra_variables", + key, value) + result["extra_variables"] = parsed_extra_variables + + result.update(self._get_legacy_vars()) + return result + + def _get_legacy_vars(self): + # Auxiliary variables for legacy consumption and try_compile cases + pkg_name = self._conanfile.ref.name + prefixes = self._cmakedeps.get_property("cmake_additional_variables_prefixes", + self._conanfile, check_type=list) or [] + + f = self._cmakedeps.get_cmake_filename(self._conanfile) + prefixes = [f] + prefixes + include_dirs = definitions = libraries = None + if not self._require.build: # To add global variables for try_compile and legacy + aggregated_cppinfo = self._full_cpp_info.aggregated_components() + # FIXME: Proper escaping of paths for CMake + incdirs = [i.replace("\\", "/") for i in aggregated_cppinfo.includedirs] + incdirs = [relativize_path(i, self._cmakedeps._conanfile, "${CMAKE_CURRENT_LIST_DIR}") + for i in incdirs] + include_dirs = ";".join(incdirs) + definitions = "" + root_target_name = self._cmakedeps.get_property("cmake_target_name", self._conanfile) + libraries = root_target_name or f"{pkg_name}::{pkg_name}" + + return {"additional_variables_prefixes": prefixes, + "version": self._conanfile.ref.version, + "include_dirs": include_dirs, + "definitions": definitions, + "libraries": libraries} @property def _template(self): @@ -79,12 +124,6 @@ def _template(self): "adding the '-DCMAKE_BUILD_TYPE=' argument.") endif() - # build_modules_paths comes from last configuration only - {% for build_module in build_modules_paths %} - message(STATUS "Conan: Including build module from '{{build_module}}'") - include("{{ build_module }}") - {% endfor %} - {% if components %} set({{filename}}_PACKAGE_PROVIDED_COMPONENTS {{components}}) foreach(comp {%raw%}${{%endraw%}{{filename}}_FIND_COMPONENTS}) @@ -96,4 +135,33 @@ def _template(self): endif() endforeach() {% endif %} + + ################# Global variables for try compile and legacy ############## + {% for prefix in additional_variables_prefixes %} + set({{ prefix }}_VERSION_STRING "{{ version }}") + {% if include_dirs is not none %} + set({{ prefix }}_INCLUDE_DIRS "{{ include_dirs }}" ) + set({{ prefix }}_INCLUDE_DIR "{{ include_dirs }}" ) + {% endif %} + {% if libraries is not none %} + set({{ prefix }}_LIBRARIES {{ libraries }} ) + {% endif %} + {% if definitions is not none %} + set({{ prefix }}_DEFINITIONS {{ definitions}} ) + {% endif %} + {% endfor %} + + # build_modules_paths comes from last configuration only + # Some build modules in ConanCenter use try_compile variables and legacy, so this + # include() needs to happen after the above variables are defined + {% for build_module in build_modules_paths %} + message(STATUS "Conan: Including build module from '{{build_module}}'") + include("{{ build_module }}") + {% endfor %} + + # Definition of extra CMake variables from cmake_extra_variables + + {% for key, value in extra_variables.items() %} + set({{ key }} {{ value }}) + {% endfor %} """) diff --git a/conan/tools/cmake/cmakedeps2/config_version.py b/conan/tools/cmake/cmakeconfigdeps/config_version.py similarity index 97% rename from conan/tools/cmake/cmakedeps2/config_version.py rename to conan/tools/cmake/cmakeconfigdeps/config_version.py index 4a3212ba21c..51ae75d01a8 100644 --- a/conan/tools/cmake/cmakedeps2/config_version.py +++ b/conan/tools/cmake/cmakeconfigdeps/config_version.py @@ -22,7 +22,7 @@ def content(self): @property def filename(self): f = self._cmakedeps.get_cmake_filename(self._conanfile) - return f"{f}-config-version.cmake" if f == f.lower() else f"{f}ConfigVersion.cmake" + return f"{f}-config-version.cmake" if f == f.lower() else f"{f}ConfigVersion.cmake" @property def _context(self): diff --git a/conan/tools/cmake/cmakedeps2/target_configuration.py b/conan/tools/cmake/cmakeconfigdeps/target_configuration.py similarity index 86% rename from conan/tools/cmake/cmakedeps2/target_configuration.py rename to conan/tools/cmake/cmakeconfigdeps/target_configuration.py index cf11208af48..e867eba75da 100644 --- a/conan/tools/cmake/cmakedeps2/target_configuration.py +++ b/conan/tools/cmake/cmakeconfigdeps/target_configuration.py @@ -8,16 +8,18 @@ from conan.internal.api.install.generators import relativize_path from conan.internal.model.pkg_type import PackageType from conan.internal.graph.graph import CONTEXT_BUILD, CONTEXT_HOST +from conan.tools.cmake.utils import cmake_escape_value class TargetConfigurationTemplate2: """ FooTarget-release.cmake """ - def __init__(self, cmakedeps, conanfile, require): + def __init__(self, cmakedeps, conanfile, require, full_cpp_info): self._cmakedeps = cmakedeps self._conanfile = conanfile # The dependency conanfile, not the consumer one self._require = require + self._full_cpp_info = full_cpp_info def content(self): auto_link = self._cmakedeps.get_property("cmake_set_interface_link_directories", @@ -85,14 +87,18 @@ def _requires(self, info, components): f"'{required_pkg}::{required_comp}' but component " f"'{required_comp}' not found in {required_pkg}") raise ConanException(msg) + if dep.package_type is PackageType.APP: + continue # It doesn't make sense to link a package that is an App comp = None default_target = f"{dep.ref.name}::{dep.ref.name}" # replace_requires link = pkg_type is not PackageType.SHARED else: + if dep_comp.type is PackageType.APP or dep_comp.exe: + continue # It doesn't make sense to link a package that is an App comp = required_comp default_target = f"{required_pkg}::{required_comp}" link = not (pkg_type is PackageType.SHARED and - dep_comp.type is PackageType.SHARED) + dep_comp.type is PackageType.SHARED) dep_target = self._cmakedeps.get_property("cmake_target_name", dep, comp) dep_target = dep_target or default_target @@ -102,7 +108,7 @@ def _requires(self, info, components): @property def _context(self): - cpp_info = self._conanfile.cpp_info.deduce_full_cpp_info(self._conanfile) + cpp_info = self._full_cpp_info assert isinstance(cpp_info.type, PackageType) pkg_name = self._conanfile.ref.name # fallback to consumer configuration if it doesn't have build_type @@ -115,13 +121,11 @@ def _context(self): libs = {} # The BUILD context does not generate libraries targets atm - if self._conanfile.context == CONTEXT_HOST: + if not self._require.build: libs = self._get_libs(cpp_info, pkg_name, pkg_folder, pkg_folder_var) self._add_root_lib_target(libs, pkg_name, cpp_info) exes = self._get_exes(cpp_info, pkg_name, pkg_folder, pkg_folder_var) - prefixes = self._cmakedeps.get_property("cmake_additional_variables_prefixes", - self._conanfile, check_type=list) or [] seen_aliases = set() root_target_name = self._cmakedeps.get_property("cmake_target_name", self._conanfile) root_target_name = root_target_name or f"{pkg_name}::{pkg_name}" @@ -139,20 +143,6 @@ def _context(self): raise ConanException(f"Alias '{alias}' already defined as a target in " f"{self._conanfile}. ") - f = self._cmakedeps.get_cmake_filename(self._conanfile) - prefixes = [f] + prefixes - include_dirs = definitions = libraries = None - if not self._require.build: # To add global variables for try_compile and legacy - aggregated_cppinfo = cpp_info.aggregated_components() - # FIXME: Proper escaping of paths for CMake - incdirs = [i.replace("\\", "/") for i in aggregated_cppinfo.includedirs] - incdirs = [relativize_path(i, self._cmakedeps._conanfile, "${CMAKE_CURRENT_LIST_DIR}") - for i in incdirs] - include_dirs = ";".join(incdirs) - definitions = "" - root_target_name = self._cmakedeps.get_property("cmake_target_name", self._conanfile) - libraries = root_target_name or f"{pkg_name}::{pkg_name}" - pkg_folder = relativize_path(pkg_folder, self._cmakedeps._conanfile, "${CMAKE_CURRENT_LIST_DIR}") dependencies = self._get_dependencies() @@ -162,13 +152,7 @@ def _context(self): "config": config, "exes": exes, "libs": libs, - "context": self._conanfile.context, - # Extra global variables - "additional_variables_prefixes": prefixes, - "version": self._conanfile.ref.version, - "include_dirs": include_dirs, - "definitions": definitions, - "libraries": libraries, + "context": self._conanfile.context } def _get_libs(self, cpp_info, pkg_name, pkg_folder, pkg_folder_var) -> dict: @@ -179,7 +163,7 @@ def _get_libs(self, cpp_info, pkg_name, pkg_folder, pkg_folder_var) -> dict: name) target_name = target_name or f"{pkg_name}::{name}" target = self._get_cmake_lib(component, cpp_info.components, pkg_folder, - pkg_folder_var) + pkg_folder_var, comp_name=name) if target is not None: cmake_target_aliases = self._get_aliases(name) target["cmake_target_aliases"] = cmake_target_aliases @@ -194,31 +178,33 @@ def _get_libs(self, cpp_info, pkg_name, pkg_folder, pkg_folder_var) -> dict: libs[target_name] = target return libs - def _get_cmake_lib(self, info, components, pkg_folder, pkg_folder_var): - if info.exe or not (info.package_framework or info.includedirs or info.libs or info.system_libs): + def _get_cmake_lib(self, info, components, pkg_folder, pkg_folder_var, comp_name=None): + if info.exe or not (info.package_framework or info.frameworks or info.includedirs or info.libs + or info.system_libs or info.defines): return includedirs = ";".join(self._path(i, pkg_folder, pkg_folder_var) for i in info.includedirs) if info.includedirs else "" requires = self._requires(info, components) assert isinstance(requires, dict) - defines = " ".join(info.defines) - # TODO: Missing escaping? + defines = ";".join(cmake_escape_value(f) for f in info.defines) # FIXME: Filter by lib traits!!!!! if not self._require.headers: # If not depending on headers, paths and includedirs = defines = None + extra_libs = self._cmakedeps.get_property("cmake_extra_interface_libs", self._conanfile, + comp_name=comp_name, check_type=list) or [] sources = [self._path(source, pkg_folder, pkg_folder_var) for source in info.sources] target = {"type": "INTERFACE", "includedirs": includedirs, "defines": defines, "requires": requires, - "cxxflags": " ".join(info.cxxflags), - "cflags": " ".join(info.cflags), - "sharedlinkflags": " ".join(info.sharedlinkflags), - "exelinkflags": " ".join(info.exelinkflags), - "system_libs": " ".join(info.system_libs), + "cxxflags": ";".join(cmake_escape_value(f) for f in info.cxxflags), + "cflags": ";".join(cmake_escape_value(f) for f in info.cflags), + "sharedlinkflags": ";".join(cmake_escape_value(v) for v in info.sharedlinkflags), + "exelinkflags": ";".join(cmake_escape_value(v) for v in info.exelinkflags), + "system_libs": " ".join(info.system_libs + extra_libs), "sources": " ".join(sources) - } + } # System frameworks (only Apple OS) if info.frameworks: target['frameworks'] = " ".join([f"-framework {frw}" for frw in info.frameworks]) @@ -231,10 +217,12 @@ def _get_cmake_lib(self, info, components, pkg_folder, pkg_folder_var): assert lib_type, f"Unknown package type {info.type}" assert info.location, f"cpp_info.location missing for framework {info.package_framework}" target["type"] = lib_type - target["package_framework"]["location"] = self._path(info.location, pkg_folder, pkg_folder_var) - target["includedirs"] = [] # empty array as frameworks have their own way to inject headers + target["package_framework"]["location"] = self._path(info.location, pkg_folder, + pkg_folder_var) + target["includedirs"] = [] # empty as frameworks have their own way to inject headers # FIXME: This is not needed for CMake < 3.24. Remove it when Conan requires CMake >= 3.24 - target["package_framework"]["frameworkdir"] = self._path(pkg_folder, pkg_folder, pkg_folder_var) + target["package_framework"]["frameworkdir"] = self._path(pkg_folder, pkg_folder, + pkg_folder_var) if info.libs: if len(info.libs) != 1: raise ConanException(f"New CMakeDeps only allows 1 lib per component:\n" @@ -312,6 +300,9 @@ def _get_dependencies(self): transitive_reqs = self._cmakedeps.get_transitive_requires(self._conanfile) # FIXME: Hardcoded CONFIG ret = {self._cmakedeps.get_cmake_filename(r): "CONFIG" for r in transitive_reqs.values()} + extra_mods = self._cmakedeps.get_property("cmake_extra_dependencies", self._conanfile, + check_type=list) or [] + ret.update({extra_mod: "" for extra_mod in extra_mods}) return ret @staticmethod @@ -327,11 +318,6 @@ def escape(p_): return escape(p) return f"${{{pkg_folder_var}}}/{escape(p)}" - @staticmethod - def _escape_cmake_string(values): - return " ".join(v.replace("\\", "\\\\").replace('$', '\\$').replace('"', '\\"') - for v in values) - @property def _template(self): # TODO: CMake 3.24: Apple Frameworks: https://cmake.org/cmake/help/latest/manual/cmake-generator-expressions.7.html#genex:LINK_LIBRARY @@ -374,26 +360,26 @@ def _template(self): {% endif %} {% if lib_info.get("defines") %} set_property(TARGET {{lib}} APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS - {{config_wrapper(config, lib_info["defines"])}}) + "{{config_wrapper(config, lib_info["defines"])}}") {% endif %} {% if lib_info.get("cxxflags") %} set_property(TARGET {{lib}} APPEND PROPERTY INTERFACE_COMPILE_OPTIONS - $<$:{{config_wrapper(config, lib_info["cxxflags"])}}>) + "$<$:{{config_wrapper(config, lib_info["cxxflags"])}}>") {% endif %} {% if lib_info.get("cflags") %} set_property(TARGET {{lib}} APPEND PROPERTY INTERFACE_COMPILE_OPTIONS - $<$:{{config_wrapper(config, lib_info["cflags"])}}>) + "$<$:{{config_wrapper(config, lib_info["cflags"])}}>") {% endif %} {% if lib_info.get("sharedlinkflags") %} {% set linkflags = config_wrapper(config, lib_info["sharedlinkflags"]) %} set_property(TARGET {{lib}} APPEND PROPERTY INTERFACE_LINK_OPTIONS - $<$,SHARED_LIBRARY>:{{linkflags}}> - $<$,MODULE_LIBRARY>:{{linkflags}}>) + "$<$,SHARED_LIBRARY>:{{linkflags}}>" + "$<$,MODULE_LIBRARY>:{{linkflags}}>") {% endif %} {% if lib_info.get("exelinkflags") %} {% set exeflags = config_wrapper(config, lib_info["exelinkflags"]) %} set_property(TARGET {{lib}} APPEND PROPERTY INTERFACE_LINK_OPTIONS - $<$,EXECUTABLE>:{{exeflags}}>) + "$<$,EXECUTABLE>:{{exeflags}}>") {% endif %} {% if lib_info.get("link_languages") %} @@ -467,21 +453,6 @@ def _template(self): {% endif %} {% endfor %} - ################# Global variables for try compile and legacy ############## - {% for prefix in additional_variables_prefixes %} - set({{ prefix }}_VERSION_STRING "{{ version }}") - {% if include_dirs is not none %} - set({{ prefix }}_INCLUDE_DIRS "{{ include_dirs }}" ) - set({{ prefix }}_INCLUDE_DIR "{{ include_dirs }}" ) - {% endif %} - {% if libraries is not none %} - set({{ prefix }}_LIBRARIES {{ libraries }} ) - {% endif %} - {% if definitions is not none %} - set({{ prefix }}_DEFINITIONS {{ definitions}} ) - {% endif %} - {% endfor %} - ################# Exes information ############## {% for exe, location in exes.items() %} #################### {{exe}} #################### diff --git a/conan/tools/cmake/cmakedeps2/targets.py b/conan/tools/cmake/cmakeconfigdeps/targets.py similarity index 100% rename from conan/tools/cmake/cmakedeps2/targets.py rename to conan/tools/cmake/cmakeconfigdeps/targets.py diff --git a/conan/tools/cmake/cmakedeps/cmakedeps.py b/conan/tools/cmake/cmakedeps/cmakedeps.py index d65ad659f87..db797866d2b 100644 --- a/conan/tools/cmake/cmakedeps/cmakedeps.py +++ b/conan/tools/cmake/cmakedeps/cmakedeps.py @@ -18,7 +18,7 @@ from conan.internal.model.dependencies import get_transitive_requires -class CMakeDeps(object): +class CMakeDeps: def __init__(self, conanfile): self._conanfile = conanfile @@ -39,7 +39,7 @@ def __init__(self, conanfile): def generate(self): """ - This method will save the generated files to the conanfile.generators_folder + This method will save the generated files to the ``conanfile.generators_folder`` folder """ check_duplicated_generator(self, self._conanfile) @@ -135,10 +135,10 @@ def _generate_files(self, require, dep, ret, find_module_mode): def set_property(self, dep, prop, value, build_context=False): """ Using this method you can overwrite the :ref:`property` values set by - the Conan recipes from the consumer. This can be done for `cmake_file_name`, `cmake_target_name`, - `cmake_find_mode`, `cmake_module_file_name`, `cmake_module_target_name`, `cmake_additional_variables_prefixes`, - `cmake_config_version_compat`, `system_package_version`, - `cmake_build_modules`, `nosoname`, and `cmake_target_aliases`. + the Conan recipes from the consumer. This can be done for ``cmake_file_name``, ``cmake_target_name``, + ``cmake_find_mode``, ``cmake_module_file_name``, ``cmake_module_target_name``, ``cmake_additional_variables_prefixes``, + ``cmake_config_version_compat``, ``system_package_version``, + ``cmake_build_modules``, ``nosoname``, ``cmake_target_aliases`` and ``cmake_extra_variables``. :param dep: Name of the dependency to set the :ref:`property`. For components use the syntax: ``dep_name::component_name``. @@ -167,7 +167,7 @@ def get_property(self, prop, dep, comp_name=None, check_type=None): else dep.cpp_info.components[comp_name].get_property(prop, check_type=check_type) def get_cmake_package_name(self, dep, module_mode=None): - """Get the name of the file for the find_package(XXX)""" + """Get the name of the file for the ``find_package(XXX)`` call""" # This is used by CMakeDeps to determine: # - The filename to generate (XXX-config.cmake or FindXXX.cmake) # - The name of the defined XXX_DIR variables @@ -182,7 +182,7 @@ def get_cmake_package_name(self, dep, module_mode=None): def get_find_mode(self, dep): """ :param dep: requirement - :return: "none" or "config" or "module" or "both" or "config" when not set + :return: One of ``"none"``, ``"config"``, ``"module"`` or ``"both"``. Defaults to ``"config"`` when not set """ tmp = self.get_property("cmake_find_mode", dep) if tmp is None: diff --git a/conan/tools/cmake/cmakedeps/templates/__init__.py b/conan/tools/cmake/cmakedeps/templates/__init__.py index 64e90955bf1..e35d7c44fe9 100644 --- a/conan/tools/cmake/cmakedeps/templates/__init__.py +++ b/conan/tools/cmake/cmakedeps/templates/__init__.py @@ -4,7 +4,7 @@ from conan.errors import ConanException -class CMakeDepsFileTemplate(object): +class CMakeDepsFileTemplate: def __init__(self, cmakedeps, require, conanfile, generating_module=False): self.cmakedeps = cmakedeps diff --git a/conan/tools/cmake/cmakedeps/templates/config.py b/conan/tools/cmake/cmakedeps/templates/config.py index f7e9fe1ba60..6cd5070419b 100644 --- a/conan/tools/cmake/cmakedeps/templates/config.py +++ b/conan/tools/cmake/cmakedeps/templates/config.py @@ -28,6 +28,23 @@ def additional_variables_prefixes(self): self.cmakedeps.get_property("cmake_additional_variables_prefixes", self.conanfile, check_type=list) or []) return list(set([self.file_name] + prefix_list)) + @property + def parsed_extra_variables(self): + # Reading configuration from "cmake_extra_variables" property + from conan.tools.cmake.utils import parse_extra_variable + conf_extra_variables = self.conanfile.conf.get("tools.cmake.cmaketoolchain:extra_variables", + default={}, check_type=dict) + dep_extra_variables = self.cmakedeps.get_property("cmake_extra_variables", self.conanfile, + check_type=dict) or {} + # The configuration variables have precedence over the dependency ones + extra_variables = {dep: value for dep, value in dep_extra_variables.items() + if dep not in conf_extra_variables} + parsed_extra_variables = {} + for key, value in extra_variables.items(): + parsed_extra_variables[key] = parse_extra_variable("cmake_extra_variables", + key, value) + return parsed_extra_variables + @property def context(self): targets_include = "" if not self.generating_module else "module-" @@ -36,6 +53,7 @@ def context(self): "version": self.conanfile.ref.version, "file_name": self.file_name, "additional_variables_prefixes": self.additional_variables_prefixes, + "extra_variables": self.parsed_extra_variables, "pkg_name": self.pkg_name, "config_suffix": self.config_suffix, "check_components_exist": self.cmakedeps.check_components_exist, @@ -83,6 +101,12 @@ def template(self): {% endfor %} + # Definition of extra CMake variables from cmake_extra_variables + + {% for key, value in extra_variables.items() %} + set({{ key }} {{ value }}) + {% endfor %} + # Only the last installed configuration BUILD_MODULES are included to avoid the collision foreach(_BUILD_MODULE {{ pkg_var(pkg_name, 'BUILD_MODULES_PATHS', config_suffix) }} ) message({% raw %}${{% endraw %}{{ file_name }}_MESSAGE_MODE} "Conan: Including build module from '${_BUILD_MODULE}'") diff --git a/conan/tools/cmake/cmakedeps/templates/target_data.py b/conan/tools/cmake/cmakedeps/templates/target_data.py index 1e5a6e2862a..cff1a5a0874 100644 --- a/conan/tools/cmake/cmakedeps/templates/target_data.py +++ b/conan/tools/cmake/cmakedeps/templates/target_data.py @@ -6,6 +6,7 @@ from conan.tools.cmake.cmakedeps.templates import CMakeDepsFileTemplate from conan.errors import ConanException from conan.internal.api.install.generators import relativize_path +from conan.tools.cmake.utils import cmake_escape_value """ @@ -268,7 +269,7 @@ def _get_dependencies_find_modes(self): return ret -class _TargetDataContext(object): +class _TargetDataContext: def __init__(self, cpp_info, pfolder_var_name, package_folder, require, library_type, is_host_windows, conanfile, cmakedeps, comp_name=None): @@ -296,15 +297,12 @@ def join_paths(paths): def join_flags(separator, values): # Flags have to be escaped - ret = separator.join(v.replace('\\', '\\\\').replace('$', '\\$').replace('"', '\\"') - for v in values) + ret = separator.join(cmake_escape_value(v) for v in values) return ret def join_defines(values, prefix=""): # Defines have to be escaped, included spaces - return "\n\t\t\t".join('"%s%s"' % (prefix, v.replace('\\', '\\\\').replace('$', '\\$'). - replace('"', '\\"')) - for v in values) + return "\n\t\t\t".join('"%s%s"' % (prefix, cmake_escape_value(v)) for v in values) self.include_paths = join_paths(cpp_info.includedirs) self.lib_paths = join_paths(cpp_info.libdirs) diff --git a/conan/tools/cmake/layout.py b/conan/tools/cmake/layout.py index 9cf75c98a4a..19fb2d954ad 100644 --- a/conan/tools/cmake/layout.py +++ b/conan/tools/cmake/layout.py @@ -8,7 +8,10 @@ def cmake_layout(conanfile, generator=None, src_folder=".", build_folder="build"): """ :param conanfile: The current recipe object. Always use ``self``. - :param generator: Allow defining the CMake generator. In most cases it doesn't need to be passed, as it will get the value from the configuration ``tools.cmake.cmaketoolchain:generator``, or it will automatically deduce the generator from the ``settings`` + :param generator: Allow defining the CMake generator. In most cases it doesn't need to be passed, + as it will get the value from the configuration + ``tools.cmake.cmaketoolchain:generator``, or it will automatically deduce + the generator from the ``settings`` :param src_folder: Value for ``conanfile.folders.source``, change it if your source code (and CMakeLists.txt) is in a subfolder. :param build_folder: Specify the name of the "base" build folder. The default is "build", but @@ -85,6 +88,8 @@ def get_build_folder_custom_vars(conanfile): tmp = None if group == "settings": tmp = conanfile.settings.get_safe(var) + if tmp and var == "arch": # handle Apple multi-arch/universal binaries + tmp = tmp.replace("|", "_") elif group == "options": value = conanfile.options.get_safe(var) if value is not None: diff --git a/conan/tools/cmake/presets.py b/conan/tools/cmake/presets.py index 6ebe55a0228..ba1c7151ac5 100644 --- a/conan/tools/cmake/presets.py +++ b/conan/tools/cmake/presets.py @@ -18,8 +18,8 @@ def write_cmake_presets(conanfile, toolchain_file, generator, cache_variables, user_presets_path=None, preset_prefix=None, buildenv=None, runenv=None, cmake_executable=None, absolute_paths=None): preset_path, preset_data = _CMakePresets.generate(conanfile, toolchain_file, generator, - cache_variables, preset_prefix, buildenv, runenv, - cmake_executable, absolute_paths) + cache_variables, preset_prefix, buildenv, + runenv, cmake_executable, absolute_paths) _IncludingPresets.generate(conanfile, preset_path, user_presets_path, preset_prefix, preset_data, absolute_paths) @@ -28,8 +28,8 @@ class _CMakePresets: """ Conan generated main CMakePresets.json inside the generators_folder """ @staticmethod - def generate(conanfile, toolchain_file, generator, cache_variables, preset_prefix, buildenv, runenv, - cmake_executable, absolute_paths): + def generate(conanfile, toolchain_file, generator, cache_variables, preset_prefix, buildenv, + runenv, cmake_executable, absolute_paths): toolchain_file = os.path.abspath(os.path.join(conanfile.generators_folder, toolchain_file)) if not absolute_paths: try: # Make it relative to the build dir if possible @@ -104,7 +104,8 @@ def _contents(conanfile, toolchain_file, cache_variables, generator, preset_pref """ multiconfig = is_multi_configuration(generator) conf = _CMakePresets._configure_preset(conanfile, generator, cache_variables, toolchain_file, - multiconfig, preset_prefix, buildenv, cmake_executable) + multiconfig, preset_prefix, buildenv, + cmake_executable) build = _CMakePresets._build_preset_fields(conanfile, multiconfig, preset_prefix) test = _CMakePresets._test_preset_fields(conanfile, multiconfig, preset_prefix, runenv) ret = {"version": 3, @@ -174,11 +175,11 @@ def _configure_preset(conanfile, generator, cache_variables, toolchain_file, mul if conanfile.build_folder: # If we are installing a ref: "conan install ", we don't have build_folder, because # we don't even have a conanfile with a `layout()` to determine the build folder. - # If we install a local conanfile: "conan install ." with a layout(), it will be available. + # If we install a local conanfile "conan install ." with a layout(), it will be available ret["binaryDir"] = conanfile.build_folder def _format_val(val): - return f'"{val}"' if type(val) == str and " " in val else f"{val}" + return f'"{val}"' if type(val) is str and " " in val else f"{val}" # https://github.com/conan-io/conan/pull/12034#issuecomment-1253776285 cache_variables_info = " ".join( @@ -216,12 +217,16 @@ def _common_preset_fields(conanfile, multiconfig, preset_prefix): def _build_preset_fields(conanfile, multiconfig, preset_prefix): ret = _CMakePresets._common_preset_fields(conanfile, multiconfig, preset_prefix) build_preset_jobs = build_jobs(conanfile) - ret["jobs"] = build_preset_jobs + if build_preset_jobs: + ret["jobs"] = build_preset_jobs return ret @staticmethod def _test_preset_fields(conanfile, multiconfig, preset_prefix, runenv): ret = _CMakePresets._common_preset_fields(conanfile, multiconfig, preset_prefix) + build_preset_jobs = build_jobs(conanfile) + if build_preset_jobs: + ret.setdefault("execution", {})["jobs"] = build_preset_jobs if runenv: ret["environment"] = runenv return ret diff --git a/conan/tools/cmake/toolchain/blocks.py b/conan/tools/cmake/toolchain/blocks.py index 902993fd110..3bd91eab96b 100644 --- a/conan/tools/cmake/toolchain/blocks.py +++ b/conan/tools/cmake/toolchain/blocks.py @@ -266,7 +266,9 @@ def context(self): thread_flags_list = " ".join(threads_flags(self._conanfile)) if not arch_flag and not arch_link_flag and not thread_flags_list: return - return {"arch_flag": arch_flag, "arch_link_flag": arch_link_flag, "thread_flags_list": thread_flags_list} + return {"arch_flag": arch_flag, "arch_link_flag": arch_link_flag, + "thread_flags_list": thread_flags_list} + class LinkerScriptsBlock(Block): template = textwrap.dedent("""\ @@ -834,7 +836,7 @@ def context(self): "cflags": cflags, "sharedlinkflags": sharedlinkflags, "exelinkflags": exelinkflags, - "defines": [define.replace('"', '\\"') for define in defines] + "defines": [define.replace('"', '\\"') for define in defines], } @@ -1003,11 +1005,11 @@ def get_toolset(generator, conanfile): toolset += ",version=14.{}{}".format(compiler_version[-1], compiler_update) elif compiler == "clang": if generator and "Visual" in generator: - if "Visual Studio 16" in generator or "Visual Studio 17" in generator: + if any(f"Visual Studio {v}" in generator for v in ("16", "17", "18")): toolset = "ClangCL" else: raise ConanException("CMakeToolchain with compiler=clang and a CMake " - "'Visual Studio' generator requires VS16 or VS17") + "'Visual Studio' generator requires VS16, VS17 or VS18") toolset_arch = conanfile.conf.get("tools.cmake.cmaketoolchain:toolset_arch") if toolset_arch is not None: toolset_arch = "host={}".format(toolset_arch) @@ -1199,44 +1201,32 @@ class ExtraVariablesBlock(Block): {% endif %} """) - CMAKE_CACHE_TYPES = ["BOOL", "FILEPATH", "PATH", "STRING", "INTERNAL"] - - def get_exact_type(self, key, value): - if isinstance(value, str): - return f"\"{value}\"" - elif isinstance(value, (int, float)): - return value - elif isinstance(value, dict): - var_value = self.get_exact_type(key, value.get("value")) - is_force = value.get("force") - if is_force: - if not isinstance(is_force, bool): - raise ConanException(f'tools.cmake.cmaketoolchain:extra_variables "{key}" "force" must be a boolean') - is_cache = value.get("cache") - if is_cache: - if not isinstance(is_cache, bool): - raise ConanException(f'tools.cmake.cmaketoolchain:extra_variables "{key}" "cache" must be a boolean') - var_type = value.get("type") - if not var_type: - raise ConanException(f'tools.cmake.cmaketoolchain:extra_variables "{key}" needs "type" defined for cache variable') - if var_type not in self.CMAKE_CACHE_TYPES: - raise ConanException(f'tools.cmake.cmaketoolchain:extra_variables "{key}" invalid type "{var_type}" for cache variable. Possible types: {", ".join(self.CMAKE_CACHE_TYPES)}') - # Set docstring as variable name if not defined - docstring = value.get("docstring") or key - force_str = " FORCE" if is_force else "" # Support python < 3.11 - return f"{var_value} CACHE {var_type} \"{docstring}\"{force_str}" - else: - if is_force: - raise ConanException(f'tools.cmake.cmaketoolchain:extra_variables "{key}" "force" is only allowed for cache variables') - return var_value - def context(self): + from conan.tools.cmake.utils import parse_extra_variable # Reading configuration from "tools.cmake.cmaketoolchain:extra_variables" extra_variables = self._conanfile.conf.get("tools.cmake.cmaketoolchain:extra_variables", default={}, check_type=dict) + compilation_verbosity = self._conanfile.conf.get("tools.compilation:verbosity", + choices=("quiet", "verbose")) + build_verbosity = self._conanfile.conf.get("tools.build:verbosity", + choices=("quiet", "verbose")) + if build_verbosity == "quiet": + build_verbosity = "error" + + if compilation_verbosity == "verbose": + extra_variables.setdefault("CMAKE_VERBOSE_MAKEFILE", + {"cache": True, "type": "BOOL", + "value": "ON"}) + + if build_verbosity: + extra_variables.setdefault("CMAKE_MESSAGE_LOG_LEVEL", + {"cache": True, "type": "STRING", + "value": build_verbosity.upper()}) + parsed_extra_variables = {} for key, value in extra_variables.items(): - parsed_extra_variables[key] = self.get_exact_type(key, value) + parsed_extra_variables[key] = parse_extra_variable("tools.cmake.cmaketoolchain:extra_variables", + key, value) return {"extra_variables": parsed_extra_variables} diff --git a/conan/tools/cmake/toolchain/toolchain.py b/conan/tools/cmake/toolchain/toolchain.py index daeb86e90fe..6ff9a8b1138 100644 --- a/conan/tools/cmake/toolchain/toolchain.py +++ b/conan/tools/cmake/toolchain/toolchain.py @@ -9,12 +9,16 @@ from conan.tools.build import use_win_mingw from conan.tools.cmake.presets import write_cmake_presets from conan.tools.cmake.toolchain import CONAN_TOOLCHAIN_FILENAME -from conan.tools.cmake.toolchain.blocks import ExtraVariablesBlock, ToolchainBlocks, UserToolchain, \ - GenericSystemBlock, \ - AndroidSystemBlock, AppleSystemBlock, FPicBlock, ArchitectureBlock, GLibCXXBlock, VSRuntimeBlock, \ - CppStdBlock, ParallelBlock, CMakeFlagsInitBlock, TryCompileBlock, FindFiles, PkgConfigBlock, \ - SkipRPath, SharedLibBock, OutputDirsBlock, ExtraFlagsBlock, CompilersBlock, LinkerScriptsBlock, \ - VSDebuggerEnvironment, VariablesBlock, PreprocessorBlock +from conan.tools.cmake.toolchain.blocks import (ExtraVariablesBlock, ToolchainBlocks, + UserToolchain, GenericSystemBlock, + AndroidSystemBlock, AppleSystemBlock, FPicBlock, + ArchitectureBlock, GLibCXXBlock, VSRuntimeBlock, + CppStdBlock, ParallelBlock, CMakeFlagsInitBlock, + TryCompileBlock, FindFiles, PkgConfigBlock, + SkipRPath, SharedLibBock, OutputDirsBlock, + ExtraFlagsBlock, CompilersBlock, LinkerScriptsBlock, + VSDebuggerEnvironment, VariablesBlock, + PreprocessorBlock) from conan.tools.cmake.utils import is_multi_configuration from conan.tools.env import VirtualBuildEnv, VirtualRunEnv from conan.tools.intel import IntelCC @@ -201,15 +205,20 @@ def generate(self): if self._conanfile.conf.get("tools.cmake.cmaketoolchain:presets_environment", default="", check_type=str, choices=("disabled", "")) != "disabled": - build_env = self.presets_build_environment.vars(self._conanfile) if self.presets_build_environment else VirtualBuildEnv(self._conanfile, auto_generate=True).vars() - run_env = self.presets_run_environment.vars(self._conanfile) if self.presets_run_environment else VirtualRunEnv(self._conanfile, auto_generate=True).vars() + build_env = self.presets_build_environment.vars(self._conanfile) \ + if self.presets_build_environment \ + else VirtualBuildEnv(self._conanfile, auto_generate=True).vars() + run_env = self.presets_run_environment.vars(self._conanfile) \ + if self.presets_run_environment \ + else VirtualRunEnv(self._conanfile, auto_generate=True).vars() buildenv = {name: value for name, value in build_env.items(variable_reference="$penv{{{name}}}")} runenv = {name: value for name, value in run_env.items(variable_reference="$penv{{{name}}}")} - cmake_executable = self._conanfile.conf.get("tools.cmake:cmake_program", None) or self._find_cmake_exe() + cmake_executable = self._conanfile.conf.get("tools.cmake:cmake_program", None) + cmake_executable = cmake_executable or self._find_cmake_exe() user_presets = self.user_presets_path try: # TODO: Refactor this repeated pattern to deduce "is-consumer" @@ -249,7 +258,8 @@ def _get_generator(self, recipe_generator): '14': '14 2015', '15': '15 2017', '16': '16 2019', - '17': '17 2022'} + '17': '17 2022', + '18': '18 2026'} if compiler == "msvc": if compiler_version is None: diff --git a/conan/tools/cmake/utils.py b/conan/tools/cmake/utils.py index ffd9fcbd640..757a63a3568 100644 --- a/conan/tools/cmake/utils.py +++ b/conan/tools/cmake/utils.py @@ -1,5 +1,45 @@ +from conan.errors import ConanException + def is_multi_configuration(generator): if not generator: return False return "Visual" in generator or "Xcode" in generator or "Multi-Config" in generator + + +def parse_extra_variable(source, key, value): + CMAKE_CACHE_TYPES = ["BOOL", "FILEPATH", "PATH", "STRING", "INTERNAL"] + if isinstance(value, str): + return f"\"{value}\"" + elif isinstance(value, (int, float)): + return value + elif isinstance(value, dict): + var_value = parse_extra_variable(source, key, value.get("value")) + is_force = value.get("force") + if is_force: + if not isinstance(is_force, bool): + raise ConanException(f'{source} "{key}" "force" must be a boolean') + is_cache = value.get("cache") + if is_cache: + if not isinstance(is_cache, bool): + raise ConanException(f'{source} "{key}" "cache" must be a boolean') + var_type = value.get("type") + if not var_type: + raise ConanException(f'{source} "{key}" needs "type" defined for cache variable') + if var_type not in CMAKE_CACHE_TYPES: + raise ConanException(f'{source} "{key}" invalid type "{var_type}" for cache variable.' + f' Possible types: {", ".join(CMAKE_CACHE_TYPES)}') + # Set docstring as variable name if not defined + docstring = value.get("docstring") or key + force_str = " FORCE" if is_force else "" # Support python < 3.11 + return f"{var_value} CACHE {var_type} \"{docstring}\"{force_str}" + else: + if is_force: + raise ConanException(f'{source} "{key}" "force" is only allowed for cache variables') + return var_value + raise ConanException(f'{source} "{key}" has invalid type. Allowed types: str, int, float, dict,' + f' got {type(value)}') + + +def cmake_escape_value(v): + return v.replace('\\', '\\\\').replace('$', '\\$').replace('"', '\\"') diff --git a/conan/tools/env/environment.py b/conan/tools/env/environment.py index 24703c6a4e5..5c158129383 100644 --- a/conan/tools/env/environment.py +++ b/conan/tools/env/environment.py @@ -1,5 +1,6 @@ import os import textwrap +from shlex import quote from collections import OrderedDict from contextlib import contextmanager @@ -81,6 +82,9 @@ def __init__(self, name, value=None, separator=" ", path=False): self._path = path self._sep = separator + def __bool__(self): + return bool(self._values) # Empty means unset + def dumps(self): result = [] path = "(path)" if self._path else "" @@ -345,6 +349,7 @@ def __init__(self, conanfile, values, scope): self._conanfile = conanfile self._scope = scope self._subsystem = deduce_subsystem(conanfile, scope) + self._deactivation_mode = conanfile.conf.get("tools.env:deactivation_mode", default=None, check_type=str) @property def _pathsep(self): @@ -403,6 +408,14 @@ def apply(self): os.environ.clear() os.environ.update(old_env) + def save_dotenv(self, file_location): + result = [] + for varname, varvalues in self._values.items(): + value = varvalues.get_value(subsystem=self._subsystem, pathsep=self._pathsep) + result.append('{}="{}"'.format(varname, value)) + content = "\n".join(result) + save(file_location, content) + def save_bat(self, file_location, generate_deactivate=True): _, filename = os.path.split(file_location) deactivate_file = "deactivate_{}".format(filename) @@ -442,44 +455,24 @@ def save_bat(self, file_location, generate_deactivate=True): with open(file_location, "w", encoding="utf-8") as f: f.write(content) - def save_ps1(self, file_location, generate_deactivate=True,): + def save_ps1(self, file_location, generate_deactivate=True): _, filename = os.path.split(file_location) - deactivate_file = "deactivate_{}".format(filename) - deactivate = textwrap.dedent("""\ - Push-Location $PSScriptRoot - "echo `"Restoring environment`"" | Out-File -FilePath "{deactivate_file}" - $vars = (Get-ChildItem env:*).name - $updated_vars = @({vars}) - - foreach ($var in $updated_vars) - {{ - if ($var -in $vars) - {{ - $var_value = (Get-ChildItem env:$var).value - Add-Content "{deactivate_file}" "`n`$env:$var = `"$var_value`"" - }} - else - {{ - Add-Content "{deactivate_file}" "`nif (Test-Path env:$var) {{ Remove-Item env:$var }}" - }} - }} - Pop-Location - """).format( - deactivate_file=deactivate_file, - vars=",".join(['"{}"'.format(var) for var in self._values.keys()]) - ) - capture = textwrap.dedent("""\ - {deactivate} - """).format(deactivate=deactivate if generate_deactivate else "") - result = [capture] + result = [] + if generate_deactivate: + result.append(_ps1_deactivate_contents(self._deactivation_mode, self._values, filename)) abs_base_path, new_path = relativize_paths(self._conanfile, "$PSScriptRoot") for varname, varvalues in self._values.items(): value = varvalues.get_str("$env:{name}", subsystem=self._subsystem, pathsep=self._pathsep, root_path=abs_base_path, script_path=new_path) - if value: + if generate_deactivate and self._deactivation_mode == "function": + # Check environment variable existence before saving value + result.append( + f'if ($env:{varname}) {{ $env:{_old_env_prefix(filename)}_{varname} = $env:{varname} }}' + ) + if varvalues: value = value.replace('"', '`"') # escape quotes - result.append('$env:{}="{}"'.format(varname, value)) + result.append(f'$env:{varname}="{value}"') else: result.append('if (Test-Path env:{0}) {{ Remove-Item env:{0} }}'.format(varname)) @@ -487,38 +480,30 @@ def save_ps1(self, file_location, generate_deactivate=True,): # It is very important to save it correctly with utf-16, the Conan util save() is broken # and powershell uses utf-16 files!!! os.makedirs(os.path.dirname(os.path.abspath(file_location)), exist_ok=True) - open(file_location, "w", encoding="utf-16").write(content) + with open(file_location, "w", encoding="utf-16") as f: + f.write(content) def save_sh(self, file_location, generate_deactivate=True): filepath, filename = os.path.split(file_location) - deactivate_file = os.path.join("$script_folder", "deactivate_{}".format(filename)) - deactivate = textwrap.dedent("""\ - echo "echo Restoring environment" > "{deactivate_file}" - for v in {vars} - do - is_defined="true" - value=$(printenv $v) || is_defined="" || true - if [ -n "$value" ] || [ -n "$is_defined" ] - then - echo export "$v='$value'" >> "{deactivate_file}" - else - echo unset $v >> "{deactivate_file}" - fi - done - """.format(deactivate_file=deactivate_file, vars=" ".join(self._values.keys()))) - capture = textwrap.dedent("""\ - {deactivate} - """).format(deactivate=deactivate if generate_deactivate else "") - result = [capture] + result = [] + if generate_deactivate: + result.append(_sh_deactivate_contents(self._deactivation_mode, self._values, filename)) abs_base_path, new_path = relativize_paths(self._conanfile, "$script_folder") for varname, varvalues in self._values.items(): value = varvalues.get_str("${name}", self._subsystem, pathsep=self._pathsep, root_path=abs_base_path, script_path=new_path) value = value.replace('"', '\\"') - if value: - result.append('export {}="{}"'.format(varname, value)) + if generate_deactivate and self._deactivation_mode == "function": + # Check environment variable existence before saving value + result.append( + f'if [ -n "${{{varname}+x}}" ]; then ' + f'export {_old_env_prefix(filename)}_{varname}="${{{varname}}}"; ' + f'fi;' + ) + if varvalues: + result.append(f'export {varname}="{value}"') else: - result.append('unset {}'.format(varname)) + result.append(f'unset {varname}') content = "\n".join(result) content = f'script_folder="{os.path.abspath(filepath)}"\n' + content @@ -568,10 +553,114 @@ def save_script(self, filename): else: self.save_sh(path) + if self._conanfile.conf.get("tools.env:dotenv", check_type=bool): + bt = self._conanfile.settings.get_safe("build_type") + arch = self._conanfile.settings.get_safe("arch") + name = name.replace(bt.lower(), bt) if bt else name + name = name.replace(arch.lower(), arch) if arch else name + ConanOutput().warning(f"Creating dotenv file: {name}.env\n" + "Files generated with absolute paths, not interpolated.\n" + "When https://github.com/microsoft/vscode-cpptools/issues/13781 " + "solved, it will get interpolation", warn_tag="experimental") + self.save_dotenv(f"{name}.env") + if self._scope: register_env_script(self._conanfile, path, self._scope) +def _deactivate_func_name(filename): + return os.path.splitext(os.path.basename(filename))[0].replace("-", "_") + + +def _old_env_prefix(filename): + return f"_CONAN_OLD_{_deactivate_func_name(filename).upper()}" + + +def _ps1_deactivate_contents(deactivation_mode, values, filename): + vars_list = ", ".join(f'"{v}"' for v in values.keys()) + if deactivation_mode == "function": + var_prefix = _old_env_prefix(filename) + func_name = _deactivate_func_name(filename) + return textwrap.dedent(f"""\ + function global:deactivate_{func_name} {{ + Write-Host "Restoring environment" + foreach ($v in @({vars_list})) {{ + $oldVarName = "{var_prefix}_$v" + $oldValue = Get-Item -Path "Env:$oldVarName" -ErrorAction SilentlyContinue + if (Test-Path env:$oldValue) {{ + Remove-Item -Path "Env:$v" -ErrorAction SilentlyContinue + }} else {{ + Set-Item -Path "Env:$v" -Value $oldValue.Value + }} + Remove-Item -Path "Env:$oldVarName" -ErrorAction SilentlyContinue + }} + Remove-Item -Path function:deactivate_{func_name} -ErrorAction SilentlyContinue + }} + """) + + deactivate_file = "deactivate_{}".format(filename) + return textwrap.dedent(f"""\ + Push-Location $PSScriptRoot + "echo `"Restoring environment`"" | Out-File -FilePath "{deactivate_file}" + $vars = (Get-ChildItem env:*).name + $updated_vars = @({vars_list}) + + foreach ($var in $updated_vars) + {{ + if ($var -in $vars) + {{ + $var_value = (Get-ChildItem env:$var).value + Add-Content "{deactivate_file}" "`n`$env:$var = `"$var_value`"" + }} + else + {{ + Add-Content "{deactivate_file}" "`nif (Test-Path env:$var) {{ Remove-Item env:$var }}" + }} + }} + Pop-Location + """) + + +def _sh_deactivate_contents(deactivation_mode, values, filename): + vars_list = " ".join(quote(v) for v in values.keys()) + if deactivation_mode == "function": + func_name = _deactivate_func_name(filename) + return textwrap.dedent(f"""\ + # sh-like function to restore environment + deactivate_{func_name} () {{ + echo "Restoring environment" + for v in {vars_list}; do + old_var="{_old_env_prefix(filename)}_${{v}}" + # Use eval for indirect expansion (POSIX safe) + eval "is_set=\\${{${{old_var}}+x}}" + if [ -n "${{is_set}}" ]; then + eval "old_value=\\${{${{old_var}}}}" + eval "export ${{v}}=\\${{old_value}}" + else + unset "${{v}}" + fi + unset "${{old_var}}" + done + unset -f deactivate_{func_name} + }} + """) + deactivate_file = os.path.join("$script_folder", "deactivate_{}".format(filename)) + return textwrap.dedent(f"""\ + echo "echo Restoring environment" > "{deactivate_file}" + for v in {vars_list} + do + is_defined="true" + value=$(printenv $v) || is_defined="" || true + if [ -n "$value" ] || [ -n "$is_defined" ] + then + echo export "$v='$value'" >> "{deactivate_file}" + else + echo unset $v >> "{deactivate_file}" + fi + done + """) + + class ProfileEnvironment: def __init__(self): self._environments = OrderedDict() diff --git a/conan/tools/files/__init__.py b/conan/tools/files/__init__.py index e183b6c4e33..1e5f2aeeec0 100644 --- a/conan/tools/files/__init__.py +++ b/conan/tools/files/__init__.py @@ -3,7 +3,6 @@ move_folder_contents, chmod from conan.tools.files.patches import patch, apply_conandata_patches, export_conandata_patches -from conan.tools.files.packager import AutoPackager from conan.tools.files.symlinks import symlinks from conan.tools.files.copy_pattern import copy from conan.tools.files.conandata import update_conandata, trim_conandata diff --git a/conan/tools/files/conandata.py b/conan/tools/files/conandata.py index a9b0a70ac3d..84eb4d0859b 100644 --- a/conan/tools/files/conandata.py +++ b/conan/tools/files/conandata.py @@ -47,7 +47,8 @@ def trim_conandata(conanfile, raise_if_missing=True): only """ if not hasattr(conanfile, "export_folder") or conanfile.export_folder is None: - raise ConanException("The 'trim_conandata()' tool can only be used in the 'export()' method or 'post_export()' hook") + raise ConanException("The 'trim_conandata()' tool can only be used in " + "the 'export()' method or 'post_export()' hook") path = os.path.join(conanfile.export_folder, "conandata.yml") if not os.path.exists(path): if raise_if_missing: diff --git a/conan/tools/files/copy_pattern.py b/conan/tools/files/copy_pattern.py index 588eab93bfe..f1640264fb6 100644 --- a/conan/tools/files/copy_pattern.py +++ b/conan/tools/files/copy_pattern.py @@ -62,6 +62,7 @@ def _filter_files(src, pattern, excludes, ignore_case, excluded_folder): filenames = [] files_symlinked_to_folders = [] + pattern = pattern.lower() if ignore_case else pattern if excludes: if not isinstance(excludes, (tuple, list)): excludes = (excludes, ) @@ -77,9 +78,10 @@ def _filter_files(src, pattern, excludes, ignore_case, excluded_folder): # Check if any of the subfolders is a symlink for subfolder in subfolders: - relative_path = os.path.relpath(os.path.join(root, subfolder), src) if os.path.islink(os.path.join(root, subfolder)): - if fnmatch.fnmatch(os.path.normpath(relative_path.lower()), pattern): + relative_path = os.path.relpath(os.path.join(root, subfolder), src) + compare_relative_path = relative_path.lower() if ignore_case else relative_path + if fnmatch.fnmatch(os.path.normpath(compare_relative_path), pattern): files_symlinked_to_folders.append(relative_path) relative_path = os.path.relpath(root, src) @@ -96,7 +98,6 @@ def _filter_files(src, pattern, excludes, ignore_case, excluded_folder): filenames.append(relative_name) if ignore_case: - pattern = pattern.lower() files_to_copy = [n for n in filenames if fnmatch.fnmatch(os.path.normpath(n.lower()), pattern)] else: diff --git a/conan/tools/files/files.py b/conan/tools/files/files.py index 729d1f2b404..bf58a15ff4d 100644 --- a/conan/tools/files/files.py +++ b/conan/tools/files/files.py @@ -94,7 +94,7 @@ def rm(conanfile, pattern, folder, recursive=False, excludes=None): def get(conanfile, url, md5=None, sha1=None, sha256=None, destination=".", filename="", keep_permissions=False, pattern=None, verify=True, retry=None, retry_wait=None, - auth=None, headers=None, strip_root=False, extract_filter=None): + auth=None, headers=None, strip_root=False, extract_filter=None, excludes=None): """ High level download and decompressing of a tgz, zip or other compressed format file. Just a high level wrapper for download, unzip, and remove the temporary zip file once unzipped. @@ -118,6 +118,7 @@ def get(conanfile, url, md5=None, sha1=None, sha256=None, destination=".", filen :param headers: forwarded to ``tools.file.download()``. :param strip_root: forwarded to ``tools.file.unzip()``. :param extract_filter: forwarded to ``tools.file.unzip()``. + :param excludes: forwarded to ``tools.file.unzip()``. """ if not filename: # deduce filename from the URL @@ -131,7 +132,8 @@ def get(conanfile, url, md5=None, sha1=None, sha256=None, destination=".", filen retry=retry, retry_wait=retry_wait, auth=auth, headers=headers, md5=md5, sha1=sha1, sha256=sha256) unzip(conanfile, filename, destination=destination, keep_permissions=keep_permissions, - pattern=pattern, strip_root=strip_root, extract_filter=extract_filter) + pattern=pattern, strip_root=strip_root, extract_filter=extract_filter, + excludes=excludes) os.unlink(filename) @@ -265,7 +267,8 @@ def chdir(conanfile, newdir): os.chdir(old_path) -def chmod(conanfile, path:str, read:Optional[bool]=None, write:Optional[bool]=None, execute:Optional[bool]=None, recursive:bool=False): +def chmod(conanfile, path: str, read: Optional[bool] = None, write: Optional[bool] = None, + execute: Optional[bool] = None, recursive: bool = False): """Change file or directory permissions cross-platform. .. versionadded:: 2.15 @@ -344,7 +347,7 @@ def _change_permission(it_path:str): def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern=None, - strip_root=False, extract_filter=None): + strip_root=False, extract_filter=None, excludes=None): """ Extract different compressed formats @@ -362,6 +365,9 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern= :param extract_filter: (Optional, defaulted to None). When extracting a tar file, use the tar extracting filters define by Python in https://docs.python.org/3/library/tarfile.html + :param excludes: (Optional, defaulted to None). When extracting a file, + exclude paths matching any of the patterns. This should be a Unix shell-style wildcard, + see fnmatch documentation for more details. """ output = conanfile.output @@ -370,7 +376,8 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern= if (filename.endswith(".tar.gz") or filename.endswith(".tgz") or filename.endswith(".tbz2") or filename.endswith(".tar.bz2") or filename.endswith(".tar")): - return untargz(filename, destination, pattern, strip_root, extract_filter) + return untargz(filename, destination, pattern, strip_root, extract_filter, + excludes=excludes) if filename.endswith(".gz"): target_name = filename[:-3] if destination == "." else destination target_dir = os.path.dirname(target_name) @@ -381,7 +388,8 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern= shutil.copyfileobj(fin, fout) return if filename.endswith(".tar.xz") or filename.endswith(".txz"): - return untargz(filename, destination, pattern, strip_root, extract_filter) + return untargz(filename, destination, pattern, strip_root, extract_filter, + excludes=excludes) import zipfile full_path = os.path.normpath(os.path.join(os.getcwd(), destination)) @@ -390,6 +398,9 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern= zip_info = z.infolist() if pattern: zip_info = [zi for zi in zip_info if fnmatch(zi.filename, pattern)] + if excludes: + zip_info = [zi for zi in zip_info + if not any(fnmatch(zi.filename, pat) for pat in excludes)] if strip_root: names = [zi.filename.replace("\\", "/") for zi in zip_info] common_folder = os.path.commonprefix(names).split("/", 1)[0] @@ -433,7 +444,8 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern= output.writeln("") -def untargz(filename, destination=".", pattern=None, strip_root=False, extract_filter=None): +def untargz(filename, destination=".", pattern=None, strip_root=False, extract_filter=None, + excludes=None): # NOT EXPOSED at `conan.tools.files` but used in tests import tarfile with tarfile.TarFile.open(filename, mode='r:*') as tarredgzippedFile: @@ -443,7 +455,7 @@ def untargz(filename, destination=".", pattern=None, strip_root=False, extract_f # File I/O functions in the Windows API convert "/" to "\" as part of converting # the name to an NT-style name, except when using the "\\?\" prefix using_long_path_prefix = destination.startswith("\\\\?\\") - if not pattern and not strip_root and not using_long_path_prefix: + if not pattern and not excludes and not strip_root and not using_long_path_prefix: tarredgzippedFile.extractall(destination) else: common_folder = None @@ -451,6 +463,8 @@ def untargz(filename, destination=".", pattern=None, strip_root=False, extract_f for member in tarredgzippedFile: if pattern and not fnmatch(member.name, pattern): continue # Skip files that don’t match the pattern + if excludes and any(fnmatch(member.name, pat) for pat in excludes): + continue # Skip files that match the excludes if strip_root: name = member.name.replace("\\", "/") diff --git a/conan/tools/files/packager.py b/conan/tools/files/packager.py deleted file mode 100644 index cc03fd0c2b6..00000000000 --- a/conan/tools/files/packager.py +++ /dev/null @@ -1,99 +0,0 @@ -import os - -from conan.errors import ConanException - - -class _PatternEntry(object): - - def __init__(self): - self.include = [] - self.lib = [] - self.bin = [] - self.src = [] - self.build = [] - self.res = [] - self.framework = [] - - -class _Patterns(object): - - def __init__(self): - self.source = _PatternEntry() - self.build = _PatternEntry() - - -class AutoPackager(object): - - def __init__(self, conanfile): - """ - :param conanfile: The current recipe object. Always use ``self``. - """ - self._conanfile = conanfile - self.patterns = _Patterns() - - self.patterns.source.include = ["*.h", "*.hpp", "*.hxx"] - self.patterns.source.lib = [] - self.patterns.source.bin = [] - - self.patterns.build.include = ["*.h", "*.hpp", "*.hxx"] - self.patterns.build.lib = ["*.so", "*.so.*", "*.a", "*.lib", "*.dylib"] - self.patterns.build.bin = ["*.exe", "*.dll"] - conanfile.output.warning("AutoPackager is **** deprecated ****", warn_tag="deprecated") - conanfile.output.warning("AutoPackager **** will be removed ****", warn_tag="deprecated") - conanfile.output.warning("Use explicit copy() calls instead", warn_tag="deprecated") - - def run(self): - cf = self._conanfile - # Check that the components declared in source/build are in package - cnames = set(cf.cpp.source.components) - cnames = cnames.union(set(cf.cpp.build.components)) - if cnames.difference(set(cf.cpp.package.components)): - # TODO: Raise? Warning? Ignore? - raise ConanException("There are components declared in cpp.source.components" - " or in cpp.build.components that are not declared in" - " cpp.package.components") - if cnames: - for cname in cnames: - if cname in cf.cpp.source.components: - self._package_cppinfo("source", cf.cpp.source.components[cname], - cf.cpp.package.components[cname]) - if cname in cf.cpp.build.components: - self._package_cppinfo("build", cf.cpp.build.components[cname], - cf.cpp.package.components[cname]) - else: # No components declared - self._package_cppinfo("source", cf.cpp.source, cf.cpp.package) - self._package_cppinfo("build", cf.cpp.build, cf.cpp.package) - - def _package_cppinfo(self, origin_name, origin_cppinfo, dest_cppinfo): - """ - @param origin_name: one from ["source", "build"] - @param origin_cppinfo: cpp_info object of an origin (can be a component cppinfo too) - @param dest_cppinfo: cpp_info object of the package or a component from package - """ - - patterns_var = getattr(self.patterns, origin_name) - base_folder = getattr(self._conanfile, "{}_folder".format(origin_name)) - - for var in ["include", "lib", "bin", "framework", "src", "build", "res"]: - dirs_var_name = "{}dirs".format(var) - origin_paths = getattr(origin_cppinfo, dirs_var_name) - if not origin_paths: - continue - patterns = getattr(patterns_var, var) - destinations = getattr(dest_cppinfo, dirs_var_name) - - if not destinations: # For example: Not declared "includedirs" in package.cpp_info - continue - - if len(destinations) > 1: - # Check if there is only one possible destination at package, otherwise the - # copy would need to be done manually - err_msg = "The package has more than 1 cpp_info.{}, cannot package automatically" - raise ConanException(err_msg.format(dirs_var_name)) - - dst_folder = os.path.join(self._conanfile.folders.base_package, destinations[0]) - from conan.tools.files import copy - for d in origin_paths: - src_folder = os.path.join(base_folder, d) - for pattern in patterns: - copy(self._conanfile, pattern, src_folder, dst_folder) diff --git a/conan/tools/gnu/autotools.py b/conan/tools/gnu/autotools.py index d66dcdf6013..bf3cf257f6e 100644 --- a/conan/tools/gnu/autotools.py +++ b/conan/tools/gnu/autotools.py @@ -11,7 +11,7 @@ def join_arguments(args): return " ".join(filter(None, args)) -class Autotools(object): +class Autotools: def __init__(self, conanfile, namespace=None): """ @@ -84,9 +84,9 @@ def make(self, target=None, args=None, makefile=None): command = join_arguments([make_program, str_makefile, target, str_args, str_extra_args, jobs]) self._conanfile.run(command) - def install(self, args=None, target="install", makefile=None): + def install(self, args=None, target=None, makefile=None): """ - This is just an "alias" of ``self.make(target="install")`` + This is just an "alias" of ``self.make(target="install")`` or ``self.make(target="install-strip")`` :param args: (Optional, Defaulted to ``None``): List of arguments to use for the ``make`` call. By default an argument ``DESTDIR=unix_path(self.package_folder)`` @@ -95,6 +95,11 @@ def install(self, args=None, target="install", makefile=None): :param target: (Optional, Defaulted to ``None``): Choose which target to install. :param makefile: (Optional, Defaulted to ``None``): Allow specifying a custom makefile to use instead of default "Makefile" """ + if target is None: + target = "install" + do_strip = self._conanfile.conf.get("tools.build:install_strip", check_type=bool) + if do_strip: + target += "-strip" args = args if args else [] str_args = " ".join(args) if "DESTDIR=" not in str_args: diff --git a/conan/tools/gnu/autotoolstoolchain.py b/conan/tools/gnu/autotoolstoolchain.py index e4c008553ce..52b644382a9 100644 --- a/conan/tools/gnu/autotoolstoolchain.py +++ b/conan/tools/gnu/autotoolstoolchain.py @@ -2,7 +2,7 @@ from conan.errors import ConanException from conan.internal import check_duplicated_generator -from conan.internal.internal_tools import raise_on_universal_arch +from conan.internal.internal_tools import is_universal_arch from conan.tools.apple.apple import is_apple_os, resolve_apple_flags, apple_extra_flags from conan.tools.build import cmd_args_to_string, save_toolchain_args from conan.tools.build.cross_building import cross_building @@ -27,7 +27,6 @@ def __init__(self, conanfile, namespace=None, prefix="/"): helper so that it reads the information from the proper file. :param prefix: Folder to use for ``--prefix`` argument ("/" by default). """ - raise_on_universal_arch(conanfile) self._conanfile = conanfile self._namespace = namespace @@ -62,13 +61,19 @@ def __init__(self, conanfile, namespace=None, prefix="/"): if llvm_clang_front(self._conanfile) == "clang": self.msvc_runtime_link_flags = ["-fuse-ld=lld-link"] + self._is_universal_arch = is_universal_arch(conanfile.settings.get_safe("arch"), + conanfile.settings.possible_values().get("arch")) + if self._is_universal_arch and not is_apple_os(self._conanfile): + arch_str = conanfile.settings.get_safe('arch') + raise ConanException(f"Universal arch '{arch_str}' is only supported in Apple OSes") + # Cross build triplets self._host = self._conanfile.conf.get("tools.gnu:host_triplet") self._build = self._conanfile.conf.get("tools.gnu:build_triplet") self._target = None self.android_cross_flags = {} - self._is_cross_building = cross_building(self._conanfile) + self._is_cross_building = not self._is_universal_arch and cross_building(self._conanfile) if self._is_cross_building: compiler = self._conanfile.settings.get_safe("compiler") # If cross-building and tools.android:ndk_path is defined, let's try to guess the Android @@ -86,8 +91,13 @@ def __init__(self, conanfile, namespace=None, prefix="/"): self._build = _get_gnu_triplet(os_build, arch_build, compiler=compiler)["triplet"] sysroot = self._conanfile.conf.get("tools.build:sysroot") - sysroot = sysroot.replace("\\", "/") if sysroot is not None else None - self.sysroot_flag = "--sysroot {}".format(sysroot) if sysroot else None + if sysroot: + root = sysroot.replace("\\", "/") + compiler = self._conanfile.settings.get_safe("compiler") + self.sysroot_flag = f"--sysroot {root}" if compiler != "qcc" else f"-Wc,-isysroot,{root}" + else: + self.sysroot_flag = None + extra_configure_args = self._conanfile.conf.get("tools.gnu:extra_configure_args", check_type=list, default=[]) @@ -101,12 +111,15 @@ def __init__(self, conanfile, namespace=None, prefix="/"): # Apple stuff is_cross_building_osx = (self._is_cross_building and conanfile.settings_build.get_safe('os') == "Macos" - and is_apple_os(conanfile)) - min_flag, arch_flag, isysroot_flag = ( - resolve_apple_flags(conanfile, is_cross_building=is_cross_building_osx) + and is_apple_os(conanfile) + and not self._is_universal_arch) + + min_flag, arch_flags, isysroot_flag = ( + resolve_apple_flags(conanfile, is_cross_building=is_cross_building_osx, + is_universal=self._is_universal_arch) ) # https://man.archlinux.org/man/clang.1.en#Target_Selection_Options - self.apple_arch_flag = arch_flag + self.apple_arch_flag = arch_flags # -isysroot makes all includes for your library relative to the build directory self.apple_isysroot_flag = isysroot_flag self.apple_min_version_flag = min_flag diff --git a/conan/tools/gnu/gnudeps_flags.py b/conan/tools/gnu/gnudeps_flags.py index ed3a00198e9..2e2948ba6ef 100644 --- a/conan/tools/gnu/gnudeps_flags.py +++ b/conan/tools/gnu/gnudeps_flags.py @@ -8,7 +8,7 @@ from conan.internal.subsystems import subsystem_path, deduce_subsystem -class GnuDepsFlags(object): +class GnuDepsFlags: def __init__(self, conanfile, cpp_info): self._conanfile = conanfile diff --git a/conan/tools/gnu/gnutoolchain.py b/conan/tools/gnu/gnutoolchain.py index 421be382a47..5a1701d9b58 100644 --- a/conan/tools/gnu/gnutoolchain.py +++ b/conan/tools/gnu/gnutoolchain.py @@ -1,7 +1,8 @@ import os +from conan.errors import ConanException from conan.internal import check_duplicated_generator -from conan.internal.internal_tools import raise_on_universal_arch +from conan.internal.internal_tools import is_universal_arch from conan.tools.apple.apple import is_apple_os, resolve_apple_flags, apple_extra_flags from conan.tools.build import cmd_args_to_string, save_toolchain_args from conan.tools.build.cross_building import cross_building @@ -33,7 +34,6 @@ def __init__(self, conanfile, namespace=None, prefix="/"): helper so that it reads the information from the proper file. :param prefix: Folder to use for ``--prefix`` argument ("/" by default). """ - raise_on_universal_arch(conanfile) self._conanfile = conanfile self._namespace = namespace self._is_apple_system = is_apple_os(self._conanfile) @@ -66,6 +66,13 @@ def __init__(self, conanfile, namespace=None, prefix="/"): self.msvc_runtime_link_flags = [] if llvm_clang_front(self._conanfile) == "clang": self.msvc_runtime_link_flags = ["-fuse-ld=lld-link"] + + self._is_universal_arch = is_universal_arch(conanfile.settings.get_safe("arch"), + conanfile.settings.possible_values().get("arch")) + if self._is_universal_arch and not is_apple_os(self._conanfile): + arch_str = conanfile.settings.get_safe('arch') + raise ConanException(f"Universal arch '{arch_str}' is only supported in Apple OSes") + extra_configure_args = self._conanfile.conf.get("tools.gnu:extra_configure_args", check_type=list, default=[]) @@ -76,7 +83,7 @@ def __init__(self, conanfile, namespace=None, prefix="/"): "host": {"triplet": self._conanfile.conf.get("tools.gnu:host_triplet")}, "build": {"triplet": self._conanfile.conf.get("tools.gnu:build_triplet")} } - self._is_cross_building = cross_building(self._conanfile) + self._is_cross_building = not self._is_universal_arch and cross_building(self._conanfile) if self._is_cross_building: compiler = self._conanfile.settings.get_safe("compiler") # Host triplet @@ -91,8 +98,13 @@ def __init__(self, conanfile, namespace=None, prefix="/"): self.triplets_info["build"] = _get_gnu_triplet(os_build, arch_build, compiler=compiler) sysroot = self._conanfile.conf.get("tools.build:sysroot") - sysroot = sysroot.replace("\\", "/") if sysroot is not None else None - self.sysroot_flag = "--sysroot {}".format(sysroot) if sysroot else None + if sysroot: + root = sysroot.replace("\\", "/") + compiler = self._conanfile.settings.get_safe("compiler") + self.sysroot_flag = f"--sysroot {root}" if compiler != "qcc" else f"-Wc,-isysroot,{root}" + else: + self.sysroot_flag = None + self.configure_args = {} self.autoreconf_args = {"--force": None, "--install": None} self.make_args = {} @@ -104,12 +116,15 @@ def __init__(self, conanfile, namespace=None, prefix="/"): # Apple stuff is_cross_building_osx = (self._is_cross_building and conanfile.settings_build.get_safe('os') == "Macos" - and is_apple_os(conanfile)) - min_flag, arch_flag, isysroot_flag = ( - resolve_apple_flags(conanfile, is_cross_building=is_cross_building_osx) + and is_apple_os(conanfile) + and not self._is_universal_arch) + + min_flag, arch_flags, isysroot_flag = ( + resolve_apple_flags(conanfile, is_cross_building=is_cross_building_osx, + is_universal=self._is_universal_arch) ) # https://man.archlinux.org/man/clang.1.en#Target_Selection_Options - self.apple_arch_flag = arch_flag + self.apple_arch_flag = arch_flags # -isysroot makes all includes for your library relative to the build directory self.apple_isysroot_flag = isysroot_flag self.apple_min_version_flag = min_flag diff --git a/conan/tools/google/bazel.py b/conan/tools/google/bazel.py index 55702d2db26..06cb0409318 100644 --- a/conan/tools/google/bazel.py +++ b/conan/tools/google/bazel.py @@ -4,7 +4,7 @@ from conan.tools.google import BazelToolchain -class Bazel(object): +class Bazel: def __init__(self, conanfile): """ diff --git a/conan/tools/google/bazeldeps.py b/conan/tools/google/bazeldeps.py index ad77246f311..c0a4590c396 100644 --- a/conan/tools/google/bazeldeps.py +++ b/conan/tools/google/bazeldeps.py @@ -53,6 +53,13 @@ class _BazelDepBuildGenerator: {% if lib_info['import_lib_path'] %} interface_library = "{{ lib_info['import_lib_path'] }}", {% endif %} + {% if lib_info["linkopts"] %} + linkopts = [ + {% for linkopt in lib_info["linkopts"] %} + {{ linkopt }}, + {% endfor %} + ], + {% endif %} ) {% endfor %} {% endmacro %} @@ -64,7 +71,9 @@ class _BazelDepBuildGenerator: {% for header in obj["headers"] %} {{ header }}, {% endfor %} - ]), + ], + allow_empty = True + ), {% endif %} {% if obj["includes"] %} includes = [ @@ -214,7 +223,13 @@ def _get_linkopts(self, cpp_info): link_opt = '/DEFAULTLIB:{}' if os_build == "Windows" else '-l{}' system_libs = [link_opt.format(lib) for lib in cpp_info.system_libs] shared_flags = cpp_info.sharedlinkflags + cpp_info.exelinkflags - return [f'"{flag}"' for flag in (system_libs + shared_flags)] + frameworkdirs_flags = [] + framework_flags = [] + for frw in cpp_info.frameworks: + framework_flags.extend(["-framework", frw]) + for frw_dir in cpp_info.frameworkdirs: + frameworkdirs_flags.extend(["-F", frw_dir.replace("\\", "/")]) + return [f'"{flag}"' for flag in (system_libs + shared_flags + framework_flags + frameworkdirs_flags)] def _get_copts(self, cpp_info): # FIXME: long discussions between copts (-Iflag) vs includes in Bazel. Not sure yet @@ -268,12 +283,20 @@ def package_info(self): def _get_lib_info(self, cpp_info, deduced_cpp_info, component_name=None): def _lib_info(lib_name, virtual_cpp_info): - return { + info = { "name": lib_name, "is_shared": virtual_cpp_info.type == PackageType.SHARED, "lib_path": _relativize_path(virtual_cpp_info.location, self._package_folder), - "import_lib_path": _relativize_path(virtual_cpp_info.link_location, self._package_folder) + "import_lib_path": _relativize_path(virtual_cpp_info.link_location, self._package_folder), + "linkopts": [] } + if info['is_shared'] and info["lib_path"] and not info["lib_path"].endswith(".dll"): + # Issue: https://github.com/conan-io/conan/issues/19190 + # Issue: https://github.com/conan-io/conan/issues/19135 + # (UNIX) Adding the rpath flag as any application could link through the library + # which points out a symlink, but that name does not appear in the library location + info["linkopts"] = [f'"-Wl,-rpath,{libdir}"' for libdir in cpp_info.libdirs] + return info libs = cpp_info.libs libs_info = [] diff --git a/conan/tools/intel/intel_cc.py b/conan/tools/intel/intel_cc.py index 2722b475788..a2ab6d08211 100644 --- a/conan/tools/intel/intel_cc.py +++ b/conan/tools/intel/intel_cc.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- """ Intel generator module for oneAPI Toolkits. @@ -75,6 +73,10 @@ def ms_toolset(self): def generate(self, scope="build"): """Generate the Conan Intel file to be loaded in build environment by default""" check_duplicated_generator(self, self._conanfile) + intel_cc_path = self._conanfile.conf.get("tools.intel:installation_path") + if intel_cc_path == "": + return + if platform.system() == "Windows" and not self._conanfile.win_bash: content = textwrap.dedent("""\ @echo off diff --git a/conan/tools/layout/__init__.py b/conan/tools/layout/__init__.py index 5d5213fdffe..4dc15dfcf94 100644 --- a/conan/tools/layout/__init__.py +++ b/conan/tools/layout/__init__.py @@ -14,3 +14,6 @@ def basic_layout(conanfile, src_folder=".", build_folder=None): conanfile.folders.generators = os.path.join(conanfile.folders.build, "conan") conanfile.cpp.build.bindirs = ["."] conanfile.cpp.build.libdirs = ["."] + + if not conanfile.cpp.source.includedirs: + conanfile.cpp.source.includedirs = ["include"] diff --git a/conan/tools/meson/helpers.py b/conan/tools/meson/helpers.py index 1d3c812fffb..49330b481d6 100644 --- a/conan/tools/meson/helpers.py +++ b/conan/tools/meson/helpers.py @@ -1,5 +1,5 @@ from conan.api.output import ConanOutput -from conan.tools.build.flags import cppstd_msvc_flag +from conan.tools.build.flags import cppstd_msvc_flag, disable_flag from conan.internal.model.options import _PackageOption # https://mesonbuild.com/Reference-tables.html#operating-system-names @@ -107,9 +107,9 @@ def to_meson_value(value): return value -def to_cppstd_flag(compiler, compiler_version, cppstd): +def to_cppstd_flag(conanfile, compiler, compiler_version, cppstd): """Gets a valid cppstd flag. - + :param conanfile: ``ConanFile`` instance. :param compiler: ``str`` compiler name. :param compiler_version: ``str`` compiler version. :param cppstd: ``str`` cppstd version. @@ -117,6 +117,8 @@ def to_cppstd_flag(compiler, compiler_version, cppstd): """ if cppstd is None: return None + if disable_flag(conanfile, "cppstd"): + return None if compiler == "msvc": # Meson's logic with 'vc++X' vs 'c++X' is possibly a little outdated. # Presumably the intent is 'vc++X' is permissive and 'c++X' is not, @@ -127,10 +129,11 @@ def to_cppstd_flag(compiler, compiler_version, cppstd): return f"gnu++{cppstd[3:]}" if cppstd.startswith("gnu") else f"c++{cppstd}" -def to_cstd_flag(cstd): +def to_cstd_flag(conanfile, cstd): """Gets a valid cstd flag. """ if cstd is None: return None - else: - return cstd if cstd.startswith("gnu") else f"c{cstd}" + if disable_flag(conanfile, "cppstd"): + return None + return cstd if cstd.startswith("gnu") else f"c{cstd}" diff --git a/conan/tools/meson/meson.py b/conan/tools/meson/meson.py index 9604e5a732f..bca877e6624 100644 --- a/conan/tools/meson/meson.py +++ b/conan/tools/meson/meson.py @@ -4,7 +4,7 @@ from conan.tools.meson.toolchain import MesonToolchain -class Meson(object): +class Meson: """ This class calls Meson commands when a package is being built. Notice that this one should be used together with the ``MesonToolchain`` generator. @@ -71,9 +71,12 @@ def build(self, target=None): self._conanfile.output.info("Meson build cmd: {}".format(cmd)) self._conanfile.run(cmd) - def install(self): + def install(self, cli_args=None): """ - Runs ``meson install -C "." --destdir`` in the build folder. + Runs ``meson install -C "." --destdir ..`` in the build folder. + + :param cli_args: List of arguments to be added to the command: + ``meson install -C "." --destdir ... arg1 arg2`` """ meson_build_folder = self._conanfile.build_folder.replace("\\", "/") meson_package_folder = self._conanfile.package_folder.replace("\\", "/") @@ -84,6 +87,8 @@ def install(self): cmd += " " + verbosity if self._conanfile.conf.get("tools.build:install_strip", check_type=bool): cmd += " --strip" + if cli_args: + cmd += " " + " ".join(cli_args) self._conanfile.run(cmd) def test(self): diff --git a/conan/tools/meson/toolchain.py b/conan/tools/meson/toolchain.py index 2d5d1acac31..08503ce964a 100644 --- a/conan/tools/meson/toolchain.py +++ b/conan/tools/meson/toolchain.py @@ -8,7 +8,7 @@ from conan.internal.internal_tools import raise_on_universal_arch from conan.tools.apple.apple import is_apple_os, apple_min_version_flag, \ resolve_apple_flags, apple_extra_flags -from conan.tools.build.cross_building import cross_building +from conan.tools.build.cross_building import cross_building, can_run from conan.tools.build.flags import (architecture_link_flag, libcxx_flags, architecture_flag, threads_flags) from conan.tools.env import VirtualBuildEnv @@ -201,9 +201,9 @@ def __init__(self, conanfile, backend=None, native=False): cppstd = self._conanfile.settings.get_safe("compiler.cppstd") cstd = self._conanfile.settings.get_safe("compiler.cstd") #: C++ language standard to use. Defined by ``to_cppstd_flag()`` by default. - self.cpp_std = to_cppstd_flag(compiler, compiler_version, cppstd) + self.cpp_std = to_cppstd_flag(self._conanfile, compiler, compiler_version, cppstd) #: C language standard to use. Defined by ``to_cstd_flag()`` by default. - self.c_std = to_cstd_flag(cstd) + self.c_std = to_cstd_flag(self._conanfile, cstd) #: VS runtime library to use. Defined by ``msvc_runtime_flag()`` by default. self.b_vscrt = None if compiler in ("msvc", "clang"): @@ -266,7 +266,8 @@ def __init__(self, conanfile, backend=None, native=False): sdk_host = conanfile.settings.get_safe("os.sdk") self.cross_build["host"]["subsystem"] = get_apple_subsystem(sdk_host) self.cross_build["build"]["subsystem"] = get_apple_subsystem(sdk_build) - self.properties["needs_exe_wrapper"] = True + # Issue: https://github.com/conan-io/conan/issues/19217 + self.properties["needs_exe_wrapper"] = not can_run(self._conanfile) if hasattr(conanfile, 'settings_target') and conanfile.settings_target: settings_target = conanfile.settings_target os_target = settings_target.get_safe("os") @@ -289,8 +290,6 @@ def __init__(self, conanfile, backend=None, native=False): # Read configuration for sys_root property (honoring existing conf) self._sys_root = self._conanfile_conf.get("tools.build:sysroot", check_type=str) - if self._sys_root: - self.properties["sys_root"] = self._sys_root # Read configuration for compilers compilers_by_conf = self._conanfile_conf.get("tools.build:compiler_executables", default={}, @@ -414,7 +413,6 @@ def _resolve_apple_flags_and_variables(self, build_env, compilers_by_conf): self.apple_isysroot_flag = isysroot_flag.split() if isysroot_flag else [] self.apple_min_version_flag = [apple_min_version_flag(self._conanfile)] # Objective C/C++ ones - flags = [] self.objc = compilers_by_conf.get("objc", "clang") self.objcpp = compilers_by_conf.get("objcpp", "clang++") enable_arc = self._conanfile.conf.get("tools.apple:enable_arc", check_type=bool) @@ -463,7 +461,7 @@ def _get_extra_flags(self): check_type=list) linker_scripts = self._conanfile_conf.get("tools.build:linker_scripts", default=[], check_type=list) - linker_script_flags = ['-T"' + linker_script + '"' for linker_script in linker_scripts] + linker_script_flags = ['-T' + linker_script for linker_script in linker_scripts] defines = self._conanfile_conf.get("tools.build:defines", default=[], check_type=list) sys_root = [f"--sysroot={self._sys_root}"] if self._sys_root else [""] ld = (sharedlinkflags + exelinkflags + linker_script_flags + sys_root + self.extra_ldflags diff --git a/conan/tools/microsoft/msbuild.py b/conan/tools/microsoft/msbuild.py index 4f4dd52e15e..b18c3fc9e2c 100644 --- a/conan/tools/microsoft/msbuild.py +++ b/conan/tools/microsoft/msbuild.py @@ -50,7 +50,7 @@ def command(self, sln, targets=None): :return: ``str`` msbuild command line. """ # TODO: Enable output_binary_log via config - cmd = ('msbuild.exe "%s" -p:Configuration="%s" -p:Platform=%s' + cmd = ('msbuild.exe "%s" -p:Configuration="%s" -p:Platform="%s"' % (sln, self.build_type, self.platform)) verbosity = msbuild_verbosity_cmd_line_arg(self._conanfile) @@ -60,12 +60,12 @@ def command(self, sln, targets=None): maxcpucount = self._conanfile.conf.get("tools.microsoft.msbuild:max_cpu_count", check_type=int) if maxcpucount is not None: - cmd += f" -m:{maxcpucount}" if maxcpucount > 0 else " -m" + cmd += f' -m:"{maxcpucount}"' if maxcpucount > 0 else " -m" if targets: if not isinstance(targets, list): raise ConanException("targets argument should be a list") - cmd += " -target:{}".format(";".join(targets)) + cmd += ' -target:"{}"'.format(";".join(targets)) return cmd diff --git a/conan/tools/microsoft/nmakedeps.py b/conan/tools/microsoft/nmakedeps.py index b14451769b0..e2af344b441 100644 --- a/conan/tools/microsoft/nmakedeps.py +++ b/conan/tools/microsoft/nmakedeps.py @@ -5,7 +5,7 @@ from conan.tools.env import Environment -class NMakeDeps(object): +class NMakeDeps: def __init__(self, conanfile): """ diff --git a/conan/tools/microsoft/nmaketoolchain.py b/conan/tools/microsoft/nmaketoolchain.py index e7e0151a236..c20b575f11a 100644 --- a/conan/tools/microsoft/nmaketoolchain.py +++ b/conan/tools/microsoft/nmaketoolchain.py @@ -5,7 +5,7 @@ from conan.tools.microsoft.visual import msvc_runtime_flag, VCVars -class NMakeToolchain(object): +class NMakeToolchain: """ https://learn.microsoft.com/en-us/cpp/build/reference/running-nmake?view=msvc-170#toolsini-and-nmake We have also explored the usage of Tools.ini: @@ -25,10 +25,12 @@ def __init__(self, conanfile): self.extra_ldflags = [] self.extra_defines = [] - def _format_options(self, options): + @staticmethod + def _format_options(options): return [f"{opt[0].replace('-', '/')}{opt[1:]}" for opt in options if len(opt) > 1] - def _format_defines(self, defines): + @staticmethod + def _format_defines(defines): formated_defines = [] for define in defines: if "=" in define: @@ -57,7 +59,8 @@ def _cl(self): cppstd = cppstd_flag(self._conanfile) if cppstd: cxxflags.append(cppstd) - cxxflags.extend(self._conanfile.conf.get("tools.build:cxxflags", default=[], check_type=list)) + cxxflags.extend(self._conanfile.conf.get("tools.build:cxxflags", default=[], + check_type=list)) cxxflags.extend(self.extra_cxxflags) defines = [] @@ -67,9 +70,8 @@ def _cl(self): defines.extend(self._conanfile.conf.get("tools.build:defines", default=[], check_type=list)) defines.extend(self.extra_defines) - return ["/nologo"] + \ - self._format_options(bt_flags + rt_flags + cflags + cxxflags) + \ - self._format_defines(defines) + return (["/nologo"] + self._format_options(bt_flags + rt_flags + cflags + cxxflags) + + self._format_defines(defines)) @property def _link(self): @@ -78,8 +80,10 @@ def _link(self): ldflags = [] ldflags.extend(bt_ldflags) - ldflags.extend(self._conanfile.conf.get("tools.build:sharedlinkflags", default=[], check_type=list)) - ldflags.extend(self._conanfile.conf.get("tools.build:exelinkflags", default=[], check_type=list)) + ldflags.extend(self._conanfile.conf.get("tools.build:sharedlinkflags", default=[], + check_type=list)) + ldflags.extend(self._conanfile.conf.get("tools.build:exelinkflags", default=[], + check_type=list)) ldflags.extend(self.extra_ldflags) return ["/nologo"] + self._format_options(ldflags) @@ -94,7 +98,8 @@ def environment(self): env.append("_LINK_", self._link) # Also define some special env-vars which can override special NMake macros: # https://learn.microsoft.com/en-us/cpp/build/reference/special-nmake-macros - conf_compilers = self._conanfile.conf.get("tools.build:compiler_executables", default={}, check_type=dict) + conf_compilers = self._conanfile.conf.get("tools.build:compiler_executables", default={}, + check_type=dict) if conf_compilers: compilers_mapping = { "AS": "asm", diff --git a/conan/tools/microsoft/toolchain.py b/conan/tools/microsoft/toolchain.py index ab92b3e28e2..e1f58bcf283 100644 --- a/conan/tools/microsoft/toolchain.py +++ b/conan/tools/microsoft/toolchain.py @@ -5,15 +5,16 @@ from jinja2 import Template from conan.internal import check_duplicated_generator +from conan.internal.api.detect.detect_vs import vs_installation_path from conan.tools.build import build_jobs from conan.tools.intel.intel_cc import IntelCC from conan.tools.microsoft.visual import VCVars, msvs_toolset, msvc_runtime_flag, \ - msvc_platform_from_arch + msvc_platform_from_arch, vs_ide_version from conan.errors import ConanException from conan.internal.util.files import save, load -class MSBuildToolchain(object): +class MSBuildToolchain: """ MSBuildToolchain class generator """ @@ -23,6 +24,9 @@ class MSBuildToolchain(object): _config_toolchain_props = textwrap.dedent("""\ + {% if toolset_version_full_path %} + + {% endif %} {{ defines }}%(PreprocessorDefinitions) @@ -57,7 +61,7 @@ def __init__(self, conanfile): self._conanfile = conanfile #: Dict-like that defines the preprocessor definitions self.preprocessor_definitions = {} - #: Dict-like that defines the preprocessor definitions + #: Dict with compile options that will be added as value in the ClCompile section self.compile_options = {} #: List of all the CXX flags self.cxxflags = [] @@ -76,6 +80,7 @@ def __init__(self, conanfile): #: setting, else, it'll be based on ``msvc`` version. self.toolset = msvs_toolset(conanfile) self.properties = {} + self.toolset_version_full_path = _get_toolset_props(conanfile) def _name_condition(self, settings): platform = msvc_platform_from_arch(settings.get_safe("arch")) @@ -130,9 +135,9 @@ def format_macro(key, value): cstd = f"stdc{self.cstd}" if self.cstd else "" runtime_library = self.runtime_library toolset = self.toolset or "" - compile_options = self._conanfile.conf.get("tools.microsoft.msbuildtoolchain:compile_options", - default={}, check_type=dict) - self.compile_options.update(compile_options) + conf_options = self._conanfile.conf.get("tools.microsoft.msbuildtoolchain:compile_options", + default={}, check_type=dict) + self.compile_options.update(conf_options) parallel = "" njobs = build_jobs(self._conanfile) if njobs: @@ -156,7 +161,8 @@ def format_macro(key, value): "compile_options": compile_options, "parallel": parallel, "properties": self.properties, - "winsdk_version": winsdk_version + "winsdk_version": winsdk_version, + "toolset_version_full_path": self.toolset_version_full_path } def _write_config_toolchain(self, config_filename): @@ -213,7 +219,35 @@ def _get_extra_flags(self): # Now, it's time to get all the flags defined by the user cxxflags = self._conanfile.conf.get("tools.build:cxxflags", default=[], check_type=list) cflags = self._conanfile.conf.get("tools.build:cflags", default=[], check_type=list) - sharedlinkflags = self._conanfile.conf.get("tools.build:sharedlinkflags", default=[], check_type=list) - exelinkflags = self._conanfile.conf.get("tools.build:exelinkflags", default=[], check_type=list) + sharedlinkflags = self._conanfile.conf.get("tools.build:sharedlinkflags", default=[], + check_type=list) + exelinkflags = self._conanfile.conf.get("tools.build:exelinkflags", default=[], + check_type=list) defines = self._conanfile.conf.get("tools.build:defines", default=[], check_type=list) return cxxflags, cflags, defines, sharedlinkflags, exelinkflags + + +def _get_toolset_props(conanfile): + msvc_update = conanfile.conf.get("tools.microsoft:msvc_update") + compiler_update = msvc_update or conanfile.settings.get_safe("compiler.update") + if compiler_update is None: + return + + vs_version = vs_ide_version(conanfile) + if int(vs_version) <= 14: + return + vs_install_path = conanfile.conf.get("tools.microsoft.msbuild:installation_path") + vs_path = vs_install_path or vs_installation_path(vs_version) + if not vs_path or not os.path.isdir(vs_path): + return + + basebuild = os.path.normpath(os.path.join(vs_path, "VC/Auxiliary/Build")) + # The equivalent of compiler 19.26 is toolset 14.26 + compiler_version = str(conanfile.settings.compiler.version) + vcvars_ver = "14.{}{}".format(compiler_version[-1], compiler_update) + for folder in os.listdir(basebuild): + if not os.path.isdir(os.path.join(basebuild, folder)): + continue + if folder.startswith(vcvars_ver): + result = folder + return os.path.join(basebuild, result, f"Microsoft.VCToolsVersion.{result}.props") diff --git a/conan/tools/microsoft/visual.py b/conan/tools/microsoft/visual.py index 3e7a87b5a55..3dc394de683 100644 --- a/conan/tools/microsoft/visual.py +++ b/conan/tools/microsoft/visual.py @@ -70,7 +70,8 @@ def msvc_version_to_vs_ide_version(version): '191': '15', '192': '16', '193': '17', - '194': '17'} # Note both 193 and 194 belong to VS 17 2022 + '194': '17', # Note both 193 and 194 belong to VS 17 2022 + '195': '18'} return _visuals[str(version)] @@ -87,7 +88,8 @@ def msvc_version_to_toolset_version(version): '191': 'v141', '192': 'v142', "193": 'v143', - "194": 'v143'} + "194": 'v143', + "195": 'v145'} return toolsets.get(str(version)) @@ -169,10 +171,10 @@ def generate(self, scope="build"): if is_ps1 is not None: ConanOutput().warning( "Boolean values for 'tools.env.virtualenv:powershell' are deprecated. " - "Please specify 'powershell.exe' or 'pwsh' instead, appending arguments if needed " - "(for example: 'powershell.exe -argument'). " - "To unset this configuration, use `tools.env.virtualenv:powershell=!`, which matches " - "the previous 'False' behavior.", + "Please specify 'powershell.exe' or 'pwsh' instead, appending arguments " + "if needed (for example: 'powershell.exe -argument'). " + "To unset this configuration, use `tools.env.virtualenv:powershell=!`, " + "which matches the previous 'False' behavior.", warn_tag="deprecated" ) except ConanException: @@ -197,7 +199,7 @@ def generate(self, scope="build"): def _create_deactivate_vcvars_file(conanfile, filename): deactivate_filename = f"deactivate_{filename}" - message = f"[{deactivate_filename}]: vcvars env cannot be deactivated" + message = f"[{deactivate_filename}]: *** vcvars env cannot be deactivated ***\n" is_ps1 = filename.endswith(".ps1") if is_ps1: content = f"Write-Host {message}" @@ -296,6 +298,7 @@ def _vcvars_path(version, vs_install_path): vcpath = os.path.join(vs_path, "VC/Auxiliary/Build/vcvarsall.bat") else: vcpath = os.path.join(vs_path, "VC/vcvarsall.bat") + vcpath = os.path.normpath(vcpath) return vcpath @@ -312,15 +315,18 @@ def _vcvars_versions(conanfile): "v141": "15", "v142": "16", "v143": "17", - "v144": "17"}.get(toolset_version) + "v144": "17", + "v145": "18"}.get(toolset_version) if vs_version is None: - raise ConanException("Visual Studio Runtime version (v140-v144) not defined. Please, " - "add the compiler.runtime_version=[v140-v144] setting to your profile.") + raise ConanException("Visual Studio Runtime version (v140-v145) not defined. Please, " + "add the compiler.runtime_version=[v140-v145] setting to your " + "profile.") vcvars_ver = {"v140": "14.0", "v141": "14.1", "v142": "14.2", "v143": "14.3", - "v144": "14.4"}.get(toolset_version) + "v144": "14.4", + "v145": "14.5"}.get(toolset_version) if vcvars_ver and msvc_update is not None: vcvars_ver += f"{msvc_update}" else: diff --git a/conan/tools/premake/premake.py b/conan/tools/premake/premake.py index e123d31db43..f70f3809e3d 100644 --- a/conan/tools/premake/premake.py +++ b/conan/tools/premake/premake.py @@ -17,8 +17,10 @@ '192': '2019', '193': '2022', '194': '2022', # still 2022 + '195': '2026' } + class Premake: """ This class calls Premake commands when a package is being built. Notice that @@ -52,7 +54,7 @@ def __init__(self, conanfile): msvc_version = PREMAKE_VS_VERSION.get(str(self._conanfile.settings.compiler.version)) self.action = f'vs{msvc_version}' else: - self.action = "gmake" # New generator (old gmakelegacy is deprecated) + self.action = "gmake" # New generator (old gmakelegacy is deprecated) self._premake_conan_toolchain = Path(self._conanfile.generators_folder) / PremakeToolchain.filename @@ -98,7 +100,7 @@ def _compilation_verbosity(self): # --verbose does not print compilation commands but internal Makefile progress logic return " verbose=1" if verbosity == "verbose" else "" - def build(self, workspace, targets=None, msbuild_platform=None): + def build(self, workspace, targets=None, configuration=None, msbuild_platform=None): """ Depending on the action, this method will run either ``msbuild`` or ``make`` with ``N_JOBS``. You can specify ``N_JOBS`` through the configuration line ``tools.build:jobs=N_JOBS`` @@ -106,18 +108,21 @@ def build(self, workspace, targets=None, msbuild_platform=None): :param workspace: ``str`` Specifies the solution to be compiled (only used by ``MSBuild``). :param targets: ``List[str]`` Declare the projects to be built (None to build all projects). + :param configuration: ``str`` Specify the configuration build type, default to build_type ("Release" or "Debug"), + but this allow setting custom configuration type. :param msbuild_platform: ``str`` Specify the platform for the internal MSBuild generator (only used by ``MSBuild``). """ if not self._premake_conan_toolchain.exists(): raise ConanException("Premake.build() method requires PremakeToolchain to work properly") + build_type = configuration or str(self._conanfile.settings.build_type) if self.action.startswith("vs"): msbuild = MSBuild(self._conanfile) if msbuild_platform: msbuild.platform = msbuild_platform + msbuild.build_type = build_type msbuild.build(sln=f"{workspace}.sln", targets=targets) else: - build_type = str(self._conanfile.settings.build_type) targets = "all" if targets is None else " ".join(targets) njobs = build_jobs(self._conanfile) self._conanfile.run(f"make config={build_type.lower()} {targets} -j{njobs}{self._compilation_verbosity}") diff --git a/conan/tools/premake/premakedeps.py b/conan/tools/premake/premakedeps.py index 94501b35465..9987570bcaa 100644 --- a/conan/tools/premake/premakedeps.py +++ b/conan/tools/premake/premakedeps.py @@ -95,7 +95,7 @@ # Helper class that expands cpp_info meta information in lua readable string sequences -class _PremakeTemplate(object): +class _PremakeTemplate: def __init__(self, req, dep_cpp_info): def _format_paths(paths): if not paths: diff --git a/conan/tools/qbs/qbs.py b/conan/tools/qbs/qbs.py index 1673d8b6d22..7d25994561a 100644 --- a/conan/tools/qbs/qbs.py +++ b/conan/tools/qbs/qbs.py @@ -18,7 +18,7 @@ def _configuration_dict_to_commandlist(name, config_dict): return command_list -class Qbs(object): +class Qbs: """ Qbs helper to use together with the QbsDeps feature. This class provides helper methods that wraps calls to the Qbs tool. diff --git a/conan/tools/ros/rosenv.py b/conan/tools/ros/rosenv.py index fabe6866a03..8d38b6d082e 100644 --- a/conan/tools/ros/rosenv.py +++ b/conan/tools/ros/rosenv.py @@ -1,8 +1,7 @@ import os from conan.api.output import Color -from conan.tools.env import VirtualBuildEnv, Environment +from conan.tools.env import Environment from conan.tools.env.environment import create_env_script -from conan.tools.files import save class ROSEnv: @@ -18,8 +17,10 @@ def __init__(self, conanfile): """ self._conanfile = conanfile self.variables = {} - self._build_script_file = "conanrosenv-build.sh" - self._wrapper_script_file = "conanrosenv.sh" + self._build_script_sh_file = "conanrosenv-build.sh" + self._build_script_bat_file = "conanrosenv-build.bat" + self._wrapper_script_sh_file = "conanrosenv.sh" + self._wrapper_script_bat_file = "conanrosenv.bat" def generate(self): """ @@ -37,17 +38,41 @@ def generate(self): rosbuildenv = Environment() for k, v in self.variables.items(): rosbuildenv.define(k, v) - rosbuildenv.vars(self._conanfile, "build").save_script(self._build_script_file) + for build_script_file in [self._build_script_sh_file, self._build_script_bat_file]: + rosbuildenv.vars(self._conanfile, "build").save_script(build_script_file) + # TODO: Add powrshell support generating .ps1 files + self._generate_sh_files() + self._generate_bat_files() + + def _generate_sh_files(self): # Generate conanrosenv.sh script wrapper that calls conanbuild.sh and conanrun.sh - # TODO: Windows .bat/.ps1 files still not supported for the wrapper - conanbuild_path = os.path.join(self._conanfile.generators_folder, "conanbuild.sh") - conanrun_path = os.path.join(self._conanfile.generators_folder, "conanrun.sh") - rosenv_wrapper_content = [f". \"{conanbuild_path}\"", f". \"{conanrun_path}\""] - create_env_script(self._conanfile, "\n".join(rosenv_wrapper_content), - self._wrapper_script_file, None) - - conanrosenv_path = os.path.join(self._conanfile.generators_folder, self._wrapper_script_file) - msg = f"Generated ROSEnv Conan file: {self._wrapper_script_file}\n" + \ + conanbuild_sh_path = os.path.join(self._conanfile.generators_folder, "conanbuild.sh") + conanrun_sh_path = os.path.join(self._conanfile.generators_folder, "conanrun.sh") + rosenv_sh_wrapper_content = [f". \"{conanbuild_sh_path}\"", f". \"{conanrun_sh_path}\""] + create_env_script(self._conanfile, "\n".join(rosenv_sh_wrapper_content), + self._wrapper_script_sh_file, None) + + conanrosenv_path = os.path.join(self._conanfile.generators_folder, + self._wrapper_script_sh_file) + msg = f"Generated ROSEnv Conan file: {self._wrapper_script_sh_file}\n" + \ f"Use 'source {conanrosenv_path}' to set the ROSEnv Conan before 'colcon build'" self._conanfile.output.info(msg, fg=Color.CYAN) + + def _generate_bat_files(self): + # Generate conanrosenv.bat script wrapper that calls conanbuild.bat and conanrun.bat + conanbuild_bat_path = os.path.join(self._conanfile.generators_folder, "conanbuild.bat") + conanrun_bat_path = os.path.join(self._conanfile.generators_folder, "conanrun.bat") + rosenv_bat_wrapper_content = [ + "@echo off", + f"CALL \"{conanbuild_bat_path}\"", + f"CALL \"{conanrun_bat_path}\"" + ] + create_env_script(self._conanfile, "\n".join(rosenv_bat_wrapper_content), + self._wrapper_script_bat_file, None) + + conanrosenv_path = os.path.join(self._conanfile.generators_folder, + self._wrapper_script_bat_file) + msg = f"Generated ROSEnv Conan file: {self._wrapper_script_bat_file}\n" + \ + f"Use 'call {conanrosenv_path}' to set the ROSEnv Conan before 'colcon build'" + self._conanfile.output.info(msg, fg=Color.CYAN) diff --git a/conan/tools/sbom/cyclonedx.py b/conan/tools/sbom/cyclonedx.py index 521cdf4428a..06ddc649b04 100644 --- a/conan/tools/sbom/cyclonedx.py +++ b/conan/tools/sbom/cyclonedx.py @@ -1,5 +1,6 @@ from conan import conan_version + def cyclonedx_1_4(conanfile, name=None, add_build=False, add_tests=False, **kwargs): """ (Experimental) Generate cyclone 1.4 SBOM with JSON format @@ -28,25 +29,30 @@ def cyclonedx_1_4(conanfile, name=None, add_build=False, add_tests=False, **kwar from datetime import datetime, timezone graph = conanfile.subgraph - has_special_root_node = not (getattr(graph.root.ref, "name", False) and getattr(graph.root.ref, "version", False) and getattr(graph.root.ref, "revision", False)) + has_special_root_node = not (getattr(graph.root.ref, "name", False) + and getattr(graph.root.ref, "version", False) + and getattr(graph.root.ref, "revision", False)) special_id = str(uuid.uuid4()) name_default = getattr(graph.root.ref, "name", False) or "conan-sbom" - name_default += f"/{graph.root.ref.version}" if bool(getattr(graph.root.ref, "version", False)) else "" - nodes = [node for node in graph.nodes if (node.context == "host" or add_build) and (not node.test or add_tests)] + name_default += f"/{graph.root.ref.version}" if getattr(graph.root.ref, "version", False) else "" + + nodes = [node for node in graph.nodes if should_add_node(node, add_build, add_tests)] if has_special_root_node: nodes = nodes[1:] dependencies = [] if has_special_root_node: deps = {"ref": special_id, - "dependsOn": [_calculate_bomref(d.dst) for d in graph.root.edges]} + "dependsOn": [_calculate_bomref(d.dst) for d in graph.root.edges + if should_add_node(d.dst, add_build, add_tests)]} dependencies.append(deps) for c in nodes: deps = {"ref": _calculate_bomref(c)} - dep = [d for d in c.edges if (d.dst.context == "host" or add_build) and (not d.dst.test or add_tests)] + dep = [d for d in c.edges if should_add_node(d.dst, add_build, add_tests)] - depends_on = [_calculate_bomref(d.dst) for d in dep] + depends_on = [_calculate_bomref(d.dst) for d in dep + if should_add_node(d.dst, add_build, add_tests)] if depends_on: deps["dependsOn"] = depends_on dependencies.append(deps) @@ -90,6 +96,7 @@ def cyclonedx_1_4(conanfile, name=None, add_build=False, add_tests=False, **kwar } return sbom_cyclonedx_1_4 + def cyclonedx_1_6(conanfile, name=None, add_build=False, add_tests=False, **kwargs): """ (Experimental) Generate cyclone 1.6 SBOM with JSON format @@ -118,12 +125,15 @@ def cyclonedx_1_6(conanfile, name=None, add_build=False, add_tests=False, **kwar from datetime import datetime, timezone graph = conanfile.subgraph - has_special_root_node = not (getattr(graph.root.ref, "name", False) and getattr(graph.root.ref, "version", False) and getattr(graph.root.ref, "revision", False)) + has_special_root_node = not (getattr(graph.root.ref, "name", False) + and getattr(graph.root.ref, "version", False) + and getattr(graph.root.ref, "revision", False)) special_id = str(uuid.uuid4()) name_default = getattr(graph.root.ref, "name", False) or "conan-sbom" - name_default += f"/{graph.root.ref.version}" if bool(getattr(graph.root.ref, "version", False)) else "" - nodes = [node for node in graph.nodes if (node.context == "host" or add_build) and (not node.test or add_tests)] + name_default += f"/{graph.root.ref.version}" if getattr(graph.root.ref, "version", False) else "" + + nodes = [node for node in graph.nodes if should_add_node(node, add_build, add_tests)] if has_special_root_node: nodes = nodes[1:] @@ -131,13 +141,15 @@ def cyclonedx_1_6(conanfile, name=None, add_build=False, add_tests=False, **kwar if has_special_root_node: deps = {"ref": special_id, "dependsOn": [_calculate_bomref(d.dst) - for d in graph.root.edges]} + for d in graph.root.edges + if should_add_node(d.dst, add_build, add_tests)]} dependencies.append(deps) for c in nodes: deps = {"ref": _calculate_bomref(c)} - dep = [d for d in c.edges if (d.dst.context == "host" or add_build) and (not d.dst.test or add_tests)] + dep = [d for d in c.edges if should_add_node(d.dst, add_build, add_tests)] - depends_on = [_calculate_bomref(d.dst) for d in dep] + depends_on = [_calculate_bomref(d.dst) for d in dep + if should_add_node(d.dst, add_build, add_tests)] if depends_on: deps["dependsOn"] = depends_on dependencies.append(deps) @@ -167,7 +179,7 @@ def cyclonedx_1_6(conanfile, name=None, add_build=False, add_tests=False, **kwar }, "timestamp": f"{datetime.fromtimestamp(time.time(), tz=timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}", "tools": { - "components":[{ + "components": [{ "type": "application", "name": "Conan-io", "version": str(conan_version), @@ -186,21 +198,24 @@ def _calculate_licenses(component): from conan.tools.sbom.spdx_licenses import NORMALIZED_VALID_SPDX_LICENSES licenses = component.conanfile.license - if isinstance(licenses, str): # Just one license + if isinstance(licenses, str): # Just one license field = "id" if licenses.lower() in NORMALIZED_VALID_SPDX_LICENSES else "name" - return [{"license":{ field: licenses }}] + return [{"license": {field: licenses}}] - return [ # More than one license + return [ + # More than one license {"license": { - "id" if l.lower() in NORMALIZED_VALID_SPDX_LICENSES else "name": l + "id" if lic.lower() in NORMALIZED_VALID_SPDX_LICENSES else "name": lic }} - for l in licenses + for lic in licenses ] + def _calculate_bomref(component): user = f"&user={component.ref.user}" if component.ref.user else "" channel = f"&channel={component.ref.channel}" if component.ref.channel else "" return f"pkg:conan/{component.name}@{component.ref.version}?rref={component.ref.revision}{user}{channel}" - +def should_add_node(node, add_build, add_tests): + return (node.context == "host" or add_build) and (not node.test or add_tests) diff --git a/conan/tools/sbom/spdx_licenses.py b/conan/tools/sbom/spdx_licenses.py index b64d9d906f4..0dfbc5d9914 100644 --- a/conan/tools/sbom/spdx_licenses.py +++ b/conan/tools/sbom/spdx_licenses.py @@ -683,7 +683,6 @@ } - NORMALIZED_VALID_SPDX_LICENSES = { "0bsd", "3d-slicer-1.0", diff --git a/conan/tools/system/__init__.py b/conan/tools/system/__init__.py index e69de29bb2d..39ba4469a13 100644 --- a/conan/tools/system/__init__.py +++ b/conan/tools/system/__init__.py @@ -0,0 +1 @@ +from conan.tools.system.pip_manager import PipEnv diff --git a/conan/tools/system/package_manager.py b/conan/tools/system/package_manager.py index 570744d5e0a..b3ed377168f 100644 --- a/conan/tools/system/package_manager.py +++ b/conan/tools/system/package_manager.py @@ -5,15 +5,18 @@ from conan.errors import ConanException -class _SystemPackageManagerTool(object): +class _SystemPackageManagerTool: mode_check = "check" # Check if installed, fail if not mode_install = "install" mode_report = "report" # Only report what would be installed, no check (can run in any system) mode_report_installed = "report-installed" # report installed and missing packages tool_name = None + version_separator = "" install_command = "" update_command = "" check_command = "" + check_version_command = "" + full_package_name = "{name}{arch_separator}{arch_name}" accepted_install_codes = [0] accepted_update_codes = [0] accepted_check_codes = [0, 1] @@ -67,15 +70,30 @@ def get_default_tool(self): self._conanfile.output.info("A default system package manager couldn't be found for {}, " "system packages will not be installed.".format(os_name)) + def _split_package_name(self, package, host_package): + + name, version = (package.split("=")[0], package.split("=")[1]) if "=" in package else (package, "") + arch_separator, arch_name = "", "" + version_separator = self.version_separator if version else "" + + if self._arch in self._arch_names and cross_building(self._conanfile) and host_package: + arch_separator = self._arch_separator + arch_name = self._arch_names.get(self._arch) + return name, version, arch_separator, arch_name, version_separator + def get_package_name(self, package, host_package=True): # Only if the package is for building, for example a library, # we should add the host arch when cross building. # If the package is a tool that should be installed on the current build # machine we should not add the arch. - if self._arch in self._arch_names and cross_building(self._conanfile) and host_package: - return "{}{}{}".format(package, self._arch_separator, - self._arch_names.get(self._arch)) - return package + + name, version, arch_separator, arch_name, version_separator = self._split_package_name(package, host_package) + + return self.full_package_name.format(name=name, + arch_separator=arch_separator, + arch_name=arch_name, + version_separator=version_separator, + version=version) @property def sudo_str(self): @@ -87,9 +105,9 @@ def run(self, method, *args, **kwargs): if self._active_tool == self.__class__.tool_name: return method(*args, **kwargs) - def _conanfile_run(self, command, accepted_returns): + def _conanfile_run(self, command, accepted_returns, quiet=True): # When checking multiple packages, this is too noisy - ret = self._conanfile.run(command, ignore_errors=True, quiet=True) + ret = self._conanfile.run(command, ignore_errors=True, quiet=quiet) if ret not in accepted_returns: raise ConanException("Command '%s' failed" % command) return ret @@ -169,7 +187,6 @@ def _install(self, packages, update=False, check=True, host_package=True, **kwar packages = self.check(packages, host_package=host_package) missing_pkgs = pkgs.setdefault("missing", []) missing_pkgs.extend(p for p in packages if p not in missing_pkgs) - if self._mode == self.mode_report_installed: return @@ -186,14 +203,13 @@ def _install(self, packages, update=False, check=True, host_package=True, **kwar elif packages: if update: self.update() - packages_arch = [self.get_package_name(package, host_package=host_package) for package in packages] if packages_arch: command = self.install_command.format(sudo=self.sudo_str, tool=self.tool_name, packages=" ".join(packages_arch), **kwargs) - return self._conanfile_run(command, self.accepted_install_codes) + return self._conanfile_run(command, self.accepted_install_codes, quiet=False) else: self._conanfile.output.info("System requirements: {} already " "installed".format(" ".join(packages))) @@ -203,24 +219,43 @@ def _update(self): # in case we are in check mode just ignore if self._mode == self.mode_install: command = self.update_command.format(sudo=self.sudo_str, tool=self.tool_name) - return self._conanfile_run(command, self.accepted_update_codes) + return self._conanfile_run(command, self.accepted_update_codes, quiet=False) def _check(self, packages, host_package=True): - missing = [pkg for pkg in packages if self.check_package(self.get_package_name(pkg, host_package=host_package)) != 0] + missing = [pkg for pkg in packages if self.check_package(pkg, host_package) != 0] return missing - def check_package(self, package): - command = self.check_command.format(tool=self.tool_name, - package=package) + def check_package(self, package, host_package=True): + name, version, arch_separator, arch_name, _ = self._split_package_name(package, host_package) + + check_arch = self._arch if host_package else self._conanfile.settings_build.get_safe('arch') + arch_package = arch_name or self._arch_names.get(check_arch) + package = self.full_package_name.format(name=name, + arch_separator=arch_separator, + arch_name=arch_name, + version="", + version_separator="") + command = self.check_command.format(tool=self.tool_name, package=package, arch_package=arch_package, base_name=name) + + if version: + if self.check_version_command: + command = self.check_version_command.format(tool=self.tool_name, package=package, version=version, arch_package=arch_package, + base_name=name) + else: + self._conanfile.output.warning(f"System requirements: \"{self.tool_name}\" doesn't support package versions," + f" \"{package}\" will be installed without a specific version.") return self._conanfile_run(command, self.accepted_check_codes) class Apt(_SystemPackageManagerTool): # TODO: apt? apt-get? tool_name = "apt-get" + version_separator = "=" + full_package_name = "{name}{arch_separator}{arch_name}{version_separator}{version}" install_command = "{sudo}{tool} install -y {recommends}{packages}" update_command = "{sudo}{tool} update" - check_command = "dpkg-query -W -f='${{Status}}' {package} | grep -q \"ok installed\"" + check_command = r"dpkg-query -W -f='${{Architecture}}\n' {base_name} | grep -qEx '({arch_package}|all)'" + check_version_command = r"dpkg-query -W -f='${{Architecture}} ${{Version}}\n' {base_name} | grep -qEx '({arch_package}|all) {version}'" def __init__(self, conanfile, arch_names=None): """ @@ -265,9 +300,12 @@ def install(self, packages, update=False, check=True, recommends=False, host_pac class Yum(_SystemPackageManagerTool): tool_name = "yum" + version_separator = "-" + full_package_name = "{name}{version_separator}{version}{arch_separator}{arch_name}" install_command = "{sudo}{tool} install -y {packages}" update_command = "{sudo}{tool} check-update -y" check_command = "rpm -q {package}" + check_version_command = "rpm -q {package}-{version}" accepted_update_codes = [0, 100] def __init__(self, conanfile, arch_names=None): @@ -293,24 +331,35 @@ def __init__(self, conanfile, arch_names=None): class Dnf(Yum): tool_name = "dnf" + version_separator = "-" + full_package_name = "{name}{version_separator}{version}{arch_separator}{arch_name}" + check_version_command = "rpm -q {package}-{version}" class Brew(_SystemPackageManagerTool): tool_name = "brew" + version_separator = "@" + full_package_name = "{name}{version_separator}{version}" install_command = "{sudo}{tool} install {packages}" update_command = "{sudo}{tool} update" check_command = 'test -n "$({tool} ls --versions {package})"' + check_version_command = 'brew list --versions {package} | grep "{version}"' class Pkg(_SystemPackageManagerTool): tool_name = "pkg" + version_separator = "-" + full_package_name = "{name}{version_separator}{version}" install_command = "{sudo}{tool} install -y {packages}" update_command = "{sudo}{tool} update" check_command = "{tool} info {package}" + check_version_command = "{tool} info {package} | grep \"Version: {version}\"" class PkgUtil(_SystemPackageManagerTool): tool_name = "pkgutil" + version_separator = "@" + full_package_name = "{name}{version_separator}{version}" install_command = "{sudo}{tool} --install --yes {packages}" update_command = "{sudo}{tool} --catalog" check_command = 'test -n "`{tool} --list {package}`"' @@ -318,9 +367,12 @@ class PkgUtil(_SystemPackageManagerTool): class Chocolatey(_SystemPackageManagerTool): tool_name = "choco" + version_separator = " --version " + full_package_name = "{name}{version_separator}{version}" install_command = "{tool} install --yes {packages}" update_command = "{tool} outdated" check_command = '{tool} list --exact {package} | findstr /c:"1 packages installed."' + check_version_command = '{tool} list --local-only {package} | findstr /i "{version}"' class PacMan(_SystemPackageManagerTool): @@ -345,9 +397,12 @@ def __init__(self, conanfile, arch_names=None): class Apk(_SystemPackageManagerTool): tool_name = "apk" + version_separator = "=" + full_package_name = "{name}{version_separator}{version}" install_command = "{sudo}{tool} add --no-cache {packages}" update_command = "{sudo}{tool} update" check_command = "{tool} info -e {package}" + check_version_command = "{tool} info {package} | grep \"{version}\"" def __init__(self, conanfile, _arch_names=None): """ @@ -364,6 +419,9 @@ def __init__(self, conanfile, _arch_names=None): class Zypper(_SystemPackageManagerTool): tool_name = "zypper" + version_separator = "=" # < or > + full_package_name = "{name}{arch_separator}{arch_name}{version_separator}{version}" install_command = "{sudo}{tool} --non-interactive in {packages}" update_command = "{sudo}{tool} --non-interactive ref" check_command = "rpm -q {package}" + check_version_command = "rpm -q {package}-{version}" diff --git a/conan/tools/system/pip_manager.py b/conan/tools/system/pip_manager.py new file mode 100644 index 00000000000..f028703d2a5 --- /dev/null +++ b/conan/tools/system/pip_manager.py @@ -0,0 +1,73 @@ +import platform +import os +import shutil + +from conan.tools.build import cmd_args_to_string +from conan.tools.env.environment import Environment +from conan.errors import ConanException + + +class PipEnv: + + def __init__(self, conanfile, folder=None, name=""): + """ + :param conanfile: The current conanfile "self" + :param folder: Optional folder, by default the "build_folder" + :param name: Optional name for the virtualenv, by default "conan_pipenv" + """ + self._conanfile = conanfile + self.env_name = f"conan_pipenv{f'_{name}' if name else ''}" + self._env_dir = os.path.abspath(os.path.join(folder or conanfile.build_folder, + self.env_name)) + bins = "Scripts" if platform.system() == "Windows" else "bin" + self.bin_dir = os.path.join(self._env_dir, bins) + pyexe = "python.exe" if platform.system() == "Windows" else "python" + self._python_exe = os.path.join(self.bin_dir, pyexe) + self._create_venv() + + def generate(self): + """ + Create a conan environment to use the python venv in the next steps of the conanfile. + """ + env = Environment() + env.prepend_path("PATH", self.bin_dir) + env.vars(self._conanfile).save_script(self.env_name) + + @staticmethod + def _default_python(): + python = "python" if platform.system() == "Windows" else "python3" + default_python = shutil.which(python) + return os.path.realpath(default_python) if default_python else None + + def _create_venv(self): + python_interpreter = self._conanfile.conf.get("tools.system.pipenv:python_interpreter") + python_interpreter = python_interpreter or self._default_python() + if not python_interpreter: + raise ConanException("PipEnv could not find a Python executable path. Please, install " + "Python system-wide or set the " + "'tools.system.pipenv:python_interpreter' " + "conf to the full path of a Python executable") + + try: + self._conanfile.run(cmd_args_to_string([python_interpreter, '-m', 'venv', + self._env_dir])) + except ConanException as e: + raise ConanException(f"PipEnv could not create a Python virtual " + f"environment using '{python_interpreter}': {e}") + + def install(self, packages, pip_args=None): + """ + Will try to install the list of pip packages passed as a parameter. + + :param packages: try to install the list of pip packages passed as a parameter. + :param pip_args: additional argument list to be passed to the 'pip install' command, + e.g.: ['--no-cache-dir', '--index-url', 'https://my.pypi.org/simple']. + Defaults to ``None``. + :return: the return code of the executed pip command. + """ + args = [self._python_exe, "-m", "pip", "install", "--disable-pip-version-check"] + if pip_args: + args += list(pip_args) + args += list(packages) + command = cmd_args_to_string(args) + return self._conanfile.run(command) diff --git a/conans/migrations.py b/conans/migrations.py index a9fc7394e8a..3efa116536c 100644 --- a/conans/migrations.py +++ b/conans/migrations.py @@ -10,7 +10,7 @@ CONAN_VERSION = "version.txt" -class Migrator(object): +class Migrator: def __init__(self, conf_path, current_version): self.conf_path = conf_path diff --git a/conans/model/package_ref.py b/conans/model/package_ref.py deleted file mode 100644 index 55fe086f391..00000000000 --- a/conans/model/package_ref.py +++ /dev/null @@ -1,31 +0,0 @@ -from conan.api.output import ConanOutput -from conan.api.model import PkgReference as _PkgReference - -_msg = """ -******************************************************************* -Private '{}' -detected in user code (custom commands, extensions, recipes, etc). -Please stop using it, use only public documented APIs, , such as -'from conan.api.model import PkgReference' -as this will break in next releases. -******************************************************************* -""" - - -class PkgReference(_PkgReference): - usage_counter = 0 - - def __init__(self, *args, **kwargs): - if PkgReference.usage_counter == 0: - ConanOutput().warning(_msg.format("from conans.model.package_ref import PkgReference"), - warn_tag="deprecated") - PkgReference.usage_counter += 1 - super(PkgReference, self).__init__(*args, **kwargs) - - @staticmethod - def loads(*args, **kwargs): - if PkgReference.usage_counter == 0: - ConanOutput().warning(_msg.format("from conans.model.package_ref import PkgReference"), - warn_tag="deprecated") - PkgReference.usage_counter += 1 - return _PkgReference.loads(*args, **kwargs) diff --git a/conans/model/recipe_ref.py b/conans/model/recipe_ref.py deleted file mode 100644 index 97960babed2..00000000000 --- a/conans/model/recipe_ref.py +++ /dev/null @@ -1,31 +0,0 @@ -from conan.api.output import ConanOutput -from conan.api.model import RecipeReference as _RecipeReference - -_msg = """ -******************************************************************* -Private '{}' -detected in user code (custom commands, extensions, recipes, etc). -Please stop using it, use only public documented APIs, such as -'from conan.api.model import RecipeReference' -as this will break in next releases. -******************************************************************* -""" - - -class RecipeReference(_RecipeReference): - usage_counter = 0 - - def __init__(self, *args, **kwargs): - if RecipeReference.usage_counter == 0: - ConanOutput().warning(_msg.format("from conans.model.recipe_ref import RecipeReference"), - warn_tag="deprecated") - RecipeReference.usage_counter += 1 - super(RecipeReference, self).__init__(*args, **kwargs) - - @staticmethod - def loads(*args, **kwargs): - if RecipeReference.usage_counter == 0: - ConanOutput().warning(_msg.format("from conans.model.recipe_ref import RecipeReference"), - warn_tag="deprecated") - RecipeReference.usage_counter += 1 - return _RecipeReference.loads(*args, **kwargs) diff --git a/conans/requirements.txt b/conans/requirements.txt index d1b66f81ee0..a747fbf673a 100644 --- a/conans/requirements.txt +++ b/conans/requirements.txt @@ -1,9 +1,9 @@ requests>=2.25, <3.0.0 -urllib3>=1.26.6, <2.1 +urllib3>=1.26.6, <3.0 colorama>=0.4.3, <0.5.0 PyYAML>=6.0, <7.0 patch-ng>=1.18.0, <1.19 fasteners>=0.15 -distro>=1.4.0, <=1.8.0; platform_system == 'Linux' or platform_system == 'FreeBSD' +distro>=1.4.0, <2.0.0; platform_system == 'Linux' or platform_system == 'FreeBSD' Jinja2>=3.0, <4.0.0 python-dateutil>=2.8.0, <3 diff --git a/conans/requirements_dev.txt b/conans/requirements_dev.txt index b2876e233bb..7d8ca9b1989 100644 --- a/conans/requirements_dev.txt +++ b/conans/requirements_dev.txt @@ -1,7 +1,5 @@ pytest>=7, <8.0.0 pytest-xdist # To launch in N cores with pytest -n -parameterized>=0.6.3 -mock>=1.3.0, <1.4.0 WebTest>=3.0.0, <4 bottle PyJWT diff --git a/conans/server/rest/controller/v2/search.py b/conans/server/rest/controller/v2/search.py index 16b63f36a4e..d223674fecd 100644 --- a/conans/server/rest/controller/v2/search.py +++ b/conans/server/rest/controller/v2/search.py @@ -28,7 +28,10 @@ def search(auth_user): @app.route(r.common_search_packages_revision, method=["GET"]) def search_packages(name, version, username, channel, auth_user, revision=None): # query is no longer server side + list_only = request.params.get("list_only", False) + if isinstance(list_only, str): + list_only = False if 'false' == list_only.lower() else True search_service = SearchService(app.authorizer, app.server_store, auth_user) ref = RecipeReference(name, version, username, channel, revision) - info = search_service.search_packages(ref) + info = search_service.search_packages(ref, list_only) return info diff --git a/conans/server/service/authorize.py b/conans/server/service/authorize.py index 29ae2edfda4..542bf68478a 100644 --- a/conans/server/service/authorize.py +++ b/conans/server/service/authorize.py @@ -1,13 +1,11 @@ -''' - +""" This module handles user authentication and permissions for read or write a conans or package. This only reads from file the users and permissions. Replace this module with other that keeps the interface or super class. - -''' +""" from abc import ABCMeta, abstractmethod diff --git a/conans/server/service/v2/search.py b/conans/server/service/v2/search.py index dd52bd27996..5eba6ed979c 100644 --- a/conans/server/service/v2/search.py +++ b/conans/server/service/v2/search.py @@ -11,7 +11,7 @@ from conan.internal.util.files import load -def _get_local_infos_min(server_store, ref): +def _get_local_infos_min(server_store, ref, list_only): result = {} new_ref = ref subdirs = list_folder_subdirs(server_store.packages(new_ref), level=1) @@ -19,6 +19,9 @@ def _get_local_infos_min(server_store, ref): for package_id in sorted(subdirs): if package_id in result: continue + if list_only: + result[package_id] = {} + continue # Read conaninfo pref = PkgReference(new_ref, package_id) revision_entry = server_store.get_last_package_revision(pref) @@ -35,7 +38,7 @@ def _get_local_infos_min(server_store, ref): return result -def search_packages(server_store, ref): +def search_packages(server_store, ref, list_only): """ Return a dict like this: @@ -52,7 +55,7 @@ def search_packages(server_store, ref): ref_norev.revision = None if not os.path.exists(server_store.conan_revisions_root(ref_norev)): raise RecipeNotFoundException(ref) - infos = _get_local_infos_min(server_store, ref) + infos = _get_local_infos_min(server_store, ref, list_only) return infos @@ -63,10 +66,10 @@ def __init__(self, authorizer, server_store, auth_user): self._server_store = server_store self._auth_user = auth_user - def search_packages(self, reference): + def search_packages(self, reference, list_only=False): """Shared between v1 and v2, v1 will iterate rrevs""" self._authorizer.check_read_conan(self._auth_user, reference) - info = search_packages(self._server_store, reference) + info = search_packages(self._server_store, reference, list_only) return info def _search_recipes(self, pattern=None, ignorecase=True): diff --git a/setup.py b/setup.py index d2d685a7dc8..e032eac7f5a 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ def generate_long_description_file(): setup( name='conan', - python_requires='>=3.6', + python_requires='>=3.7', # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # https://packaging.python.org/en/latest/single_source_version.html @@ -84,9 +84,13 @@ def generate_long_description_file(): 'Topic :: Software Development :: Build Tools', 'License :: OSI Approved :: MIT License', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8' + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13' ], # What does your project relate to? diff --git a/setup_server.py b/setup_server.py index 8a6254b73f4..1353e9fc8ae 100644 --- a/setup_server.py +++ b/setup_server.py @@ -12,7 +12,6 @@ import os import re -from os import path # The tests utils are used by conan-package-tools @@ -79,9 +78,13 @@ def generate_long_description_file(): 'Topic :: Software Development :: Build Tools', 'License :: OSI Approved :: MIT License', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8' + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13' ], # What does your project relate to? diff --git a/test/README.md b/test/README.md index 22809fc9f90..c1b4e5c965f 100644 --- a/test/README.md +++ b/test/README.md @@ -64,13 +64,15 @@ external tools: `cmake`, `gcc`, `clang`, `visual_studio`, `mingw`, `autotools`, ```python @pytest.mark.skipif(platform.system() != "Windows", reason="Needs windows for vcvars") -@pytest.mark.visual_studio +@pytest.mark.tool("visual_studio") def test_vcvars_priority(self): client = TestClient() ... ``` If the test needs any of those tools to run it should be marked as using that tool and moved to the `test/functional` folder. +Note that only tests in ``test/functional`` might need the ``@pytest.mark.tool`` annotation. Tests in ``integration`` or ``unittest`` should never require an extra tool. + ### Parametrizing tests @@ -87,11 +89,23 @@ Please, if you need to run several combinations of the same testing code use par If you want to run the Coman test suite locally, please check the [README on the front page](https://github.com/conan-io/conan#running-the-tests). +Recall it is not expected for contributors to run the full test suite locally, only: + +- Run the ``unittest`` and ``integration`` tests. These shouldn't require any external tools +- If doing modifications to some specific build-system integration, locate the relevant folder under ``functional/toolchains`` and run those tests only. + + +The reason is that the ``functional`` test suite uses too many different external tools, and installing all of them can be tedious. +The Conan CI system will run those tests. + ## Installation of tools Work in progress! +Note the ``test/conftest.py`` file contains the upstream configuration of tools. +This file should not be changed, but users can create a ``test/conftest_user.py`` file containing their local definitions of tools, that will override the ``conftest.py`` definitions. + ### Windows msys2, mingw64 and mingw32 Download msys2 (64 bit) from msys2.org diff --git a/test/conftest.py b/test/conftest.py index 0d6d3c76891..50b4fd65ae8 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -48,7 +48,13 @@ msys2_path = os.getenv("MSYS2_PATH", "C:/msys64") tools_locations = { - "clang": {"disabled": True}, + "clang": { + "exe": "clang", + "default": "20", + "20": { + "path": {'Windows': 'C:/Program Files/LLVM/bin'} # by choco + } + }, 'visual_studio': {"default": "15", "15": {}, "16": {"disabled": True}, @@ -87,10 +93,10 @@ 'Darwin': '/Users/runner/Applications/CMake/3.27.9/bin', 'Linux': "/usr/share/cmake-3.27.9/bin"} }, - "4.0": { - "path": {'Windows': 'C:/tools/cmake/4.0.0-rc3/cmake-4.0.0-rc3-windows-x86_64/bin', - 'Darwin': '/Users/runner/Applications/CMake/4.0.0-rc3/bin', - 'Linux': "/usr/share/cmake-4.0.0-rc3/bin"} + "4.2": { + "path": {'Windows': 'C:/tools/cmake/4.2.1/cmake-4.2.1-windows-x86_64/bin', + 'Darwin': '/Users/runner/Applications/CMake/4.2.1/bin', + 'Linux': "/usr/share/cmake-4.2.1/bin"} } }, 'ninja': { @@ -153,16 +159,16 @@ "system": {"path": {'Windows': "C:/tools/cygwin/bin"}}, }, 'bazel': { - "default": "7", - "6.5.0": {"path": {'Linux': '/usr/share/bazel-6.5.0/bin', + "default": "7.x", + "6.x": {"path": {'Linux': '/usr/share/bazel-6.5.0/bin', 'Windows': 'C:/tools/bazel/6.5.0', 'Darwin': '/Users/runner/Applications/bazel/6.5.0'}}, - "7.4.1": {"path": {'Linux': '/usr/share/bazel-7.4.1/bin', - 'Windows': 'C:/tools/bazel/7.4.1', - 'Darwin': '/Users/runner/Applications/bazel/7.4.1'}}, - "8.0.0": {"path": {'Linux': '/usr/share/bazel-8.0.0/bin', - 'Windows': 'C:/tools/bazel/8.0.0', - 'Darwin': '/Users/runner/Applications/bazel/8.0.0'}}, + "7.x": {"path": {'Linux': '/usr/share/bazel-7.6.2/bin', + 'Windows': 'C:/tools/bazel/7.6.2', + 'Darwin': '/Users/runner/Applications/bazel/7.6.2'}}, + "8.x": {"path": {'Linux': '/usr/share/bazel-8.4.2/bin', + 'Windows': 'C:/tools/bazel/8.4.2', + 'Darwin': '/Users/runner/Applications/bazel/8.4.2'}}, }, 'premake': { "exe": "premake5", diff --git a/test/functional/command/report_test.py b/test/functional/command/report_test.py index fa3b0270ccb..302c1679530 100644 --- a/test/functional/command/report_test.py +++ b/test/functional/command/report_test.py @@ -7,56 +7,44 @@ from conan.test.utils.tools import TestClient -@pytest.mark.parametrize("old_args", [ - "-op=v1 -or=pkg/1.0", - "-or=pkg/1.0" -]) -@pytest.mark.parametrize("new_args", [ - "-np=v2 -nr=pkg/2.0", - "-nr=pkg/2.0#{revision}", -]) -@pytest.mark.parametrize("formatter", [ - "-f=json --out-file=output.json", - "-f=html --out-file=output.html" -]) -@pytest.mark.tool("git") -def test_compare_paths(old_args, new_args, formatter): +@pytest.fixture +def fixture_client(): tc = TestClient(light=True, default_server_user=True) patch_file = textwrap.dedent(""" - diff --git a/foo.txt b/foo.txt - new file mode 100644 - index 0000000..e311fbb - --- /dev/null - +++ b/foo.txt - @@ -0,0 +1 @@ - +{version} - """) + diff --git a/foo.txt b/foo.txt + new file mode 100644 + index 0000000..e311fbb + --- /dev/null + +++ b/foo.txt + @@ -0,0 +1 @@ + +{version} + """) conandata_yml = textwrap.dedent(""" - patches: - "{version}": - - patch_file: "patches/patch.patch" - """) + patches: + "{version}": + - patch_file: "patches/patch.patch" + """) conanfile = textwrap.dedent(""" - from conan import ConanFile - from conan.tools.files import apply_conandata_patches, export_conandata_patches, save + from conan import ConanFile + from conan.tools.files import apply_conandata_patches, export_conandata_patches, save - class TestConan(ConanFile): - name = "pkg" + class TestConan(ConanFile): + name = "pkg" - exports = "{version}.txt" + exports = "{version}.txt" - def export_sources(self): - export_conandata_patches(self) + def export_sources(self): + export_conandata_patches(self) - def source(self): - save(self, "myfile.txt", "{version}") - if self.version == "2.0": - save(self, "new-file-for-v2.txt", "a new file for the new version") - apply_conandata_patches(self) - """) + def source(self): + save(self, "myfile.txt", "{version}") + if self.version == "2.0": + save(self, "new-file-for-v2.txt", "a new file for the new version") + apply_conandata_patches(self) + """) tc.save({ "v1/conanfile.py": conanfile.format(version="v1"), @@ -72,36 +60,58 @@ def source(self): tc.run("create v1 --version=1.0") v1_layout = tc.exported_layout() - v1_path = v1_layout.export() + tc.test_v1_path = v1_layout.export() tc.run("create v2 --version=2.0") - v2_revision = tc.exported_layout().reference.revision + tc.test_v2_revision = tc.exported_layout().reference.revision tc.run("upload * -r=default -c") tc.run("remove * -c") + return tc + + +@pytest.mark.parametrize("old_args", [ + "-op=v1 -or=pkg/1.0", + "-or=pkg/1.0" +]) +@pytest.mark.parametrize("new_args", [ + "-np=v2 -nr=pkg/2.0", + "-nr=pkg/2.0#{revision}", +]) +@pytest.mark.parametrize("formatter", [ + "-f=json --out-file=output.json", + "-f=html --out-file=output.html" +]) +@pytest.mark.tool("git") +def test_compare_paths(fixture_client, old_args, new_args, formatter): + tc = fixture_client + v2_revision = fixture_client.test_v2_revision + v1_path = tc.test_v1_path.replace("\\", "/") + new_args = new_args.format(revision=v2_revision) tc.run(f'report diff {old_args} {new_args} {formatter}') if "json" in formatter: output_json = json.loads(tc.load("output.json")) - assert any(os.path.join(v1_path, "conanfile.py").replace("\\", "/") in k for k in output_json.keys()) - assert any(os.path.join(v1_path, "conanmanifest.txt").replace("\\", "/") in k for k in output_json.keys()) + assert any(os.path.join(v1_path, "conanfile.py").replace("\\", "/") in k + for k in output_json.keys()) + assert any(os.path.join(v1_path, "conanmanifest.txt").replace("\\", "/") in k + for k in output_json.keys()) # We have patch information - assert any(os.path.abspath(os.path.join(v1_path, "..", "es", "patches", "patch.patch")).replace("\\", "/") in k for k in output_json.keys()) - # '/private/var/folders/lw/6bflvp3s3t5b56n2p_bj_vx80000gn/T/tmptcto18wfconans/path with spaces/.conan2/p/pkg8fd97e4b9cf55/es/patches/patch.patch' + assert any(os.path.abspath(os.path.join(v1_path, "..", "es", "patches", "patch.patch")).replace("\\", "/") in k + for k in output_json.keys()) # We have exports information - assert any(os.path.abspath(os.path.join(v1_path, "..", "e", "v1.txt")).replace("\\", "/") in k for k in output_json.keys()) + assert any(os.path.abspath(os.path.join(v1_path, "..", "e", "v1.txt")).replace("\\", "/") in k + for k in output_json.keys()) # New files assert any("new-file-for-v2.txt" in k for k in output_json.keys()) elif "html" in formatter: output_html = tc.load("output.html") # We have patch information - assert f"""(new)/es/patches/patch.patch""" in output_html + assert "(old)/es/patches/patch.patch" in output_html + assert "(new)/s/new-file-for-v2.txt" in output_html # We have exports information - assert f"""(old)/e/v1.txt""" in output_html - # New files appear both in the menu and in the diff - assert output_html.count("(new)/s/new-file-for-v2.txt") == 3 - + assert "(old)/e/v1.txt" in output_html diff --git a/test/functional/command/config_install_test.py b/test/functional/command/test_config_install.py similarity index 50% rename from test/functional/command/config_install_test.py rename to test/functional/command/test_config_install.py index c317bece80f..85143b5e71c 100644 --- a/test/functional/command/config_install_test.py +++ b/test/functional/command/test_config_install.py @@ -3,13 +3,16 @@ import shutil import stat import textwrap -import unittest import pytest -from mock import patch +from unittest.mock import patch -from conan.api.model import Remote +import yaml + +from conan.api.model import Remote, RecipeReference from conan.internal.api.config.config_installer import _hide_password + +from conan.internal.cache.home_paths import HomePaths from conan.internal.rest.file_downloader import FileDownloader from conan.internal.paths import DEFAULT_CONAN_HOME from conan.test.assets.genconanfile import GenConanfile @@ -24,14 +27,6 @@ def make_file_read_only(file_path): os.chmod(file_path, mode & ~ stat.S_IWRITE) -win_profile = """[settings] - os: Windows -""" - -linux_profile = """[settings] - os: Linux -""" - remotes = """{ "remotes": [ { @@ -54,48 +49,23 @@ def make_file_read_only(file_path): arch: [x86, x86_64] """ -cache_conan_conf = """ -[log] -run_to_output = False # environment CONAN_LOG_RUN_TO_OUTPUT -level = 10 # environment CONAN_LOGGING_LEVEL - -[general] -cpu_count = 1 # environment CONAN_CPU_COUNT - -[proxies] -# Empty (or missing) section will try to use system proxies. -# As documented in https://requests.readthedocs.io/en/master/user/advanced/#proxies -http = http://user:pass@10.10.1.10:3128/ -https = None -# http = http://10.10.1.10:3128 -# https = http://10.10.1.10:1080 -""" - -myfuncpy = """def mycooladd(a, b): - return a + b -""" - -class ConfigInstallTest(unittest.TestCase): - def setUp(self): - self.client = TestClient() - save(os.path.join(self.client.paths.profiles_path, "default"), "#default profile empty") - save(os.path.join(self.client.paths.profiles_path, "linux"), "#empty linux profile") +class TestConfigInstall: @staticmethod def _create_profile_folder(folder=None): folder = folder or temp_folder(path_with_spaces=False) save_files(folder, {"settings.yml": settings_yml, "remotes.json": remotes, - "profiles/linux": linux_profile, - "profiles/windows": win_profile, + "profiles/linux": "#linuxprofile", + "profiles/windows": "#winprofile", "hooks/dummy": "#hook dummy", "hooks/foo.py": "#hook foo", "hooks/custom/custom.py": "#hook custom", ".git/hooks/foo": "foo", "hooks/.git/hooks/before_push": "before_push", "pylintrc": "#Custom pylint", - "python/myfuncs.py": myfuncpy, + "python/myfuncs.py": "does not matter", "python/__init__.py": "" }) return folder @@ -108,9 +78,8 @@ def test_config_fails_no_storage(self): client.run("create . --name=pkg --version=1.0") client.run('config install "%s"' % folder) client.run("remote list") - self.assertIn("myrepo1: https://myrepourl.net [Verify SSL: False, Enabled: True]", - client.out) - self.assertIn("my-repo-2: https://myrepo2.com [Verify SSL: True, Enabled: True]", client.out) + assert "myrepo1: https://myrepourl.net [Verify SSL: False, Enabled: True]" in client.out + assert "my-repo-2: https://myrepo2.com [Verify SSL: True, Enabled: True]" in client.out def _create_zip(self, zippath=None): folder = self._create_profile_folder() @@ -133,90 +102,84 @@ def _create_tgz(self, tgz_path=None): files = self._get_files(folder) return tgz_with_contents(files, tgz_path) - def _check(self, params): - settings_path = self.client.paths.settings_path - self.assertEqual(load(settings_path).splitlines(), settings_yml.splitlines()) - api = self.client.api + @staticmethod + def _check(c): + settings_path = c.paths.settings_path + assert load(settings_path).splitlines() == settings_yml.splitlines() + api = c.api cache_remotes = api.remotes.list() - self.assertEqual(list(cache_remotes), [ + assert list(cache_remotes) == [ Remote("myrepo1", "https://myrepourl.net", False, False), Remote("my-repo-2", "https://myrepo2.com", True, False), - ]) - self.assertEqual(sorted(os.listdir(self.client.paths.profiles_path)), - sorted(["default", "linux", "windows"])) - self.assertEqual(load(os.path.join(self.client.paths.profiles_path, "linux")).splitlines(), - linux_profile.splitlines()) - self.assertEqual(load(os.path.join(self.client.paths.profiles_path, - "windows")).splitlines(), - win_profile.splitlines()) - self.assertEqual("#Custom pylint", - load(os.path.join(self.client.cache_folder, "pylintrc"))) - self.assertEqual("", - load(os.path.join(self.client.cache_folder, "python", - "__init__.py"))) - self.assertEqual("#hook dummy", - load(os.path.join(self.client.cache_folder, "hooks", "dummy"))) - self.assertEqual("#hook foo", - load(os.path.join(self.client.cache_folder, "hooks", "foo.py"))) - self.assertEqual("#hook custom", - load(os.path.join(self.client.cache_folder, "hooks", "custom", - "custom.py"))) - self.assertFalse(os.path.exists(os.path.join(self.client.cache_folder, "hooks", - ".git"))) - self.assertFalse(os.path.exists(os.path.join(self.client.cache_folder, ".git"))) + ] + assert sorted(os.listdir(c.paths.profiles_path)) == sorted(["default", "linux", "windows"]) + assert c.load_home("profiles/linux") == "#linuxprofile" + assert c.load_home("profiles/windows") == "#winprofile" + assert "#Custom pylint" == c.load_home("pylintrc") + assert "" == c.load_home("python/__init__.py") + assert "#hook dummy" == c.load_home("hooks/dummy") + assert "#hook foo" == c.load_home("hooks/foo.py") + assert "#hook custom" == c.load_home("hooks/custom/custom.py") + assert not os.path.exists(os.path.join(c.cache_folder, "hooks", ".git")) + assert not os.path.exists(os.path.join(c.cache_folder, ".git")) def test_install_file(self): """ should install from a file in current dir """ zippath = self._create_zip() + c = TestClient(light=True) for filetype in ["", "--type=file"]: - self.client.run('config install "%s" %s' % (zippath, filetype)) - self._check("file, %s, True, None" % zippath) - self.assertTrue(os.path.exists(zippath)) + c.run('config install "%s" %s' % (zippath, filetype)) + self._check(c) + assert os.path.exists(zippath) def test_install_config_file(self): """ should install from a settings and remotes file in configuration directory """ import tempfile profile_folder = self._create_profile_folder() - self.assertTrue(os.path.isdir(profile_folder)) + assert os.path.isdir(profile_folder) src_setting_file = os.path.join(profile_folder, "settings.yml") src_remote_file = os.path.join(profile_folder, "remotes.json") - # Install profile_folder without settings.yml + remotes.json in order to install them manually + # Install profile_folder without settings.yml remotes.json in order to install them manually tmp_dir = tempfile.mkdtemp() dest_setting_file = os.path.join(tmp_dir, "settings.yml") dest_remote_file = os.path.join(tmp_dir, "remotes.json") shutil.move(src_setting_file, dest_setting_file) shutil.move(src_remote_file, dest_remote_file) - self.client.run('config install "%s"' % profile_folder) + + c = TestClient(light=True) + c.run('config install "%s"' % profile_folder) shutil.move(dest_setting_file, src_setting_file) shutil.move(dest_remote_file, src_remote_file) shutil.rmtree(tmp_dir) for cmd_option in ["", "--type=file"]: - self.client.run('config install "%s" %s' % (src_setting_file, cmd_option)) - self.client.run('config install "%s" %s' % (src_remote_file, cmd_option)) - self._check("file, %s, True, None" % src_remote_file) + c.run('config install "%s" %s' % (src_setting_file, cmd_option)) + c.run('config install "%s" %s' % (src_remote_file, cmd_option)) + self._check(c) def test_install_dir(self): """ should install from a dir in current dir """ folder = self._create_profile_folder() - self.assertTrue(os.path.isdir(folder)) + assert os.path.isdir(folder) + c = TestClient(light=True) for dirtype in ["", "--type=dir"]: - self.client.run('config install "%s" %s' % (folder, dirtype)) - self._check("dir, %s, True, None" % folder) + c.run('config install "%s" %s' % (folder, dirtype)) + self._check(c) def test_install_source_target_folders(self): folder = temp_folder() save_files(folder, {"subf/file.txt": "hello", "subf/subf/file2.txt": "bye"}) - self.client.run('config install "%s" -sf=subf -tf=newsubf' % folder) - content = load(os.path.join(self.client.cache_folder, "newsubf/file.txt")) - self.assertEqual(content, "hello") - content = load(os.path.join(self.client.cache_folder, "newsubf/subf/file2.txt")) - self.assertEqual(content, "bye") + c = TestClient(light=True) + c.run('config install "%s" -sf=subf -tf=newsubf' % folder) + content = c.load_home("newsubf/file.txt") + assert content == "hello" + content = c.load_home("newsubf/subf/file2.txt") + assert content == "bye" def test_install_remotes_json(self): folder = temp_folder() @@ -239,170 +202,173 @@ def test_install_remotes_json(self): save_files(folder, {"remotes.json": remotes_json, "remotes.txt": remotes_txt}) - self.client.run(f'config install "{folder}"') - assert "Defining remotes from remotes.json" in self.client.out + c = TestClient(light=True) + c.run(f'config install "{folder}"') + assert "Defining remotes from remotes.json" in c.out - self.client.run('remote list') + c.run('remote list') - assert "repojson1: https://repojson1.net [Verify SSL: False, Enabled: True]" in self.client.out - assert "repojson2: https://repojson2.com [Verify SSL: True, Enabled: True]" in self.client.out + assert "repojson1: https://repojson1.net [Verify SSL: False, Enabled: True]" in c.out + assert "repojson2: https://repojson2.com [Verify SSL: True, Enabled: True]" in c.out # We only install remotes.json folder = temp_folder() save_files(folder, {"remotes.json": remotes_json}) - self.client.run(f'config install "{folder}"') - assert "Defining remotes from remotes.json" in self.client.out + c.run(f'config install "{folder}"') + assert "Defining remotes from remotes.json" in c.out - self.client.run('remote list') + c.run('remote list') - assert "repojson1: https://repojson1.net [Verify SSL: False, Enabled: True]" in self.client.out - assert "repojson2: https://repojson2.com [Verify SSL: True, Enabled: True]" in self.client.out + assert "repojson1: https://repojson1.net [Verify SSL: False, Enabled: True]" in c.out + assert "repojson2: https://repojson2.com [Verify SSL: True, Enabled: True]" in c.out def test_without_profile_folder(self): - shutil.rmtree(self.client.paths.profiles_path) + c = TestClient(light=True) + shutil.rmtree(c.paths.profiles_path) zippath = self._create_zip() - self.client.run('config install "%s"' % zippath) - self.assertEqual(sorted(os.listdir(self.client.paths.profiles_path)), - sorted(["linux", "windows"])) - self.assertEqual(load(os.path.join(self.client.paths.profiles_path, "linux")).splitlines(), - linux_profile.splitlines()) + + c.run('config install "%s"' % zippath) + assert sorted(os.listdir(c.paths.profiles_path)) == sorted(["linux", "windows"]) + assert c.load_home("profiles/linux") == "#linuxprofile" def test_install_url(self): """ should install from a URL """ - + c = TestClient(light=True) for origin in ["", "--type=url"]: - def my_download(obj, url, file_path, **kwargs): # @UnusedVariable + def my_download(obj, url, file_path, **kwargs): # noqa self._create_zip(file_path) with patch.object(FileDownloader, 'download', new=my_download): - self.client.run("config install http://myfakeurl.com/myconf.zip %s" % origin) - self._check("url, http://myfakeurl.com/myconf.zip, True, None") + c.run("config install http://myfakeurl.com/myconf.zip %s" % origin) + self._check(c) # repeat the process to check - self.client.run("config install http://myfakeurl.com/myconf.zip %s" % origin) - self._check("url, http://myfakeurl.com/myconf.zip, True, None") + c.run("config install http://myfakeurl.com/myconf.zip %s" % origin) + self._check(c) def test_install_url_query(self): """ should install from a URL """ + c = TestClient(light=True) - def my_download(obj, url, file_path, **kwargs): # @UnusedVariable + def my_download(obj, url, file_path, **kwargs): # noqa self._create_zip(file_path) with patch.object(FileDownloader, 'download', new=my_download): # repeat the process to check it works with ?args - self.client.run("config install http://myfakeurl.com/myconf.zip?sha=1") - self._check("url, http://myfakeurl.com/myconf.zip?sha=1, True, None") + c.run("config install http://myfakeurl.com/myconf.zip?sha=1") + self._check(c) def test_install_change_only_verify_ssl(self): - def my_download(obj, url, file_path, **kwargs): # @UnusedVariable + def my_download(obj, url, file_path, **kwargs): # noqa self._create_zip(file_path) + c = TestClient(light=True) with patch.object(FileDownloader, 'download', new=my_download): - self.client.run("config install http://myfakeurl.com/myconf.zip") - self._check("url, http://myfakeurl.com/myconf.zip, True, None") + c.run("config install http://myfakeurl.com/myconf.zip") + self._check(c) # repeat the process to check - self.client.run("config install http://myfakeurl.com/myconf.zip --verify-ssl=False") - self._check("url, http://myfakeurl.com/myconf.zip, False, None") + c.run("config install http://myfakeurl.com/myconf.zip --verify-ssl=False") + self._check(c) def test_install_url_tgz(self): """ should install from a URL to tar.gz """ + c = TestClient(light=True) - def my_download(obj, url, file_path, **kwargs): # @UnusedVariable + def my_download(obj, url, file_path, **kwargs): # noqa self._create_tgz(file_path) with patch.object(FileDownloader, 'download', new=my_download): - self.client.run("config install http://myfakeurl.com/myconf.tar.gz") - self._check("url, http://myfakeurl.com/myconf.tar.gz, True, None") + c.run("config install http://myfakeurl.com/myconf.tar.gz") + self._check(c) def test_failed_install_repo(self): """ should install from a git repo """ - self.client.run('config install notexistingrepo.git', assert_error=True) - self.assertIn("ERROR: Failed conan config install: Can't clone repo", self.client.out) + c = TestClient(light=True) + c.run('config install notexistingrepo.git', assert_error=True) + assert "ERROR: Failed conan config install: Can't clone repo" in c.out def test_failed_install_http(self): """ should install from a http zip """ - self.client.run('config install httpnonexisting', assert_error=True) - self.assertIn("ERROR: Failed conan config install: " - "Error while installing config from httpnonexisting", self.client.out) + c = TestClient(light=True) + c.run('config install httpnonexisting', assert_error=True) + assert ("ERROR: Failed conan config install: " + "Error while installing config from httpnonexisting") in c.out @pytest.mark.tool("git") def test_install_repo(self): """ should install from a git repo """ - + c = TestClient(light=True) folder = self._create_profile_folder() - with self.client.chdir(folder): - self.client.run_command('git init .') - self.client.run_command('git add .') - self.client.run_command('git config user.name myname') - self.client.run_command('git config user.email myname@mycompany.com') - self.client.run_command('git commit -m "mymsg"') + with c.chdir(folder): + c.run_command('git init .') + c.run_command('git add .') + c.run_command('git config user.name myname') + c.run_command('git config user.email myname@mycompany.com') + c.run_command('git commit -m "mymsg"') - self.client.run('config install "%s/.git"' % folder) - check_path = os.path.join(folder, ".git") - self._check("git, %s, True, None" % check_path) + c.run('config install "%s/.git"' % folder) + self._check(c) @pytest.mark.tool("git") def test_install_repo_relative(self): + c = TestClient(light=True) relative_folder = "./config" - absolute_folder = os.path.join(self.client.current_folder, "config") + absolute_folder = os.path.join(c.current_folder, "config") mkdir(absolute_folder) folder = self._create_profile_folder(absolute_folder) - with self.client.chdir(folder): - self.client.run_command('git init .') - self.client.run_command('git add .') - self.client.run_command('git config user.name myname') - self.client.run_command('git config user.email myname@mycompany.com') - self.client.run_command('git commit -m "mymsg"') + with c.chdir(folder): + c.run_command('git init .') + c.run_command('git add .') + c.run_command('git config user.name myname') + c.run_command('git config user.email myname@mycompany.com') + c.run_command('git commit -m "mymsg"') - self.client.run('config install "%s/.git"' % relative_folder) - self._check("git, %s, True, None" % os.path.join("%s" % folder, ".git")) + c.run('config install "%s/.git"' % relative_folder) + self._check(c) @pytest.mark.tool("git") def test_install_custom_args(self): """ should install from a git repo """ - + c = TestClient(light=True) folder = self._create_profile_folder() - with self.client.chdir(folder): - self.client.run_command('git init .') - self.client.run_command('git add .') - self.client.run_command('git config user.name myname') - self.client.run_command('git config user.email myname@mycompany.com') - self.client.run_command('git commit -m "mymsg"') + with c.chdir(folder): + c.run_command('git init .') + c.run_command('git add .') + c.run_command('git config user.name myname') + c.run_command('git config user.email myname@mycompany.com') + c.run_command('git commit -m "mymsg"') - self.client.run('config install "%s/.git" --args="-c init.templateDir=value"' % folder) - check_path = os.path.join(folder, ".git") - self._check("git, %s, True, -c init.templateDir=value" % check_path) + c.run('config install "%s/.git" --args="-c init.templateDir=value"' % folder) + self._check(c) def test_force_git_type(self): - client = TestClient() + client = TestClient(light=True) client.run('config install httpnonexisting --type=git', assert_error=True) - self.assertIn("Can't clone repo", client.out) + assert "Can't clone repo" in client.out def test_force_dir_type(self): - client = TestClient() - client.run('config install httpnonexisting --type=dir', assert_error=True) - self.assertIn("ERROR: Failed conan config install: No such directory: 'httpnonexisting'", - client.out) + c = TestClient(light=True) + c.run('config install httpnonexisting --type=dir', assert_error=True) + assert "ERROR: Failed conan config install: No such directory: 'httpnonexisting'" in c.out def test_force_file_type(self): - client = TestClient() + client = TestClient(light=True) client.run('config install httpnonexisting --type=file', assert_error=True) - self.assertIn("No such file or directory: 'httpnonexisting'", client.out) + assert "No such file or directory: 'httpnonexisting'" in client.out def test_force_url_type(self): - client = TestClient() + client = TestClient(light=True) client.run('config install httpnonexisting --type=url', assert_error=True) - self.assertIn("Error downloading file httpnonexisting: 'Invalid URL 'httpnonexisting'", - client.out) + assert "Error downloading file httpnonexisting: 'Invalid URL 'httpnonexisting'" in client.out def test_removed_credentials_from_url_unit(self): """ @@ -414,24 +380,24 @@ def test_removed_credentials_from_url_unit(self): url_hidden_password = r"https://test_username:@server.com/resource.zip" # Check url is the same when not using credentials - self.assertEqual(_hide_password(url_without_credentials), url_without_credentials) + assert _hide_password(url_without_credentials) == url_without_credentials # Check password is hidden using url with credentials - self.assertEqual(_hide_password(url_with_credentials), url_hidden_password) + assert _hide_password(url_with_credentials) == url_hidden_password # Check that it works with other protocols ftp ftp_with_credentials = r"ftp://test_username_ftp:test_password_321@server.com/resurce.zip" ftp_hidden_password = r"ftp://test_username_ftp:@server.com/resurce.zip" - self.assertEqual(_hide_password(ftp_with_credentials), ftp_hidden_password) + assert _hide_password(ftp_with_credentials) == ftp_hidden_password # Check function also works for file paths *unix/windows unix_file_path = r"/tmp/test" - self.assertEqual(_hide_password(unix_file_path), unix_file_path) + assert _hide_password(unix_file_path) == unix_file_path windows_file_path = r"c:\windows\test" - self.assertEqual(_hide_password(windows_file_path), windows_file_path) + assert _hide_password(windows_file_path) == windows_file_path # Check works with empty string - self.assertEqual(_hide_password(''), '') + assert _hide_password('') == '' def test_remove_credentials_config_installer(self): """ Functional test to check credentials are not displayed in output but are still present @@ -441,144 +407,149 @@ def test_remove_credentials_config_installer(self): fake_url_with_credentials = "http://test_user:test_password@myfakeurl.com/myconf.zip" fake_url_hidden_password = "http://test_user:@myfakeurl.com/myconf.zip" - def my_download(obj, url, file_path, **kwargs): # @UnusedVariable - self.assertEqual(url, fake_url_with_credentials) + def my_download(obj, url, file_path, **kwargs): # noqa + assert url == fake_url_with_credentials self._create_zip(file_path) + c = TestClient(light=True) with patch.object(FileDownloader, 'download', new=my_download): - self.client.run("config install %s" % fake_url_with_credentials) + c.run("config install %s" % fake_url_with_credentials) # Check credentials are not displayed in output - self.assertNotIn(fake_url_with_credentials, self.client.out) - self.assertIn(fake_url_hidden_password, self.client.out) + assert fake_url_with_credentials not in c.out + assert fake_url_hidden_password in c.out # Check credentials still stored in configuration - self._check("url, %s, True, None" % fake_url_with_credentials) + self._check(c) def test_ssl_verify(self): + c = TestClient(light=True) + fake_url = "https://fakeurl.com/myconf.zip" - def download_verify_false(obj, url, file_path, **kwargs): # @UnusedVariable + def download_verify_false(obj, url, file_path, **kwargs): # noqa assert kwargs["verify_ssl"] is False self._create_zip(file_path) - def download_verify_true(obj, url, file_path, **kwargs): # @UnusedVariable + def download_verify_true(obj, url, file_path, **kwargs): # noqa assert kwargs["verify_ssl"] is True self._create_zip(file_path) with patch.object(FileDownloader, 'download', new=download_verify_false): - self.client.run("config install %s --verify-ssl=False" % fake_url) + c.run("config install %s --verify-ssl=False" % fake_url) with patch.object(FileDownloader, 'download', new=download_verify_true): - self.client.run("config install %s --verify-ssl=True" % fake_url) + c.run("config install %s --verify-ssl=True" % fake_url) with patch.object(FileDownloader, 'download', new=download_verify_true): - self.client.run(f"config install {fake_url}") + c.run(f"config install {fake_url}") with patch.object(FileDownloader, 'download', new=download_verify_false): - self.client.run(f"config install {fake_url} --insecure") + c.run(f"config install {fake_url} --insecure") @pytest.mark.tool("git") def test_git_checkout_is_possible(self): folder = self._create_profile_folder() - with self.client.chdir(folder): - self.client.run_command('git init .') - self.client.run_command('git checkout -b master') - self.client.run_command('git add .') - self.client.run_command('git config user.name myname') - self.client.run_command('git config user.email myname@mycompany.com') - self.client.run_command('git commit -m "mymsg"') - self.client.run_command('git checkout -b other_branch') + c = TestClient(light=True) + with c.chdir(folder): + c.run_command('git init .') + c.run_command('git checkout -b master') + c.run_command('git add .') + c.run_command('git config user.name myname') + c.run_command('git config user.email myname@mycompany.com') + c.run_command('git commit -m "mymsg"') + c.run_command('git checkout -b other_branch') save(os.path.join(folder, "extensions", "hooks", "cust", "cust.py"), "") - self.client.run_command('git add .') - self.client.run_command('git commit -m "my file"') + c.run_command('git add .') + c.run_command('git commit -m "my file"') - self.client.run('config install "%s/.git" --args "-b other_branch"' % folder) - check_path = os.path.join(folder, ".git") - self._check("git, %s, True, -b other_branch" % check_path) - file_path = os.path.join(self.client.paths.hooks_path, "cust", "cust.py") + c.run('config install "%s/.git" --args "-b other_branch"' % folder) + self._check(c) + file_path = os.path.join(c.paths.hooks_path, "cust", "cust.py") assert load(file_path) == "" # Add changes to that branch and update - with self.client.chdir(folder): + with c.chdir(folder): save(os.path.join(folder, "extensions", "hooks", "cust", "cust.py"), "new content") - self.client.run_command('git add .') - self.client.run_command('git commit -m "my other file"') - self.client.run_command('git checkout master') - self.client.run('config install "%s/.git" --args "-b other_branch"' % folder) - check_path = os.path.join(folder, ".git") - self._check("git, %s, True, -b other_branch" % check_path) + c.run_command('git add .') + c.run_command('git commit -m "my other file"') + c.run_command('git checkout master') + c.run('config install "%s/.git" --args "-b other_branch"' % folder) + self._check(c) assert load(file_path) == "new content" def test_config_install_requester(self): # https://github.com/conan-io/conan/issues/4169 path = self._create_zip() + c = TestClient(light=True) file_server = TestFileServer(os.path.dirname(path)) - self.client.servers["file_server"] = file_server - - self.client.run(f"config install {file_server.fake_url}/myconfig.zip") - assert "Defining remotes from remotes.json" in self.client.out - assert "Copying file myfuncs.py" in self.client.out + c.servers["file_server"] = file_server + c.run(f"config install {file_server.fake_url}/myconfig.zip") + assert "Defining remotes from remotes.json" in c.out + assert "Copying file myfuncs.py" in c.out def test_overwrite_read_only_file(self): + c = TestClient(light=True) source_folder = self._create_profile_folder() - self.client.run('config install "%s"' % source_folder) + c.run('config install "%s"' % source_folder) # make existing settings.yml read-only - make_file_read_only(self.client.paths.settings_path) - self.assertFalse(os.access(self.client.paths.settings_path, os.W_OK)) + make_file_read_only(c.paths.settings_path) + assert not os.access(c.paths.settings_path, os.W_OK) # config install should overwrite the existing read-only file - self.client.run('config install "%s"' % source_folder) - self.assertTrue(os.access(self.client.paths.settings_path, os.W_OK)) + c.run('config install "%s"' % source_folder) + assert os.access(c.paths.settings_path, os.W_OK) def test_dont_copy_file_permissions(self): source_folder = self._create_profile_folder() # make source settings.yml read-only make_file_read_only(os.path.join(source_folder, 'remotes.json')) - self.client.run('config install "%s"' % source_folder) - self.assertTrue(os.access(self.client.paths.settings_path, os.W_OK)) - + c = TestClient(light=True) + c.run('config install "%s"' % source_folder) + assert os.access(c.paths.settings_path, os.W_OK) -class ConfigInstallSchedTest(unittest.TestCase): - def setUp(self): - self.folder = temp_folder(path_with_spaces=False) - # FIXME: This is broken, this config no longer exist, but it doesn't raise, fix it - save_files(self.folder, {"global.conf": "core:config_install_interval=5m"}) - self.client = TestClient() - self.client.save({"conanfile.txt": ""}) +class TestConfigInstallSched: def test_execute_more_than_once(self): """ Once executed by the scheduler, conan config install must executed again when invoked manually """ - self.client.run('config install "%s"' % self.folder) - self.assertIn("Copying file global.conf", self.client.out) + folder = temp_folder(path_with_spaces=False) + save_files(folder, {"global.conf": "core.download:parallel=0"}) + c = TestClient(light=True) + c.run('config install "%s"' % folder) + assert "Copying file global.conf" in c.out - self.client.run('config install "%s"' % self.folder) - self.assertIn("Copying file global.conf", self.client.out) + c.run('config install "%s"' % folder) + assert "Copying file global.conf" in c.out @pytest.mark.tool("git") def test_config_install_remove_git_repo(self): """ config_install_interval must break when remote git has been removed """ - with self.client.chdir(self.folder): - self.client.run_command('git init .') - self.client.run_command('git add .') - self.client.run_command('git config user.name myname') - self.client.run_command('git config user.email myname@mycompany.com') - self.client.run_command('git commit -m "mymsg"') - self.client.run('config install "%s/.git" --type git' % self.folder) - self.assertIn("Copying file global.conf", self.client.out) - self.assertIn("Repo cloned!", self.client.out) # git clone executed by scheduled task + folder = temp_folder(path_with_spaces=False) + save_files(folder, {"global.conf": "core.download:parallel=0"}) + c = TestClient(light=True) + with c.chdir(folder): + c.run_command('git init .') + c.run_command('git add .') + c.run_command('git config user.name myname') + c.run_command('git config user.email myname@mycompany.com') + c.run_command('git commit -m "mymsg"') + c.run('config install "%s/.git" --type git' % folder) + assert "Copying file global.conf" in c.out + assert "Repo cloned!" in c.out # git clone executed by scheduled task def test_config_fails_git_folder(self): # https://github.com/conan-io/conan/issues/8594 + config_folder = temp_folder(path_with_spaces=False) + save_files(config_folder, {"global.conf": "core.download:parallel=0"}) folder = os.path.join(temp_folder(), ".gitlab-conan", DEFAULT_CONAN_HOME) client = TestClient(cache_folder=folder) - with client.chdir(self.folder): + with client.chdir(config_folder): client.run_command('git init .') client.run_command('git add .') client.run_command('git config user.name myname') @@ -586,13 +557,13 @@ def test_config_fails_git_folder(self): client.run_command('git commit -m "mymsg"') assert ".gitlab-conan" in client.cache_folder assert os.path.basename(client.cache_folder) == DEFAULT_CONAN_HOME - client.run('config install "%s/.git" --type git' % self.folder) + client.run('config install "%s/.git" --type git' % config_folder) client.load_home("global.conf") # check it is there dirs = os.listdir(client.cache_folder) assert ".git" not in dirs -class TestConfigInstall: +class TestConfigInstall2: def test_config_install_reestructuring_source(self): """ https://github.com/conan-io/conan/issues/9885 """ @@ -622,67 +593,119 @@ def test_config_install_reestructuring_source(self): class TestConfigInstallPkg: - conanfile = textwrap.dedent(""" - from conan import ConanFile - from conan.tools.files import copy - class Conf(ConanFile): - name = "myconf" - version = "0.1" - package_type = "configuration" - def package(self): - copy(self, "*.conf", src=self.build_folder, dst=self.package_folder) - """) - @pytest.fixture() - def client(self): - c = TestClient(default_server_user=True) - c.save({"conanfile.py": self.conanfile, - "global.conf": "user.myteam:myconf=myvalue"}) - c.run("export-pkg .") + @pytest.fixture(scope="class") + def servers(self): + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.files import copy + class Conf(ConanFile): + package_type = "configuration" + def package(self): + copy(self, "*.conf", src=self.build_folder, dst=self.package_folder) + """) + c = TestClient(default_server_user=True, light=True) + c.save({"conanfile.py": conanfile}) + for pkg in ("myconf_a", "myconf_b", "myconf_c"): + for version in ("0.1", "0.2"): + c.save({"global.conf": f"user.myteam:myconf=my_{pkg}/{version}_value"}) + c.run(f"export-pkg . --name={pkg} --version={version}") c.run("upload * -r=default -c") c.run("remove * -c") - return c + return c.servers - def test_config_install_from_pkg(self, client): - # Now install it - c = client - c.run("config install-pkg myconf/[*]") - assert "myconf/0.1: Downloaded package revision" in c.out - assert "Copying file global.conf" in c.out - c.run("config show *") - assert "user.myteam:myconf: myvalue" in c.out - - # Just to make sure it doesn't crash in the update - c.run("config install-pkg myconf/[*]") - # Conan will not re-download fromthe server the same revision - assert "myconf/0.1: Downloaded package revision" not in c.out - # It doesn't re-install either - assert "Copying file global.conf" not in c.out - c.run("config show *") - assert "user.myteam:myconf: myvalue" in c.out - - # We can force the re-installation - c.run("config install-pkg myconf/[*] --force") - assert "Copying file global.conf" in c.out - c.run("config show *") - assert "user.myteam:myconf: myvalue" in c.out - - def test_update_flow(self, client): - # Now try the update flow - c = client - c2 = TestClient(servers=c.servers, inputs=["admin", "password"]) - c2.save({"conanfile.py": self.conanfile, - "global.conf": "user.myteam:myconf=othervalue"}) - c2.run("export-pkg .") - c2.run("upload * -r=default -c") - - c.run("config install-pkg myconf/[*]") - assert "myconf/0.1: Downloaded package revision" in c.out + @staticmethod + def _check_conf(c, pkg): c.run("config show *") - assert "user.myteam:myconf: othervalue" in c.out + assert f"user.myteam:myconf: my_{pkg}_value" in c.out - def test_cant_use_as_dependency(self): - c = TestClient() + @staticmethod + def _check_conf_file(c, refs): + content = json.loads(c.load_home("config_version.json"))["config_version"] + for a, b in zip(refs, content): + assert a in b + + def test_install_pkg(self, servers): + c = TestClient(servers=servers, light=True) + c.run("config install-pkg myconf_a/0.1") + assert "Installing new or updating configuration packages" in c.out + self._check_conf(c, "myconf_a/0.1") + self._check_conf_file(c, ["myconf_a/0.1"]) + cache_files = os.listdir(c.cache_folder) + assert "conaninfo.txt" not in cache_files + assert "conanmanifest.txt" not in cache_files + + # skip reinstall + c.run("config install-pkg myconf_a/0.1") + assert ("The requested configurations are identical to the already installed ones, " + "skipping re-installation") in c.out + self._check_conf(c, "myconf_a/0.1") + self._check_conf_file(c, ["myconf_a/0.1"]) + + # forced reinstall + c.save_home({"global.conf": ""}) + c.run("config install-pkg myconf_a/0.1 --force") + assert "forcing re-installation because --force" in c.out + self._check_conf(c, "myconf_a/0.1") + self._check_conf_file(c, ["myconf_a/0.1"]) + + def test_with_url(self, servers): + c = TestClient(servers=servers, light=True) + url = servers["default"].fake_url + c.run("remote remove default") + c.run(f"config install-pkg myconf_a/0.1 --url={url}") + assert "Installing new or updating configuration packages" in c.out + self._check_conf(c, "myconf_a/0.1") + self._check_conf_file(c, ["myconf_a/0.1"]) + + def test_update(self, servers): + c = TestClient(servers=servers, light=True) + c.run("config install-pkg myconf_a/0.1") + c.run("config install-pkg myconf_a/0.2") + assert "Installing new or updating configuration packages" in c.out + self._check_conf(c, "myconf_a/0.2") + self._check_conf_file(c, ["myconf_a/0.2"]) + + def test_addition(self, servers): + c = TestClient(servers=servers, light=True) + c.run("config install-pkg myconf_a/0.1") + c.run("config install-pkg myconf_b/0.1") + assert "Installing new or updating configuration packages" in c.out + self._check_conf(c, "myconf_b/0.1") + self._check_conf_file(c, ["myconf_a/0.1", "myconf_b/0.1"]) + + def test_update_first(self, servers): + c = TestClient(servers=servers, light=True) + c.run("config install-pkg myconf_a/0.1") + c.run("config install-pkg myconf_b/0.1") + + # update of the first fail + c.run("config install-pkg myconf_a/[*]", assert_error=True) + assert "ERROR: Installing these configuration packages will break" in c.out + assert "use 'conan config install-pkg --force' to force" in c.out + assert "Use 'conan config clean' first to fully reset your configuration" in c.out + self._check_conf(c, "myconf_b/0.1") + self._check_conf_file(c, ["myconf_a/0.1", "myconf_b/0.1"]) + + # Now force the first + c.run("config install-pkg myconf_a/[*] --force") + assert "Forcing the installation because --force was defined" in c.out + self._check_conf(c, "myconf_a/0.2") + self._check_conf_file(c, ["myconf_b/0.1", "myconf_a/0.2"]) + + def test_update_second(self, servers): + c = TestClient(servers=servers, light=True) + c.run("config install-pkg myconf_a/0.1") + c.run("config install-pkg myconf_b/0.1") + + # update of the second works without problem + c.run("config install-pkg myconf_b/[*]") + assert "Installing new or updating configuration packages" in c.out + self._check_conf(c, "myconf_b/0.2") + self._check_conf_file(c, ["myconf_a/0.1", "myconf_b/0.2"]) + + def test_error_cant_use_as_dependency(self): + c = TestClient(light=True) conanfile = GenConanfile("myconf", "0.1").with_package_type("configuration") c.save({"myconf/conanfile.py": conanfile, "pkg/conanfile.py": GenConanfile("pkg", "0.1").with_requires("myconf/0.1")}) @@ -691,31 +714,13 @@ def test_cant_use_as_dependency(self): assert "ERROR: Configuration package myconf/0.1 cannot be used as requirement, " \ "but pkg/0.1 is requiring it" in c.out - def test_cant_use_without_type(self): - c = TestClient() - conanfile = GenConanfile("myconf", "0.1") - c.save({"myconf/conanfile.py": conanfile}) + def test_error_cant_use_without_type(self): + c = TestClient(light=True) + c.save({"myconf/conanfile.py": GenConanfile("myconf", "0.1")}) c.run("create myconf") c.run("config install-pkg myconf/[*]", assert_error=True) assert 'ERROR: myconf/0.1 is not of package_type="configuration"' in c.out - def test_lockfile(self, client): - """ it should be able to install the config using a lockfile - """ - c = client - c.run("config install-pkg myconf/[*] --lockfile-out=config.lock") - - c2 = TestClient(servers=c.servers, inputs=["admin", "password"]) - # Make sure we bump the version, otherwise only a package revision will be created - c2.save({"conanfile.py": self.conanfile.replace("0.1", "0.2"), - "global.conf": "user.myteam:myconf=othervalue"}) - c2.run("export-pkg .") - c2.run("upload * -r=default -c") - - c.run("config install-pkg myconf/[*] --lockfile=config.lock") - c.run("config show *") - assert "user.myteam:myconf: myvalue" in c.out - def test_create_also(self): conanfile = textwrap.dedent(""" from conan import ConanFile @@ -729,35 +734,147 @@ def package(self): copy(self, "*.conf", src=self.build_folder, dst=self.package_folder) """) - c = TestClient(default_server_user=True) + c = TestClient(default_server_user=True, light=True) c.save({"conanfile.py": conanfile, - "global.conf": "user.myteam:myconf=myvalue"}) + "global.conf": "user.myteam:myconf=my_myconf/0.1_value"}) c.run("create .") c.run("upload * -r=default -c") c.run("remove * -c") c.run("config install-pkg myconf/[*]") - c.run("config show *") - assert "user.myteam:myconf: myvalue" in c.out - - def test_without_initial_remote(self): - c = TestClient(default_server_user=True) - c.save({"conanfile.py": self.conanfile, - "global.conf": "user.myteam:myconf=myvalue"}) - c.run("export-pkg .") - c.run("upload * -r=default -c") - c.run("remove * -c") - c.run("remote list --format=json") - list_remotes = json.loads(c.stdout) - remote_url = list_remotes[0]["url"] + self._check_conf(c, "myconf/0.1") + self._check_conf_file(c, ["myconf/0.1"]) - # This uses the same server and URL, because the TestClient+TestServer - # does not allow atm to test this, as it requires the remote to be defined - c.run(f"config install-pkg myconf/[*] --url={remote_url}") - assert "myconf/0.1: Downloaded package revision" in c.out - assert "Copying file global.conf" in c.out - c.run("config show *") - assert "user.myteam:myconf: myvalue" in c.out + def test_lockfile(self, servers): + """ it should be able to install the config older version using a lockfile + """ + c = TestClient(servers=servers, light=True) + c.run("config install-pkg myconf_a/0.1 --lockfile-out=config.lock") + c.run("config install-pkg myconf_a/[*] --lockfile=config.lock") + assert ("The requested configurations are identical to the already installed ones, " + "skipping re-installation") in c.out + self._check_conf(c, "myconf_a/0.1") + self._check_conf_file(c, ["myconf_a/0.1"]) + + # Without the lockfile, it is free to update + c.run("config install-pkg myconf_a/[*] --lockfile-out=config.lock") + assert "Installing new or updating configuration packages" + self._check_conf(c, "myconf_a/0.2") + self._check_conf_file(c, ["myconf_a/0.2"]) + result = json.loads(c.load("config.lock")) + assert "myconf_a/0.2" in result["config_requires"][0] + + def test_install_from_file(self, servers): + c = TestClient(servers=servers, light=True) + # To make explicit the format of the conanconfig file + conanconfig = textwrap.dedent("""\ + packages: + - myconf_a/0.1 + - myconf_b/0.1 + """) + c.save({"conanconfig.yml": conanconfig}) + # Admits the default location . + c.run("config install-pkg") + self._check_conf(c, "myconf_b/0.1") + self._check_conf_file(c, ["myconf_a/0.1", "myconf_b/0.1"]) + + # install same file again: + c.run("config install-pkg .") + assert ("The requested configurations are identical to the already installed ones, " + "skipping re-installation") in c.out + self._check_conf(c, "myconf_b/0.1") + self._check_conf_file(c, ["myconf_a/0.1", "myconf_b/0.1"]) + + # Forced re-install + c.save_home({"global.conf": ""}) + c.run("config install-pkg . --force") + assert "forcing re-installation because --force" in c.out + self._check_conf(c, "myconf_b/0.1") + self._check_conf_file(c, ["myconf_a/0.1", "myconf_b/0.1"]) + + def test_install_from_file_with_url(self, servers): + c = TestClient(servers=servers, light=True) + c.run("remote remove default") + server_url = servers["default"].fake_url + conanconfig = yaml.dump({"packages": ["myconf_a/0.1"], "urls": [server_url]}) + c.save({"conanconfig.yml": conanconfig}) + c.run("config install-pkg .") + self._check_conf(c, "myconf_a/0.1") + self._check_conf_file(c, ["myconf_a/0.1"]) + + def test_update_with_file(self, servers): + c = TestClient(servers=servers, light=True) + conanconfig = yaml.dump({"packages": ["myconf_a/0.1", "myconf_b/0.1"]}) + c.save({"conanconfig.yml": conanconfig}) + c.run("config install-pkg .") + + # Now installing "updates" without disrupting the order + c.run("config install-pkg myconf_c/0.1") + assert "Installing new or updating configuration packages" + self._check_conf(c, "myconf_c/0.1") + self._check_conf_file(c, ["myconf_a/0.1", "myconf_b/0.1", "myconf_c/0.1"]) + + # Now installing "updates" without disrupting the order + conanconfig = textwrap.dedent("""\ + packages: + - myconf_a/0.1 + - myconf_b/0.1 + - myconf_c/[*] + """) + c.save({"conanconfig.yml": conanconfig}) + c.run("config install-pkg .") + assert "Installing new or updating configuration packages" in c.out + self._check_conf(c, "myconf_c/0.2") + self._check_conf_file(c, ["myconf_a/0.1", "myconf_b/0.1", "myconf_c/0.2"]) + + # This is also an update of the existing ones, keeping the order + conanconfig = textwrap.dedent("""\ + packages: + - myconf_a/[*] + - myconf_b/0.1 + - myconf_c/[*] + """) + c.save({"conanconfig.yml": conanconfig}) + c.run("config install-pkg .") + assert "Installing new or updating configuration packages" in c.out + self._check_conf(c, "myconf_c/0.2") + self._check_conf_file(c, ["myconf_a/0.2", "myconf_b/0.1", "myconf_c/0.2"]) + + def test_failed_update_force(self, servers): + c = TestClient(servers=servers, light=True) + conanconfig = yaml.dump({"packages": ["myconf_a/0.1", "myconf_b/0.1"]}) + c.save({"conanconfig.yml": conanconfig}) + c.run("config install-pkg .") + + # Now installing "updates" without disrupting the order + conanconfig = textwrap.dedent("""\ + packages: + - myconf_a/0.2 + """) + c.save({"conanconfig.yml": conanconfig}) + c.run("config install-pkg .", assert_error=True) + assert "use 'conan config install-pkg --force' to force" in c.out + assert "Use 'conan config clean' first to fully reset" in c.out + self._check_conf(c, "myconf_b/0.1") + self._check_conf_file(c, ["myconf_a/0.1", "myconf_b/0.1"]) + + c.run("config install-pkg . --force") + assert "Forcing the installation because --force was defined" in c.out + self._check_conf(c, "myconf_a/0.2") + self._check_conf_file(c, ["myconf_b/0.1", "myconf_a/0.2"]) + + def test_install_from_file_with_lockfile(self, servers): + # it should stay in version 0.1 + c = TestClient(servers=servers) + c.run("config install-pkg myconf_a/0.1 --lockfile-out=conan.lock") + conanconfig = yaml.dump({"packages": ["myconf_a/[>=0.1 <1.0]"]}) + c.save({"conanconfig.yml": conanconfig}) + c.run("config install-pkg .") + path = HomePaths(c.cache_folder).config_version_path + # First are the newest installed + configs = json.loads(load(path))["config_version"] + configs = [str(RecipeReference.loads(r)) for r in configs] + assert configs == ["myconf_a/0.1"] class TestConfigInstallPkgSettings: @@ -800,7 +917,7 @@ def test_config_install_from_pkg(self, client, default_profile): c.run("config show *") assert "user.myteam:myconf: mywinvalue" in c.out - c.run("config install-pkg myconf/[*] -s os=Linux") + c.run("config install-pkg myconf/[*] -s os=Linux --force") assert "myconf/0.1: Downloaded package revision" in c.out assert "Copying file global.conf" in c.out c.run("config show *") @@ -824,7 +941,7 @@ def test_config_install_from_pkg_profile(self, client): c.run("config show *") assert "user.myteam:myconf: mywinvalue" in c.out - c.run("config install-pkg myconf/[*] -pr=nix.profile") + c.run("config install-pkg myconf/[*] -pr=nix.profile --force") assert "myconf/0.1: Downloaded package revision" in c.out assert "Copying file global.conf" in c.out c.run("config show *") @@ -841,7 +958,7 @@ def test_config_install_from_pkg_profile_default(self, client): assert "user.myteam:myconf: mywinvalue" in c.out c.save_home({"profiles/default": "[settings]\nos=Linux"}) - c.run("config install-pkg myconf/[*]") + c.run("config install-pkg myconf/[*] --force") assert "myconf/0.1: Downloaded package revision" in c.out assert "Copying file global.conf" in c.out c.run("config show *") @@ -889,7 +1006,7 @@ def test_config_install_from_pkg(self, client, default_profile): c.run("config show *") assert "user.myteam:myconf: my1value" in c.out - c.run("config install-pkg myconf/[*] -o &:project=project2") + c.run("config install-pkg myconf/[*] -o &:project=project2 --force") assert "myconf/0.1: Downloaded package revision" in c.out assert "Copying file global.conf" in c.out c.run("config show *") @@ -914,7 +1031,7 @@ def test_config_install_from_pkg_profile(self, client): c.run("config show *") assert "user.myteam:myconf: my1value" in c.out - c.run("config install-pkg myconf/[*] -pr=nix.profile") + c.run("config install-pkg myconf/[*] -pr=nix.profile --force") assert "myconf/0.1: Downloaded package revision" in c.out assert "Copying file global.conf" in c.out c.run("config show *") @@ -931,7 +1048,7 @@ def test_config_install_from_pkg_profile_default(self, client): assert "user.myteam:myconf: my1value" in c.out c.save_home({"profiles/default": "[options]\n&:project=project2"}) - c.run("config install-pkg myconf/[*]") + c.run("config install-pkg myconf/[*] --force") assert "myconf/0.1: Downloaded package revision" in c.out assert "Copying file global.conf" in c.out c.run("config show *") diff --git a/test/functional/command/test_install_deploy.py b/test/functional/command/test_install_deploy.py index b01ab3a321c..37b4d5d2599 100644 --- a/test/functional/command/test_install_deploy.py +++ b/test/functional/command/test_install_deploy.py @@ -508,8 +508,9 @@ def package_info(self): c.run("install --requires=pkga/1.0 --requires=pkgb/1.0 --deployer=runtime_deploy " "--deployer-folder=myruntime -vvv") - expected = sorted(["pkga.so", "pkgb.so", "pkga.dll"]) - assert sorted(os.listdir(os.path.join(c.current_folder, "myruntime"))) == expected + assert sorted(os.listdir(os.path.join(c.current_folder, "myruntime"))) == sorted(["bin", "lib"]) + assert sorted(os.listdir(os.path.join(c.current_folder, "myruntime", "lib"))) == sorted(['pkga.so', 'pkgb.so']) + assert sorted(os.listdir(os.path.join(c.current_folder, "myruntime", "bin"))) == sorted(['pkga.dll']) @pytest.mark.parametrize("symlink, expected", @@ -554,6 +555,68 @@ def package(self): assert not os.path.islink(lib) +def test_runtime_deploy_subfolder(): + """ The deployer runtime_deploy should preserve subfolder structure when deploying shared libraries + """ + c = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.files import copy + import os + class Pkg(ConanFile): + package_type = "shared-library" + def package(self): + copy(self, "*.so*", src=self.build_folder, dst=self.package_folder, keep_path=True) + """) + c.save({"foo/conanfile.py": conanfile, + "foo/lib/libfoo.so": "", + "foo/lib/subfolder/libbar.so": "", + "foo/lib/subfolder/subsubfolder/libqux.so": "",}) + c.run("export-pkg foo/ --name=foo --version=0.1.0") + c.run(f"install --requires=foo/0.1.0 --deployer=runtime_deploy --deployer-folder=output") + + assert sorted(os.listdir(os.path.join(c.current_folder, "output"))) == ["libfoo.so", "subfolder"] + assert sorted(os.listdir(os.path.join(c.current_folder, "output", "subfolder"))) == ["libbar.so", "subsubfolder"] + assert sorted(os.listdir(os.path.join(c.current_folder, "output", "subfolder", "subsubfolder"))) == ["libqux.so"] + + +def test_runtime_deploy_subfolder_symlink(): + """ The deployer runtime_deploy should preserve subfolder structure when deploying shared + libraries with symlinks + """ + c = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.files import copy, chdir, mkdir + import os + class Pkg(ConanFile): + package_type = "shared-library" + def package(self): + copy(self, "*.so*", src=self.build_folder, dst=self.package_folder) + with chdir(self, os.path.join(self.package_folder, "lib")): + os.symlink(src="subfolder/libfoo.so.1.0", dst="libfoo.so.1") + os.symlink(src="libfoo.so.1", dst="libfoo.so") + """) + c.save({"foo/conanfile.py": conanfile, + "foo/lib/subfolder/libfoo.so.1.0": "",}) + c.run("export-pkg foo/ --name=foo --version=0.1.0") + c.run(f"install --requires=foo/0.1.0 --deployer=runtime_deploy --deployer-folder=output -c:a tools.deployer:symlinks=True") + + assert os.listdir(os.path.join(c.current_folder, "output", "subfolder")) == ["libfoo.so.1.0"] + # INFO: This test requires in Windows to have symlinks enabled, otherwise it will fail + if platform.system() != "Windows": + assert sorted(os.listdir(os.path.join(c.current_folder, "output"))) == ["libfoo.so", "libfoo.so.1" , "subfolder"] + link_so_0 = os.path.join(c.current_folder, "output", "libfoo.so.1") + link_so = os.path.join(c.current_folder, "output", "libfoo.so") + lib = os.path.join(c.current_folder, "output", "subfolder", "libfoo.so.1.0") + assert os.path.islink(link_so_0) + assert os.path.islink(link_so) + assert not os.path.isabs(os.readlink(link_so_0)) + assert not os.path.isabs(os.readlink(os.path.join(link_so))) + assert os.path.realpath(link_so) == os.path.realpath(link_so_0) + assert os.path.realpath(link_so_0) == os.path.realpath(lib) + + def test_deployer_errors(): c = TestClient() c.save({"conanfile.txt": "", diff --git a/test/functional/conftest.py b/test/functional/conftest.py index 3dd4c911b98..25e064e179f 100644 --- a/test/functional/conftest.py +++ b/test/functional/conftest.py @@ -240,7 +240,7 @@ def _matrix_c_interface_client(): FILE "${CMAKE_CURRENT_BINARY_DIR}/matrixConfig.cmake" ) install(EXPORT matrixConfig - DESTINATION "${CMAKE_INSTALL_PREFIX}/matrix/cmake" + DESTINATION "matrix/cmake" NAMESPACE matrix:: ) """) diff --git a/test/functional/layout/test_build_system_layout_helpers.py b/test/functional/layout/test_build_system_layout_helpers.py index 0eb287dc808..b06b1b8a5d7 100644 --- a/test/functional/layout/test_build_system_layout_helpers.py +++ b/test/functional/layout/test_build_system_layout_helpers.py @@ -15,7 +15,7 @@ def conanfile(): .with_import("import os") .with_setting("build_type").with_setting("arch") .with_import("from conan.tools.microsoft import vs_layout") - .with_import("from conan.tools.files import AutoPackager, save")) + .with_import("from conan.tools.files import save, copy")) conanfile += """ def source(self): @@ -28,7 +28,8 @@ def layout(self): vs_layout(self) def package(self): - AutoPackager(self).run() + copy(self, "*", self.source_folder, self.package_folder) + copy(self, "*", self.build_folder, os.path.join(self.package_folder, "lib"), keep_path=False) """ return conanfile diff --git a/test/functional/layout/test_layout_autopackage.py b/test/functional/layout/test_layout_autopackage.py deleted file mode 100644 index 43b7436e3a4..00000000000 --- a/test/functional/layout/test_layout_autopackage.py +++ /dev/null @@ -1,393 +0,0 @@ -import os - -from conan.api.model import PkgReference -from conan.api.model import RecipeReference -from conan.test.assets.genconanfile import GenConanfile -from conan.test.utils.tools import TestClient - - -def get_latest_package_reference(cache, ref, pkgid): - latest_rrev = cache.get_latest_recipe_reference(ref) - pref = PkgReference(latest_rrev, pkgid) - prefs = cache.get_package_revisions_references(pref, only_latest_prev=True) - return prefs[0] - - -def test_auto_package_no_components(): - client = TestClient() - conan_file = str(GenConanfile().with_settings("build_type") - .with_import("from conan.tools.files import AutoPackager, save")) - conan_file += """ - - def source(self): - save(self, "source_sources/source_stuff.cpp", "") - save(self, "source_includes/include1.hpp", "") - save(self, "source_includes/include2.hpp", "") - save(self, "source_includes2/include3.h", "") - save(self, "source_libs/slibone.a", "") - save(self, "source_libs/slibtwo.a", "") - save(self, "source_libs/bin_to_discard.exe", "") - save(self, "source_bins/source_bin.exe", "") - save(self, "source_frameworks/sframe1/include/include.h", "") - save(self, "source_frameworks/sframe2/include/include.h", "") - save(self, "source_frameworks/sframe1/lib/libframework.lib", "") - save(self, "source_frameworks/sframe2/lib/libframework.lib", "") - save(self, "source_frameworks/sframe2/foo/bar.txt", "") - - def build(self): - save(self, "build_sources/build_stuff.cpp", "") - save(self, "build_sources/subdir/othersubdir/selective_stuff.cpp", "") - save(self, "build_includes/include3.h", "") - save(self, "build_includes/include4.hpp", "") - save(self, "build_libs/blibone.a", "") - save(self, "build_libs/blibtwo.a", "") - save(self, "build_bins/build_bin.exe", "") - save(self, "build_frameworks/bframe1/include/include.h", "") - save(self, "build_frameworks/bframe2/include/include.h", "") - - def layout(self): - self.folders.source = "my_source" - self.folders.build = "my_build" - self.folders.generators = "my_build/generators" - - # Package locations - self.cpp.package.includedirs = ["my_includes"] - self.cpp.package.srcdirs = ["my_sources"] - self.cpp.package.bindirs = ["my_bins"] - self.cpp.package.libdirs = ["my_libs"] - self.cpp.package.frameworkdirs = ["my_frameworks"] - - # Source CPP INFO - self.cpp.source.srcdirs = ["source_sources"] - self.cpp.source.includedirs = ["source_includes", "source_includes2"] - self.cpp.source.libdirs = ["source_libs"] - self.cpp.source.bindirs = ["source_bins"] - self.cpp.source.frameworkdirs = ["source_frameworks"] - - # Build CPP INFO - self.cpp.build.srcdirs = ["build_sources", - "build_sources/subdir/othersubdir"] - self.cpp.build.includedirs = ["build_includes"] - self.cpp.build.libdirs = ["build_libs"] - self.cpp.build.bindirs = ["build_bins"] - self.cpp.build.frameworkdirs = ["build_frameworks"] - - def package(self): - # Discard include3.h from source - packager = AutoPackager(self) - packager.patterns.build.include = ["*.hpp", "*.h", "include3.h"] - packager.patterns.build.lib = ["*.a"] - packager.patterns.build.bin = ["*.exe"] - packager.patterns.build.src = ["*.cpp"] - packager.patterns.build.framework = ["sframe*", "bframe*"] - packager.patterns.source.include = ["*.hpp"] # Discard include3.h from source - packager.patterns.source.lib = ["*.a"] - packager.patterns.source.bin = ["*.exe"] - packager.patterns.source.src = ["*.cpp"] - packager.patterns.source.framework = ["sframe*"] - packager.run() - """ - client.save({"conanfile.py": conan_file}) - client.run("create . --name=lib --version=1.0") - assert "AutoPackager is **** deprecated ****" in client.out - package_id = client.created_package_id("lib/1.0") - - ref = RecipeReference.loads("lib/1.0@") - prev = get_latest_package_reference(client.cache, ref, package_id) - - p_folder = client.cache.pkg_layout(prev).package() - - def p_path(path): - return os.path.join(p_folder, path) - - # Check directories that shouldn't exist - assert not os.path.exists(p_path("include")) - assert not os.path.exists(p_path("bin")) - assert not os.path.exists(p_path("lib")) - - # Check files source - assert os.path.exists(p_path("my_sources")) - assert os.path.exists(p_path("my_sources/source_stuff.cpp")) - - assert os.path.exists(p_path("my_includes/include1.hpp")) - assert os.path.exists(p_path("my_includes/include2.hpp")) - assert not os.path.exists(p_path("my_includes/include.h")) - - assert os.path.exists(p_path("my_libs/slibone.a")) - assert os.path.exists(p_path("my_libs/slibtwo.a")) - - assert os.path.exists(p_path("my_bins/source_bin.exe")) - assert not os.path.exists(p_path("my_bins/bin_to_discard.exe")) - - assert os.path.exists(p_path("my_frameworks/sframe1/include/include.h")) - assert os.path.exists(p_path("my_frameworks/sframe2/include/include.h")) - assert os.path.exists(p_path("my_frameworks/sframe1/lib/libframework.lib")) - assert os.path.exists(p_path("my_frameworks/sframe2/lib/libframework.lib")) - assert os.path.exists(p_path("my_frameworks/sframe2/foo/bar.txt")) - - # Check files build - assert os.path.exists(p_path("my_sources/build_stuff.cpp")) - # note: as the "selective_stuff.cpp" is included in another my_sources dir, is copied twice - # it would need manual adjustement of the copy to avoid it - assert os.path.exists(p_path("my_sources/selective_stuff.cpp")) - assert os.path.exists(p_path("my_sources/subdir/othersubdir/selective_stuff.cpp")) - - assert os.path.exists(p_path("my_includes/include3.h")) - assert not os.path.exists(p_path("my_includes/include4.h")) - - assert os.path.exists(p_path("my_libs/blibone.a")) - assert os.path.exists(p_path("my_libs/blibtwo.a")) - - assert os.path.exists(p_path("my_bins/build_bin.exe")) - - assert os.path.exists(p_path("my_frameworks/sframe1/include/include.h")) - assert os.path.exists(p_path("my_frameworks/sframe2/include/include.h")) - - -def test_auto_package_with_components(): - """The files from the components are not mixed in package if they belong to different dirs""" - client = TestClient() - conan_file = str(GenConanfile() - .with_settings("build_type") - .with_import("from conan.tools.files import AutoPackager, save")) - conan_file += """ - - def source(self): - save(self, "includes1/component1.hpp", "") - save(self, "includes2/component2.hpp", "") - - def build(self): - save(self, "build_libs/component1.a", "") - save(self, "build_libs/component2.a", "") - save(self, "build_bins/component3.exe", "") - - def layout(self): - # Build and source infos - self.cpp.source.components["component1"].includedirs = ["includes1"] - self.cpp.source.components["component2"].includedirs = ["includes2"] - self.cpp.build.components["component1"].libdirs = ["build_libs"] - self.cpp.build.components["component2"].libdirs = ["build_libs"] - self.cpp.build.components["component3"].bindirs = ["build_bins"] - - # Package infos - self.cpp.package.components["component1"].includedirs = ["include"] - self.cpp.package.components["component2"].includedirs = ["include2"] - self.cpp.package.components["component1"].libdirs = ["lib"] - self.cpp.package.components["component2"].libdirs = ["lib"] - self.cpp.package.components["component3"].bindirs = ["bin"] - - def package(self): - packager = AutoPackager(self) - packager.run() - """ - client.save({"conanfile.py": conan_file}) - client.run("create . --name=lib --version=1.0") - package_id = client.created_package_id("lib/1.0") - - ref = RecipeReference.loads("lib/1.0@") - pref = get_latest_package_reference(client.cache, ref, package_id) - p_folder = client.cache.pkg_layout(pref).package() - - def p_path(path): - return os.path.join(p_folder, path) - - # Check directories that shouldn't exist - assert not os.path.exists(p_path("includes1")) - assert not os.path.exists(p_path("includes2")) - assert not os.path.exists(p_path("build_libs")) - - # Check includes - assert os.path.exists(p_path("include")) - assert os.path.exists(p_path("include/component1.hpp")) - assert not os.path.exists(p_path("include/component2.hpp")) - assert os.path.exists(p_path("include2/component2.hpp")) - assert not os.path.exists(p_path("include2/component1.hpp")) - - # Check libs - assert os.path.exists(p_path("lib/component1.a")) - assert os.path.exists(p_path("lib/component2.a")) - - # Check app - assert os.path.exists(p_path("bin/component3.exe")) - - -def test_auto_package_with_components_declared_badly(): - client = TestClient() - conan_file = str(GenConanfile().with_settings("build_type") - .with_import("from conan.tools.files import AutoPackager")) - conan_file += """ - - def layout(self): - # Build and source infos - self.cpp.source.components["component1"].includedirs = ["includes1"] - self.cpp.source.components["component2"].includedirs = ["includes2"] - self.cpp.build.components["component1"].libdirs = ["build_libs"] - self.cpp.build.components["component2"].libdirs = ["build_libs"] - self.cpp.build.components["component3"].bindirs = ["build_bins"] - - # Package infos BUT NOT DECLARING component2 - self.cpp.package.components["component1"].includedirs = ["include"] - self.cpp.package.components["component1"].libdirs = ["lib"] - self.cpp.package.components["component3"].bindirs = ["bin"] - - def package(self): - AutoPackager(self).run() - """ - - client.save({"conanfile.py": conan_file}) - client.run("create . --name=lib --version=1.0", assert_error=True) - assert "There are components declared in cpp.source.components or in " \ - "cpp.build.components that are not declared in " \ - "cpp.package.components" in client.out - - -def test_auto_package_default_patterns(): - """By default: - self.patterns.source.include = ["*.h", "*.hpp", "*.hxx"] - self.patterns.build.lib = ["*.so", "*.so.*", "*.a", "*.lib", "*.dylib"] - self.patterns.build.bin = ["*.exe", "*.dll"] - """ - client = TestClient() - conan_file = str(GenConanfile().with_settings("build_type") - .with_import("import os") - .with_import("from conan.tools.files import AutoPackager, save")) - conan_file += """ - def source(self): - save(self, "myincludes/mylib.header","") - save(self, "myincludes/mylib.h","") - save(self, "myincludes/mylib.hpp","") - save(self, "myincludes/mylib.hxx","") - - def build(self): - save(self, "ugly_build/mylib.a", "") - save(self, "ugly_build/mylib.so", "") - save(self, "ugly_build/mylib.so.0", "") - save(self, "ugly_build/mylib.lib", "") - save(self, "ugly_build/mylib.dylib", "") - save(self, "ugly_build/app.exe", "") - save(self, "ugly_build/app.dll", "") - save(self, "ugly_build/mylib.janderclander", "") - - def layout(self): - self.cpp.source.includedirs = ["myincludes"] - self.cpp.build.libdirs = ["ugly_build"] - self.cpp.build.bindirs = ["ugly_build"] - - def package(self): - AutoPackager(self).run() - """ - client.save({"conanfile.py": conan_file}) - client.run("create . --name=lib --version=1.0") - package_id = client.created_package_id("lib/1.0") - ref = RecipeReference.loads("lib/1.0@") - pref = get_latest_package_reference(client.cache, ref, package_id) - p_folder = client.cache.pkg_layout(pref).package() - - assert set(os.listdir(os.path.join(p_folder, "lib"))) == {"mylib.a", "mylib.so", "mylib.so.0", - "mylib.dylib", "mylib.lib"} - assert set(os.listdir(os.path.join(p_folder, "include"))) == {"mylib.h", "mylib.hpp", - "mylib.hxx"} - assert set(os.listdir(os.path.join(p_folder, "bin"))) == {"app.exe", "app.dll"} - - -def test_auto_package_default_folders_with_components(): - """By default, the cpp_info of the components are empty""" - client = TestClient() - conan_file = str(GenConanfile().with_settings("build_type") - .with_import("import os") - .with_import("from conan.tools.files import AutoPackager")) - conan_file += """ - def layout(self): - for el in [self.cpp.source, self.cpp.build]: - assert el.components["foo"].includedirs == [] - assert el.components["foo"].libdirs == [] - assert el.components["foo"].bindirs == [] - assert el.components["foo"].frameworkdirs == [] - assert el.components["foo"].srcdirs == [] - assert el.components["foo"].resdirs == [] - - assert self.cpp.package.components["foo"].includedirs == ["include"] - assert self.cpp.package.components["foo"].libdirs == ["lib"] - assert self.cpp.package.components["foo"].bindirs == ["bin"] - assert self.cpp.package.components["foo"].frameworkdirs == [] - assert self.cpp.package.components["foo"].srcdirs == [] - assert self.cpp.package.components["foo"].resdirs == [] - - def package(self): - AutoPackager(self).run() - """ - client.save({"conanfile.py": conan_file}) - client.run("create . --name=lib --version=1.0") - - -def test_auto_package_with_custom_package_too(): - """We can also declare the package() method and call explicitly to the - self.folders.package_files()""" - client = TestClient() - conan_file = str(GenConanfile().with_settings("build_type") - .with_import("import os") - .with_import("from conan.tools.files import AutoPackager, save")) - conan_file += """ - def source(self): - save(self, "myincludes/mylib.header","") - - def build(self): - save(self, "ugly_build/mylib.a", "") - - def layout(self): - self.cpp.source.includedirs = ["myincludes"] - self.cpp.build.libdirs = ["ugly_build"] - - def package(self): - packager = AutoPackager(self) - packager.patterns.source.include = ["*.header"] - packager.patterns.build.lib = ["*.a"] - packager.run() - assert os.path.exists(os.path.join(self.package_folder, "include", "mylib.header")) - assert os.path.exists(os.path.join(self.package_folder, "lib", "mylib.a")) - self.output.warning("Package method called!") - - """ - client.save({"conanfile.py": conan_file}) - client.run("create . --name=lib --version=1.0") - assert "Package method called!" in client.out - - -def test_auto_package_only_one_destination(): - """If the layout declares more than one destination folder it fails, because it cannot guess - where to put the artifacts (very weird situation a package with two include/)""" - client = TestClient() - conan_file = str(GenConanfile().with_settings("build_type") - .with_import("import os") - .with_import("from conan.tools.files import AutoPackager, save")) - conan_file += """ - def source(self): - save(self, "myincludes/mylib.header","") - - def build(self): - save(self, "ugly_build/mylib.a", "") - - def layout(self): - self.cpp.source.includedirs = ["myincludes"] - self.cpp.build.libdirs = ["ugly_build"] - self.cpp.build.bindirs = ["ugly_build"] - self.cpp.build.frameworkdirs = ["ugly_build"] - self.cpp.build.srcdirs = ["ugly_build"] - self.cpp.build.builddirs = ["ugly_build"] - self.cpp.build.resdirs = ["ugly_build"] - - self.cpp.package.{} = ["folder1", "folder2"] - - def package(self): - packager = AutoPackager(self) - packager.patterns.source.include = ["*.header"] - packager.patterns.build.lib = ["*.a"] - packager.run() - - """ - for dirs in ["includedirs", "builddirs", "bindirs", "srcdirs", "frameworkdirs", "libdirs", - "resdirs"]: - client.save({"conanfile.py": conan_file.format(dirs)}) - client.run("create . --name=lib --version=1.0", assert_error=True) - assert "The package has more than 1 cpp_info.{}, " \ - "cannot package automatically".format(dirs) in client.out diff --git a/test/functional/layout/test_source_folder.py b/test/functional/layout/test_source_folder.py index 59c237106d5..dbca7744bce 100644 --- a/test/functional/layout/test_source_folder.py +++ b/test/functional/layout/test_source_folder.py @@ -2,7 +2,7 @@ import platform from shutil import copy -import mock +from unittest import mock import pytest from conan.test.assets.cmake import gen_cmakelists diff --git a/test/functional/only_source_test.py b/test/functional/only_source_test.py deleted file mode 100644 index cd6ccc811f7..00000000000 --- a/test/functional/only_source_test.py +++ /dev/null @@ -1,151 +0,0 @@ -import os -import unittest - -from conan.api.model import RecipeReference -from conan.test.assets.genconanfile import GenConanfile -from conan.test.utils.tools import TestClient - - -class OnlySourceTest(unittest.TestCase): - - def test_conan_test(self): - # Checks --build in test command - client = TestClient() - client.save({"conanfile.py": GenConanfile("hello0", "0.0")}) - client.run("export . --user=lasote --channel=stable") - client.save({"conanfile.py": GenConanfile("hello1", "1.1"). - with_require("hello0/0.0@lasote/stable")}) - client.run("export . --user=lasote --channel=stable") - - # Now test out Hello2 - client.save({"conanfile.py": GenConanfile("hello2", "2.2"). - with_require("hello1/1.1@lasote/stable"), - "test/conanfile.py": GenConanfile().with_test("pass")}) - - # Should recognize the hello package - # Will Fail because hello0/0.0 and hello1/1.1 has not built packages - # and by default no packages are built - client.run("create . --user=lasote --channel=stable", assert_error=True) - self.assertIn("Try to build locally from sources using the '--build=hello0/0.0@lasote/stable " - "--build=hello1/1.1@lasote/stable'", - client.out) - # Only 1 reference! - assert "List all available packages using 'conan list \"hello0/0.0@lasote/stable:*\" -r=remote'" in client.out - - # We generate the package for hello0/0.0 - client.run("install --requires=hello0/0.0@lasote/stable --build hello0*") - - # Still missing hello1/1.1 - client.run("create . --user=lasote --channel=stable", assert_error=True) - self.assertIn("Try to build locally from sources using the " - "'--build=hello1/1.1@lasote/stable'", client.out) - - # We generate the package for hello1/1.1 - client.run("install --requires=hello1/1.1@lasote/stable --build hello1*") - - # Now Hello2 should be built and not fail - client.run("create . --user=lasote --channel=stable") - self.assertNotIn("Can't find a 'hello2/2.2@lasote/stable' package", client.out) - self.assertIn('hello2/2.2@lasote/stable: Forced build from source', client.out) - - # Now package is generated but should be built again - client.run("create . --user=lasote --channel=stable") - self.assertIn('hello2/2.2@lasote/stable: Forced build from source', client.out) - - def test_build_policies_in_conanfile(self): - client = TestClient(default_server_user=True) - base = GenConanfile("hello0", "1.0").with_exports("*") - conanfile = str(base) + "\n build_policy = 'missing'" - client.save({"conanfile.py": conanfile}) - client.run("export . --user=lasote --channel=stable") - - # Install, it will build automatically if missing (without the --build missing option) - client.run("install --requires=hello0/1.0@lasote/stable") - self.assertIn("Building", client.out) - - # Try to do it again, now we have the package, so no build is done - client.run("install --requires=hello0/1.0@lasote/stable") - self.assertNotIn("Building", client.out) - - # Try now to upload all packages, should not crash because of the "missing" build policy - client.run("upload hello0/1.0@lasote/stable -r default") - - # --- Build policy to always --- - conanfile = str(base) + "\n build_policy = 'always'" - client.save({"conanfile.py": conanfile}, clean_first=True) - client.run("export . --user=lasote --channel=stable") - - # Install, it will build automatically if missing (without the --build missing option) - client.run("install --requires=hello0/1.0@lasote/stable", assert_error=True) - self.assertIn("ERROR: hello0/1.0@lasote/stable: build_policy='always' has been removed", - client.out) - - def test_reuse(self): - client = TestClient(default_server_user=True) - ref = RecipeReference.loads("hello0/0.1@lasote/stable") - client.save({"conanfile.py": GenConanfile("hello0", "0.1")}) - client.run("export . --user=lasote --channel=stable") - client.run("install --requires=%s --build missing" % str(ref)) - pkg_layout = client.created_layout() - self.assertTrue(os.path.exists(pkg_layout.build())) - self.assertTrue(os.path.exists(pkg_layout.package())) - - # Upload - client.run("upload %s -r default" % str(ref)) - - # Now from other "computer" install the uploaded conans with same options (nothing) - other_client = TestClient(servers=client.servers) - other_client.run("install --requires=%s --build missing" % str(ref)) - pref = client.get_latest_package_reference(ref) - self.assertFalse(os.path.exists(other_client.get_latest_pkg_layout(pref).build())) - self.assertTrue(os.path.exists(other_client.get_latest_pkg_layout(pref).package())) - - # Now from other "computer" install the uploaded conans with same options (nothing) - other_client = TestClient(servers=client.servers) - other_client.run("install --requires=%s --build='*'" % str(ref)) - pkg_layout = other_client.created_layout() - self.assertTrue(os.path.exists(pkg_layout.build())) - self.assertTrue(os.path.exists(pkg_layout.package())) - - # Use an invalid pattern and check that its not builded from source - other_client = TestClient(servers=client.servers) - other_client.run("install --requires=%s --build HelloInvalid" % str(ref)) - - # pref = client.get_latest_package_reference(ref) - # self.assertIn("No package matching 'HelloInvalid' pattern", other_client.out) - # self.assertFalse(os.path.exists(other_client.get_latest_pkg_layout(pref).build())) - - # Use another valid pattern and check that its not builded from source - other_client = TestClient(servers=client.servers) - other_client.run("install --requires=%s --build HelloInvalid -b hello" % str(ref)) - # self.assertIn("No package matching 'HelloInvalid' pattern", other_client.out) - - # Now even if the package is in local store, check that's rebuilded - other_client.run("install --requires=%s -b hello*" % str(ref)) - self.assertIn("Copying sources to build folder", other_client.out) - - other_client.run("install --requires=%s" % str(ref)) - self.assertNotIn("Copying sources to build folder", other_client.out) - - -def test_build_policy_missing(): - c = TestClient(default_server_user=True) - conanfile = GenConanfile("pkg", "1.0").with_class_attribute('build_policy = "missing"')\ - .with_class_attribute('upload_policy = "skip"') - c.save({"conanfile.py": conanfile}) - c.run("export .") - - # the --build=never has higher priority - c.run("install --requires=pkg/1.0@ --build=never", assert_error=True) - assert "ERROR: Missing prebuilt package for 'pkg/1.0'" in c.out - - c.run("install --requires=pkg/1.0@") - assert "pkg/1.0: Building package from source as defined by build_policy='missing'" in c.out - - # If binary already there it should do nothing - c.run("install --requires=pkg/1.0@") - assert "pkg/1.0: Building package from source" not in c.out - - c.run("upload * -r=default -c") - assert "Uploading package" not in c.out - assert "pkg/1.0: Skipping upload of binaries, because upload_policy='skip'" in c.out diff --git a/test/functional/revisions_test.py b/test/functional/revisions_test.py index cb23887d905..8b699af35f6 100644 --- a/test/functional/revisions_test.py +++ b/test/functional/revisions_test.py @@ -1,10 +1,9 @@ import copy import time -import unittest from collections import OrderedDict import pytest -from mock import patch +from unittest.mock import patch from conan.test.utils.env import environment_update from conan.internal.errors import RecipeNotFoundException @@ -26,9 +25,10 @@ def _create(c_v2, ref, conanfile=None, args=None, assert_error=False): @pytest.mark.artifactory_ready -class InstallingPackagesWithRevisionsTest(unittest.TestCase): +class TestInstallingPackagesWithRevisions: - def setUp(self): + @pytest.fixture(autouse=True) + def setup(self): self.server = TestServer() self.server2 = TestServer() self.servers = OrderedDict([("default", self.server), @@ -39,13 +39,13 @@ def setUp(self): def recipe_revision(self, ref): tmp = copy.copy(ref) tmp.revision = None - latest_rrev = self.c_v2.cache.get_latest_recipe_reference(tmp) + latest_rrev = self.c_v2.cache.get_latest_recipe_revision(tmp) return latest_rrev.revision def package_revision(self, pref): tmp = copy.copy(pref) tmp.revision = None - latest_prev = self.c_v2.cache.get_latest_package_reference(tmp) + latest_prev = self.c_v2.cache.get_latest_package_revision(tmp) return latest_prev.revision def test_install_binary_iterating_remotes_same_rrev(self): @@ -68,12 +68,11 @@ def test_install_binary_iterating_remotes_same_rrev(self): self.c_v2.run(f"upload {self.ref} -r=remote2 -c") self.c_v2.remove_all() - self.assertEqual(pref.ref.revision, pref2.ref.revision) + assert pref.ref.revision == pref2.ref.revision self.c_v2.run("install --requires={}".format(self.ref)) self.c_v2.assert_listed_require({str(self.ref): "Downloaded (default)"}) - self.assertIn("Retrieving package {} from remote 'remote2'".format(pref.package_id), - self.c_v2.out) + assert "Retrieving package {} from remote 'remote2'".format(pref.package_id) in self.c_v2.out def test_diamond_revisions_conflict(self): """ If we have a diamond because of pinned conflicting revisions in the requirements, @@ -104,8 +103,8 @@ def test_diamond_revisions_conflict(self): _create(self.c_v2, project, conanfile=GenConanfile().with_requirement(lib2).with_requirement(lib3), assert_error=True) - self.assertIn("ERROR: Version conflict", self.c_v2.out) - # self.assertIn("Different revisions of {} has been requested".format(lib1), self.c_v2.out) + assert "ERROR: Version conflict" in self.c_v2.out + # assert "Different revisions of {} has been requested".format(lib1), self.c_v2.out) def test_alias_to_a_rrev(self): """ If an alias points to a RREV, it resolved that RREV and no other""" @@ -123,17 +122,17 @@ def test_alias_to_a_rrev(self): self.c_v2.alias("lib/latest@conan/stable", repr(pref.ref)) alias_ref = RecipeReference.loads("lib/latest@conan/stable") exported = load(self.c_v2.get_latest_ref_layout(alias_ref).conanfile()) - self.assertIn('alias = "{}"'.format(repr(pref.ref)), exported) + assert 'alias = "{}"'.format(repr(pref.ref)) in exported self.c_v2.run(f"upload lib/latest@conan/stable -r=default -c") self.c_v2.remove_all() self.c_v2.run("install --requires=lib/(latest)@conan/stable") # Shouldn't be packages in the cache - self.assertNotIn("doesn't belong to the installed recipe revision", self.c_v2.out) + assert "doesn't belong to the installed recipe revision" not in self.c_v2.out # Read current revision - self.assertEqual(pref.ref.revision, self.recipe_revision(self.ref)) + assert pref.ref.revision == self.recipe_revision(self.ref) def test_revision_metadata_update_on_install(self): """If a clean v2 client installs a RREV/PREV from a server, it get @@ -144,13 +143,13 @@ def test_revision_metadata_update_on_install(self): # Remove all from c_v2 local self.c_v2.remove_all() - assert len(self.c_v2.cache.get_recipe_revisions_references(self.ref)) == 0 + assert len(self.c_v2.cache.get_recipe_revisions(self.ref)) == 0 self.c_v2.run("install --requires={}".format(self.ref)) local_rev = self.recipe_revision(self.ref) local_prev = self.package_revision(pref) - self.assertEqual(local_rev, pref.ref.revision) - self.assertEqual(local_prev, pref.revision) + assert local_rev == pref.ref.revision + assert local_prev == pref.revision def test_revision_update_on_package_update(self): """ @@ -182,11 +181,11 @@ def test_revision_update_on_package_update(self): prev1_time_remote = self.server.package_revision_time(pref) prev2_time_remote = self.server.package_revision_time(pref2) - self.assertNotEqual(prev1_time_remote, prev2_time_remote) # Two package revisions + assert prev1_time_remote != prev2_time_remote # Two package revisions client.run("install --requires=pkg/0.1 --update") client.assert_listed_require({"pkg/0.1": "Cache (Updated date) (default)"}) - self.assertIn("Retrieving package {}".format(pref.package_id), client.out) + assert "Retrieving package {}".format(pref.package_id) in client.out def test_revision_mismatch_packages_in_local(self): """Test adapted for the new cache: we create a revision but we export again a recipe @@ -201,13 +200,13 @@ def test_revision_mismatch_packages_in_local(self): sot1.revision = None sot2 = copy.copy(ref2) sot2.revision = None - self.assertEqual(sot1, sot2) - self.assertNotEqual(pref.ref.revision, ref2.revision) + assert sot1 == sot2 + assert pref.ref.revision != ref2.revision # Now we try to install the self.ref, the binary is missing when using revisions command = "install --requires={}".format(self.ref) client.run(command, assert_error=True) - self.assertIn("ERROR: Missing prebuilt package for '{}'".format(self.ref), client.out) + assert "ERROR: Missing prebuilt package for '{}'".format(self.ref) in client.out def test_revision_install_explicit_mismatch_rrev(self): # If we have a recipe in local, but we request to install a different one with RREV @@ -216,10 +215,10 @@ def test_revision_install_explicit_mismatch_rrev(self): ref = client.export(self.ref) command = "install --requires={}#fakerevision".format(ref) client.run(command, assert_error=True) - self.assertIn("Unable to find '{}#fakerevision' in remotes".format(ref), client.out) + assert "Unable to find '{}#fakerevision' in remotes".format(ref) in client.out command = "install --requires={}#fakerevision --update".format(ref) client.run(command, assert_error=True) - self.assertIn("Unable to find '{}#fakerevision' in remotes".format(ref), client.out) + assert "Unable to find '{}#fakerevision' in remotes".format(ref) in client.out # Now create a new revision with other client and upload it, we will request it new_client = TestClient(servers=self.servers, inputs=["admin", "password"]) @@ -244,7 +243,7 @@ def test_revision_mismatch_packages_remote(self): command = "install --requires={}".format(self.ref) client.run(command, assert_error=True) - self.assertIn("Can't find a '{}' package".format(self.ref), client.out) + assert "Can't find a '{}' package".format(self.ref) in client.out def test_revision_build_requires(self): conanfile = GenConanfile() @@ -263,12 +262,13 @@ def test_revision_build_requires(self): for ref in refs: command = "install --update --tool-require={}".format(repr(ref)) client.run(command) - self.assertIn("Downloaded recipe revision {}".format(ref.revision), client.out) + assert "Downloaded recipe revision {}".format(ref.revision) in client.out -class RemoveWithRevisionsTest(unittest.TestCase): +class TestRemoveWithRevisions: - def setUp(self): + @pytest.fixture(autouse=True) + def setup(self): self.server = TestServer() self.c_v2 = TestClient(servers={"default": self.server}, inputs=["admin", "password"]) self.ref = RecipeReference.loads("lib/1.0@conan/testing") @@ -281,7 +281,7 @@ def test_remove_local_recipe(self): ref1 = client.export(self.ref) ref1.revision = None client.run("remove {} -c".format(repr(ref1))) - self.assertFalse(client.recipe_exists(self.ref)) + assert not client.recipe_exists(self.ref) # If I remove a ref with a wrong revision, the revision is not removed ref1 = client.export(self.ref) @@ -289,8 +289,8 @@ def test_remove_local_recipe(self): fakeref.revision = "fakerev" full_ref = repr(fakeref) client.run("remove {} -c".format(repr(fakeref)), assert_error=True) - self.assertIn(f"ERROR: Recipe revision '{full_ref}' not found", client.out) - self.assertTrue(client.recipe_exists(self.ref)) + assert f"ERROR: Recipe revision '{full_ref}' not found" in client.out + assert client.recipe_exists(self.ref) def test_remove_local_package(self): """Locally: @@ -305,7 +305,7 @@ def test_remove_local_package(self): tmp = copy.copy(pref1.ref) tmp.revision = None client.run("remove {} -c".format(repr(tmp))) - self.assertFalse(client.package_exists(pref1)) + assert not client.package_exists(pref1) # If I remove the ref with fake RREV, the packages are not removed pref1 = _create(self.c_v2, self.ref) @@ -313,13 +313,13 @@ def test_remove_local_package(self): fakeref.revision = "fakerev" str_ref = repr(fakeref) client.run("remove {} -c".format(repr(fakeref)), assert_error=True) - self.assertTrue(client.package_exists(pref1)) - self.assertIn("Recipe revision '{}' not found".format(str_ref), client.out) + assert client.package_exists(pref1) + assert "Recipe revision '{}' not found".format(str_ref) in client.out # If I remove the ref with valid RREV, the packages are removed pref1 = _create(self.c_v2, self.ref) client.run("remove {} -c".format(repr(pref1.ref))) - self.assertFalse(client.package_exists(pref1)) + assert not client.package_exists(pref1) # If I remove the ref without RREV but specifying PREV it raises pref1 = _create(self.c_v2, self.ref) @@ -327,20 +327,20 @@ def test_remove_local_package(self): tmp.revision = None command = "remove {}:{}#{} -c".format(repr(tmp), pref1.package_id, pref1.revision) client.run(command) - self.assertFalse(client.package_exists(pref1)) + assert not client.package_exists(pref1) # A wrong PREV doesn't remove the PREV pref1 = _create(self.c_v2, self.ref) command = "remove {}:{}#fakeprev -c".format(repr(pref1.ref), pref1.package_id) client.run(command, assert_error=True) - self.assertTrue(client.package_exists(pref1)) - self.assertIn("ERROR: Package revision", client.out) + assert client.package_exists(pref1) + assert "ERROR: Package revision" in client.out # Everything correct, removes the unique local package revision pref1 = _create(self.c_v2, self.ref) command = "remove {}:{}#{} -c".format(repr(pref1.ref), pref1.package_id, pref1.revision) client.run(command) - self.assertFalse(client.package_exists(pref1)) + assert not client.package_exists(pref1) def test_remove_remote_recipe(self): """When a client removes a reference, it removes ALL revisions, no matter @@ -351,17 +351,17 @@ def test_remove_remote_recipe(self): pref2 = _create(self.c_v2, self.ref, conanfile=GenConanfile().with_build_msg("RREV 2!")) self.c_v2.run(f"upload {pref2.ref} -r=default -c") - self.assertNotEqual(pref1, pref2) + assert pref1 != pref2 remover_client = self.c_v2 # Remove ref without revision in a remote remover_client.run("remove {} -c -r default".format(self.ref)) - self.assertFalse(self.server.recipe_exists(self.ref)) - self.assertFalse(self.server.recipe_exists(pref1.ref)) - self.assertFalse(self.server.recipe_exists(pref2.ref)) - self.assertFalse(self.server.package_exists(pref1)) - self.assertFalse(self.server.package_exists(pref2)) + assert not self.server.recipe_exists(self.ref) + assert not self.server.recipe_exists(pref1.ref) + assert not self.server.recipe_exists(pref2.ref) + assert not self.server.package_exists(pref1) + assert not self.server.package_exists(pref2) def test_remove_remote_recipe_revision(self): """If a client removes a recipe with revision: @@ -372,15 +372,15 @@ def test_remove_remote_recipe_revision(self): pref2 = _create(self.c_v2, self.ref, conanfile=GenConanfile().with_build_msg("RREV 2!")) self.c_v2.run(f"upload {pref2.ref} -r=default -c") - self.assertNotEqual(pref1, pref2) + assert pref1 != pref2 remover_client = self.c_v2 # Remove ref without revision in a remote command = "remove {} -c -r default".format(repr(pref1.ref)) remover_client.run(command) - self.assertFalse(self.server.recipe_exists(pref1.ref)) - self.assertTrue(self.server.recipe_exists(pref2.ref)) + assert not self.server.recipe_exists(pref1.ref) + assert self.server.recipe_exists(pref2.ref) def test_remove_remote_package(self): """When a client removes a package, without RREV, it removes the package from ALL @@ -391,19 +391,19 @@ def test_remove_remote_package(self): pref2 = _create(self.c_v2, self.ref, conanfile=GenConanfile().with_build_msg("RREV 2!")) self.c_v2.run(f"upload {pref2.ref} -r=default -c") - self.assertEqual(pref1.package_id, pref2.package_id) + assert pref1.package_id == pref2.package_id # Both revisions exist in 2.0 cache - self.assertTrue(self.c_v2.package_exists(pref1)) - self.assertTrue(self.c_v2.package_exists(pref2)) + assert self.c_v2.package_exists(pref1) + assert self.c_v2.package_exists(pref2) remover_client = self.c_v2 # Remove pref without RREV in a remote remover_client.run("remove {}#*:{} -c -r default".format(self.ref, pref2.package_id)) - self.assertTrue(self.server.recipe_exists(pref1.ref)) - self.assertTrue(self.server.recipe_exists(pref2.ref)) - self.assertFalse(self.server.package_exists(pref1)) - self.assertFalse(self.server.package_exists(pref2)) + assert self.server.recipe_exists(pref1.ref) + assert self.server.recipe_exists(pref2.ref) + assert not self.server.package_exists(pref1) + assert not self.server.package_exists(pref2) def test_remove_remote_package_revision(self): """When a client removes a package with PREV @@ -429,10 +429,10 @@ def test_remove_remote_package_revision(self): self.c_v2.run(f"upload {pref2b.ref} -r=default -c") # Check created revisions - self.assertEqual(pref1.package_id, pref2.package_id) - self.assertEqual(pref2.package_id, pref2b.package_id) - self.assertEqual(pref2.ref.revision, pref2b.ref.revision) - self.assertNotEqual(pref2.revision, pref2b.revision) + assert pref1.package_id == pref2.package_id + assert pref2.package_id == pref2b.package_id + assert pref2.ref.revision == pref2b.ref.revision + assert pref2.revision != pref2b.revision remover_client = self.c_v2 @@ -440,25 +440,25 @@ def test_remove_remote_package_revision(self): command = "remove {}:{}#{} -c -r default".format(self.ref, pref2.package_id, pref2.revision) remover_client.run(command) - self.assertTrue(self.server.recipe_exists(pref1.ref)) - self.assertTrue(self.server.recipe_exists(pref2.ref)) - self.assertTrue(self.server.recipe_exists(pref2b.ref)) - self.assertTrue(self.server.package_exists(pref1)) - self.assertTrue(self.server.package_exists(pref2b)) - self.assertFalse(self.server.package_exists(pref2)) + assert self.server.recipe_exists(pref1.ref) + assert self.server.recipe_exists(pref2.ref) + assert self.server.recipe_exists(pref2b.ref) + assert self.server.package_exists(pref1) + assert self.server.package_exists(pref2b) + assert not self.server.package_exists(pref2) # Try to remove a missing revision command = "remove {}:{}#fakerev -c -r default".format(repr(pref2.ref), pref2.package_id) remover_client.run(command, assert_error=True) fakeref = copy.copy(pref2) fakeref.revision = "fakerev" - self.assertIn(f"ERROR: Package revision '{fakeref.repr_notime()}' not found", - remover_client.out) + assert f"ERROR: Package revision '{fakeref.repr_notime()}' not found" in remover_client.out -class UploadPackagesWithRevisions(unittest.TestCase): +class TestUploadPackagesWithRevisions: - def setUp(self): + @pytest.fixture(autouse=True) + def setup(self): self.server = TestServer() self.c_v2 = TestClient(servers={"default": self.server}, inputs=["admin", "password"]) self.ref = RecipeReference.loads("lib/1.0@conan/testing") @@ -472,7 +472,7 @@ def test_upload_a_recipe(self): client.run(f"upload {self.ref} -r=default -c") revs = [r.revision for r in self.server.server_store.get_recipe_revisions_references(self.ref)] - self.assertEqual(revs, [pref.ref.revision]) + assert revs == [pref.ref.revision] def test_upload_no_overwrite_recipes(self): """If we upload a RREV to the server and create a new RREV in the client, @@ -488,9 +488,9 @@ def test_upload_no_overwrite_recipes(self): conanfile=GenConanfile().with_setting("os").with_build_msg("rev2"), args=" -s os=Linux") - self.assertEqual(self.server.server_store.get_last_revision(self.ref)[0], pref.ref.revision) + assert self.server.server_store.get_last_revision(self.ref)[0] == pref.ref.revision client.run(f"upload {self.ref} -r=default -c") - self.assertEqual(self.server.server_store.get_last_revision(self.ref)[0], pref2.ref.revision) + assert self.server.server_store.get_last_revision(self.ref)[0] == pref2.ref.revision def test_upload_no_overwrite_packages(self): """If we upload a PREV to the server and create a new PREV in the client, @@ -506,13 +506,11 @@ def test_upload_no_overwrite_packages(self): with environment_update({"MY_VAR": "2"}): pref2 = _create(self.c_v2, self.ref, conanfile=conanfile) - self.assertNotEqual(pref.revision, pref2.revision) + assert pref.revision != pref2.revision - self.assertEqual(self.server.server_store.get_last_package_revision(pref2).revision, - pref.revision) + assert self.server.server_store.get_last_package_revision(pref2).revision == pref.revision client.run(f"upload {self.ref} -r=default -c") - self.assertEqual(self.server.server_store.get_last_package_revision(pref2).revision, - pref2.revision) + assert self.server.server_store.get_last_package_revision(pref2).revision == pref2.revision def test_server_with_only_v2_capability(): @@ -523,9 +521,10 @@ def test_server_with_only_v2_capability(): c_v2.run(f"upload * -r=default -c") -class ServerRevisionsIndexes(unittest.TestCase): +class TestServerRevisionsIndexes: - def setUp(self): + @pytest.fixture(autouse=True) + def setup(self): self.server = TestServer() self.c_v2 = TestClient(servers={"default": self.server}, inputs=["admin", "password"]) self.ref = RecipeReference.loads("lib/1.0@conan/testing") @@ -539,28 +538,23 @@ def test_rotation_deleting_recipe_revisions(self): """ ref1 = self.c_v2.export(self.ref, conanfile=GenConanfile()) self.c_v2.run(f"upload {ref1} -r=default -c") - self.assertEqual(self.server.server_store.get_last_revision(self.ref).revision, - ref1.revision) + assert self.server.server_store.get_last_revision(self.ref).revision == ref1.revision ref2 = self.c_v2.export(self.ref, conanfile=GenConanfile().with_build_msg("I'm rev2")) self.c_v2.run(f"upload {ref2} -r=default -c") - self.assertEqual(self.server.server_store.get_last_revision(self.ref).revision, - ref2.revision) + assert self.server.server_store.get_last_revision(self.ref).revision == ref2.revision ref3 = self.c_v2.export(self.ref, conanfile=GenConanfile().with_build_msg("I'm rev3")) self.c_v2.run(f"upload {ref2} -r=default -c") - self.assertEqual(self.server.server_store.get_last_revision(self.ref).revision, - ref3.revision) + assert self.server.server_store.get_last_revision(self.ref).revision == ref3.revision revs = [r.revision for r in self.server.server_store.get_recipe_revisions_references(self.ref)] - self.assertEqual(revs, [ref3.revision, ref2.revision, ref1.revision]) - self.assertEqual(self.server.server_store.get_last_revision(self.ref).revision, - ref3.revision) + assert revs == [ref3.revision, ref2.revision, ref1.revision] + assert self.server.server_store.get_last_revision(self.ref).revision == ref3.revision # Delete the latest from the server self.c_v2.run("remove {} -r default -c".format(repr(ref3))) revs = [r.revision for r in self.server.server_store.get_recipe_revisions_references(self.ref)] - self.assertEqual(revs, [ref2.revision, ref1.revision]) - self.assertEqual(self.server.server_store.get_last_revision(self.ref).revision, - ref2.revision) + assert revs == [ref2.revision, ref1.revision] + assert self.server.server_store.get_last_revision(self.ref).revision == ref2.revision def test_rotation_deleting_package_revisions(self): """ @@ -573,39 +567,34 @@ def test_rotation_deleting_package_revisions(self): with environment_update({"MY_VAR": "1"}): pref1 = _create(self.c_v2, self.ref, conanfile=conanfile) self.c_v2.run("upload * -r=default -c") - self.assertEqual(self.server.server_store.get_last_package_revision(pref1).revision, - pref1.revision) + assert self.server.server_store.get_last_package_revision(pref1).revision == pref1.revision with environment_update({"MY_VAR": "2"}): pref2 = _create(self.c_v2, self.ref, conanfile=conanfile) self.c_v2.run("upload * -r=default -c") - self.assertEqual(self.server.server_store.get_last_package_revision(pref1).revision, - pref2.revision) + assert self.server.server_store.get_last_package_revision(pref1).revision == pref2.revision with environment_update({"MY_VAR": "3"}): pref3 = _create(self.c_v2, self.ref, conanfile=conanfile) self.c_v2.run("upload * -r=default -c") - self.assertEqual(self.server.server_store.get_last_package_revision(pref1).revision, - pref3.revision) + assert self.server.server_store.get_last_package_revision(pref1).revision == pref3.revision - self.assertEqual(pref1.ref.revision, pref2.ref.revision) - self.assertEqual(pref2.ref.revision, pref3.ref.revision) - self.assertEqual(pref3.ref.revision, pref3.ref.revision) + assert pref1.ref.revision == pref2.ref.revision + assert pref2.ref.revision == pref3.ref.revision + assert pref3.ref.revision == pref3.ref.revision pref = copy.copy(pref1) pref.revision = None revs = [r.revision for r in self.server.server_store.get_package_revisions_references(pref)] - self.assertEqual(revs, [pref3.revision, pref2.revision, pref1.revision]) - self.assertEqual(self.server.server_store.get_last_package_revision(pref).revision, - pref3.revision) + assert revs == [pref3.revision, pref2.revision, pref1.revision] + assert self.server.server_store.get_last_package_revision(pref).revision == pref3.revision # Delete the latest from the server self.c_v2.run("remove {}:{}#{} -r default -c".format(repr(pref3.ref), pref3.package_id, pref3.revision)) revs = [r.revision for r in self.server.server_store.get_package_revisions_references(pref)] - self.assertEqual(revs, [pref2.revision, pref1.revision]) - self.assertEqual(self.server.server_store.get_last_package_revision(pref).revision, - pref2.revision) + assert revs == [pref2.revision, pref1.revision] + assert self.server.server_store.get_last_package_revision(pref).revision == pref2.revision def test_deleting_all_rrevs(self): """ @@ -623,14 +612,14 @@ def test_deleting_all_rrevs(self): self.c_v2.run("remove {} -r default -c".format(repr(ref2))) self.c_v2.run("remove {} -r default -c".format(repr(ref3))) - self.assertRaises(RecipeNotFoundException, - self.server.server_store.get_recipe_revisions_references, self.ref) + with pytest.raises(RecipeNotFoundException): + self.server.server_store.get_recipe_revisions_references(self.ref) ref4 = self.c_v2.export(self.ref, conanfile=GenConanfile().with_build_msg("I'm rev4")) self.c_v2.run("upload * -r=default -c") revs = [r.revision for r in self.server.server_store.get_recipe_revisions_references(self.ref)] - self.assertEqual(revs, [ref4.revision]) + assert revs == [ref4.revision] def test_deleting_all_prevs(self): """ @@ -662,7 +651,7 @@ def test_deleting_all_prevs(self): pref.revision = None revs = [r.revision for r in self.server.server_store.get_package_revisions_references(pref)] - self.assertEqual(revs, [pref4.revision]) + assert revs == [pref4.revision] def test_touching_other_server(): diff --git a/test/functional/sbom/test_cyclonedx.py b/test/functional/sbom/test_cyclonedx.py index c661ca3aee8..162f0c98c18 100644 --- a/test/functional/sbom/test_cyclonedx.py +++ b/test/functional/sbom/test_cyclonedx.py @@ -43,17 +43,10 @@ def post_generate(conanfile): ConanOutput().success(f"CYCLONEDX CREATED - {{conanfile.generators_folder}}") """ + @pytest.mark.parametrize("cyclone_version", ["cyclonedx_1_4", "cyclonedx_1_6"]) class TestCyclonedx: - @pytest.fixture() - def hook_setup_post_package_default(self, cyclone_version): - tc = TestClient() - hook_path = os.path.join(tc.paths.hooks_path, "hook_sbom.py") - save(hook_path, sbom_hook_post_package.format(cyclone_version=cyclone_version, - add_build=False, add_tests=False)) - return tc - @pytest.fixture() def hook_setup_post_package(self, cyclone_version): tc = TestClient() @@ -385,3 +378,15 @@ def test_sbom_user_path(self, hook_setup_post_package_tl, user, channel, user_de assert content_json["components"][0]["bom-ref"].split("&user=")[1] == f"{user}&channel={channel}" if channel else user assert content_json["dependencies"][0]["dependsOn"][0].split("&user=")[1] == f"{user_dep}&channel={channel_dep}" if channel_dep else user_dep + + def test_sbom_test_requires_skipped(self, cyclone_version): + tc = TestClient() + hook_path = os.path.join(tc.paths.hooks_path, "hook_sbom.py") + save(hook_path, sbom_hook_post_generate.format(cyclone_version=cyclone_version, name=None, + add_build=False, add_tests=False)) + tc.save({"dep/conanfile.py": GenConanfile("mydep", "1.0"), + "conanfile.py": GenConanfile("foo", "1.0").with_test_requires("mydep/1.0")}) + tc.run("create dep") + tc.run("install .") + content = tc.load("sbom/sbom.cdx.json") + assert "mydep" not in content diff --git a/test/functional/subsystems_build_test.py b/test/functional/subsystems_build_test.py index 4c439c4a5cf..178880486dc 100644 --- a/test/functional/subsystems_build_test.py +++ b/test/functional/subsystems_build_test.py @@ -491,7 +491,8 @@ def test_cygwin(self): check_exe_run(client.out, "main", "gcc", None, "Debug", "x86_64", None, subsystem="cygwin") check_vs_runtime("app.exe", client, "15", "Debug", subsystem="cygwin") - @pytest.mark.tool("clang", "13") + @pytest.mark.tool("ninja") + @pytest.mark.tool("clang", "20") def test_clang(self): """ native, LLVM/Clang compiler diff --git a/test/functional/toolchains/apple/test_xcodebuild.py b/test/functional/toolchains/apple/test_xcodebuild.py index 74c7c069050..512471d346f 100644 --- a/test/functional/toolchains/apple/test_xcodebuild.py +++ b/test/functional/toolchains/apple/test_xcodebuild.py @@ -1,4 +1,5 @@ import platform +import re import textwrap import pytest @@ -18,6 +19,17 @@ Release: conan_config.xcconfig """) +xcode_project_bare = textwrap.dedent(""" + name: app + targets: + app: + type: tool + platform: macOS + sources: + - app + """) + + main = textwrap.dedent(""" #include #include "hello.h" @@ -165,3 +177,57 @@ def build(self): client.run_command("xcodegen generate") client.run("create . --build=missing -s os.sdk=macosx -s os.sdk_version=12.0 " "-c tools.apple:sdk_path=notexistingsdk", assert_error=True) + + +@pytest.mark.skipif(platform.system() != "Darwin", reason="Only for MacOS") +@pytest.mark.tool("xcodebuild") +@pytest.mark.tool("xcodegen") +@pytest.mark.parametrize("no_copy_source", [True, False]) +def test_project_xcodebuild_cli_args(client, no_copy_source): + + conanfile = textwrap.dedent(f""" + import os + from conan import ConanFile + from conan.tools.apple import XcodeBuild + from conan.tools.files import copy + class MyApplicationConan(ConanFile): + name = "myapplication" + version = "1.0" + requires = "hello/0.1" + settings = "os", "compiler", "build_type", "arch" + generators = "XcodeDeps" + exports_sources = "app.xcodeproj/*", "app/*" + package_type = "application" + no_copy_source = {str(no_copy_source)} + def build(self): + xb = XcodeBuild(self) + proj = os.path.join(self.source_folder, "app.xcodeproj") + xc = os.path.join(self.build_folder, "conan_config.xcconfig") + xb.build(proj, cli_args=["-xcconfig", xc, + f"SYMROOT={{self.build_folder}}", + f"OBJROOT={{self.build_folder}}"]) + + def package(self): + copy(self, "{{}}/app".format(self.settings.build_type), self.build_folder, + os.path.join(self.package_folder, "bin"), keep_path=False) + + def package_info(self): + self.cpp_info.bindirs = ["bin"] + """) + + client.save({"conanfile.py": conanfile, + "test_package/conanfile.py": test, + "app/main.cpp": main, + "project.yml": xcode_project_bare}, clean_first=True) + + client.run_command("xcodegen generate") + + for build_type in ["Release", "Debug"]: + client.run(f"create . --build=missing -s build_type={build_type} -c tools.build:verbosity=verbose -c tools.compilation:verbosity=verbose") + + build_folder = re.search(r"Building your package in (/.+)", client.out).group(1) + + assert f"OBJROOT = {build_folder}" + assert f"SYMROOT = {build_folder}" + assert "-xcconfig" in client.out + assert f"App {build_type}!" in client.out diff --git a/test/functional/toolchains/apple/test_xcodedeps_components.py b/test/functional/toolchains/apple/test_xcodedeps_components.py index d2276f8019c..6d78308e968 100644 --- a/test/functional/toolchains/apple/test_xcodedeps_components.py +++ b/test/functional/toolchains/apple/test_xcodedeps_components.py @@ -107,10 +107,10 @@ def package_info(self): self.cpp_info.components["core"].includedirs.append("include") self.cpp_info.components["core"].libdirs.append("lib") - self.cpp_info.components["client-test"].libs = ["client"] - self.cpp_info.components["client-test"].includedirs.append("include") - self.cpp_info.components["client-test"].libdirs.append("lib") - self.cpp_info.components["client-test"].requires.extend(["core", "tcp::tcp"]) + self.cpp_info.components["client-test++"].libs = ["client"] + self.cpp_info.components["client-test++"].includedirs.append("include") + self.cpp_info.components["client-test++"].libdirs.append("lib") + self.cpp_info.components["client-test++"].requires.extend(["core", "tcp::tcp"]) self.cpp_info.components["server"].libs = ["server"] self.cpp_info.components["server"].includedirs.append("include") @@ -138,7 +138,7 @@ def package_info(self): self.cpp_info.libs = ["chat"] self.cpp_info.includedirs.append("include") self.cpp_info.libdirs.append("lib") - self.cpp_info.requires.append("network::client-test") + self.cpp_info.requires.append("network::client-test++") """ cmakelists_chat = textwrap.dedent(""" @@ -150,7 +150,7 @@ def package_info(self): add_library(chat src/chat.cpp include/chat.h) target_include_directories(chat PUBLIC include) set_target_properties(chat PROPERTIES PUBLIC_HEADER "include/chat.h") - target_link_libraries(chat network::client-test) + target_link_libraries(chat network::client-test++) install(TARGETS chat) """) @@ -188,7 +188,7 @@ def package_info(self): client.run("install --requires=chat/1.0@ -g XcodeDeps --output-folder=conan " "-s build_type=Debug --build=missing") chat_xcconfig = client.load(os.path.join("conan", "conan_chat_chat.xcconfig")) - assert '#include "conan_network_client_test.xcconfig"' in chat_xcconfig + assert '#include "conan_network_client_test__.xcconfig"' in chat_xcconfig assert '#include "conan_network_server.xcconfig"' not in chat_xcconfig assert '#include "conan_network_network.xcconfig"' not in chat_xcconfig host_arch = client.get_default_host_profile().settings['arch'] diff --git a/test/functional/toolchains/apple/test_xcodetoolchain.py b/test/functional/toolchains/apple/test_xcodetoolchain.py index 9fd6431ce14..4ec46c7d868 100644 --- a/test/functional/toolchains/apple/test_xcodetoolchain.py +++ b/test/functional/toolchains/apple/test_xcodetoolchain.py @@ -80,7 +80,7 @@ def package_info(self): client.run_command("xcodegen generate") - sdk_version = "15.0" + sdk_version = "15.5" settings = "-s arch=x86_64 -s os.sdk_version={} -s compiler.cppstd={} " \ "-s compiler.libcxx=libc++ -s os.version={} " \ "-c tools.build.cross_building:can_run=True " \ diff --git a/conans/model/__init__.py b/test/functional/toolchains/autotools/__init__.py similarity index 100% rename from conans/model/__init__.py rename to test/functional/toolchains/autotools/__init__.py diff --git a/test/functional/toolchains/autotools/test_universal_binaries.py b/test/functional/toolchains/autotools/test_universal_binaries.py new file mode 100644 index 00000000000..987e3ddb7b0 --- /dev/null +++ b/test/functional/toolchains/autotools/test_universal_binaries.py @@ -0,0 +1,67 @@ +import os +import platform +import textwrap + +import pytest + +from conan.test.utils.tools import TestClient + + +@pytest.mark.skipif(platform.system() != "Darwin", reason="Only OSX") +def test_autotools_universal_binary(): + client = TestClient(path_with_spaces=False) + + conanfile = textwrap.dedent(""" + import os + from conan import ConanFile + from conan.tools.gnu import AutotoolsToolchain, Autotools + from conan.tools.layout import basic_layout + + class mylibraryRecipe(ConanFile): + name = "mylibrary" + version = "1.0" + package_type = "library" + settings = "os", "compiler", "build_type", "arch" + options = {"shared": [True, False], "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + + exports_sources = "configure.ac", "Makefile.am", "src/*" + + def config_options(self): + if self.settings.os == "Windows": + self.options.rm_safe("fPIC") + + def configure(self): + if self.options.shared: + self.options.rm_safe("fPIC") + + def layout(self): + basic_layout(self) + + def generate(self): + at_toolchain = AutotoolsToolchain(self) + at_toolchain.generate() + + def build(self): + autotools = Autotools(self) + autotools.autoreconf() + autotools.configure() + autotools.make() + + def package(self): + autotools = Autotools(self) + autotools.install() + self.run(f"lipo -info {os.path.join(self.package_folder, 'lib', 'libmylibrary.a')}") + + def package_info(self): + self.cpp_info.libs = ["mylibrary"] + + """) + + client.run('new autotools_lib -d name=mylibrary -d version=0.1') + + client.save({"conanfile.py": conanfile}) + + client.run('create . -s="arch=armv8|x86_64" -tf=""') + + assert "libmylibrary.a are: x86_64 arm64" in client.out diff --git a/test/functional/toolchains/cmake/cmakedeps/test_apple_frameworks.py b/test/functional/toolchains/cmake/cmakedeps/test_apple_frameworks.py index 8166187f2ec..e00f857d02f 100644 --- a/test/functional/toolchains/cmake/cmakedeps/test_apple_frameworks.py +++ b/test/functional/toolchains/cmake/cmakedeps/test_apple_frameworks.py @@ -42,7 +42,7 @@ def build(self): # needs at least 3.23.3 because of error with "empty identity" -# https://stackoverflow.com/questions/72746725/xcode-14-beta-cmake-not-able-to-resolve-cmake-c-compiler-and-cmake-cxx-compiler +# https://stackoverflow.com/questions/72746725/xcode-14-beta-cmake-not-able-to-resolve-cmake-c-compiler-and-cmake-cxx-compiler @pytest.mark.skipif(platform.system() != "Darwin", reason="Only OSX") @pytest.mark.tool("cmake", "3.23") def test_apple_framework_xcode(client): diff --git a/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_components.py b/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_components.py index 50005a06cda..f733d9b31d6 100644 --- a/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_components.py +++ b/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_components.py @@ -1,12 +1,11 @@ import textwrap -import unittest import pytest from conan.test.utils.tools import TestClient -class PropagateSpecificComponents(unittest.TestCase): +class TestPropagateSpecificComponents: """ Feature: recipes can declare the components they are consuming from their requirements, only those components should be propagated to their own consumers. If required components @@ -45,7 +44,8 @@ class Recipe(ConanFile): requires = "middle/version" """) - def setUp(self): + @pytest.fixture(autouse=True) + def setup(self): client = TestClient() client.save({ 'top.py': self.top, @@ -61,8 +61,8 @@ def test_cmakedeps_app(self): t.save({'conanfile.py': self.app}) t.run("install . -g CMakeDeps") config = t.load("middle-Target-release.cmake") - self.assertIn('top::cmp1', config) - self.assertNotIn("top::top", config) + assert 'top::cmp1' in config + assert "top::top" not in config def test_cmakedeps_multi(self): t = TestClient(cache_folder=self.cache_folder) @@ -70,12 +70,12 @@ def test_cmakedeps_multi(self): host_arch = t.get_default_host_profile().settings['arch'] content = t.load(f'middle-release-{host_arch}-data.cmake') - self.assertIn("list(APPEND middle_FIND_DEPENDENCY_NAMES top)", content) + assert "list(APPEND middle_FIND_DEPENDENCY_NAMES top)" in content content = t.load('middle-Target-release.cmake') - self.assertNotIn("top::top", content) - self.assertNotIn("top::cmp2", content) - self.assertIn("top::cmp1", content) + assert "top::top" not in content + assert "top::cmp2" not in content + assert "top::cmp1" in content @pytest.fixture @@ -158,7 +158,8 @@ def build(self): find_package(requirement) get_target_property(tmp_libs requirement::component INTERFACE_LINK_LIBRARIES) get_target_property(tmp_options requirement::component INTERFACE_LINK_OPTIONS) - get_target_property(tmp_deps requirement_requirement_component_DEPS_TARGET INTERFACE_LINK_LIBRARIES) + get_target_property(tmp_deps requirement_requirement_component_DEPS_TARGET + INTERFACE_LINK_LIBRARIES) message("component libs: ${tmp_libs}") message("component options: ${tmp_options}") message("component deps: ${tmp_deps}") @@ -166,8 +167,10 @@ def build(self): t.save({"conanfile.py": conanfile, "CMakeLists.txt": cmakelists}) t.run("create . --build missing -s build_type=Release") - assert 'component libs: $<$:>;$<$:>;requirement_requirement_component_DEPS_TARGET' in t.out - assert 'component deps: $<$:>;$<$:system_lib_component>;' in t.out + assert ('component libs: $<$:>;$<$:>;' + 'requirement_requirement_component_DEPS_TARGET') in t.out + assert ('component deps: $<$:>;$<$:' + 'system_lib_component>;') in t.out assert ('component options: ' '$<$:' '$<$,SHARED_LIBRARY>:>;' @@ -360,7 +363,9 @@ def build(self): # The boost::boost target has linked the two components assert "AGGREGATED LIBS: boost::boost" in t.out assert "AGGREGATED LINKED: boost::B;boost::A" in t.out - assert "BOOST_B LINKED: $<$:>;$<$:>;boost_boost_B_DEPS_TARGET" in t.out - assert "BOOST_A LINKED: $<$:>;$<$:>;boost_boost_A_DEPS_TARGET" in t.out + assert ("BOOST_B LINKED: $<$:>;$<$:>;" + "boost_boost_B_DEPS_TARGET") in t.out + assert ("BOOST_A LINKED: $<$:>;$<$:>;" + "boost_boost_A_DEPS_TARGET") in t.out assert "BOOST_B_DEPS LINKED: $<$:>;$<$:B_1;B_2>" in t.out assert "BOOST_A_DEPS LINKED: $<$:>;$<$:A_1;A_2>;" in t.out diff --git a/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_components_names.py b/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_components_names.py index e292ecbc41f..ea75cb3ad60 100644 --- a/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_components_names.py +++ b/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_components_names.py @@ -1,5 +1,4 @@ import os -import platform import shutil import textwrap @@ -280,27 +279,6 @@ def test_standard_names(setup_client_with_greetings): create_chat(client, "standard", package_info, cmake_find, test_cmake_find) - # Test consumer multi-config - if platform.system() == "Windows": - with client.chdir("test_package"): - # FIXME This doesn't work because test_package requires has been disabled - return - client.run("install . -s build_type=Release") - client.run("install . -s build_type=Debug") - client.run_command('cmake . -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake') - client.run_command("cmake --build . --config Debug") - client.run_command(r".\Debug\example.exe") - assert "sayhellobye: Debug!" in client.out - assert "sayhello: Debug!" in client.out - assert "hello: Debug!" in client.out - assert "bye: Debug!" in client.out - client.run_command("cmake --build . --config Release") - client.run_command(r".\Release\example.exe") - assert "sayhellobye: Release!" in client.out - assert "sayhello: Release!" in client.out - assert "hello: Release!" in client.out - assert "bye: Release!" in client.out - @pytest.mark.tool("cmake") def test_custom_names(setup_client_with_greetings): @@ -563,9 +541,6 @@ def test_component_not_found_same_name_as_pkg_require(self): .with_package_info(cpp_info={"components": {"cmp": {"requires": ["mypkg::zlib", "zlib::zlib"]}}}, env_info={}) - consumer = GenConanfile("consumer", "0.1").with_setting("build_type")\ - .with_generator("CMakeDeps")\ - .with_requirement(RecipeReference("final", "0.1", None, None)) consumer = textwrap.dedent(""" from conan import ConanFile @@ -585,32 +560,6 @@ def requirements(self): settings = "build_type" """) - def test_component_not_found(self): - conanfile = textwrap.dedent(""" - from conan import ConanFile - class GreetingsConan(ConanFile): - def package_info(self): - self.cpp_info.components["hello"].libs = ["hello"] - self.cpp_info.components["hello"].libdirs = ["lib"] - self.cpp_info.components["hello"].includedirs = ["include"] - """) - client = TestClient() - client.save({"conanfile.py": conanfile}) - client.run("create . --name=greetings --version=0.0.1") - - conanfile = textwrap.dedent(""" - from conan import ConanFile - class WorldConan(ConanFile): - requires = "greetings/0.0.1" - def package_info(self): - self.cpp_info.components["helloworld"].requires = ["greetings::non-existent"] - """) - client.save({"conanfile.py": conanfile}) - client.run("create . --name=world --version=0.0.1") - client.run("install --requires=world/0.0.1@ -g CMakeDeps", assert_error=True) - assert ("Component 'greetings::non-existent' not found in 'greetings' " - "package requirement" in client.out) - client = TestClient() client.save({"zlib.py": zlib, "mypkg.py": mypkg, "final.py": final, "consumer.py": consumer}) client.run("create zlib.py") @@ -823,7 +772,7 @@ def build(self): """) if check_components_exist is False: conanfile = conanfile.format("deps.check_components_exist=False") - elif check_components_exist is True: + elif check_components_exist: conanfile = conanfile.format("deps.check_components_exist=True") else: conanfile = conanfile.format("") diff --git a/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_custom_configs.py b/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_custom_configs.py index 8aaf2576d4d..4c43dda5dc9 100644 --- a/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_custom_configs.py +++ b/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_custom_configs.py @@ -1,7 +1,6 @@ import os import platform import textwrap -import unittest import pytest @@ -14,7 +13,7 @@ @pytest.mark.tool("cmake") @pytest.mark.skipif(platform.system() != "Windows", reason="Only for windows") -class CustomConfigurationTest(unittest.TestCase): +def test_custom_config_generator_multi(): conanfile = textwrap.dedent(""" import os from conan import ConanFile @@ -61,53 +60,51 @@ def generate(self): target_link_libraries(app PRIVATE hello::hello) """) - def setUp(self): - self.client = TestClient(path_with_spaces=False) - self.client.run("new cmake_lib -d name=hello -d version=0.1") - self.client.run("create . -s compiler.version=191 " - "-s build_type=Release -o hello/*:shared=True -tf=\"\"") - self.client.run("create . --name=hello --version=0.1 -s compiler.version=191 " - "-s build_type=Release -tf=\"\"") - - # Prepare the actual consumer package - self.client.save({"conanfile.py": self.conanfile, - "CMakeLists.txt": self.cmakelist, - "app.cpp": self.app}) - - def test_generator_multi(self): - settings = {"compiler": "msvc", - "compiler.version": "191", - "compiler.runtime": "dynamic", - "arch": "x86_64", - "build_type": "Release", - } - - settings = " ".join('-s %s="%s"' % (k, v) for k, v in settings.items() if v) - - # Run the configure corresponding to this test case - with self.client.chdir('build'): - self.client.run("install .. %s -o hello/*:shared=True -of=." % settings) - self.client.run("install .. %s -o hello/*:shared=False -of=." % settings) - self.assertTrue(os.path.isfile(os.path.join(self.client.current_folder, - "hello-Target-releaseshared.cmake"))) - self.assertTrue(os.path.isfile(os.path.join(self.client.current_folder, - "hello-Target-release.cmake"))) - - self.client.run_command('cmake .. -G "Visual Studio 15 Win64" --loglevel=DEBUG') - self.assertIn("Found DLL and STATIC", self.client.out) - self.client.run_command('cmake --build . --config Release') - self.client.run_command(r"Release\\app.exe") - self.assertIn("hello/0.1: Hello World Release!", self.client.out) - self.assertIn("main: Release!", self.client.out) - self.client.run_command('cmake --build . --config ReleaseShared') - self.client.run_command(r"ReleaseShared\\app.exe") - self.assertIn("hello/0.1: Hello World Release!", self.client.out) - self.assertIn("main: Release!", self.client.out) + c = TestClient(path_with_spaces=False) + c.run("new cmake_lib -d name=hello -d version=0.1") + c.run("create . -s compiler.version=191 " + "-s build_type=Release -o hello/*:shared=True -tf=\"\"") + c.run("create . --name=hello --version=0.1 -s compiler.version=191 " + "-s build_type=Release -tf=\"\"") + + # Prepare the actual consumer package + c.save({"conanfile.py": conanfile, + "CMakeLists.txt": cmakelist, + "app.cpp": app}) + + settings = {"compiler": "msvc", + "compiler.version": "191", + "compiler.runtime": "dynamic", + "arch": "x86_64", + "build_type": "Release", + } + + settings = " ".join('-s %s="%s"' % (k, v) for k, v in settings.items() if v) + + # Run the configure corresponding to this test case + with c.chdir('build'): + c.run("install .. %s -o hello/*:shared=True -of=." % settings) + c.run("install .. %s -o hello/*:shared=False -of=." % settings) + assert os.path.isfile(os.path.join(c.current_folder, + "hello-Target-releaseshared.cmake")) + assert os.path.isfile(os.path.join(c.current_folder, + "hello-Target-release.cmake")) + + c.run_command('cmake .. -G "Visual Studio 15 Win64" --loglevel=DEBUG') + assert "Found DLL and STATIC" in c.out + c.run_command('cmake --build . --config Release') + c.run_command(r"Release\\app.exe") + assert "hello/0.1: Hello World Release!" in c.out + assert "main: Release!" in c.out + c.run_command('cmake --build . --config ReleaseShared') + c.run_command(r"ReleaseShared\\app.exe") + assert "hello/0.1: Hello World Release!" in c.out + assert "main: Release!" in c.out @pytest.mark.tool("cmake") @pytest.mark.skipif(platform.system() != "Windows", reason="Only for windows") -class CustomSettingsTest(unittest.TestCase): +def test_custom_config_settings(): conanfile = textwrap.dedent(""" from conan import ConanFile from conan.tools.cmake import CMakeDeps @@ -148,59 +145,57 @@ def generate(self): target_link_libraries(app PRIVATE hello::hello) """) - def setUp(self): - self.client = TestClient(path_with_spaces=False) - settings = "build_type: [MyRelease]" - save(self.client.paths.settings_path_user, settings) - self.client.run("new cmake_lib -d name=hello -d version=0.1") - cmake = self.client.load("CMakeLists.txt") - - cmake = cmake.replace("cmake_minimum_required", """ - set(CMAKE_CONFIGURATION_TYPES Debug MyRelease Release CACHE STRING "Types") - - cmake_minimum_required""") - cmake = cmake.replace("add_library", textwrap.dedent(""" - set(CMAKE_CXX_FLAGS_MYRELEASE ${CMAKE_CXX_FLAGS_RELEASE}) - set(CMAKE_C_FLAGS_MYRELEASE ${CMAKE_C_FLAGS_RELEASE}) - set(CMAKE_EXE_LINKER_FLAGS_MYRELEASE ${CMAKE_EXE_LINKER_FLAGS_RELEASE}) - add_library""")) - cmake = cmake.replace("PUBLIC_HEADER", "CONFIGURATIONS MyRelease\nPUBLIC_HEADER") - self.client.save({"CMakeLists.txt": cmake}) - self.client.run("create . --name=hello --version=0.1 -s compiler.version=191 -s build_type=MyRelease " - "-s:b build_type=MyRelease -tf=\"\"") - - # Prepare the actual consumer package - self.client.save({"conanfile.py": self.conanfile, - "CMakeLists.txt": self.cmakelist, - "app.cpp": self.app}) - - def test_generator_multi(self): - settings = {"compiler": "msvc", - "compiler.version": "191", - "compiler.runtime": "dynamic", - "arch": "x86_64", - "build_type": "MyRelease", - } - - settings_h = " ".join('-s %s="%s"' % (k, v) for k, v in settings.items()) - settings_b = " ".join('-s:b %s="%s"' % (k, v) for k, v in settings.items()) - - # Run the configure corresponding to this test case - build_directory = os.path.join(self.client.current_folder, "build").replace("\\", "/") - with self.client.chdir(build_directory): - self.client.run("install .. %s %s -of=." % (settings_h, settings_b)) - toolchain = self.client.load("conan_toolchain.cmake") - # As it is a custom configuration, the TRY_COMPILE_CONFIFURATION not defined - assert "CMAKE_TRY_COMPILE_CONFIGURATION" not in toolchain - self.assertTrue(os.path.isfile(os.path.join(self.client.current_folder, - "hello-Target-myrelease.cmake"))) - - self.client.run_command('cmake .. -G "Visual Studio 15" ' - '-DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake') - self.client.run_command('cmake --build . --config MyRelease') - self.client.run_command(r"MyRelease\\app.exe") - self.assertIn("hello/0.1: Hello World Release!", self.client.out) - self.assertIn("main: Release!", self.client.out) + c = TestClient(path_with_spaces=False) + settings = "build_type: [MyRelease]" + save(c.paths.settings_path_user, settings) + c.run("new cmake_lib -d name=hello -d version=0.1") + cmake = c.load("CMakeLists.txt") + + cmake = cmake.replace("cmake_minimum_required", """ + set(CMAKE_CONFIGURATION_TYPES Debug MyRelease Release CACHE STRING "Types") + + cmake_minimum_required""") + cmake = cmake.replace("add_library", textwrap.dedent(""" + set(CMAKE_CXX_FLAGS_MYRELEASE ${CMAKE_CXX_FLAGS_RELEASE}) + set(CMAKE_C_FLAGS_MYRELEASE ${CMAKE_C_FLAGS_RELEASE}) + set(CMAKE_EXE_LINKER_FLAGS_MYRELEASE ${CMAKE_EXE_LINKER_FLAGS_RELEASE}) + add_library""")) + cmake = cmake.replace("PUBLIC_HEADER", "CONFIGURATIONS MyRelease\nPUBLIC_HEADER") + c.save({"CMakeLists.txt": cmake}) + c.run("create . --name=hello --version=0.1 -s compiler.version=191 " + "-s build_type=MyRelease -s:b build_type=MyRelease -tf=\"\"") + + # Prepare the actual consumer package + c.save({"conanfile.py": conanfile, + "CMakeLists.txt": cmakelist, + "app.cpp": app}) + + settings = {"compiler": "msvc", + "compiler.version": "191", + "compiler.runtime": "dynamic", + "arch": "x86_64", + "build_type": "MyRelease", + } + + settings_h = " ".join('-s %s="%s"' % (k, v) for k, v in settings.items()) + settings_b = " ".join('-s:b %s="%s"' % (k, v) for k, v in settings.items()) + + # Run the configure corresponding to this test case + build_directory = os.path.join(c.current_folder, "build").replace("\\", "/") + with c.chdir(build_directory): + c.run("install .. %s %s -of=." % (settings_h, settings_b)) + toolchain = c.load("conan_toolchain.cmake") + # As it is a custom configuration, the TRY_COMPILE_CONFIFURATION not defined + assert "CMAKE_TRY_COMPILE_CONFIGURATION" not in toolchain + assert os.path.isfile(os.path.join(c.current_folder, + "hello-Target-myrelease.cmake")) + + c.run_command('cmake .. -G "Visual Studio 15" ' + '-DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake') + c.run_command('cmake --build . --config MyRelease') + c.run_command(r"MyRelease\\app.exe") + assert "hello/0.1: Hello World Release!" in c.out + assert "main: Release!" in c.out @pytest.mark.tool("cmake") @@ -246,7 +241,8 @@ def package(self): # in MSVC multi-config -s pkg/*:build_type=Debug is not really necesary, toolchain do nothing # TODO: Challenge how to define consumer build_type for conanfile.txt client.run("install . -s pkg*:build_type=Debug -s build_type=Release") - client.run_command("cmake . -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Debug") + client.run_command("cmake . -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake " + "-DCMAKE_BUILD_TYPE=Debug") client.run_command("cmake --build . --config Debug") cmd = os.path.join(".", "Debug", "app") if platform.system() == "Windows" else "./app" client.run_command(cmd) diff --git a/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_find_module_and_config.py b/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_find_module_and_config.py index dcddf41a738..9bf4770ffc5 100644 --- a/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_find_module_and_config.py +++ b/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_find_module_and_config.py @@ -132,8 +132,8 @@ def test_reuse_with_modules_and_config(client): @pytest.mark.tool("cmake") -@pytest.mark.parametrize("find_mode_PKGA, find_mode_PKGB, find_mode_consumer", find_modes) -def test_transitive_modules_found(find_mode_PKGA, find_mode_PKGB, find_mode_consumer): +@pytest.mark.parametrize("find_mode_pkga, find_mode_pkgb, find_mode_consumer", find_modes) +def test_transitive_modules_found(find_mode_pkga, find_mode_pkgb, find_mode_consumer): """ related to https://github.com/conan-io/conan/issues/10224 modules files variables were set with the pkg_name_FOUND or pkg_name_VERSION @@ -187,8 +187,8 @@ def build(self): """) client.save({"pkgb.py": conan_pkg.format(requires='requires="pkga/1.0"', filename='MYPKGB', module_filename='MYPKGB', - mode=find_mode_PKGB), - "pkga.py": conan_pkg.format(requires='', filename='MYPKGA', module_filename='unicorns', mode=find_mode_PKGA), + mode=find_mode_pkgb), + "pkga.py": conan_pkg.format(requires='', filename='MYPKGA', module_filename='unicorns', mode=find_mode_pkga), "consumer.py": consumer, "CMakeLists.txt": cmakelist.format(find_mode=find_mode_consumer)}) @@ -210,6 +210,6 @@ def build(self): assert "MYPKGB_DEFINITIONS: -DDEFINE_MYPKGB" in client.out assert "Conan: Target declared 'pkga::pkga'" - if find_mode_PKGA == "module": + if find_mode_pkga == "module": assert 'Found unicorns: 1.0 (found version "1.0")' in client.out diff --git a/test/functional/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_new.py b/test/functional/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_new.py index 3ca484d4fe3..36750cbb704 100644 --- a/test/functional/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_new.py +++ b/test/functional/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_new.py @@ -851,8 +851,11 @@ def package_info(self): self.cpp_info.system_libs = ["m", "dl"] # Just to verify CMake don't break if self.settings.compiler == "gcc": - self.cpp_info.sharedlinkflags = ["-z lazy", "-s"] - self.cpp_info.exelinkflags = ["-z lazy", "-s"] + # This triggers errors when not quoted and list ";" separated + self.cpp_info.sharedlinkflags = ["SHELL:-z lazy", "SHELL:-u symbol1", + "SHELL:-u symbol2", "-s"] + self.cpp_info.exelinkflags = ["SHELL:-z lazy", "SHELL:-u symbol1", + "SHELL:-u symbol2", "-s"] """) engine_h = textwrap.dedent(""" #pragma once @@ -994,6 +997,17 @@ def test_tool_requires(self): assert 'set(tool_INCLUDE_DIR' not in tool_config assert 'set(tool_LIBRARIES' not in tool_config + def test_libs_build_context(self): + c = TestClient() + c.run("new header_lib -d name=hello -d version=1.0 -o=hello") + c.run("create hello -tf=") + c.run("new cmake_lib -d name=bye -d version=1.0 -d requires=hello/1.0 -o=bye") + # Ninja for same layout in all platforms + c.run(f"install bye --build-require -c:a tools.cmake.cmakedeps:new={new_value} " + f"-c:a tools.cmake.cmaketoolchain:generator=Ninja") + cmake = c.load("bye/build/Release/generators/hello-TargetsBuild-release.cmake") + assert "add_library(hello::hello INTERFACE IMPORTED)" in cmake + @pytest.mark.tool("cmake") @pytest.mark.parametrize("tool_requires", [True, False]) @@ -1526,7 +1540,6 @@ def package_info(self): assert "dep/0.1 cpp_info incorrect .type shared-library for .exe myexe" in c.out - def test_multiple_find_package_subfolder(): c = TestClient() conanfile = textwrap.dedent(""" @@ -1572,3 +1585,32 @@ def build(self): assert "find_package(matrix)" in c.out assert "target_link_libraries(... matrix::matrix)" in c.out assert "Conan: Target declared imported INTERFACE library 'matrix::matrix'" in c.out + + +@pytest.mark.tool("cmake", "3.27") +def test_find_package_casing(): + cmakelists = textwrap.dedent(""" + cmake_minimum_required(VERSION 3.15) + project(test NONE) + + # As long as the package generates hello-config.cmake lowercase, it works + find_package(HellO REQUIRED) # Casing!!!!! + """) + consumer = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.cmake import CMake + class Pkg(ConanFile): + requires = "hello/1.0" + generators = "CMakeToolchain", "CMakeDeps" + settings = "os", "compiler", "build_type", "arch" + def build(self): + cmake = CMake(self) + cmake.configure() + """) + client = TestClient() + client.save({"conanfile.py": GenConanfile("hello", "1.0")}) + client.run(f"create .") + + client.save({"conanfile.py": consumer, "CMakeLists.txt": cmakelists}) + client.run(f"build . -c tools.cmake.cmakedeps:new={new_value}") + assert "Conan: Configuring Targets for hello/1.0" in client.out diff --git a/test/functional/toolchains/cmake/test_cmake_and_no_soname_flag.py b/test/functional/toolchains/cmake/test_cmake_and_no_soname_flag.py index d2dc005df4d..0226f64bb18 100644 --- a/test/functional/toolchains/cmake/test_cmake_and_no_soname_flag.py +++ b/test/functional/toolchains/cmake/test_cmake_and_no_soname_flag.py @@ -63,7 +63,7 @@ def test_no_soname_flag(nosoname_property): # If `nosoname_property` is False, and we have a library without the SONAME flag, # then it should fail - if nosoname_property is False: + if not nosoname_property: client.run_command(command, assert_error=True) assert "libnosoname.so: cannot open shared object file: " \ "No such file or directory" in client.out diff --git a/test/functional/toolchains/cmake/test_cmake_extra_variables.py b/test/functional/toolchains/cmake/test_cmake_extra_variables.py new file mode 100644 index 00000000000..132352403d8 --- /dev/null +++ b/test/functional/toolchains/cmake/test_cmake_extra_variables.py @@ -0,0 +1,54 @@ +import textwrap +import pytest + +from conan.test.utils.tools import TestClient +new_value = "will_break_next" + + +@pytest.mark.tool("cmake", "3.27") +@pytest.mark.parametrize("generator", ["CMakeDeps", "CMakeConfigDeps"]) +def test_package_info_extra_variables(generator): + """ The dependencies can define extra variables to be used in CMake, + but if the user is setting the cmake_extra_variables conf, + those should have precedence. + """ + client = TestClient() + dep_conanfile = textwrap.dedent(""" + from conan import ConanFile + + class Pkg(ConanFile): + name = "dep" + version = "0.1" + + def package_info(self): + self.cpp_info.set_property("cmake_extra_variables", {"FOO": 42}) + """) + client.save({"dep/conanfile.py": dep_conanfile}) + client.run("create dep") + + cmakelists = textwrap.dedent(""" + cmake_minimum_required(VERSION 3.27) + project(myproject CXX) + find_package(dep CONFIG REQUIRED) + message(STATUS "FOO=${FOO}") + """) + + conanfile = textwrap.dedent(f""" + from conan import ConanFile + from conan.tools.cmake import CMake + + class Pkg(ConanFile): + settings = "os", "arch", "compiler", "build_type" + generators = "{generator}", "CMakeToolchain" + requires = "dep/0.1" + def build(self): + cmake = CMake(self) + cmake.configure() + """) + client.save({"CMakeLists.txt": cmakelists, + "conanfile.py": conanfile}) + conf = f"-c tools.cmake.cmakedeps:new={new_value}" if generator == "CMakeConfigDeps" else "" + client.run(f"build . {conf} " + """-c tools.cmake.cmaketoolchain:extra_variables="{'FOO': '9'}" """) + + assert "-- FOO=9" in client.out diff --git a/test/functional/toolchains/cmake/test_cmake_find_none.py b/test/functional/toolchains/cmake/test_cmake_find_none.py index a445fb4071e..08245cf5efc 100644 --- a/test/functional/toolchains/cmake/test_cmake_find_none.py +++ b/test/functional/toolchains/cmake/test_cmake_find_none.py @@ -1,5 +1,8 @@ +import platform import textwrap +import pytest + from conan.test.utils.tools import TestClient @@ -48,7 +51,7 @@ def package_info(self): FILE "${CMAKE_CURRENT_BINARY_DIR}/Qt5Config.cmake" ) install(EXPORT Qt5Config - DESTINATION "${CMAKE_INSTALL_PREFIX}/qt/cmake" + DESTINATION "qt/cmake" NAMESPACE qt:: ) """) @@ -102,3 +105,157 @@ def build(self): c.run("build .") assert "Conan: Target declared 'karchive::karchive'" in c.out # And it doesn't fail to find transitive qt + + +def test_cmake_find_none_relocation(): + c = TestClient(default_server_user=True) + c.run("new cmake_lib -d name=pkg -d version=0.1") + conanfile = c.load("conanfile.py") + conanfile = conanfile + ' self.cpp_info.set_property("cmake_find_mode", "none")' + conanfile = conanfile + '\n self.cpp_info.builddirs = ["pkg/cmake"]' + + cmake_export = textwrap.dedent(""" + cmake_minimum_required(VERSION 3.15) + project(MyHello CXX) + + add_library(pkg src/pkg.cpp) + target_include_directories(pkg PUBLIC + $ + $ + ) + set_target_properties(pkg PROPERTIES PUBLIC_HEADER "include/pkg.h") + + install(TARGETS pkg EXPORT pkgConfig) + export(TARGETS pkg + NAMESPACE pkg:: + FILE "${CMAKE_CURRENT_BINARY_DIR}/pkgConfig.cmake" + ) + install(EXPORT pkgConfig + DESTINATION "pkg/cmake" + NAMESPACE pkg:: + ) + """) + + c.save({"conanfile.py": conanfile, + "CMakeLists.txt": cmake_export}) + + c.run("create . ") + c.run("upload * -r=default -c") + c.run("remove * -c") + + c2 = TestClient(servers=c.servers) + c2.run("new cmake_exe -d name=myapp -d version=0.1 -d requires=pkg/0.1") + c2.run('build .') + # Now it builds correctly without failing, because package is relocatable + + +@pytest.mark.tool("ninja") +# It is IMPORTANT to do cmake-3.27 AFTER ninja, otherwise CI injects another cmake +@pytest.mark.tool("cmake", "3.27") +def test_cmake_find_none_relocation_multi(): + # What happens for multi-config + c = TestClient(default_server_user=True) + c.run("new cmake_lib -d name=pkg -d version=0.1") + conanfile = textwrap.dedent(""" + import os, textwrap + from conan import ConanFile + from conan.tools.files import save, load + from conan.tools.cmake import CMakeToolchain, CMake, cmake_layout, CMakeDeps + + class pkgRecipe(ConanFile): + name = "pkg" + version = "0.1" + package_type = "library" + + settings = "os", "compiler", "build_type", "arch" + options = {"shared": [True, False], "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + + # Sources are located in the same place as this recipe, copy them to the recipe + exports_sources = "CMakeLists.txt", "src/*", "include/*" + implements = ["auto_shared_fpic"] + + generators = "CMakeToolchain", "CMakeDeps" + + def layout(self): + cmake_layout(self) + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def package(self): + cmake = CMake(self) + cmake.install() + p = os.path.join(self.package_folder, "pkg", "cmake", "pkgConfig.cmake") + new_config = textwrap.dedent(''' + # Create imported target pkg::pkg + add_library(pkg::pkg STATIC IMPORTED) + + if(NOT DEFINED CONAN_pkg_DIR_MULTI) + get_filename_component(CONAN_pkg_DIR_MULTI "${CMAKE_CURRENT_LIST_FILE}" PATH) + endif() + foreach(_IMPORT_PREFIX ${CONAN_pkg_DIR_MULTI}) + get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) + get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) + + set_target_properties(pkg::pkg PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${_IMPORT_PREFIX}/include" + ) + + # Load information for each installed configuration. + file(GLOB CONFIG_FILES "${_IMPORT_PREFIX}/pkg/cmake/pkgConfig-*.cmake") + foreach(f ${CONFIG_FILES}) + include(${f}) + endforeach() + endforeach() + ''') + save(self, p, new_config) + + def package_info(self): + self.cpp_info.libs = ["pkg"] + self.cpp_info.set_property("cmake_find_mode", "none") + self.cpp_info.builddirs = ["pkg/cmake"] + """) + + cmake_export = textwrap.dedent(""" + cmake_minimum_required(VERSION 3.15) + project(MyHello CXX) + + add_library(pkg src/pkg.cpp) + target_include_directories(pkg PUBLIC + $ + $ + ) + set_target_properties(pkg PROPERTIES PUBLIC_HEADER "include/pkg.h") + + install(TARGETS pkg EXPORT pkgConfig) + export(TARGETS pkg + NAMESPACE pkg:: + FILE "${CMAKE_CURRENT_BINARY_DIR}/pkgConfig.cmake" + ) + install(EXPORT pkgConfig + DESTINATION "pkg/cmake" + NAMESPACE pkg:: + ) + """) + + c.save({"conanfile.py": conanfile, + "CMakeLists.txt": cmake_export}) + + c.run("create .") + c.run("create . -s build_type=Debug") + c.run("upload * -r=default -c") + c.run("remove * -c") + + c2 = TestClient(servers=c.servers) + c2.run("new cmake_exe -d name=myapp -d version=0.1 -d requires=pkg/0.1") + env = r".\build\generators\conanbuild.bat &&" if platform.system() == "Windows" else "" + conf = ('-c tools.cmake.cmaketoolchain:generator="Ninja Multi-Config" ' + '-c tools.cmake.cmakedeps:new=will_break_next') + c2.run(f'install . {conf}') + c2.run(f'install . {conf} -s build_type=Debug') + c2.run_command(f"{env} cmake --preset conan-default") + c2.run_command(f"{env} cmake --build --preset conan-release") + c2.run_command(f"{env} cmake --build --preset conan-debug") diff --git a/test/functional/toolchains/cmake/test_cmake_multi.py b/test/functional/toolchains/cmake/test_cmake_multi.py new file mode 100644 index 00000000000..0407765420b --- /dev/null +++ b/test/functional/toolchains/cmake/test_cmake_multi.py @@ -0,0 +1,103 @@ +import os +import platform +import textwrap + +from conan.test.utils.tools import TestClient + + +def test_multi_cmake(): + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.cmake import CMakeToolchain, CMake, cmake_layout, CMakeDeps + import os + + + class multiRecipe(ConanFile): + settings = "os", "compiler", "build_type", "arch" + + exports_sources = "cmake_one/CMakeLists.txt", "cmake_two/CMakeLists.txt", "src_one/*", "src_two/*" + + def layout(self): + cmake_layout(self) + + def generate(self): + deps = CMakeDeps(self) + deps.generate() + tc = CMakeToolchain(self) + tc.generate() + + def build_one(self): + cmake = CMake(self) + cmake.configure(build_script_folder="cmake_one", subfolder="one") + cmake.build(subfolder="one") + # cmake.install(subfolder="one") + + def build_two(self): + cmake = CMake(self) + # CMAKE_PREFIX_PATH + cmake.configure(build_script_folder="cmake_two", subfolder="two") + cmake.build(subfolder="two") + + def build(self): + self.build_one() + self.build_two() + + def package(self): + cmake = CMake(self) + cmake.install(subfolder="two") + + def package_info(self): + self.cpp_info.libs = ["hello_two"] + self.cpp_info.includedirs = ['two/include'] + self.cpp_info.libdirs = ['two/lib'] + """) + + hello_cpp = textwrap.dedent(""" + #include + #include "hello_{name}.h" + + void hello_{name}() {{ + std::cout << "Hello, World {name}!" << std::endl; + }} + """) + + hello_h = textwrap.dedent(""" + #ifndef HELLO_{name}_H + #define HELLO_{name}_H + + void hello_{name}(); + + #endif + """) + + cmakelist = textwrap.dedent(""" + cmake_minimum_required(VERSION 3.15) + project(hello_{name} LANGUAGES CXX) + + add_library(hello_{name} ../src_{name}/hello_{name}.cpp) + target_include_directories(hello_{name} PUBLIC ../src_{name}) + + set_target_properties(hello_{name} PROPERTIES PUBLIC_HEADER "../src_{name}/hello_{name}.h") + install(TARGETS hello_{name}) + """) + + client = TestClient() + client.save({"conanfile.py": conanfile, + "cmake_one/CMakeLists.txt": cmakelist.format(name="one"), + "cmake_two/CMakeLists.txt": cmakelist.format(name="two"), + "src_one/hello_one.h": hello_h.format(name="one"), + "src_one/hello_one.cpp": hello_cpp.format(name="one"), + "src_two/hello_two.h": hello_h.format(name="two"), + "src_two/hello_two.cpp": hello_cpp.format(name="two")}) + + client.run("create . --name=multi --version=0.1") + file_ext = '.a' if platform.system() != "Windows" else '.lib' + lib_prefix = 'lib' if platform.system() != "Windows" else '' + + assert "multi/0.1: package(): Packaged 1 '.h' file: hello_two.h" in client.out + assert f"multi/0.1: package(): Packaged 1 '{file_ext}' file: {lib_prefix}hello_two{file_ext}" in client.out + package_folder = client.created_layout().package() + assert not os.path.exists(os.path.join(package_folder, "one", "include", "hello_one.h")) + assert not os.path.exists(os.path.join(package_folder, "one", "lib", f"{lib_prefix}hello_one{file_ext}")) + assert os.path.exists(os.path.join(package_folder, "two", "include", "hello_two.h")) + assert os.path.exists(os.path.join(package_folder, "two", "lib", f"{lib_prefix}hello_two{file_ext}")) diff --git a/test/functional/toolchains/cmake/test_cmake_toolchain.py b/test/functional/toolchains/cmake/test_cmake_toolchain.py index 2a2f7b3cad1..d31dd380788 100644 --- a/test/functional/toolchains/cmake/test_cmake_toolchain.py +++ b/test/functional/toolchains/cmake/test_cmake_toolchain.py @@ -760,11 +760,11 @@ def package(self): """) client.save({"conanfile.py": conanfile, - "libfoo.so": "", - "foobin": "", - "host_profile": host_profile, - "build_profile": build_profile - }) + "libfoo.so": "", + "foobin": "", + "host_profile": host_profile, + "build_profile": build_profile + }) client.run("create . -pr:b build_profile -pr:h build_profile") build_context_package_folder = re.search(r"Package folder ([\w\W]+).conan2([\w\W]+)", str(client.out)).group(2).strip() @@ -1345,7 +1345,7 @@ def build(self): c.save({"android": android, "conanfile.py": conanfile, - "CMakeLists.txt": cmake,}) + "CMakeLists.txt": cmake}) # first run works ok c.run('build . --profile:host=android') assert 'sdk: 1.0.0' in c.out @@ -1442,3 +1442,77 @@ def build(self): c.run("build . ") # Release breaks the check assert "Looking for pow - found" in c.out # And it no longer crashes + + +@pytest.mark.skipif(platform.system() != "Linux", reason="Linker scripts in Linux only") +def test_cmake_linker_scripts(): + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout + + class Test(ConanFile): + exports_sources = "CMakeLists.txt", "src/*" + settings = "os", "compiler", "arch", "build_type" + generators = "CMakeToolchain" + + def layout(self): + cmake_layout(self) + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + """) + + main = "int main() {}" + + cmakelists = textwrap.dedent(""" + set(CMAKE_CXX_COMPILER_WORKS 1) + set(CMAKE_CXX_ABI_COMPILED 1) + cmake_minimum_required(VERSION 3.15) + project(Test CXX) + add_executable(example main.cpp) + """) + profile = textwrap.dedent(""" + include(default) + [conf] + tools.build:linker_scripts=['{{profile_dir}}/mylinkscript.ld'] + """) + + c = TestClient() + c.save({"conanfile.py": conanfile, + "CMakeLists.txt": cmakelists, + "main.cpp": main, + "mylinkscript.ld": "", + "profile": profile}) + c.run('build . -pr=profile', assert_error=True) + # This error means the linker script was fonud and loaded, it failed because empty + assert "PHDR segment not covered by LOAD segment" in c.out + + +def test_cmake_toolchain_verbosity_propagation(): + t = TestClient(light=True) + t.save({"conanfile.py": GenConanfile("mylib", "1.0")}) + t.run("create .") + t.run("install --requires=mylib/1.0 -g CMakeToolchain") + toolchain = t.load("conan_toolchain.cmake") + assert 'set(CMAKE_VERBOSE_MAKEFILE' not in toolchain + assert 'set(CMAKE_MESSAGE_LOG_LEVEL' not in toolchain + + t.run("install --requires=mylib/1.0 -g CMakeToolchain -c tools.build:verbosity=verbose " + "-c tools.compilation:verbosity=verbose") + toolchain = t.load("conan_toolchain.cmake") + assert 'set(CMAKE_VERBOSE_MAKEFILE "ON"' in toolchain + assert 'set(CMAKE_MESSAGE_LOG_LEVEL "VERBOSE"' in toolchain + + t.run("install --requires=mylib/1.0 -g CMakeToolchain -c tools.build:verbosity=quiet " + "-c tools.compilation:verbosity=quiet") + toolchain = t.load("conan_toolchain.cmake") + assert 'set(CMAKE_MESSAGE_LOG_LEVEL "ERROR"' in toolchain + + # Extra variables have preference + t.run("install --requires=mylib/1.0 -g CMakeToolchain -c tools.compilation:verbosity=quiet " + "-c tools.cmake.cmaketoolchain:extra_variables=\"{'CMAKE_MESSAGE_LOG_LEVEL': 'WARNING'}\"") + toolchain = t.load("conan_toolchain.cmake") + # assert 'set(CMAKE_VERBOSE_MAKEFILE "OFF"' in toolchain + assert 'set(CMAKE_MESSAGE_LOG_LEVEL "WARNING"' in toolchain diff --git a/test/functional/toolchains/cmake/test_cmake_toolchain_vxworks_clang.py b/test/functional/toolchains/cmake/test_cmake_toolchain_vxworks_clang.py deleted file mode 100644 index 2a833dbd5ff..00000000000 --- a/test/functional/toolchains/cmake/test_cmake_toolchain_vxworks_clang.py +++ /dev/null @@ -1,127 +0,0 @@ -import textwrap - -import pytest - -from conan.test.assets.cmake import gen_cmakelists -from conan.test.assets.sources import gen_function_cpp -from conan.test.utils.tools import TestClient - - -@pytest.fixture -def client(): - c = TestClient() - clang_profile = textwrap.dedent(""" - [settings] - arch=armv7 - build_type=RelWithDebInfo - compiler=clang - compiler.libcxx=libstdc++11 - compiler.version=12 - os=VxWorks - os.version=7 - - [buildenv] - CC=clang - CXX=clang++ - """) - conanfile = textwrap.dedent(""" - import os - from conan import ConanFile - from conan.tools.cmake import CMake, cmake_layout - - class Pkg(ConanFile): - settings = "os", "compiler", "build_type", "arch" - exports_sources = "*" - generators = "CMakeToolchain" - - def layout(self): - cmake_layout(self) - - def build(self): - cmake = CMake(self) - cmake.configure() - cmake.build() - cmd = os.path.join(self.cpp.build.bindirs[0], "my_app") - self.run('readelf -s ' + cmd, env=["conanrunenv"]) - """) - toolchain_file = textwrap.dedent(""" - set(vsb "/vsb") # location of VxWorks Source Build (vsb) - set(CMAKE_LINKER "ldarm") - set(CMAKE_CXX_LINK_EXECUTABLE " -o " CACHE STRING "Workaround for clang, use GNU linker not clang/lld" FORCE) - set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES - ${vsb}/usr/h - ${vsb}/usr/h/public - ${vsb}/h/config - ${vsb}/share/h/public - ) - add_compile_definitions( - ARMEL - CPU=_VX_ARMARCH7 - _REENTRANT - INET - _VSB_CONFIG_FILE="${vsb}/h/config/vsbConfig.h" - ) - add_compile_options( - -fno-builtin - -pipe - ) - add_compile_definitions( - _C99 - _HAS_C9X - _VX_CPU=_VX_ARMARCH7 - __RTP__ - TOOL=llvm - TOOL_FAMILY=llvm - CPU_FAMILY=ARM - __ELF__ - __vxworks - __VXWORKS__ - _USE_INIT_ARRAY - ) - add_compile_options( - -fasm - -fomit-frame-pointer - -ffunction-sections - -fdata-sections - --target=arm-eabi - -mabi=aapcs - -mcpu=cortex-a9 - -mfloat-abi=hard - -mfpu=vfpv3 - -mlittle-endian - -mno-implicit-float - -nostdinc++ - -nostdlibinc - -march=armv7 - -gdwarf-3 - -mllvm - -two-entry-phi-node-folding-threshold=2 - ) - add_link_options( - --target2=rel - ) - link_directories( - ${vsb}/usr/lib/common - ) - - set(CMAKE_CXX_STANDARD 14) - set(CMAKE_C_STANDARD_LIBRARIES "--start-group --as-needed -lllvm -lc -lc_internal --end-group") - set(CMAKE_CXX_STANDARD_LIBRARIES "-lnet --start-group --as-needed -lc -lc_internal -lllvm -lcplusplus -lllvmcplus -ldl --end-group") - set(CMAKE_EXE_LINKER_FLAGS "--defsym __wrs_rtp_base=0x80000000 -u __wr_need_frame_add -u __tls__ -T${vsb}/usr/ldscripts/rtp.ld -static -EL ${vsb}/usr/lib/common/crt0.o" CACHE STRING "" FORCE) - """) - c.save({"conanfile.py": conanfile, - "clang": clang_profile, - "toolchain-vxworks.cmake": toolchain_file, - "CMakeLists.txt": gen_cmakelists(appname="my_app", appsources=["src/main.cpp"]), - "src/main.cpp": gen_function_cpp(name="main")}) - return c - - -@pytest.mark.tool("cmake") -@pytest.mark.tool("clang", "12") -def test_clang_cmake_ninja(client): - client.run("create . --name=pkg --version=0.1 -pr=clang " - "-c tools.cmake.cmaketoolchain:generator=Ninja " - "-c tools.cmake.cmaketoolchain:toolchain_file=../../toolchain-vxworks.cmake") - assert 'cmake -G "Ninja"' in client.out - assert "__wrs_rtp_" in client.out diff --git a/test/functional/toolchains/cmake/test_cmake_toolchain_win_clang.py b/test/functional/toolchains/cmake/test_cmake_toolchain_win_clang.py index c11b2084661..900872e07d9 100644 --- a/test/functional/toolchains/cmake/test_cmake_toolchain_win_clang.py +++ b/test/functional/toolchains/cmake/test_cmake_toolchain_win_clang.py @@ -7,6 +7,7 @@ from conan.test.assets.cmake import gen_cmakelists from conan.test.assets.sources import gen_function_cpp, gen_function_c +from test.conftest import tools_locations from test.functional.utils import check_vs_runtime, check_exe_run from conan.test.utils.tools import TestClient @@ -56,8 +57,14 @@ def package_info(self): cmd = os.path.join(self.package_folder, "bin", "my_app") self.output.info("MYCMD={}!".format(os.path.abspath(cmd))) """) + clangpath = tools_locations["clang"]["20"]["path"]["Windows"] + llvm_clang_path = textwrap.dedent(f""" + [buildenv] + PATH=+(path){clangpath} + """) c.save({"conanfile.py": conanfile, "clang": clang_profile, + "clang_path": llvm_clang_path, "CMakeLists.txt": gen_cmakelists(appname="my_app", appsources=["src/main.cpp"], install=True), "src/main.cpp": gen_function_cpp(name="main")}) @@ -65,7 +72,7 @@ def package_info(self): @pytest.mark.tool("cmake") -@pytest.mark.tool("clang", "18") +@pytest.mark.tool("clang", "20") @pytest.mark.skipif(platform.system() != "Windows", reason="requires Win") class TestLLVMClang: """ External LLVM/clang, with different CMake generators @@ -74,19 +81,18 @@ class TestLLVMClang: @pytest.mark.tool("mingw64") @pytest.mark.tool("visual_studio", "17") - @pytest.mark.tool("clang", "18") # repeated, for priority over the mingw64 clang @pytest.mark.parametrize("runtime", ["static", "dynamic"]) def test_clang_mingw(self, client, runtime): """ compiling with an LLVM-clang installed, which uses by default the VS runtime """ - client.run("create . --name=pkg --version=0.1 -pr=clang " + client.run("create . --name=pkg --version=0.1 -pr=clang -pr=clang_path " "-s compiler.runtime_version=v144 " "-s compiler.runtime={}".format(runtime)) # clang compilations in Windows will use MinGW Makefiles by default assert 'cmake -G "MinGW Makefiles"' in client.out assert "GNU-like command-line" in client.out - assert "main __clang_major__18" in client.out + assert "main __clang_major__20" in client.out assert "main _MSC_VER194" in client.out assert "main _MSVC_LANG2014" in client.out assert "main _M_X64 defined" in client.out @@ -102,13 +108,13 @@ def test_clang_mingw(self, client, runtime): @pytest.mark.tool("visual_studio", "17") @pytest.mark.parametrize("generator", ["Ninja", "NMake Makefiles"]) def test_clang_cmake_ninja_nmake(self, client, generator): - client.run("create . --name=pkg --version=0.1 -pr=clang -s compiler.runtime=dynamic " - "-s compiler.runtime_version=v144 " + client.run("create . --name=pkg --version=0.1 -pr=clang -pr=clang_path " + "-s compiler.runtime=dynamic -s compiler.runtime_version=v144 " '-c tools.cmake.cmaketoolchain:generator="{}"'.format(generator)) assert 'cmake -G "{}"'.format(generator) in client.out assert "GNU-like command-line" in client.out - assert "main __clang_major__18" in client.out + assert "main __clang_major__20" in client.out assert "main _MSC_VER194" in client.out assert "main _MSVC_LANG2014" in client.out assert "main _M_X64 defined" in client.out @@ -126,13 +132,13 @@ def test_clang_cmake_ninja_clang_cl(self, client): """) client.save({"comp_exes": pr}) - client.run("create . --name=pkg --version=0.1 -pr=clang -pr=comp_exes " + client.run("create . --name=pkg --version=0.1 -pr=clang -pr=clang_path -pr=comp_exes " "-s compiler.runtime=dynamic -s compiler.runtime_version=v144 " '-c tools.cmake.cmaketoolchain:generator="Ninja"') assert 'cmake -G "Ninja"' in client.out assert "MSVC-like command-line" in client.out - assert "main __clang_major__18" in client.out + assert "main __clang_major__20" in client.out assert "main _MSC_VER194" in client.out assert "main _MSVC_LANG2014" in client.out assert "main _M_X64 defined" in client.out @@ -143,19 +149,19 @@ def test_clang_cmake_ninja_clang_cl(self, client): @pytest.mark.tool("ninja") @pytest.mark.tool("visual_studio", "17") - @pytest.mark.tool("clang", "18") # repeated, for priority over the mingw64 clang def test_clang_cmake_runtime_version(self, client): generator = "Ninja" # Make sure that normal CMakeLists with verify=False works client.save({"CMakeLists.txt": gen_cmakelists(verify=False, appname="my_app", appsources=["src/main.cpp"], install=True)}) - client.run("create . --name=pkg --version=0.1 -pr=clang -s compiler.runtime=dynamic -s compiler.cppstd=17 " - "-s compiler.runtime_version=v144 " + client.run("create . --name=pkg --version=0.1 -pr=clang -pr=clang_path " + "-s compiler.runtime=dynamic " + "-s compiler.cppstd=17 -s compiler.runtime_version=v144 " '-c tools.cmake.cmaketoolchain:generator="{}"'.format(generator)) assert 'cmake -G "{}"'.format(generator) in client.out assert "GNU-like command-line" in client.out - assert "main __clang_major__18" in client.out + assert "main __clang_major__20" in client.out # Check this! Clang compiler in Windows is reporting MSC_VER and MSVC_LANG! assert "main _MSC_VER194" in client.out assert "main _MSVC_LANG2017" in client.out @@ -181,7 +187,7 @@ def test_clang_visual_studio_generator(self, client): '-c tools.cmake.cmaketoolchain:generator="{}"'.format(generator)) assert 'cmake -G "{}"'.format(generator) in client.out assert "MSVC-like command-line" in client.out - assert "main __clang_major__19" in client.out + assert "main __clang_major__19" in client.out # The one inside VS is clang 19 # Check this! Clang compiler in Windows is reporting MSC_VER and MSVC_LANG! # CI forced the installation of 19.38, seems to prevail there assert "main _MSC_VER19" in client.out @@ -288,7 +294,7 @@ def test_clang_pure_c(self, client): @pytest.mark.tool("cmake") @pytest.mark.tool("ninja") -@pytest.mark.tool("clang", "18") +@pytest.mark.tool("clang", "20") @pytest.mark.skipif(platform.system() != "Windows", reason="requires Win") def test_error_clang_cmake_ninja_custom_cxx(client): clang_profile = textwrap.dedent(""" @@ -297,7 +303,7 @@ def test_error_clang_cmake_ninja_custom_cxx(client): arch=x86_64 build_type=Release compiler=clang - compiler.version=18 + compiler.version=20 [buildenv] CXX=/no/exist/clang++ """) diff --git a/test/functional/toolchains/cmake/test_cmake_toolchain_xcode_flags.py b/test/functional/toolchains/cmake/test_cmake_toolchain_xcode_flags.py index 16f0881f0e9..388d180c649 100644 --- a/test/functional/toolchains/cmake/test_cmake_toolchain_xcode_flags.py +++ b/test/functional/toolchains/cmake/test_cmake_toolchain_xcode_flags.py @@ -30,7 +30,6 @@ def test_cmake_apple_bitcode_arc_and_visibility_flags_enabled(op_system, os_vers os.sdk={} arch={} [conf] - tools.apple:enable_bitcode=True tools.apple:enable_arc=True tools.apple:enable_visibility=True """.format(op_system, os_version, sdk, arch)) @@ -41,10 +40,6 @@ def test_cmake_apple_bitcode_arc_and_visibility_flags_enabled(op_system, os_vers _add_message_status_flags(client) client.run("install . --profile:build=default --profile:host=host") toolchain = client.load(os.path.join("build", "Release", "generators", "conan_toolchain.cmake")) - # bitcode - assert 'set(CMAKE_XCODE_ATTRIBUTE_ENABLE_BITCODE "YES")' in toolchain - assert 'set(CMAKE_XCODE_ATTRIBUTE_BITCODE_GENERATION_MODE "bitcode")' in toolchain - assert 'set(BITCODE "-fembed-bitcode")' in toolchain # arc assert 'set(FOBJC_ARC "-fobjc-arc")' in toolchain assert 'set(CMAKE_XCODE_ATTRIBUTE_CLANG_ENABLE_OBJC_ARC "YES")' in toolchain @@ -54,10 +49,10 @@ def test_cmake_apple_bitcode_arc_and_visibility_flags_enabled(op_system, os_vers client.run("create . --profile:build=default --profile:host=host -tf=") # flags - assert "-- CONAN_C_FLAGS: -fembed-bitcode -fvisibility=default" in client.out - assert "-- CONAN_CXX_FLAGS: -fembed-bitcode -fvisibility=default" in client.out - assert "-- CONAN_OBJC_FLAGS: -fembed-bitcode -fvisibility=default -fobjc-arc" in client.out - assert "-- CONAN_OBJCXX_FLAGS: -fembed-bitcode -fvisibility=default -fobjc-arc" in client.out + assert "-- CONAN_C_FLAGS: -fvisibility=default" in client.out + assert "-- CONAN_CXX_FLAGS: -fvisibility=default" in client.out + assert "-- CONAN_OBJC_FLAGS: -fvisibility=default -fobjc-arc" in client.out + assert "-- CONAN_OBJCXX_FLAGS: -fvisibility=default -fobjc-arc" in client.out assert "[100%] Built target hello" in client.out diff --git a/test/functional/toolchains/cmake/test_cmaketoolchain_paths.py b/test/functional/toolchains/cmake/test_cmaketoolchain_paths.py index c77fc2366e1..638208adf61 100644 --- a/test/functional/toolchains/cmake/test_cmaketoolchain_paths.py +++ b/test/functional/toolchains/cmake/test_cmaketoolchain_paths.py @@ -214,7 +214,7 @@ def package_info(self): FILE "${CMAKE_CURRENT_BINARY_DIR}/helloConfig.cmake" ) install(EXPORT helloConfig - DESTINATION "${CMAKE_INSTALL_PREFIX}/hello/cmake" + DESTINATION "hello/cmake" NAMESPACE hello:: ) """) diff --git a/test/functional/toolchains/cmake/test_cps.py b/test/functional/toolchains/cmake/test_cps.py new file mode 100644 index 00000000000..527044e134e --- /dev/null +++ b/test/functional/toolchains/cmake/test_cps.py @@ -0,0 +1,559 @@ +import os +import platform +import shutil +import textwrap + +import pytest + +from conan.internal.api.new.cmake_lib import test_conanfile_v2 +from conan.test.assets.sources import gen_function_h, gen_function_cpp +from conan.test.utils.tools import TestClient + + +@pytest.mark.tool("cmake", "4.2") +@pytest.mark.parametrize("shared", [False, True]) +def test_cps(shared): + c = TestClient() + c.run("new cmake_lib") + conanfile = textwrap.dedent("""\ + from conan import ConanFile + from conan.tools.cmake import CMakeToolchain, CMake, cmake_layout + from conan.cps import CPS + import glob + + class mypkgRecipe(ConanFile): + name = "mypkg" + version = "0.1" + package_type = "library" + + settings = "os", "compiler", "build_type", "arch" + options = {"shared": [True, False], "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + + exports_sources = "CMakeLists.txt", "src/*", "include/*" + implements = ["auto_shared_fpic"] + generators = "CMakeToolchain" + + def layout(self): + cmake_layout(self) + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def package(self): + cmake = CMake(self) + cmake.install() + + def package_info(self): + file_loc = glob.glob("**/mypkg.cps", recursive=True) + self.cpp_info = CPS.load(file_loc[0]).to_conan() + """) + + cmake = textwrap.dedent("""\ + cmake_minimum_required(VERSION 4.2) + project(mypkg CXX) + + set(CMAKE_EXPERIMENTAL_EXPORT_PACKAGE_INFO "b80be207-778e-46ba-8080-b23bba22639e") + + add_library(mypkg src/mypkg.cpp) + target_include_directories(mypkg PUBLIC + $ + $) + + set_target_properties(mypkg PROPERTIES PUBLIC_HEADER "include/mypkg.h") + install(TARGETS mypkg EXPORT mypkg) + + install(PACKAGE_INFO mypkg EXPORT mypkg) + """) + + # First, try with the standard mypkg-config.cmake consumption + c.save({"conanfile.py": conanfile, + "CMakeLists.txt": cmake}) + + shared_arg = "-o &:shared=True" if shared else "" + c.run(f"create {shared_arg}") + assert "mypkg/0.1: Hello World Release!" in c.out + + # Lets consume directly with CPS + test_cmake = textwrap.dedent("""\ + cmake_minimum_required(VERSION 4.2) + project(PackageTest CXX) + + set(CMAKE_EXPERIMENTAL_FIND_CPS_PACKAGES e82e467b-f997-4464-8ace-b00808fff261) + + find_package(mypkg CONFIG REQUIRED) + + add_executable(example src/example.cpp) + target_link_libraries(example mypkg::mypkg) + """) + test_conanfile = textwrap.dedent("""\ + import os + + from conan import ConanFile + from conan.tools.cmake import CMake, cmake_layout, CMakeToolchain, CMakeConfigDeps + from conan.tools.build import can_run + + + class TestConan(ConanFile): + settings = "os", "compiler", "build_type", "arch" + + def requirements(self): + self.requires(self.tested_reference_str) + + def generate(self): + deps = CMakeConfigDeps(self) + deps.set_property("mypkg", "cmake_find_mode", "none") + deps.generate() + tc = CMakeToolchain(self) + tc.generate() + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def layout(self): + cmake_layout(self) + + def test(self): + if can_run(self): + cmd = os.path.join(self.cpp.build.bindir, "example") + self.run(cmd, env="conanrun") + """) + shutil.rmtree(os.path.join(c.current_folder, "test_package", "build")) + c.save({"test_package/conanfile.py": test_conanfile, + "test_package/CMakeLists.txt": test_cmake}) + c.run(f"create {shared_arg} --build=never") + assert "mypkg/0.1: Hello World Release!" in c.out + + +@pytest.mark.tool("cmake", "4.2") +@pytest.mark.parametrize("shared", [False, True]) +def test_cps_components(shared): + c = TestClient() + c.run("new cmake_lib") + conanfile = textwrap.dedent("""\ + from conan import ConanFile + from conan.tools.cmake import CMakeToolchain, CMake, cmake_layout + from conan.cps import CPS + import glob + + class mypkgRecipe(ConanFile): + name = "mypkg" + version = "0.1" + package_type = "library" + + settings = "os", "compiler", "build_type", "arch" + options = {"shared": [True, False], "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + + exports_sources = "CMakeLists.txt", "src/*", "include/*" + implements = ["auto_shared_fpic"] + generators = "CMakeToolchain" + + def layout(self): + cmake_layout(self) + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def package(self): + cmake = CMake(self) + cmake.install() + + def package_info(self): + file_loc = glob.glob("**/mypkg.cps", recursive=True) + cps_data = CPS.load(file_loc[0]) + # Convert CPS to cpp_info with components + self.cpp_info = cps_data.to_conan() + """) + + cmake = textwrap.dedent("""\ + cmake_minimum_required(VERSION 4.2) + project(mypkg CXX) + + set(CMAKE_EXPERIMENTAL_EXPORT_PACKAGE_INFO "b80be207-778e-46ba-8080-b23bba22639e") + + # First library: core + add_library(mypkg_core src/mypkg_core.cpp) + target_include_directories(mypkg_core PUBLIC + $ + $) + set_target_properties(mypkg_core PROPERTIES PUBLIC_HEADER "include/mypkg_core.h") + + # Second library: utils (independent from core) + add_library(mypkg_utils src/mypkg_utils.cpp) + target_include_directories(mypkg_utils PUBLIC + $ + $) + set_target_properties(mypkg_utils PROPERTIES PUBLIC_HEADER "include/mypkg_utils.h") + + install(TARGETS mypkg_core mypkg_utils EXPORT mypkg) + + install(PACKAGE_INFO mypkg EXPORT mypkg) + """) + + # Create source files for both libraries + core_cpp = gen_function_cpp(name="mypkg_core") + core_h = gen_function_h(name="mypkg_core") + utils_cpp = gen_function_cpp(name="mypkg_utils") + utils_h = gen_function_h(name="mypkg_utils") + + # Create test_package files for the two components + test_package_cmake = textwrap.dedent("""\ + cmake_minimum_required(VERSION 3.15) + project(PackageTest CXX) + + set(CMAKE_EXPERIMENTAL_FIND_CPS_PACKAGES e82e467b-f997-4464-8ace-b00808fff261) + + find_package(mypkg CONFIG REQUIRED) + + add_executable(example src/example.cpp) + target_link_libraries(example mypkg::mypkg_core mypkg::mypkg_utils) + """) + + test_package_example = textwrap.dedent("""\ + #include "mypkg_core.h" + #include "mypkg_utils.h" + + int main() { + mypkg_core(); + mypkg_utils(); + return 0; + } + """) + + # First, try with the standard mypkg-config.cmake consumption + c.save({"conanfile.py": conanfile, + "CMakeLists.txt": cmake, + "src/mypkg_core.cpp": core_cpp, + "include/mypkg_core.h": core_h, + "src/mypkg_utils.cpp": utils_cpp, + "include/mypkg_utils.h": utils_h, + "test_package/CMakeLists.txt": test_package_cmake, + "test_package/src/example.cpp": test_package_example}) + + shared_arg = "-o &:shared=True" if shared else "" + c.run(f"create {shared_arg}") + assert "mypkg_core: Release!" in c.out + assert "mypkg_utils: Release!" in c.out + + test_conanfile = textwrap.dedent("""\ + import os + + from conan import ConanFile + from conan.tools.cmake import CMake, cmake_layout, CMakeToolchain, CMakeConfigDeps + from conan.tools.build import can_run + + + class TestConan(ConanFile): + settings = "os", "compiler", "build_type", "arch" + + def requirements(self): + self.requires(self.tested_reference_str) + + def generate(self): + deps = CMakeConfigDeps(self) + deps.set_property("mypkg", "cmake_find_mode", "none") + deps.generate() + tc = CMakeToolchain(self) + tc.generate() + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def layout(self): + cmake_layout(self) + + def test(self): + if can_run(self): + cmd = os.path.join(self.cpp.build.bindir, "example") + self.run(cmd, env="conanrun") + """) + shutil.rmtree(os.path.join(c.current_folder, "test_package", "build")) + c.save({"test_package/conanfile.py": test_conanfile}) + c.run(f"create {shared_arg} --build=never") + assert "mypkg_core: Release!" in c.out + assert "mypkg_utils: Release!" in c.out + + +@pytest.mark.tool("cmake", "4.2") +@pytest.mark.parametrize("kind", ["static_public", "static_private", "shared_private"]) +def test_cps_components_requires(kind): + if kind == "shared_private" and platform.system() == "Linux": + pytest.skip("CPS still doesn't support this case") + c = TestClient() + conanfile = textwrap.dedent("""\ + from conan import ConanFile + from conan.tools.cmake import CMakeToolchain, CMake, cmake_layout + from conan.cps import CPS + import glob + + class Recipe(ConanFile): + name = "{name}" + version = "0.1" + package_type = "library" + + settings = "os", "compiler", "build_type", "arch" + options = {{"shared": [True, False], "fPIC": [True, False]}} + default_options = {{"shared": False, "fPIC": True}} + + exports_sources = "CMakeLists.txt", "src/*", "include/*" + implements = ["auto_shared_fpic"] + generators = "CMakeToolchain", "CMakeConfigDeps" + + def requirements(self): + {requires} + pass + + def layout(self): + cmake_layout(self) + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def package(self): + cmake = CMake(self) + cmake.install() + + def package_info(self): + file_loc = glob.glob("**/{name}.cps", recursive=True) + cps_data = CPS.load(file_loc[0]) + # Convert CPS to cpp_info with components + self.cpp_info = cps_data.to_conan() + """) + + lib_type = "PUBLIC" if "public" in kind else "PRIVATE" + cmake = textwrap.dedent("""\ + cmake_minimum_required(VERSION 4.2) + project({name} CXX) + + set(CMAKE_EXPERIMENTAL_EXPORT_PACKAGE_INFO "b80be207-778e-46ba-8080-b23bba22639e") + + {find} + # First library: core + add_library({name}_core src/{name}_core.cpp) + target_include_directories({name}_core PUBLIC + $ + $) + set_target_properties({name}_core PROPERTIES PUBLIC_HEADER "include/{name}_core.h") + {deps} + + # Second library: utils (independent from core) + add_library({name}_utils src/{name}_utils.cpp) + + target_include_directories({name}_utils PUBLIC + $ + $) + set_target_properties({name}_utils PROPERTIES PUBLIC_HEADER "include/{name}_utils.h") + + target_link_libraries({name}_utils {lib_type} {name}_core) + + install(TARGETS {name}_core EXPORT {name} + PUBLIC_HEADER DESTINATION ${{CMAKE_INSTALL_INCLUDEDIR}}/core) + install(TARGETS {name}_utils EXPORT {name} + PUBLIC_HEADER DESTINATION ${{CMAKE_INSTALL_INCLUDEDIR}}/utils) + + install(PACKAGE_INFO {name} EXPORT {name}) + """) + + # Create source files for both libraries + core_cpp = gen_function_cpp(name="liba_core") + core_h = gen_function_h(name="liba_core") + utils_cpp = gen_function_cpp(name="liba_utils", includes=["liba_core"], + calls=["liba_core"]) + if "public" in kind: + utils_h = gen_function_h(name="liba_utils", includes=["liba_core"]) + else: + utils_h = gen_function_h(name="liba_utils") + + # Create test_package files for the two components + test_package_cmake = textwrap.dedent("""\ + cmake_minimum_required(VERSION 3.15) + project(PackageTest CXX) + + set(CMAKE_EXPERIMENTAL_FIND_CPS_PACKAGES e82e467b-f997-4464-8ace-b00808fff261) + + find_package({name} CONFIG REQUIRED) + + add_executable(example src/example.cpp) + target_link_libraries(example {name}::{name}_utils) + """) + + test_package_example = textwrap.dedent("""\ + #include "{name}_utils.h" + + int main() {{ + {name}_utils(); + return 0; + }} + """) + test_conanfile = test_conanfile_v2.replace("{{package_name}}", "") + test_conanfile = test_conanfile.replace("CMakeDeps", "CMakeConfigDeps") + # First, try with the standard liba-config.cmake consumption + c.save({"conanfile.py": conanfile.format(name="liba", requires=""), + "CMakeLists.txt": cmake.format(name="liba", lib_type=lib_type, deps="", find=""), + "src/liba_core.cpp": core_cpp, + "include/liba_core.h": core_h, + "src/liba_utils.cpp": utils_cpp, + "include/liba_utils.h": utils_h, + "test_package/conanfile.py": test_conanfile, + "test_package/CMakeLists.txt": test_package_cmake.format(name="liba"), + "test_package/src/example.cpp": test_package_example.format(name="liba")}) + + shared_arg = "-o *:shared=True" if "shared" in kind else "" + c.run(f"create {shared_arg}") + assert "liba_core: Release!" in c.out + assert "liba_utils: Release!" in c.out + + test_conanfile_cps = textwrap.dedent("""\ + import os + + from conan import ConanFile + from conan.tools.cmake import CMake, cmake_layout, CMakeToolchain, CMakeConfigDeps + from conan.tools.build import can_run + + + class TestConan(ConanFile): + settings = "os", "compiler", "build_type", "arch" + + def requirements(self): + self.requires(self.tested_reference_str) + + def generate(self): + deps = CMakeConfigDeps(self) + deps.set_property("liba", "cmake_find_mode", "none") + deps.set_property("libb", "cmake_find_mode", "none") + deps.generate() + tc = CMakeToolchain(self) + tc.generate() + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def layout(self): + cmake_layout(self) + + def test(self): + if can_run(self): + cmd = os.path.join(self.cpp.build.bindir, "example") + self.run(cmd, env="conanrun") + """) + shutil.rmtree(os.path.join(c.current_folder, "test_package", "build")) + c.save({"test_package/conanfile.py": test_conanfile_cps}) + c.run(f"create {shared_arg} --build=never") + assert "liba_core: Release!" in c.out + assert "liba_utils: Release!" in c.out + + # Now a second level + core_cpp = gen_function_cpp(name="libb_core", includes=["liba_utils"], calls=["liba_utils"]) + if "public" in kind: + core_h = gen_function_h(name="libb_core", includes=["liba_utils"]) + else: + core_h = gen_function_h(name="libb_core") + utils_cpp = gen_function_cpp(name="libb_utils", includes=["libb_core"], + calls=["libb_core"]) + if "public" in kind: + utils_h = gen_function_h(name="libb_utils", includes=["libb_core"]) + else: + utils_h = gen_function_h(name="libb_utils") + + deps = f'target_link_libraries(libb_core {lib_type} liba::liba_utils)' + find = f'find_package(liba CONFIG REQUIRED)' + transitive_headers = "public" in kind + requires = f'self.requires("liba/0.1", transitive_headers={transitive_headers})' + c.save({"conanfile.py": conanfile.format(name="libb", requires=requires), + "CMakeLists.txt": cmake.format(name="libb", lib_type=lib_type, deps=deps, find=find), + "src/libb_core.cpp": core_cpp, + "include/libb_core.h": core_h, + "src/libb_utils.cpp": utils_cpp, + "include/libb_utils.h": utils_h, + "test_package/conanfile.py": test_conanfile, + "test_package/CMakeLists.txt": test_package_cmake.format(name="libb"), + "test_package/src/example.cpp": test_package_example.format(name="libb")}, + clean_first=True) + + c.run(f"create {shared_arg}") + assert "libb_core: Release!" in c.out + assert "libb_utils: Release!" in c.out + assert "liba_core: Release!" in c.out + assert "liba_utils: Release!" in c.out + + c.save({"test_package/conanfile.py": test_conanfile_cps}) + c.run(f"create {shared_arg} --build=never") + assert "libb_core: Release!" in c.out + assert "libb_utils: Release!" in c.out + assert "liba_core: Release!" in c.out + assert "liba_utils: Release!" in c.out + + +@pytest.mark.skip(reason="Just to report to CMake upstream, and CPS feature request") +@pytest.mark.tool("cmake", "4.2") +def test_pure_cmake_shared(): + c = TestClient() + + cmake = textwrap.dedent("""\ + cmake_minimum_required(VERSION 4.2) + project(myproj CXX) + + set(CMAKE_EXPERIMENTAL_EXPORT_PACKAGE_INFO "b80be207-778e-46ba-8080-b23bba22639e") + + # First library: core + add_library(mypkg_core src/mypkg_core.cpp) + target_include_directories(mypkg_core PUBLIC + $ + $) + set_target_properties(mypkg_core PROPERTIES PUBLIC_HEADER "include/mypkg_core.h") + + # Second library: utils + add_library(mypkg_utils src/mypkg_utils.cpp) + + target_include_directories(mypkg_utils PUBLIC + $ + $) + set_target_properties(mypkg_utils PROPERTIES PUBLIC_HEADER "include/mypkg_utils.h") + + target_link_libraries(mypkg_utils PRIVATE mypkg_core) + + install(TARGETS mypkg_core EXPORT mypkg + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/core) + install(TARGETS mypkg_utils EXPORT mypkg + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/utils) + + install(PACKAGE_INFO mypkg EXPORT mypkg) + """) + + # Create source files for both libraries + core_cpp = gen_function_cpp(name="mypkg_core") + core_h = gen_function_h(name="mypkg_core") + utils_cpp = gen_function_cpp(name="mypkg_utils", includes=["mypkg_core"], + calls=["mypkg_core"]) + utils_h = gen_function_h(name="mypkg_utils") + + # First, try with the standard mypkg-config.cmake consumption + c.save({"CMakeLists.txt": cmake, + "src/mypkg_core.cpp": core_cpp, + "include/mypkg_core.h": core_h, + "src/mypkg_utils.cpp": utils_cpp, + "include/mypkg_utils.h": utils_h}) + + c.run_command(f"cmake . -DBUILD_SHARED_LIBS=ON") + print(c.out) + c.run_command("cmake --build . --config Release") + print(c.out) + c.run_command("cmake --install . --config Release --prefix=mypkginstall") + print(c.out) + cps = c.load("mypkginstall/cps/mypkg.cps") + print(cps) + cps_release = c.load("mypkginstall/cps/mypkg@release.cps") + print(cps_release) diff --git a/test/functional/toolchains/cmake/test_ninja.py b/test/functional/toolchains/cmake/test_ninja.py index 7c34c142e8e..0a78d6cd708 100644 --- a/test/functional/toolchains/cmake/test_ninja.py +++ b/test/functional/toolchains/cmake/test_ninja.py @@ -58,8 +58,7 @@ def package(self): @pytest.mark.parametrize("build_type,shared", [("Release", False), ("Debug", True)]) @pytest.mark.tool("ninja") def test_locally_build_linux(build_type, shared, client): - settings = "-s os=Linux -s arch=x86_64 -s build_type={} -o hello/*:shared={}".format(build_type, - shared) + settings = f"-s os=Linux -s arch=x86_64 -s build_type={build_type} -o hello/*:shared={shared}" client.run("install . {}".format(settings)) client.run_command('cmake . -G "Ninja" -DCMAKE_TOOLCHAIN_FILE={} -DCMAKE_BUILD_TYPE={}' .format(CMakeToolchain.filename, build_type)) diff --git a/test/functional/toolchains/cmake/test_universal_binaries.py b/test/functional/toolchains/cmake/test_universal_binaries.py index a68e06a8eea..9b967b404af 100644 --- a/test/functional/toolchains/cmake/test_universal_binaries.py +++ b/test/functional/toolchains/cmake/test_universal_binaries.py @@ -94,9 +94,9 @@ def test(self): client.run('install . -s="arch=armv8|armv8.3|x86_64" ' '-c tools.cmake.cmake_layout:build_folder_vars=\'["settings.arch"]\'') - client.run_command("cmake --preset \"conan-armv8|armv8.3|x86_64-release\" ") - client.run_command("cmake --build --preset \"conan-armv8|armv8.3|x86_64-release\" ") - client.run_command("lipo -info './build/armv8|armv8.3|x86_64/Release/foo'") + client.run_command("cmake --preset \"conan-armv8_armv8.3_x86_64-release\" ") + client.run_command("cmake --build --preset \"conan-armv8_armv8.3_x86_64-release\" ") + client.run_command("lipo -info './build/armv8_armv8.3_x86_64/Release/foo'") assert "foo are: x86_64 arm64 arm64e" in client.out @@ -129,7 +129,7 @@ def test_create_universal_binary_ninja(): client.save({"conanfile.txt": conanfile, "CMakeLists.txt": cmake}, - clean_first=True) + clean_first=True) client.run('install . -s=arch="armv8|x86_64" --build=missing -of=build') diff --git a/test/unittests/client/toolchain/__init__.py b/test/functional/toolchains/emscripten/__init__.py similarity index 100% rename from test/unittests/client/toolchain/__init__.py rename to test/functional/toolchains/emscripten/__init__.py diff --git a/test/functional/toolchains/gnu/autotools/test_apple_toolchain.py b/test/functional/toolchains/gnu/autotools/test_apple_toolchain.py index e64044ae275..ad35f636ef9 100644 --- a/test/functional/toolchains/gnu/autotools/test_apple_toolchain.py +++ b/test/functional/toolchains/gnu/autotools/test_apple_toolchain.py @@ -38,8 +38,6 @@ def build(self): @pytest.mark.skipif(platform.system() != "Darwin", reason="Only OSX") @pytest.mark.parametrize("config", [("x86_64", "Macos", "10.14", None), ("armv8", "iOS", "10.0", "iphoneos"), - ("armv7", "iOS", "10.0", "iphoneos"), - ("x86", "iOS", "10.0", "iphonesimulator"), ("x86_64", "iOS", "10.0", "iphonesimulator"), ("armv8", "Macos", "10.14", None) # M1 ]) diff --git a/test/functional/toolchains/gnu/autotools/test_win_bash.py b/test/functional/toolchains/gnu/autotools/test_win_bash.py index fca0f1c9c4b..fff3b4ed1e0 100644 --- a/test/functional/toolchains/gnu/autotools/test_win_bash.py +++ b/test/functional/toolchains/gnu/autotools/test_win_bash.py @@ -65,25 +65,25 @@ def build(self): @pytest.mark.skipif(platform.system() != "Windows", reason="Requires Windows") -@pytest.mark.tool("clang", "18") @pytest.mark.tool("msys2") +@pytest.mark.tool("clang", "20") @pytest.mark.parametrize("frontend", ("clang", "clang-cl")) @pytest.mark.parametrize("runtime", ("static", "dynamic")) @pytest.mark.parametrize("build_type", ("Debug", "Release")) def test_autotools_bash_complete_clang(frontend, runtime, build_type): client = TestClient(path_with_spaces=False) # Problem is that msys2 also has clang in the path, so we need to make it explicit - clangpath = tools_locations["clang"]["18"]["path"]["Windows"] + clangpath = tools_locations["clang"]["20"]["path"]["Windows"] # compilers c, cpp = ("clang", "clang++") if frontend == "clang" else ("clang-cl", "clang-cl") - comps = f'{{"cpp":"{clangpath}/{cpp}", "c":"{clangpath}/{c}", "rc":"{clangpath}/{c}"}}' + comps = f'{{"cpp":"{cpp}", "c":"{c}", "rc":"{c}"}}' profile_win = textwrap.dedent(f""" [settings] os=Windows arch=x86_64 build_type={build_type} compiler=clang - compiler.version=18 + compiler.version=20 compiler.cppstd=14 compiler.runtime_version=v144 compiler.runtime={runtime} @@ -93,6 +93,9 @@ def test_autotools_bash_complete_clang(frontend, runtime, build_type): tools.microsoft.bash:subsystem=msys2 tools.microsoft.bash:path=bash tools.compilation:verbosity=verbose + + [buildenv] + PATH=+(path){clangpath} """) main = gen_function_cpp(name="main") @@ -131,7 +134,7 @@ def build(self): client.run("build . -pr=profile_win") client.run_command("main.exe") assert "__GNUC__" not in client.out - assert "main __clang_major__18" in client.out + assert "main __clang_major__20" in client.out check_exe_run(client.out, "main", "clang", None, build_type, "x86_64", None) bat_contents = client.load("conanbuild.bat") diff --git a/test/functional/toolchains/gnu/test_gnutoolchain_apple.py b/test/functional/toolchains/gnu/test_gnutoolchain_apple.py index e5543d54446..b405ade41cf 100644 --- a/test/functional/toolchains/gnu/test_gnutoolchain_apple.py +++ b/test/functional/toolchains/gnu/test_gnutoolchain_apple.py @@ -13,8 +13,6 @@ @pytest.mark.skipif(platform.system() != "Darwin", reason="Only OSX") @pytest.mark.parametrize("config", [("x86_64", "Macos", "10.14", None), ("armv8", "iOS", "10.0", "iphoneos"), - ("armv7", "iOS", "10.0", "iphoneos"), - ("x86", "iOS", "10.0", "iphonesimulator"), ("x86_64", "iOS", "10.0", "iphonesimulator"), ("armv8", "Macos", "10.14", None) # M1 ]) diff --git a/test/functional/toolchains/gnu/test_pkgconfigdeps.py b/test/functional/toolchains/gnu/test_pkgconfigdeps.py index 501d9969527..ef59880f5e5 100644 --- a/test/functional/toolchains/gnu/test_pkgconfigdeps.py +++ b/test/functional/toolchains/gnu/test_pkgconfigdeps.py @@ -1,5 +1,4 @@ import platform -import sys import textwrap import pytest @@ -56,7 +55,6 @@ def build_requirements(self): assert "Description: Conan package: app" in client.load("app.pc") -@pytest.mark.skipif(sys.version_info.minor < 7, reason="Meson 1.1.x version needs Python >= 3.7") @pytest.mark.skipif(platform.system() != "Windows", reason="It makes sense only for Windows") @pytest.mark.tool("meson") # https://github.com/mesonbuild/meson/pull/11649 is part of Meson 1.1.0 @pytest.mark.tool("pkg_config") diff --git a/test/functional/toolchains/gnu/test_universal_binaries.py b/test/functional/toolchains/gnu/test_universal_binaries.py new file mode 100644 index 00000000000..e23aff7fbd1 --- /dev/null +++ b/test/functional/toolchains/gnu/test_universal_binaries.py @@ -0,0 +1,66 @@ +import platform +import textwrap + +import pytest + +from conan.test.utils.tools import TestClient + + +@pytest.mark.skipif(platform.system() != "Darwin", reason="Only OSX") +def test_gnutoolchain_universal_binary(): + client = TestClient(path_with_spaces=False) + + conanfile = textwrap.dedent(""" + import os + from conan import ConanFile + from conan.tools.gnu import GnuToolchain, Autotools + from conan.tools.layout import basic_layout + + class mylibraryRecipe(ConanFile): + name = "mylibrary" + version = "1.0" + package_type = "library" + settings = "os", "compiler", "build_type", "arch" + options = {"shared": [True, False], "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + + exports_sources = "configure.ac", "Makefile.am", "src/*" + + def config_options(self): + if self.settings.os == "Windows": + self.options.rm_safe("fPIC") + + def configure(self): + if self.options.shared: + self.options.rm_safe("fPIC") + + def layout(self): + basic_layout(self) + + def generate(self): + gnu_toolchain = GnuToolchain(self) + gnu_toolchain.generate() + + def build(self): + autotools = Autotools(self) + autotools.autoreconf() + autotools.configure() + autotools.make() + + def package(self): + autotools = Autotools(self) + autotools.install() + self.run(f"lipo -info {os.path.join(self.package_folder, 'lib', 'libmylibrary.a')}") + + def package_info(self): + self.cpp_info.libs = ["mylibrary"] + + """) + + client.run('new autotools_lib -d name=mylibrary -d version=0.1') + + client.save({"conanfile.py": conanfile}) + + client.run('create . -s="arch=armv8|x86_64" -tf=""') + + assert "libmylibrary.a are: x86_64 arm64" in client.out diff --git a/test/functional/toolchains/google/test_bazel.py b/test/functional/toolchains/google/test_bazel.py index 8886211e8d0..c9d28c47220 100644 --- a/test/functional/toolchains/google/test_bazel.py +++ b/test/functional/toolchains/google/test_bazel.py @@ -3,6 +3,7 @@ import pytest +from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.test_files import temp_folder from conan.test.utils.tools import TestClient @@ -38,7 +39,7 @@ def base_profile(): @pytest.mark.parametrize("build_type", ["Debug", "Release", "RelWithDebInfo", "MinSizeRel"]) -@pytest.mark.tool("bazel", "6.5.0") +@pytest.mark.tool("bazel", "6.x") def test_basic_exe_6x(bazelrc, build_type, base_profile, bazel_output_root_dir): client = TestClient(path_with_spaces=False) client.run(f"new bazel_exe -d name=myapp -d version=1.0 -d output_root_dir={bazel_output_root_dir}") @@ -56,7 +57,7 @@ def test_basic_exe_6x(bazelrc, build_type, base_profile, bazel_output_root_dir): @pytest.mark.parametrize("build_type", ["Debug", "Release", "RelWithDebInfo", "MinSizeRel"]) -@pytest.mark.tool("bazel", "7.4.1") +@pytest.mark.tool("bazel", "7.x") def test_basic_exe(bazelrc, build_type, base_profile, bazel_output_root_dir): client = TestClient(path_with_spaces=False) client.run(f"new bazel_7_exe -d name=myapp -d version=1.0 -d output_root_dir={bazel_output_root_dir}") @@ -73,7 +74,7 @@ def test_basic_exe(bazelrc, build_type, base_profile, bazel_output_root_dir): assert "myapp/1.0: Hello World Debug!" in client.out -@pytest.mark.tool("bazel", "8.0.0") +@pytest.mark.tool("bazel", "8.x") def test_basic_lib(bazelrc, base_profile, bazel_output_root_dir): """ Issue related: https://github.com/conan-io/conan/issues/17438 @@ -85,7 +86,7 @@ def test_basic_lib(bazelrc, base_profile, bazel_output_root_dir): @pytest.mark.parametrize("shared", [False, True]) -@pytest.mark.tool("bazel", "6.5.0") +@pytest.mark.tool("bazel", "6.x") def test_transitive_libs_consuming_6x(shared, bazel_output_root_dir): """ Testing the next dependencies structure for shared/static libs @@ -211,7 +212,7 @@ def test_transitive_libs_consuming_6x(shared, bazel_output_root_dir): @pytest.mark.parametrize("shared", [False, True]) -@pytest.mark.tool("bazel", "7.4.1") +@pytest.mark.tool("bazel", "7.x") @pytest.mark.skipif(platform.system() == "Linux", reason="Conan CI fails (likely related to parallel " "tests running??). Skipping it for now!") @@ -253,6 +254,10 @@ def test_transitive_libs_consuming_7x(shared, bazel_output_root_dir): self.cpp_info.system_libs.append("m") else: self.cpp_info.system_libs.append("ws2_32") + # Issue: https://github.com/conan-io/conan/issues/18748 + from conan.tools.apple.apple import is_apple_os + if is_apple_os(self): + self.cpp_info.frameworks = ["CoreFoundation"] """ client.save({"conanfile.py": conanfile}) client.run(f"create . -o '*:shared={shared}' -tf ''") # skipping tests @@ -337,3 +342,45 @@ def test_transitive_libs_consuming_7x(shared, bazel_output_root_dir): client.run(f"create . -o '*:shared={shared}'") assert "mysecondlib() First define MY_VALUE and other define 2" in client.out assert "myfirstlib/1.2.11: Hello World Release!" + + +@pytest.mark.tool("bazel", "8.x") +def test_empty_bazel_query(): + """ + Test that following a simple steps using the BazelDeps and running + a global `bazel query //...` runs OK (bazel >= 8.0) + + Issue related: https://github.com/conan-io/conan/issues/18743 + """ + zlib = GenConanfile("zlib", "0.1") + consumer = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.google import BazelDeps, bazel_layout + + class App(ConanFile): + settings = "os", "arch", "compiler", "build_type" + requires = "zlib/0.1" + + def layout(self): + bazel_layout(self) + + def generate(self): + bz = BazelDeps(self) + bz.generate() + """) + module = textwrap.dedent("""\ + load_conan_dependencies = use_extension("//conan:conan_deps_module_extension.bzl", "conan_extension") + use_repo(load_conan_dependencies, "zlib") + """) + client = TestClient() + client.save({ + "zlib/conanfile.py": zlib, + "consumer/conanfile.py": consumer, + "consumer/MODULE.bazel": module, + }) + client.run("create zlib") + client.run("install consumer") + with client.chdir("consumer"): + client.run_command("bazel query //...") + assert "//conan/zlib:zlib" in client.out + assert "//conan/zlib:zlib_binaries" in client.out diff --git a/test/functional/toolchains/google/test_bazeltoolchain_cross_compilation.py b/test/functional/toolchains/google/test_bazeltoolchain_cross_compilation.py index 036a4445de1..e848e82e5be 100644 --- a/test/functional/toolchains/google/test_bazeltoolchain_cross_compilation.py +++ b/test/functional/toolchains/google/test_bazeltoolchain_cross_compilation.py @@ -10,7 +10,7 @@ @pytest.mark.skipif(platform.system() != "Darwin", reason="Only for Darwin") -@pytest.mark.tool("bazel", "6.5.0") # not working for Bazel 7.x +@pytest.mark.tool("bazel", "6.x") # not working for Bazel 7.x def test_bazel_simple_cross_compilation(): profile = textwrap.dedent(""" [settings] diff --git a/test/functional/toolchains/ios/test_using_cmake.py b/test/functional/toolchains/ios/test_using_cmake.py index a209f00bd5d..203d85f43c8 100644 --- a/test/functional/toolchains/ios/test_using_cmake.py +++ b/test/functional/toolchains/ios/test_using_cmake.py @@ -1,6 +1,5 @@ import platform import textwrap -import unittest import pytest @@ -10,69 +9,62 @@ @pytest.mark.skipif(platform.system() != "Darwin", reason="Requires XCode") -class ToolchainiOSTestCase(unittest.TestCase): +@pytest.mark.tool("cmake", "3.19") +def test_xcode_ios_generator(): + """ Simplest approach: + https://cmake.org/cmake/help/latest/manual/cmake-toolchains.7.html#cross-compiling-for-ios-tvos-or-watchos + """ + t = TestClient() + create_library(t) + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.cmake import CMake, CMakeToolchain - def setUp(self): - self.t = TestClient() - create_library(self.t) - self._conanfile = textwrap.dedent(""" - from conan import ConanFile - from conan.tools.cmake import CMake, CMakeToolchain + class Library(ConanFile): + name = 'hello' + version = '1.0' + settings = 'os', 'arch', 'compiler', 'build_type' + exports_sources = 'hello.h', 'hello.cpp', 'CMakeLists.txt' + options = {'shared': [True, False]} + default_options = {'shared': False} - class Library(ConanFile): - name = 'hello' - version = '1.0' - settings = 'os', 'arch', 'compiler', 'build_type' - exports_sources = 'hello.h', 'hello.cpp', 'CMakeLists.txt' - options = {{'shared': [True, False]}} - default_options = {{'shared': False}} + def generate(self): + tc = CMakeToolchain(self, generator="Xcode") + tc.generate() - def generate(self): - tc = CMakeToolchain(self, generator={generator}) - tc.generate() + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + self.run("lipo -info Release-iphoneos/libhello.a") - def build(self): - cmake = CMake(self) - cmake.configure() - cmake.build() - self.run("lipo -info Release-iphoneos/libhello.a") + def package(self): + cmake = CMake(self) + cmake.install() + """) - def package(self): - cmake = CMake(self) - cmake.install() - """) + ios_profile = textwrap.dedent(""" + [settings] + os=iOS + os.sdk=iphoneos + os.version=12.0 + arch=armv8 + compiler=apple-clang + compiler.version=12.0 + compiler.libcxx=libc++ + build_type=Release + """) - self.t.save({ - 'ios_profile': textwrap.dedent(""" - [settings] - os=iOS - os.sdk=iphoneos - os.version=12.0 - arch=armv8 - compiler=apple-clang - compiler.version=12.0 - compiler.libcxx=libc++ - build_type=Release - """) - }) + t.save({'conanfile.py': conanfile, + "ios_profile": ios_profile}) - @pytest.mark.tool("cmake", "3.19") - def test_xcode_generator(self): - """ Simplest approach: - https://cmake.org/cmake/help/latest/manual/cmake-toolchains.7.html#cross-compiling-for-ios-tvos-or-watchos - """ - self.t.save({'conanfile.py': self._conanfile.format(generator='"Xcode"')}) + # Build in the cache + t.run('create . --profile:build=default --profile:host=ios_profile') + assert "Non-fat file: Release-iphoneos/libhello.a is architecture: arm64" in t.out - # Build in the cache - self.t.run('create . --profile:build=default --profile:host=ios_profile') - self.assertIn("Non-fat file: Release-iphoneos/libhello.a is architecture: arm64", self.t.out) - - # Build locally - self.t.run('install . --profile:host=ios_profile --profile:build=default') - self.t.run_command('cmake . -G"Xcode" -DCMAKE_TOOLCHAIN_FILE={}'.format(CMakeToolchain.filename)) - self.t.run_command('cmake --build . --config Release') - self.t.run_command("lipo -info Release-iphoneos/libhello.a") - self.assertIn("Non-fat file: Release-iphoneos/libhello.a is architecture: arm64", self.t.out) - - def test_unix_makefiles_generator(self): - pass + # Build locally + t.run('install . --profile:host=ios_profile --profile:build=default') + t.run_command('cmake . -G"Xcode" -DCMAKE_TOOLCHAIN_FILE={}'.format(CMakeToolchain.filename)) + t.run_command('cmake --build . --config Release') + t.run_command("lipo -info Release-iphoneos/libhello.a") + assert "Non-fat file: Release-iphoneos/libhello.a is architecture: arm64" in t.out diff --git a/test/functional/toolchains/meson/_base.py b/test/functional/toolchains/meson/_base.py index edc202bda93..f055848c379 100644 --- a/test/functional/toolchains/meson/_base.py +++ b/test/functional/toolchains/meson/_base.py @@ -1,36 +1,24 @@ import platform -import unittest -import pytest -from conan.test.utils.tools import TestClient - - -@pytest.mark.tool("meson") -@pytest.mark.skipif(platform.system() not in ("Darwin", "Windows", "Linux"), - reason="Not tested for not mainstream boring operating systems") -class TestMesonBase(unittest.TestCase): - def setUp(self): - self.t = TestClient() - - def _check_binary(self): - # FIXME: Some values are hardcoded to match the CI setup - host_arch = self.t.get_default_host_profile().settings['arch'] - arch_macro = { - "gcc": {"armv8": "__aarch64__", "x86_64": "__x86_64__"}, - "msvc": {"armv8": "_M_ARM64", "x86_64": "_M_X64"} - } - if platform.system() == "Darwin": - self.assertIn(f"main {arch_macro['gcc'][host_arch]} defined", self.t.out) - self.assertIn("main __apple_build_version__", self.t.out) - self.assertIn("main __clang_major__16", self.t.out) - # TODO: check why __clang_minor__ seems to be not defined in XCode 12 - # commented while migrating to XCode12 CI - # self.assertIn("main __clang_minor__0", self.t.out) - elif platform.system() == "Windows": - self.assertIn(f"main {arch_macro['msvc'][host_arch]} defined", self.t.out) - self.assertIn("main _MSC_VER19", self.t.out) - self.assertIn("main _MSVC_LANG2014", self.t.out) - elif platform.system() == "Linux": - self.assertIn(f"main {arch_macro['gcc'][host_arch]} defined", self.t.out) - self.assertIn("main __GNUC__9", self.t.out) +def check_binary(t): + # FIXME: Some values are hardcoded to match the CI setup + host_arch = t.get_default_host_profile().settings['arch'] + arch_macro = { + "gcc": {"armv8": "__aarch64__", "x86_64": "__x86_64__"}, + "msvc": {"armv8": "_M_ARM64", "x86_64": "_M_X64"} + } + if platform.system() == "Darwin": + assert f"main {arch_macro['gcc'][host_arch]} defined" in t.out + assert "main __apple_build_version__" in t.out + assert "main __clang_major__17" in t.out + # TODO: check why __clang_minor__ seems to be not defined in XCode 12 + # commented while migrating to XCode12 CI + # assert ("main __clang_minor__0" in t.out + elif platform.system() == "Windows": + assert f"main {arch_macro['msvc'][host_arch]} defined" in t.out + assert "main _MSC_VER19" in t.out + assert "main _MSVC_LANG2014" in t.out + elif platform.system() == "Linux": + assert f"main {arch_macro['gcc'][host_arch]} defined" in t.out + assert "main __GNUC__9" in t.out diff --git a/test/functional/toolchains/meson/test_backend.py b/test/functional/toolchains/meson/test_backend.py index b6b6d1dfce7..9464b626e5a 100644 --- a/test/functional/toolchains/meson/test_backend.py +++ b/test/functional/toolchains/meson/test_backend.py @@ -1,6 +1,5 @@ import os import platform -import sys import textwrap import pytest @@ -10,7 +9,6 @@ @pytest.mark.tool("meson") -@pytest.mark.skipif(sys.version_info.major == 2, reason="Meson not supported in Py2") @pytest.mark.skipif(platform.system() != "Windows", reason="requires Windows") def test_cross_x86(): conanfile_py = textwrap.dedent(""" diff --git a/test/functional/toolchains/meson/test_cross_compilation.py b/test/functional/toolchains/meson/test_cross_compilation.py index 689fb352af8..88322b792f5 100644 --- a/test/functional/toolchains/meson/test_cross_compilation.py +++ b/test/functional/toolchains/meson/test_cross_compilation.py @@ -59,8 +59,6 @@ def build(self): @pytest.mark.skipif(platform.system() != "Darwin", reason="requires Xcode") @pytest.mark.parametrize("arch, os_, os_version, os_sdk", [ ('armv8', 'iOS', '17.1', 'iphoneos'), - ('armv7', 'iOS', '10.0', 'iphoneos'), - ('x86', 'iOS', '10.0', 'iphonesimulator'), ('x86_64', 'iOS', '10.0', 'iphonesimulator'), ('armv8' if platform.machine() == "x86_64" else "x86_64", 'Macos', None, None), ('armv8' if platform.machine() == "x86_64" else "x86_64", 'Macos', '10.11', None), @@ -127,6 +125,7 @@ def test_apple_meson_toolchain_cross_compiling(arch, os_, os_version, os_sdk): assert f"'-mmacosx-version-min={os_version}'" in content +@pytest.mark.tool("ninja") @pytest.mark.tool("meson") # for Linux, build for x86 will require a multilib compiler # for macOS, build for x86 is no longer supported by modern Xcode @@ -246,3 +245,49 @@ def test_use_meson_toolchain(): # Build locally c.run('build . --profile:host=android') assert "conanfile.py (hello/0.1): Calling build()" in c.out + + +@pytest.mark.tool("meson") +@pytest.mark.tool("ninja") +@pytest.mark.skipif(platform.system() != "Linux", reason="Linker scripts in Linux only") +def test_linker_script(): + # https://github.com/conan-io/conan/issues/18535 + conanfile_py = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.meson import Meson + from conan.tools.files import load + + class App(ConanFile): + settings = "os", "arch", "compiler", "build_type" + generators = "MesonToolchain" + + def layout(self): + self.folders.build = "build" + + def build(self): + meson = Meson(self) + try: + meson.configure() + finally: + errors = load(self, "meson-logs/meson-log.txt") + self.output.info(errors) + """) + meson_build = textwrap.dedent(""" + project('tutorial', 'cpp') + executable('demo', 'main.cpp') + """) + main_cpp = gen_function_cpp(name="main") + profile = textwrap.dedent(""" + include(default) + [conf] + tools.build:linker_scripts=['{{profile_dir}}/mylinkscript.ld'] + """) + c = TestClient() + c.save({"conanfile.py": conanfile_py, + "meson.build": meson_build, + "main.cpp": main_cpp, + "mylinkscript.ld": "", + "profile": profile}) + c.run('build . -pr=profile', assert_error=True) + # This error means the linker script was fonud and loaded, it failed because empty + assert "PHDR segment not covered by LOAD segment" in c.out diff --git a/test/functional/toolchains/meson/test_install.py b/test/functional/toolchains/meson/test_install.py index 819526e499d..8ffeb264bab 100644 --- a/test/functional/toolchains/meson/test_install.py +++ b/test/functional/toolchains/meson/test_install.py @@ -4,10 +4,11 @@ import pytest from conan.test.assets.sources import gen_function_cpp, gen_function_h -from test.functional.toolchains.meson._base import TestMesonBase +from conan.test.utils.tools import TestClient +from test.functional.toolchains.meson._base import check_binary -class MesonInstall(TestMesonBase): +class TestMesonInstall: _conanfile_py = textwrap.dedent(""" import os import shutil @@ -98,21 +99,23 @@ def test(self): target_link_libraries(${PROJECT_NAME} hello::hello) """) + @pytest.mark.tool("ninja") @pytest.mark.tool("meson") @pytest.mark.tool("cmake") def test_install(self): + t = TestClient() hello_cpp = gen_function_cpp(name="hello") hello_h = gen_function_h(name="hello") test_package_cpp = gen_function_cpp(name="main", includes=["hello"], calls=["hello"]) - self.t.save({"conanfile.py": self._conanfile_py, - "meson.build": self._meson_build, - "hello.cpp": hello_cpp, - "hello.h": hello_h, - os.path.join("test_package", "conanfile.py"): self._test_package_conanfile_py, - os.path.join("test_package", "CMakeLists.txt"): self._test_package_cmake_lists, - os.path.join("test_package", "src", "test_package.cpp"): test_package_cpp}) - - self.t.run("create . --name=hello --version=0.1 -c tools.compilation:verbosity=verbose") - assert "--verbose" in self.t.out - self._check_binary() + t.save({"conanfile.py": self._conanfile_py, + "meson.build": self._meson_build, + "hello.cpp": hello_cpp, + "hello.h": hello_h, + os.path.join("test_package", "conanfile.py"): self._test_package_conanfile_py, + os.path.join("test_package", "CMakeLists.txt"): self._test_package_cmake_lists, + os.path.join("test_package", "src", "test_package.cpp"): test_package_cpp}) + + t.run("create . --name=hello --version=0.1 -c tools.compilation:verbosity=verbose") + assert "--verbose" in t.out + check_binary(t) diff --git a/test/functional/toolchains/meson/test_meson.py b/test/functional/toolchains/meson/test_meson.py index 889ec8a3ba7..d3997e1bc57 100644 --- a/test/functional/toolchains/meson/test_meson.py +++ b/test/functional/toolchains/meson/test_meson.py @@ -9,12 +9,15 @@ from conan.test.assets.sources import gen_function_cpp, gen_function_h from conan.test.utils.tools import TestClient from conan.api.model import RecipeReference -from test.functional.toolchains.meson._base import TestMesonBase +from test.functional.toolchains.meson._base import check_binary -class MesonToolchainTest(TestMesonBase): +@pytest.mark.tool("ninja") +@pytest.mark.tool("meson") +class TestMesonToolchain: def test_definition_of_global_options(self): + t = TestClient() conanfile_py = textwrap.dedent(""" from conan import ConanFile from conan.tools.meson import Meson, MesonToolchain @@ -83,45 +86,46 @@ def build(self): # Issue related: https://github.com/conan-io/conan/issues/14453 # At first, let's add a raw option to see that it fails - self.t.save({"conanfile.py": conanfile_py.format(msg="self.options.msg"), - "meson.build": meson_build, - "meson_options.txt": meson_options_txt, - "hello.h": hello_h, - "hello.cpp": hello_cpp, - "main.cpp": app}) - self.t.run("build .", assert_error=True) + t.save({"conanfile.py": conanfile_py.format(msg="self.options.msg"), + "meson.build": meson_build, + "meson_options.txt": meson_options_txt, + "hello.h": hello_h, + "hello.cpp": hello_cpp, + "main.cpp": app}) + t.run("build .", assert_error=True) # Using directly the options, the result is -> HELLO_MSG = Hi everyone, which is incorrect assert "WARN: deprecated: Please, do not use a Conan option value directly. " \ - "Convert 'options.shared' into" in self.t.out + "Convert 'options.shared' into" in t.out assert "WARN: deprecated: Please, do not use a Conan option value directly. " \ - "Convert 'options.msg' into" in self.t.out - assert "Malformed value" in self.t.out + "Convert 'options.msg' into" in t.out + assert "Malformed value" in t.out # Let's transform the Conan option into other allowed data type to solve the issue - self.t.save({"conanfile.py": conanfile_py.format(msg="str(self.options.msg)")}) - self.t.run("build .") - content = self.t.load(os.path.join("build", "gen_folder", "conan_meson_native.ini")) - self.assertIn("[project options]", content) - self.assertIn("STRING_DEFINITION = 'Text'", content) - self.assertIn("TRUE_DEFINITION = true", content) - self.assertIn("FALSE_DEFINITION = false", content) - self.assertIn("DYNAMIC = False", content) # Meson transforms correctly this value into bool - self.assertIn("HELLO_MSG = 'Hi everyone!'", content) - self.assertIn("INT_DEFINITION = 42", content) - self.assertIn("ARRAY_DEFINITION = ['Text1', 'Text2']", content) - self.assertIn("[built-in options]", content) - self.assertIn("buildtype = 'release'", content) - - self.t.run_command(os.path.join("build", "demo")) - self.assertIn("hello: Release!", self.t.out) - self.assertIn("STRING_DEFINITION: Text", self.t.out) - self.assertIn("[properties]", content) - self.assertNotIn("needs_exe_wrapper", content) - - self._check_binary() + t.save({"conanfile.py": conanfile_py.format(msg="str(self.options.msg)")}) + t.run("build .") + content = t.load(os.path.join("build", "gen_folder", "conan_meson_native.ini")) + assert "[project options]" in content + assert "STRING_DEFINITION = 'Text'" in content + assert "TRUE_DEFINITION = true" in content + assert "FALSE_DEFINITION = false" in content + assert "DYNAMIC = False" in content # Meson transforms correctly this value into bool + assert "HELLO_MSG = 'Hi everyone!'" in content + assert "INT_DEFINITION = 42" in content + assert "ARRAY_DEFINITION = ['Text1', 'Text2']" in content + assert "[built-in options]" in content + assert "buildtype = 'release'" in content + + t.run_command(os.path.join("build", "demo")) + assert "hello: Release!" in t.out + assert "STRING_DEFINITION: Text" in t.out + assert "[properties]" in content + assert "needs_exe_wrapper" not in content + + check_binary(t) def test_meson_default_dirs(self): - self.t.run("new meson_exe -d name=hello -d version=1.0") - # self.t.run("new meson_exe -d name=hello -d version=1.0 -m meson_exe") + t = TestClient() + t.run("new meson_exe -d name=hello -d version=1.0") + # t.run("new meson_exe -d name=hello -d version=1.0 -m meson_exe") meson_build = textwrap.dedent(""" project('tutorial', 'cpp') @@ -171,20 +175,20 @@ def test(self): self.run("demo", env="conanrun") """) # Replace meson.build, conanfile.py and test_package/conanfile.py - self.t.save({"meson.build": meson_build, - "conanfile.py": conanfile, - "test_package/conanfile.py": test_conanfile, - "src/file1.txt": "", "src/file2.txt": ""}) - self.t.run("create . -c tools.build:verbosity=quiet -c tools.compilation:verbosity=verbose") + t.save({"meson.build": meson_build, + "conanfile.py": conanfile, + "test_package/conanfile.py": test_conanfile, + "src/file1.txt": "", "src/file2.txt": ""}) + t.run("create . -c tools.build:verbosity=quiet -c tools.compilation:verbosity=verbose") # Check verbosity control - assert "unrecognized arguments" not in self.t.out - assert re.search("meson compile .*? --verbose", self.t.out) - assert re.search("meson install .*? --quiet", self.t.out) + assert "unrecognized arguments" not in t.out + assert re.search("meson compile .*? --verbose", t.out) + assert re.search("meson install .*? --quiet", t.out) # Check if all the files are in the final directories ref = RecipeReference.loads("hello/1.0") - pref = self.t.get_latest_package_reference(ref) - package_folder = self.t.get_latest_pkg_layout(pref).package() + pref = t.get_latest_package_reference(ref) + package_folder = t.get_latest_pkg_layout(pref).package() if platform.system() == "Windows": assert os.path.exists(os.path.join(package_folder, "lib", "hello.lib")) assert os.path.exists(os.path.join(package_folder, "bin", "hello.dll")) @@ -250,6 +254,7 @@ def build(self): assert match +@pytest.mark.tool("ninja") @pytest.mark.tool("meson") @pytest.mark.skipif(platform.system() != "Windows", reason="Only for Windows") @pytest.mark.skipif(sys.version_info.minor < 8, reason="Latest Meson versions needs Python >= 3.8") diff --git a/test/functional/toolchains/meson/test_meson_and_gnu_deps_flags.py b/test/functional/toolchains/meson/test_meson_and_gnu_deps_flags.py index db87fd29017..fae6e7cbbe7 100644 --- a/test/functional/toolchains/meson/test_meson_and_gnu_deps_flags.py +++ b/test/functional/toolchains/meson/test_meson_and_gnu_deps_flags.py @@ -4,12 +4,12 @@ import pytest -from test.functional.toolchains.meson._base import TestMesonBase from conan.test.utils.tools import TestClient -class TestMesonToolchainAndGnuFlags(TestMesonBase): +class TestMesonToolchainAndGnuFlags: + @pytest.mark.tool("ninja") @pytest.mark.tool("meson") @pytest.mark.tool("pkg_config") def test_mesondeps_flags_are_being_appended_and_not_replacing_toolchain_ones(self): diff --git a/test/functional/toolchains/meson/test_meson_and_objc.py b/test/functional/toolchains/meson/test_meson_and_objc.py index 81d88e0f217..c7bb5033ad4 100644 --- a/test/functional/toolchains/meson/test_meson_and_objc.py +++ b/test/functional/toolchains/meson/test_meson_and_objc.py @@ -1,6 +1,5 @@ import os import platform -import sys import textwrap import pytest @@ -48,7 +47,6 @@ def build(self): @pytest.mark.tool("meson") -@pytest.mark.skipif(sys.version_info.major == 2, reason="Meson not supported in Py2") @pytest.mark.skipif(platform.system() != "Darwin", reason="requires Xcode") def test_apple_meson_toolchain_native_compilation_objective_c(): t = TestClient() @@ -84,13 +82,10 @@ def test_apple_meson_toolchain_native_compilation_objective_c(): @pytest.mark.parametrize("arch, os_, os_version, sdk", [ ('armv8', 'iOS', '10.0', 'iphoneos'), - ('armv7', 'iOS', '10.0', 'iphoneos'), - ('x86', 'iOS', '10.0', 'iphonesimulator'), ('x86_64', 'iOS', '10.0', 'iphonesimulator'), ('armv8', 'Macos', '11.0', None) # Apple Silicon ]) @pytest.mark.tool("meson") -@pytest.mark.skipif(sys.version_info.major == 2, reason="Meson not supported in Py2") @pytest.mark.skipif(platform.system() != "Darwin", reason="requires Xcode") def test_apple_meson_toolchain_cross_compiling_and_objective_c(arch, os_, os_version, sdk): profile = textwrap.dedent(""" diff --git a/test/functional/toolchains/meson/test_pkg_config_reuse.py b/test/functional/toolchains/meson/test_pkg_config_reuse.py index 273049f86a7..b69c7e36729 100644 --- a/test/functional/toolchains/meson/test_pkg_config_reuse.py +++ b/test/functional/toolchains/meson/test_pkg_config_reuse.py @@ -4,11 +4,14 @@ import textwrap from conan.test.assets.sources import gen_function_cpp -from test.functional.toolchains.meson._base import TestMesonBase +from conan.test.utils.tools import TestClient +from test.functional.toolchains.meson._base import check_binary +@pytest.mark.tool("ninja") +@pytest.mark.tool("meson") @pytest.mark.tool("pkg_config") -class MesonPkgConfigTest(TestMesonBase): +class TestMesonPkgConfig: _conanfile_py = textwrap.dedent(""" from conan import ConanFile from conan.tools.meson import Meson, MesonToolchain @@ -39,20 +42,21 @@ def build(self): """) def test_reuse(self): - self.t.run("new cmake_lib -d name=hello -d version=0.1") - self.t.run("create . -tf=\"\"") + t = TestClient() + t.run("new cmake_lib -d name=hello -d version=0.1") + t.run("create . -tf=\"\"") app = gen_function_cpp(name="main", includes=["hello"], calls=["hello"]) # Prepare the actual consumer package - self.t.save({"conanfile.py": self._conanfile_py, - "meson.build": self._meson_build, - "main.cpp": app}, - clean_first=True) + t.save({"conanfile.py": self._conanfile_py, + "meson.build": self._meson_build, + "main.cpp": app}, + clean_first=True) # Build in the cache - self.t.run("build .") - self.t.run_command(os.path.join("build", "demo")) + t.run("build .") + t.run_command(os.path.join("build", "demo")) - self.assertIn("Hello World Release!", self.t.out) + assert "Hello World Release!" in t.out - self._check_binary() + check_binary(t) diff --git a/test/functional/toolchains/meson/test_preprocessor_definitions.py b/test/functional/toolchains/meson/test_preprocessor_definitions.py index b3b772a4246..c243073bdbd 100644 --- a/test/functional/toolchains/meson/test_preprocessor_definitions.py +++ b/test/functional/toolchains/meson/test_preprocessor_definitions.py @@ -1,11 +1,14 @@ import os import textwrap +import pytest + from conan.test.assets.sources import gen_function_cpp, gen_function_h -from test.functional.toolchains.meson._base import TestMesonBase +from conan.test.utils.tools import TestClient +from test.functional.toolchains.meson._base import check_binary -class MesonPreprocessorDefinitionsTest(TestMesonBase): +class TestMesonPreprocessorDefinitionsTest: _conanfile_py = textwrap.dedent(""" from conan import ConanFile from conan.tools.meson import Meson, MesonToolchain @@ -41,30 +44,33 @@ def build(self): executable('demo', 'main.cpp', link_with: hello) """) + @pytest.mark.tool("ninja") + @pytest.mark.tool("meson") def test_build(self): hello_h = gen_function_h(name="hello") hello_cpp = gen_function_cpp(name="hello", preprocessor=["TEST_DEFINITION1", "TEST_DEFINITION2"]) app = gen_function_cpp(name="main", includes=["hello"], calls=["hello"]) - self.t.save({"conanfile.py": self._conanfile_py, - "meson.build": self._meson_build, - "hello.h": hello_h, - "hello.cpp": hello_cpp, - "main.cpp": app}) + t = TestClient() + t.save({"conanfile.py": self._conanfile_py, + "meson.build": self._meson_build, + "hello.h": hello_h, + "hello.cpp": hello_cpp, + "main.cpp": app}) - self.t.run("install .") + t.run("install .") - content = self.t.load("conan_meson_native.ini") + content = t.load("conan_meson_native.ini") - self.assertIn("[built-in options]", content) - self.assertIn("buildtype = 'release'", content) + assert "[built-in options]" in content + assert "buildtype = 'release'" in content - self.t.run("build .") - self.t.run_command(os.path.join("build", "demo")) + t.run("build .") + t.run_command(os.path.join("build", "demo")) - self.assertIn("hello: Release!", self.t.out) - self.assertIn("TEST_DEFINITION1: TestPpdValue1", self.t.out) - self.assertIn("TEST_DEFINITION2: TestPpdValue2", self.t.out) + assert "hello: Release!" in t.out + assert "TEST_DEFINITION1: TestPpdValue1" in t.out + assert "TEST_DEFINITION2: TestPpdValue2" in t.out - self._check_binary() + check_binary(t) diff --git a/test/functional/toolchains/meson/test_subproject.py b/test/functional/toolchains/meson/test_subproject.py index 88738dc9a68..7415bb0388b 100644 --- a/test/functional/toolchains/meson/test_subproject.py +++ b/test/functional/toolchains/meson/test_subproject.py @@ -1,4 +1,3 @@ -import os import textwrap import pytest @@ -160,6 +159,7 @@ def test(self): """) +@pytest.mark.tool("ninja") @pytest.mark.tool("cmake") @pytest.mark.tool("meson") def test_subproject(): @@ -169,14 +169,14 @@ def test_subproject(): "meson.build": _meson_build, "greeter.c": _greeter_c, "greeter.h": _greeter_h, - os.path.join("include", "greeter.h"): _greeter_h, - os.path.join("subprojects", "hello", "include", "hello.h"): _hello_h, - os.path.join("subprojects", "hello", "hello.c"): _hello_c, - os.path.join("subprojects", "hello", "meson.build"): _meson_subproject_build, - os.path.join("subprojects", "hello", "meson_options.txt"): _meson_subproject_options, - os.path.join("test_package", "conanfile.py"): _test_package_conanfile_py, - os.path.join("test_package", "CMakeLists.txt"): _test_package_cmake_lists, - os.path.join("test_package", "src", "test_package.c"): test_package_c}) + "include/greeter.h": _greeter_h, + "subprojects/hello/include/hello.h": _hello_h, + "subprojects/hello/hello.c": _hello_c, + "subprojects/hello/meson.build": _meson_subproject_build, + "subprojects/hello/meson_options.txt": _meson_subproject_options, + "test_package/conanfile.py": _test_package_conanfile_py, + "test_package/CMakeLists.txt": _test_package_cmake_lists, + "test_package/src/test_package.c": test_package_c}) client.run("create . --name=greeter --version=0.1") assert "Hello from subproject" in client.out assert "Le sous-projet vous salut" not in client.out diff --git a/test/functional/toolchains/meson/test_test.py b/test/functional/toolchains/meson/test_test.py index 2995c38123a..1f9aa991973 100644 --- a/test/functional/toolchains/meson/test_test.py +++ b/test/functional/toolchains/meson/test_test.py @@ -1,14 +1,15 @@ -import os - import pytest import textwrap from conan.test.assets.sources import gen_function_cpp -from test.functional.toolchains.meson._base import TestMesonBase +from conan.test.utils.tools import TestClient +from test.functional.toolchains.meson._base import check_binary +@pytest.mark.tool("ninja") +@pytest.mark.tool("meson") @pytest.mark.tool("pkg_config") -class MesonTest(TestMesonBase): +class TestMeson: _test_package_meson_build = textwrap.dedent(""" project('test_package', 'cpp') hello = dependency('hello', version : '>=0.1') @@ -48,14 +49,15 @@ def test(self): """) def test_reuse(self): - self.t.run("new cmake_lib -d name=hello -d version=0.1") + t = TestClient() + t.run("new cmake_lib -d name=hello -d version=0.1") test_package_cpp = gen_function_cpp(name="main", includes=["hello"], calls=["hello"]) - self.t.save({os.path.join("test_package", "conanfile.py"): self._test_package_conanfile_py, - os.path.join("test_package", "meson.build"): self._test_package_meson_build, - os.path.join("test_package", "test_package.cpp"): test_package_cpp}) + t.save({"test_package/conanfile.py": self._test_package_conanfile_py, + "test_package/meson.build": self._test_package_meson_build, + "test_package/test_package.cpp": test_package_cpp}) - self.t.run("create . --name=hello --version=0.1") + t.run("create . --name=hello --version=0.1") - self._check_binary() + check_binary(t) diff --git a/test/functional/toolchains/meson/test_v2_meson_template.py b/test/functional/toolchains/meson/test_v2_meson_template.py index c26ed1f7972..4babaeb78e6 100644 --- a/test/functional/toolchains/meson/test_v2_meson_template.py +++ b/test/functional/toolchains/meson/test_v2_meson_template.py @@ -5,6 +5,7 @@ from conan.test.utils.tools import TestClient +@pytest.mark.tool("ninja") @pytest.mark.tool("meson") @pytest.mark.tool("pkg_config") def test_meson_lib_template(): @@ -31,6 +32,7 @@ def test_meson_lib_template(): assert "hello/0.1: Hello World Release!" in client.out +@pytest.mark.tool("ninja") @pytest.mark.tool("meson") def test_meson_exe_template(): client = TestClient(path_with_spaces=False) diff --git a/test/functional/toolchains/microsoft/test_msbuilddeps.py b/test/functional/toolchains/microsoft/test_msbuilddeps.py index 0143c210d6e..6e55397910e 100644 --- a/test/functional/toolchains/microsoft/test_msbuilddeps.py +++ b/test/functional/toolchains/microsoft/test_msbuilddeps.py @@ -1,17 +1,15 @@ import os import platform import textwrap -import unittest import pytest -from parameterized import parameterized_class from conan.test.assets.genconanfile import GenConanfile from conan.test.assets.sources import gen_function_cpp, gen_function_h from conan.test.assets.visual_project_files import get_vs_project_files -from test.conftest import tools_locations from conan.test.utils.tools import TestClient, NO_SETTINGS_PACKAGE_ID + sln_file = r""" Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 15 @@ -397,256 +395,259 @@ """ -vs_versions = [{"vs_version": "191", "msvc_version": "191", "ide_year": "2017", "toolset": "v141"}] +@pytest.mark.tool("cmake") +@pytest.mark.tool("visual_studio") +@pytest.mark.skipif(platform.system() != "Windows", reason="Requires MSBuild") +def test_msbuild_generator(): + client = TestClient() + client.run("new cmake_lib -d name=hello0 -d version=1.0") + client.run("create . -tf=") + client.save({}, clean_first=True) + client.run("new cmake_lib -d name=hello3 -d version=1.0") + client.run("create . -tf=") + client.save({}, clean_first=True) + client.run("new cmake_lib -d name=hello1 -d version=1.0 -d requires=hello0/1.0") + client.run("create . -tf=") + client.save({}, clean_first=True) -if "17" in tools_locations['visual_studio'] and not tools_locations['visual_studio']['17'].get('disabled', False): - vs_versions.append({"vs_version": "193", "msvc_version": "19.3", "ide_year": "2022", "toolset": "v143"}) + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.microsoft import MSBuild + class HelloConan(ConanFile): + settings = "os", "build_type", "compiler", "arch" + requires = "hello1/1.0", "hello3/1.0" + generators = "MSBuildDeps", "MSBuildToolchain" + def build(self): + msbuild = MSBuild(self) + msbuild.build("MyProject.sln") + """) + myapp_cpp = gen_function_cpp(name="main", msg="MyApp", + includes=["hello1"], calls=["hello1"]) + myproject_cpp = gen_function_cpp(name="main", msg="MyProject", includes=["hello3"], + calls=["hello3"]) + files = {"MyProject.sln": sln_file, + "MyProject/MyProject.vcxproj": myproject_vcxproj, + "MyProject/MyProject.cpp": myproject_cpp, + "MyApp/MyApp.vcxproj": myapp_vcxproj, + "MyApp/MyApp.cpp": myapp_cpp, + "conanfile.py": conanfile} + + client.save(files, clean_first=True) + client.run("install .") + client.run("build .") + assert "warning MSB4011" not in client.out + client.run_command(r"x64\Release\MyProject.exe") + assert "MyProject: Release!" in client.out + assert "hello3/1.0: Hello World Release!" in client.out + client.run_command(r"x64\Release\MyApp.exe") + assert "MyApp: Release!" in client.out + assert "hello0/1.0: Hello World Release!" in client.out + assert "hello1/1.0: Hello World Release" in client.out -@parameterized_class(vs_versions) @pytest.mark.tool("visual_studio") @pytest.mark.skipif(platform.system() != "Windows", reason="Requires MSBuild") -class MSBuildGeneratorTest(unittest.TestCase): - - @pytest.mark.tool("cmake") - def test_msbuild_generator(self): - client = TestClient() - client.run("new cmake_lib -d name=hello0 -d version=1.0") - client.run("create . -tf=") - client.save({}, clean_first=True) - client.run("new cmake_lib -d name=hello3 -d version=1.0") - client.run("create . -tf=") - client.save({}, clean_first=True) - client.run("new cmake_lib -d name=hello1 -d version=1.0 -d requires=hello0/1.0") - client.run("create . -tf=") - client.save({}, clean_first=True) - - conanfile = textwrap.dedent(""" - from conan import ConanFile - from conan.tools.microsoft import MSBuild - class HelloConan(ConanFile): - settings = "os", "build_type", "compiler", "arch" - requires = "hello1/1.0", "hello3/1.0" - generators = "MSBuildDeps", "MSBuildToolchain" - def build(self): - msbuild = MSBuild(self) - msbuild.build("MyProject.sln") - """) - myapp_cpp = gen_function_cpp(name="main", msg="MyApp", - includes=["hello1"], calls=["hello1"]) - myproject_cpp = gen_function_cpp(name="main", msg="MyProject", includes=["hello3"], - calls=["hello3"]) - files = {"MyProject.sln": sln_file, - "MyProject/MyProject.vcxproj": myproject_vcxproj, - "MyProject/MyProject.cpp": myproject_cpp, - "MyApp/MyApp.vcxproj": myapp_vcxproj, - "MyApp/MyApp.cpp": myapp_cpp, - "conanfile.py": conanfile} - - client.save(files, clean_first=True) - client.run("install .") - client.run("build .") - self.assertNotIn("warning MSB4011", client.out) - client.run_command(r"x64\Release\MyProject.exe") - self.assertIn("MyProject: Release!", client.out) - self.assertIn("hello3/1.0: Hello World Release!", client.out) - client.run_command(r"x64\Release\MyApp.exe") - self.assertIn("MyApp: Release!", client.out) - self.assertIn("hello0/1.0: Hello World Release!", client.out) - self.assertIn("hello1/1.0: Hello World Release", client.out) - - def test_install_reference(self): - client = TestClient() - client.save({"conanfile.py": GenConanfile()}) - client.run("create . --name=mypkg --version=0.1") - client.run("install --requires=mypkg/0.1@ -g MSBuildDeps") - self.assertIn("Generator 'MSBuildDeps' calling 'generate()'", client.out) - # https://github.com/conan-io/conan/issues/8163 - props = client.load("conan_mypkg_vars_release_x64.props") # default Release/x64 - folder = props[props.find("")+len("") - :props.find("")] - self.assertTrue(os.path.isfile(os.path.join(folder, "conaninfo.txt"))) - - def test_install_reference_gcc(self): - client = TestClient() - client.save({"conanfile.py": GenConanfile()}) - client.run("create . --name=pkg --version=1.0") - - conanfile = textwrap.dedent(""" - from conan import ConanFile - class Pkg(ConanFile): - settings = "os", "compiler", "arch", "build_type" - generators = "MSBuildDeps" - requires = "pkg/1.0" - """) - client.save({"conanfile.py": conanfile}) - - client.run('install . -s os=Windows -s compiler=msvc ' - '-s compiler.version={vs_version}' - ' -s compiler.runtime=dynamic'.format(vs_version=self.vs_version)) - self.assertIn("conanfile.py: Generator 'MSBuildDeps' calling 'generate()'", client.out) - props = client.load("conan_pkg_release_x64.props") - self.assertIn('', props) - # This will overwrite the existing one, cause configuration and arch is the same - client.run("install . -s os=Linux -s compiler=gcc -s compiler.version=5.2 '" - "'-s compiler.libcxx=libstdc++") - self.assertIn("conanfile.py: Generator 'MSBuildDeps' calling 'generate()'", client.out) - pkg_props = client.load("conan_pkg.props") - self.assertIn('Project="conan_pkg_release_x64.props"', pkg_props) - - def test_no_build_type_error(self): - client = TestClient() - client.save({"conanfile.py": GenConanfile(), - "profile": "[settings]\nos=Windows\ncompiler=msvc\narch=x86_64"}) - client.run("create . --name=mypkg --version=0.1") - client.run("install --requires=mypkg/0.1@ -g MSBuildDeps -pr=profile", assert_error=True) - self.assertIn("The 'msbuild' generator requires a 'build_type' setting value", client.out) - - def test_custom_configuration(self): - client = TestClient() - client.save({"conanfile.py": GenConanfile()}) - client.run("create . --name=pkg --version=1.0") - - conanfile = textwrap.dedent(""" - from conan import ConanFile - from conan.tools.microsoft import MSBuildDeps - class Pkg(ConanFile): - settings = "os", "compiler", "arch", "build_type" - requires = "pkg/1.0" - def generate(self): - ms = MSBuildDeps(self) - ms.configuration = "My"+str(self.settings.build_type) - ms.platform = "My"+str(self.settings.arch) - ms.generate() - """) - client.save({"conanfile.py": conanfile}) - - client.run('install . -s os=Windows -s compiler=msvc ' - '-s compiler.version={vs_version}' - ' -s compiler.runtime=dynamic'.format(vs_version=self.vs_version)) - props = client.load("conan_pkg_myrelease_myx86_64.props") - self.assertIn('', props) - client.run('install . -s os=Windows -s compiler=msvc ' - '-s compiler.version={vs_version}' - ' -s compiler.runtime=dynamic -s arch=x86 ' - '-s build_type=Debug'.format(vs_version=self.vs_version)) - props = client.load("conan_pkg_mydebug_myx86.props") - self.assertIn('', props) - props = client.load("conan_pkg.props") - self.assertIn("conan_pkg_myrelease_myx86_64.props", props) - self.assertIn("conan_pkg_mydebug_myx86.props", props) - - def test_custom_configuration_errors(self): - client = TestClient() - client.save({"conanfile.py": GenConanfile()}) - client.run("create . --name=pkg --version=1.0") - - conanfile = textwrap.dedent(""" - from conan import ConanFile - from conan.tools.microsoft import MSBuildDeps - class Pkg(ConanFile): - settings = "os", "compiler", "arch", "build_type" - requires = "pkg/1.0" - def generate(self): - ms = MSBuildDeps(self) - ms.configuration = None - ms.generate() - """) - client.save({"conanfile.py": conanfile}) - - client.run('install . -s os=Windows -s compiler=msvc' - ' -s compiler.version={vs_version}' - ' -s compiler.runtime=dynamic'.format(vs_version=self.vs_version), assert_error=True) - self.assertIn("MSBuildDeps.configuration is None, it should have a value", client.out) - client.save({"conanfile.py": conanfile.replace("configuration", "platform")}) - - client.run('install . -s os=Windows -s compiler=msvc' - ' -s compiler.version={vs_version}' - ' -s compiler.runtime=dynamic'.format(vs_version=self.vs_version), assert_error=True) - self.assertIn("MSBuildDeps.platform is None, it should have a value", client.out) - - def test_install_transitive(self): - # https://github.com/conan-io/conan/issues/8065 - client = TestClient() - client.save({"conanfile.py": GenConanfile()}) - client.run("create . --name=pkga --version=1.0") - client.save({"conanfile.py": GenConanfile().with_requires("pkga/1.0")}) - client.run("create . --name=pkgb --version=1.0") - - conanfile = textwrap.dedent(""" - from conan import ConanFile - from conan.tools.microsoft import MSBuild - class HelloConan(ConanFile): - settings = "os", "build_type", "compiler", "arch" - requires = "pkgb/1.0@", "pkga/1.0" - generators = "MSBuildDeps", "MSBuildToolchain" - def build(self): - msbuild = MSBuild(self) - msbuild.build("MyProject.sln") - """) - myapp_cpp = gen_function_cpp(name="main", msg="MyApp") - myproject_cpp = gen_function_cpp(name="main", msg="MyProject") - files = {"MyProject.sln": sln_file, - "MyProject/MyProject.vcxproj": myproject_vcxproj.replace("conan_hello3.props", - "conandeps.props"), - "MyProject/MyProject.cpp": myproject_cpp, - "MyApp/MyApp.vcxproj": myapp_vcxproj.replace("conan_hello1.props", - "conandeps.props"), - "MyApp/MyApp.cpp": myapp_cpp, - "conanfile.py": conanfile} - - client.save(files, clean_first=True) - client.run("install .") - client.run("build .") - self.assertNotIn("warning MSB4011", client.out) - - def test_install_build_requires(self): - # https://github.com/conan-io/conan/issues/8170 - client = TestClient() - client.save({"conanfile.py": GenConanfile()}) - client.run("create . --name=tool --version=1.0") - - conanfile = textwrap.dedent(""" - from conan import ConanFile - from conan.tools.files import load - class HelloConan(ConanFile): - settings = "os", "build_type", "compiler", "arch" - build_requires = "tool/1.0" - generators = "MSBuildDeps" - - def build(self): - deps = load(self, "conandeps.props") - assert "conan_tool.props" not in deps - self.output.info("Conan_tools.props not in deps") - """) - client.save({"conanfile.py": conanfile}) - client.run("install .") - deps = client.load("conandeps.props") - self.assertNotIn("conan_tool.props", deps) - client.run("create . --name=pkg --version=0.1") - self.assertIn("Conan_tools.props not in deps", client.out) - - def test_install_transitive_build_requires(self): - # https://github.com/conan-io/conan/issues/8170 - client = TestClient() - client.save({"conanfile.py": GenConanfile()}) - client.run("create . --name=dep --version=1.0") - client.run("create . --name=tool_build --version=1.0") - client.run("create . --name=tool_test --version=1.0") - conanfile = GenConanfile().with_requires("dep/1.0").\ - with_tool_requires("tool_build/1.0").\ - with_test_requires("tool_test/1.0") - client.save({"conanfile.py": conanfile}) - client.run("create . --name=pkg --version=1.0") - - client.save({"conanfile.py": GenConanfile(). - with_settings("os", "compiler", "arch", "build_type"). - with_requires("pkg/1.0")}, clean_first=True) - client.run("install . -g MSBuildDeps -pr:b=default -pr:h=default") - pkg = client.load("conan_pkg_release_x64.props") - assert "conan_dep.props" in pkg - assert "tool_test" not in pkg # test requires are not there - assert "tool_build" not in pkg +def test_install_reference_gcc(): + client = TestClient() + client.save({"conanfile.py": GenConanfile()}) + client.run("create . --name=pkg --version=1.0") + + conanfile = textwrap.dedent(""" + from conan import ConanFile + class Pkg(ConanFile): + settings = "os", "compiler", "arch", "build_type" + generators = "MSBuildDeps" + requires = "pkg/1.0" + """) + client.save({"conanfile.py": conanfile}) + + client.run('install . -s os=Windows -s compiler=msvc ' + '-s compiler.version=194 -s compiler.runtime=dynamic') + assert "conanfile.py: Generator 'MSBuildDeps' calling 'generate()'" in client.out + props = client.load("conan_pkg_release_x64.props") + assert '' in props + # This will overwrite the existing one, cause configuration and arch is the same + client.run("install . -s os=Linux -s compiler=gcc -s compiler.version=5.2 '" + "'-s compiler.libcxx=libstdc++") + assert "conanfile.py: Generator 'MSBuildDeps' calling 'generate()'" in client.out + pkg_props = client.load("conan_pkg.props") + assert 'Project="conan_pkg_release_x64.props"' in pkg_props + + +@pytest.mark.tool("visual_studio") +@pytest.mark.skipif(platform.system() != "Windows", reason="Requires MSBuild") +def test_custom_configuration(): + client = TestClient() + client.save({"conanfile.py": GenConanfile()}) + client.run("create . --name=pkg --version=1.0") + + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.microsoft import MSBuildDeps + class Pkg(ConanFile): + settings = "os", "compiler", "arch", "build_type" + requires = "pkg/1.0" + def generate(self): + ms = MSBuildDeps(self) + ms.configuration = "My"+str(self.settings.build_type) + ms.platform = "My"+str(self.settings.arch) + ms.generate() + """) + client.save({"conanfile.py": conanfile}) + + client.run('install . -s os=Windows -s compiler=msvc ' + '-s compiler.version=194 -s compiler.runtime=dynamic') + props = client.load("conan_pkg_myrelease_myx86_64.props") + assert '' in props + client.run('install . -s os=Windows -s compiler=msvc ' + '-s compiler.version=194 -s compiler.runtime=dynamic -s arch=x86 ' + '-s build_type=Debug') + props = client.load("conan_pkg_mydebug_myx86.props") + assert '' in props + props = client.load("conan_pkg.props") + assert "conan_pkg_myrelease_myx86_64.props" in props + assert "conan_pkg_mydebug_myx86.props" in props + + +def test_custom_configuration_errors(): + client = TestClient() + client.save({"conanfile.py": GenConanfile()}) + client.run("create . --name=pkg --version=1.0") + + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.microsoft import MSBuildDeps + class Pkg(ConanFile): + settings = "os", "compiler", "arch", "build_type" + requires = "pkg/1.0" + def generate(self): + ms = MSBuildDeps(self) + ms.configuration = None + ms.generate() + """) + client.save({"conanfile.py": conanfile}) + + client.run('install . -s os=Windows -s compiler=msvc' + ' -s compiler.version=194 -s compiler.runtime=dynamic', assert_error=True) + assert "MSBuildDeps.configuration is None, it should have a value" in client.out + client.save({"conanfile.py": conanfile.replace("configuration", "platform")}) + + client.run('install . -s os=Windows -s compiler=msvc' + ' -s compiler.version=194 -s compiler.runtime=dynamic', assert_error=True) + assert "MSBuildDeps.platform is None, it should have a value" in client.out + + +@pytest.mark.tool("visual_studio") +@pytest.mark.skipif(platform.system() != "Windows", reason="Requires MSBuild") +def test_install_transitive(): + # https://github.com/conan-io/conan/issues/8065 + client = TestClient() + client.save({"conanfile.py": GenConanfile()}) + client.run("create . --name=pkga --version=1.0") + client.save({"conanfile.py": GenConanfile().with_requires("pkga/1.0")}) + client.run("create . --name=pkgb --version=1.0") + + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.microsoft import MSBuild + class HelloConan(ConanFile): + settings = "os", "build_type", "compiler", "arch" + requires = "pkgb/1.0@", "pkga/1.0" + generators = "MSBuildDeps", "MSBuildToolchain" + def build(self): + msbuild = MSBuild(self) + msbuild.build("MyProject.sln") + """) + myapp_cpp = gen_function_cpp(name="main", msg="MyApp") + myproject_cpp = gen_function_cpp(name="main", msg="MyProject") + files = {"MyProject.sln": sln_file, + "MyProject/MyProject.vcxproj": myproject_vcxproj.replace("conan_hello3.props", + "conandeps.props"), + "MyProject/MyProject.cpp": myproject_cpp, + "MyApp/MyApp.vcxproj": myapp_vcxproj.replace("conan_hello1.props", + "conandeps.props"), + "MyApp/MyApp.cpp": myapp_cpp, + "conanfile.py": conanfile} + + client.save(files, clean_first=True) + client.run("build .") + assert "warning MSB4011" not in client.out + + +def test_no_build_type_error(): + client = TestClient() + client.save({"conanfile.py": GenConanfile(), + "profile": "[settings]\nos=Windows\ncompiler=msvc\narch=x86_64"}) + client.run("create . --name=mypkg --version=0.1") + client.run("install --requires=mypkg/0.1@ -g MSBuildDeps -pr=profile", assert_error=True) + assert "The 'msbuild' generator requires a 'build_type' setting value" in client.out + + +def test_install_build_requires(): + # https://github.com/conan-io/conan/issues/8170 + client = TestClient() + client.save({"conanfile.py": GenConanfile()}) + client.run("create . --name=tool --version=1.0") + + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.files import load + class HelloConan(ConanFile): + settings = "os", "build_type", "compiler", "arch" + build_requires = "tool/1.0" + generators = "MSBuildDeps" + + def build(self): + deps = load(self, "conandeps.props") + assert "conan_tool.props" not in deps + self.output.info("Conan_tools.props not in deps") + """) + client.save({"conanfile.py": conanfile}) + client.run("install .") + deps = client.load("conandeps.props") + assert "conan_tool.props" not in deps + client.run("create . --name=pkg --version=0.1") + assert "Conan_tools.props not in deps" in client.out + + +@pytest.mark.tool("visual_studio") +@pytest.mark.skipif(platform.system() != "Windows", reason="Requires MSBuild") +def test_install_transitive_build_requires(): + # https://github.com/conan-io/conan/issues/8170 + client = TestClient() + client.save({"conanfile.py": GenConanfile()}) + client.run("create . --name=dep --version=1.0") + client.run("create . --name=tool_build --version=1.0") + client.run("create . --name=tool_test --version=1.0") + conanfile = GenConanfile().with_requires("dep/1.0").\ + with_tool_requires("tool_build/1.0").\ + with_test_requires("tool_test/1.0") + client.save({"conanfile.py": conanfile}) + client.run("create . --name=pkg --version=1.0") + + client.save({"conanfile.py": GenConanfile(). + with_settings("os", "compiler", "arch", "build_type"). + with_requires("pkg/1.0")}, clean_first=True) + client.run("install . -g MSBuildDeps -pr:b=default -pr:h=default") + pkg = client.load("conan_pkg_release_x64.props") + assert "conan_dep.props" in pkg + assert "tool_test" not in pkg # test requires are not there + assert "tool_build" not in pkg + + +@pytest.mark.tool("visual_studio") +@pytest.mark.skipif(platform.system() != "Windows", reason="Requires MSBuild") +def test_install_reference(): + client = TestClient() + client.save({"conanfile.py": GenConanfile()}) + client.run("create . --name=mypkg --version=0.1") + client.run("install --requires=mypkg/0.1@ -g MSBuildDeps") + assert "Generator 'MSBuildDeps' calling 'generate()'" in client.out + # https://github.com/conan-io/conan/issues/8163 + props = client.load("conan_mypkg_vars_release_x64.props") # default Release/x64 + folder = props[props.find("")+len("") + :props.find("")] + assert os.path.isfile(os.path.join(folder, "conaninfo.txt")) @pytest.mark.parametrize("pattern,exclude_a,exclude_b", @@ -833,7 +834,7 @@ class HelloConan(ConanFile): settings = "os", "build_type", "compiler", "arch" generators = "MSBuildDeps", "MSBuildToolchain" - def requirements(self): + def build_requirements(self): self.test_requires("mydep.pkg.team/0.1") def build(self): @@ -922,14 +923,16 @@ def build(self): - + Application false v141 true Unicode - + Application false v141 diff --git a/test/functional/toolchains/microsoft/test_msbuildtoolchain.py b/test/functional/toolchains/microsoft/test_msbuildtoolchain.py index 60b1d32893b..46f3743d12e 100644 --- a/test/functional/toolchains/microsoft/test_msbuildtoolchain.py +++ b/test/functional/toolchains/microsoft/test_msbuildtoolchain.py @@ -48,3 +48,43 @@ def test_msbuildtoolchain_winsdk_version(): # I have verified also opening VS IDE that the setting is correctly configured # because the test always run over vcvars that already activates it assert "amd64 - winsdk_version=10.0 - vcvars_ver=14.3" in client.out + + +@pytest.mark.skipif(platform.system() != "Windows", reason="Requires Windows") +def test_msbuildtoolchain_compiler_update(): + # It only works for update=8, because 19.38 is the compiler in Github actions! + client = TestClient(path_with_spaces=False) + client.run("new msbuild_lib -d name=hello -d version=0.1") + # conantoolchain.props is already imported in the msbuild_exe tempalte + client.run("create . -s arch=x86_64 -s compiler.version=193 -s compiler.update=8") + # I have verified also opening VS IDE that the setting is correctly configured + # because the test always run over vcvars that already activates it + assert "amd64 - winsdk_version=None - vcvars_ver=14.38" in client.out + assert "hello/0.1: _MSC_VER1938" in client.out + + +@pytest.mark.tool("visual_studio") +@pytest.mark.skipif(platform.system() != "Windows", reason="Only for windows") +class TestTxtCommandLineMSBuild: + + def test_declarative(self): + conanfile = textwrap.dedent(""" + [generators] + MSBuildToolchain + """) + client = TestClient() + client.save({"conanfile.txt": conanfile}) + client.run("install .") + self._check(client) + + @staticmethod + def _check(client): + assert "conanfile.txt: Generator 'MSBuildToolchain' calling 'generate()'" in client.out + toolchain = client.load("conantoolchain.props") + assert "Pkg", conan_toolchain_props) - self.assertIn("0.1", conan_toolchain_props) + assert "Pkg" in conan_toolchain_props + assert "0.1" in conan_toolchain_props diff --git a/test/functional/toolchains/test_nmake_toolchain.py b/test/functional/toolchains/test_nmake_toolchain.py index 36a4121e4dc..b1267378c57 100644 --- a/test/functional/toolchains/test_nmake_toolchain.py +++ b/test/functional/toolchains/test_nmake_toolchain.py @@ -79,41 +79,37 @@ def build(self): """) client.save({"conanfile.py": conanfile, "makefile": makefile, - "simple.cpp": gen_function_cpp(name="main", includes=["dep"], calls=["dep"], preprocessor=conf_preprocessors.keys())}, + "simple.cpp": gen_function_cpp(name="main", includes=["dep"], calls=["dep"], + preprocessor=conf_preprocessors.keys())}, clean_first=True) client.run(f"build . {settings} {conf}") client.run_command("simple.exe") assert "dep/1.0" in client.out - check_exe_run(client.out, "main", "msvc", version, build_type, "x86_64", cppstd, conf_preprocessors) + check_exe_run(client.out, "main", "msvc", version, build_type, "x86_64", cppstd, + conf_preprocessors) @pytest.mark.skipif(platform.system() != "Windows", reason="Requires Windows") -@pytest.mark.tool("clang", "18") +@pytest.mark.tool("cmake", "3.23") +# This test uses clang inside Visual Studio, not managed by mark.tool +@pytest.mark.tool("visual_studio") def test_toolchain_nmake_clang(): - compiler = "clang" - version = "18" - runtime = "dynamic" cppstd = "14" build_type = "Debug" - defines = ["TEST_DEFINITION1", "TEST_DEFINITION2=0", "TEST_DEFINITION3=", "TEST_DEFINITION4=TestPpdValue4", - "TEST_DEFINITION5=__declspec(dllexport)", "TEST_DEFINITION6=foo bar"] - cflags = cxxflags = sharedlinkflags = exelinkflags = [] + defines = ["TEST_DEFINITION1", "TEST_DEFINITION2=0", "TEST_DEFINITION3=", + "TEST_DEFINITION4=TestPpdValue4", "TEST_DEFINITION5=__declspec(dllexport)", + "TEST_DEFINITION6=foo bar"] client = TestClient(path_with_spaces=False) - settings = {"compiler": compiler, - "compiler.version": version, - "compiler.cppstd": cppstd, - "compiler.runtime": runtime, + settings = {"compiler": "clang", + "compiler.version": "20", + "compiler.cppstd": "14", + "compiler.runtime": "dynamic", "build_type": build_type, "compiler.runtime_version": "v144", "arch": "x86_64"} - serialize_array = lambda arr: "[{}]".format(",".join([f"'{v}'" for v in arr])) conf = { - "tools.build:defines": serialize_array(defines) if defines else "", - "tools.build:cflags": serialize_array(cflags) if cflags else "", - "tools.build:cxxflags": serialize_array(cxxflags) if cxxflags else "", - "tools.build:sharedlinkflags": serialize_array(sharedlinkflags) if sharedlinkflags else "", - "tools.build:exelinkflags": serialize_array(exelinkflags) if exelinkflags else "", + "tools.build:defines": "[{}]".format(",".join([f"'{v}'" for v in defines])), "tools.build:compiler_executables": r'{\"c\": \"clang-cl\", \"cpp\": \"clang-cl\"}', "tools.cmake.cmaketoolchain:generator": "Visual Studio 17", } @@ -154,9 +150,11 @@ def build(self): """) client.save({"conanfile.py": conanfile, "makefile": makefile, - "simple.cpp": gen_function_cpp(name="main", includes=["dep"], calls=["dep"], preprocessor=conf_preprocessors.keys())}, + "simple.cpp": gen_function_cpp(name="main", includes=["dep"], calls=["dep"], + preprocessor=conf_preprocessors.keys())}, clean_first=True) client.run(f"build . {settings} {conf}") client.run_command("simple.exe") assert "dep/1.0" in client.out - check_exe_run(client.out, "main", "clang", "19.1", build_type, "x86_64", cppstd, conf_preprocessors) + check_exe_run(client.out, "main", "clang", "19.1", build_type, "x86_64", cppstd, + conf_preprocessors) diff --git a/test/functional/toolchains/test_premake.py b/test/functional/toolchains/test_premake.py index 59a47b4f2a7..85921c6eb9a 100644 --- a/test/functional/toolchains/test_premake.py +++ b/test/functional/toolchains/test_premake.py @@ -10,7 +10,9 @@ from conan.test.utils.tools import TestClient from conan.test.assets.sources import gen_function_cpp -@pytest.mark.skipif(platform.machine() != "x86_64", reason="Premake Legacy generator only supports x86_64 machines") + +@pytest.mark.skipif(platform.machine() != "x86_64", + reason="Premake Legacy generator only supports x86_64 machines") @pytest.mark.tool("premake") def test_premake_legacy(matrix_client): c = matrix_client @@ -78,7 +80,6 @@ def build(self): assert "matrix/1.0: Hello World Debug!" in c.out - @pytest.mark.tool("premake") def test_premake_new_generator(): c = TestClient() @@ -112,7 +113,8 @@ def test_premake_components(transitive_libs): includedirs=[".", "libb1", "libb2"], projects=[ {"name": "libb1", "files": ["libb1/*.h", "libb1/*.cpp"], "kind": "StaticLib"}, - {"name": "libb2", "files": ["libb2/*.h", "libb2/*.cpp"], "kind": "StaticLib", "links": ["libb1"]} + {"name": "libb2", "files": ["libb2/*.h", "libb2/*.cpp"], "kind": "StaticLib", + "links": ["libb1"]} ] ) libb_conanfile = textwrap.dedent(""" @@ -144,11 +146,14 @@ def build(self): def package(self): for lib in ("libb1", "libb2"): - copy(self, "*.h", os.path.join(self.source_folder, lib), os.path.join(self.package_folder, "include", lib)) + copy(self, "*.h", os.path.join(self.source_folder, lib), + os.path.join(self.package_folder, "include", lib)) for pattern in ("*.lib", "*.a", "*.so*", "*.dylib"): - copy(self, pattern, os.path.join(self.build_folder, "bin"), os.path.join(self.package_folder, "lib")) - copy(self, "*.dll", os.path.join(self.build_folder, "bin"), os.path.join(self.package_folder, "bin")) + copy(self, pattern, os.path.join(self.build_folder, "bin"), + os.path.join(self.package_folder, "lib")) + copy(self, "*.dll", os.path.join(self.build_folder, "bin"), + os.path.join(self.package_folder, "bin")) def package_info(self): self.cpp_info.components["libb1"].libs = ["libb1"] @@ -187,9 +192,11 @@ def package_info(self): # Create a consumer application which depends on libb c.run("new premake_exe -d name=consumer -d version=1.0 -o consumer -d requires=libb/1.0") # Adapt includes and usage of libb in the consumer application - replace_in_file(ConanFileMock(), os.path.join(c.current_folder, "consumer", "src", "consumer.cpp"), + replace_in_file(ConanFileMock(), + os.path.join(c.current_folder, "consumer", "src", "consumer.cpp"), '#include "libb.h"', '#include "libb1/libb1.h"\n#include "libb2/libb2.h"') - replace_in_file(ConanFileMock(), os.path.join(c.current_folder, "consumer", "src", "consumer.cpp"), + replace_in_file(ConanFileMock(), + os.path.join(c.current_folder, "consumer", "src", "consumer.cpp"), 'libb()', 'libb1();libb2()') c.run("create liba") @@ -250,3 +257,57 @@ def build(self): c.run("build .", assert_error=True) # Error should be about not finding matrix +@pytest.mark.tool("premake") +def test_premake_custom_configuration(transitive_libraries): + c = transitive_libraries + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.layout import basic_layout + from conan.tools.premake import Premake, PremakeDeps, PremakeToolchain + + class ConsumerRecipe(ConanFile): + name = "consumer" + version = "1.0" + package_type = "application" + settings = "os", "compiler", "build_type", "arch" + options = {"sanitizer": [True, False]} + default_options = {"sanitizer": False} + exports_sources = "*" + + def layout(self): + basic_layout(self) + + @property + def _premake_configuration(self): + return str(self.settings.build_type) + "Sanitizer" if self.options.sanitizer else "" + + def generate(self): + deps = PremakeDeps(self) + deps.configuration = self._premake_configuration + deps.generate() + tc = PremakeToolchain(self) + tc.generate() + + def build(self): + premake = Premake(self) + premake.configure() + # premake.build(workspace="Consumer") + premake.build(workspace="Consumer", configuration=self._premake_configuration) + """) + + c.save({"src/main.cpp": gen_function_cpp(name="main"), + "premake5.lua": gen_premake5( + workspace="Consumer", + projects=[ + { + "name": "consumer", + "files": ["src/main.cpp"], + "kind": "ConsoleApp", + } + ], + configurations=["Debug", "Release", "DebugSanitizer", "ReleaseSanitizer"], + ), + "conanfile.py": conanfile, + }) + # Test it builds successfully with the custom configuration + c.run("build . -o sanitizer=True") diff --git a/test/functional/toolchains/test_txt_cmdline.py b/test/functional/toolchains/test_txt_cmdline.py deleted file mode 100644 index 8e827822705..00000000000 --- a/test/functional/toolchains/test_txt_cmdline.py +++ /dev/null @@ -1,61 +0,0 @@ -import platform -import textwrap -import unittest - -import pytest - -from conan.test.utils.tools import TestClient - - -class TestTxtCommandLine(unittest.TestCase): - - def test_declarative(self): - conanfile = textwrap.dedent(""" - [generators] - CMakeToolchain - MesonToolchain - """) - client = TestClient() - client.save({"conanfile.txt": conanfile}) - client.run("install .") - self._check(client) - - def _check(self, client): - self.assertIn("conanfile.txt: Generator 'CMakeToolchain' calling 'generate()'", client.out) - self.assertIn("conanfile.txt: Generator 'MesonToolchain' calling 'generate()'", client.out) - toolchain = client.load("conan_toolchain.cmake") - self.assertIn("Conan automatically generated toolchain file", toolchain) - toolchain = client.load("conan_meson_native.ini") - self.assertIn("[project options]", toolchain) - - def test_command_line(self): - client = TestClient() - client.save({"conanfile.txt": ""}) - client.run("install . -g CMakeToolchain -g MesonToolchain ") - self._check(client) - - -@pytest.mark.tool("visual_studio") -@pytest.mark.skipif(platform.system() != "Windows", reason="Only for windows") -class TestTxtCommandLineMSBuild(unittest.TestCase): - - def test_declarative(self): - conanfile = textwrap.dedent(""" - [generators] - MSBuildToolchain - """) - client = TestClient() - client.save({"conanfile.txt": conanfile}) - client.run("install .") - self._check(client) - - def _check(self, client): - self.assertIn("conanfile.txt: Generator 'MSBuildToolchain' calling 'generate()'", client.out) - toolchain = client.load("conantoolchain.props") - self.assertIn(" protobuf (shared) + # \---tool-require-> grpc/ (shared) -> protobuf (shared) + # \--tool-require-------(host)----------------/ + tc = TestClient(light=True) + tc.save({"grpc/conanfile.py": GenConanfile("grpc", "0.1").with_shared_option(shared) + .with_requirement("protobuf/0.1"), + "protobuf/conanfile.py": GenConanfile("protobuf", "0.1") + .with_shared_option(shared), + "conanfile.py": GenConanfile("app", "0.1").with_requires("protobuf/[*]") + .with_tool_requirement("grpc/[*]") + .with_tool_requirement("protobuf/") + }) + tc.run("export protobuf") + tc.run("export grpc") + tc.run("graph info .") + assert "Conflict between" not in tc.out + + @pytest.mark.parametrize("shared", [True, False]) + def test_host_version_transitive_contexts2(self, shared): + # app -> grpc (shared) -> protobuf (shared) + # \-----------------------/ + # \---tool-require-> grpc/ (shared) -> protobuf (shared) + # \--tool-require-------(host)----------------/ + tc = TestClient(light=True) + tc.save({"grpc/conanfile.py": GenConanfile("grpc", "0.1").with_shared_option(shared) + .with_requirement("protobuf/0.1"), + "protobuf/conanfile.py": GenConanfile("protobuf", "0.1") + .with_shared_option(shared), + "conanfile.py": GenConanfile("app", "0.1").with_requires("grpc/0.1") + .with_requires("protobuf/[*]") + .with_tool_requirement("grpc/") + .with_tool_requirement("protobuf/") + }) + tc.run("export protobuf") + tc.run("export grpc") + tc.run("graph info .") + assert "Conflict between" not in tc.out + + def test_host_version_transitive_contexts_orphan(self): + tc = TestClient(light=True) + tc.save({"grpc/conanfile.py": GenConanfile("grpc", "0.1") + .with_requirement("protobuf/0.1").with_shared_option(False), + "protobuf/conanfile.py": GenConanfile("protobuf", "0.1").with_shared_option(False), + "conanfile.py": GenConanfile("app", "0.1").with_requires("grpc/0.1") + .with_requires("protobuf/[*]") + .with_tool_requirement("protobuf/") + .with_tool_requirement("grpc/")}) + tc.run("export protobuf") + tc.run("export grpc") + tc.run("graph info . -o:a=*:shared=True -f=json", redirect_stdout="graph.json") + data = json.loads(tc.load("graph.json")) + + def _assert_no_orphan(deps_graph): + ids = set(deps_graph["nodes"].keys()) + seen = set(deps_graph["root"].keys()) + for node in deps_graph["nodes"].values(): + seen.update(node["dependencies"].keys()) + assert not ids - seen, f"Orphan nodes found: {ids - seen}" + _assert_no_orphan(data["graph"]) + + def test_user_channel_error(self): + lib = textwrap.dedent(""" + from conan import ConanFile + class LibWithToolConan(ConanFile): + name = "lib_with_tool" + package_type = "static-library" + settings = "os", "arch" + # {} + """) + c = TestClient(default_server_user=True) + c.save({"conanfile.py": lib.format(1)}) + c.run("create . --version 1.0.0 --user foobar") + c.save({"conanfile.py": lib.format(2)}) + c.run("create . --version 1.1.0 --user foobar") + rev1 = c.exported_recipe_revision() + c.save({"conanfile.py": lib.format(3)}) + c.run("create . --version 1.1.0 --user foobar") + rev2 = c.exported_recipe_revision() + assert rev1 != rev2 + + c.run(f"upload lib_with_tool/1.1.0@foobar#{rev2} -r=default") + c.run(f"remove lib_with_tool/1.1.0@foobar#{rev2} -c") + + dep = textwrap.dedent(""" + from conan import ConanFile + class DepLibConan(ConanFile): + name = "dep_lib" + + package_type = "static-library" + settings = "os", "arch" + + def build_requirements(self): + self.tool_requires("lib_with_tool/@foobar") + + def requirements(self): + self.requires("lib_with_tool/1.1.0@foobar") + """) + c.save({"conanfile.py": dep}, clean_first=True) + c.run("create . --version=1.0.0 --user=foobar") + + conanfile = textwrap.dedent(f""" + [requires] + dep_lib/1.0.0@foobar + lib_with_tool/1.1.0@foobar#{rev2} + """) + c.save({"conanfile.txt": conanfile}, clean_first=True) + c.run("install .") + # No longer produces a conflict + def test_build_missing_build_requires(): c = TestClient(light=True) @@ -602,7 +713,7 @@ def configure(self): def test_transitive_build_scripts_error(): # https://github.com/conan-io/conan/issues/18235 - c = TestClient() + c = TestClient(light=True) meta = textwrap.dedent(""" from conan import ConanFile @@ -631,3 +742,29 @@ def requirements(self): c.run("create meta") c.run("install product") # It doesn't crash + + +def test_transitive_build_scripts_library_error(): + # https://github.com/conan-io/conan/issues/18856 + c = TestClient(light=True) + tool = textwrap.dedent(""" + from conan import ConanFile + + class Product(ConanFile): + name = "tool" + version = "0.1" + package_type = "build-scripts" + def requirements(self): + self.requires("dep/0.1") + """) + + c.save({"dep/conanfile.py": GenConanfile("dep", "0.1").with_package_type("static-library"), + "tool/conanfile.py": tool}) + c.run("create dep") + c.run("create tool") + # Tool-requires, not an issue + c.run("install --tool-requires=tool/0.1") + # requires, it crashed with traceback, now a clear error message + c.run("install --requires=tool/0.1", assert_error=True) + assert ("ERROR: Package 'tool/0.1' with type 'build-scripts' cannot have " + "a 'static-library' dependency to 'dep/0.1'") in c.out diff --git a/test/integration/build_requires/profile_build_requires_test.py b/test/integration/build_requires/profile_build_requires_test.py index 03fd0b2beeb..f293f7b4e1d 100644 --- a/test/integration/build_requires/profile_build_requires_test.py +++ b/test/integration/build_requires/profile_build_requires_test.py @@ -79,7 +79,8 @@ def test_profile_requires(self, client): \\--(requires)->mylib/0.1 -(tool_requires)->tool/0.1 (skipped) """ client.run("export . --user=lasote --channel=stable") - client.run("install --requires=mylib/0.1@lasote/stable --profile ./profile.txt --build missing") + client.run("install --requires=mylib/0.1@lasote/stable " + "--profile ./profile.txt --build missing") assert "Hello World!" in client.out client.run("install --requires=mylib/0.1@lasote/stable --profile ./profile2.txt --build='*'") diff --git a/test/integration/build_requires/test_install_test_build_require.py b/test/integration/build_requires/test_install_test_build_require.py index bb8d6771456..830b77b9747 100644 --- a/test/integration/build_requires/test_install_test_build_require.py +++ b/test/integration/build_requires/test_install_test_build_require.py @@ -152,7 +152,8 @@ def test_build_require_conanfile_text(client): client.save({"conanfile.txt": "[tool_requires]\nmycmake/1.0"}, clean_first=True) client.run("install . -g VirtualBuildEnv") ext = ".bat" if platform.system() == "Windows" else ".sh" - cmd = environment_wrap_command(ConanFileMock(),"conanbuild", client.current_folder, f"mycmake{ext}") + cmd = environment_wrap_command(ConanFileMock(), "conanbuild", client.current_folder, + f"mycmake{ext}") client.run_command(cmd) system = {"Darwin": "Macos"}.get(platform.system(), platform.system()) assert "MYCMAKE={}!!".format(system) in client.out @@ -162,7 +163,8 @@ def test_build_require_conanfile_text(client): def test_build_require_command_line_build_context(client): client.run("install --tool-requires=mycmake/1.0@ -g VirtualBuildEnv -pr:b=default") ext = ".bat" if platform.system() == "Windows" else ".sh" - cmd = environment_wrap_command(ConanFileMock(),"conanbuild", client.current_folder, f"mycmake{ext}") + cmd = environment_wrap_command(ConanFileMock(), "conanbuild", client.current_folder, + f"mycmake{ext}") client.run_command(cmd) system = {"Darwin": "Macos"}.get(platform.system(), platform.system()) assert "MYCMAKE={}!!".format(system) in client.out diff --git a/test/integration/build_requires/test_toolchain_packages.py b/test/integration/build_requires/test_toolchain_packages.py index d6d1109f9f9..14c39d7244f 100644 --- a/test/integration/build_requires/test_toolchain_packages.py +++ b/test/integration/build_requires/test_toolchain_packages.py @@ -380,7 +380,8 @@ def test(self): # check the list packages c.run("list gcc/0.1:* --format=json", redirect_stdout="packages.json") pkgs_json = json.loads(c.load("packages.json")) - pref = PkgReference.loads("gcc/0.1#a8d725d9988de633accf410fb04cd162:6190ea2804cd4777609ec7174ccfdee22c6318c3") + pref = PkgReference.loads("gcc/0.1#a8d725d9988de633accf410fb04cd162:" + "6190ea2804cd4777609ec7174ccfdee22c6318c3") settings_target = { "arch": "armv7", "build_type": "Release", diff --git a/test/integration/cache/backup_sources_test.py b/test/integration/cache/backup_sources_test.py index 94e5184b97e..e7d67bc3eac 100644 --- a/test/integration/cache/backup_sources_test.py +++ b/test/integration/cache/backup_sources_test.py @@ -510,7 +510,7 @@ def source(self): assert f"Sources for {self.file_server.fake_url}/internet/myfile.txt found in remote backup {self.file_server.fake_url}/downloader/" in self.client.out def test_list_urls_miss(self): - def custom_download(this, url, *args, **kwargs): + def custom_download(this, url, *args, **kwargs): # noqa raise NotFoundException() with mock.patch("conan.internal.rest.file_downloader.FileDownloader.download", @@ -849,10 +849,10 @@ def source(self): self.client.run("source .") assert f"{sha256} is dirty, removing it" in self.client.out - @pytest.mark.skip("Recover when .dirty files are properly handled") + @pytest.mark.parametrize("exception", [Exception, ConanException]) def test_backup_source_upload_when_dirty(self, exception): - def custom_download(this, *args, **kwargs): + def custom_download(this, *args, **kwargs): # noqa raise exception() mkdir(os.path.join(self.download_cache_folder, "s")) @@ -872,15 +872,16 @@ def source(self): self.client.save_home( {"global.conf": f"core.sources:download_cache={self.download_cache_folder}\n" - f"core.sources:download_urls=['{self.file_server.fake_url}/backup/', 'origin']\n" - f"core.sources:upload_url={self.file_server.fake_url}/backup/"}) + "tools.files.download:retry=0\n" + f"core.sources:upload_url={self.file_server.fake_url}/backup/\n"}) self.client.save({"conanfile.py": conanfile}) with mock.patch("conan.internal.rest.file_downloader.FileDownloader._download_file", custom_download): self.client.run("source .", assert_error=True) - # A .dirty file was created, now try to source again, it should detect the dirty download and re-download it + # A .dirty file was created, now try to source again, + # it should detect the dirty download and clean it, but not re-source it self.client.run("export . --name=pkg2 --version=1.0") self.client.run("upload * -c -r=default") assert "No backup sources files to upload" in self.client.out diff --git a/test/integration/cache/cache2_update_test.py b/test/integration/cache/cache2_update_test.py index 6659733aa0d..7ffa41aeb89 100644 --- a/test/integration/cache/cache2_update_test.py +++ b/test/integration/cache/cache2_update_test.py @@ -1,8 +1,10 @@ import copy +import json +import textwrap from collections import OrderedDict import pytest -from mock import patch +from unittest.mock import patch from conan.api.model import RecipeReference from conans.server.revision_list import RevisionList @@ -99,7 +101,7 @@ def test_revision_fixed_version(self): assert f"liba/1.0.0: Retrieving package {NO_SETTINGS_PACKAGE_ID}" \ " from remote 'server0'" in self.client.out - latest_rrev = self.client.cache.get_latest_recipe_reference(RecipeReference.loads("liba/1.0.0@")) + latest_rrev = self.client.cache.get_latest_recipe_revision(RecipeReference.loads("liba/1.0.0@")) # check that we have stored REV1 in client with the same date from the server0 assert latest_rrev.timestamp == self.server_times["server0"] assert latest_rrev.timestamp == self.server_times["server0"] @@ -138,7 +140,7 @@ def test_revision_fixed_version(self): self.client.run("install --requires=liba/1.0.0@ --update") # --> result: Update date and server because server0 has a newer date - latest_rrev = self.client.cache.get_latest_recipe_reference(self.liba) + latest_rrev = self.client.cache.get_latest_recipe_revision(self.liba) self.client.assert_listed_require({"liba/1.0.0": "Updated (server2)"}) assert "liba/1.0.0: Downloaded package" in self.client.out assert latest_rrev.timestamp == self.server_times["server2"] @@ -171,7 +173,7 @@ def test_revision_fixed_version(self): # create newer revisions in servers so that the ones from the clients are older self.client.save({"conanfile.py": GenConanfile("liba", "1.0.0").with_build_msg("REV3")}) self.client.run("create .") - rev_to_upload = self.client.cache.get_latest_recipe_reference(self.liba) + rev_to_upload = self.client.cache.get_latest_recipe_revision(self.liba) # the future self.the_time = 3000000000.0 self._upload_ref_to_all_servers(repr(rev_to_upload), self.client) @@ -194,7 +196,7 @@ def test_revision_fixed_version(self): " from remote 'server2'" in self.client2.out check_ref = RecipeReference.loads(str(rev_to_upload)) # without revision - rev_to_upload = self.client2.cache.get_latest_recipe_reference(check_ref) + rev_to_upload = self.client2.cache.get_latest_recipe_revision(check_ref) assert rev_to_upload.timestamp == self.server_times["server2"] # | CLIENT | CLIENT2 | SERVER0 | SERVER1 | SERVER2 | @@ -210,7 +212,7 @@ def test_revision_fixed_version(self): # check one revision we already have will not be installed # we search for that revision in the cache, we found it # --> result: don't install that - latest_rrev = self.client.cache.get_latest_recipe_reference(self.liba) + latest_rrev = self.client.cache.get_latest_recipe_revision(self.liba) self.client.run(f"install --requires={latest_rrev}@#{latest_rrev.revision}") self.client.assert_listed_require({"liba/1.0.0": "Cache"}) assert "liba/1.0.0: Already installed!" in self.client.out @@ -224,7 +226,7 @@ def test_revision_fixed_version(self): # create new older revisions in servers self.client.save({"conanfile.py": GenConanfile("liba", "1.0.0").with_build_msg("REV4")}) self.client.run("create .") - server_rrev = self.client.cache.get_latest_recipe_reference(self.liba) + server_rrev = self.client.cache.get_latest_recipe_revision(self.liba) self.the_time = 0.0 self._upload_ref_to_all_servers("liba/1.0.0", self.client) @@ -260,7 +262,7 @@ def test_revision_fixed_version(self): assert "liba/1.0.0: Checking remote: server2" not in self.client.out server_rrev_norev = copy.copy(server_rrev) server_rrev_norev.revision = None - latest_cache_revision = self.client.cache.get_latest_recipe_reference(server_rrev_norev) + latest_cache_revision = self.client.cache.get_latest_recipe_revision(server_rrev_norev) assert latest_cache_revision != server_rrev # | CLIENT | CLIENT2 | SERVER0 | SERVER1 | SERVER2 | @@ -276,7 +278,7 @@ def test_revision_fixed_version(self): # from the remote # --> result: update REV4 date to 30 but it won't be latest - latest_cache_revision = self.client.cache.get_latest_recipe_reference(server_rrev_norev) + latest_cache_revision = self.client.cache.get_latest_recipe_revision(server_rrev_norev) assert latest_cache_revision != server_rrev latest_cache_revision = self.client.cache.recipe_layout(server_rrev).reference @@ -290,7 +292,7 @@ def test_revision_fixed_version(self): self.client.save({"conanfile.py": GenConanfile("liba", "1.0.0").with_build_msg("REV6")}) self.client.run("create .") - server_rrev = self.client.cache.get_latest_recipe_reference(self.liba) + server_rrev = self.client.cache.get_latest_recipe_revision(self.liba) self.the_time = 3000000020.0 self._upload_ref_to_all_servers(repr(server_rrev), self.client) @@ -311,7 +313,7 @@ def test_revision_fixed_version(self): # install anything, we are considering revisions fully immutable in 2.0 # --> results: update revision date in cache, do not install anything - latest_rrev_cache = self.client.cache.get_latest_recipe_reference(self.liba) + latest_rrev_cache = self.client.cache.get_latest_recipe_revision(self.liba) assert latest_server_time == latest_rrev_cache.timestamp self.client.assert_listed_require({"liba/1.0.0": "Cache (Updated date) (server2)"}) @@ -336,7 +338,7 @@ def test_revision_fixed_version(self): # the revision from the server that has the latest date # --> results: install from server2 - latest_rrev_cache = self.client.cache.get_latest_recipe_reference(self.liba) + latest_rrev_cache = self.client.cache.get_latest_recipe_revision(self.liba) assert latest_server_time == latest_rrev_cache.timestamp self.client.assert_listed_require({"liba/1.0.0": "Downloaded (server2)"}) @@ -384,7 +386,7 @@ def test_version_ranges(self): assert "liba/[>0.9.0]: liba/1.0.0" in self.client.out self.client.assert_listed_require({"liba/1.0.0": "Downloaded (server0)"}) - latest_rrev = self.client.cache.get_latest_recipe_reference(RecipeReference.loads("liba/1.0.0@")) + latest_rrev = self.client.cache.get_latest_recipe_revision(RecipeReference.loads("liba/1.0.0@")) assert latest_rrev.timestamp == self.server_times["server0"] # | CLIENT | CLIENT2 | SERVER0 | SERVER1 | SERVER2 | @@ -447,9 +449,9 @@ def test_version_ranges(self): ["libc", {"liba/1.0": "Cache", "libb/1.0": "Cache"}], ["liba", {"liba/1.1": "Downloaded (default)", - "libb/1.0": "Cache"}], + "libb/1.0": "Cache"}], ["libb", {"liba/1.0": "Cache", - "libb/1.1": "Downloaded (default)"}], + "libb/1.1": "Downloaded (default)"}], ["", {"liba/1.0": "Cache", "libb/1.0": "Cache"}], # Patterns not supported, only full name match @@ -480,3 +482,49 @@ def test_muliref_update_pattern(update, result): tc.run(f'install --requires="liba/[>=1.0]" --requires="libb/[>=1.0]" -r default {update_flag}') tc.assert_listed_require(result) + + +def test_update_remote_older_revision(): + # https://github.com/conan-io/conan/issues/19313 + c = TestClient(light=True, default_server_user=True) + zlib = textwrap.dedent(""" + from conan import ConanFile + class Zlib(ConanFile): + name = "zlib" + version = "1.2.11" + exports_sources = "*" + """) + c.save({"conanfile.py": zlib, + "file.h": "//myheader"}) + c.run("export") + c.run("upload * -r=default -c") + + c2 = TestClient(light=True, servers=c.servers) + c2.run("graph info --requires=zlib/[*]") + rev1 = "2e87959c586811f8a4eaf12a327cc042" + c2.assert_listed_require({f"zlib/1.2.11#{rev1}": "Downloaded (default)"}) + + # Modify zlib code + c.save({"file.h": "//myheader 2"}) + c.run("export") + c.run("upload * -r=default -c") + + c2.run("graph info --requires=zlib/[*] --update") + rev2 = "934b3de03768d9030b61127d588d0a96" + c2.assert_listed_require({f"zlib/1.2.11#{rev2}": "Updated (default)"}) + + # revert this to old, make it latestlatest + c.save({"file.h": "//myheader"}) + c.run("export") + c.run("remove * -r=default -c") + c.run("upload * -r=default -c") + + c2.run("graph info --requires=zlib/[*] --update") + assert ("Latest from 'default' was found in " + "the cache, using it and updating its timestamp") in c2.out + c2.assert_listed_require({f"zlib/1.2.11#{rev1}": "Updated (default)"}) + + c2.run("list *#* --format=json") + revs = json.loads(c2.stdout)["Local Cache"]["zlib/1.2.11"]["revisions"] + # we check that the update from the server has made rev1 the latest in the cache again + assert revs[rev1]["timestamp"] > revs[rev2]["timestamp"] diff --git a/test/integration/cache/storage_path_test.py b/test/integration/cache/storage_path_test.py index 851078babda..2cac6d7b2ee 100644 --- a/test/integration/cache/storage_path_test.py +++ b/test/integration/cache/storage_path_test.py @@ -1,4 +1,6 @@ import os +import textwrap +from hashlib import sha256 from conan.test.utils.test_files import temp_folder from conan.test.utils.tools import TestClient, GenConanfile @@ -22,3 +24,18 @@ def test_wrong_home_error(): client.save_home({"global.conf": "core.cache:storage_path=//"}) client.run("list *") assert "Couldn't initialize storage in" in client.out + + +def test_short_storage_path(): + c = TestClient() + global_conf = textwrap.dedent("""\ + {% set h = hashlib.new("sha256", conan_home_folder.encode(), + usedforsecurity=False).hexdigest() %} + core.cache:storage_path=C:/conan_{{h[:6]}} + """) + c.save_home({"global.conf": global_conf}) + c.run("config show *") + myhash = sha256() + myhash.update(c.cache_folder.replace("\\", "/").encode()) + myhash = myhash.hexdigest()[:6] + assert f"core.cache:storage_path: C:/conan_{myhash}" in c.out diff --git a/test/integration/cache/test_home_special_char.py b/test/integration/cache/test_home_special_char.py index cce04e926f0..0ec98ebcba7 100644 --- a/test/integration/cache/test_home_special_char.py +++ b/test/integration/cache/test_home_special_char.py @@ -1,6 +1,5 @@ import os import platform -import sys import pytest @@ -57,8 +56,6 @@ def build(self): return c -@pytest.mark.skipif(platform.system() == "Linux" and sys.version_info.minor <= 6, - reason="It is failing in CI in python3.6 Linux docker image") def test_reuse_buildenv(client_with_special_chars): c = client_with_special_chars # Need the 2 profile to work correctly buildenv diff --git a/test/integration/cache/test_package_revisions.py b/test/integration/cache/test_package_revisions.py index e0af866a656..6ba56aa7b65 100644 --- a/test/integration/cache/test_package_revisions.py +++ b/test/integration/cache/test_package_revisions.py @@ -1,4 +1,4 @@ -from mock import mock +from unittest import mock from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient diff --git a/test/integration/command/cache/test_cache_clean.py b/test/integration/command/cache/test_cache_clean.py index 834336cc67c..9a4e2293d07 100644 --- a/test/integration/command/cache/test_cache_clean.py +++ b/test/integration/command/cache/test_cache_clean.py @@ -11,7 +11,7 @@ @pytest.mark.parametrize("use_pkglist", [True, False]) def test_cache_clean(use_pkglist): - c = TestClient(default_server_user=True) + c = TestClient(default_server_user=True, light=True) c.save({"conanfile.py": GenConanfile("pkg", "0.1").with_exports("*").with_exports_sources("*"), "sorces/file.txt": ""}) c.run("create .") @@ -25,7 +25,6 @@ def test_cache_clean(use_pkglist): if use_pkglist: c.run("list *:*#* -f=json", redirect_stdout="pkglist.json") - pkglist = c.load("pkglist.json") arg = "--list=pkglist.json" if use_pkglist else "*" c.run(f'cache clean {arg} -s -b -v') assert f"{ref_layout.reference.repr_notime()}: Cleaning recipe cache contents" in c.out @@ -44,7 +43,7 @@ def test_cache_clean(use_pkglist): def test_cache_clean_all(): - c = TestClient() + c = TestClient(light=True) c.save({"pkg1/conanfile.py": GenConanfile("pkg", "0.1").with_class_attribute("revision_mode='scm'"), "pkg2/conanfile.py": GenConanfile("pkg", "0.2").with_package("error"), "pkg3/conanfile.py": GenConanfile("pkg", "0.3")}) @@ -52,12 +51,12 @@ def test_cache_clean_all(): c.run("create pkg2", assert_error=True) c.run("create pkg3") pref = c.created_package_reference("pkg/0.3") - temp_folder = os.path.join(c.cache_folder, "p", "t") - assert len(os.listdir(temp_folder)) == 1 # Failed export was here + tmp_folder = os.path.join(c.cache_folder, "p", "t") + assert len(os.listdir(tmp_folder)) == 1 # Failed export was here builds_folder = os.path.join(c.cache_folder, "p", "b") assert len(os.listdir(builds_folder)) == 2 # both builds are here c.run('cache clean') - assert not os.path.exists(temp_folder) + assert not os.path.exists(tmp_folder) assert len(os.listdir(builds_folder)) == 1 # only correct pkg/0.3 remains # Check correct package removed all ref_layout = c.get_latest_ref_layout(pref.ref) @@ -70,7 +69,7 @@ def test_cache_clean_all(): # A second clean like this used to crash # as it tried to delete a folder that was not there and tripped shutils up c.run('cache clean') - assert not os.path.exists(temp_folder) + assert not os.path.exists(tmp_folder) def test_cache_multiple_builds_same_prev_clean(): @@ -78,7 +77,7 @@ def test_cache_multiple_builds_same_prev_clean(): Different consecutive builds will create exactly the same folder, for the same exact prev, not leaving trailing non-referenced folders """ - c = TestClient() + c = TestClient(light=True) c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) c.run("create .") create_out = c.out @@ -105,7 +104,7 @@ def test_cache_multiple_builds_diff_prev_clean(): Different consecutive builds will create different folders, even if for the same exact prev, leaving trailing non-referenced folders """ - c = TestClient() + c = TestClient(light=True) package_lines = 'save(self, os.path.join(self.package_folder, "foo.txt"), str(time.time()))' gen = GenConanfile("pkg", "0.1").with_package(package_lines).with_import("import os, time") \ .with_import("from conan.tools.files import save") diff --git a/test/integration/command/cache/test_cache_integrity.py b/test/integration/command/cache/test_cache_integrity.py index b40a2f9a676..a40cf817445 100644 --- a/test/integration/command/cache/test_cache_integrity.py +++ b/test/integration/command/cache/test_cache_integrity.py @@ -8,7 +8,8 @@ @pytest.mark.parametrize("use_pkglist", [True, False]) -def test_cache_integrity(use_pkglist): +@pytest.mark.parametrize("output_pkglist", [True, False]) +def test_cache_integrity(use_pkglist, output_pkglist): t = TestClient() t.save({"conanfile.py": GenConanfile()}) t.run("create . --name pkg1 --version 1.0") @@ -29,7 +30,14 @@ def test_cache_integrity(use_pkglist): t.run("list *:*#* -f=json", redirect_stdout="pkglist.json") arg = "--list=pkglist.json" if use_pkglist else "*" - t.run(f"cache check-integrity {arg}", assert_error=True) + if output_pkglist: + arg += " --format=json" + + t.run( + f"cache check-integrity {arg}", + assert_error=True, + redirect_stdout="pkglist.json" if output_pkglist else None, + ) assert "pkg1/1.0#4d670581ccb765839f2239cc8dff8fbd: Integrity check: ok" in t.out assert "pkg1/1.0#4d670581ccb765839f2239cc8dff8fbd:da39a3ee5e6b4b0d3255bfef95601890afd80709" \ "#0ba8627bd47edc3a501e8f0eb9a79e5e: Integrity check: ok" in t.out @@ -40,9 +48,12 @@ def test_cache_integrity(use_pkglist): assert "pkg4/4.0#4d670581ccb765839f2239cc8dff8fbd:da39a3ee5e6b4b0d3255bfef95601890afd80709" \ "#0ba8627bd47edc3a501e8f0eb9a79e5e: ERROR: \nManifest mismatch" in t.out - t.run("remove pkg2/2.0:da39a3ee5e6b4b0d3255bfef95601890afd80709 -c") - t.run("remove pkg3/3.0:da39a3ee5e6b4b0d3255bfef95601890afd80709 -c") - t.run("remove pkg4/4.0:da39a3ee5e6b4b0d3255bfef95601890afd80709 -c") + if output_pkglist: + t.run("remove --list=pkglist.json -c") + else: + t.run("remove pkg2/2.0:da39a3ee5e6b4b0d3255bfef95601890afd80709 -c") + t.run("remove pkg3/3.0:da39a3ee5e6b4b0d3255bfef95601890afd80709 -c") + t.run("remove pkg4/4.0:da39a3ee5e6b4b0d3255bfef95601890afd80709 -c") t.run("cache check-integrity *") assert "pkg1/1.0#4d670581ccb765839f2239cc8dff8fbd: Integrity check: ok" in t.out assert "pkg2/2.0#4d670581ccb765839f2239cc8dff8fbd: Integrity check: ok" in t.out @@ -50,7 +61,8 @@ def test_cache_integrity(use_pkglist): assert "pkg4/4.0#4d670581ccb765839f2239cc8dff8fbd: Integrity check: ok" in t.out -def test_cache_integrity_missing_recipe_manifest(): +@pytest.mark.parametrize("output_pkglist", [True, False]) +def test_cache_integrity_missing_recipe_manifest(output_pkglist): t = TestClient() t.save({"conanfile.py": GenConanfile()}) t.run("create . --name pkg1 --version 1.0") @@ -60,13 +72,16 @@ def test_cache_integrity_missing_recipe_manifest(): os.remove(manifest) t.run("create . --name pkg3 --version=3.0") - t.run("cache check-integrity *", assert_error=True) + if output_pkglist: + t.run("cache check-integrity * -f json", assert_error=True, redirect_stdout="pkglist.json") + else: + t.run("cache check-integrity *", assert_error=True) assert "pkg1/1.0#4d670581ccb765839f2239cc8dff8fbd: Integrity check: ok" in t.out assert "pkg2/2.0#4d670581ccb765839f2239cc8dff8fbd: ERROR: Manifest missing" in t.out assert "pkg3/3.0#4d670581ccb765839f2239cc8dff8fbd: Integrity check: ok" in t.out assert "ERROR: There are corrupted artifacts, check the error logs" in t.out - t.run("remove pkg2* -c") + t.run(f"remove {'--list pkglist.json' if output_pkglist else 'pkg2*'} -c") t.run("cache check-integrity *") assert "pkg1/1.0#4d670581ccb765839f2239cc8dff8fbd:da39a3ee5e6b4b0d3255bfef95601890afd80709" \ "#0ba8627bd47edc3a501e8f0eb9a79e5e: Integrity check: ok" in t.out @@ -74,7 +89,8 @@ def test_cache_integrity_missing_recipe_manifest(): assert "Integrity check: ok" in t.out -def test_cache_integrity_missing_package_manifest(): +@pytest.mark.parametrize("output_pkglist", [True, False]) +def test_cache_integrity_missing_package_manifest(output_pkglist): t = TestClient() t.save({"conanfile.py": GenConanfile()}) t.run("create . --name pkg1 --version 1.0") @@ -84,14 +100,17 @@ def test_cache_integrity_missing_package_manifest(): os.remove(manifest) t.run("create . --name pkg3 --version=3.0") - t.run("cache check-integrity *", assert_error=True) + if output_pkglist: + t.run("cache check-integrity * -f json", assert_error=True, redirect_stdout="pkglist.json") + else: + t.run("cache check-integrity *", assert_error=True) assert "pkg1/1.0#4d670581ccb765839f2239cc8dff8fbd: Integrity check: ok" in t.out assert "pkg2/2.0#4d670581ccb765839f2239cc8dff8fbd:da39a3ee5e6b4b0d3255bfef95601890afd80709" \ "#0ba8627bd47edc3a501e8f0eb9a79e5e: ERROR: Manifest missing" in t.out assert "pkg3/3.0#4d670581ccb765839f2239cc8dff8fbd: Integrity check: ok" in t.out assert "ERROR: There are corrupted artifacts, check the error logs" in t.out - t.run("remove pkg2* -c") + t.run(f"remove {'--list pkglist.json' if output_pkglist else 'pkg2*'} -c") t.run("cache check-integrity *") assert "pkg1/1.0#4d670581ccb765839f2239cc8dff8fbd:da39a3ee5e6b4b0d3255bfef95601890afd80709" \ "#0ba8627bd47edc3a501e8f0eb9a79e5e: Integrity check: ok" in t.out @@ -99,7 +118,8 @@ def test_cache_integrity_missing_package_manifest(): assert "Integrity check: ok" in t.out -def test_cache_integrity_missing_package_conaninfo(): +@pytest.mark.parametrize("output_pkglist", [True, False]) +def test_cache_integrity_missing_package_conaninfo(output_pkglist): t = TestClient() t.save({"conanfile.py": GenConanfile()}) t.run("create . --name pkg1 --version 1.0") @@ -108,12 +128,15 @@ def test_cache_integrity_missing_package_conaninfo(): conaninfo = os.path.join(layout.package(), "conaninfo.txt") os.remove(conaninfo) - t.run("cache check-integrity *", assert_error=True) + if output_pkglist: + t.run("cache check-integrity * -f json", assert_error=True, redirect_stdout="pkglist.json") + else: + t.run("cache check-integrity *", assert_error=True) assert "pkg1/1.0#4d670581ccb765839f2239cc8dff8fbd: Integrity check: ok" in t.out assert "pkg2/2.0#4d670581ccb765839f2239cc8dff8fbd:da39a3ee5e6b4b0d3255bfef95601890afd80709" \ "#0ba8627bd47edc3a501e8f0eb9a79e5e: ERROR: \nManifest mismatch" in t.out - t.run("remove pkg2* -c") + t.run(f"remove {'--list pkglist.json' if output_pkglist else 'pkg2*'} -c") t.run("cache check-integrity *") assert "pkg1/1.0#4d670581ccb765839f2239cc8dff8fbd:da39a3ee5e6b4b0d3255bfef95601890afd80709" \ "#0ba8627bd47edc3a501e8f0eb9a79e5e: Integrity check: ok" in t.out diff --git a/test/integration/command/cache/test_cache_save_restore.py b/test/integration/command/cache/test_cache_save_restore.py index 113a0dd0ac5..77c4774d3cd 100644 --- a/test/integration/command/cache/test_cache_save_restore.py +++ b/test/integration/command/cache/test_cache_save_restore.py @@ -2,6 +2,7 @@ import os import platform import shutil +import sys import tarfile import time @@ -14,7 +15,6 @@ from conan.internal.util.files import save, load - def test_cache_save_restore(): c = TestClient() c.save({"conanfile.py": GenConanfile().with_settings("os")}) @@ -203,7 +203,8 @@ def test_cache_save_restore_metadata(): # FIXME: check the timestamps of the conan cache restore -@pytest.mark.skipif(platform.system() == "Windows", reason="Fails in windows in ci because of the low precission of the clock") +@pytest.mark.skipif(platform.system() == "Windows", + reason="Fails in windows in ci because of the low precission of the clock") def test_cache_save_restore_multiple_revisions(): c = TestClient() c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) @@ -218,7 +219,6 @@ def test_cache_save_restore_multiple_revisions(): c.run("create .") rrev3 = c.exported_recipe_revision() - def check_ordered_revisions(client): client.run("list *#* --format=json") revisions = json.loads(client.stdout)["Local Cache"]["pkg/0.1"]["revisions"] @@ -297,3 +297,29 @@ def test_cache_save_restore_custom_storage_path(src_store, dst_store): c2.run("cache restore conan_cache_save.tgz") c2.run("list *:*") assert "pkg/1.0" in c2.out + + +@pytest.mark.parametrize("compress", ["gz", "xz", "zst"]) +def test_cache_save_restore_compressions(compress): + """ we accept different compressions formats""" + if compress == "zst" and sys.version_info.minor < 14: + pytest.skip("Skipping zst compression tests") + + conan_file = GenConanfile() \ + .with_settings("os") \ + .with_package_file("bin/file.txt", "content!!") + + client = TestClient() + client.save({"conanfile.py": conan_file}) + client.run("create . --name=pkg --version=1.0 -s os=Linux") + client.run(f"cache save pkg/*:* --file=mysave.t{compress}") + if compress in ("xz", "zst"): + assert f"WARN: risk: The '{compress}' compression is experimental." in client.out + cache_path = os.path.join(client.current_folder, f"mysave.t{compress}") + assert os.path.exists(cache_path) + + c2 = TestClient() + shutil.copy2(cache_path, c2.current_folder) + c2.run(f"cache restore mysave.t{compress}") + c2.run("list *:*#*") + assert "pkg/1.0" in c2.out diff --git a/test/integration/command/create_test.py b/test/integration/command/create_test.py index f929b314a86..852c9c0a191 100644 --- a/test/integration/command/create_test.py +++ b/test/integration/command/create_test.py @@ -3,7 +3,10 @@ import re import textwrap +import pytest + from conan.api.model import RecipeReference +from conan.cli.exit_codes import ERROR_GENERAL from conan.test.utils.tools import TestClient, NO_SETTINGS_PACKAGE_ID, GenConanfile from conan.internal.util.files import load @@ -54,7 +57,7 @@ def system_requirements(self): self.output.info("Running system requirements!!") """) client.save({"conanfile.py": conanfile}) - client.run("create . --name=pkg --version=0.1 --user=lasote --channel=testing") + client.run("create --name=pkg --version=0.1 --user=lasote --channel=testing") assert "Profile host:\n[settings]" in client.out assert "pkg/0.1@lasote/testing: Generating the package" in client.out assert "Running system requirements!!" in client.out @@ -891,3 +894,44 @@ def test(self): c.run("create . -tm --build=missing") assert "Testing the package" in c.out assert "pkg/0.1 (test package): TEST!!!" in c.out + + +@pytest.mark.parametrize("command", ["create", "install"]) +@pytest.mark.parametrize("out_file", [False, True]) +def test_create_build_fail_generate_outfile(command, out_file): + c = TestClient() + c.save({"pkga/conanfile.py": GenConanfile("pkga", "0.1"), + "pkgb/conanfile.py": GenConanfile("pkgb", "0.1").with_requires("pkga/0.1"), + "pkgc/conanfile.py": GenConanfile("pkgc", "0.1") + .with_requires("pkgb/0.1") + .with_package("raise Exception('myerror')"), + "pkgd/conanfile.py": GenConanfile("pkgd", "0.1").with_requires("pkgc/0.1") + .with_settings("build_type") + .with_generator("CMakeDeps"), + }) + c.run("export pkga") + c.run("export pkgb") + c.run("export pkgc") + if out_file: + error = c.run(f"{command} pkgd --build=missing --format=json --out-file=graph.json", + assert_error=True) + else: + error = c.run(f"{command} pkgd --build=missing --format=json", assert_error=True, + redirect_stdout="graph.json") + assert error == ERROR_GENERAL + assert "pkgc/0.1: Error in package() method, line 8" in c.out + graph = json.loads(c.load("graph.json")) + nodeid = "1" if command == "create" else "0" + assert graph["graph"]["nodes"][nodeid]["name"] == "pkgd" + + # We can construct a package list from it + c.run("list -g=graph.json --graph-binaries=Build --format=json") + pkglist = json.loads(c.stdout) + # not built packages don't have revisions + rrev = pkglist["Local Cache"]["pkgc/0.1"]["revisions"]["b7f74fa20b19f1daac67db49318b7197"] + assert "revisions" not in rrev["packages"]["4a8d7d78a454700be1ab74b4a77fd7f36a44d122"] + # built packages do have package revisions + rrev = pkglist["Local Cache"]["pkgb/0.1"]["revisions"]["5b1ae5e3c1f718c0fd90d4dd8d9b57fb"] + assert "revisions" in rrev["packages"]["47a5f20ec8fb480e1c5794462089b01a3548fdc5"] + rrev = pkglist["Local Cache"]["pkga/0.1"]["revisions"]["57ece23aeb368b634896004ad579767a"] + assert "revisions" in rrev["packages"]["da39a3ee5e6b4b0d3255bfef95601890afd80709"] diff --git a/test/integration/command/custom_commands_test.py b/test/integration/command/custom_commands_test.py index a8824f3e120..8714c8a3b5c 100644 --- a/test/integration/command/custom_commands_test.py +++ b/test/integration/command/custom_commands_test.py @@ -147,6 +147,14 @@ def bye_say(conan_api, parser, *args, **kwargs): client.run("-h") assert "greet:bye" in client.out + # Ensure the prog has the full command name + client.run("hello -h") + assert "conan hello" in client.out + client.run("greet:bye -h") + assert "conan greet:bye" in client.out + client.run("greet:bye say -h") + assert "conan greet:bye say" in client.out + def test_custom_command_with_subcommands(self): complex_command = textwrap.dedent(""" import json @@ -488,3 +496,28 @@ def mycache(conan_api, parser, *args, **kwargs): c.run("mycache") # Does not break unexpectedly, caching is working assert "WARN: Cache invalidated, as expected: Invalid URL 'broken" in c.out + + +def test_custom_command_settings_access(): + tc = TestClient() + mycommand = textwrap.dedent(""" + from conan.cli.command import conan_command + from conan.api.output import ConanOutput + + @conan_command(group="custom commands") + def mycommand(conan_api, parser, *args, **kwargs): + \""" + test + \""" + settings = conan_api.config.settings_yml + ConanOutput().info(f"settings.fields: {settings.fields}") + ConanOutput().info(f"settings.possible_values(): {settings.possible_values()}") + ConanOutput().info(f"settings.compiler: {settings.compiler}") + ConanOutput().info(f"settings.compiler.possible_values(): {settings.compiler.possible_values()}") + """) + command_file_path = os.path.join('extensions', 'commands', 'cmd_mycommand.py') + tc.save_home({f"{command_file_path}": mycommand}) + tc.run("mycommand") + assert "settings.fields: ['arch', 'build_type', 'compiler', 'os']" in tc.out + # No values here, but as it does not fail, the interface is working fine + assert "settings.compiler: None" in tc.out diff --git a/test/integration/command/download/download_selected_packages_test.py b/test/integration/command/download/download_selected_packages_test.py index 528614582c6..096d138471c 100644 --- a/test/integration/command/download/download_selected_packages_test.py +++ b/test/integration/command/download/download_selected_packages_test.py @@ -19,7 +19,7 @@ def setup(): client.run("install --requires={} -s os=Linux --build missing".format(ref)) client.run("install --requires={} -s os=Linux -s arch=x86 --build missing".format(ref)) client.run("upload {} -r default".format(ref)) - latest_rrev = client.cache.get_latest_recipe_reference(ref) + latest_rrev = client.cache.get_latest_recipe_revision(ref) packages = client.cache.get_package_references(latest_rrev) package_ids = [package.package_id for package in packages] return client, ref, package_ids, str(conanfile) diff --git a/test/integration/command/export/export_path_test.py b/test/integration/command/export/export_path_test.py index f02688b17bd..ce9419ce8c1 100644 --- a/test/integration/command/export/export_path_test.py +++ b/test/integration/command/export/export_path_test.py @@ -59,7 +59,7 @@ def export(self): client.save({"current/conanfile.py": conanfile, "source/main.cpp": "mymain"}) with client.chdir("current"): - client.run("export . --name=hello --version=0.1 --user=lasote --channel=stable") + client.run("export --name=hello --version=0.1 --user=lasote --channel=stable") ref_layoyt = client.exported_layout() reg_path = ref_layoyt.export() manif = FileTreeManifest.load(reg_path) diff --git a/test/integration/export_sources_test.py b/test/integration/command/export/export_sources_test.py similarity index 100% rename from test/integration/export_sources_test.py rename to test/integration/command/export/export_sources_test.py diff --git a/test/integration/command/export/export_test.py b/test/integration/command/export/export_test.py index 01b976c85dd..0f7b19abe14 100644 --- a/test/integration/command/export/export_test.py +++ b/test/integration/command/export/export_test.py @@ -5,7 +5,6 @@ import textwrap import pytest -from parameterized import parameterized from conan.internal.model.manifest import FileTreeManifest from conan.api.model import RecipeReference @@ -49,7 +48,6 @@ def test_export_without_full_reference(self): client.run("export . --channel=channel") assert "lib/1.0@user/channel: Exported" in client.out - def test_export_read_only(self): client = TestClient(light=True) conanfile = textwrap.dedent(""" @@ -191,7 +189,7 @@ def export_sources(self): export_path = layout.export_sources() assert sorted(['file.txt', 'file.cpp', 'file.h']) == sorted(os.listdir(export_path)) - @parameterized.expand([("myconanfile.py", ), ("Conanfile.py", )]) + @pytest.mark.parametrize("filename", ["myconanfile.py", "Conanfile.py"]) def test_filename(self, filename): client = TestClient(light=True) client.save({filename: GenConanfile("hello", "1.2")}) diff --git a/test/integration/command/export/test_export.py b/test/integration/command/export/test_export.py index d0ecfda4817..25cf40cbfa0 100644 --- a/test/integration/command/export/test_export.py +++ b/test/integration/command/export/test_export.py @@ -1,5 +1,6 @@ import textwrap +from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient @@ -21,3 +22,20 @@ class TestConan(ConanFile): client.run("create .") assert "hello/1.2@myuser/mychannel" in client.out assert "hello/1.2:" not in client.out + + +def test_export_cci(): + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile("mypkg", "cci.123")}) + c.run("export .") + assert ("WARN: risk: Version 'cci.123' contains an alphanumeric " + "major alongside a minor version") in c.out + + c.save({"conanfile.py": GenConanfile("mypkg", "develop")}) + c.run("export .") + assert "WARN" not in c.out + + c.save({"conanfile.py": GenConanfile("mypkg", "cci.123") + .with_class_attribute("package_id_embed_mode='full_mode'")}) + c.run("export .") + assert "WARN" not in c.out diff --git a/test/integration/command/export_pkg_test.py b/test/integration/command/export_pkg_test.py index 110acf1f09f..c0c01cc7e7a 100644 --- a/test/integration/command/export_pkg_test.py +++ b/test/integration/command/export_pkg_test.py @@ -46,7 +46,7 @@ def test_package_folder_errors(self): # https://github.com/conan-io/conan/issues/2350 client = TestClient() client.save({CONANFILE: GenConanfile()}) - client.run("export-pkg . --name=hello --version=0.1 --user=lasote --channel=stable") + client.run("export-pkg --name=hello --version=0.1 --user=lasote --channel=stable") assert ("conanfile.py (hello/0.1@lasote/stable): package(): " "WARN: No files in this package!") in client.out diff --git a/test/integration/command/help_test.py b/test/integration/command/help_test.py index 25bb560bf93..252ef40e6ed 100644 --- a/test/integration/command/help_test.py +++ b/test/integration/command/help_test.py @@ -1,7 +1,4 @@ -import argparse - -import pytest -from mock import patch +from unittest.mock import patch from conan import __version__ from conan.test.utils.env import environment_update diff --git a/test/integration/command/info/info_test.py b/test/integration/command/info/info_test.py index 9ef293701d1..875df40f138 100644 --- a/test/integration/command/info/info_test.py +++ b/test/integration/command/info/info_test.py @@ -71,9 +71,9 @@ class MyTest(ConanFile): client.save({"conanfile.py": conanfile}) client.run("graph info . --format=json") recipe = json.loads(client.stdout)["graph"]["nodes"]["0"] - assert type(recipe["topics"]) == list + assert type(recipe["topics"]) is list assert recipe["topics"] == ["foo"] - assert type(recipe["provides"]) == list + assert type(recipe["provides"]) is list assert recipe["provides"] == ["bar"] # But this used to fail, @@ -91,9 +91,9 @@ class MyTest(ConanFile): client2.save({"conanfile.py": conanfile2}) client2.run("graph info . --format=json") recipe = json.loads(client2.stdout)["graph"]["nodes"]["0"] - assert type(recipe["topics"]) == list + assert type(recipe["topics"]) is list assert recipe["topics"] == ["foo"] - assert type(recipe["provides"]) == list + assert type(recipe["provides"]) is list assert recipe["provides"] == ["bar"] @@ -111,7 +111,7 @@ def test_wrong_path_parameter(self): client = TestClient() client.run("graph info", assert_error=True) - assert "ERROR: Please specify a path" in client.out + assert "ERROR: Conanfile not found" in client.out client.run("graph info not_real_path", assert_error=True) assert "ERROR: Conanfile not found" in client.out @@ -119,6 +119,13 @@ def test_wrong_path_parameter(self): client.run("graph info conanfile.txt", assert_error=True) assert "ERROR: Conanfile not found" in client.out + client.run("graph info --requires=foo/1.0", assert_error=True) + assert "ERROR: Package 'foo/1.0' not resolved: No remote defined." in client.out + + client.run("graph info . --requires=foo/1.0", assert_error=True) + assert ("--requires and --tool-requires arguments are incompatible " + "with [path] '.' argument") in client.out + class TestFilters: @@ -128,7 +135,7 @@ def test_filter_fields(self): c.save({"conanfile.py": GenConanfile() .with_class_attribute("author = 'myself'") .with_class_attribute("license = 'MIT'") - .with_class_attribute("url = 'http://url.com'")}) + .with_class_attribute("url = 'https://url.com'")}) c.run("graph info . ") assert "license: MIT" in c.out assert "author: myself" in c.out @@ -148,7 +155,7 @@ def test_filter_fields_json(self): c.save({"conanfile.py": GenConanfile() .with_class_attribute("author = 'myself'") .with_class_attribute("license = 'MIT'") - .with_class_attribute("url = 'http://url.com'")}) + .with_class_attribute("url = 'https://url.com'")}) c.run("graph info . --filter=license --format=json") assert "author" not in c.out assert '"license": "MIT"' in c.out @@ -158,16 +165,37 @@ def test_filter_fields_json(self): class TestJsonOutput: def test_json_package_filter(self): - # Formatted output like json or html doesn't make sense to be filtered client = TestClient() - conanfile = GenConanfile("pkg", "0.1").with_setting("build_type") - client.save({"conanfile.py": conanfile}) + conanfile = GenConanfile("pkg", "0.1").with_setting("build_type").with_requires("dep/0.1") + client.save({"dep/conanfile.py": GenConanfile("dep", "0.1"), + "conanfile.py": conanfile}) + client.run("create dep") client.run("graph info . --package-filter=nothing --format=json") assert '"nodes": {}' in client.out client.run("graph info . --package-filter=pkg* --format=json") graph = json.loads(client.stdout) + assert len(graph["graph"]["nodes"]) == 1 assert graph["graph"]["nodes"]["0"]["ref"] == "pkg/0.1" + # Consumer & filtering + client.run("graph info . --package-filter=& --format=json") + graph = json.loads(client.stdout) + assert len(graph["graph"]["nodes"]) == 1 + assert graph["graph"]["nodes"]["0"]["ref"] == "pkg/0.1" + + # This returns nothing, doesn't work that way, serialized graph doesn't have the consumer + client.run("graph info --requires=dep/0.1 --package-filter=& --format=json") + graph = json.loads(client.stdout) + assert len(graph["graph"]["nodes"]) == 1 + assert graph["graph"]["nodes"]["1"]["name"] == "dep" + + # Filter matches all + client.run("graph info . --package-filter=* --format=json") + graph = json.loads(client.stdout) + assert len(graph["graph"]["nodes"]) == 2 + assert graph["graph"]["nodes"]["0"]["ref"] == "pkg/0.1" + assert graph["graph"]["nodes"]["1"]["name"] == "dep" + def test_json_info_outputs(self): client = TestClient() conanfile = GenConanfile("pkg", "0.1").with_setting("build_type") @@ -375,9 +403,8 @@ def validate(self): c.save({ "dep/conanfile.py": GenConanfile("dep").with_version("0.1"), "dep_invalid/conanfile.py": dep_invalid, - "consumer/conanfile.py": GenConanfile("consumer") - .with_requires("dep/0.1") - .with_requires("dep_invalid/0.1"), + "consumer/conanfile.py": GenConanfile("consumer").with_requires("dep/0.1") + .with_requires("dep_invalid/0.1"), }) c.run("export dep") c.run("export dep_invalid") @@ -388,7 +415,6 @@ def validate(self): assert exit_code == 0 - class TestInfoUpdate: def test_update(self): @@ -475,3 +501,17 @@ def requirements(self): c.run("graph info . -c tools.graph:vendor=build --build='lib*'") c.assert_listed_binary({"lib/1.0": (NO_SETTINGS_PACKAGE_ID, "Build")}) + + +def test_graph_info_provides_conflict_html(): + """ This test also makes sure that the serialization of the GraphProvidesConflict works""" + tc = TestClient(light=True) + tc.save({"bar/conanfile.py": GenConanfile("bar", "1.0"), + "foo/conanfile.py": GenConanfile("foo", "1.0").with_provides("bar")}) + + tc.run("export bar") + tc.run("export foo") + tc.run("graph info --requires=bar/1.0 --requires=foo/1.0 --format=html", assert_error=True) + # It got properly serialized + assert '"type": "provide_conflict"' in tc.out + assert "Both 'bar/1.0' and 'foo/1.0' provide '['bar']'" diff --git a/test/integration/command/info/test_graph_info_graphical.py b/test/integration/command/info/test_graph_info_graphical.py index 58d681d0ee6..27aa57bfd1a 100644 --- a/test/integration/command/info/test_graph_info_graphical.py +++ b/test/integration/command/info/test_graph_info_graphical.py @@ -159,3 +159,14 @@ def test_graph_conflict_diamond(): c.run("graph info game --format=html", assert_error=True, redirect_stdout="graph.html") # check that it doesn't crash assert "ERROR: Version conflict: Conflict between math/1.0.1 and math/1.0 in the graph." in c.out + + +def test_graph_missing_error(): + c = TestClient() + c.save({"engine/conanfile.py": GenConanfile("engine", "1.0").with_requires("math/1.0"), + "game/conanfile.py": GenConanfile("game", "1.0").with_requires("engine/1.0"), + }) + c.run("export engine") + c.run("graph info game --format=html", assert_error=True, redirect_stdout="graph.html") + # check that it doesn't crash, tested manually + assert "ERROR: Package 'math/1.0' not resolved: No remote defined" in c.out diff --git a/test/integration/command/info/test_info_build_order.py b/test/integration/command/info/test_info_build_order.py index 4a39b80e6f9..f21fe24d3b8 100644 --- a/test/integration/command/info/test_info_build_order.py +++ b/test/integration/command/info/test_info_build_order.py @@ -887,7 +887,7 @@ def test_info_build_order_editable(): "consumer/conanfile.txt": "[requires]\npkg/0.1"}) c.run("editable add dep") c.run("export pkg") - + c.run("graph build-order consumer --build=missing --build=editable -f=json --order-by=recipe") bo_json = json.loads(c.stdout) pkg = bo_json["order"][0][0]["packages"][0][0] @@ -900,3 +900,17 @@ def test_info_build_order_editable(): pkg = bo_json["order"][0][0] assert pkg["binary"] == "EditableBuild" assert pkg["build_args"] == "--requires=dep/0.1 --build=dep/0.1" + + +def test_build_order_path_reqs_mixed_args(): + # This used not to crash in previous versions + # Also make sure we can properly used them separately + tc = TestClient(light=True) + tc.run("graph build-order . --requires=foo/1.0", assert_error=True) + assert "ERROR: --requires and --tool-requires arguments are incompatible with [path] '.' argument" in tc.out + + tc.run("graph build-order --requires=foo/1.0", assert_error=True) + assert "Package 'foo/1.0' not resolved: No remote defined" in tc.out + + tc.run("graph build-order .", assert_error=True) + assert "Conanfile not found" in tc.out diff --git a/test/integration/command/install/install_test.py b/test/integration/command/install/install_test.py index e0e25276483..e0c26dd5d75 100644 --- a/test/integration/command/install/install_test.py +++ b/test/integration/command/install/install_test.py @@ -191,9 +191,13 @@ def requirements(self): def test_install_with_path_errors(client): - # Install without path param not allowed + # Install without path param allowed, but nothing found client.run("install", assert_error=True) - assert "ERROR: Please specify a path" in client.out + assert "Conanfile not found" in client.out + + # Install without path param allowed, but nothing found + client.run("install . --requires=foo/1.0", assert_error=True) + assert "--requires and --tool-requires arguments are incompatible with [path]" in client.out # Path with wrong conanfile.txt path client.run("install not_real_dir/conanfile.txt", assert_error=True) diff --git a/test/integration/command/install/install_update_test.py b/test/integration/command/install/install_update_test.py index e4d6fd0bee8..32a4f7f6826 100644 --- a/test/integration/command/install/install_update_test.py +++ b/test/integration/command/install/install_update_test.py @@ -76,7 +76,7 @@ def test_update_not_date(): ref = RecipeReference.loads("hello0/1.0@lasote/stable") - initial_recipe_timestamp = client.cache.get_latest_recipe_reference(ref).timestamp + initial_recipe_timestamp = client.cache.get_latest_recipe_revision(ref).timestamp initial_package_timestamp = prev.timestamp # Change and rebuild package @@ -85,7 +85,7 @@ def test_update_not_date(): client.run("export . --user=lasote --channel=stable") client.run("install --requires=hello0/1.0@lasote/stable --build='*'") - rebuild_recipe_timestamp = client.cache.get_latest_recipe_reference(ref).timestamp + rebuild_recipe_timestamp = client.cache.get_latest_recipe_revision(ref).timestamp rebuild_package_timestamp = client.get_latest_package_reference(ref).timestamp assert rebuild_recipe_timestamp != initial_recipe_timestamp @@ -100,7 +100,7 @@ def test_update_not_date(): client.assert_listed_require({"hello0/1.0@lasote/stable": "Newer"}) - failed_update_recipe_timestamp = client.cache.get_latest_recipe_reference(ref).timestamp + failed_update_recipe_timestamp = client.cache.get_latest_recipe_revision(ref).timestamp failed_update_package_timestamp = client.get_latest_package_reference(ref).timestamp assert rebuild_recipe_timestamp == failed_update_recipe_timestamp diff --git a/test/integration/command/install/test_graph_build_mode.py b/test/integration/command/install/test_graph_build_mode.py index 2dcd8c5862c..dc0966ad397 100644 --- a/test/integration/command/install/test_graph_build_mode.py +++ b/test/integration/command/install/test_graph_build_mode.py @@ -1,3 +1,5 @@ +import json + import pytest from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient @@ -137,3 +139,32 @@ def test_install_build_all_with_double_skip(build_arg, bar, foo, foobar, build_a "foo/1.0@user/testing": (foo_id, foo), "foobar/1.0@user/testing": (foobar_id, foobar), }) + + +def test_build_consumer(): + """ If the consumer is built from sources, the dependencies must be built too + """ + client = TestClient() + client.save({"dep1/conanfile.py": GenConanfile("dep1", "1.0").with_build_msg("TEST: Building dep1"), + "dep2/conanfile.py": GenConanfile("dep2", "1.0").with_build_msg("TEST: Building dep2"), + "conanfile.py": GenConanfile("consumer", "1.0") + .with_build_msg("TEST: Building consumer").with_requires("dep1/1.0", "dep2/1.0")}) + client.run("create dep1") + client.run("create dep2") + client.run("export .") + + # Still makes little sense, but just to test the logic + client.run("create . --build=* --build=!&", assert_error=True) + assert "TEST: Building consumer" not in client.out + + client.run(f"graph info --requires=consumer/1.0 --build=& -f=json", redirect_stdout="graph.json") + graph = json.loads(client.load("graph.json")) + assert graph["graph"]["nodes"]["1"]["binary"] == "Build" # Would be missing if not built + assert graph["graph"]["nodes"]["2"]["binary"] == "Cache" + assert graph["graph"]["nodes"]["3"]["binary"] == "Cache" + + client.run(f"graph info . --build=& -f=json", redirect_stdout="graph.json") + graph = json.loads(client.load("graph.json")) + assert graph["graph"]["nodes"]["0"]["binary"] is None + assert graph["graph"]["nodes"]["1"]["binary"] == "Cache" + assert graph["graph"]["nodes"]["2"]["binary"] == "Cache" diff --git a/test/integration/command/list/list_test.py b/test/integration/command/list/list_test.py index 5d18a29e804..037dda3b1ef 100644 --- a/test/integration/command/list/list_test.py +++ b/test/integration/command/list/list_test.py @@ -8,16 +8,12 @@ import pytest -from conan.internal.errors import ConanConnectionError from conan.errors import ConanException -from conan.test.assets.cmake import gen_cmakelists +from conan.internal.errors import ConanConnectionError +from conan.internal.util.files import save from conan.test.assets.genconanfile import GenConanfile -from conan.test.assets.sources import gen_function_h, gen_function_cpp -from conan.test.utils.file_server import TestFileServer -from conan.test.utils.test_files import temp_folder -from conan.test.utils.tools import TestClient, TestServer, NO_SETTINGS_PACKAGE_ID, zipdir from conan.test.utils.env import environment_update -from conan.internal.util.files import save, save_files, sha256sum +from conan.test.utils.tools import TestClient, TestServer, NO_SETTINGS_PACKAGE_ID class TestParamErrors: @@ -70,7 +66,8 @@ def test_graph_file_error(self): c.run("list --graph graph.json --format=json", redirect_stdout="pkglist.json") c.run("list --graph pkglist.json", assert_error=True) assert ( - 'Expected a graph file but found an unexpected JSON file format. You can create a "graph" JSON file by running' + 'Expected a graph file but found an unexpected JSON file format. ' + 'You can create a "graph" JSON file by running' in c.out ) assert ( @@ -78,6 +75,7 @@ def test_graph_file_error(self): in c.out ) + @pytest.fixture(scope="module") def client(): servers = OrderedDict([("default", TestServer()), @@ -230,6 +228,17 @@ def test_list_recipe_version_ranges_patterns(self, client, remote): 'zlix/1.0.0': {}} self.check_json(client, pattern, remote, expected_json) + def test_version_range_prerelease(self): + tc = TestClient(light=True) + tc.save({"conanfile.py": GenConanfile("foo", "1.0-pre.1")}) + tc.run("export .") + tc.run("list * ",) + assert "foo/1.0-pre.1" in tc.out + tc.run("list foo/[>=1.0]") + assert "foo/1.0-pre.1" not in tc.out + tc.run("list foo/[>=1.0] -cc core.version_ranges:resolve_prereleases=True") + assert "foo/1.0-pre.1" in tc.out + @pytest.mark.parametrize("remote", [True, False]) def test_list_recipe_versions_exact(self, client, remote): pattern = "zli/1.0.0" @@ -857,7 +866,8 @@ def test_list_compact_no_settings_no_options(self): "pkg/1.0#a69a86bbd19ae2ef7eedc64ae645c531:*", "pkg/1.0#a69a86bbd19ae2ef7eedc64ae645c531:*#*", "pkg/1.0#a69a86bbd19ae2ef7eedc64ae645c531:da39a3ee5e6b4b0d3255bfef95601890afd80709#*", - "pkg/1.0#a69a86bbd19ae2ef7eedc64ae645c531:da39a3ee5e6b4b0d3255bfef95601890afd80709#0ba8627bd47edc3a501e8f0eb9a79e5e" + "pkg/1.0#a69a86bbd19ae2ef7eedc64ae645c531:da39a3ee5e6b4b0d3255bfef95601890afd80709#" + "0ba8627bd47edc3a501e8f0eb9a79e5e" ]) def test_list_compact_patterns(self, pattern): c = TestClient(light=True) @@ -952,6 +962,7 @@ def test_overlapping_versions(): results = json.loads(tc.load("list.json")) assert len(results["Local Cache"]) == 2 + def test_list_local_recipe_index(): c = TestClient(light=True) c.run(f"new local_recipes_index -d name=pkg -d version=0.1 -d url='https://conan-fake-url.com' ") diff --git a/test/integration/command/list/test_combined_pkglist_flows.py b/test/integration/command/list/test_combined_pkglist_flows.py index 6eff4ebc742..d980d60de99 100644 --- a/test/integration/command/list/test_combined_pkglist_flows.py +++ b/test/integration/command/list/test_combined_pkglist_flows.py @@ -4,6 +4,7 @@ import pytest from conan.test.assets.genconanfile import GenConanfile +from conan.test.utils.env import environment_update from conan.test.utils.tools import TestClient, TestServer @@ -214,6 +215,115 @@ def test_graph_2_pkg_list_remotes(self): assert "app/1.0: Retrieving recipe metadata from remote 'remote2'" in c.out assert "app/1.0: Retrieving package metadata" in c.out + def test_input_only_name_version(self): + c = TestClient(default_server_user=True, light=True) + c.save({"zlib/conanfile.py": GenConanfile("zlib", "1.0")}) + c.run("create zlib") + c.run("upload zlib* -c -r=default") + + # Create input pkglist for find-remote + c.run(f"list * --format=json", redirect_stdout="mylist.json") + pkglist = json.loads(c.load("mylist.json")) + expected = {"zlib/1.0": {}} + assert pkglist["Local Cache"] == expected + + c.run("pkglist find-remote mylist.json --format=json --remote default") + pkglist = json.loads(c.stdout) + assert pkglist["default"] == expected + + def test_input_recipe_revisions(self): + c = TestClient(default_server_user=True, light=True) + c.save({"zlib/conanfile.py": GenConanfile("zlib", "1.0")}) + c.run("create zlib") + c.run("upload zlib* -c -r=default") + + # Create input pkglist for find-remote + c.run(f"list *#* --format=json", redirect_stdout="mylist.json") + + def _check(origin): + pkglist = json.loads(c.load("mylist.json")) + revs = pkglist[origin]["zlib/1.0"]["revisions"] + assert list(revs) == ["c570d63921c5f2070567da4bf64ff261"] + assert "packages" not in revs["c570d63921c5f2070567da4bf64ff261"] + + _check("Local Cache") + + c.run("pkglist find-remote mylist.json --format=json --remote default", + redirect_stdout="mylist.json") + _check("default") + + def test_input_only_package_ids(self): + c = TestClient(default_server_user=True, light=True) + c.save({"zlib/conanfile.py": GenConanfile("zlib", "1.0").with_settings("os")}) + c.run("create zlib -s os=Linux") + c.run("upload zlib* -c -r=default") + + # Create input pkglist for find-remote + c.run(f"list *:* --format=json", redirect_stdout="mylist.json") + + def _check(origin): + pkglist = json.loads(c.load("mylist.json")) + revs = pkglist[origin]["zlib/1.0"]["revisions"] + assert list(revs) == ["1cb7410d0365f87510a6767c7bef804e"] + info = {"info": {'settings': {'os': 'Linux'}}} + expected = {"9a4eb3c8701508aa9458b1a73d0633783ecc2270": info} + assert revs["1cb7410d0365f87510a6767c7bef804e"]["packages"] == expected + + c.run("pkglist find-remote mylist.json --format=json --remote default", + redirect_stdout="mylist.json") + _check("default") + + def test_graph_pkg_list_of_recipes_and_binaries(self): + c = TestClient(default_server_user=True, light=True) + c.save({"zlib/conanfile.py": GenConanfile("zlib", "1.0").with_settings("os")}) + c.run("create zlib -s os=Linux") + c.run("upload zlib* -c -r=default") + + # Create input pkglist for find-remote + c.run(f"list *#*:*#* --format=json", redirect_stdout="mylist.json") + + def _check(origin): + pkglist = json.loads(c.load("mylist.json")) + revs = pkglist[origin]["zlib/1.0"]["revisions"] + assert list(revs) == ["1cb7410d0365f87510a6767c7bef804e"] + expected = {'settings': {'os': 'Linux'}} + pkgs = revs["1cb7410d0365f87510a6767c7bef804e"]["packages"] + assert list(pkgs) == ["9a4eb3c8701508aa9458b1a73d0633783ecc2270"] + pkg = pkgs["9a4eb3c8701508aa9458b1a73d0633783ecc2270"] + assert pkg["info"] == expected + assert list(pkg["revisions"]) == ["1d3c57385f4133c1fbd6d13bd538496e"] + + _check("Local Cache") + c.run("pkglist find-remote mylist.json --format=json --remote default", + redirect_stdout="mylist.json") + _check("default") + + def test_graph_pkg_list_counter_example(self): + c = TestClient(default_server_user=True, light=True) + c.save({"conanfile.py": GenConanfile("zlib", "1.0").with_package_file("file.txt", + env_var="MY_VAR")}) + + with environment_update({"MY_VAR": "1"}): + c.run("create .") + with environment_update({"MY_VAR": "2"}): + c.run("create .") + c.run("upload zlib*:*#* -c -r=default") + + # Create input pkglist for find-remote + c.run(f"list zlib/1.0#latest:*#latest --format=json", redirect_stdout="mylist.json") + + def _check(origin): + pkglist = json.loads(c.load("mylist.json")) + prevs = pkglist[origin]["zlib/1.0"]["revisions"] + input_pkgs = prevs["212b9babae6a4b8a8362703cec4257ad"]["packages"] + prevs = input_pkgs["da39a3ee5e6b4b0d3255bfef95601890afd80709"]["revisions"] + assert len(prevs) == 1 + + _check("Local Cache") + c.run("pkglist find-remote mylist.json --format=json --remote default", + redirect_stdout="mylist.json") + _check("default") + class TestPkgListMerge: """ deep merge lists @@ -273,11 +383,12 @@ def test_pkglist_file_error(self): c.run("create . --version=1.0 --format=json", redirect_stdout="out/graph.json") c.run("pkglist merge -l out/graph.json", assert_error=True) assert ( - 'ERROR: Expected a package list file but found a graph file. You can create a "package list" JSON file by running:' - in c.out + 'ERROR: Expected a package list file but found a graph file. ' + 'You can create a "package list" JSON file by running:' in c.out ) assert "conan list --graph graph.json --format=json > pkglist.json" in c.out + class TestDownloadUpload: @pytest.fixture() def client(self): @@ -312,6 +423,21 @@ def test_download_upload_all(self, client, prev_list): assert "Uploading package 'zlib/1.0.0" in client.out assert "Uploading package 'zli/" not in client.out + def test_download_upload_all_no_removed(self): + c = TestClient(default_server_user=True, light=True) + c.save({"zlib.py": GenConanfile("zlib", "0.1")}) + c.run("create zlib.py ") + c.run("upload * -r=default -c --format=json", redirect_stdout="pkglist.json") + c.run("remove * -c") + c.run(f"download --list=pkglist.json -r=default --format=json", + redirect_stdout="pkglist.json") + # The original "files" and "upload-urls" fields of first upload are removed + pkglist_json = c.load("pkglist.json") + assert "files" not in pkglist_json + assert "upload-urls" not in pkglist_json + c.run("upload --list=pkglist.json -r=default --format=json") + # It doesn't crash anymore + @pytest.mark.parametrize("prev_list", [False, True]) def test_download_upload_only_recipes(self, client, prev_list): if prev_list: @@ -351,43 +477,66 @@ def client(self): def test_remove_nothing_only_refs(self, client): # It is necessary to do *#* for actually removing something client.run(f"list * --format=json", redirect_stdout="pkglist.json") - client.run(f"remove --list=pkglist.json -c") + client.run(f"remove --list=pkglist.json -c --format=json") assert "Nothing to remove, package list do not contain recipe revisions" in client.out + result = json.loads(client.stdout) + assert result["Local Cache"] == {} # Nothing was removed @pytest.mark.parametrize("remote", [False, True]) def test_remove_all(self, client, remote): # It is necessary to do *#* for actually removing something remote = "-r=default" if remote else "" client.run(f"list *#* {remote} --format=json", redirect_stdout="pkglist.json") - client.run(f"remove --list=pkglist.json {remote} -c") + client.run(f"remove --list=pkglist.json {remote} -c --dry-run") assert "zli/1.0.0#f034dc90894493961d92dd32a9ee3b78:" \ " Removed recipe and all binaries" in client.out assert "zlib/1.0.0@user/channel#ffd4bc45820ddb320ab224685b9ba3fb:" \ " Removed recipe and all binaries" in client.out + + client.run(f"remove --list=pkglist.json {remote} -c --format=json") + result = json.loads(client.stdout) + origin = "Local Cache" if not remote else "default" + assert len(result[origin]["zli/1.0.0"]["revisions"]) == 1 + assert len(result[origin]["zlib/1.0.0@user/channel"]["revisions"]) == 1 + assert "packages" not in client.stdout # Packages are not listed at all + client.run(f"list * {remote}") assert "There are no matching recipe references" in client.out @pytest.mark.parametrize("remote", [False, True]) def test_remove_packages_no_revisions(self, client, remote): - # It is necessary to do *#* for actually removing something + # It is necessary to do *#*:*#* for actually removing binaries remote = "-r=default" if remote else "" client.run(f"list *#*:* {remote} --format=json", redirect_stdout="pkglist.json") - client.run(f"remove --list=pkglist.json {remote} -c") + client.run(f"remove --list=pkglist.json {remote} -c --format=json") assert "No binaries to remove for 'zli/1.0.0#f034dc90894493961d92dd32a9ee3b78'" in client.out assert "No binaries to remove for 'zlib/1.0.0@user/channel" \ "#ffd4bc45820ddb320ab224685b9ba3fb" in client.out + result = json.loads(client.stdout) + origin = "Local Cache" if not remote else "default" + assert result[origin] == {} # Nothing was removed @pytest.mark.parametrize("remote", [False, True]) def test_remove_packages(self, client, remote): - # It is necessary to do *#* for actually removing something + # It is necessary to do *#*:*#* for actually removing binaries remote = "-r=default" if remote else "" client.run(f"list *#*:*#* {remote} --format=json", redirect_stdout="pkglist.json") - client.run(f"remove --list=pkglist.json {remote} -c") - + client.run(f"remove --list=pkglist.json {remote} -c --dry-run") assert "Removed recipe and all binaries" not in client.out assert "zli/1.0.0#f034dc90894493961d92dd32a9ee3b78: Removed binaries" in client.out assert "zlib/1.0.0@user/channel#ffd4bc45820ddb320ab224685b9ba3fb: " \ "Removed binaries" in client.out + + client.run(f"remove --list=pkglist.json {remote} -c --format=json") + result = json.loads(client.stdout) + origin = "Local Cache" if not remote else "default" + zli_revs = result[origin]["zli/1.0.0"]["revisions"] + zli_uc_revs = result[origin]["zlib/1.0.0@user/channel"]["revisions"] + assert len(zli_revs) == 1 + assert len(zli_uc_revs) == 1 + assert len(zli_revs["f034dc90894493961d92dd32a9ee3b78"]["packages"]) == 1 + assert len(zli_uc_revs["ffd4bc45820ddb320ab224685b9ba3fb"]["packages"]) == 1 + client.run(f"list *:* {remote}") assert "zli/1.0.0" in client.out assert "zlib/1.0.0@user/channel" in client.out @@ -442,7 +591,8 @@ def test_graph_list(self): tc = TestClient(light=True) tc.save({"conanfile.py": GenConanfile("lib", "1.0")}) - tc.run("graph info . -f json --build=never --no-remote --filter=context", redirect_stdout="graph_context.json") + tc.run("graph info . -f json --build=never --no-remote --filter=context", + redirect_stdout="graph_context.json") tc.run("list --graph=graph_context.json --graph-context=build-only --format=json", assert_error=True) diff --git a/test/integration/command/list/test_list_lru.py b/test/integration/command/list/test_list_lru.py index 54962ed4ad1..d150e02cec1 100644 --- a/test/integration/command/list/test_list_lru.py +++ b/test/integration/command/list/test_list_lru.py @@ -52,6 +52,23 @@ def test_cache_clean_lru(self, method): assert "da39a3ee5e6b4b0d3255bfef95601890afd80709" not in c.out assert "dep" not in c.out + def test_lru_just_created(self): + c = TestClient() + c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) + c.run("create .") + c.run("list * --lru=1d", assert_error=True) + assert "'--lru' must be used with recipe revision pattern" in c.out + c.run("list *#* --lru=1d") + assert "pkg/0.1" not in c.out + # What if no revision, but package id is passed + c.run("list pkg/0.1:* --lru=1d", assert_error=True) + assert "'--lru' must be used with package revision pattern" in c.out + # Doesn't fail, but packages is empty + c.run("list pkg/0.1:*#* --lru=1d --format=json") + pkgs = json.loads(c.stdout) + rev = pkgs["Local Cache"]["pkg/0.1"]["revisions"]["485dad6cb11e2fa99d9afbe44a57a164"] + assert rev["packages"] == {} + def test_update_lru_when_used_as_dependency(self): """Show that using a recipe as a dependency will update its LRU""" c = TestClient() diff --git a/test/unittests/client/toolchain/autotools/__init__.py b/test/integration/command/remote/__init__.py similarity index 100% rename from test/unittests/client/toolchain/autotools/__init__.py rename to test/integration/command/remote/__init__.py diff --git a/test/integration/command/remote_test.py b/test/integration/command/remote/remote_test.py similarity index 90% rename from test/integration/command/remote_test.py rename to test/integration/command/remote/remote_test.py index 610c9b4685e..45e57366103 100644 --- a/test/integration/command/remote_test.py +++ b/test/integration/command/remote/remote_test.py @@ -107,8 +107,8 @@ def test_duplicated_error(self): """ check remote name are not duplicated """ self.client.run("remote add remote1 http://otherurl", assert_error=True) - assert "ERROR: Remote 'remote1' already exists in remotes (use --force to continue)" \ - in self.client.out + assert ("ERROR: Remote 'remote1' already exists in remotes (use --force to continue)" + in self.client.out) self.client.run("remote list") assert "otherurl" not in self.client.out @@ -124,19 +124,19 @@ def test_missing_subarguments(self): def test_invalid_url(self): self.client.run("remote add foobar foobar.com") - assert "WARN: The URL 'foobar.com' is invalid. It must contain scheme and hostname." \ - in self.client.out + assert ("WARN: The URL 'foobar.com' is invalid. It must contain scheme and hostname." + in self.client.out) self.client.run("remote list") assert "foobar.com" in self.client.out self.client.run("remote update foobar --url pepe.org") - assert "WARN: The URL 'pepe.org' is invalid. It must contain scheme and hostname." \ - in self.client.out + assert ("WARN: The URL 'pepe.org' is invalid. It must contain scheme and hostname." + in self.client.out) self.client.run("remote list") assert "pepe.org" in self.client.out def test_errors(self): - self.client.run("remote update origin --url http://foo.com", assert_error=True) + self.client.run("remote update origin --url https://foo.com", assert_error=True) assert "ERROR: Remote 'origin' doesn't exist" in self.client.out self.client.run("remote remove origin", assert_error=True) @@ -205,15 +205,15 @@ def test_verify_ssl(self): data = json.loads(registry) assert data["remotes"][0]["name"] == "my-remote" assert data["remotes"][0]["url"] == "http://someurl" - assert data["remotes"][0]["verify_ssl"] == True + assert data["remotes"][0]["verify_ssl"] is True assert data["remotes"][1]["name"] == "my-remote2" assert data["remotes"][1]["url"] == "http://someurl2" - assert data["remotes"][1]["verify_ssl"] == False + assert data["remotes"][1]["verify_ssl"] is False assert data["remotes"][2]["name"] == "my-remote3" assert data["remotes"][2]["url"] == "http://someurl3" - assert data["remotes"][2]["verify_ssl"] == True + assert data["remotes"][2]["verify_ssl"] is True def test_remote_disable(self): client = TestClient(light=True) @@ -231,10 +231,10 @@ def test_remote_disable(self): data = json.loads(registry) assert data["remotes"][0]["name"] == "my-remote0" assert data["remotes"][0]["url"] == "http://someurl0" - assert data["remotes"][0]["disabled"] == True + assert data["remotes"][0]["disabled"] is True assert data["remotes"][3]["name"] == "my-remote3" assert data["remotes"][3]["url"] == "http://someurl3" - assert data["remotes"][3]["disabled"] == True + assert data["remotes"][3]["disabled"] is True # check that they are still listed, as disabled client.run("remote list") @@ -246,7 +246,7 @@ def test_remote_disable(self): registry = load(client.paths.remotes_path) data = json.loads(registry) for remote in data["remotes"]: - assert remote["disabled"] == True + assert remote["disabled"] is True data = json.loads(client.stdout) for remote in data: @@ -350,25 +350,31 @@ def test_using_frozen_center(): """ c = TestClient(light=True) c.run("remote add whatever https://center.conan.io") - assert "The remote 'https://center.conan.io' is now frozen and has been replaced by 'https://center2.conan.io'." in c.out + assert ("The remote 'https://center.conan.io' is now frozen and has been " + "replaced by 'https://center2.conan.io'.") in c.out assert 'conan remote update whatever --url="https://center2.conan.io"' in c.out c.run("remote list") - assert "The remote 'https://center.conan.io' is now frozen and has been replaced by 'https://center2.conan.io'." in c.out + assert ("The remote 'https://center.conan.io' is now frozen and has been " + "replaced by 'https://center2.conan.io'.") in c.out c.run("remote remove whatever") c.run("remote add conancenter https://center2.conan.io") - assert "The remote 'https://center.conan.io' is now frozen and has been replaced by 'https://center2.conan.io'." not in c.out + assert ("The remote 'https://center.conan.io' is now frozen and has been " + "replaced by 'https://center2.conan.io'.") not in c.out c.run("remote list") - assert "The remote 'https://center.conan.io' is now frozen and has been replaced by 'https://center2.conan.io'." not in c.out + assert ("The remote 'https://center.conan.io' is now frozen and has been " + "replaced by 'https://center2.conan.io'.") not in c.out c.run("remote update conancenter --url=https://center.conan.io") - assert "The remote 'https://center.conan.io' is now frozen and has been replaced by 'https://center2.conan.io'." in c.out + assert ("The remote 'https://center.conan.io' is now frozen and has been " + "replaced by 'https://center2.conan.io'.") in c.out c.run("remote disable conancenter") - assert "The remote 'https://center.conan.io' is now frozen and has been replaced by 'https://center2.conan.io'." not in c.out + assert ("The remote 'https://center.conan.io' is now frozen and has been " + "replaced by 'https://center2.conan.io'.") not in c.out def test_wrong_remotes_json_file(): @@ -402,6 +408,7 @@ def test_allowed_packages_remotes(): assert "app/1.0: Downloaded recipe revision 04ab3bc4b945a2ee44285962c277906d" in tc.out assert "liba/1.0: Downloaded recipe revision 4d670581ccb765839f2239cc8dff8fbd" not in tc.out assert "ERROR: Package 'liba/1.0' not resolved: Unable to find 'liba/1.0' in remotes" in tc.out + assert "Required by 'app/1.0'" in tc.out tc.run("remove * -c") tc.run('remote update default --allowed-packages="liba/*" --allowed-packages="app/*"') @@ -449,4 +456,5 @@ def test_remote_allowed_negation(): assert "lib/1.0: Downloaded recipe revision" in tc.out tc.run("install --requires=config/1.0 -r=default", assert_error=True) - assert "ERROR: Package 'config/1.0' not resolved: Unable to find 'config/1.0' in remotes" in tc.out + assert ("ERROR: Package 'config/1.0' not resolved: Unable to find " + "'config/1.0' in remotes") in tc.out diff --git a/test/integration/command/remote_verify_ssl_test.py b/test/integration/command/remote/remote_verify_ssl_test.py similarity index 100% rename from test/integration/command/remote_verify_ssl_test.py rename to test/integration/command/remote/remote_verify_ssl_test.py diff --git a/test/integration/command/test_remote_users.py b/test/integration/command/remote/test_remote_users.py similarity index 99% rename from test/integration/command/test_remote_users.py rename to test/integration/command/remote/test_remote_users.py index 9e2ad9bd72a..a96ac618028 100644 --- a/test/integration/command/test_remote_users.py +++ b/test/integration/command/remote/test_remote_users.py @@ -4,7 +4,7 @@ from collections import OrderedDict from datetime import timedelta -from mock import patch +from unittest.mock import patch from conan.internal.api.remotes.localdb import LocalDB from conan.test.utils.tools import TestClient, TestServer diff --git a/test/integration/command/remove_test.py b/test/integration/command/remove_test.py index 771c47507b8..ee6de258d38 100644 --- a/test/integration/command/remove_test.py +++ b/test/integration/command/remove_test.py @@ -105,7 +105,7 @@ def test_remove_all_packages_but_the_recipe_at_remote(self): c.run("list *:* -r default") assert "foobar/0.1@user/testing" in c.out assert "arch: x86_64" in c.out - assert "arch: x86" in c.out + assert "arch: x86" in c.out c.run("remove -c foobar/0.1@user/testing:* -r default") c.run("list *:* -r default") diff --git a/test/integration/command/source_test.py b/test/integration/command/source_test.py index f1021bbe3ee..f818ec3133a 100644 --- a/test/integration/command/source_test.py +++ b/test/integration/command/source_test.py @@ -133,7 +133,7 @@ def source(self): # Fix the error and repeat client.save({CONANFILE: conanfile.replace("err", "")}) - client.run("source .") + client.run("source") assert "conanfile.py: Calling source() in" in client.out assert "conanfile.py: Running source!" in client.out assert "Hello World" == client.load("file1.txt") diff --git a/test/integration/command/test_audit.py b/test/integration/command/test_audit.py index ca9433bfa44..6ea38735a11 100644 --- a/test/integration/command/test_audit.py +++ b/test/integration/command/test_audit.py @@ -5,7 +5,7 @@ from contextlib import contextmanager import pytest -from mock.mock import patch, MagicMock +from unittest.mock import patch, MagicMock from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.env import environment_update @@ -56,7 +56,7 @@ @contextmanager def proxy_response(status, data, retry_after=60): - with patch("conan.api.conan_api.RemotesAPI.requester") as conanRequesterMock: + with patch("conan.api.conan_api.ConanAPI._ApiHelpers.requester") as conanRequesterMock: return_status = MagicMock() return_status.status_code = status return_status.json = MagicMock(return_value=data) @@ -140,6 +140,13 @@ def test_conan_audit_proxy(): tc.run("audit scan --requires=zlib/1.2.11") assert "zlib/1.2.11 1 vulnerability found" in tc.out + tc.save({"conanfile.txt": "[requires]\nzlib/1.2.11\n"}, clean_first=True) + tc.run("audit scan") + assert "zlib/1.2.11 1 vulnerability found" in tc.out + + tc.run("audit scan . --requires=zlib/1.2.11", assert_error=True) + assert "--requires and --tool-requires arguments are incompatible with [path] '.' argument" in tc.out + # Now some common errors, like rate limited or missing lib, but it should not fail! with proxy_response(429, {"error": "Rate limit exceeded"}): tc.run("audit list zlib/1.2.11", assert_error=True) @@ -350,7 +357,7 @@ def fake_post(url, headers, json): # noqa return response with environment_update({"CONAN_AUDIT_PROVIDER_TOKEN_CONANCENTER": "env_token_value"}): - with patch("conan.api.conan_api.RemotesAPI.requester", + with patch("conan.api.conan_api.ConanAPI._ApiHelpers.requester", new_callable=MagicMock) as requester_mock: requester_mock.post = fake_post tc.run("audit list zlib/1.2.11") @@ -472,3 +479,23 @@ def test_parse_error_crash_when_no_edges(): assert "conan_error" in scan_result assert "zlib/1.2.11" in scan_result["conan_error"] assert "7.0" in scan_result["conan_error"] + + +@pytest.mark.parametrize("package_context", ["build", "host"]) +@pytest.mark.parametrize("filter_context", ["build", "host", None]) +def test_audit_scan_context_filter(package_context, filter_context): + tc = TestClient(light=True) + + tc.save({"conanfile.py": GenConanfile("zlib", "1.2.11")}) + tc.run("export .") + tc.run("audit provider auth conancenter --token=valid_token") + + requires = "requires" if package_context == "host" else "tool-requires" + context = "" if filter_context is None else f"--context={filter_context}" + + with proxy_response(200, {}): + tc.run(f"audit scan --{requires}=zlib/1.2.11 {context}") + if filter_context is None or filter_context == package_context: + assert "Requesting vulnerability info for: zlib/1.2.11" in tc.out + else: + assert "Requesting vulnerability info for: zlib/1.2.11" not in tc.out diff --git a/test/integration/command/test_build.py b/test/integration/command/test_build.py index 0d8cbd0f046..a3881465c6a 100644 --- a/test/integration/command/test_build.py +++ b/test/integration/command/test_build.py @@ -29,7 +29,7 @@ def test_build_output_json(): client = TestClient() conanfile = GenConanfile() client.save({"conanfile.py": conanfile}) - client.run("build . --format=json", redirect_stdout=_OUTPUT_FILE) + client.run("build --format=json", redirect_stdout=_OUTPUT_FILE) output = json.loads(client.load(_OUTPUT_FILE)) assert "graph" in output diff --git a/test/integration/command/test_graph_find_binaries.py b/test/integration/command/test_graph_find_binaries.py index fd7e0eb1585..2af0c0b11fd 100644 --- a/test/integration/command/test_graph_find_binaries.py +++ b/test/integration/command/test_graph_find_binaries.py @@ -29,8 +29,8 @@ def test_exact_match(self, client): """) assert textwrap.indent(expected, " ") in c.out c.run("graph explain --requires=lib/1.0 --missing=lib/1.0 -s os=Windows --format=json") - cache = json.loads(c.stdout)["closest_binaries"] - revisions = cache["lib/1.0"]["revisions"] + closest = json.loads(c.stdout)["closest_binaries"] + revisions = closest["lib/1.0"]["revisions"] pkgs = revisions["5313a980ea0c56baeb582c510d6d9fbc"]["packages"] assert len(pkgs) == 1 pkg1 = pkgs["3d714b452400b3c3d6a964f42d5ec5004a6f22dc"] @@ -55,8 +55,8 @@ def test_settings_incomplete(self, client): """) assert textwrap.indent(expected, " ") in c.out c.run("graph explain --requires=lib/1.0 -pr windows --format=json") - cache = json.loads(c.stdout)["closest_binaries"] - revisions = cache["lib/1.0"]["revisions"] + closest = json.loads(c.stdout)["closest_binaries"] + revisions = closest["lib/1.0"]["revisions"] pkgs = revisions["5313a980ea0c56baeb582c510d6d9fbc"]["packages"] assert len(pkgs) == 1 pkg1 = pkgs["3d714b452400b3c3d6a964f42d5ec5004a6f22dc"] @@ -99,8 +99,8 @@ def test_different_option(self, client): """) assert textwrap.indent(expected, " ") in c.out c.run("graph explain --requires=lib/1.0 -pr linux --format=json") - cache = json.loads(c.stdout)["closest_binaries"] - revisions = cache["lib/1.0"]["revisions"] + closest = json.loads(c.stdout)["closest_binaries"] + revisions = closest["lib/1.0"]["revisions"] pkgs = revisions["5313a980ea0c56baeb582c510d6d9fbc"]["packages"] assert len(pkgs) == 1 pkg1 = pkgs["499989797d9192081b8f16f7d797b107a2edd8da"] @@ -158,8 +158,8 @@ def test_different_setting(self, client): """) assert textwrap.indent(expected, " ") in c.out c.run("graph explain --requires=lib/1.0 -pr windows --format=json") - cache = json.loads(c.stdout)["closest_binaries"] - revisions = cache["lib/1.0"]["revisions"] + closest = json.loads(c.stdout)["closest_binaries"] + revisions = closest["lib/1.0"]["revisions"] pkgs = revisions["5313a980ea0c56baeb582c510d6d9fbc"]["packages"] assert len(pkgs) == 1 pkg1 = pkgs["c2dd2d51b5074bdb5b7d717929372de09830017b"] @@ -196,8 +196,8 @@ def package_id(self): """) assert textwrap.indent(expected, " ") in c.out c.run("graph explain --tool-requires=tool/1.0 -s:b os=Windows -s:h os=Macos --format=json") - cache = json.loads(c.stdout)["closest_binaries"] - revisions = cache["tool/1.0"]["revisions"] + closest = json.loads(c.stdout)["closest_binaries"] + revisions = closest["tool/1.0"]["revisions"] pkgs = revisions["4cc4b286a46dc2ed188d8c417eadb4e6"]["packages"] assert len(pkgs) == 1 pkg1 = pkgs["d66135125c07cc240b8d6adda090b76d60341205"] @@ -225,8 +225,8 @@ def test_different_platform(self, client): """) assert textwrap.indent(expected, " ") in c.out c.run("graph explain --requires=lib/1.0 -pr macos --format=json") - cache = json.loads(c.stdout)["closest_binaries"] - revisions = cache["lib/1.0"]["revisions"] + closest = json.loads(c.stdout)["closest_binaries"] + revisions = closest["lib/1.0"]["revisions"] pkgs = revisions["5313a980ea0c56baeb582c510d6d9fbc"]["packages"] assert len(pkgs) == 1 pkg1 = pkgs["c2dd2d51b5074bdb5b7d717929372de09830017b"] @@ -254,8 +254,8 @@ def test_different_conf(self, client): """) assert textwrap.indent(expected, " ") in c.out c.run("graph explain --requires=lib/1.0 -c user.foo:bar=42 -s os=Linux -f=json") - cache = json.loads(c.stdout)["closest_binaries"] - revisions = cache["lib/1.0"]["revisions"] + closest = json.loads(c.stdout)["closest_binaries"] + revisions = closest["lib/1.0"]["revisions"] pkgs = revisions["5313a980ea0c56baeb582c510d6d9fbc"]["packages"] assert len(pkgs) == 1 pkg1 = pkgs["499989797d9192081b8f16f7d797b107a2edd8da"] @@ -336,8 +336,8 @@ def test_different_python_requires(self): """) assert textwrap.indent(expected, " ") in c.out c.run("graph explain --requires=lib/1.0 -s os=Linux --format=json") - cache = json.loads(c.stdout)["closest_binaries"] - revisions = cache["lib/1.0"]["revisions"] + closest = json.loads(c.stdout)["closest_binaries"] + revisions = closest["lib/1.0"]["revisions"] pkgs = revisions["7bf17caa5bf9d2ed1dd8b337e9623fc0"]["packages"] assert len(pkgs) == 1 pkg1 = pkgs["5ccdb706197ca94edc0ecee9ef0d0b11b887d937"] @@ -370,8 +370,8 @@ def test_build_requires(self): """) assert textwrap.indent(expected, " ") in c.out c.run("graph explain --requires=lib/1.0 -s os=Linux --format=json") - cache = json.loads(c.stdout)["closest_binaries"] - revisions = cache["lib/1.0"]["revisions"] + closest = json.loads(c.stdout)["closest_binaries"] + revisions = closest["lib/1.0"]["revisions"] pkgs = revisions["4790f7f1561b52be5d39f0bc8e9acbed"]["packages"] assert len(pkgs) == 1 pkg1 = pkgs["c9d96a611b8c819f35728d58d743f6a78a1b5942"] diff --git a/test/integration/command/test_inspect.py b/test/integration/command/test_inspect.py index e23ee392aa2..55636cd1a74 100644 --- a/test/integration/command/test_inspect.py +++ b/test/integration/command/test_inspect.py @@ -51,7 +51,7 @@ def test_missing_conanfile(): def test_dot_and_folder_conanfile(): t = TestClient() t.save({"conanfile.py": GenConanfile().with_name("foo")}) - t.run("inspect .") + t.run("inspect") assert 'name: foo' in t.out t.save({"foo/conanfile.py": GenConanfile().with_name("foo")}, clean_first=True) t.run("inspect foo") diff --git a/test/integration/command/test_new.py b/test/integration/command/test_new.py index 0cadbc55141..ac7e8e5973f 100644 --- a/test/integration/command/test_new.py +++ b/test/integration/command/test_new.py @@ -51,10 +51,18 @@ def test_new_defaults(self): c = TestClient(light=True) for t in ("cmake_lib", "cmake_exe", "meson_lib", "meson_exe", "msbuild_lib", "msbuild_exe", "bazel_lib", "bazel_exe", "autotools_lib", "autotools_exe"): - c.run(f"new {t} -f") + c.save({}, clean_first=True) + c.run(f"new {t}") conanfile = c.load("conanfile.py") assert 'name = "mypkg"' in conanfile assert 'version = "0.1"' in conanfile + if t == "cmake_lib": + cmake = c.load("CMakeLists.txt") + assert "src/mypkg.cpp" in cmake + cpp = c.load("src/mypkg.cpp") + assert "mypkg()" in cpp + hpp = c.load("include/mypkg.h") + assert "void mypkg();" in hpp c.run(f"new {t} -d name=mylib -f") conanfile = c.load("conanfile.py") diff --git a/test/integration/command/test_outdated.py b/test/integration/command/test_outdated.py index c5294cf7372..4c94e38bc7a 100644 --- a/test/integration/command/test_outdated.py +++ b/test/integration/command/test_outdated.py @@ -9,7 +9,7 @@ @pytest.fixture def create_libs(): - tc = TestClient(default_server_user=True) + tc = TestClient(default_server_user=True, light=True) tc.save({"conanfile.py": GenConanfile()}) tc.run("create . --name=zlib --version=1.0") tc.run("create . --name=zlib --version=2.0") diff --git a/test/integration/command/test_output.py b/test/integration/command/test_output.py index bd383905f53..261f5ec0b49 100644 --- a/test/integration/command/test_output.py +++ b/test/integration/command/test_output.py @@ -12,9 +12,7 @@ def test_invalid_output_level(self): t = TestClient(light=True) t.save({"conanfile.py": GenConanfile("foo", "1.0")}) t.run("create . -vfooling", assert_error=True) - assert "Invalid argument '-vfooling'" in t.out - assert "Allowed values: quiet, error, warning, notice, status, verbose, " \ - "debug(v), trace(vv)" in t.out + assert "argument -v: invalid choice: 'fooling'" in t.out with environment_update({"CONAN_LOG_LEVEL": "fail"}): t.run("create .", assert_error=True) diff --git a/test/integration/command/test_package_test.py b/test/integration/command/test_package_test.py index a48c1f1e0a8..b9b3ea7a8d9 100644 --- a/test/integration/command/test_package_test.py +++ b/test/integration/command/test_package_test.py @@ -12,10 +12,22 @@ class TestPackageTest: def test_basic(self): client = TestClient() + test_package = textwrap.dedent(""" + from conan import ConanFile + class TestPackage(ConanFile): + def requirements(self): + self.requires(self.tested_reference_str) + def test(self): + self.output.info("TESTING") + """) client.save({CONANFILE: GenConanfile("hello", "0.1"), - "test_package/conanfile.py": GenConanfile().with_test("pass")}) + "test_package/conanfile.py": test_package}) client.run("create . --user=lasote --channel=stable") assert "hello/0.1@lasote/stable: Created package" in client.out + client.run("test test_package hello/0.1@lasote/stable") + assert "hello/0.1@lasote/stable (test package): TESTING" in client.out + client.run("test hello/0.1@lasote/stable") + assert "hello/0.1@lasote/stable (test package): TESTING" in client.out def test_basic_json(self): client = TestClient() diff --git a/test/integration/command/test_run.py b/test/integration/command/test_run.py new file mode 100644 index 00000000000..7a56eb371db --- /dev/null +++ b/test/integration/command/test_run.py @@ -0,0 +1,105 @@ +import textwrap + +import pytest +import platform + +from conan.test.assets.genconanfile import GenConanfile +from conan.test.utils.tools import TestClient + + +executable = "myapp.bat" if platform.system() == "Windows" else "myapp.sh" + + +@pytest.fixture(scope="module") +def client(): + tc = TestClient(default_server_user=True) + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.files import save + import os + + class Pkg(ConanFile): + name = "pkg" + version = "0.1" + # So that the requirement is run=True even for --requires + package_type = "application" + options = {"foo": [True, False, "bar"]} + default_options = {"foo": True} + settings = "os" + + def package(self): + executable = os.path.join(self.package_folder, "bin", '""" + executable + """') + save(self, executable, f"echo Hello World! foo={self.options.foo}") + # Make it executable + os.chmod(executable, 0o755) + """) + tc.save({"pkg/conanfile.py": conanfile}) + tc.run("create pkg") + return tc + + +@pytest.mark.parametrize("context_flag", ["host", "build", None]) +@pytest.mark.parametrize("requires_context", ["host", "build",]) +@pytest.mark.parametrize("use_conanfile", [True, False]) +def test_run(client, context_flag, requires_context, use_conanfile): + context_arg = { + "host": "--context=host", + "build": "--context=build", + None: "", + }.get(context_flag) + should_find_binary = (context_flag == requires_context) or (context_flag is None) + if use_conanfile: + conanfile_consumer = GenConanfile("consumer", "1.0").with_settings("os") + if requires_context == "host": + conanfile_consumer.with_requires("pkg/0.1") + else: + conanfile_consumer.with_tool_requires("pkg/0.1") + + client.save({"conanfile.py": conanfile_consumer}) + client.run(f"run {executable} {context_arg}", assert_error=not should_find_binary) + else: + requires = "requires" if requires_context == "host" else "tool-requires" + client.run(f"run {executable} --{requires}=pkg/0.1 {context_arg}", + assert_error=not should_find_binary) + if should_find_binary: + assert "Hello World!" in client.out + else: + if platform.system() == "Windows": + assert "not recognized as an internal or external command" in client.out + else: + assert "Error 127 while executing" in client.out + + +def test_run_context_priority(client): + client.run("create pkg -o=pkg/*:foo=False") + + client.run(f"run {executable} --requires=pkg/0.1 --tool-requires=pkg/0.1 -o:b=pkg/*:foo=False") + # True is host, False is build, run gives priority to host + assert "Hello World! foo=True" in client.out + + +def test_run_missing_executable(client): + client.run(f"run a-binary-name-that-does-not-exist --requires=pkg/0.1", assert_error=True) + if platform.system() == "Windows": + assert "not recognized as an internal or external command" in client.out + else: + assert "Error 127 while executing" in client.out + + +def test_run_missing_binary(client): + client.run("run foo --requires=pkg/0.1 -o=pkg/*:foo=bar", assert_error=True) + assert "Error installing the dependencies" in client.out + assert "Missing prebuilt package for 'pkg/0.1'" in client.out + + +def test_run_missing_package(client): + client.run("run foo --requires=pkg/2.1", assert_error=True) + assert "Error installing the dependencies" in client.out + assert "Package 'pkg/2.1' not resolved" in client.out + + +@pytest.mark.skipif(platform.system() == "Windows", reason="Unix only") +def test_run_status_is_propagated(client): + client.run("run false --requires=pkg/0.1", assert_error=True) + assert "Error installing the dependencies" not in client.out + assert "ERROR: Error 1 while executing" in client.out diff --git a/test/integration/command/upload/test_upload_bundle.py b/test/integration/command/upload/test_upload_bundle.py index 3e3bf7c4b68..e7effa3b539 100644 --- a/test/integration/command/upload/test_upload_bundle.py +++ b/test/integration/command/upload/test_upload_bundle.py @@ -6,10 +6,10 @@ from conan.test.utils.tools import TestClient -def test_upload_bundle(): - """ Test how a custom command can create an upload bundle and print it +def test_upload_pkg_list(): + """ Test how a custom command can create an upload pkglist and print it """ - c = TestClient(default_server_user=True) + c = TestClient(default_server_user=True, light=True) mycommand = textwrap.dedent(""" import json import os @@ -19,9 +19,9 @@ def test_upload_bundle(): from conan.api.output import cli_out_write @conan_command(group="custom commands") - def upload_bundle(conan_api, parser, *args, **kwargs): + def upload_pkglist(conan_api, parser, *args, **kwargs): \""" - create an upload bundle + create an upload pkglist \""" parser.add_argument('reference', help="Recipe reference or package reference, can contain * as " @@ -36,7 +36,7 @@ def upload_bundle(conan_api, parser, *args, **kwargs): ref_pattern = ListPattern(args.reference, package_id="*") package_list = conan_api.list.select(ref_pattern) - if not package_list.recipes: + if not package_list: raise ConanException("No recipes found matching pattern '{}'".format(args.reference)) # Check if the recipes/packages are in the remote @@ -46,11 +46,11 @@ def upload_bundle(conan_api, parser, *args, **kwargs): """) command_file_path = os.path.join(c.cache_folder, 'extensions', - 'commands', 'cmd_upload_bundle.py') + 'commands', 'cmd_upload_pkglist.py') c.save({command_file_path: mycommand}) c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) c.run("create .") - c.run('upload-bundle "*" -r=default', redirect_stdout="mybundle.json") - bundle = c.load("mybundle.json") - bundle = json.loads(bundle) - assert bundle["pkg/0.1"]["revisions"]["485dad6cb11e2fa99d9afbe44a57a164"]["upload"] is True + c.run('upload-pkglist "*" -r=default', redirect_stdout="mypkglist.json") + pkglist = c.load("mypkglist.json") + pkglist = json.loads(pkglist) + assert pkglist["pkg/0.1"]["revisions"]["485dad6cb11e2fa99d9afbe44a57a164"]["upload"] is True diff --git a/test/integration/command/upload/upload_compression_test.py b/test/integration/command/upload/upload_compression_test.py index 3d0768d97ce..43285c81c5c 100644 --- a/test/integration/command/upload/upload_compression_test.py +++ b/test/integration/command/upload/upload_compression_test.py @@ -67,7 +67,7 @@ def test_upload_only_tgz_if_needed(): conan_path = server_paths.conan_revisions_root(ref) assert os.path.exists(conan_path) - latest_rrev = client.cache.get_latest_recipe_reference(ref) + latest_rrev = client.cache.get_latest_recipe_revision(ref) package_ids = client.cache.get_package_references(latest_rrev) pref = package_ids[0] diff --git a/test/integration/command/upload/upload_test.py b/test/integration/command/upload/upload_test.py index 7146a13db0a..30de88a034d 100644 --- a/test/integration/command/upload/upload_test.py +++ b/test/integration/command/upload/upload_test.py @@ -6,13 +6,12 @@ from collections import OrderedDict import pytest -from mock import patch +from unittest.mock import patch from requests import Response from conan.errors import ConanException from conan.api.model import PkgReference from conan.internal.api.uploader import gzopen_without_timestamps -from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME, PACKAGE_TGZ_NAME from conan.test.utils.tools import NO_SETTINGS_PACKAGE_ID, TestClient, TestServer, \ GenConanfile, TestRequester, TestingResponse from conan.internal.util.files import is_dirty, save, set_dirty, sha1sum @@ -129,7 +128,7 @@ def gzopen_patched(name, mode="r", fileobj=None, **kwargs): # noqa export_download_folder = layout.download_export() - tgz = os.path.join(export_download_folder, EXPORT_SOURCES_TGZ_NAME) + tgz = os.path.join(export_download_folder, "conan_sources.tgz") assert os.path.exists(tgz) assert is_dirty(tgz) @@ -147,7 +146,7 @@ def test_broken_package_tgz(self): pref = client.created_layout().reference def gzopen_patched(name, fileobj, compresslevel=None): # noqa - if name == PACKAGE_TGZ_NAME: + if name == "conan_package.tgz": raise ConanException("Error gzopen %s" % name) return gzopen_without_timestamps(name, fileobj) with patch('conan.internal.api.uploader.gzopen_without_timestamps', new=gzopen_patched): @@ -155,7 +154,7 @@ def gzopen_patched(name, fileobj, compresslevel=None): # noqa assert "Error gzopen conan_package.tgz" in client.out download_folder = client.get_latest_pkg_layout(pref).download_package() - tgz = os.path.join(download_folder, PACKAGE_TGZ_NAME) + tgz = os.path.join(download_folder, "conan_package.tgz") assert os.path.exists(tgz) assert is_dirty(tgz) diff --git a/test/integration/conanfile/init_test.py b/test/integration/conanfile/init_test.py index a342784bcda..2a82ec42453 100644 --- a/test/integration/conanfile/init_test.py +++ b/test/integration/conanfile/init_test.py @@ -49,3 +49,36 @@ def build(self): client.run("create . --name=pkg --version=0.1 --user=user --channel=testing") assert "description: MyDescription" in client.out assert "license: MIT" in client.out + + def test_options_from_yml(self): + client = TestClient() + conanfile = textwrap.dedent("""\ + from conan import ConanFile + + class Lib(ConanFile): + name = "pkg" + version = "0.1" + + def init(self): + self.options.update(self.conan_data["options"], + self.conan_data["default_options"]) + """) + conandata = textwrap.dedent("""\ + options: + myopt1: [1, 2, 3] + myopt2: [null, potato] + myopt3: [null, ANY] + default_options: + myopt1: 2 + """) + client.save({"conanfile.py": conanfile, + "conandata.yml": conandata}) + + client.run("create . ") + client.run("create . -o &:myopt1=1 -o &:myopt2=potato -o &:myopt3=whatever") + client.run("list *:*") + + assert "myopt1: 2" in client.out + assert "myopt1: 1" in client.out + assert "myopt2: potato" in client.out + assert "myopt3: whatever" in client.out diff --git a/test/integration/conanfile/required_conan_version_test.py b/test/integration/conanfile/required_conan_version_test.py index cc63cc089af..3e2db6bd222 100644 --- a/test/integration/conanfile/required_conan_version_test.py +++ b/test/integration/conanfile/required_conan_version_test.py @@ -1,6 +1,6 @@ import textwrap -import mock +from unittest import mock from conan import __version__ from conan.test.utils.tools import TestClient diff --git a/test/integration/conanfile/test_deprecated.py b/test/integration/conanfile/test_deprecated.py index d6a2b466c11..128cc536246 100644 --- a/test/integration/conanfile/test_deprecated.py +++ b/test/integration/conanfile/test_deprecated.py @@ -1,7 +1,5 @@ import textwrap -import pytest - from conan.test.utils.tools import TestClient, GenConanfile @@ -65,18 +63,3 @@ def configure(self): tc.run("graph info . --version=2.0") assert "deprecated: None" in tc.out - -class TestDeprecatedIncludesMessages: - @pytest.mark.parametrize("import_stm,way", [ - ["from conans.model.package_ref import PkgReference", "PkgReference()"], - ["from conans.model.package_ref import PkgReference", "PkgReference.loads('foo/1.0:123')"], - ["from conans.model.recipe_ref import RecipeReference", "RecipeReference()"], - ["from conans.model.recipe_ref import RecipeReference", "RecipeReference.loads('foo/1.0@user/channel')"] - ]) - def test_deprecated_package_ref_include(self, import_stm, way): - t = TestClient(light=True) - t.save({"conanfile.py": GenConanfile("pkg", "0.1") - .with_import(import_stm) - .with_package_id(f"ref = {way}")}) - t.run("create conanfile.py") - assert f"Private '{import_stm}'" in t.out diff --git a/test/integration/configuration/conf/test_auth_source_plugin.py b/test/integration/configuration/conf/test_auth_source_plugin.py index a90e316f5a2..b8067cec75f 100644 --- a/test/integration/configuration/conf/test_auth_source_plugin.py +++ b/test/integration/configuration/conf/test_auth_source_plugin.py @@ -1,9 +1,14 @@ import json import os import textwrap +from unittest import mock +from unittest.mock import MagicMock import pytest +from conan import __version__ +from conan.internal.model.conf import ConfDefinition +from conan.internal.rest.conan_requester import ConanRequester from conan.test.utils.file_server import TestFileServer from conan.test.utils.test_files import temp_folder from conan.test.utils.tools import TestClient @@ -13,30 +18,29 @@ class TestAuthSourcePlugin: @pytest.fixture def setup_test_client(self): - self.client = TestClient(default_server_user=True, light=True) - self.file_server = TestFileServer() - self.client.servers["file_server"] = self.file_server - self.download_cache_folder = temp_folder() - self.client.save_home({"global.conf": "tools.files.download:retry=0"}) + client = TestClient(default_server_user=True, light=True) + file_server = TestFileServer() + client.servers["file_server"] = file_server + client.save_home({"global.conf": "tools.files.download:retry=0"}) - save(os.path.join(self.file_server.store, "myfile.txt"), "Bye, world!") + save(os.path.join(file_server.store, "myfile.txt"), "Bye, world!") conanfile = textwrap.dedent(f""" - from conan import ConanFile - from conan.tools.files import download - class Pkg2(ConanFile): - name = "pkg" - version = "1.0" - def source(self): - download(self, "{self.file_server.fake_url}/basic-auth/myfile.txt", "myfile.txt") - """) - self.client.save({"conanfile.py": conanfile}) + from conan import ConanFile + from conan.tools.files import download + class Pkg2(ConanFile): + name = "pkg" + version = "1.0" + def source(self): + download(self, "{file_server.fake_url}/basic-auth/myfile.txt", "myfile.txt") + """) + client.save({"conanfile.py": conanfile}) - return self.client, self.file_server.fake_url + return client, file_server.fake_url - """ Test when the plugin fails, we want a clear message and a helpful trace - """ def test_error_source_plugin(self, setup_test_client): + """ Test when the plugin fails, we want a clear message and a helpful trace + """ c, url = setup_test_client auth_plugin = textwrap.dedent("""\ def auth_source_plugin(url): @@ -46,16 +50,16 @@ def auth_source_plugin(url): c.run("source conanfile.py", assert_error=True) assert "Test Error" in c.out - """ Test when the plugin give a correct and wrong password, we want a message about the success - or fail in login - """ @pytest.mark.parametrize("password", ["password", "bad-password"]) def test_auth_source_plugin_direct_credentials(self, password, setup_test_client): + """ Test when the plugin give a correct and wrong password, we want a message about + the success or fail in login + """ should_fail = password == "bad-password" c, url = setup_test_client auth_plugin = textwrap.dedent(f"""\ def auth_source_plugin(url): - return { json.dumps({'user': 'user', 'password': password}) } + return {json.dumps({'user': 'user', 'password': password})} """) c.save_home({"extensions/plugins/auth_source.py": auth_plugin}) c.run("source conanfile.py", assert_error=should_fail) @@ -64,10 +68,10 @@ def auth_source_plugin(url): else: assert os.path.exists(os.path.join(c.current_folder, "myfile.txt")) - """ Test when the plugin do not give any user or password, we want the code to continue with - the rest of the input methods - """ def test_auth_source_plugin_fallback(self, setup_test_client): + """ Test when the plugin do not give any user or password, we want the code to continue with + the rest of the input methods + """ c, url = setup_test_client auth_plugin = textwrap.dedent("""\ def auth_source_plugin(url): @@ -75,8 +79,38 @@ def auth_source_plugin(url): """) c.save_home({"extensions/plugins/auth_source.py": auth_plugin}) source_credentials = json.dumps({"credentials": [{"url": url, "token": "password"}]}) - save(os.path.join(c.cache_folder, "source_credentials.json"), source_credentials) + c.save_home({"source_credentials.json": source_credentials}) c.run("source conanfile.py") # As the auth plugin is not returning any password the code is falling back to the rest of # the input methods in this case provided by source_credentials.json. assert os.path.exists(os.path.join(c.current_folder, "myfile.txt")) + + def test_source_auth_plugin_headers(self): + auth_plugin = textwrap.dedent(f"""\ + def auth_source_plugin(url): + return {json.dumps({'headers': {"myheader": "myvalue"}})} + """) + cache_folder = temp_folder() + save(os.path.join(cache_folder, "extensions/plugins/auth_source.py"), auth_plugin) + + mock_http_requester = MagicMock() + with mock.patch("conan.internal.rest.conan_requester.requests", mock_http_requester): + requester = ConanRequester(ConfDefinition(), cache_folder) + requester.get(url="aaa", source_credentials=True) + headers = requester._http_requester.get.call_args[1]["headers"] + assert headers["myheader"] == "myvalue" + assert f"Conan/{__version__}" in headers["User-Agent"] + + def test_source_credentials_headers(self): + cache_folder = temp_folder() + source_credentials = json.dumps({"credentials": [{"url": "aaa", "token": "mytok", + "headers": {"myheader2": "myvalue2"}}]}) + save(os.path.join(cache_folder, "source_credentials.json"), source_credentials) + + mock_http_requester = MagicMock() + with mock.patch("conan.internal.rest.conan_requester.requests", mock_http_requester): + requester = ConanRequester(ConfDefinition(), cache_folder) + requester.get(url="aaa", source_credentials=True) + headers = requester._http_requester.get.call_args[1]["headers"] + assert headers["myheader2"] == "myvalue2" + assert f"Conan/{__version__}" in headers["User-Agent"] diff --git a/test/integration/configuration/conf/test_conf.py b/test/integration/configuration/conf/test_conf.py index 5b5c3b3a2d5..03654b0cf7b 100644 --- a/test/integration/configuration/conf/test_conf.py +++ b/test/integration/configuration/conf/test_conf.py @@ -3,7 +3,7 @@ import textwrap import pytest -from mock import patch +from unittest.mock import patch from conan import conan_version from conan.internal.api.detect import detect_api diff --git a/test/integration/configuration/custom_setting_test_package_test.py b/test/integration/configuration/custom_setting_test_package_test.py index 53011e57db6..d290e64758a 100644 --- a/test/integration/configuration/custom_setting_test_package_test.py +++ b/test/integration/configuration/custom_setting_test_package_test.py @@ -26,4 +26,4 @@ def test(self): client.save({"conanfile.py": conanfile, "test_package/conanfile.py": test_conanfile}) client.run("create . -s os=Windows -s product=onion -s build_type=Release") - assert"hello/0.1 (test package): TestSettings: Windows, Release, onion", client.out + assert "hello/0.1 (test package): TestSettings: Windows, Release, onion" in client.out diff --git a/test/integration/configuration/default_profile_test.py b/test/integration/configuration/default_profile_test.py index 6cacd4cc6a0..11e444c9adf 100644 --- a/test/integration/configuration/default_profile_test.py +++ b/test/integration/configuration/default_profile_test.py @@ -18,8 +18,7 @@ def test_conanfile_txt_incomplete_profile(self): client.save({CONANFILE: conanfile}) client.run("create . --name=pkg --version=0.1 --user=lasote --channel=stable") - assert "pkg/0.1@lasote/stable: Package '%s' created" % NO_SETTINGS_PACKAGE_ID \ - in client.out + assert "pkg/0.1@lasote/stable: Package '%s' created" % NO_SETTINGS_PACKAGE_ID in client.out client.save({"conanfile.txt": "[requires]\npkg/0.1@lasote/stable"}, clean_first=True) client.run('install .') @@ -143,6 +142,16 @@ def build(self): assert_error=True) assert "You need to create a default profile" in client.out + def test_default_profile(self): + c = TestClient() + # Test with a profile set using and environment variable + tmp = temp_folder() + profile = os.path.join(tmp, 'myprofile') + save(profile, "[settings]\nos=FreeBSD\n") + with environment_update({'CONAN_DEFAULT_BUILD_PROFILE': profile}): + c.run("profile show") + assert "os=FreeBSD" in c.out # build profiles + def test_conf_default_two_profiles(): client = TestClient() diff --git a/test/integration/configuration/profile_test.py b/test/integration/configuration/profile_test.py index f031fded2cd..ff59ed1ec63 100644 --- a/test/integration/configuration/profile_test.py +++ b/test/integration/configuration/profile_test.py @@ -6,7 +6,6 @@ from textwrap import dedent import pytest -from parameterized import parameterized from conan.internal.paths import CONANFILE from conan.test.assets.genconanfile import GenConanfile @@ -637,6 +636,7 @@ def test_consumer_invalid_profile_multiple_groups(): assert ("ERROR: Error reading 'myprofs/myprofile' profile: ConfigParser: " "Duplicated section: [settings]") in client.out + def test_compose_numeric_values_scoped_pkg(): tc = TestClient(light=True) tc.save({"conanfile.py": GenConanfile("hello", "0.1") @@ -645,3 +645,33 @@ def test_compose_numeric_values_scoped_pkg(): tc.run("create . -pr=profile") assert "user.var.value: 10" in tc.out + + +def test_nullify_settings(): + c = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + class Pkg(ConanFile): + settings = "build_type" + def configure(self): + if not self.settings.build_type: + self.output.info("BUILD TYPE NOT DEFINED!!!") + else: + self.output.info(f"I am a {self.settings.build_type} pkg!!!") + """) + c.save({"conanfile.py": conanfile, + "profile": "[settings]\nbuild_type=Release\npkg1/*:build_type=~"}) + c.run("create . --name=pkg1 --version=0.1") + c.run("create . --name=pkg2 --version=0.1") + c.run("create . --name=pkg3 --version=0.1") + c.save({"conanfile.py": GenConanfile().with_requires("pkg1/0.1", "pkg2/0.1", "pkg3/0.1")}) + + c.run("graph info . -s build_type=Release -s pkg1/*:build_type=~") + assert "pkg1/0.1: BUILD TYPE NOT DEFINED!!!" in c.out + assert "pkg2/0.1: I am a Release pkg!!!" in c.out + assert "pkg3/0.1: I am a Release pkg!!!" in c.out + + c.run("graph info . -pr=profile") + assert "pkg1/0.1: BUILD TYPE NOT DEFINED!!!" in c.out + assert "pkg2/0.1: I am a Release pkg!!!" in c.out + assert "pkg3/0.1: I am a Release pkg!!!" in c.out diff --git a/test/integration/configuration/requester_test.py b/test/integration/configuration/requester_test.py index 71816d51ddf..ba72cc69dfb 100644 --- a/test/integration/configuration/requester_test.py +++ b/test/integration/configuration/requester_test.py @@ -1,7 +1,7 @@ import os -import mock -from mock import Mock, MagicMock +from unittest import mock +from unittest.mock import Mock, MagicMock from conan import __version__ from conan.internal.rest.conan_requester import ConanRequester @@ -23,14 +23,14 @@ def test_default_no_verify(self): with mock.patch("conan.internal.rest.conan_requester.requests", mocked_requester): requester = ConanRequester(ConfDefinition()) requester.get(url="aaa", verify=False) - assert requester._http_requester.verify == False + assert requester._http_requester.verify is False def test_default_verify(self): mocked_requester = MockRequesterGet() with mock.patch("conan.internal.rest.conan_requester.requests", mocked_requester): requester = ConanRequester(ConfDefinition()) requester.get(url="aaa", verify=True) - assert requester._http_requester.verify == True + assert requester._http_requester.verify is True def test_cache_config(self): file_path = os.path.join(temp_folder(), "whatever_cacert") diff --git a/test/integration/configuration/required_version_test.py b/test/integration/configuration/required_version_test.py index bea17ea0dbe..1898c198a86 100644 --- a/test/integration/configuration/required_version_test.py +++ b/test/integration/configuration/required_version_test.py @@ -1,5 +1,5 @@ import os -import mock +from unittest import mock from conan import __version__ from conan.test.utils.test_files import temp_folder diff --git a/test/integration/configuration/test_auth_remote_plugin.py b/test/integration/configuration/test_auth_remote_plugin.py index 2087d5b6e93..8bc4ce07838 100644 --- a/test/integration/configuration/test_auth_remote_plugin.py +++ b/test/integration/configuration/test_auth_remote_plugin.py @@ -6,9 +6,10 @@ class TestAuthRemotePlugin: - """ Test when the plugin fails, we want a clear message and a helpful trace - """ + def test_error_auth_remote_plugin(self): + """ Test when the plugin fails, we want a clear message and a helpful trace + """ c = TestClient(default_server_user=True) auth_plugin = textwrap.dedent("""\ def auth_remote_plugin(remote, user=None): @@ -20,11 +21,11 @@ def auth_remote_plugin(remote, user=None): assert "Error while processing 'auth_remote.py' plugin" in c.out assert "ERROR: Error while processing 'auth_remote.py' plugin, line " in c.out - """ Test when the plugin give a correct and wrong password, we want a message about the success - or fail in login - """ @pytest.mark.parametrize("password", ["password", "bad-password"]) def test_auth_remote_plugin_direct_credentials(self, password): + """ Test when the plugin give a correct and wrong password, we want a message about + the success or fail in login + """ should_fail = password == "bad-password" c = TestClient(default_server_user=True) auth_plugin = textwrap.dedent(f"""\ @@ -37,7 +38,8 @@ def auth_remote_plugin(remote, user=None): if should_fail: assert "ERROR: Wrong user or password. [Remote: default]" in c.out else: - assert "Changed user of remote 'default' from 'None' (anonymous) to 'admin' (authenticated)" in c.out + assert ("Changed user of remote 'default' from 'None' (anonymous) to " + "'admin' (authenticated)") in c.out def test_auth_remote_plugin_fallback(self): """ Test when the plugin do not give any user or password, we want the code to continue with @@ -53,4 +55,5 @@ def auth_remote_plugin(remote, user=None): c.run("remote login default") # As the auth plugin is not returning any password the code is falling back to the rest of # the input methods in this case the stdin provided by TestClient. - assert "Changed user of remote 'default' from 'None' (anonymous) to 'admin' (authenticated)" in c.out + assert ("Changed user of remote 'default' from 'None' (anonymous) to " + "'admin' (authenticated)") in c.out diff --git a/test/integration/configuration/test_profile_jinja.py b/test/integration/configuration/test_profile_jinja.py index f3e697af7cb..3532852848d 100644 --- a/test/integration/configuration/test_profile_jinja.py +++ b/test/integration/configuration/test_profile_jinja.py @@ -232,6 +232,24 @@ def configure(self): assert "conanfile.py: PROFILE NAME: include_default" in client.out +def test_profile_root_name(): + c = TestClient() + root = textwrap.dedent(""" + [conf] + user.profile:name = {{ profile_name }} + user.profile:root_name = {{ root_profile_name }} + """) + tpl = textwrap.dedent(""" + include(myprofile) + """) + + c.save({"myprofile": root, + "other": tpl}) + c.run("profile show -pr=other") + assert "user.profile:name=myprofile" in c.out + assert "user.profile:root_name=other" in c.out + + class TestProfileDetectAPI: def test_profile_detect_os_arch(self): """ testing OS & ARCH just to test the UX and interface diff --git a/test/integration/cps/test_cps.py b/test/integration/cps/test_cps.py index 407f91a13ba..6dfbcee5338 100644 --- a/test/integration/cps/test_cps.py +++ b/test/integration/cps/test_cps.py @@ -122,38 +122,106 @@ def package_info(self): assert 'set(zlib_LIBS_RELEASE zlib)' in cmake -def test_cps_merge(): - folder = temp_folder() - cps_base = textwrap.dedent(""" +def test_cps_shared_in_pkg(): + c = TestClient() + cps = textwrap.dedent("""\ { - "components" : { - "mypkg" : { + "components" : + { + "mypkg" : + { "includes" : [ "@prefix@/include" ], - "type" : "archive" + "type" : "dylib" } }, - "cps_version" : "0.12.0", - "name" : "mypkg", - "version" : "1.0" + "cps_path" : "@prefix@/cps", + "cps_version" : "0.13.0", + "name" : "mypkg" } """) - cps_conf = textwrap.dedent(""" + cps_release = textwrap.dedent("""\ { - "components" : { - "mypkg" : { - "link_languages" : [ "cpp" ], - "location" : "@prefix@/lib/mypkg.lib" + "components" : + { + "mypkg" : + { + "link_location" : "@prefix@/lib/mypkg.lib", + "location" : "@prefix@/bin/mypkg.dll" } }, "configuration" : "Release", "name" : "mypkg" } """) + cps = "".join(cps.splitlines()) + cps_release = "".join(cps_release.splitlines()) + conanfile = textwrap.dedent(f""" + import os, json + from conan.tools.files import save + from conan import ConanFile + class Pkg(ConanFile): + name = "mypkg" + version = "1.0" + + def package(self): + cps = '{cps}' + cps_path = os.path.join(self.package_folder, "mypkg.cps") + save(self, cps_path, cps) + cps = '{cps_release}' + cps_path = os.path.join(self.package_folder, "mypkg@release.cps") + save(self, cps_path, cps) + + def package_info(self): + from conan.cps import CPS + self.cpp_info = CPS.load("mypkg.cps").to_conan() + """) + c.save({"pkg/conanfile.py": conanfile}) + c.run("create pkg") + + settings = "-s os=Windows -s compiler=msvc -s compiler.version=191 -s arch=x86_64" + + c.run(f"install --requires=mypkg/1.0 {settings} -g CMakeDeps") + cmake = c.load("mypkg-release-x86_64-data.cmake") + assert 'set(mypkg_INCLUDE_DIRS_RELEASE "${mypkg_PACKAGE_FOLDER_RELEASE}/include")' in cmake + assert 'set(mypkg_LIB_DIRS_RELEASE "${mypkg_PACKAGE_FOLDER_RELEASE}/lib")' + assert 'set(mypkg_LIBS_RELEASE mypkg)' in cmake + + +def test_cps_merge(): + folder = temp_folder() + + cps_base = textwrap.dedent("""{ + "components": { + "mypkg":{ + "includes": ["@prefix@/include"], + "type": "archive" + } + }, + "cps_path": "@prefix@/cps", + "cps_version": "0.13.0", + "name": "mypkg" + } + """) + cps_conf = textwrap.dedent("""{ + "components" : { + "mypkg" : { + "link_languages" : [ "cpp" ], + "location" : "@prefix@/lib/mypkg.lib" + } + }, + "configuration" : "Release", + "name" : "mypkg" + } + """) save_files(folder, {"mypkg.cps": cps_base, "mypkg@release.cps": cps_conf}) cps = CPS.load(os.path.join(folder, "mypkg.cps")) json_cps = cps.serialize() - print(json.dumps(json_cps, indent=2)) + mypkg = json_cps["components"]["mypkg"] + assert mypkg["includes"] == ["@prefix@/include"] + assert mypkg["location"] == "@prefix@/lib/mypkg.lib" + assert mypkg["type"] == "archive" + assert mypkg["link_languages"] == ["cpp"] def test_extended_cpp_info(): diff --git a/test/integration/editable/editable_add_test.py b/test/integration/editable/editable_add_test.py index 74621932f52..cb418090c27 100644 --- a/test/integration/editable/editable_add_test.py +++ b/test/integration/editable/editable_add_test.py @@ -8,7 +8,7 @@ def test_install_ok(self): ref = "--name=lib --version=version --user=user --channel=name" t = TestClient() t.save({'conanfile.py': GenConanfile()}) - t.run('editable add . {}'.format(ref)) + t.run('editable add {}'.format(ref)) assert "Reference 'lib/version@user/name' in editable mode" in t.out def test_editable_list_search(self): diff --git a/test/integration/editable/editable_remove_test.py b/test/integration/editable/editable_remove_test.py index c44213b0c49..42e31601b4f 100644 --- a/test/integration/editable/editable_remove_test.py +++ b/test/integration/editable/editable_remove_test.py @@ -31,7 +31,7 @@ def test_unlink_pattern(self, client): assert "lib" not in client.out def test_remove_path(self, client): - client.run("editable remove .") + client.run("editable remove") assert "Removed editable 'lib/version@user/name':" in client.out client.run("editable list") assert "lib" not in client.out diff --git a/test/integration/editable/test_editable_envvars.py b/test/integration/editable/test_editable_envvars.py index ac4223ade6b..8896e63f8bd 100644 --- a/test/integration/editable/test_editable_envvars.py +++ b/test/integration/editable/test_editable_envvars.py @@ -27,10 +27,7 @@ def package_info(self): """) c.save({"dep/conanfile.py": dep, - "pkg/conanfile.py": GenConanfile().with_settings("os") - .with_requires("dep/1.0") - .with_generator("VirtualBuildEnv") - .with_generator("VirtualRunEnv")}) + "pkg/conanfile.py": GenConanfile().with_settings("os").with_requires("dep/1.0")}) c.run("editable add dep --name=dep --version=1.0") c.run("install pkg -s os=Linux -s:b os=Linux") build_path = os.path.join(c.current_folder, "dep", "mybuild", "mylocalbuild") @@ -56,6 +53,103 @@ def package_info(self): assert "mylocalsrc" not in runenv +def test_editable_envvars_package_info(): + """ if the ``layout()`` defines cpp.build.runenv_info/conf_info and the ``package_info()`` + also defines them, the ``package_info()`` values have precedence. This is not symmetrical + with ``cpp_info.includedirs/xxxdirs``. The recommended solution is to use + ``self.layouts.package.runenvinfo/conf_info`` instead. + """ + c = TestClient() + dep = textwrap.dedent(""" + from conan import ConanFile + class Dep(ConanFile): + name = "dep" + version = "1.0" + def layout(self): + self.layouts.build.runenv_info.define("SOME_PATH", "mypath_layout") + self.cpp.build.includedirs = ["mylayoutinclude"] + self.layouts.build.conf_info.define("user:myconf", "mylayoutconf") + + def package_info(self): + print("Running package_info!!") + self.runenv_info.define("SOME_PATH", "mypath_pkginfo") + self.cpp_info.includedirs = ["mypkginfoinclude"] + self.conf_info.define("user:myconf", "mypkginfoconf") + """) + pkg = textwrap.dedent(""" + from conan import ConanFile + class Dep(ConanFile): + name = "pkg" + version = "1.0" + settings = "os", "build_type" + requires = "dep/1.0" + def generate(self): + myconf = self.dependencies["dep"].conf_info.get("user:myconf") + self.output.info(f"DEP CONFINFO {myconf}") + """) + + c.save({"dep/conanfile.py": dep, + "pkg/conanfile.py": pkg}) + c.run("editable add dep ") + c.run("install pkg -s os=Linux -s:b os=Linux -g CMakeDeps") + assert "conanfile.py (pkg/1.0): DEP CONFINFO mylayoutconf" in c.out + cmake = c.load("pkg/dep-release-data.cmake") + assert 'set(dep_INCLUDE_DIRS_RELEASE "${dep_PACKAGE_FOLDER_RELEASE}/mylayoutinclude")' in cmake + runenv = c.load("pkg/conanrunenv-release.sh") + assert f'export SOME_PATH="mypath_layout"' in runenv + + +def test_editable_envvars_package(): + """ This test shows that ``self.layouts.package.runenvinfo/conf_info`` works, ignored + while in editable, but used when regular package + """ + c = TestClient() + dep = textwrap.dedent(""" + from conan import ConanFile + class Dep(ConanFile): + name = "dep" + version = "1.0" + def layout(self): + self.layouts.build.runenv_info.define("SOME_VALUE", "mypath_layout") + self.cpp.build.includedirs = ["mylayoutinclude"] + self.layouts.build.conf_info.define("user:myconf", "mylayoutconf") + + self.layouts.package.runenv_info.define("SOME_VALUE", "mypath_pkginfo") + self.cpp.package.includedirs = ["mypkginfoinclude"] + self.layouts.package.conf_info.define("user:myconf", "mypkginfoconf") + """) + pkg = textwrap.dedent(""" + from conan import ConanFile + class Dep(ConanFile): + name = "pkg" + version = "1.0" + settings = "os", "build_type" + requires = "dep/1.0" + def generate(self): + myconf = self.dependencies["dep"].conf_info.get("user:myconf") + self.output.info(f"DEP CONFINFO {myconf}") + """) + + c.save({"dep/conanfile.py": dep, + "pkg/conanfile.py": pkg}) + c.run("editable add dep ") + c.run("install pkg -s os=Linux -s:b os=Linux -g CMakeDeps") + assert "conanfile.py (pkg/1.0): DEP CONFINFO mylayoutconf" in c.out + cmake = c.load("pkg/dep-release-data.cmake") + assert 'set(dep_INCLUDE_DIRS_RELEASE "${dep_PACKAGE_FOLDER_RELEASE}/mylayoutinclude")' in cmake + runenv = c.load("pkg/conanrunenv-release.sh") + assert f'export SOME_VALUE="mypath_layout"' in runenv + + c.run("editable remove dep") + c.run("create dep") + c.run("install pkg -s os=Linux -s:b os=Linux -g CMakeDeps") + assert "conanfile.py (pkg/1.0): DEP CONFINFO mypkginfoconf" in c.out + cmake = c.load("pkg/dep-release-data.cmake") + assert 'set(dep_INCLUDE_DIRS_RELEASE "${dep_PACKAGE_FOLDER_RELEASE}/mypkginfoinclude")' in cmake + runenv = c.load("pkg/conanrunenv-release.sh") + assert f'export SOME_VALUE="mypath_pkginfo"' in runenv + + def test_editable_conf(): c = TestClient() # TODO: Define if we want conf.xxxx_path(), instead of (..., path=True) methods @@ -67,7 +161,8 @@ def layout(self): self.folders.build = "mybuild" self.layouts.source.conf_info.append_path("user:myconf", "mylocalsrc") self.layouts.build.conf_info.append_path("user:myconf", "mylocalbuild") - self.layouts.build.conf_info.update_path("user:mydictconf", {"a": "mypatha", "b": "mypathb"}) + self.layouts.build.conf_info.update_path("user:mydictconf", {"a": "mypatha", + "b": "mypathb"}) self.layouts.build.conf_info.define_path("user:mydictconf2", {"c": "mypathc"}) """) @@ -80,7 +175,8 @@ def generate(self): self.output.info(f"CONF: {conf}") dictconf = self.dependencies["dep"].conf_info.get("user:mydictconf", check_type=dict) self.output.info(f"CONFDICT: {dictconf}") - dictconf2 = self.dependencies["dep"].conf_info.get("user:mydictconf2", check_type=dict) + dictconf2 = self.dependencies["dep"].conf_info.get("user:mydictconf2", + check_type=dict) self.output.info(f"CONFDICT: {dictconf2}") """) c.save({"dep/conanfile.py": dep, diff --git a/test/integration/environment/test_buildenv_profile.py b/test/integration/environment/test_buildenv_profile.py index 546c688084b..772fd0a349a 100644 --- a/test/integration/environment/test_buildenv_profile.py +++ b/test/integration/environment/test_buildenv_profile.py @@ -132,3 +132,20 @@ def generate(self): assert "WARN: dep ENV:Foo" in client.out assert "WARN: pkg ENV:Foo2" in client.out assert "WARN: None ENV:Var" in client.out + + +def test_buildenv_error_unset(): + # https://github.com/conan-io/conan/issues/19285#issuecomment-3569891282 + c = TestClient() + profile = textwrap.dedent(""" + [buildenv] + CLASSPATH=! + OTHERPATH= + """) + c.save({"conanfile.txt": "", + "profile": profile}) + + c.run("install . -pr=profile -s:a os=Linux") + env = c.load("conanbuildenv.sh") + assert "unset CLASSPATH" in env + assert 'export OTHERPATH=""' in env diff --git a/test/integration/environment/test_env.py b/test/integration/environment/test_env.py index 293371d284d..945352597fa 100644 --- a/test/integration/environment/test_env.py +++ b/test/integration/environment/test_env.py @@ -486,7 +486,8 @@ class Pkg(ConanFile): assert "LD_LIBRARY_PATH" in conanrunenv -def test_multiple_deactivate(): +@pytest.mark.parametrize("deactivation_mode", ["function", None]) +def test_multiple_deactivate(deactivation_mode): conanfile = textwrap.dedent(r""" from conan import ConanFile from conan.tools.env import Environment @@ -513,13 +514,14 @@ def generate(self): "display.bat": display_bat, "display.sh": display_sh}) os.chmod(os.path.join(client.current_folder, "display.sh"), 0o777) - client.run("install .") + client.run(f"install . {f'-c=tools.env:deactivation_mode={deactivation_mode}' if deactivation_mode else ''} ") for _ in range(2): # Just repeat it, so we can check things keep working if platform.system() == "Windows": cmd = "conanbuild.bat && display.bat && deactivate_conanbuild.bat && display.bat" else: - cmd = '. ./conanbuild.sh && ./display.sh && . ./deactivate_conanbuild.sh && ./display.sh' + deactivate_cmd = "deactivate_conanbuild" if deactivation_mode else ". ./deactivate_conanbuild.sh" + cmd = f'. ./conanbuild.sh && ./display.sh && {deactivate_cmd} && ./display.sh' out, _ = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, cwd=client.current_folder).communicate() out = out.decode() @@ -530,7 +532,8 @@ def generate(self): assert "VAR2=!!" in out -def test_multiple_deactivate_order(): +@pytest.mark.parametrize("deactivation_mode", ["function", None]) +def test_multiple_deactivate_order(deactivation_mode): """ https://github.com/conan-io/conan/issues/13693 """ @@ -558,13 +561,14 @@ def generate(self): "display.bat": display_bat, "display.sh": display_sh}) os.chmod(os.path.join(client.current_folder, "display.sh"), 0o777) - client.run("install .") + client.run(f"install . {f'-c=tools.env:deactivation_mode={deactivation_mode}' if deactivation_mode else ''} ") for _ in range(2): # Just repeat it, so we can check things keep working if platform.system() == "Windows": cmd = "conanbuild.bat && display.bat && deactivate_conanbuild.bat && display.bat" else: - cmd = '. ./conanbuild.sh && ./display.sh && . ./deactivate_conanbuild.sh && ./display.sh' + deactivate_cmd = "deactivate_conanbuild" if deactivation_mode else ". ./deactivate_conanbuild.sh" + cmd = f'. ./conanbuild.sh && ./display.sh && {deactivate_cmd} && ./display.sh' out, _ = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, cwd=client.current_folder).communicate() out = out.decode() @@ -573,6 +577,70 @@ def generate(self): assert "MYVAR=!!" in out +@pytest.mark.skipif(platform.system() == "Windows", reason="Shell script test") +@pytest.mark.parametrize("deactivation_mode", ["function", None]) +def test_deactivate_missing_vars_stay_missing(deactivation_mode): + """ Tests that these two cases preserve variable status + 1. + export FOO= + ./conanrunenv.sh that changes FOO to something with a value + ./deactivate_conanrunenv.sh + FOO is still empty + + 2. + BAR is not defined + ./conanrunenv.sh that changes BAR to something with a value + ./deactivate_conanrunenv.sh + BAR is still not defined + + 3. + BAZ is defined to some value + ./conanrunenv.sh that unsets BAZ + ./deactivate_conanrunenv.sh + BAZ is still defined to some value + + 4. + FOOBAR is empty + ./conanrunenv.sh that unsets FOOBAR + ./deactivate_conanrunenv.sh + FOOBAR is still empty + """ + conanfile = textwrap.dedent(r""" + from conan import ConanFile + from conan.tools.env import Environment + class Pkg(ConanFile): + def generate(self): + e1 = Environment() + e1.define("FOO", "Value1") + e1.define("BAR", "Value2") + e1.unset("BAZ") + e1.unset("FOOBAR") + e1.vars(self).save_script("mybuild1") + """) + display_sh = textwrap.dedent("""\ + echo FOO=$FOO!! + if [ -n "${BAR+x}" ]; then echo "BAR EXISTS!!"; fi; + echo BAZ=$BAZ!! + echo FOOBAR=$FOOBAR!! + """) + client = TestClient(light=True) + client.save({"conanfile.py": conanfile, + "display.sh": display_sh}) + os.chmod(os.path.join(client.current_folder, "display.sh"), 0o777) + client.run(f"install . {f'-c=tools.env:deactivation_mode={deactivation_mode}' if deactivation_mode else ''} ") + + deactivate_cmd = "deactivate_conanbuild" if deactivation_mode else ". ./deactivate_conanbuild.sh" + cmd = (f'export FOO=&& export BAZ=Value3 && export FOOBAR=' + f'&& . ./conanbuild.sh && {deactivate_cmd} && ./display.sh') + out, _ = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + shell=True, cwd=client.current_folder).communicate() + out = out.decode() + assert "FOO=!!" in out + assert "BAR EXISTS!!" not in out + assert "BAZ=Value3!!" in out + assert "FOOBAR=!!" in out + + @pytest.mark.skipif(platform.system() != "Windows", reason="Path problem in Windows only") @pytest.mark.parametrize("num_deps", [3, ]) def test_massive_paths(num_deps): @@ -645,7 +713,8 @@ class Pkg(ConanFile): assert "MYTOOL {}!!".format(i) in client.out -def test_profile_build_env_spaces(): +@pytest.mark.parametrize("deactivation_mode", ["function", None]) +def test_profile_build_env_spaces(deactivation_mode): display_bat = textwrap.dedent("""\ @echo off echo VAR1=%VAR1%!! @@ -659,12 +728,13 @@ def test_profile_build_env_spaces(): "display.bat": display_bat, "display.sh": display_sh}) os.chmod(os.path.join(client.current_folder, "display.sh"), 0o777) - client.run("install . -g VirtualBuildEnv -pr=profile") + client.run(f"install . -g VirtualBuildEnv -pr=profile {f'-c=tools.env:deactivation_mode={deactivation_mode}' if deactivation_mode else ''} ") if platform.system() == "Windows": cmd = "conanbuild.bat && display.bat && deactivate_conanbuild.bat && display.bat" else: - cmd = '. ./conanbuild.sh && ./display.sh && . ./deactivate_conanbuild.sh && ./display.sh' + deactivate_cmd = "deactivate_conanbuild" if deactivation_mode else ". ./deactivate_conanbuild.sh" + cmd = f'. ./conanbuild.sh && ./display.sh && {deactivate_cmd} && ./display.sh' out, _ = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, cwd=client.current_folder).communicate() out = out.decode() @@ -852,3 +922,56 @@ def test_deactivate_relocatable_substitute(): conanbuild = c.load("conanbuildenv.sh") result = os.path.join("$script_folder", "deactivate_conanbuildenv.sh") assert f'"{result}"' in conanbuild + + +class TestDotEnv: + def test_dotenv(self): + c = TestClient() + other_path = os.path.join(c.current_folder, "my", "rel", "path") + myprofile = textwrap.dedent(f""" + [buildenv] + MYVAR1=MyVal1 + MYVAR3+=MyVal3 + MYPATH+=(path)/some/path/here + MYOTHER_PATH=+(path){other_path} + + [runenv] + MYRUNVAR=SomeVal1 + + [conf] + tools.env:dotenv=True + """) + c.save({"conanfile.txt": "", + "myprofile": myprofile}) + c.run("install . -pr=myprofile -s build_type=Release") + dotenv = c.load("conanbuildenv-Release.env") + expected = os.path.join(c.current_folder, "my", "rel", "path") + assert f'MYOTHER_PATH="{expected}"' in dotenv + assert f'MYPATH="/some/path/here"' in dotenv + assert 'MYVAR3="MyVal3"' in dotenv + assert 'MYVAR1="MyVal1"' in dotenv + dotenv = c.load("conanrunenv-Release.env") + assert f'MYRUNVAR="SomeVal1"' in dotenv + + def test_generate_only_dotenv(self): + # If for some reason a recipe only wants the dot env files, doable + c = TestClient() + myprofile = textwrap.dedent(f""" + [buildenv] + MYVAR1=MyVal1 + [runenv] + MYRUNVAR=SomeVal1 + """) + conanfile = textwrap.dedent(f""" + from conan import ConanFile + from conan.tools.env import VirtualBuildEnv, VirtualRunEnv + class Lib(ConanFile): + def generate(self): + VirtualBuildEnv(self).vars().save_dotenv("mybuild.env") + VirtualRunEnv(self) + """) + c.save({"conanfile.py": conanfile, + "myprofile": myprofile}) + c.run("install . -pr=myprofile") + assert set(os.listdir(c.current_folder)) == {'conanfile.py', 'mybuild.env', 'myprofile'} + assert 'MYVAR1="MyVal1"' in c.load("mybuild.env") diff --git a/test/integration/extensions/hooks/hook_test.py b/test/integration/extensions/hooks/hook_test.py index 9cdd7a88837..f363ef6f109 100644 --- a/test/integration/extensions/hooks/hook_test.py +++ b/test/integration/extensions/hooks/hook_test.py @@ -71,6 +71,9 @@ def pre_package_info(conanfile): def post_package_info(conanfile): conanfile.output.info("Hello") + +def post_package_id(conanfile): + conanfile.output.info("Hello") """ @@ -88,10 +91,14 @@ def test_complete_hook(self): assert f"conanfile.py (pkg/0.1): {hook_msg} post_source(): Hello" in c.out c.run("install .") + assert f"conanfile.py (pkg/0.1): {hook_msg} pre_validate(): Hello" in c.out + assert f"conanfile.py (pkg/0.1): {hook_msg} post_validate(): Hello" in c.out assert f"conanfile.py (pkg/0.1): {hook_msg} pre_generate(): Hello" in c.out assert f"conanfile.py (pkg/0.1): {hook_msg} post_generate(): Hello" in c.out c.run("build .") + assert f"conanfile.py (pkg/0.1): {hook_msg} pre_validate(): Hello" in c.out + assert f"conanfile.py (pkg/0.1): {hook_msg} post_validate(): Hello" in c.out assert f"conanfile.py (pkg/0.1): {hook_msg} pre_build(): Hello" in c.out assert f"conanfile.py (pkg/0.1): {hook_msg} post_build(): Hello" in c.out assert f"conanfile.py (pkg/0.1): {hook_msg} pre_generate(): Hello" in c.out @@ -106,8 +113,11 @@ def test_complete_hook(self): assert f"pkg/0.1: {hook_msg} post_export(): Hello" in c.out assert f"conanfile.py (pkg/0.1): {hook_msg} pre_package(): Hello" in c.out assert f"conanfile.py (pkg/0.1): {hook_msg} post_package(): Hello" in c.out + assert f"conanfile.py (pkg/0.1): {hook_msg} post_package_id(): Hello" in c.out c.run("create . ") + assert f"pkg/0.1: {hook_msg} pre_validate(): Hello" in c.out + assert f"pkg/0.1: {hook_msg} post_validate(): Hello" in c.out assert f"pkg/0.1: {hook_msg} pre_export(): Hello" in c.out assert f"pkg/0.1: {hook_msg} post_export(): Hello" in c.out assert f"pkg/0.1: {hook_msg} pre_source(): Hello" in c.out @@ -118,6 +128,7 @@ def test_complete_hook(self): assert f"pkg/0.1: {hook_msg} post_package(): Hello" in c.out assert f"pkg/0.1: {hook_msg} pre_package_info(): Hello" in c.out assert f"pkg/0.1: {hook_msg} post_package_info(): Hello" in c.out + assert f"pkg/0.1: {hook_msg} post_package_id(): Hello" in c.out def test_import_hook(self): """ Test that a hook can import another random python file diff --git a/test/integration/graph/core/graph_manager_base.py b/test/integration/graph/core/graph_manager_base.py index 0ea633cfbf5..b1e54480cb2 100644 --- a/test/integration/graph/core/graph_manager_base.py +++ b/test/integration/graph/core/graph_manager_base.py @@ -125,7 +125,8 @@ def build_consumer(self, path, profile_build_requires=None, install=True, option return deps_graph - def _check_node(self, node, ref, deps=None, dependents=None, settings=None, options=None): + @staticmethod + def _check_node(node, ref, deps=None, dependents=None, settings=None, options=None): dependents = dependents or [] deps = deps or [] diff --git a/test/integration/graph/core/graph_manager_test.py b/test/integration/graph/core/graph_manager_test.py index 4e2da8cabf6..ccca1384154 100644 --- a/test/integration/graph/core/graph_manager_test.py +++ b/test/integration/graph/core/graph_manager_test.py @@ -1,5 +1,4 @@ import pytest -from parameterized import parameterized from conan.internal.graph.graph_error import GraphMissingError, GraphLoopError, GraphConflictError from conan.errors import ConanException @@ -19,7 +18,7 @@ def _check_transitive(node, transitive_deps): assert v1.require.headers is v2[1], f"{v1.node}!={v2[0]} headers" assert v1.require.libs is v2[2], f"{v1.node}!={v2[0]} libs" assert v1.require.build is v2[3], f"{v1.node}!={v2[0]} build" - assert v1.require.run is v2[4], f"{v1.node}!={v2[0]} run" + assert v1.require.run is v2[4], f"{v1.node}!={v2[0]} run" assert len(v2) <= 5 @@ -49,7 +48,7 @@ def test_dependency_missing(self): deps_graph = self.build_consumer(consumer, install=False) # TODO: Better error handling - assert type(deps_graph.error) == GraphMissingError + assert type(deps_graph.error) is GraphMissingError assert 1 == len(deps_graph.nodes) app = deps_graph.root @@ -342,8 +341,8 @@ def test_build_script_requirement(self): _check_transitive(app, [(libb, True, True, False, False)]) _check_transitive(libb, [(liba, False, False, True, True)]) - @parameterized.expand([("application",), ("shared-library",), ("static-library",), - ("header-library",), ("build-scripts",), (None,)]) + @pytest.mark.parametrize("package_type", ["application", "shared-library", "static-library", + "header-library", "build-scripts", None]) def test_generic_build_require_adjust_run_with_package_type(self, package_type): # app --br-> cmake (app) self.recipe_conanfile("cmake/0.1", GenConanfile().with_package_type(package_type)) @@ -588,8 +587,7 @@ def test_negate_headers(self): (libb, True, True, False, False), (liba, False, True, False, False)]) - @parameterized.expand([("static-library", ), - ("shared-library", )]) + @pytest.mark.parametrize("library_type", ["static-library", "shared-library"]) def test_libraries_transitive_headers(self, library_type): # app -> libc/0.1 -> libb0.1 -> liba0.1 # All with transitive_headers, the final application shoud get all headers @@ -798,8 +796,7 @@ def test_force_run(self): (libb, False, True, False, False), (liba, False, True, False, True)]) - @parameterized.expand([(True,), - (False,)]) + @pytest.mark.parametrize("run", [True, False]) def test_header_only_run(self, run): # app -> libc/0.1 -> libb0.1 -> liba0.1 # many header-onlys @@ -1353,7 +1350,7 @@ def test_diamond(self): (libc, True, True, False, False), (liba, True, True, False, False)]) - @parameterized.expand([(True, ), (False, )]) + @pytest.mark.parametrize("order", [True, False]) def test_diamond_additive(self, order): # app -> libb0.1 ---------> liba0.1 # \-> libc0.1 (run=True)->/ @@ -1551,7 +1548,7 @@ def test_diamond_conflict(self): consumer = self.recipe_consumer("app/0.1", ["libb/0.1", "libc/0.1"]) deps_graph = self.build_consumer(consumer, install=False) - assert type(deps_graph.error) == GraphConflictError + assert type(deps_graph.error) is GraphConflictError assert 4 == len(deps_graph.nodes) app = deps_graph.root @@ -1575,7 +1572,7 @@ def test_shared_conflict_shared(self): deps_graph = self.build_consumer(consumer, install=False) - assert type(deps_graph.error) == GraphConflictError + assert type(deps_graph.error) is GraphConflictError assert 4 == len(deps_graph.nodes) app = deps_graph.root @@ -1604,7 +1601,7 @@ def test_private_conflict(self): deps_graph = self.build_consumer(consumer, install=False) - assert type(deps_graph.error) == GraphConflictError + assert type(deps_graph.error) is GraphConflictError assert 5 == len(deps_graph.nodes) app = deps_graph.root @@ -1851,7 +1848,7 @@ def test_loop(self): deps_graph = self.build_consumer(consumer, install=False) # TODO: Better error modeling - assert type(deps_graph.error) == GraphLoopError + assert type(deps_graph.error) is GraphLoopError assert 4 == len(deps_graph.nodes) @@ -1866,7 +1863,7 @@ def test_loop(self): self._check_node(liba, "liba/0.1#123", deps=[], dependents=[libb]) -class TransitiveOverridesGraphTest(GraphManagerTest): +class TestTransitiveOverridesGraph(GraphManagerTest): def test_diamond(self): # app -> libb0.1 -> liba0.2 (overriden to lib0.2) @@ -1901,7 +1898,7 @@ def test_diamond_conflict(self): deps_graph = self.build_consumer(consumer, install=False) assert deps_graph.error is not False - assert type(deps_graph.error) == GraphConflictError + assert type(deps_graph.error) is GraphConflictError assert 2 == len(deps_graph.nodes) app = deps_graph.root @@ -1972,7 +1969,7 @@ def test_diamond_reverse_order_conflict(self): consumer = self.recipe_consumer("app/0.1", ["dep1/2.0", "dep2/1.0"]) deps_graph = self.build_consumer(consumer, install=False) - assert type(deps_graph.error) == GraphConflictError + assert type(deps_graph.error) is GraphConflictError assert 3 == len(deps_graph.nodes) app = deps_graph.root @@ -2007,7 +2004,7 @@ def test_invisible_not_forced(self): self._check_node(liba2, "liba/0.2#123", dependents=[app]) -class PureOverrideTest(GraphManagerTest): +class TestPureOverride(GraphManagerTest): def test_diamond(self): # app -> libb0.1 -> liba0.2 (overriden to lib0.2) @@ -2143,7 +2140,7 @@ def test_override_not_used(self): (liba, True, True, False, False)]) -class PackageIDDeductions(GraphManagerTest): +class TestPackageIDDeductions(GraphManagerTest): def test_static_dep_to_shared(self): # project -> app1 -> lib @@ -2235,7 +2232,7 @@ def test_project_require_transitive_conflict(self): build=False, run=True), install=False) - assert type(deps_graph.error) == GraphConflictError + assert type(deps_graph.error) is GraphConflictError def test_project_require_apps_transitive(self): # project -> app1 (app type) -> lib @@ -2284,7 +2281,7 @@ def test_project_require_apps_transitive_conflict(self): "app2/0.1"), install=False) - assert type(deps_graph.error) == GraphConflictError + assert type(deps_graph.error) is GraphConflictError def test_project_require_private(self): # project -(!visible)-> app1 -> lib1 diff --git a/test/integration/graph/core/test_alias.py b/test/integration/graph/core/test_alias.py index 629ac1d5945..bf7eb87b0cf 100644 --- a/test/integration/graph/core/test_alias.py +++ b/test/integration/graph/core/test_alias.py @@ -156,7 +156,7 @@ def test_alias_tansitive(self): _check_transitive(app, [(liba, True, True, False, False)]) -class AliasBuildRequiresTest(GraphManagerTest): +class TestAliasBuildRequires(GraphManagerTest): def test_non_conflicting_alias(self): # https://github.com/conan-io/conan/issues/5468 diff --git a/test/integration/graph/core/test_auto_package_type.py b/test/integration/graph/core/test_auto_package_type.py index 9e12c11ccfa..3db133079c8 100644 --- a/test/integration/graph/core/test_auto_package_type.py +++ b/test/integration/graph/core/test_auto_package_type.py @@ -47,6 +47,7 @@ def test_auto_package_type(conanfile): c.run("graph info . --filter package_type -o header_only=True -o shared=False") assert "package_type: header-library" in c.out + def test_package_type_and_header_library(): """ Show that forcing a package_type and header_only=True does not change the package_type""" tc = TestClient(light=True) diff --git a/test/integration/graph/core/test_build_requires.py b/test/integration/graph/core/test_build_requires.py index fb6e0f8c6e0..e3569700c03 100644 --- a/test/integration/graph/core/test_build_requires.py +++ b/test/integration/graph/core/test_build_requires.py @@ -2,8 +2,6 @@ import pytest -from parameterized import parameterized - from conan.internal.graph.graph_error import GraphConflictError, GraphLoopError from conan.api.model import RecipeReference from test.integration.graph.core.graph_manager_base import GraphManagerTest @@ -26,9 +24,9 @@ def _check_transitive(node, transitive_deps): if v1.require.test is not v2[5]: raise Exception(f"{v1.node}!={v2[0]} test") -class BuildRequiresGraphTest(GraphManagerTest): +class TestBuildRequiresGraph(GraphManagerTest): - @parameterized.expand([("recipe", ), ("profile", )]) + @pytest.mark.parametrize("build_require", ["recipe", "profile"]) def test_basic(self, build_require): # app -(br)-> cmake self._cache_recipe("cmake/0.1", GenConanfile()) @@ -69,7 +67,7 @@ def test_lib_build_require(self): _check_transitive(app, [(lib, True, True, False, False)]) _check_transitive(lib, [(cmake, False, False, True, True)]) - @parameterized.expand([("shared", ), ("static", ), ("notrun", ), ("run", )]) + @pytest.mark.parametrize("cmakelib_type", ["shared", "static", "notrun", "run"]) def test_build_require_transitive(self, cmakelib_type): # app -> lib -(br)-> cmake -> cmakelib (cmakelib_type) @@ -167,8 +165,7 @@ def test_build_require_transitive_static(self): self._cache_recipe("zlib/0.2", GenConanfile().with_shared_option(False)) self._cache_recipe("cmake/0.1", GenConanfile().with_require("zlib/0.1")) self._cache_recipe("mingw/0.1", GenConanfile().with_require("zlib/0.2")) - self._cache_recipe("lib/0.1", GenConanfile().with_tool_requires("cmake/0.1", - "mingw/0.1")) + self._cache_recipe("lib/0.1", GenConanfile().with_tool_requires("cmake/0.1", "mingw/0.1")) deps_graph = self.build_graph(GenConanfile("app", "0.1").with_require("lib/0.1")) assert 6 == len(deps_graph.nodes) @@ -202,7 +199,7 @@ def test_build_require_transitive_shared(self): deps_graph = self.build_graph(GenConanfile("app", "0.1").with_require("lib/0.1"), install=False) - assert type(deps_graph.error) == GraphConflictError + assert type(deps_graph.error) is GraphConflictError out = str(deps_graph.error) assert "Version conflict: Conflict between zlib/0.2 and zlib/0.1 in the graph." in out assert "Conflict originates from lib/0.1" in out @@ -332,7 +329,7 @@ def test_lib_build_require_transitive(self): _check_transitive(app, [(lib, True, True, False, False)]) _check_transitive(lib, [(gtest, True, True, False, False)]) - @parameterized.expand([("shared",), ("static",), ("notrun",), ("run",)]) + @pytest.mark.parametrize("gtestlib_type", ["shared", "static", "notrun", "run"]) def test_test_require_transitive(self, gtestlib_type): # app -> lib -(tr)-> gtest -> gtestlib (gtestlib_type) @@ -449,7 +446,7 @@ def _check_graph(self, deps_graph, reverse): _check_transitive(lib_c, [(lib_a, True, True, False, True), (util, True, True, False, True)]) - @parameterized.expand([(True,), (False,)]) + @pytest.mark.parametrize("reverse", [True, False]) def test_fixed_versions(self, reverse): # lib_c -(tr)-> lib_a -0.1--> util # \--------(tr)----0.1------/ @@ -461,7 +458,7 @@ def test_fixed_versions(self, reverse): deps_graph = self.build_graph(GenConanfile("lib_c", "0.1").with_test_requires(*deps)) self._check_graph(deps_graph, reverse) - @parameterized.expand([(True,), (False,)]) + @pytest.mark.parametrize("reverse", [True, False]) def test_fixed_versions_conflict(self, reverse): # lib_c -(tr)-> lib_a -0.1--> util # \--------(tr)----0.2------/ @@ -473,9 +470,9 @@ def test_fixed_versions_conflict(self, reverse): deps = ("lib_a/0.1", "util/0.2") if not reverse else ("util/0.2", "lib_a/0.1") conanfile = GenConanfile("lib_c", "0.1").with_test_requires(*deps) deps_graph = self.build_graph(conanfile, install=False) - assert type(deps_graph.error) == GraphConflictError + assert type(deps_graph.error) is GraphConflictError - @parameterized.expand([(True,), (False,)]) + @pytest.mark.parametrize("reverse", [True, False]) def test_fixed_versions_hybrid(self, reverse): # lib_c -----> lib_a--0.1--> util # \--------(tr)----0.1------/ @@ -492,7 +489,7 @@ def test_fixed_versions_hybrid(self, reverse): deps_graph = self.build_graph(conanfile) self._check_graph(deps_graph, reverse=reverse) - @parameterized.expand([(True,), (False,)]) + @pytest.mark.parametrize("reverse", [True, False]) def test_fixed_versions_hybrid_conflict(self, reverse): # lib_c -----> lib_a--0.1---> util # \--------(tr)----0.2------/ @@ -507,9 +504,9 @@ def test_fixed_versions_hybrid_conflict(self, reverse): else: conanfile = conanfile.with_test_requires("lib_a/0.1").with_requires("util/0.2") deps_graph = self.build_graph(conanfile, install=False) - assert type(deps_graph.error) == GraphConflictError + assert type(deps_graph.error) is GraphConflictError - @parameterized.expand([(True,), (False,)]) + @pytest.mark.parametrize("reverse", [True, False]) def test_version_ranges(self, reverse): # lib_c -(tr)-> lib_a -> util # \--------(tr)-------/ @@ -521,7 +518,7 @@ def test_version_ranges(self, reverse): deps_graph = self.build_graph(GenConanfile("lib_c", "0.1").with_test_requires(*deps)) self._check_graph(deps_graph, reverse) - @parameterized.expand([(True,), (False,)]) + @pytest.mark.parametrize("reverse", [True, False]) def test_version_ranges_conflict(self, reverse): # lib_c -(tr)-> lib_a -> util/0.1 # \--------(tr)------> util/1.0 @@ -532,9 +529,9 @@ def test_version_ranges_conflict(self, reverse): deps = ("lib_a/[>=0]", "util/[>=1]") if not reverse else ("util/[>=1]", "lib_a/[>=0]") deps_graph = self.build_graph(GenConanfile("lib_c", "0.1").with_test_requires(*deps), install=False) - assert type(deps_graph.error) == GraphConflictError + assert type(deps_graph.error) is GraphConflictError - @parameterized.expand([(True,), (False,)]) + @pytest.mark.parametrize("reverse", [True, False]) def test_version_ranges_hybrid(self, reverse): # lib_c ---> lib_a -> util # \--------(tr)-------/ @@ -550,7 +547,7 @@ def test_version_ranges_hybrid(self, reverse): deps_graph = self.build_graph(conanfile) self._check_graph(deps_graph, reverse) - @parameterized.expand([(True,), (False,)]) + @pytest.mark.parametrize("reverse", [True, False]) def test_version_ranges_hybrid_conflict(self, reverse): # lib_c -(tr)-> lib_a -> util/0.1 # \--------(tr)------> util/1.0 @@ -564,10 +561,10 @@ def test_version_ranges_hybrid_conflict(self, reverse): else: conanfile = conanfile.with_test_requires("lib_a/[>=0.1]").with_requires("util/[>=1]") deps_graph = self.build_graph(conanfile, install=False) - assert type(deps_graph.error) == GraphConflictError + assert type(deps_graph.error) is GraphConflictError -class BuildRequiresPackageIDTest(GraphManagerTest): +class TestBuildRequiresPackageID(GraphManagerTest): def test_default_no_affect(self,): # app -> lib -(br)-> cmake @@ -618,7 +615,7 @@ def test_minor_mode(self,): assert lib.package_id != NO_SETTINGS_PACKAGE_ID -class PublicBuildRequiresTest(GraphManagerTest): +class TestPublicBuildRequires(GraphManagerTest): def test_simple(self): # app -> lib -(br public)-> cmake @@ -658,7 +655,7 @@ def test_deep_dependency_tree(self): deps_graph = self.build_graph(GenConanfile("app", "0.1").with_requirement("liba/0.1", run=True)) # Build requires always apply to the consumer - self.assertEqual(8 + 4, len(deps_graph.nodes)) + assert 8 + 4 == len(deps_graph.nodes) app = deps_graph.root liba = app.edges[0].dst libb = liba.edges[0].dst @@ -739,7 +736,7 @@ def test_conflict_diamond(self): "libc/0.1"), install=False) - assert type(deps_graph.error) == GraphConflictError + assert type(deps_graph.error) is GraphConflictError # Build requires always apply to the consumer assert 4 == len(deps_graph.nodes) @@ -768,7 +765,7 @@ def test_conflict_diamond_two_levels(self): "libe/0.1"), install=False) - assert type(deps_graph.error) == GraphConflictError + assert type(deps_graph.error) is GraphConflictError # Build requires always apply to the consumer assert 6 == len(deps_graph.nodes) @@ -882,7 +879,7 @@ def test_direct_loop_error(self): deps_graph = self.build_graph(GenConanfile("app", "0.1").with_build_requires("cmake/0.1"), install=False) - assert type(deps_graph.error) == GraphLoopError + assert type(deps_graph.error) is GraphLoopError # Build requires always apply to the consumer assert 4 == len(deps_graph.nodes) @@ -904,7 +901,7 @@ def test_indirect_loop_error(self): deps_graph = self.build_graph(GenConanfile().with_build_requires("cmake/0.1"), install=False) - assert type(deps_graph.error) == GraphLoopError + assert type(deps_graph.error) is GraphLoopError # Build requires always apply to the consumer assert 6 == len(deps_graph.nodes) diff --git a/test/integration/graph/core/test_provides.py b/test/integration/graph/core/test_provides.py index ec3c2d095b0..a955f1d62b4 100644 --- a/test/integration/graph/core/test_provides.py +++ b/test/integration/graph/core/test_provides.py @@ -1,6 +1,6 @@ import textwrap -from parameterized import parameterized +import pytest from conan.internal.graph.graph_error import GraphProvidesError from test.integration.graph.core.graph_manager_base import GraphManagerTest @@ -17,7 +17,7 @@ def test_direct_conflict(self): with_requires("libb/0.1")) deps_graph = self.build_consumer(consumer, install=False) - assert type(deps_graph.error) == GraphProvidesError + assert type(deps_graph.error) is GraphProvidesError assert 2 == len(deps_graph.nodes) app = deps_graph.root @@ -34,7 +34,7 @@ def test_transitive_conflict(self): with_requires("libb/0.1")) deps_graph = self.build_consumer(consumer, install=False) - assert type(deps_graph.error) == GraphProvidesError + assert type(deps_graph.error) is GraphProvidesError assert 3 == len(deps_graph.nodes) app = deps_graph.root @@ -45,7 +45,7 @@ def test_transitive_conflict(self): self._check_node(libb, "libb/0.1#123", deps=[libc], dependents=[app]) self._check_node(libc, "libc/0.1#123", deps=[], dependents=[libb]) - @parameterized.expand([(True,), (False,)]) + @pytest.mark.parametrize("private", [True, False]) def test_branches_conflict(self, private): # app -> libb/0.1 (provides feature) # \ -> libc/0.1 (provides feature) @@ -60,7 +60,7 @@ def test_branches_conflict(self, private): with_requires("libb/0.1", "libc/0.1")) deps_graph = self.build_consumer(consumer, install=False) - assert type(deps_graph.error) == GraphProvidesError + assert type(deps_graph.error) is GraphProvidesError assert 3 == len(deps_graph.nodes) app = deps_graph.root @@ -105,7 +105,7 @@ def test_diamond_conflict(self): consumer = self.recipe_consumer("app/0.1", ["libb/0.1", "libc/0.1"]) deps_graph = self.build_consumer(consumer, install=False) - assert type(deps_graph.error) == GraphProvidesError + assert type(deps_graph.error) is GraphProvidesError assert 5 == len(deps_graph.nodes) app = deps_graph.root @@ -131,7 +131,7 @@ def test_loop(self): consumer = self.recipe_consumer("app/0.1", ["libc/0.1"]) deps_graph = self.build_consumer(consumer, install=False) - assert type(deps_graph.error) == GraphProvidesError + assert type(deps_graph.error) is GraphProvidesError assert 4 == len(deps_graph.nodes) @@ -146,7 +146,7 @@ def test_loop(self): self._check_node(liba, "liba/0.1#123", dependents=[libb]) -class ProvidesBuildRequireTest(GraphManagerTest): +class TestProvidesBuildRequire(GraphManagerTest): def test_build_require_no_conflict(self): # app (provides libjpeg) -(build)-> br/v1 -> br_lib/v1(provides libjpeg) self.recipe_conanfile("br_lib/0.1", GenConanfile().with_provides("libjpeg")) @@ -200,7 +200,7 @@ def test_transitive_test_require_conflict(self): deps_graph = self.build_consumer(path, install=False) - assert type(deps_graph.error) == GraphProvidesError + assert type(deps_graph.error) is GraphProvidesError assert 3 == len(deps_graph.nodes) @@ -221,7 +221,7 @@ def test_two_br_conflict(self): .with_tool_requires("br1/0.1", "br2/0.1")) deps_graph = self.build_consumer(path, install=False) - assert type(deps_graph.error) == GraphProvidesError + assert type(deps_graph.error) is GraphProvidesError assert 3 == len(deps_graph.nodes) @@ -292,4 +292,5 @@ def test_name_provide_error_message(): tc.run("graph info --requires=mozjpeg/0.1 --requires=libjpeg/0.1", assert_error=True) # This used to report that None was provided, but now it reports the name of the provides - assert "ERROR: Provide Conflict: Both 'libjpeg/0.1' and 'mozjpeg/0.1' provide '['libjpeg']'" in tc.out + assert ("ERROR: Provide Conflict: Both 'libjpeg/0.1' and " + "'mozjpeg/0.1' provide '['libjpeg']'") in tc.out diff --git a/test/integration/graph/core/test_version_ranges.py b/test/integration/graph/core/test_version_ranges.py index b5cdafb2f03..4f9a19d65d6 100644 --- a/test/integration/graph/core/test_version_ranges.py +++ b/test/integration/graph/core/test_version_ranges.py @@ -53,7 +53,7 @@ def test_missing(self): deps_graph = self.build_consumer(consumer, install=False) - assert type(deps_graph.error) == GraphMissingError + assert type(deps_graph.error) is GraphMissingError assert 1 == len(deps_graph.nodes) app = deps_graph.root @@ -66,7 +66,7 @@ def test_userchannel_no_match(self): deps_graph = self.build_consumer(consumer, install=False) - assert type(deps_graph.error) == GraphMissingError + assert type(deps_graph.error) is GraphMissingError assert 1 == len(deps_graph.nodes) app = deps_graph.root @@ -80,7 +80,7 @@ def test_required_userchannel_no_match(self): deps_graph = self.build_consumer(consumer, install=False) - assert type(deps_graph.error) == GraphMissingError + assert type(deps_graph.error) is GraphMissingError assert 1 == len(deps_graph.nodes) app = deps_graph.root @@ -94,7 +94,7 @@ def test_transitive_out_range(self): deps_graph = self.build_consumer(consumer, install=False) - assert type(deps_graph.error) == GraphMissingError + assert type(deps_graph.error) is GraphMissingError assert 1 == len(deps_graph.nodes) app = deps_graph.root @@ -176,7 +176,7 @@ def test_transitive_conflict(self): consumer = self.recipe_consumer("app/0.1", ["libb/0.1", "libc/0.1"]) deps_graph = self.build_consumer(consumer, install=False) - assert type(deps_graph.error) == GraphConflictError + assert type(deps_graph.error) is GraphConflictError assert 4 == len(deps_graph.nodes) app = deps_graph.root @@ -199,7 +199,7 @@ def test_transitive_fixed_conflict(self): consumer = self.recipe_consumer("app/0.1", ["libb/0.1", "libc/0.1"]) deps_graph = self.build_consumer(consumer, install=False) - assert type(deps_graph.error) == GraphConflictError + assert type(deps_graph.error) is GraphConflictError assert 4 == len(deps_graph.nodes) app = deps_graph.root @@ -277,7 +277,7 @@ def test_transitive_fixed_conflict(self): consumer = self.recipe_consumer("app/0.1", ["libb/0.1", "liba/[>1.0]"]) deps_graph = self.build_consumer(consumer, install=False) - assert type(deps_graph.error) == GraphConflictError + assert type(deps_graph.error) is GraphConflictError assert 2 == len(deps_graph.nodes) app = deps_graph.root diff --git a/test/integration/graph/test_dependencies_visit.py b/test/integration/graph/test_dependencies_visit.py index 63d3b4191d2..8829aca47b2 100644 --- a/test/integration/graph/test_dependencies_visit.py +++ b/test/integration/graph/test_dependencies_visit.py @@ -45,15 +45,15 @@ def generate(self): client.save({"conanfile.py": conanfile}) client.run("install .") - refs = client.cache.get_latest_recipe_reference(RecipeReference.loads("openssl/0.1")) + refs = client.cache.get_latest_recipe_revision(RecipeReference.loads("openssl/0.1")) pkgs = client.cache.get_package_references(refs) - prev1 = client.cache.get_latest_package_reference(pkgs[0]) + prev1 = client.cache.get_latest_package_revision(pkgs[0]) assert f"DefRef: {repr(prev1.ref)}!!!" in client.out assert f"DefPRef: {prev1.repr_notime()}!!!" in client.out - refs = client.cache.get_latest_recipe_reference(RecipeReference.loads("openssl/0.2")) + refs = client.cache.get_latest_recipe_revision(RecipeReference.loads("openssl/0.2")) pkgs = client.cache.get_package_references(refs) - prev2 = client.cache.get_latest_package_reference(pkgs[0]) + prev2 = client.cache.get_latest_package_revision(pkgs[0]) assert f"DefRefBuild: {repr(prev2.ref)}!!!" in client.out assert f"DefPRefBuild: {prev2.repr_notime()}!!!" in client.out @@ -255,11 +255,13 @@ class Pkg(ConanFile): class User(ConanFile): requires = "dep/1.0" def generate(self): - self.output.info("HOME: {}".format(self.dependencies["dep"].homepage)) - self.output.info("URL: {}".format(self.dependencies["dep"].url)) - self.output.info("LICENSE: {}".format(self.dependencies["dep"].license)) - self.output.info("RECIPE: {}".format(self.dependencies["dep"].recipe_folder)) - self.output.info("CONANDATA: {}".format(self.dependencies["dep"].conan_data)) + dep = self.dependencies["dep"] + self.output.info("HOME: {}".format(dep.homepage)) + self.output.info("URL: {}".format(dep.url)) + self.output.info("LICENSE: {}".format(dep.license)) + self.output.info("RECIPE FOLDER: {}".format(dep.recipe_folder)) + self.output.info("CONANDATA: {}".format(dep.conan_data)) + self.output.info("RECIPE: {}".format(dep.recipe)) """) c.save({"dep/conanfile.py": conanfile, @@ -270,8 +272,9 @@ def generate(self): assert "conanfile.py: HOME: myhome" in c.out assert "conanfile.py: URL: myurl" in c.out assert "conanfile.py: LICENSE: MIT" in c.out - assert "conanfile.py: RECIPE:" in c.out + assert "conanfile.py: RECIPE FOLDER:" in c.out assert "conanfile.py: CONANDATA: {}" in c.out + assert "conanfile.py: RECIPE: Cache" in c.out def test_dependency_interface_validate(): diff --git a/test/integration/graph/test_divergent_cppstd_build_host.py b/test/integration/graph/test_divergent_cppstd_build_host.py index 637786d50e8..2bdd43fcab1 100644 --- a/test/integration/graph/test_divergent_cppstd_build_host.py +++ b/test/integration/graph/test_divergent_cppstd_build_host.py @@ -8,7 +8,7 @@ def test_divergent_cppstd_build_host(): c = TestClient() - + conanfile = textwrap.dedent(""" [requires] waterfall/1.0 @@ -21,13 +21,12 @@ def test_divergent_cppstd_build_host(): .with_requires("waterfall/1.0"), "conanfile.txt": conanfile}) - c.run("export waterfall") c.run("export rainbow") - c.run(f"install . --build=missing -s compiler.cppstd=14 -s:b compiler.cppstd=17 --format=json", redirect_stdout="graph.json") + c.run(f"install . --build=missing -s compiler.cppstd=14 -s:b compiler.cppstd=17 --format=json", + redirect_stdout="graph.json") graph = json.loads(c.load("graph.json")) - # waterfall is twice in the graph: as a direct host dependency, and an indirect build dependency assert graph['graph']['nodes']['1']['ref'] == "waterfall/1.0#821e924dcef2f185dd651e6d434f9f95" assert graph['graph']['nodes']['1']['context'] == "host" @@ -37,7 +36,7 @@ def test_divergent_cppstd_build_host(): # is this the right behaviour? # without the compatibility plugin, we would require two different package_id in the same graph, - # but because of compatibility plugin, Conan graph uses a "compatible" package_id in the build context - # rather than the exact one + # but because of compatibility plugin, Conan graph uses a "compatible" package_id in the + # build context rather than the exact one assert graph['graph']['nodes']['1']['package_id'] == graph['graph']['nodes']['3']['package_id'] diff --git a/test/integration/graph/test_platform_requires.py b/test/integration/graph/test_platform_requires.py index 2d60277f96e..d7301afb4c4 100644 --- a/test/integration/graph/test_platform_requires.py +++ b/test/integration/graph/test_platform_requires.py @@ -236,7 +236,7 @@ def test_package_id_explicit_revision(self): """ client = TestClient(light=True) client.save_home({"global.conf": - "core.package_id:default_unknown_mode=recipe_revision_mode"}) + "core.package_id:default_unknown_mode=recipe_revision_mode"}) client.save({"conanfile.py": GenConanfile("pkg", "1.0").with_requires("dep/1.0"), "profile": "[platform_requires]\ndep/1.0#r1", "profile2": "[platform_requires]\ndep/1.0#r2"}) diff --git a/test/integration/graph/test_replace_requires.py b/test/integration/graph/test_replace_requires.py index ef6e86bb4ba..0bd78898349 100644 --- a/test/integration/graph/test_replace_requires.py +++ b/test/integration/graph/test_replace_requires.py @@ -213,6 +213,10 @@ def package_info(self): assert f"zlib/0.1: {name}/{version}" in c.out assert f"app/0.1: DEP ZLIB generate: {name}!" in c.out assert f"app/0.1: DEP ZLIB build: {name}!" in c.out + if name == "zlib-ng": + # CMakeDeps can not be used to consume replaced requires for different packages + # only CMakeConfigDeps has this capability + c.run("install --requires=app/0.1 -pr=profile -g CMakeDeps", assert_error=True) def test_replace_requires_consumer_references_error_multiple(): @@ -728,3 +732,29 @@ class App(ConanFile): else: assert "" in props + + +@pytest.mark.parametrize("require, pattern, alternative, expected", [ + # Version range as pattern + # PINNED REQUIRE VERSION + ("dep/1.0", "dep/[>=1.0 <2]", "dep/1.3", "dep/1.3"), + ("dep/1.0", "dep/[>=1.5 <2]", "dep/1.3", False), + # RANGE REQUIRE VERSION + ("dep/[>=1.2 <2]", "dep/[>=1.0 <2]", "dep/1.3", "dep/1.3"), + ("dep/[>=1.0 <1.5]", "dep/[>=1.2 <2]", "dep/1.3", "dep/1.3"), + ("dep/[>=1.0 <1.5]", "dep/[>=1.5 <2]", "dep/1.3", False) + ] + ) +def test_replace_requires_ranges(require, pattern, alternative, expected): + c = TestClient(light=True) + c.save({"dep/conanfile.py": GenConanfile("dep"), + "app/conanfile.py": GenConanfile().with_requires(require), + "profile": f"[replace_requires]\n{pattern}: {alternative}"}) + c.run("create dep --version=1.0") + c.run("create dep --version=1.3") + c.run("graph info app -pr=profile") + if expected: + assert "Replaced requires" in c.out + assert f"{require}: {expected}" in c.out + else: + assert "Replaced requires" not in c.out diff --git a/test/integration/graph/test_require_same_pkg_versions.py b/test/integration/graph/test_require_same_pkg_versions.py index 68d52ab20a4..c97d9a00a54 100644 --- a/test/integration/graph/test_require_same_pkg_versions.py +++ b/test/integration/graph/test_require_same_pkg_versions.py @@ -277,3 +277,145 @@ def build(self): c.run("remove * -c") c.run("install --requires=consumer/1.0") assert re.search(r"Skipped binaries(\s*)myqemu/1.0, myqemu/2.0, snippy/1.0, valgrind/1.0", c.out) + + +class TestTransitiveBuild: + # project --(tool-requires)--> wrapper_a -(requires)-> gcc/X + # \-----(tool-requires)--> wrapper_b -(requires)-> gcc/Y + gcc = textwrap.dedent(r""" + import os + from conan import ConanFile + from conan.tools.files import save + class Pkg(ConanFile): + name = "gcc" + package_type = "application" + def package(self): + echo = f"@echo off\necho MYGCC={self.version}!!" + save(self, os.path.join(self.package_folder, "bin", "mygcc.bat"), echo) + save(self, os.path.join(self.package_folder, "bin", "mygcc.sh"), echo) + os.chmod(os.path.join(self.package_folder, "bin", "mygcc.sh"), 0o777) + """) + + project = textwrap.dedent(""" + import platform + from conan import ConanFile + class Pkg(ConanFile): + def build_requirements(self): + self.tool_requires("wrappera/1.0") + self.tool_requires("wrapperb/1.0") + def build(self): + ext = "bat" if platform.system() == "Windows" else "sh" + """) + + def test_require_different_versions_transitive_noconflict(self): + """ Same dependency to gcc/1.0, no conflict + """ + c = TestClient() + project = r"self.run(f'mygcc.{ext}')" + c.save({"gcc/conanfile.py": self.gcc, + "wrappera/conanfile.py": GenConanfile("wrappera", "1.0").with_requires("gcc/1.0"), + "wrapperb/conanfile.py": GenConanfile("wrapperb", "1.0").with_requires("gcc/1.0"), + "project/conanfile.py": self.project + textwrap.indent(project, " ")}) + + c.run("create gcc --version=1.0") + c.run("create wrappera") + c.run("create wrapperb") + + c.run("build project") + assert "MYGCC=1.0!!" in c.out + + def test_require_different_versions_transitive_conflict(self): + c = TestClient() + project = textwrap.dedent(r""" + self.run(f'wrappera.{ext}') + self.run(f'wrapperb.{ext}') + """) + c.save({"gcc/conanfile.py": self.gcc, + "wrappera/conanfile.py": GenConanfile("wrappera", "1.0").with_requires("gcc/1.0"), + "wrapperb/conanfile.py": GenConanfile("wrapperb", "1.0").with_requires("gcc/2.0"), + "project/conanfile.py": self.project + textwrap.indent(project, " ")}) + + c.run("create gcc --version=1.0") + c.run("create gcc --version=2.0") + c.run("create wrappera") + c.run("create wrapperb") + c.run("install project", assert_error=True) + assert "Version conflict: Conflict between gcc/2.0 and gcc/1.0 in the graph" in c.out + + # disabling ``run=False`` trait + c.save({"wrapa/conanfile.py": GenConanfile("wrappera", "1.0").with_requirement("gcc/1.0", + run=False), + "wrapb/conanfile.py": GenConanfile("wrapperb", "1.0").with_requirement("gcc/2.0", + run=False)}) + c.run("create wrapa") + c.run("create wrapb") + c.run("install project") # Now it works! + + # If the run is not propagated, the consumer must do it explicitly to differentiate: + project = textwrap.dedent(r""" + import os, platform + from conan import ConanFile + from conan.tools.files import save, chdir + class Pkg(ConanFile): + name = "project" + version = "1.0" + def build_requirements(self): + self.tool_requires("wrappera/1.0") + self.tool_requires("wrapperb/1.0") + self.tool_requires("gcc/1.0", run=False) + self.tool_requires("gcc/2.0", run=False) + def build(self): + ext = "bat" if platform.system() == "Windows" else "sh" + path_a = self.dependencies.build["gcc/1.0"].cpp_info.bindir + path_b = self.dependencies.build["gcc/2.0"].cpp_info.bindir + self.run(f'"{path_a}/mygcc.{ext}"') + self.run(f'"{path_b}/mygcc.{ext}"') + """) + c.save({"project/conanfile.py": project}) + c.run("build project") + assert "MYGCC=1.0!!" in c.out + assert "MYGCC=2.0!!" in c.out + + def test_require_different_versions_transitive_vendored(self): + c = TestClient() + wrapper = textwrap.dedent(r""" + import os + from conan import ConanFile + from conan.tools.files import save, copy + class Pkg(ConanFile): + name = "{name}" + version = "1.0" + package_type = "application" + + def requirements(self): + self.requires("gcc/{gcc_version}", visible=False, run=False) + + def package(self): + copy(self, "*", src=self.dependencies["gcc"].cpp_info.bindir, + dst=os.path.join(self.package_folder, "bin")) + echo_bat = f'@echo off\necho {name}={{self.version}}!!\ncall "%~dp0/mygcc.bat"' + # Trick for cwd in Linux + echo_sh = '@echo off\necho {name}=1.0!!\n"${{0%/*}}/mygcc.sh\"' + save(self, os.path.join(self.package_folder, "bin", "{name}.bat"), echo_bat) + save(self, os.path.join(self.package_folder, "bin", "{name}.sh"), echo_sh) + os.chmod(os.path.join(self.package_folder, "bin", "{name}.sh"), 0o777) + """) + + project = textwrap.dedent(r""" + self.run(f'wrappera.{ext}') + self.run(f'wrapperb.{ext}') + """) + c.save({"gcc/conanfile.py": self.gcc, + "wrappera/conanfile.py": wrapper.format(name="wrappera", gcc_version="1.0"), + "wrapperb/conanfile.py": wrapper.format(name="wrapperb", gcc_version="2.0"), + "project/conanfile.py": self.project + textwrap.indent(project, " ")}) + + c.run("create gcc --version=1.0") + c.run("create gcc --version=2.0") + c.run("create wrappera") + c.run("create wrapperb") + c.run("build project") + assert "wrappera=1.0!!" in c.out + assert "MYGCC=1.0!!" in c.out + assert "wrapperb=1.0!!" in c.out + assert "MYGCC=2.0!!" in c.out diff --git a/test/integration/graph/test_system_tools.py b/test/integration/graph/test_system_tools.py index 04e8ff3646c..c66494d3654 100644 --- a/test/integration/graph/test_system_tools.py +++ b/test/integration/graph/test_system_tools.py @@ -6,7 +6,6 @@ from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient -from conan.internal.util.files import save class TestToolRequires: diff --git a/test/integration/graph/test_test_requires.py b/test/integration/graph/test_test_requires.py index d74bec93a6c..71d415df1b9 100644 --- a/test/integration/graph/test_test_requires.py +++ b/test/integration/graph/test_test_requires.py @@ -35,6 +35,9 @@ def test_test_requires_half_diamond_change_order(self): from conan import ConanFile class Pkg(ConanFile): def requirements(self): + # Best practice is to declare test_requires in the build_requirements() method, + # but this ensures it is also possible to declare them in requirements(), + # as has historically been allowed. self.test_requires("gtest/1.0") self.requires("zlib/1.0") """) @@ -70,8 +73,9 @@ def test_test_requires_diamond_change_order(self): game = textwrap.dedent(""" from conan import ConanFile class Pkg(ConanFile): - def requirements(self): + def build_requirements(self): self.test_requires("gtest/1.0") + def requirements(self): self.requires("engine/1.0") """) c.save({"zlib/conanfile.py": GenConanfile("zlib", "1.0"), @@ -92,7 +96,7 @@ def test_test_requires_conflict_force(self): game = textwrap.dedent(""" from conan import ConanFile class Pkg(ConanFile): - def requirements(self): + def build_requirements(self): self.test_requires("gtest/1.0", force=True) self.test_requires("rapidcheck/1.0") """) @@ -214,8 +218,9 @@ class LibC(ConanFile): name = "libc" version = "0.1" - def requirements(self): + def build_requirements(self): self.test_requires("liba/1.0") + def requirements(self): self.requires("libb/1.0") def package_info(self): diff --git a/test/integration/graph/version_ranges/version_ranges_cached_test.py b/test/integration/graph/version_ranges/version_ranges_cached_test.py index c7e9357b480..7aeaa764ae9 100644 --- a/test/integration/graph/version_ranges/version_ranges_cached_test.py +++ b/test/integration/graph/version_ranges/version_ranges_cached_test.py @@ -1,7 +1,7 @@ from collections import OrderedDict import pytest -from mock import patch +from unittest.mock import patch from conan.internal.rest.remote_manager import RemoteManager from conan.api.model import RecipeReference diff --git a/test/integration/layout/devflow_test.py b/test/integration/layout/devflow_test.py index 86841f7a38d..8f231e59c20 100644 --- a/test/integration/layout/devflow_test.py +++ b/test/integration/layout/devflow_test.py @@ -1,4 +1,5 @@ import os +import textwrap from conan.test.utils.tools import TestClient from conan.internal.util.files import load, mkdir @@ -31,17 +32,15 @@ def package(self): class TestDevInSourceFlow: - def _assert_pkg(self, folder): + @staticmethod + def _assert_pkg(folder): assert sorted(['file.h', 'myartifact.lib', 'subdir', 'conaninfo.txt', - 'conanmanifest.txt']) == \ - sorted(os.listdir(folder)) - assert load(os.path.join(folder, "myartifact.lib")) == \ - "artifact contents!" - assert load(os.path.join(folder, "subdir/myartifact2.lib")) == \ - "artifact2 contents!" + 'conanmanifest.txt']) == sorted(os.listdir(folder)) + assert load(os.path.join(folder, "myartifact.lib")) == "artifact contents!" + assert load(os.path.join(folder, "subdir/myartifact2.lib")) == "artifact2 contents!" def test_parallel_folders(self): - client = TestClient() + client = TestClient(light=True) repo_folder = os.path.join(client.current_folder, "recipe") build_folder = os.path.join(client.current_folder, "build") mkdir(repo_folder) @@ -62,7 +61,7 @@ def test_parallel_folders(self): self._assert_pkg(cache_package_folder) def test_insource_build(self): - client = TestClient() + client = TestClient(light=True) repo_folder = client.current_folder package_folder = os.path.join(client.current_folder, "pkg") mkdir(package_folder) @@ -79,7 +78,7 @@ def test_insource_build(self): self._assert_pkg(cache_package_folder) def test_child_build(self): - client = TestClient() + client = TestClient(light=True) build_folder = os.path.join(client.current_folder, "build") mkdir(build_folder) package_folder = os.path.join(build_folder, "package") @@ -121,12 +120,13 @@ def package(self): class TestDevOutSourceFlow: - def _assert_pkg(self, folder): + @staticmethod + def _assert_pkg(folder): assert sorted(['file.h', 'myartifact.lib', 'conaninfo.txt', 'conanmanifest.txt']) == \ sorted(os.listdir(folder)) def test_parallel_folders(self): - client = TestClient() + client = TestClient(light=True) repo_folder = os.path.join(client.current_folder, "recipe") src_folder = os.path.join(client.current_folder, "src") build_folder = os.path.join(client.current_folder, "build") @@ -156,7 +156,7 @@ def layout(self): self._assert_pkg(cache_package_folder) def test_insource_build(self): - client = TestClient() + client = TestClient(light=True) repo_folder = client.current_folder client.save({"conanfile.py": conanfile_out}) @@ -172,7 +172,7 @@ def test_insource_build(self): self._assert_pkg(cache_package_folder) def test_child_build(self): - client = TestClient() + client = TestClient(light=True) repo_folder = client.current_folder build_folder = os.path.join(client.current_folder, "build") mkdir(build_folder) @@ -194,3 +194,18 @@ def layout(self): cache_package_folder = client.created_layout().package() self._assert_pkg(cache_package_folder) + + +def test_error_build_file(): + c = TestClient(light=True) + conanfile_error = textwrap.dedent(r""" + from conan import ConanFile + class Test(ConanFile): + def layout(self): + self.folders.build = "build" + """) + + c.save({"conanfile.py": conanfile_error, + "build": "# Some existing BUILD file like bazel"}) + c.run("build . ", assert_error=True) + assert "ERROR: conanfile.py: Failed to create build folder, there is already a file" in c.out diff --git a/test/integration/lockfile/test_lock_pyrequires.py b/test/integration/lockfile/test_lock_pyrequires.py index 7ec6198ec2f..cf1ba0d698b 100644 --- a/test/integration/lockfile/test_lock_pyrequires.py +++ b/test/integration/lockfile/test_lock_pyrequires.py @@ -96,7 +96,8 @@ def test_lock_pyrequires_prereleases(): dep_rrev = tc.exported_recipe_revision() tc.run("lock create app --lockfile-out=app.lock", assert_error=True) # Makes sense, prereleases are not active - assert "Version range '>=0' from requirement 'dep/[>=0]' required by 'python_requires' could not be resolved" + assert ("Version range '>=0' from requirement 'dep/[>=0]' required by " + "'python_requires' could not be resolved") in tc.out tc.save_home({"global.conf": "core.version_ranges:resolve_prereleases=True"}) # This used to crash even with the conf activated @@ -136,9 +137,84 @@ def test_lock_pyrequires_export_transitive(): def test_lock_export_transitive_pyrequire(): c = TestClient(light=True) c.save({"dep/conanfile.py": GenConanfile("dep", "0.1").with_package_type("python-require"), - "pkg/conanfile.py": GenConanfile("pkg", "0.1") .with_python_requires("dep/0.1")}) + "pkg/conanfile.py": GenConanfile("pkg", "0.1").with_python_requires("dep/0.1")}) c.run("export dep") c.run("export pkg --lockfile-out=conan.lock") lock = json.loads(c.load("conan.lock")) assert "pkg/0.1#dfd6bc1becb3915043a671111860baee" in lock["requires"][0] assert "dep/0.1#5d31586a2a4355d68898875dc591009a" in lock["python_requires"][0] + + +def test_pyrequires_test_package_lockfile_error(): + # https://github.com/conan-io/conan/issues/19340 + c = TestClient(light=True) + c.save({"utils/conanfile.py": GenConanfile("utils").with_package_type("python-require"), + "bar/conanfile.py": GenConanfile("bar", "8.0").with_python_requires("utils/[^1]"), + "bar/test_package/conanfile.py": GenConanfile().with_test("pass") + .with_python_requires("utils/[^1]"), + "foo/conanfile.py": GenConanfile("foo", "1.0").with_requires("bar/[*]") + .with_python_requires("utils/1.7.1")}) + + c.run("create utils --version=1.7.1") + c.run("create utils --version=1.8.0") + c.run("create bar --lockfile-out=bar.lock") + c.run("create foo", assert_error=True) + assert "Missing prebuilt package for 'bar/8.0'" in c.out + + c.run("create foo --lockfile=bar.lock --lockfile-partial --lockfile-out=foo.lock") + c.assert_listed_require({"utils/1.7.1": "Cache", + "utils/1.8.0": "Cache"}, python=True) + lock = json.loads(c.load("foo.lock")) + assert "utils/1.8.0" in lock["python_requires"][0] + assert "utils/1.7.1" in lock["python_requires"][1] + + # The lockfile works, because the fixed version is an older one, and the lock can have + # both, the range resolves to 1.8.0, and the fixed resolves to 1.7.1 + # This lockfile can be used with same results + c.run("create foo --lockfile=foo.lock") + c.assert_listed_require({"utils/1.7.1": "Cache", + "utils/1.8.0": "Cache"}, python=True) + + +def test_pyrequires_test_package_lockfile_error_forward(): + # https://github.com/conan-io/conan/issues/19340 + c = TestClient(light=True) + c.save({"utils/conanfile.py": GenConanfile("utils").with_package_type("python-require"), + "bar/conanfile.py": GenConanfile("bar", "8.0").with_python_requires("utils/[^1]"), + "bar/test_package/conanfile.py": GenConanfile().with_test("pass") + .with_python_requires("utils/[^1]"), + "foo/conanfile.py": GenConanfile("foo", "1.0").with_requires("bar/[*]") + .with_python_requires("utils/1.9.0")}) + + # bar binary is built with utils/1.8.0 + c.run("create utils --version=1.8.0") + c.run("create bar --lockfile-out=bar.lock") + c.assert_listed_require({"utils/1.8.0": "Cache"}, python=True) + lock = json.loads(c.load("bar.lock")) + assert "utils/1.8.0" in lock["python_requires"][0] + + # now a new utils/1.9.0 is out, so the regular consumption of bar will fail with missing binary + # this will happen even if foo doesn't declare python-requires + c.run("create utils --version=1.9.0") + c.run("create foo", assert_error=True) + assert "Missing prebuilt package for 'bar/8.0'" in c.out + c.assert_listed_require({"utils/1.9.0": "Cache"}, python=True) + + # If we try to force bar strict dependencies with lockfile, it will fail, as expected + c.run("create foo --lockfile=bar.lock", assert_error=True) + assert "Requirement 'utils/1.9.0' not in lockfile 'python_requires'" in c.out + + # we can relax the lockfile, and it will be able to resolve, + # but it will still fail with missing binary for bar + c.run("create foo --lockfile=bar.lock --lockfile-partial", assert_error=True) + assert "Missing prebuilt package for 'bar/8.0'" in c.out + + c.run("create foo --lockfile=bar.lock --lockfile-partial --build=missing " + "--lockfile-out=foo.lock") + c.assert_listed_require({"utils/1.9.0": "Cache"}, python=True) + assert "utils/1.8.0" not in c.out + + # This build is reproducible with this lockfile + c.run("create foo --lockfile=foo.lock") + c.assert_listed_require({"utils/1.9.0": "Cache"}, python=True) + assert "utils/1.8.0" not in c.out diff --git a/test/integration/lockfile/test_lock_requires.py b/test/integration/lockfile/test_lock_requires.py index 657abefd553..1fb8ff353de 100644 --- a/test/integration/lockfile/test_lock_requires.py +++ b/test/integration/lockfile/test_lock_requires.py @@ -395,7 +395,7 @@ def test_lock_tool_requires_test(self, client): # https://github.com/conan-io/conan/issues/11763 c = client with c.chdir("app"): - c.run("lock create .") + c.run("lock create") lock = c.load("conan.lock") assert "cmake/1.0" not in lock assert "dep/1.0" in lock diff --git a/test/integration/lockfile/test_user_overrides.py b/test/integration/lockfile/test_user_overrides.py index 69b1cbed1fc..5107fe9d7c0 100644 --- a/test/integration/lockfile/test_user_overrides.py +++ b/test/integration/lockfile/test_user_overrides.py @@ -4,7 +4,7 @@ import pytest from conan.test.assets.genconanfile import GenConanfile -from conan.test.utils.tools import NO_SETTINGS_PACKAGE_ID, TestClient +from conan.test.utils.tools import TestClient def test_user_overrides(): @@ -362,11 +362,10 @@ def test_lock_update(self, kind, old, new): assert new in lock - class TestLockUpgrade: @pytest.mark.parametrize("kind, pkg, old, new", [ ("requires", "math", "math/1.0", "math/1.1"), - ("build-requires", "cmake", "cmake/1.0", "cmake/1.1"), # TODO there is not a --build-requires + ("build-requires", "cmake", "cmake/1.0", "cmake/1.1"), # TODO there isn't a --build-requires # ("python-requires", "mytool", "mytool/1.0", "mytool/1.1"), # TODO nor a --python-requires ]) def test_lock_upgrade(self, kind, pkg, old, new): @@ -384,11 +383,9 @@ def test_lock_upgrade(self, kind, pkg, old, new): rev1 = c.exported_recipe_revision() c.run(f"lock upgrade --{kind_create}={pkg}/[*] --update-{kind}={pkg}/[*]") lock = c.load("conan.lock") - print(lock) assert f"{old}#{rev0}" not in lock assert f"{new}#{rev1}" in lock - def test_lock_upgrade_path(self): c = TestClient(light=True) c.save({"liba/conanfile.py": GenConanfile("liba"), @@ -436,7 +433,6 @@ def test_lock_upgrade_path(self): assert "libd/1.1" in lock assert "libd/1.2" not in lock - def test_lock_upgrade_new_requirement(self): c = TestClient(light=True) c.save({"liba/conanfile.py": GenConanfile("liba").with_requires("libb/1.0"), @@ -457,3 +453,49 @@ def test_lock_upgrade_new_requirement(self): assert "libb/2.0" in lock assert "libd/1.0" in lock assert "libc/1.0" in lock # TODO: libc should be removed from lockfile? It is not required anymore... + + def test_config_upgrade(self): + """ Test that it is possible to lock also config-requires + """ + c = TestClient(light=True) + + c.save({"config/conanfile.py": GenConanfile("config").with_package_type("configuration"), + "conanfile.py": GenConanfile("pkg", "1.0")}) + c.run("create config --version=1.0") + c.run("create config --version=2.0") + c.run("config install-pkg config/1.0 --lockfile-out=conan.lock") + + def _check(refs): + reqs = json.loads(c.load("conan.lock"))["config_requires"] + for a, b in zip(refs, reqs): + assert a in b + + def _check_cache(refs): + reqs = json.loads(c.load_home("config_version.json"))["config_version"] + for a, b in zip(refs, reqs): + assert a in b + + _check(["config/1.0"]) + _check_cache(["config/1.0"]) + + c.run("lock upgrade-config --requires=config/[*] --update-config-requires=config/*") + _check(["config/2.0"]) + _check_cache(["config/1.0"]) + + c.run("config install-pkg config/1.0 --lockfile=conan.lock", assert_error=True) + assert "ERROR: Requirement 'config/1.0' not in lockfile 'config_requires'" in c.out + c.run("config install-pkg config/2.0 --lockfile=conan.lock") + _check(["config/2.0"]) + _check_cache(["config/2.0"]) + + # Force downgrade + c.run("config install-pkg config/1.0 --lockfile= --lockfile-out=conan.lock") + _check(["config/1.0"]) + _check_cache(["config/1.0"]) + + c.save({"conanconfig.yml": "packages:\n - config/[*]"}) + c.run("lock upgrade-config . --update-config-requires=config/1.0") + _check(["config/2.0"]) + _check_cache(["config/1.0"]) + c.run("config install-pkg . --lockfile=conan.lock") + _check_cache(["config/2.0"]) diff --git a/test/integration/metadata/test_metadata_commands.py b/test/integration/metadata/test_metadata_commands.py index e9c91f55e30..cdffeee9cad 100644 --- a/test/integration/metadata/test_metadata_commands.py +++ b/test/integration/metadata/test_metadata_commands.py @@ -11,17 +11,18 @@ class TestMetadataCommands: @pytest.fixture def create_conan_pkg(self): - client = TestClient(default_server_user=True) + client = TestClient(default_server_user=True, light=True) client.save({"conanfile.py": GenConanfile("pkg", "0.1")}) client.run("create .") pid = client.created_package_id("pkg/0.1") return client, pid - def save_metadata_file(self, client, pkg_ref, filename="somefile.log"): + @staticmethod + def save_metadata_file(client, pkg_ref, filename="somefile.log", content=None): client.run(f"cache path {pkg_ref} --folder=metadata") metadata_path = str(client.stdout).strip() myfile = os.path.join(metadata_path, "logs", filename) - save(myfile, f"{pkg_ref}!!!!") + save(myfile, f"{content or str(pkg_ref)}!!!!") return metadata_path, myfile def test_upload(self, create_conan_pkg): @@ -46,10 +47,12 @@ def test_upload(self, create_conan_pkg): c.run("remove * -c") c.run("install --requires=pkg/0.1") # wont install metadata by default - c.run("cache path pkg/0.1 --folder=metadata", assert_error=True) - assert "'metadata' folder does not exist for the reference pkg/0.1" in c.out - c.run(f"cache path pkg/0.1:{pid} --folder=metadata", assert_error=True) - assert f"'metadata' folder does not exist for the reference pkg/0.1:{pid}" in c.out + c.run("cache path pkg/0.1 --folder=metadata") + metadata_path = str(c.stdout).strip() + assert os.listdir(metadata_path) == [] + c.run(f"cache path pkg/0.1:{pid} --folder=metadata") + metadata_path = str(c.stdout).strip() + assert os.listdir(metadata_path) == [] # Forcing the download of the metadata of cache-existing things with the "download" command c.run("download pkg/0.1 -r=default --metadata=*") @@ -63,7 +66,7 @@ def test_upload(self, create_conan_pkg): assert os.path.isfile(os.path.join(pkg_metadata_path, f)) def test_update_contents(self): - c = TestClient(default_server_user=True) + c = TestClient(default_server_user=True, light=True) c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) c.run("export .") @@ -89,6 +92,72 @@ def test_update_contents(self): content = load(os.path.join(metadata_path, "logs", "mylogs.txt")) assert "mylogs2!!!!" in content + def test_append_contents(self): + # I can add extra files to metadata without necessarily downloading it first + c = TestClient(default_server_user=True, light=True) + c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) + c.run("create .") + pkgid = "da39a3ee5e6b4b0d3255bfef95601890afd80709" + # Add some metadata + self.save_metadata_file(c, "pkg/0.1", "mylogs.txt", content="mylogs") + self.save_metadata_file(c, f"pkg/0.1:{pkgid}", "mybin.txt", content="mybin") + + # Now upload everything + c.run("upload * -c -r=default") + assert "pkg/0.1: Recipe metadata: 1 files" in c.out + + c2 = TestClient(servers=c.servers, light=True, inputs=["admin", "password"]) + c2.run("install --requires=pkg/0.1") + self.save_metadata_file(c2, "pkg/0.1", "mylogs2.txt", content="mylogs2") + self.save_metadata_file(c2, f"pkg/0.1:{pkgid}", "mybin2.txt", content="mybin2") + c2.run("upload * -c -r=default --metadata=*") + assert "pkg/0.1: Recipe metadata: 1 files" in c2.out + + c.run("remove * -c") + c.run("download pkg/0.1 -r=default --metadata=*") + + c.run("cache path pkg/0.1 --folder=metadata") + metadata_path = str(c.stdout).strip() + assert "mylogs!!!!" in load(os.path.join(metadata_path, "logs", "mylogs.txt")) + assert "mylogs2!!!!" in load(os.path.join(metadata_path, "logs", "mylogs2.txt")) + + c.run(f"cache path pkg/0.1:{pkgid} --folder=metadata") + metadata_path = str(c.stdout).strip() + assert "mybin!!!!" in load(os.path.join(metadata_path, "logs", "mybin.txt")) + assert "mybin2!!!!" in load(os.path.join(metadata_path, "logs", "mybin2.txt")) + + def test_overwrite_server_contents(self): + # I can overwrite server contents + c = TestClient(default_server_user=True, light=True) + c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) + c.run("create .") + pkgid = "da39a3ee5e6b4b0d3255bfef95601890afd80709" + # Add some metadata + self.save_metadata_file(c, "pkg/0.1", "mylogs.txt", content="mylogs") + self.save_metadata_file(c, f"pkg/0.1:{pkgid}", "mybin.txt", content="mybin") + + # Now upload everything + c.run("upload * -c -r=default") + assert "pkg/0.1: Recipe metadata: 1 files" in c.out + + c2 = TestClient(servers=c.servers, light=True, inputs=["admin", "password"]) + c2.run("install --requires=pkg/0.1") + self.save_metadata_file(c2, "pkg/0.1", "mylogs.txt", content="mylogs2") + self.save_metadata_file(c2, f"pkg/0.1:{pkgid}", "mybin.txt", content="mybin2") + c2.run("upload * -c -r=default --metadata=*") + assert "pkg/0.1: Recipe metadata: 1 files" in c2.out + + c.run("remove * -c") + c.run("download pkg/0.1 -r=default --metadata=*") + + c.run("cache path pkg/0.1 --folder=metadata") + metadata_path = str(c.stdout).strip() + assert "mylogs2!!!!" in load(os.path.join(metadata_path, "logs", "mylogs.txt")) + + c.run(f"cache path pkg/0.1:{pkgid} --folder=metadata") + metadata_path = str(c.stdout).strip() + assert "mybin2!!!!" in load(os.path.join(metadata_path, "logs", "mybin.txt")) + def test_folder_exist(self, create_conan_pkg): """ so we can cp -R to the metadata folder, having to create the folder in the cache is weird @@ -148,7 +217,7 @@ def test_no_download_cached(self, create_conan_pkg): assert "pkg/0.1: Recipe metadata: 1 files" in c.out assert "pkg/0.1:da39a3ee5e6b4b0d3255bfef95601890afd80709: Package metadata: 1 files" in c.out - c2 = TestClient(servers=c.servers) + c2 = TestClient(servers=c.servers, light=True) tmp_folder = temp_folder() # MOST important part: activate cache c2.save_home({"global.conf": f"core.download:download_cache={tmp_folder}\n"}) @@ -196,7 +265,7 @@ def test_upload_ignored_metadata_with_pattern(self, create_conan_pkg): """ Upload command should fail when passing --metadata="" and a pattern """ - client = TestClient(default_server_user=True) + client = TestClient(default_server_user=True, light=True) client.save({"conanfile.py": GenConanfile("pkg", "0.1")}) client.run("export .") diff --git a/test/integration/metadata/test_metadata_deploy.py b/test/integration/metadata/test_metadata_deploy.py index ab6216568ab..1737c61aaaf 100644 --- a/test/integration/metadata/test_metadata_deploy.py +++ b/test/integration/metadata/test_metadata_deploy.py @@ -35,7 +35,7 @@ def build(self): def deploy(graph, output_folder, **kwargs): conanfile = graph.root.conanfile for r, d in conanfile.dependencies.items(): - if not os.path.exists(d.package_metadata_folder): + if not os.listdir(d.package_metadata_folder): continue shutil.copytree(d.package_metadata_folder, os.path.join(output_folder, "pkgs", d.ref.name)) diff --git a/test/integration/options/options_test.py b/test/integration/options/options_test.py index 96d54c3061f..f51fcbb2c41 100644 --- a/test/integration/options/options_test.py +++ b/test/integration/options/options_test.py @@ -505,6 +505,35 @@ def configure(self): assert "dep3/1.0: SHARED: True!!" in c.out assert "dep4/1.0: SHARED: True!!" in c.out + def test_pattern_version_range_warn(self): + c = TestClient(light=True) + profile = textwrap.dedent(""" + include(default) + + [tool_requires] + fmt/[*]:cmake/3.31.0 + + [options] + fmt/[*]:shared=True + + [settings] + fmt/[*]:compiler.cppstd=17 + """) + c.save({ + "profile": profile, + "conanfile.py": GenConanfile("fmt", "1.0")}) + c.run("create -pr=profile") + assert "WARN: risk: Settings pattern fmt/[*] contains a version range" in c.out + assert "WARN: risk: Options pattern fmt/[*] contains a version range" in c.out + assert "WARN: risk: Tool requires pattern fmt/[*] contains a version range" in c.out + + def test_pattern_version_range_wrong_split(self): + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile("foo", "1.0")}) + c.run("create -o=foo/[>=1]:shared=True", assert_error=True) + # We split by the first =, so this version range we can't catch and warn before it breaks + assert "option 'foo/[>' doesn't exist" in c.out + class TestTransitiveOptionsShared: """ @@ -772,6 +801,43 @@ def package_id(self): c.run("graph info app -o *:myoption!=4") assert "liba/0.1: MYOPTION: 4" in c.out + def test_important_options_recipe_priority_conditional(self): + c = TestClient() + + liba = textwrap.dedent(""" + from conan import ConanFile + class Lib(ConanFile): + name = "liba" + version = "0.1" + settings = "os" + options = {"myoption": [1, 2, 3]} + + def config_options(self): + if self.settings.os == "Windows": + setattr(self.options, "myoption!", 2) + + def package_id(self): + self.output.info(f"MYOPTION: {self.info.options.myoption}") + """) + + c.save({"liba/conanfile.py": liba, + "app/conanfile.py": GenConanfile().with_requires("liba/0.1")}) + c.run("export liba") + + c.run("graph info app -s os=Linux -o *:myoption=3") + assert "liba/0.1: MYOPTION: 3" in c.out + c.run("graph info app -s os=Linux -o *:myoption=2") + assert "liba/0.1: MYOPTION: 2" in c.out + c.run("graph info app -s os=Linux -o *:myoption=1") + assert "liba/0.1: MYOPTION: 1" in c.out + + c.run("graph info app -s os=Windows -o *:myoption=3") + assert "liba/0.1: MYOPTION: 2" in c.out + c.run("graph info app -s os=Windows -o *:myoption=2") + assert "liba/0.1: MYOPTION: 2" in c.out + c.run("graph info app -s os=Windows -o *:myoption=1") + assert "liba/0.1: MYOPTION: 2" in c.out + def test_wrong_option_syntax_no_trace(self): tc = TestClient(light=True) tc.save({"conanfile.py": GenConanfile().with_option("myoption", [1, 2, 3])}) @@ -819,9 +885,20 @@ def test_get_safe_none_option_checks(): .with_package("self.output.info(f'get_safe is None: {self.options.get_safe(\"myoption\") is None}')", "self.output.info(f'get_safe is not None: {self.options.get_safe(\"myoption\") is not None}')", "self.output.info(f'get_safe == None: {self.options.get_safe(\"myoption\") == None}')", - "self.output.info(f'get_safe != None: {self.options.get_safe(\"myoption\") != None}')" )}) + "self.output.info(f'get_safe != None: {self.options.get_safe(\"myoption\") != None}')")}) tc.run("create .") assert "get_safe is None: True" in tc.out assert "get_safe is not None: False" in tc.out assert "get_safe == None: True" in tc.out assert "get_safe != None: False" in tc.out + + +def test_option_apply_version_range(): + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile("dep", "0.1").with_shared_option(False)}) + c.run("create -o shared=True") + c.run("install --requires=dep/0.1 -o shared=True") # This worked without problem + c.run("install --requires=dep/[*] -o shared=True") + assert "WARN: risk" not in c.out + # This failed because of dep/[*] not matching pattern, now it works + assert "Install finished successfully" in c.out diff --git a/test/integration/options/test_configure_options.py b/test/integration/options/test_configure_options.py index 4782744b4e5..c6a52b45eb0 100644 --- a/test/integration/options/test_configure_options.py +++ b/test/integration/options/test_configure_options.py @@ -1,7 +1,7 @@ import os import textwrap -from parameterized import parameterized +import pytest from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient @@ -13,7 +13,7 @@ class TestConfigureOptions: header_only options automatically. """ - @parameterized.expand([ + @pytest.mark.parametrize("settings_os, shared, fpic, header_only, result", [ ["Linux", False, False, False, [False, False, False]], ["Windows", False, False, False, [False, None, False]], ["Windows", True, False, False, [True, None, False]], @@ -53,9 +53,9 @@ def build(self): result = f"shared: {result[0]}, fPIC: {result[1]}, header only: {result[2]}" assert result in client.out if header_only: - assert "Package 'da39a3ee5e6b4b0d3255bfef95601890afd80709' created"in client.out + assert "Package 'da39a3ee5e6b4b0d3255bfef95601890afd80709' created" in client.out - @parameterized.expand([ + @pytest.mark.parametrize("settings_os, shared, fpic, header_only, result", [ ["Linux", False, False, False, [False, False, False]], ["Linux", False, False, True, [False, False, True]], ["Linux", False, True, False, [False, True, False]], @@ -105,7 +105,7 @@ def build(self): result = f"shared: {result[0]}, fPIC: {result[1]}, header only: {result[2]}" assert result in client.out if header_only: - assert "Package 'da39a3ee5e6b4b0d3255bfef95601890afd80709' created"in client.out + assert "Package 'da39a3ee5e6b4b0d3255bfef95601890afd80709' created" in client.out def test_header_package_type_pid(self): """ diff --git a/test/integration/package_id/compatible_test.py b/test/integration/package_id/compatible_test.py index b224f2f8406..f2b6fea53a3 100644 --- a/test/integration/package_id/compatible_test.py +++ b/test/integration/package_id/compatible_test.py @@ -1,7 +1,10 @@ import json import textwrap -from conan.test.utils.tools import TestClient, GenConanfile +import pytest + +from conan.test.utils.env import environment_update +from conan.test.utils.tools import TestClient, GenConanfile, TestServer class TestCompatibleIDsTest: @@ -37,14 +40,17 @@ def package_info(self): client.run("export . --name=lib --version=0.1 --user=user --channel=stable") # package can be used with a profile gcc 4.9 falling back to 4.8 binary - client.save({"conanfile.py": GenConanfile().with_requires("pkg/0.1@user/stable", "lib/0.1@user/stable")}) + client.save({"conanfile.py": GenConanfile().with_requires("pkg/0.1@user/stable", + "lib/0.1@user/stable")}) # No fallback client.run("install . -pr=myprofile --build=missing -u=lib") assert "pkg/0.1@user/stable: PackageInfo!: Gcc version: 4.9!" in client.out client.assert_listed_binary({"pkg/0.1@user/stable": ("1ded27c9546219fbd04d4440e05b2298f8230047", "Build")}) - assert "lib/0.1@user/stable: Compatible configurations not found in cache, checking servers" not in client.out - assert "pkg/0.1@user/stable: Compatible configurations not found in cache, checking servers" in client.out + assert ("lib/0.1@user/stable: Compatible configurations not found in cache, " + "checking servers") not in client.out + assert ("pkg/0.1@user/stable: Compatible configurations not found in cache, " + "checking servers") in client.out def test_compatible_setting_no_user_channel(self): client = TestClient() @@ -137,7 +143,7 @@ def package_info(self): compiler.libcxx=libstdc++ """) client.save_home({"global.conf": - "core.package_id:default_unknown_mode=recipe_revision_mode"}) + "core.package_id:default_unknown_mode=recipe_revision_mode"}) client.save({"conanfile.py": conanfile, "myprofile": profile}) # Create package with gcc 4.8 @@ -724,3 +730,236 @@ def libc_compat(conanfile): tc.run("install --requires=dep/1.0 -s=libc_version=3 -s=compiler.cppstd=14") assert f"dep/1.0: Found compatible package '{dep_package_id}': compiler.cppstd=17, " \ f"libc_version=2" in tc.out + + +class TestListOnlyCompatibilityOptimization: + + @pytest.fixture() + def client(self): + tc = TestClient(default_server_user=True, light=True) + compiler_settings = textwrap.dedent(""" + compiler: + foo: + version: [1] + cppstd: [7, 11, 14, 17, 20, 23]""") + tc.run("version") + tc.save_home({"settings_user.yml": compiler_settings}) + compat = tc.load_home("extensions/plugins/compatibility/compatibility.py") + compat = compat.replace("cppstd_possible_values = supported_cppstd(conanfile)", + "cppstd_possible_values = [7, 11, 14, 17, 20, 23]") + tc.save_home({"extensions/plugins/compatibility/compatibility.py": compat}) + return tc + + @pytest.mark.parametrize("update", [True, False]) + def test_remote_compatible_package(self, client, update): + tc = client + update_arg = "-u" if update else "" + compiler_args = "-s compiler=foo -s compiler.version=1" + tc.save({"conanfile.py": GenConanfile("pkg", "0.1").with_settings("compiler")}) + tc.run(f"create . {compiler_args} -s=compiler.cppstd=14") + std14_id = tc.created_layout().reference.package_id + tc.run(f"create . {compiler_args} -s=compiler.cppstd=17") + std17_id = tc.created_layout().reference.package_id + tc.run(f"create . {compiler_args} -s=compiler.cppstd=20") + std20_id = tc.created_layout().reference.package_id + + tc.run(f"upload pkg/0.1:{std20_id} -r=default -c") + tc.run(f"upload pkg/0.1:{std14_id} -r=default -c") + tc.run(f"upload pkg/0.1:{std17_id} -r=default -c") + + tc.run("remove * -c") + tc.run(f"install --requires=pkg/0.1 {compiler_args} -s=compiler.cppstd=11 {update_arg} " + "-cc core.graph:compatibility_mode=optimized") + assert f"Found compatible package '{std14_id}'" in tc.out + + tc.run("remove * -c") + tc.run(f"remove pkg/0.1:{std17_id} -r=default -c") + tc.run(f"install --requires=pkg/0.1 {compiler_args} -s=compiler.cppstd=17 {update_arg} " + "-cc core.graph:compatibility_mode=optimized") + if not update: + assert "Found 2 compatible configurations in remotes" in tc.out + assert f"Found compatible package '{std14_id}'" in tc.out + + tc.run(f"create . {compiler_args} -s=compiler.cppstd=11") + std11_layout = tc.created_layout() + std11_id = tc.created_layout().reference.package_id + tc.run(f"upload pkg/0.1:{std11_id} -r=default -c") + tc.run("remove * -c") + tc.run(f"install --requires=pkg/0.1 {compiler_args} -s=compiler.cppstd=17 -vvv {update_arg} " + "-cc core.graph:compatibility_mode=optimized") + assert f"Found compatible package '{std11_id}'" in tc.out + # An HTTP request is made to the server to search for compatible packages + if not update: + assert "Found 3 compatible configurations in remotes" in tc.out + assert f"{std11_layout.reference.ref.revision}/search?list_only=True" in tc.out + + def test_remote_compatible_package_update_cache(self, client): + tc = client + compiler_args = "-s compiler=foo -s compiler.version=1" + tc.save({"conanfile.py": GenConanfile("pkg", "0.1") + .with_settings("compiler") + .with_import("import os, time") + .with_import("from conan.tools.files import save") + .with_package("save(self, os.path.join(self.package_folder, 'data.txt'), str(time.time()))") + }) + tc.run(f"create . {compiler_args} -s=compiler.cppstd=17") + std17_old_ref = tc.created_layout().reference + std17_id = std17_old_ref.package_id + tc.run(f"create . {compiler_args} -s=compiler.cppstd=17") + std17_new_ref = tc.created_layout().reference + assert std17_old_ref.revision != std17_new_ref.revision + + tc.run(f"upload pkg/0.1:{std17_id}#latest -r=default -c") + tc.run(f"remove pkg/0.1:{std17_id}#latest -c") + + tc.run(f"install --requires=pkg/0.1 {compiler_args} -s=compiler.cppstd=11 -r=default -u " + "-cc core.graph:compatibility_mode=optimized") + assert "Current package revision is older than the remote one " + assert f"Found compatible package '{std17_id}'" in tc.out + assert std17_old_ref.revision not in tc.out + assert std17_new_ref.revision in tc.out + + tc.run("remove * -c") + tc.run("remove * -r=default -c") + + tc.run(f"create . {compiler_args} -s=compiler.cppstd=20") + std20_old_ref = tc.created_layout().reference + std20_id = std20_old_ref.package_id + tc.run(f"create . {compiler_args} -s=compiler.cppstd=20") + std20_new_ref = tc.created_layout().reference + assert std20_old_ref.revision != std20_new_ref.revision + + tc.run(f"upload pkg/0.1:{std20_id}#* -r=default -c") + tc.run(f"remove pkg/0.1:{std20_id} -c") + + tc.run(f"install --requires=pkg/0.1 {compiler_args} -s=compiler.cppstd=11 -r=default -u " + "-cc core.graph:compatibility_mode=optimized") + assert f"Found compatible package '{std20_id}'" in tc.out + assert std20_old_ref.revision not in tc.out + assert std20_new_ref.revision in tc.out + + @pytest.mark.parametrize("enable", [True, False]) + @pytest.mark.parametrize("from_remote", [True, False]) + def test_message_if_not_enabled(self, enable, from_remote): + tc = TestClient(default_server_user=True) + tc.save({"conanfile.py": GenConanfile("pkg", "0.1").with_settings("compiler")}) + tc.run("create . -s=compiler.cppstd=17") + if from_remote: + tc.run("upload * -r=default -c") + tc.run("remove * -c") + arg = "-cc core.graph:compatibility_mode=optimized" if enable else "" + tc.run(f"install --requires=pkg/0.1 -s=compiler.cppstd=14 {arg}") + if not enable and from_remote: + assert "A new experimental approach for binary compatibility detection is available" in tc.out + else: + assert "A new experimental approach for binary compatibility detection is available" not in tc.out + + @pytest.mark.parametrize("from_remote", [True, False]) + @pytest.mark.parametrize("update", [True, False]) + def test_compatibility_different_settings_per_context(self, from_remote, update): + tc = TestClient(default_server_user=True) + tc.save({"protobuf/conanfile.py": GenConanfile("protobuf", "1.0") + .with_settings("compiler"), + "conanfile.py": GenConanfile("consumer", "1.0") + .with_require("protobuf/1.0") + .with_tool_requires("protobuf/1.0") + }) + tc.run("create protobuf -s=compiler.cppstd=14") + if from_remote: + tc.run("upload * -r=default -c") + tc.run("remove * -c") + update_arg = "--update" if update else "" + tc.run( + f"install . -s=compiler.cppstd=14 -s:b=compiler.cppstd=17 --build=missing {update_arg} " + "-cc core.graph:compatibility_mode=optimized") + + @pytest.mark.parametrize("update", [True, False]) + def test_compatibility_different_settings_per_context_prevs(self, update): + tc = TestClient(default_server_user=True) + proto = GenConanfile("protobuf", "1.0").with_settings("compiler") + proto.with_package_file("file.txt", env_var="MY_VAR") + consumer = GenConanfile().with_requires("protobuf/1.0").with_tool_requires("protobuf/1.0") + tc.save({"protobuf/conanfile.py": proto, + "conanfile.py": consumer}) + + settings = "-s:a compiler=gcc -s:a compiler.version=9 -s:a compiler.libcxx=libstdc++" + with environment_update({"MY_VAR": "value"}): + tc.run(f"create protobuf {settings} -s=compiler.cppstd=14") + tc.run("upload * -r=default -c") + + tc2 = TestClient(servers=tc.servers) + tc2.save({"conanfile.py": consumer}) + update_arg = "--update" if update else "" + tc2.run(f"install . {settings} -s=compiler.cppstd=14 -s:b=compiler.cppstd=17 {update_arg} " + "-cc core.graph:compatibility_mode=optimized") + tc2.assert_listed_binary({"protobuf/1.0": ("36d978cbb4dc35906d0fd438732d5e17cd1e388d", + "Download (default)")}) + + with environment_update({"MY_VAR": "value2"}): + tc.run(f"create protobuf {settings} -s=compiler.cppstd=14") + tc.run("upload * -r=default -c") + tc2.run(f"install . {settings} -s=compiler.cppstd=14 -s:b=compiler.cppstd=17 {update_arg} " + "-cc core.graph:compatibility_mode=optimized") + origin = "Cache" if not update else "Update (default)" + tc2.assert_listed_binary({"protobuf/1.0": ("36d978cbb4dc35906d0fd438732d5e17cd1e388d", + origin)}) + + def test_multi_remote(self): + # https://github.com/conan-io/conan/issues/19342 + c = TestClient(servers={"r1": TestServer(), "r2": TestServer()}, + inputs=["admin", "password"] * 2) + + c.save({"conanfile.py": GenConanfile("pkg", "0.1").with_settings("compiler")}) + c.run(f"create . -s=compiler.cppstd=14") + + c.run(f"upload * -r=r1 -c") + c.run(f"upload * -r=r2 -c") + c.run("remove * -c") + c.run(f"install --requires=pkg/0.1 -s=compiler.cppstd=17 " + "-cc core.graph:compatibility_mode=optimized") + # It doesn't crash + assert "pkg/0.1: Found 1 compatible configurations in remotes" in c.out + + +def test_compatibility_remove_cppstd(): + """ This test tries to reflect the following scenario: + - User recently added compiler.cppstd to their settings + - But up until now, no package was built with that setting + - At the user's own risk, we can tell Conan to accept packages built without that setting + """ + tc = TestClient() + profile = textwrap.dedent(""" + [settings] + compiler=gcc + compiler.version=11 + compiler.libcxx=libstdc++11 + """) + tc.save({"conanfile.py": GenConanfile("dep", "1.0").with_settings("compiler"), + "profile": profile}) + tc.run("create . -pr=profile") + dep_package_id = tc.created_package_id("dep/1.0") + tc.run("install --requires=dep/1.0 -pr=profile -s=compiler.cppstd=17", assert_error=True) + # We can't compile, because the dep is not compatible, it's looking for a package with cppstd + assert "Missing prebuilt package for 'dep/1.0'" in tc.out + + # Let's create a compatibility extensions + no_cppstd_compat = textwrap.dedent(""" + from conan.tools.scm import Version + + def no_cppstd_compat(conanfile): + # Do we have the setting? + cppstd_version = conanfile.settings.get_safe("compiler.cppstd") + if cppstd_version is None: + return [] + return [{"compiler.cppstd": None}] + """) + compat = tc.load_home("extensions/plugins/compatibility/compatibility.py") + compat = "from no_cppstd_compat import no_cppstd_compat\n" + compat + compat = compat.replace("# Append more factors for your custom compatibility rules here", + "factors.append(no_cppstd_compat(conanfile))") + tc.save_home({"extensions/plugins/compatibility/no_cppstd_compat.py": no_cppstd_compat, + "extensions/plugins/compatibility/compatibility.py": compat}) + + # Now we try again, this time app will find the compatible dep without cppstd + tc.run("install --requires=dep/1.0 -pr=profile -s=compiler.cppstd=17") + assert f"dep/1.0: Found compatible package '{dep_package_id}'" in tc.out diff --git a/test/integration/package_id/package_id_requires_modes_test.py b/test/integration/package_id/package_id_requires_modes_test.py index cad10498cf6..d01fa89f657 100644 --- a/test/integration/package_id/package_id_requires_modes_test.py +++ b/test/integration/package_id/package_id_requires_modes_test.py @@ -34,15 +34,15 @@ def test(self, mode, accepted_version, rejected_version, pattern): @pytest.mark.parametrize("mode, pkg_id", - [("unrelated_mode", "da39a3ee5e6b4b0d3255bfef95601890afd80709"), - ("semver_mode", "13b9e753af3958dd1b2d4b3f935b04b8fb6b6760"), - ("patch_mode", "38d7a3ec6a09165ab3e5306f81c539a2e0a784bd"), - ("minor_mode", "a5e7ad26ccf4a5049090976846da1c6ed165cced"), - ("major_mode", "6ac597ffb99c3747ed78699f206dc1041537a8df"), - # This is equal to semver_mode for 0.X.Y.Z.. - ("full_version_mode", "13b9e753af3958dd1b2d4b3f935b04b8fb6b6760"), - ("revision_mode", "ae5b9eeb74880aeb1cfa3db7f84c007a05ce3a76"), - ("full_mode", "d1b2a9538cd69363b4bae7e66c9f900b8f4c58bb")]) + [("unrelated_mode", "da39a3ee5e6b4b0d3255bfef95601890afd80709"), + ("semver_mode", "13b9e753af3958dd1b2d4b3f935b04b8fb6b6760"), + ("patch_mode", "38d7a3ec6a09165ab3e5306f81c539a2e0a784bd"), + ("minor_mode", "a5e7ad26ccf4a5049090976846da1c6ed165cced"), + ("major_mode", "6ac597ffb99c3747ed78699f206dc1041537a8df"), + # This is equal to semver_mode for 0.X.Y.Z.. + ("full_version_mode", "13b9e753af3958dd1b2d4b3f935b04b8fb6b6760"), + ("revision_mode", "ae5b9eeb74880aeb1cfa3db7f84c007a05ce3a76"), + ("full_mode", "d1b2a9538cd69363b4bae7e66c9f900b8f4c58bb")]) def test_modes(mode, pkg_id): c = TestClient(light=True) package_id_text = f'self.info.requires.{mode}()' diff --git a/test/integration/package_id/python_requires_package_id_test.py b/test/integration/package_id/python_requires_package_id_test.py index a0312a7f348..0ad771ae905 100644 --- a/test/integration/package_id/python_requires_package_id_test.py +++ b/test/integration/package_id/python_requires_package_id_test.py @@ -56,7 +56,7 @@ def test_change_mode_conf(self): self.client.run("export . --name=tool --version=1.1.2") self.client2.run("create . --name=pkg --version=0.1") assert "tool/1.1.2" in self.client2.out - self.client2.assert_listed_binary({"pkg/0.1": (PKG_ID_2,"Build")}) + self.client2.assert_listed_binary({"pkg/0.1": (PKG_ID_2, "Build")}) def test_unrelated_conf(self): # change the policy in conan.conf diff --git a/test/integration/package_id/test_cache_compatibles.py b/test/integration/package_id/test_cache_compatibles.py index 2642915f4df..0241a3c8683 100644 --- a/test/integration/package_id/test_cache_compatibles.py +++ b/test/integration/package_id/test_cache_compatibles.py @@ -180,6 +180,15 @@ def compatibility(conanfile): c.assert_listed_binary({"dep/0.1": ("ce92fac7c26ace631e30875ddbb3a58a190eb601", "Download (default)")}) + # Now what happens if we have it in cache already + c.run(f"install consumer {base_settings} -s compiler.cppstd=17 --update") + assert "dep/0.1: Checking 3 compatible configurations" in c.out + assert "dep/0.1: Main binary package '6179018ccb6b15e6443829bf3640e25f2718b931' missing" in c.out + assert "Found compatible package 'ce92fac7c26ace631e30875ddbb3a58a190eb601': " \ + "compiler.cppstd=14" in c.out + c.assert_listed_binary({"dep/0.1": ("ce92fac7c26ace631e30875ddbb3a58a190eb601", + "Cache")}) + class TestDefaultCompat: @@ -475,3 +484,14 @@ def test_remove_plugin_file(self): c.save({"conanfile.txt": ""}) c.run("install .", assert_error=True) assert "ERROR: The 'compatibility.py' plugin file doesn't exist" in c.out + + +def test_compatible_prev_from_cache(): + tc = TestClient() + tc.save({"conanfile.py": GenConanfile("pkg", "0.1").with_settings("compiler")}) + settings = ("-s os=Linux -s arch=x86_64 -s compiler=gcc -s compiler.version=8 " + "-s compiler.libcxx=libstdc++11") + tc.run(f"create . {settings} -s=compiler.cppstd=17") + tc.run(f"install --requires=pkg/0.1 {settings} -s=compiler.cppstd=14 -u") + tc.assert_listed_binary({"pkg/0.1": ("6179018ccb6b15e6443829bf3640e25f2718b931", + "Cache")}) diff --git a/test/integration/package_id/test_config_package_id.py b/test/integration/package_id/test_config_package_id.py index cf2383b3638..c21848e6cba 100644 --- a/test/integration/package_id/test_config_package_id.py +++ b/test/integration/package_id/test_config_package_id.py @@ -8,10 +8,10 @@ @pytest.mark.parametrize("config_version, mode, result", [ - ("myconfig/1.2.3#rev1:pid1#prev1", "minor_mode", "myconfig/1.2.Z"), - ("myconfig/1.2.3#rev1:pid1#prev1", "patch_mode", "myconfig/1.2.3"), - ("myconfig/1.2.3#rev1:pid1#prev1", "full_mode", "myconfig/1.2.3#rev1:pid1"), - ("myconfig/1.2.3#rev1:pid1#prev1", "revision_mode", "myconfig/1.2.3#rev1"), + ("myconfig/1.2.3#rev1", "minor_mode", "myconfig/1.2.Z"), + ("myconfig/1.2.3#rev1", "patch_mode", "myconfig/1.2.3"), + ("myconfig/1.2.3#rev1", "full_mode", "myconfig/1.2.3#rev1"), + ("myconfig/1.2.3#rev1", "revision_mode", "myconfig/1.2.3#rev1"), ("myconfig/1.2.3", "minor_mode", "myconfig/1.2.Z")]) def test_config_package_id(config_version, mode, result): c = TestClient() @@ -26,7 +26,7 @@ def test_config_package_id(config_version, mode, result): rrev = info["Local Cache"]["pkg/0.1"]["revisions"]["485dad6cb11e2fa99d9afbe44a57a164"] package_id = {"myconfig/1.2.Z": "c78b4d8224154390356fe04fe598d67aec930199", "myconfig/1.2.3": "60005f5b11bef3ddd686b13f5c6bf576a9b882b8", - "myconfig/1.2.3#rev1:pid1": "b1525975eb5420cef45b4ddd1544f87c29c773a5", + "myconfig/1.2.3#rev1:pid1": "3e4270809028e4566b55d3958e94ae3d6f0c92a7", "myconfig/1.2.3#rev1": "aae875ae226416f177bf386a3e4ad6aaffce09e7"} package_id = package_id.get(result) pkg = rrev["packages"][package_id] @@ -43,8 +43,8 @@ def test_error_config_package_id(): @pytest.mark.parametrize("config_version, mode, result", [ - ("myconfig/1.2.3#rev1:pid1#prev1", "minor_mode", "myconfig/1.2.Z"), - ("myconfig/1.2.3#rev1:pid1#prev1", "patch_mode", "myconfig/1.2.3"), + ("myconfig/1.2.3#rev1", "minor_mode", "myconfig/1.2.Z"), + ("myconfig/1.2.3#rev1", "patch_mode", "myconfig/1.2.3"), ]) def test_config_package_id_clear(config_version, mode, result): c = TestClient(light=True) diff --git a/test/integration/package_id/test_validate.py b/test/integration/package_id/test_validate.py index 0ca8ac91609..9642fdb8a1c 100644 --- a/test/integration/package_id/test_validate.py +++ b/test/integration/package_id/test_validate.py @@ -35,10 +35,10 @@ def validate(self): error = client.run("create . --name=pkg --version=0.1 -s os=Windows", assert_error=True) assert error == ERROR_INVALID_CONFIGURATION - assert "pkg/0.1: Invalid: Windows not supported"in client.out + assert "pkg/0.1: Invalid: Windows not supported" in client.out client.run("graph info --require pkg/0.1 -s os=Windows") - assert "binary: Invalid"in client.out + assert "binary: Invalid" in client.out assert "info_invalid: Windows not supported" in client.out client.run("graph info --require pkg/0.1 -s os=Windows --format json") @@ -132,26 +132,26 @@ def compatibility(self): client.run("install --requires=pkg/0.1@ -s os=Windows --build=pkg*", assert_error=True) assert "pkg/0.1: Cannot build for this configuration: Windows not supported" in client.out - assert "Windows not supported"in client.out + assert "Windows not supported" in client.out client.run("install --requires=pkg/0.1@ -s os=Windows") - assert f"pkg/0.1: Main binary package '{missing_id}' missing"in client.out - assert f"Found compatible package '{package_id}'"in client.out + assert f"pkg/0.1: Main binary package '{missing_id}' missing" in client.out + assert f"Found compatible package '{package_id}'" in client.out client.assert_listed_binary({"pkg/0.1": (package_id, "Cache")}) # --build=missing means "use existing binary if possible", and compatibles are valid binaries client.run("install --requires=pkg/0.1@ -s os=Windows --build=missing") - assert f"pkg/0.1: Main binary package '{missing_id}' missing"in client.out - assert f"Found compatible package '{package_id}'"in client.out + assert f"pkg/0.1: Main binary package '{missing_id}' missing" in client.out + assert f"Found compatible package '{package_id}'" in client.out client.assert_listed_binary({"pkg/0.1": (package_id, "Cache")}) client.run("graph info --requires=pkg/0.1@ -s os=Windows") - assert f"pkg/0.1: Main binary package '{missing_id}' missing"in client.out - assert f"Found compatible package '{package_id}'"in client.out - assert f"package_id: {package_id}"in client.out + assert f"pkg/0.1: Main binary package '{missing_id}' missing" in client.out + assert f"Found compatible package '{package_id}'" in client.out + assert f"package_id: {package_id}" in client.out client.run("graph info --requires=pkg/0.1@ -s os=Windows --build=pkg*") - assert "binary: Invalid"in client.out + assert "binary: Invalid" in client.out def test_validate_remove_package_id_create(self): client = TestClient() @@ -172,15 +172,15 @@ def package_id(self): client.save({"conanfile.py": conanfile}) client.run("create . --name=pkg --version=0.1 -s os=Linux") - assert "pkg/0.1: Package '{}' created".format(NO_SETTINGS_PACKAGE_ID)in client.out + assert "pkg/0.1: Package '{}' created".format(NO_SETTINGS_PACKAGE_ID) in client.out client.run("create . --name=pkg --version=0.1 -s os=Windows", assert_error=True) - assert "pkg/0.1: Invalid: Windows not supported"in client.out + assert "pkg/0.1: Invalid: Windows not supported" in client.out client.assert_listed_binary({"pkg/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Invalid")}) client.run("graph info --requires=pkg/0.1@ -s os=Windows") - assert "package_id: {}".format(NO_SETTINGS_PACKAGE_ID)in client.out + assert "package_id: {}".format(NO_SETTINGS_PACKAGE_ID) in client.out def test_validate_compatible_also_invalid(self): client = TestClient() @@ -212,7 +212,7 @@ def compatibility(self): assert_error=True) assert error == ERROR_INVALID_CONFIGURATION - assert "pkg/0.1: Invalid: Windows not supported"in client.out + assert "pkg/0.1: Invalid: Windows not supported" in client.out client.run("graph info --requires=pkg/0.1@ -s os=Windows") assert "binary: Invalid" in client.out @@ -245,24 +245,24 @@ def compatibility(self): # Windows invalid configuration error = client.run("create . --name=pkg --version=0.1 -s os=Windows -s build_type=Release", assert_error=True) - assert error== ERROR_INVALID_CONFIGURATION - assert "pkg/0.1: Invalid: Windows not supported"in client.out + assert error == ERROR_INVALID_CONFIGURATION + assert "pkg/0.1: Invalid: Windows not supported" in client.out error = client.run("install --requires=pkg/0.1@ -s os=Windows -s build_type=Release", assert_error=True) - assert error== ERROR_INVALID_CONFIGURATION - assert "pkg/0.1: Invalid: Windows not supported"in client.out + assert error == ERROR_INVALID_CONFIGURATION + assert "pkg/0.1: Invalid: Windows not supported" in client.out # Windows missing binary: INVALID error = client.run("install --requires=pkg/0.1@ -s os=Windows -s build_type=Debug", assert_error=True) - assert error== ERROR_INVALID_CONFIGURATION - assert "pkg/0.1: Invalid: Windows not supported"in client.out + assert error == ERROR_INVALID_CONFIGURATION + assert "pkg/0.1: Invalid: Windows not supported" in client.out error = client.run("create . --name=pkg --version=0.1 -s os=Windows -s build_type=Debug", assert_error=True) - assert error== ERROR_INVALID_CONFIGURATION - assert "pkg/0.1: Invalid: Windows not supported"in client.out + assert error == ERROR_INVALID_CONFIGURATION + assert "pkg/0.1: Invalid: Windows not supported" in client.out # info client.run("graph info --requires=pkg/0.1@ -s os=Windows") @@ -298,8 +298,8 @@ def validate(self): client.save({"conanfile.py": GenConanfile().with_requires("pkg2/0.1", "pkg1/0.1")}) error = client.run("install .", assert_error=True) - assert error== ERROR_INVALID_CONFIGURATION - assert "pkg1/0.1: Invalid: Option 2 of 'dep' not supported"in client.out + assert error == ERROR_INVALID_CONFIGURATION + assert "pkg1/0.1: Invalid: Option 2 of 'dep' not supported" in client.out def test_validate_requires(self): client = TestClient() @@ -324,15 +324,15 @@ def validate(self): .with_requirement("pkg1/0.1") .with_requirement("dep/0.2", override=True)}) error = client.run("install .", assert_error=True) - assert error== ERROR_INVALID_CONFIGURATION - assert "pkg1/0.1: Invalid: dep> 0.1 is not supported"in client.out + assert error == ERROR_INVALID_CONFIGURATION + assert "pkg1/0.1: Invalid: dep> 0.1 is not supported" in client.out client.save({"conanfile.py": GenConanfile() .with_requirement("pkg1/0.1") .with_requirement("dep/0.2", force=True)}) error = client.run("install .", assert_error=True) - assert error== ERROR_INVALID_CONFIGURATION - assert "pkg1/0.1: Invalid: dep> 0.1 is not supported"in client.out + assert error == ERROR_INVALID_CONFIGURATION + assert "pkg1/0.1: Invalid: dep> 0.1 is not supported" in client.out def test_validate_package_id_mode(self): client = TestClient() @@ -352,13 +352,13 @@ def validate(self): client.save({"conanfile.py": GenConanfile().with_requires("dep/0.1")}) error = client.run("create . --name=pkg --version=0.1 -s os=Windows", assert_error=True) - assert error== ERROR_INVALID_CONFIGURATION + assert error == ERROR_INVALID_CONFIGURATION client.assert_listed_binary({"dep/0.1": ("ebec3dc6d7f6b907b3ada0c3d3cdc83613a2b715", "Invalid")}) client.assert_listed_binary({"pkg/0.1": ("19ad5731bb09f24646c81060bd7730d6cb5b6108", "Build")}) - assert "ERROR: There are invalid packages:"in client.out - assert "dep/0.1: Invalid: Windows not supported"in client.out + assert "ERROR: There are invalid packages:" in client.out + assert "dep/0.1: Invalid: Windows not supported" in client.out def test_validate_export_pkg(self): # https://github.com/conan-io/conan/issues/9797 @@ -694,7 +694,8 @@ def compatibility(self): assert "compiler.cppstd=14" in client.out assert "compiler.cppstd=17" not in client.out - @pytest.mark.skipif(platform.system() == "Windows", reason="Needs at least 3 cppstd values available, msvc 191 does not") + @pytest.mark.skipif(platform.system() == "Windows", + reason="Needs at least 3 cppstd values available, msvc 191 does not") def test_extension_properties_cppstd_compat_non_transitiveness(self): """ The cppstd_compat is not transitive, so if a recipe has cppstd_compat=False, @@ -702,13 +703,11 @@ def test_extension_properties_cppstd_compat_non_transitiveness(self): """ tc = TestClient() tc.save({"dep/conanfile.py": GenConanfile("dep", "1.0").with_setting("compiler"), - "lib/conanfile.py": GenConanfile("lib", "1.0") - .with_setting("compiler") - .with_class_attribute('extension_properties = {"compatibility_cppstd": False}') - .with_requirement("dep/1.0"), - "app/conanfile.py": GenConanfile("app", "1.0") - .with_setting("compiler") - .with_requirement("lib/1.0")}) + "lib/conanfile.py": GenConanfile("lib", "1.0").with_setting("compiler") + .with_class_attribute('extension_properties = {"compatibility_cppstd": False}') + .with_requirement("dep/1.0"), + "app/conanfile.py": GenConanfile("app", "1.0").with_setting("compiler") + .with_requirement("lib/1.0")}) tc.run("create dep -s=compiler.cppstd=20") tc.run("create lib -s=compiler.cppstd=17") tc.run("create app -s=compiler.cppstd=14", assert_error=True) @@ -724,28 +723,27 @@ def test_extension_properties_make_transitive(self): """ tc = TestClient() tc.save({"lib/conanfile.py": GenConanfile("lib", "1.0").with_setting("compiler"), - "dep/conanfile.py": GenConanfile("dep", "1.0") - .with_setting("compiler") - .with_class_attribute('extension_properties = {"compatibility_cppstd": False}') - .with_requirement("lib/1.0"), - "conanfile.py": GenConanfile("app", "1.0") - .with_setting("compiler") - .with_requirement("dep/1.0")}) + "dep/conanfile.py": GenConanfile("dep", "1.0").with_setting("compiler") + .with_class_attribute('extension_properties = {"compatibility_cppstd": False}') + .with_requirement("lib/1.0"), + "conanfile.py": GenConanfile("app", "1.0").with_setting("compiler") + .with_requirement("dep/1.0")}) tc.run("create lib -s=compiler.cppstd=17") tc.run("create dep -s=compiler.cppstd=14") tc.run("create . -s=compiler.cppstd=14") - compat_path = os.path.join(tc.cache_folder, "extensions/plugins/compatibility/compatibility.py") + compat_path = os.path.join(tc.cache_folder, + "extensions/plugins/compatibility/compatibility.py") compat_contents = load(compat_path) transitive_expansions = textwrap.indent(textwrap.dedent(""" - extension_properties = getattr(conanfile, "extension_properties", {}).copy() - for dep in conanfile.dependencies.values(): - if not dep.extension_properties.get("compatibility_cppstd", True): - extension_properties["compatibility_cppstd"] = False - """), " ") - - compat_contents = compat_contents.replace('extension_properties = getattr(conanfile, "extension_properties", {})', transitive_expansions) + extension_properties = getattr(conanfile, "extension_properties", {}).copy() + for dep in conanfile.dependencies.values(): + if not dep.extension_properties.get("compatibility_cppstd", True): + extension_properties["compatibility_cppstd"] = False + """), " ") + existing = 'extension_properties = getattr(conanfile, "extension_properties", {})' + compat_contents = compat_contents.replace(existing, transitive_expansions) save(compat_path, compat_contents) tc.run("install --requires=app/1.0 -s=compiler.cppstd=17", assert_error=True) @@ -875,7 +873,7 @@ def requirements(self): error = client.run("create . --name=app --version=0.1 -s os=Linux -s:h arch=armv7", assert_error=True) assert error == ERROR_GENERAL - assert "ERROR: Missing prebuilt package for 'tool/0.1'"in client.out + assert "ERROR: Missing prebuilt package for 'tool/0.1'" in client.out def test_no_settings_target_in_recipe_but_in_global_compatibility(self): client = TestClient() @@ -913,9 +911,10 @@ def requirements(self): """) client.save({"conanfile.py": app_conanfile}) - error = client.run("create . --name=app --version=0.1 -s os=Linux -s:h arch=armv7", assert_error=True) + error = client.run("create . --name=app --version=0.1 -s os=Linux -s:h arch=armv7", + assert_error=True) assert error == ERROR_GENERAL - assert "ERROR: Missing prebuilt package for 'tool/0.1'"in client.out + assert "ERROR: Missing prebuilt package for 'tool/0.1'" in client.out def test_three_packages_with_and_without_settings_target(self): client = TestClient() @@ -945,7 +944,8 @@ def package_id(self): """) client.save({"conanfile.py": tool_a_conanfile}) - client.run("create . --name=tool_a --version=0.1 -s os=Linux -s:h arch=armv6 -s:b arch=x86_64 --build-require") + client.run("create . --name=tool_a --version=0.1 " + "-s os=Linux -s:h arch=armv6 -s:b arch=x86_64 --build-require") package_id_tool_a = client.created_package_id("tool_a/0.1") assert f"tool_a/0.1: Package '{package_id_tool_a}' created" in client.out @@ -957,7 +957,8 @@ class Pkg(ConanFile): """) client.save({"conanfile.py": tool_b_conanfile}) - client.run("create . --name=tool_b --version=0.1 -s os=Linux -s arch=x86_64 -s:b arch=x86_64") + client.run("create . --name=tool_b --version=0.1 -s os=Linux " + "-s arch=x86_64 -s:b arch=x86_64") package_id_tool_b = client.created_package_id("tool_b/0.1") assert f"tool_b/0.1: Package '{package_id_tool_b}' created" in client.out diff --git a/test/integration/package_id/transitive_header_only_test.py b/test/integration/package_id/transitive_header_only_test.py index a5399801cb1..6c98a12a579 100644 --- a/test/integration/package_id/transitive_header_only_test.py +++ b/test/integration/package_id/transitive_header_only_test.py @@ -1,5 +1,4 @@ from conan.test.utils.tools import TestClient, GenConanfile -from conan.internal.util.files import save class TestTransitiveIds: diff --git a/test/integration/package_id/transitive_options_affect_id_test.py b/test/integration/package_id/transitive_options_affect_id_test.py index 8b73a20172d..a26df74b3a5 100644 --- a/test/integration/package_id/transitive_options_affect_id_test.py +++ b/test/integration/package_id/transitive_options_affect_id_test.py @@ -53,3 +53,45 @@ def package_id(self): client.run("create . -o pkg*:noaffect=2 -o pkg*:affect=True") assert "EMBED MODE!!" in client.out client.assert_listed_binary({"app": (different_pkg_id, "Build")}) + + def test_transitive_shared(self): + # https://github.com/conan-io/conan/issues/18900 + c = TestClient() + + lib1 = GenConanfile("lib1", "0.1").with_shared_option(True) + lib2 = (GenConanfile("lib2", "0.1").with_shared_option(True) + .with_requirement("lib1/0.1", transitive_libs=True)) + lib3 = (GenConanfile("lib3", "0.1").with_shared_option(True) + .with_requirement("lib2/0.1")) + lib4 = (GenConanfile("lib4", "0.1").with_shared_option(True) + .with_requirement("lib3/0.1")) + c.save({"lib1/conanfile.py": lib1, + "lib2/conanfile.py": lib2, + "lib3/conanfile.py": lib3, + "lib4/conanfile.py": lib4}) + c.run("create lib1") + c.run("create lib2") + c.run("create lib3") + c.run("create lib4") + + c.run("install --requires=lib4/0.1 -o lib1/*:shared=False --build=missing") + c.assert_listed_binary({"lib1/0.1": ("55c609fe8808aa5308134cb5989d23d3caffccf2", "Build"), + "lib2/0.1": ("76844632e497abea8503d65ffd8324460dc70745", "Build"), + "lib3/0.1": ("7b10301e532fc0269d6ac70470aee5780f0836cd", "Build"), + "lib4/0.1": ("635372f179ad582c713637e361a7cd7ac7cd1d09", "Cache"), + }) + + lib3 = (GenConanfile("lib3", "0.1").with_shared_option(True) + .with_requirement("lib2/0.1", transitive_libs=True)) + c.save({"lib3/conanfile.py": lib3}) + c.run("remove * -c") + c.run("create lib1") + c.run("create lib2") + c.run("create lib3") + c.run("create lib4") + c.run("install --requires=lib4/0.1 -o lib1/*:shared=False --build=missing") + c.assert_listed_binary({"lib1/0.1": ("55c609fe8808aa5308134cb5989d23d3caffccf2", "Build"), + "lib2/0.1": ("76844632e497abea8503d65ffd8324460dc70745", "Build"), + "lib3/0.1": ("7b10301e532fc0269d6ac70470aee5780f0836cd", "Build"), + "lib4/0.1": ("dd8f5355b399fd7d96c883ddd39b992ae968cb14", "Build"), + }) diff --git a/test/integration/py_requires/python_requires_test.py b/test/integration/py_requires/python_requires_test.py index 921a320e812..04792b88496 100644 --- a/test/integration/py_requires/python_requires_test.py +++ b/test/integration/py_requires/python_requires_test.py @@ -438,7 +438,7 @@ def test_basic_option_inherit(self): client = TestClient() base = textwrap.dedent(""" from conan import ConanFile - class MyBase(object): + class MyBase: settings = "os", "build_type", "arch" options = {"base_option": [1, 2, 3]} default_options = {"base_option": 2} diff --git a/test/integration/remote/broken_download_test.py b/test/integration/remote/broken_download_test.py index 8f65bf2f6f4..fff7898910e 100644 --- a/test/integration/remote/broken_download_test.py +++ b/test/integration/remote/broken_download_test.py @@ -107,7 +107,7 @@ def get(self, url, **kwargs): else: return super(DownloadFilesBrokenRequester, self).get(url, **kwargs) - def DownloadFilesBrokenRequesterTimesOne(*args, **kwargs): + def DownloadFilesBrokenRequesterTimesOne(*args, **kwargs): # noqa return DownloadFilesBrokenRequester(1, *args, **kwargs) client = TestClient(servers=servers, inputs=["admin", "password"], requester_class=DownloadFilesBrokenRequesterTimesOne) @@ -122,12 +122,12 @@ def DownloadFilesBrokenRequesterTimesOne(*args, **kwargs): client.run("install --requires=lib/1.0@lasote/stable") assert 1 == str(client.out).count("Waiting 1 seconds to retry...") - def DownloadFilesBrokenRequesterTimesTen(*args, **kwargs): + def DownloadFilesBrokenRequesterTimesTen(*args, **kwargs): # noqa return DownloadFilesBrokenRequester(10, *args, **kwargs) client = TestClient(servers=servers, inputs=["admin", "password"], requester_class=DownloadFilesBrokenRequesterTimesTen) client.save_home({"global.conf": "core.download:retry_wait=0\n" - "core.download:retry=11"}) + "core.download:retry=11"}) client.run("install --requires=lib/1.0@lasote/stable") assert 10 == str(client.out).count("Waiting 0 seconds to retry...") diff --git a/test/integration/remote/rest_api_test.py b/test/integration/remote/rest_api_test.py index c9f8beb75c9..4d9f7d198dc 100644 --- a/test/integration/remote/rest_api_test.py +++ b/test/integration/remote/rest_api_test.py @@ -169,7 +169,8 @@ def _upload_package(self, package_reference, base_files=None): files = {"conanfile.py": GenConanfile("3").with_requires("1", "12").with_exports("*"), "hello.cpp": "hello", - "conanmanifest.txt": ""} + "conanmanifest.txt": "", + "conan_package.tgz": ""} if base_files: files.update(base_files) diff --git a/test/integration/remote/retry_test.py b/test/integration/remote/retry_test.py index 179e4ab4e5b..09316cda7a3 100644 --- a/test/integration/remote/retry_test.py +++ b/test/integration/remote/retry_test.py @@ -39,7 +39,7 @@ def put(self, *args, **kwargs): class _ConfigMock: - def get(self, conf_name, default=None, check_type=None): + def get(self, conf_name, default=None, check_type=None): # noqa return 0 @@ -89,14 +89,14 @@ def test_error_403_authentication(self): assert counter["Waiting 0 seconds to retry..."] == 0 def test_error_requests(self): - class _RequesterMock: + class _RequesterExcp: def put(self, *args, **kwargs): raise Exception("any exception") output = RedirectedTestOutput() with redirect_output(output): - uploader = FileUploader(requester=_RequesterMock(), verify=False, config=_ConfigMock()) + uploader = FileUploader(requester=_RequesterExcp(), verify=False, config=_ConfigMock()) with pytest.raises(Exception, match="any exception"): uploader.upload(url="fake", abs_path=self.filename, retry=2) output_lines = output.getvalue().splitlines() diff --git a/test/integration/remote/test_local_recipes_index.py b/test/integration/remote/test_local_recipes_index.py index 96d803e7a50..365c0effdf7 100644 --- a/test/integration/remote/test_local_recipes_index.py +++ b/test/integration/remote/test_local_recipes_index.py @@ -552,3 +552,48 @@ def build(self): "requested, but it doesn't match the current available revision in source") in c.out assert ("ERROR: The 'zlib/1.2.11' package has 'exports_sources' but sources " "not found in local cache") in c.out + + def test_reverting_to_older_revision(self): + # https://github.com/conan-io/conan/issues/19313 + folder = temp_folder() + recipes_folder = os.path.join(folder, "recipes") + zlib_config = textwrap.dedent(""" + versions: + "1.2.11": + folder: all + """) + zlib = textwrap.dedent(""" + from conan import ConanFile + class Zlib(ConanFile): + name = "zlib" + exports_sources = "*" + """) + save_files(recipes_folder, + {"zlib/config.yml": zlib_config, + "zlib/all/conanfile.py": zlib, + "zlib/all/conandata.yml": "", + "zlib/all/file.h": "//myheader"}) + + c = TestClient(light=True) + c.run(f"remote add local '{folder}'") + c.run("install --requires=zlib/[*] --build=missing") + rev1 = "169da4321a56b77e8538821613a81f1d" + c.assert_listed_require({f"zlib/1.2.11#{rev1}": "Downloaded (local)"}) + + # Modify zlib code + save_files(recipes_folder, {"zlib/all/file.h": "//myheader 222"}) + c.run("install --requires=zlib/[*] --build=missing --update") + rev2 = "8e9fa314a3fd51ab4c930f7e4972f3e7" + c.assert_listed_require({f"zlib/1.2.11#{rev2}": "Updated (local)"}) + + # Modify zlib code again to a new one + save_files(recipes_folder, {"zlib/all/file.h": "//myheader 333"}) + c.run("install --requires=zlib/[*] --update --build=missing") + rev3 = "dd54aebe11b96b3661d8360c68619a72" + c.assert_listed_require({f"zlib/1.2.11#{rev3}": "Updated (local)"}) + + # Revert to previous one + save_files(recipes_folder, {"zlib/all/file.h": "//myheader"}) + c.run("install --requires=zlib/[*] --update --build=missing") + # This crashed https://github.com/conan-io/conan/issues/19313 + c.assert_listed_require({f"zlib/1.2.11#{rev1}": "Updated (local)"}) diff --git a/test/integration/remote/test_remote_recipes_only.py b/test/integration/remote/test_remote_recipes_only.py new file mode 100644 index 00000000000..68256a4aaf1 --- /dev/null +++ b/test/integration/remote/test_remote_recipes_only.py @@ -0,0 +1,84 @@ +import json +import os + +import pytest + +from conan.api.model import PkgReference +from conan.internal.util.files import save +from conan.test.assets.genconanfile import GenConanfile +from conan.test.utils.tools import TestClient + +PKG_ID = "da39a3ee5e6b4b0d3255bfef95601890afd80709" +RREV = "a9ec2e5fbb166568d4670a9cd1ef4b26" + + +@pytest.fixture(scope="module") +def client(): + tc = TestClient(light=True, default_server_user=True) + tc.save({"conanfile.py": GenConanfile("pkg")}) + tc.run("create . --version=1.0") + assert RREV == tc.exported_recipe_revision() + assert PKG_ID == tc.created_package_id("pkg/1.0") + tc.run("upload * -r=default -c") + tc.run("remove * -c") + tc.run("remote update default --recipes-only") + return tc + + +def test_graph_binary_analyze(client): + client.run("install --requires=pkg/1.0 -f=json", assert_error=True) + client.assert_listed_binary({"pkg/1.0": (PKG_ID, "Missing")}) + + client.run("graph explain --requires=pkg/1.0") + assert "ERROR: No package binaries exist" in client.out + + +def test_list_packages(client): + client.run("list pkg/1.0:* -r=default -f=json", redirect_stdout="list.json") + result = json.loads(client.load("list.json")) + assert len(result["default"]["pkg/1.0"]["revisions"][RREV]["packages"]) == 0 + + client.run(f"list pkg/1.0:{PKG_ID} -r=default -f=json", redirect_stdout="list.json") + result = json.loads(client.load("list.json")) + assert "error" in result["default"] # not found error + + +def test_download_package(client): + client.run("download pkg/1.0:* -r=default -f=json", redirect_stdout="download.json") + result = json.loads(client.load("download.json")) + assert len(result["Local Cache"]["pkg/1.0"]["revisions"][RREV]["packages"]) == 0 + + +def test_remove_package(client): + client.run(f"remove pkg/1.0:{PKG_ID} -r=default -c -f=json", redirect_stdout="remove.json") + result = json.loads(client.load("remove.json")) + assert result["default"] == {} + + client.run(f"remove pkg/1.0:* -r=default -c -f=json", redirect_stdout="remove.json") + result = json.loads(client.load("remove.json")) + assert result["default"] == {} + + client.run(f"remove pkg/1.0 -r=default -c -f=json", redirect_stdout="remove.json") + result = json.loads(client.load("remove.json")) + assert "packages" not in result["default"]["pkg/1.0"]["revisions"][RREV] + + +@pytest.mark.parametrize("pattern", ["*", "pkg/2.0", "pkg/2.0:*", f"pkg/2.0:{PKG_ID}"]) +def test_upload_still_works(client, pattern): + client.run("create . --version=2.0") + layout = client.created_layout() + save(os.path.join(layout.metadata(), "metadata.log"), "New hello") + + client.run(f"upload {pattern} -r=default -c") + + # Now check that the binary was in fact uploaded + client.run("remote update default --recipes-only=False") + client.run("remove * -c") + client.run("list pkg/2.0:* -r=default -f=json", redirect_stdout="list.json") + result = json.loads(client.load("list.json")) + assert len(result["default"]["pkg/2.0"]["revisions"][RREV]["packages"]) == 1 + assert PKG_ID in result["default"]["pkg/2.0"]["revisions"][RREV]["packages"] + + client.run(f"download {pattern} -r=default -f=json --metadata=*", redirect_stdout="download.json") + pkg2_layout = client.get_latest_pkg_layout(PkgReference().loads(f"pkg/2.0#{RREV}:{PKG_ID}")) + assert os.path.exists(os.path.join(pkg2_layout.metadata(), "metadata.log")) diff --git a/test/integration/settings/settings_override_test.py b/test/integration/settings/settings_override_test.py index 30cdc1d9e8e..8b0d43a5303 100644 --- a/test/integration/settings/settings_override_test.py +++ b/test/integration/settings/settings_override_test.py @@ -36,10 +36,10 @@ def test_override(client): assert "COMPILER=> visual msvc" in client.out # CHECK CONANINFO FILE - latest_rrev = client.cache.get_latest_recipe_reference( + latest_rrev = client.cache.get_latest_recipe_revision( RecipeReference.loads("mingw/0.1@lasote/testing")) pkg_ids = client.cache.get_package_references(latest_rrev) - latest_prev = client.cache.get_latest_package_reference(pkg_ids[0]) + latest_prev = client.cache.get_latest_package_revision(pkg_ids[0]) package_path = client.cache.pkg_layout(latest_prev).package() conaninfo = load(os.path.join(package_path, CONANINFO)) assert "compiler=gcc" in conaninfo diff --git a/test/integration/symlinks/symlinks_test.py b/test/integration/symlinks/symlinks_test.py index 7bd09064c86..50ee1fdcc2b 100644 --- a/test/integration/symlinks/symlinks_test.py +++ b/test/integration/symlinks/symlinks_test.py @@ -6,7 +6,6 @@ from conan.api.model import PkgReference from conan.api.model import RecipeReference -from conan.internal.paths import PACKAGE_TGZ_NAME from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.test_files import temp_folder from conan.test.utils.tools import TestClient, TestServer @@ -279,6 +278,58 @@ def test_exports_does_not_follow_symlink(): assert not os.path.exists(os.path.join(package_folder, "linked_folder", "source.cpp")) +@pytest.mark.skipif(platform.system() == "Windows", reason="Symlinks not in Windows") +@pytest.mark.parametrize("uppercase", [False, True]) +def test_exports_with_uppercase_symlink_folder(uppercase): + # BaseFolder + # |- Folder + # |- source.cpp + # |- Symlink (Points to "./Folder") + base_dir_name = "BaseFolder" if uppercase else "basefolder" + + client = TestClient(default_server_user=True) + conanfile = GenConanfile("lib", "1.0")\ + .with_package('copy(self, "*", self.source_folder, self.package_folder, ignore_case=False)')\ + .with_exports_sources(base_dir_name + "/*")\ + .with_import("from conan.tools.files import copy") + + folder_dir_name = "Folder" if uppercase else "folder" + folder_dir_path = os.path.join(base_dir_name, folder_dir_name) + + symlink_name = "SymLink" if uppercase else "symlink" + symlink_path = os.path.join(base_dir_name, symlink_name) + + dummy_file_name = "source.cpp" + dummy_file_path = os.path.join(folder_dir_path, dummy_file_name) + + client.save({"conanfile.py": conanfile, + dummy_file_path: "foo"}) + + os.symlink(folder_dir_name, os.path.join(client.current_folder, symlink_path)) + + client.run("create . ") + + exports_sources_folder = client.exported_layout().export_sources() + assert os.path.islink(os.path.join(exports_sources_folder, symlink_path)) + assert os.path.exists(os.path.join(exports_sources_folder, symlink_path, dummy_file_name)) + + # Check files have been copied to the build + build_folder = client.created_layout().build() + assert os.path.islink(os.path.join(build_folder, symlink_path)) + assert os.path.exists(os.path.join(build_folder, symlink_path, dummy_file_name)) + + # Check package files are there + package_folder = client.created_layout().package() + assert os.path.islink(os.path.join(package_folder, symlink_path)) + assert os.path.exists(os.path.join(package_folder, symlink_path, dummy_file_name)) + + # Check that the manifest doesn't contain the symlink to the source.cpp + contents = load(os.path.join(package_folder, "conanmanifest.txt")) + assert symlink_path not in contents + assert dummy_file_path in contents + assert os.path.join(symlink_path, dummy_file_name) not in contents + + @pytest.mark.skipif(platform.system() != "Linux", reason="Only linux") def test_package_symlinks_zero_size(): server = TestServer() @@ -306,7 +357,7 @@ def package(self): client.run("upload * -r=default -c") # We can uncompress it without warns - tgz = os.path.join(p_folder, PACKAGE_TGZ_NAME) + tgz = os.path.join(p_folder, "conan_package.tgz") client.run_command('gzip -d "{}"'.format(tgz)) client.run_command('tar tvf "{}"'.format(os.path.join(p_folder, "conan_package.tar"))) lines = str(client.out).splitlines() diff --git a/test/integration/test_components.py b/test/integration/test_components.py index cbe29f118ab..7dab045c9c9 100644 --- a/test/integration/test_components.py +++ b/test/integration/test_components.py @@ -150,6 +150,7 @@ def package_info(self): c.run("create app") assert "app/0.1: Created package" in c.out + def test_duplication_component_properties(): """ Regression for PR 17503 - component lists would be incorrectly aggregated """ tc = TestClient(light=True) diff --git a/test/integration/test_components_error.py b/test/integration/test_components_error.py index 8d925008951..4b4cb47d9ce 100644 --- a/test/integration/test_components_error.py +++ b/test/integration/test_components_error.py @@ -1,4 +1,5 @@ import os +import re import textwrap import pytest @@ -99,6 +100,30 @@ def package_info(self): "by any '(cpp_info/components).requires'." in t.out +def test_unused_requirement_not_propagated(): + # https://github.com/conan-io/conan/issues/19026 + t = TestClient() + conanfile = textwrap.dedent(f""" + from conan import ConanFile + class Consumer(ConanFile): + name = "pkg" + version = "0.1" + requires = "header/0.1", "lib/0.1" + + def package_info(self): + self.cpp_info.requires = ["lib::lib", "header::header"] + """) + t.save({"header/conanfile.py": GenConanfile("header", "0.1").with_package_type("header-library"), + "lib/conanfile.py": GenConanfile("lib", "0.1").with_package_type("static-library"), + "pkg/conanfile.py": conanfile, + "app/conanfile.py": GenConanfile().with_requires("pkg/0.1")}) + t.run('create header') + t.run('create lib') + t.run('create pkg') + t.run("install app") + assert re.search(r"Skipped binaries(\s*)header/0.1", t.out) + + @pytest.mark.parametrize("component", [True, False]) def test_wrong_requirement(component): """ If we require a wrong requirement, we get a meaninful error. diff --git a/test/integration/test_compressions.py b/test/integration/test_compressions.py new file mode 100644 index 00000000000..07c06a9f902 --- /dev/null +++ b/test/integration/test_compressions.py @@ -0,0 +1,143 @@ +import json +import os +import sys +import textwrap + +import pytest + +from conan.api.model import RecipeReference, PkgReference +from conan.internal.util import load +from conan.test.assets.genconanfile import GenConanfile +from conan.test.utils.tools import TestClient + + +@pytest.mark.parametrize("compress", ["gz", "xz", "zst"]) +def test_xz(compress): + if compress == "zst" and sys.version_info.minor < 14: + pytest.skip("Skipping zst compression tests") + + c = TestClient(default_server_user=True) + c.save_home({"global.conf": f"core.upload:compression_format={compress}"}) + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.files import copy + + class Pkg(ConanFile): + name = "pkg" + version = "0.1" + exports_sources = "*.h" + exports = "*.yml" + def package(self): + copy(self, "*.h", self.source_folder, self.package_folder) + """) + + c.save({"conanfile.py": conanfile, + "header.h": "myheader", + "myfile.yml": "myyml"}) + c.run("create -tf=") + c.run("upload * -r=default -c --format=json") + + # Verify the uploaded files are all txz + upload_json = json.loads(c.stdout) + rrev = upload_json["default"]["pkg/0.1"]["revisions"]["4e81a0b14da7ae918cf3dba3a07578d6"] + rfiles = rrev["files"] + assert f"conan_export.t{compress}" in rfiles + assert f"conan_sources.t{compress}" in rfiles + prevs = rrev["packages"]["da39a3ee5e6b4b0d3255bfef95601890afd80709"]["revisions"] + prev = prevs["13eb72928af98144fa7bf104b69663bc"] + pfiles = prev["files"] + assert f"conan_package.t{compress}" in pfiles + + # decompress should work anyway + c.save_home({"global.conf": ""}) + c.run("remove * -c") + c.run("install --requires=pkg/0.1") + + # checking the recipe + ref = RecipeReference.loads("pkg/0.1") + rlayout = c.get_latest_ref_layout(ref) + downloaded_files = os.listdir(rlayout.download_export()) + assert f"conan_export.t{compress}" in downloaded_files + assert f"conan_sources.t{compress}" not in downloaded_files + assert "myyml" == load(os.path.join(rlayout.export(), "myfile.yml")) + + # checking the package + pref = PkgReference(rlayout.reference, "da39a3ee5e6b4b0d3255bfef95601890afd80709") + playout = c.get_latest_pkg_layout(pref) + downloaded_files = os.listdir(playout.download_package()) + assert f"conan_package.t{compress}" in downloaded_files + assert "myheader" == load(os.path.join(playout.package(), "header.h")) + + # Force the build from source + c.run("install --requires=pkg/0.1 --build=*") + downloaded_files = os.listdir(rlayout.download_export()) + assert f"conan_export.t{compress}" in downloaded_files + assert f"conan_sources.t{compress}" in downloaded_files + + +@pytest.mark.skipif(sys.version_info.minor >= 14, reason="validate zstd error in python<314") +def test_unsupported_zstd(): + c = TestClient(default_server_user=True) + c.save({"conanfile.py": GenConanfile("pkg", "0.1").with_package_file("myfile.h", "contents")}) + c.run("create") + playout = c.created_layout() + c.run("upload * -r=default -c -cc core.upload:compression_format=zst", assert_error=True) + assert "ERROR: The 'core.upload:compression_format=zst' is only for Python>=3.14" in c.out + + # Lets cheat, creating a fake zstd to test download + c.run("upload * -r=default -c --dry-run") + os.rename(os.path.join(playout.download_package(), "conan_package.tgz"), + os.path.join(playout.download_package(), "conan_package.tzst")) + c.run("upload * -r=default -c") + c.run("remove * -c") + c.run("install --requires=pkg/0.1", assert_error=True) + assert ("ERROR: File conan_package.tzst compressed with 'zst', unsupported " + "for Python<3.14") in c.out + + +class TestDuplicatedInServerErrors: + + def test_duplicated_export(self): + c = TestClient(default_server_user=True) + c.save({"conanfile.py": GenConanfile("pkg", "0.1"), + "conandata.yml": ""}) + c.run("export") + c.run("upload * -r=default -c") + c.run("remove * -c") + c.run("export") + c.run("upload * -r=default -c -cc core.upload:compression_format=xz --force") + assert "WARN: risk: The 'xz' compression is experimental" in c.out + + c.run("remove * -c") + c.run("install --requires=pkg/0.1", assert_error=True) + assert ("it contains more than one compressed file: " + "['conan_export.tgz', 'conan_export.txz']") in c.out + + def test_duplicated_source(self): + c = TestClient(default_server_user=True) + c.save({"conanfile.py": GenConanfile("pkg", "0.1").with_exports_sources("*.h"), + "myheader.h": "content"}) + c.run("export") + c.run("upload * -r=default -c") + c.run("remove * -c") + c.run("export") + c.run("upload * -r=default -c -cc core.upload:compression_format=xz --force") + + c.run("remove * -c") + c.run("install --requires=pkg/0.1 --build=missing", assert_error=True) + assert ("it contains more than one compressed file: " + "['conan_sources.tgz', 'conan_sources.txz']") in c.out + + def test_duplicated_package(self): + c = TestClient(default_server_user=True) + c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) + c.run("create") + c.run("upload * -r=default -c") + c.run("remove * -c") + c.run("create") + c.run("upload * -r=default -c -cc core.upload:compression_format=xz --force") + + c.run("remove * -c") + c.run("install --requires=pkg/0.1", assert_error=True) + assert ("it contains more than one compressed file: " + "['conan_package.tgz', 'conan_package.txz']") in c.out diff --git a/test/integration/test_migrations.py b/test/integration/test_migrations.py index 86843da1a16..98e3f8a9b9e 100644 --- a/test/integration/test_migrations.py +++ b/test/integration/test_migrations.py @@ -15,7 +15,7 @@ [("profile.py", "msvc", "EME_ESE_VC"), ("compatibility/compatibility.py", "conanfile", "conian_file")]) def test_migration_profile_checker_plugin(plugin_path, string_replace, new_string): - t = TestClient() + t = TestClient(light=True) # Any command that checks the package cache generates the DB t.run("list") assert os.path.exists(os.path.join(t.cache_folder, "p", "cache.sqlite3")) @@ -41,7 +41,7 @@ def test_migration_profile_checker_plugin(plugin_path, string_replace, new_strin assert new_string not in contents # New client, everything new - t2 = TestClient() + t2 = TestClient(light=True) # This generates the new plugin file t2.run("list") @@ -68,7 +68,7 @@ def test_migration_profile_checker_plugin(plugin_path, string_replace, new_strin def test_back_migrations(): - t = TestClient() + t = TestClient(light=True) # add 3 migrations for number in (1, 2, 3): @@ -97,7 +97,7 @@ def migrate(cache_folder): def test_back_default_compatibility_migration(): - t = TestClient() + t = TestClient(light=True) t.run("-v") # Fire the backward migration migration_file = os.path.join(t.cache_folder, "migrations", "2.4_1-migrate.py") assert os.path.exists(migration_file) @@ -113,7 +113,7 @@ def test_back_default_compatibility_migration(): class TestMigrationCppstdCompat: def test_migration(self): - t = TestClient() + t = TestClient(light=True) t.run("-v") cppstd_compat_path = "extensions/plugins/compatibility/cppstd_compat.py" compatibility_path = "extensions/plugins/compatibility/compatibility.py" @@ -125,7 +125,7 @@ def test_migration(self): assert not os.path.exists(os.path.join(t.cache_folder, cppstd_compat_path)) def test_cppstd_modified(self): - t = TestClient() + t = TestClient(light=True) t.run("-v") cppstd_compat_path = "extensions/plugins/compatibility/cppstd_compat.py" compatibility_path = "extensions/plugins/compatibility/compatibility.py" @@ -139,7 +139,7 @@ def test_cppstd_modified(self): assert "def cppstd_compat(conanfile)" not in t.load_home(compatibility_path) def test_compatibility_modified(self): - t = TestClient() + t = TestClient(light=True) t.run("-v") cppstd_compat_path = "extensions/plugins/compatibility/cppstd_compat.py" compatibility_path = "extensions/plugins/compatibility/compatibility.py" @@ -150,3 +150,26 @@ def test_compatibility_modified(self): assert t.load_home(compatibility_path) == "Modified file" # not Removed because compatibility was modified assert "POTATO" in t.load_home(cppstd_compat_path) + + def test_compatibility_modified_by_conan(self): + t = TestClient(light=True) + t.run("-v") + compatibility_path = "extensions/plugins/compatibility/compatibility.py" + old = t.load_home(compatibility_path) + # In old versions of Conan, assume POTATO was present in the file + new = old + "\n# POTATO" + t.save_home({"version.txt": "2.11", + compatibility_path: new}) + # But now with the newer version, Conan removed it + # so the triggered migration should remove it + t.run("-v") + assert "Successfully updated compatibility.py" in t.out + assert "# POTATO" not in t.load_home(compatibility_path) + + def test_compatibility_no_rewrite(self): + t = TestClient(light=True) + t.run("-v") + # compatibility.py exists and is not modified + t.save_home({"version.txt": "2.11"}) + t.run("-v") + assert "Successfully updated compatibility.py" not in t.out diff --git a/test/integration/test_recipe_policies.py b/test/integration/test_recipe_policies.py new file mode 100644 index 00000000000..6d5708004ba --- /dev/null +++ b/test/integration/test_recipe_policies.py @@ -0,0 +1,53 @@ +from conan.test.assets.genconanfile import GenConanfile +from conan.test.utils.tools import TestClient + + +def test_build_policies_in_conanfile(): + client = TestClient(default_server_user=True, light=True) + base = GenConanfile("hello0", "1.0").with_exports("*") + conanfile = str(base) + "\n build_policy = 'missing'" + client.save({"conanfile.py": conanfile}) + client.run("export . --user=lasote --channel=stable") + + # Install, it will build automatically if missing (without the --build missing option) + client.run("install --requires=hello0/1.0@lasote/stable") + assert "Building" in client.out + + # Try to do it again, now we have the package, so no build is done + client.run("install --requires=hello0/1.0@lasote/stable") + assert "Building" not in client.out + + # Try now to upload all packages, should not crash because of the "missing" build policy + client.run("upload hello0/1.0@lasote/stable -r default") + + # --- Build policy to always --- + conanfile = str(base) + "\n build_policy = 'always'" + client.save({"conanfile.py": conanfile}, clean_first=True) + client.run("export . --user=lasote --channel=stable") + + # Install, it will build automatically if missing (without the --build missing option) + client.run("install --requires=hello0/1.0@lasote/stable", assert_error=True) + assert "ERROR: hello0/1.0@lasote/stable: build_policy='always' has been removed" in client.out + + +def test_build_policy_missing(): + c = TestClient(default_server_user=True, light=True) + conanfile = GenConanfile("pkg", "1.0").with_class_attribute('build_policy = "missing"')\ + .with_class_attribute('upload_policy = "skip"') + c.save({"conanfile.py": conanfile}) + c.run("export .") + + # the --build=never has higher priority + c.run("install --requires=pkg/1.0@ --build=never", assert_error=True) + assert "ERROR: Missing prebuilt package for 'pkg/1.0'" in c.out + + c.run("install --requires=pkg/1.0@") + assert "pkg/1.0: Building package from source as defined by build_policy='missing'" in c.out + + # If binary already there it should do nothing + c.run("install --requires=pkg/1.0@") + assert "pkg/1.0: Building package from source" not in c.out + + c.run("upload * -r=default -c") + assert "Uploading package" not in c.out + assert "pkg/1.0: Skipping upload of binaries, because upload_policy='skip'" in c.out diff --git a/test/integration/tgz_macos_dot_files_test.py b/test/integration/tgz_macos_dot_files_test.py index b05700ae0b7..44340726c8f 100644 --- a/test/integration/tgz_macos_dot_files_test.py +++ b/test/integration/tgz_macos_dot_files_test.py @@ -9,14 +9,14 @@ from conan.internal.rest.remote_manager import uncompress_file from conan.api.model import RecipeReference -from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME from conan.test.utils.tools import TestClient, NO_SETTINGS_PACKAGE_ID @pytest.mark.skipif(platform.system() != "Darwin", reason="Requires OSX") class TestTgzMacosDotFiles: - def _test_for_metadata_in_zip_file(self, tgz, annotated_file, dot_file_expected): + @staticmethod + def _test_for_metadata_in_zip_file(tgz, annotated_file, dot_file_expected): tmp_folder = tempfile.mkdtemp() try: uncompress_file(src_path=tgz, dest_folder=tmp_folder) @@ -93,7 +93,7 @@ def _add_macos_metadata_to_file(filepath): # 3) In the upload process, the metadata is lost again export_download_folder = t.get_latest_ref_layout(pref.ref).download_export() - tgz = os.path.join(export_download_folder, EXPORT_SOURCES_TGZ_NAME) + tgz = os.path.join(export_download_folder, "conan_sources.tgz") assert not os.path.exists(tgz) t.run("upload lib/version@user/channel -r default --only-recipe") self._test_for_metadata_in_zip_file(tgz, 'file.txt', dot_file_expected=False) diff --git a/test/integration/toolchains/apple/test_xcodedeps.py b/test/integration/toolchains/apple/test_xcodedeps.py index 070b5918d79..8cadf9f89a9 100644 --- a/test/integration/toolchains/apple/test_xcodedeps.py +++ b/test/integration/toolchains/apple/test_xcodedeps.py @@ -582,3 +582,62 @@ def package_info(self): conan_uses_xcconfig = client.load("conan_uses_components_uses_components.xcconfig") assert '#include "conan_has_components_first.xcconfig"' in conan_uses_xcconfig assert '#include "conan_has_components_second.xcconfig"' not in conan_uses_xcconfig + + +def test_dont_add_skipped_xcconfigs_when_required_by_components(): + client = TestClient() + regular_lib = textwrap.dedent(""" + from conan import ConanFile + class PkgWithComponents(ConanFile): + name = 'regular_lib' + version = '1.0' + settings = 'os', 'compiler', 'arch', 'build_type' + def requirements(self): + self.requires('header_skip/1.0') + self.requires('header_transitive/1.0', transitive_headers=True) + def package_info(self): + self.cpp_info.components['component'].requires = ['header_skip::header_skip', + 'header_transitive::header_transitive'] + """) + + header_transitive = textwrap.dedent(""" + from conan import ConanFile + class PkgUsesComponent(ConanFile): + name = 'header_transitive' + version = '1.0' + settings = 'os', 'compiler', 'arch', 'build_type' + package_type = 'header-library' + def package_info(self): + self.cpp_info.includedirs = ["include"] + """) + + header_skip = textwrap.dedent(""" + from conan import ConanFile + class PkgUsesComponent(ConanFile): + name = 'header_skip' + version = '1.0' + settings = 'os', 'compiler', 'arch', 'build_type' + package_type = 'header-library' + def package_info(self): + self.cpp_info.includedirs = ["include"] + """) + + client.save({"header_transitive.py": header_transitive, + "header_skip.py": header_skip, + "regular_lib.py": regular_lib}) + client.run("create header_transitive.py") + client.run("create header_skip.py") + client.run("create regular_lib.py") + client.run("install --requires=regular_lib/1.0 -g XcodeDeps") + + conandeps = client.load("conan_regular_lib_component.xcconfig") + assert '#include "conan_header_skip.xcconfig"' not in conandeps + assert '#include "conan_header_transitive.xcconfig"' in conandeps + + # Verify that header_skip xcconfig files are NOT generated (skipped dependency) + skip_files = [f for f in os.listdir(client.current_folder) if 'header_skip' in f and f.endswith('.xcconfig')] + assert len(skip_files) == 0, f"Header skip files should not be generated: {skip_files}" + + # Verify that header_transitive xcconfig files ARE generated (transitive dependency) + transitive_files = [f for f in os.listdir(client.current_folder) if 'header_transitive' in f and f.endswith('.xcconfig')] + assert len(transitive_files) > 0, f"Header transitive files should be generated: {transitive_files}" diff --git a/test/integration/toolchains/cmake/cmakedeps/test_cmakedeps.py b/test/integration/toolchains/cmake/cmakedeps/test_cmakedeps.py index 457300b91bd..539f2876909 100644 --- a/test/integration/toolchains/cmake/cmakedeps/test_cmakedeps.py +++ b/test/integration/toolchains/cmake/cmakedeps/test_cmakedeps.py @@ -922,3 +922,34 @@ def generate(self): assert "add_library(component_alias" in targets_data assert "add_library(dep::my_aliased_component" in targets_data + + +def test_package_info_extra_variables(): + """ Test extra_variables property - This just shows that it works, + there are tests for cmaketoolchain that check the actual behavior + of parsing the variables""" + client = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + + class Pkg(ConanFile): + name = "pkg" + version = "0.1" + + def package_info(self): + self.cpp_info.set_property("cmake_extra_variables", {"FOO": 42, + "BAR": 42, + "CMAKE_GENERATOR_INSTANCE": "${GENERATOR_INSTANCE}/buildTools/", + "CACHE_VAR_DEFAULT_DOC": {"value": "hello world", + "cache": True, "type": "PATH"}}) + """) + client.save({"conanfile.py": conanfile}) + client.run("create .") + + client.run("install --requires=pkg/0.1 -g CMakeDeps " + """-c tools.cmake.cmaketoolchain:extra_variables="{'BAR': 9}" """) + target = client.load("pkg-config.cmake") + assert 'set(BAR' not in target + assert 'set(CMAKE_GENERATOR_INSTANCE "${GENERATOR_INSTANCE}/buildTools/")' in target + assert 'set(FOO 42)' in target + assert 'set(CACHE_VAR_DEFAULT_DOC "hello world" CACHE PATH "CACHE_VAR_DEFAULT_DOC")' in target diff --git a/test/integration/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_frameworks.py b/test/integration/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_frameworks.py index ebbd2853314..f21e110c256 100644 --- a/test/integration/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_frameworks.py +++ b/test/integration/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_frameworks.py @@ -42,3 +42,27 @@ def requirements(self): }) client.run(f"create . -c tools.cmake.cmakedeps:new={new_value}", assert_error=True) assert "Error in generator 'CMakeConfigDeps': cpp_info.location missing for framework MyFramework" in client.out + + +def test_framework_only_component_generates_target(): + conanfile = textwrap.dedent(f""" + import os + from conan import ConanFile + + class MyFramework(ConanFile): + name = "frame" + version = "1.0" + settings = "os", "arch", "compiler", "build_type" + + def package_info(self): + self.cpp_info.frameworks = ["MyFramework"] + self.cpp_info.libdirs = [] + self.cpp_info.includedirs = [] + """) + + tc = TestClient() + tc.save({"conanfile.py": conanfile}) + tc.run("create .") + tc.run(f"install --requires=frame/1.0 -g CMakeConfigDeps -c=tools.cmake.cmakedeps:new={new_value}") + targets = tc.load("frame-Targets-release.cmake") + assert "add_library(frame::frame INTERFACE IMPORTED)" in targets diff --git a/test/integration/toolchains/cmake/cmakedeps2/test_cmakedeps.py b/test/integration/toolchains/cmake/cmakedeps2/test_cmakedeps.py index 0f329d469de..aa0d54f807e 100644 --- a/test/integration/toolchains/cmake/cmakedeps2/test_cmakedeps.py +++ b/test/integration/toolchains/cmake/cmakedeps2/test_cmakedeps.py @@ -34,6 +34,7 @@ class PkgConan(ConanFile): c.save({"conanfile.py": conanfile}, clean_first=True) c.run(f"install . -c tools.cmake.cmakedeps:new={new_value}") cmake_paths = c.load("conan_cmakedeps_paths.cmake") + assert "set(CMAKE_FIND_PACKAGE_PREFER_CONFIG ON)" in cmake_paths assert re.search(r"list\(PREPEND CMAKE_PROGRAM_PATH \".*/bin\"", cmake_paths) # default assert re.search(r"list\(PREPEND CMAKE_LIBRARY_PATH \".*/mylib\"", cmake_paths) assert re.search(r"list\(PREPEND CMAKE_INCLUDE_PATH \".*/myincludes\"", cmake_paths) @@ -144,11 +145,12 @@ class PkgConan(ConanFile): liba_config = c.load("liba-config.cmake") assert ('include("${CMAKE_CURRENT_LIST_DIR}/full_deploy/' 'host/liba/1.0/share/cmake/crypto.cmake")') in liba_config + assert ('set(liba_INCLUDE_DIRS "${CMAKE_CURRENT_LIST_DIR}/full_deploy/' + 'host/liba/1.0/includea" )') in liba_config liba_targets = c.load("liba-Targets-release.cmake") assert ('set(liba_PACKAGE_FOLDER_RELEASE "${CMAKE_CURRENT_LIST_DIR}/full_deploy/' 'host/liba/1.0")') in liba_targets - assert ('set(liba_INCLUDE_DIRS "${CMAKE_CURRENT_LIST_DIR}/full_deploy/' - 'host/liba/1.0/includea" )') in liba_targets + def test_cmakeconfigdeps_recipe(): c = TestClient() @@ -165,16 +167,8 @@ def generate(self): c.save({"dep/conanfile.py": GenConanfile("dep", "0.1"), "app/conanfile.py": conanfile}) c.run("create dep") - c.run("install app", assert_error=True) - assert "CMakeConfigDeps is being used in conanfile, but the conf " \ - "'tools.cmake.cmakedeps:new' is not enabled" in c.out - c.run("install app -c tools.cmake.cmakedeps:new=will_break_next") - # will not fail, still warn - assert "WARN: Using the new CMakeConfigDeps generator" in c.out - # The only-recipe also not fails - c.run("install app -c tools.cmake.cmakedeps:new=recipe_will_break") - # will not fail - assert "WARN: Using the new CMakeConfigDeps generator" in c.out + c.run("install app") + assert "WARN: experimental: CMakeConfigDeps is experimental" in c.out # attribute generator conanfile = textwrap.dedent(""" @@ -186,13 +180,8 @@ class TestConan(ConanFile): generators = "CMakeConfigDeps" """) c.save({"app/conanfile.py": conanfile}, clean_first=True) - c.run("install app", assert_error=True) - assert "CMakeConfigDeps is being used in conanfile, but the conf " \ - "'tools.cmake.cmakedeps:new' is not enabled" in c.out - c.run("install app -c tools.cmake.cmakedeps:new=will_break_next") - assert "WARN: Using the new CMakeConfigDeps generator" in c.out - c.run("install app -c tools.cmake.cmakedeps:new=recipe_will_break") - assert "WARN: Using the new CMakeConfigDeps generator" in c.out + c.run("install app") + assert "WARN: experimental: CMakeConfigDeps is experimental" in c.out # conanfile.txt conanfile = textwrap.dedent(""" @@ -202,13 +191,8 @@ class TestConan(ConanFile): CMakeConfigDeps """) c.save({"app/conanfile.txt": conanfile}, clean_first=True) - c.run("install app", assert_error=True) - assert "CMakeConfigDeps is being used in conanfile, but the conf " \ - "'tools.cmake.cmakedeps:new' is not enabled" in c.out - c.run("install app -c tools.cmake.cmakedeps:new=will_break_next") - assert "WARN: Using the new CMakeConfigDeps generator" in c.out - c.run("install app -c tools.cmake.cmakedeps:new=recipe_will_break") - assert "WARN: Using the new CMakeConfigDeps generator" in c.out + c.run("install app") + assert "WARN: experimental: CMakeConfigDeps is experimental" in c.out def test_system_wrappers(): @@ -257,6 +241,7 @@ def package_info(self): assert "CMakeConfigDeps: cmake_set_interface_link_directories deprecated and invalid. " \ "The package 'package_info()' must correctly define the (CPS) information" in c.out + def test_consuming_cpp_info_with_components_dependency_from_same_package(): c = TestClient() conanfile = textwrap.dedent(""" @@ -360,7 +345,7 @@ def test(self): """) c.save({"dependent/conanfile.py": dependent_conanfile, "main/conanfile.py": conanfile, - "main/test_package/conanfile.py":test_package}) + "main/test_package/conanfile.py": test_package}) c.run("create ./dependent/ --name=dependent --version=0.1 " f"-c tools.cmake.cmakedeps:new={new_value}") c.run(f"create ./main/ --name=pkg --version=0.1 -c tools.cmake.cmakedeps:new={new_value}") @@ -385,103 +370,185 @@ def package_info(self): assert "CMakeConfigDeps does not support module find mode" -def test_requires_to_application(): - c = TestClient() - automake = textwrap.dedent(""" - from conan import ConanFile - class Dependent(ConanFile): - name = "automake" - version = "0.1" - package_type = "application" - """) - - conanfile = textwrap.dedent(""" +def test_cmake_extra_dependencies(): + tc = TestClient() + dep = textwrap.dedent(""" from conan import ConanFile class Pkg(ConanFile): - name = "libtool" - version = "0.1" - package_type = "static-library" - - def requirements(self): - self.requires('automake/0.1') - """) - test_package = textwrap.dedent(""" - from conan import ConanFile - from conan.tools.cmake import CMake - - class TestPkg(ConanFile): - settings = "os", "compiler", "arch", "build_type" - generators = "CMakeDeps", "CMakeToolchain" - - def requirements(self): - self.requires(self.tested_reference_str) - - def test(self): - pass - """) - - c.save({"automake/conanfile.py": automake, - "libtool/conanfile.py": conanfile, - "libtool/test_package/conanfile.py":test_package}) - c.run("create automake") - c.run(f"create libtool -c tools.cmake.cmakedeps:new={new_value}") - targets = c.load("libtool/test_package/libtool-Targets-release.cmake") - # The libtool shouldn't depend on the automake::automake target - assert "automake::automake" not in targets - - -def test_requires_to_application_component(): - c = TestClient() - automake = textwrap.dedent(""" - from conan import ConanFile - class Dependent(ConanFile): - name = "automake" + name = "dep" version = "0.1" - package_type = "application" - def package_info(self): - self.cpp_info.components["myapp"].exe = "myapp" - self.cpp_info.components["myapp"].location = "path/to/myapp" - self.cpp_info.components["mylibapp"].type = "header-library" + self.cpp_info.set_property("cmake_extra_dependencies", ["MyOpenMPI"]) + self.cpp_info.set_property("cmake_extra_interface_libs", ["MyOpenMPILib"]) """) + tc.save({"conanfile.py": dep}) + tc.run("create .") + args = f"-g CMakeDeps -c tools.cmake.cmakedeps:new={new_value}" + tc.run(f"install --requires=dep/0.1 {args}") + dep = tc.load("dep-Targets-release.cmake") + assert "find_dependency(MyOpenMPI REQUIRED )" in dep + assert "set_property(TARGET dep::dep APPEND PROPERTY INTERFACE_LINK_LIBRARIES\n" \ + " $<$:MyOpenMPILib>)" in dep - conanfile = textwrap.dedent(""" + +def test_cmake_extra_dependencies_components(): + tc = TestClient() + dep = textwrap.dedent(""" from conan import ConanFile class Pkg(ConanFile): - name = "libtool" + name = "dep" version = "0.1" - package_type = "static-library" - - def requirements(self): - self.requires('automake/0.1') def package_info(self): - self.cpp_info.requires = ["automake::mylibapp"] - """) - test_package = textwrap.dedent(""" - from conan import ConanFile - from conan.tools.cmake import CMake - - class TestPkg(ConanFile): - settings = "os", "compiler", "arch", "build_type" - generators = "CMakeDeps", "CMakeToolchain" - - def requirements(self): - self.requires(self.tested_reference_str) - - def test(self): - pass + self.cpp_info.set_property("cmake_extra_dependencies", ["MyOpenMPI"]) + self.cpp_info.components["mycomp"].set_property("cmake_extra_interface_libs", + ["MyOpenMPILib"]) + self.cpp_info.components["mycomp"].libs = ["mycomplib"] + self.cpp_info.components["mycomp"].type = "static-library" + self.cpp_info.components["mycomp"].location = "lib/mycomp.a" """) - - c.save({"automake/conanfile.py": automake, - "libtool/conanfile.py": conanfile, - "libtool/test_package/conanfile.py":test_package}) - c.run("create automake") - c.run(f"create libtool -c tools.cmake.cmakedeps:new={new_value}") - targets = c.load("libtool/test_package/libtool-Targets-release.cmake") - # The libtool shouldn't depend on the automake::automake target - assert "automake::automake" not in targets - assert "# Requirement automake::mylibapp => Full link: True" in targets - assert "$<$:automake::mylibapp>" in targets + tc.save({"conanfile.py": dep}) + tc.run("create .") + args = f"-g CMakeDeps -c tools.cmake.cmakedeps:new={new_value}" + tc.run(f"install --requires=dep/0.1 {args}") + dep = tc.load("dep-Targets-release.cmake") + assert "find_dependency(MyOpenMPI REQUIRED )" in dep + assert "set_property(TARGET dep::mycomp APPEND PROPERTY INTERFACE_LINK_LIBRARIES\n" \ + " $<$:MyOpenMPILib>)" in dep + + +class TestRequiresToApp: + def test_requires_to_application(self): + c = TestClient() + automake = GenConanfile("automake", "0.1").with_package_type("application") + conanfile = (GenConanfile("libtool", "0.1").with_package_type("static-library") + .with_requirement("automake/0.1")) + test_package = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.cmake import CMake + + class TestPkg(ConanFile): + settings = "os", "compiler", "arch", "build_type" + generators = "CMakeDeps", "CMakeToolchain" + + def requirements(self): + self.requires(self.tested_reference_str) + + def test(self): + pass + """) + + c.save({"automake/conanfile.py": automake, + "libtool/conanfile.py": conanfile, + "libtool/test_package/conanfile.py": test_package}) + c.run("create automake") + c.run(f"create libtool -c tools.cmake.cmakedeps:new={new_value}") + targets = c.load("libtool/test_package/libtool-Targets-release.cmake") + # The libtool shouldn't depend on the automake::automake target + assert "automake::automake" not in targets + + def test_requires_to_application_component(self): + c = TestClient() + automake = textwrap.dedent(""" + from conan import ConanFile + class Dependent(ConanFile): + name = "automake" + version = "0.1" + package_type = "application" + + def package_info(self): + self.cpp_info.components["myapp"].exe = "myapp" + self.cpp_info.components["myapp"].location = "path/to/myapp" + self.cpp_info.components["mylibapp"].type = "header-library" + """) + + conanfile = textwrap.dedent(""" + from conan import ConanFile + class Pkg(ConanFile): + name = "libtool" + version = "0.1" + package_type = "static-library" + + def requirements(self): + self.requires('automake/0.1') + def package_info(self): + self.cpp_info.requires = ["automake::mylibapp"] + """) + + c.save({"automake/conanfile.py": automake, + "libtool/conanfile.py": conanfile}) + c.run("create automake") + + c.run("create libtool") + c.run("install --requires=libtool/0.1 -g CMakeDeps " + f"-c tools.cmake.cmakedeps:new={new_value}") + targets = c.load("libtool-Targets-release.cmake") + # The libtool shouldn't depend on the automake::automake target + assert "automake::automake" not in targets + assert "# Requirement automake::mylibapp => Full link: True" in targets + assert "$<$:automake::mylibapp>" in targets + + def test_requires_from_library_component(self): + c = TestClient() + automake = GenConanfile("automake", "0.1").with_package_type("application") + conanfile = textwrap.dedent(""" + from conan import ConanFile + class Pkg(ConanFile): + name = "libtool" + version = "0.1" + package_type = "static-library" + + def requirements(self): + self.requires('automake/0.1') + def package_info(self): + self.cpp_info.components["mycomp"].requires = ["automake::automake"] + """) + + c.save({"automake/conanfile.py": automake, + "libtool/conanfile.py": conanfile}) + c.run("create automake") + c.run("create libtool") + c.run("install --requires=libtool/0.1 -g CMakeDeps " + f"-c tools.cmake.cmakedeps:new={new_value}") + targets = c.load("libtool-Targets-release.cmake") + # The libtool shouldn't depend on the automake::automake target + assert "automake::automake" not in targets + + def test_requires_from_library_component_to_app_component(self): + c = TestClient() + automake = textwrap.dedent(""" + from conan import ConanFile + class Dependent(ConanFile): + name = "automake" + version = "0.1" + + def package_info(self): + self.cpp_info.components["myapp"].exe = "myapp" + self.cpp_info.components["myapp"].location = "path/to/myapp" + self.cpp_info.components["mylibapp"].type = "header-library" + """) + conanfile = textwrap.dedent(""" + from conan import ConanFile + class Pkg(ConanFile): + name = "libtool" + version = "0.1" + package_type = "static-library" + + def requirements(self): + self.requires('automake/0.1') + def package_info(self): + self.cpp_info.components["mycomp"].requires = ["automake::myapp"] + """) + + c.save({"automake/conanfile.py": automake, + "libtool/conanfile.py": conanfile}) + c.run("create automake") + c.run("create libtool") + c.run("install --requires=libtool/0.1 -g CMakeDeps " + f"-c tools.cmake.cmakedeps:new={new_value}") + targets = c.load("libtool-Targets-release.cmake") + # The libtool shouldn't depend on the automake::automake target + assert "automake::myapp" not in targets + assert "automake::automake" not in targets def test_alias_cmakedeps_set_property(): @@ -494,7 +561,8 @@ class Dep(ConanFile): version = "1.0" settings = "os", "compiler", "build_type", "arch" def package_info(self): - self.cpp_info.components["mycomp"].set_property("cmake_target_name", "dep::mycomponent") + self.cpp_info.components["mycomp"].set_property("cmake_target_name", + "dep::mycomponent") """), "conanfile.py": textwrap.dedent(""" from conan import ConanFile @@ -509,7 +577,8 @@ class Pkg(ConanFile): def generate(self): deps = CMakeDeps(self) deps.set_property("dep", "cmake_target_aliases", ["alias", "dep::other_name"]) - deps.set_property("dep::mycomp", "cmake_target_aliases", ["component_alias", "dep::my_aliased_component"]) + deps.set_property("dep::mycomp", "cmake_target_aliases", + ["component_alias", "dep::my_aliased_component"]) deps.generate() """)}) tc.run("create dep") @@ -521,3 +590,59 @@ def generate(self): assert "add_library(component_alias" in targets_data assert "add_library(dep::my_aliased_component" in targets_data + + +def test_package_info_extra_variables(): + """ Test extra_variables property - This just shows that it works, + there are tests for cmaketoolchain that check the actual behavior + of parsing the variables""" + client = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + + class Pkg(ConanFile): + name = "pkg" + version = "0.1" + + def package_info(self): + self.cpp_info.set_property("cmake_extra_variables", {"FOO": 42, + "BAR": 42, + "CMAKE_GENERATOR_INSTANCE": + "${GENERATOR_INSTANCE}/buildTools/", + "CACHE_VAR_DEFAULT_DOC": {"value": "hello world", + "cache": True, "type": "PATH"}}) + """) + client.save({"conanfile.py": conanfile}) + client.run("create .") + + client.run(f"install --requires=pkg/0.1 -g CMakeDeps -c tools.cmake.cmakedeps:new={new_value} " + """-c tools.cmake.cmaketoolchain:extra_variables="{'BAR': 9}" """) + target = client.load("pkg-config.cmake") + assert 'set(BAR' not in target + assert 'set(CMAKE_GENERATOR_INSTANCE "${GENERATOR_INSTANCE}/buildTools/")' in target + assert 'set(FOO 42)' in target + assert 'set(CACHE_VAR_DEFAULT_DOC "hello world" CACHE PATH "CACHE_VAR_DEFAULT_DOC")' in target + + +def test_target_defines_only(): + client = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + + class Pkg(ConanFile): + name = "pkg" + version = "0.1" + + def package_info(self): + self.cpp_info.components["base"].includedirs = [] + self.cpp_info.components["base"].defines = ["FOO=1"] + self.cpp_info.components["comp"].includedirs = ["include"] + self.cpp_info.components["comp"].requires = ["base"] + """) + client.save({"conanfile.py": conanfile}) + client.run("create .") + + client.run(f"install --requires=pkg/0.1 -g CMakeDeps -c tools.cmake.cmakedeps:new={new_value}") + target = client.load("pkg-Targets-release.cmake") + assert 'add_library(pkg::base INTERFACE IMPORTED)' in target + assert "# Requirement pkg::base => Full link: True" in target diff --git a/test/integration/toolchains/cmake/test_cmaketoolchain.py b/test/integration/toolchains/cmake/test_cmaketoolchain.py index d61f84574b8..6a424eac5e5 100644 --- a/test/integration/toolchains/cmake/test_cmaketoolchain.py +++ b/test/integration/toolchains/cmake/test_cmaketoolchain.py @@ -5,7 +5,7 @@ import textwrap import pytest -from mock import mock +from unittest import mock from conan.tools.cmake.presets import load_cmake_presets from conan.test.assets.genconanfile import GenConanfile @@ -523,6 +523,31 @@ def test_extra_flags_via_conf(): assert 'add_compile_definitions( "D1" "D2")' in toolchain +def test_bitcode_enable_flag(): + profile = textwrap.dedent(""" + [settings] + os=Macos + arch=armv8 + compiler=apple-clang + compiler.version=17 + compiler.libcxx=libc++ + build_type=Release + + [conf] + tools.apple:enable_bitcode=True + """) + + client = TestClient(path_with_spaces=False) + + conanfile = GenConanfile().with_settings("os", "arch", "compiler", "build_type") \ + .with_generator("CMakeToolchain") + client.save({"conanfile.py": conanfile, + "profile": profile}) + client.run("install . --profile:build=profile --profile:host=profile") + toolchain = client.load("conan_toolchain.cmake") + assert 'set(BITCODE "-fembed-bitcode")' in toolchain + + def test_cmake_presets_binary_dir_available(): client = TestClient(path_with_spaces=False) conanfile = textwrap.dedent(""" @@ -1371,6 +1396,11 @@ def test_presets_njobs(): c.run('install . -g CMakeToolchain -c tools.build:jobs=42') presets = json.loads(c.load("CMakePresets.json")) assert presets["buildPresets"][0]["jobs"] == 42 + assert presets["testPresets"][0]["execution"]["jobs"] == 42 + c.run('install . -g CMakeToolchain -c tools.build:jobs=0') + presets = json.loads(c.load("CMakePresets.json")) + assert "jobs" not in presets["buildPresets"][0] + assert "execution" not in presets["testPresets"][0] def test_add_cmakeexe_to_presets(): diff --git a/test/integration/toolchains/env/test_buildenv.py b/test/integration/toolchains/env/test_buildenv.py index 181c3c373fd..3ade358a5df 100644 --- a/test/integration/toolchains/env/test_buildenv.py +++ b/test/integration/toolchains/env/test_buildenv.py @@ -35,3 +35,43 @@ def build(self): client.save({"conanfile.py": conanfile, "build_profile": build_profile}) client.run("create . -pr:b=build_profile") assert "Python" in client.out + + +def test_quoted_vars(): + # https://github.com/conan-io/conan/issues/18761 + c = TestClient() + + dep = textwrap.dedent(""" + from conan import ConanFile + class Pkg(ConanFile): + name = "dep" + version = "0.1" + settings = "os" + def package_info(self): + v = 'MyVar:"content!! | other!!"' + v = v.replace("|", "^|") if self.settings.os == "Windows" else v # escape + self.buildenv_info.define("MyCustomEscapedVar", v) + """) + + conanfile = textwrap.dedent(""" + import os, platform + from conan import ConanFile + + class ConanFileToolsTest(ConanFile): + settings = "os" + requires = "dep/0.1" + def build(self): + if platform.system() == "Windows": + self.run("set MyCustomEscapedVar") + else: + self.run("printenv MyCustomEscapedVar") + self.run("echo Hello!!!!") + """) + + c.save({"dep/conanfile.py": dep, + "consumer/conanfile.py": conanfile}) + + c.run("create dep") + c.run("build consumer") + assert 'MyVar:"content!! | other!!"' in c.out + assert "Hello!!!!" in c.out diff --git a/test/integration/toolchains/gnu/test_autotoolstoolchain.py b/test/integration/toolchains/gnu/test_autotoolstoolchain.py index 7da8302b162..4f3f2e26ff2 100644 --- a/test/integration/toolchains/gnu/test_autotoolstoolchain.py +++ b/test/integration/toolchains/gnu/test_autotoolstoolchain.py @@ -369,12 +369,13 @@ def test_conf_build_does_not_exist(): assert 'export CC_FOR_BUILD="x86_64-linux-gnu-gcc"' in tc assert 'export CXX_FOR_BUILD="x86_64-linux-gnu-g++"' in tc + @pytest.mark.parametrize( "threads, flags", [("posix", "-pthread"), ("wasm_workers", "-sWASM_WORKERS=1")], ) def test_thread_flags(threads, flags): - os = platform.system() + is_win = platform.system() == "Windows" client = TestClient() profile = textwrap.dedent(f""" [settings] @@ -396,8 +397,8 @@ def test_thread_flags(threads, flags): } ) client.run("install . -pr=./profile") - toolchain = client.load("conanautotoolstoolchain{}".format('.bat' if os == "Windows" else '.sh')) - if os == "Windows": + toolchain = client.load("conanautotoolstoolchain{}".format('.bat' if is_win else '.sh')) + if is_win: assert f'set "CXXFLAGS=%CXXFLAGS% -stdlib=libc++ {flags}"' in toolchain assert f'set "CFLAGS=%CFLAGS% {flags}"' in toolchain assert f'set "LDFLAGS=%LDFLAGS% {flags}' in toolchain diff --git a/test/integration/toolchains/gnu/test_basic_layout.py b/test/integration/toolchains/gnu/test_basic_layout.py index 5dc589c9dc9..d99a4672cba 100644 --- a/test/integration/toolchains/gnu/test_basic_layout.py +++ b/test/integration/toolchains/gnu/test_basic_layout.py @@ -3,8 +3,10 @@ import textwrap import pytest +from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient + @pytest.mark.parametrize("basic_layout, expected_path", [ ('basic_layout(self)', 'build-release'), ('basic_layout(self, build_folder="custom_build_folder")', 'custom_build_folder')]) @@ -26,3 +28,58 @@ def layout(self): ext = "sh" if platform.system() != "Windows" else "bat" assert os.path.isfile(os.path.join(c.current_folder, "pkg", expected_path, "conan", "conanautotoolstoolchain.{}".format(ext))) + + +def test_editable_includes(): + c = TestClient() + dep = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.layout import basic_layout + + class Dep(ConanFile): + name = "dep" + version = "0.1" + def layout(self): + basic_layout(self, src_folder="src") + """) + c.save({"dep/conanfile.py": dep, + "pkg/conanfile.py": GenConanfile("pkg", "0.1").with_settings("build_type", "arch") + .with_requires("dep/0.1")}) + c.run("editable add dep") + c.run("install pkg -s arch=x86_64 -g CMakeDeps -g PkgConfigDeps -g XcodeDeps") + data = c.load(f"pkg/dep-release-x86_64-data.cmake") + assert 'set(dep_INCLUDE_DIRS_RELEASE "${dep_PACKAGE_FOLDER_RELEASE}/src/include")' in data + pc = c.load("pkg/dep.pc") + assert "includedir=${prefix}/src/include" in pc + xcode = c.load("pkg/conan_dep_dep_release_x86_64.xcconfig") + dep_path = os.path.join(c.current_folder, "dep", "src", "include") + assert f"SYSTEM_HEADER_SEARCH_PATHS_dep_dep[config=Release][arch=x86_64][sdk=*] = \"{dep_path}\"" in xcode + + +def test_editable_includes_previously_defined(): + # if someone already defined self.cpp.source.includedirs make sure we don't overwrite + # their value + c = TestClient() + dep = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.layout import basic_layout + + class Dep(ConanFile): + name = "dep" + version = "0.1" + def layout(self): + self.cpp.source.includedirs = ["somefolder"] + basic_layout(self, src_folder="src") + """) + c.save({"dep/conanfile.py": dep, + "pkg/conanfile.py": GenConanfile("pkg", "0.1").with_settings("build_type", "arch") + .with_requires("dep/0.1")}) + c.run("editable add dep") + c.run("install pkg -s arch=x86_64 -g CMakeDeps -g PkgConfigDeps -g XcodeDeps") + data = c.load(f"pkg/dep-release-x86_64-data.cmake") + assert 'set(dep_INCLUDE_DIRS_RELEASE "${dep_PACKAGE_FOLDER_RELEASE}/src/somefolder")' in data + pc = c.load("pkg/dep.pc") + assert "includedir=${prefix}/src/somefolder" in pc + xcode = c.load("pkg/conan_dep_dep_release_x86_64.xcconfig") + dep_path = os.path.join(c.current_folder, "dep", "src", "somefolder") + assert f"SYSTEM_HEADER_SEARCH_PATHS_dep_dep[config=Release][arch=x86_64][sdk=*] = \"{dep_path}\"" in xcode diff --git a/test/integration/toolchains/gnu/test_gnutoolchain.py b/test/integration/toolchains/gnu/test_gnutoolchain.py index 5294123ed57..acea79bee59 100644 --- a/test/integration/toolchains/gnu/test_gnutoolchain.py +++ b/test/integration/toolchains/gnu/test_gnutoolchain.py @@ -620,11 +620,11 @@ def test_thread_flags(threads, flags): } ) client.run("install . -pr=./profile") - os = platform.system() + is_win = platform.system() == "Windows" toolchain = client.load( - "conangnutoolchain{}".format('.bat' if os == "Windows" else '.sh')) + "conangnutoolchain{}".format('.bat' if is_win else '.sh')) - if os == "Windows": + if is_win: assert f'set "CXXFLAGS=%CXXFLAGS% -stdlib=libc++ {flags}"' in toolchain assert f'set "CFLAGS=%CFLAGS% {flags}"' in toolchain assert f'set "LDFLAGS=%LDFLAGS% {flags}' in toolchain diff --git a/test/integration/toolchains/gnu/test_makedeps.py b/test/integration/toolchains/gnu/test_makedeps.py index 253998aad80..060c70ca554 100644 --- a/test/integration/toolchains/gnu/test_makedeps.py +++ b/test/integration/toolchains/gnu/test_makedeps.py @@ -416,7 +416,6 @@ def package_info(self): ("sysroot = ['/foo/bar/sysroot']", True, 'CONAN_SYSROOT_PACKAGE = /foo/bar/sysroot') if platform.system() != "Windows" else ("sysroot = ['C:/my_sysroot']", True, 'CONAN_SYSROOT_PACKAGE = C:/my_sysroot')]) - def test_makefile_sysroot(pattern, result, expected): """ The MakeDeps should not enforce sysroot in case not defined diff --git a/test/integration/toolchains/google/test_bazeldeps.py b/test/integration/toolchains/google/test_bazeldeps.py index d243b34cddf..533c60eaf46 100644 --- a/test/integration/toolchains/google/test_bazeldeps.py +++ b/test/integration/toolchains/google/test_bazeldeps.py @@ -1,7 +1,11 @@ import os import pathlib +import platform +import re import textwrap +import pytest + from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient from conan.tools.files import load @@ -64,7 +68,9 @@ def layout(self): name = "dep", hdrs = glob([ "include/**", - ]), + ], + allow_empty = True + ), includes = [ "include", ], @@ -165,7 +171,9 @@ def package_info(self): name = "dep", hdrs = glob([ "include/**", - ]), + ], + allow_empty = True + ), includes = [ "include", ], @@ -270,7 +278,9 @@ def package_info(self): name = "third", hdrs = glob([ "include/**", - ]), + ], + allow_empty = True + ), includes = [ "include", ], @@ -288,7 +298,9 @@ def package_info(self): name = "second-mycomponent", hdrs = glob([ "include/**", - ]), + ], + allow_empty = True + ), includes = [ "include", ], @@ -303,7 +315,9 @@ def package_info(self): name = "second-myfirstcomp", hdrs = glob([ "include/**", - ]), + ], + allow_empty = True + ), includes = [ "include", ], @@ -319,7 +333,9 @@ def package_info(self): name = "second", hdrs = glob([ "include/**", - ]), + ], + allow_empty = True + ), includes = [ "include", ], @@ -346,7 +362,9 @@ def package_info(self): name = "myfirstlib-cmp1", hdrs = glob([ "include/**", - ]), + ], + allow_empty = True + ), includes = [ "include", ], @@ -362,7 +380,9 @@ def package_info(self): name = "myfirstlib", hdrs = glob([ "include/**", - ]), + ], + allow_empty = True + ), includes = [ "include", ], @@ -448,7 +468,9 @@ def package_info(self): name = "pkg", hdrs = glob([ "include/**", - ]), + ], + allow_empty = True + ), includes = [ "include", ], @@ -479,7 +501,9 @@ def package_info(self): name = "component1", hdrs = glob([ "include/**", - ]), + ], + allow_empty = True + ), includes = [ "include", ], @@ -494,7 +518,9 @@ def package_info(self): name = "fancy_name-cmp2", hdrs = glob([ "include/**", - ]), + ], + allow_empty = True + ), includes = [ "include", ], @@ -509,7 +535,9 @@ def package_info(self): name = "component3", hdrs = glob([ "include/**", - ]), + ], + allow_empty = True + ), includes = [ "include", ], @@ -525,7 +553,9 @@ def package_info(self): name = "fancy_name", hdrs = glob([ "include/**", - ]), + ], + allow_empty = True + ), includes = [ "include", ], @@ -588,7 +618,9 @@ def build_requirements(self): name = "{0}", hdrs = glob([ "include/**", - ]), + ], + allow_empty = True + ), includes = [ "include", ], @@ -664,7 +696,9 @@ def _load_dependencies_impl(mctx): name = "dep", hdrs = glob([ "include/**", - ]), + ], + allow_empty = True + ), includes = [ "include", ], @@ -762,7 +796,9 @@ def generate(self): name = "component1", hdrs = glob([ "include/**", - ]), + ], + allow_empty = True + ), includes = [ "include", ], @@ -777,7 +813,9 @@ def generate(self): name = "libother-cmp2", hdrs = glob([ "include/**", - ]), + ], + allow_empty = True + ), includes = [ "include", ], @@ -792,7 +830,9 @@ def generate(self): name = "component3", hdrs = glob([ "include/**", - ]), + ], + allow_empty = True + ), includes = [ "include", ], @@ -808,7 +848,9 @@ def generate(self): name = "libother", hdrs = glob([ "include/**", - ]), + ], + allow_empty = True + ), includes = [ "include", ], @@ -1051,7 +1093,6 @@ def package_info(self): assert "Install finished successfully" in c.out # the asserts in build() didn't fail - def test_shared_windows_find_libraries(): """ Testing the ``_get_libs`` mechanism in Windows, the shared libraries and their @@ -1157,8 +1198,8 @@ def package_info(self): "zlib/conanfile.py": zlib, "openssl/conanfile.py": openssl, "libcurl/conanfile.py": libcurl, - "libiconv/conanfile.py": libiconv, - }) + "libiconv/conanfile.py": libiconv + }) c.run("export-pkg zlib -o:a shared=True") c.run("export-pkg openssl -o:a shared=True") c.run("export-pkg libcurl -o:a shared=True") @@ -1245,7 +1286,9 @@ def package_info(self): name = "mylib-myfirstcomp", hdrs = glob([ "include/**", - ]), + ], + allow_empty = True + ), includes = [ "include", ], @@ -1257,3 +1300,95 @@ def package_info(self): ) """) assert myfirstcomp_expected in build_content + + +@pytest.mark.skipif(platform.system() == "Windows", reason="Unix paths only") +def test_apple_frameworks_and_frameworkdirs(): + """ + Testing that Apple frameworks are included as linkopts + Issue: https://github.com/conan-io/conan/issues/18748 + """ + client = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + + class PkgConfigConan(ConanFile): + def package_info(self): + self.cpp_info.frameworks = ["CoreFoundation", "Cocoa"] + self.cpp_info.frameworkdirs = ["/my/path/to/frw1"] + + """) + client.save({"conanfile.py": conanfile}, clean_first=True) + client.run("create . --name=mylib --version=0.1") + client.save({"conanfile.py": GenConanfile("pkg", "0.1").with_require("mylib/0.1")}, + clean_first=True) + client.run("install . -g BazelDeps") + build_content = load(None, os.path.join(client.current_folder, "mylib", "BUILD.bazel")) + build_file_expected = textwrap.dedent("""\ + cc_library( + name = "mylib", + hdrs = glob([ + "include/**", + ], + allow_empty = True + ), + includes = [ + "include", + ], + linkopts = [ + "-framework", + "CoreFoundation", + "-framework", + "Cocoa", + "-F", + "/my/path/to/frw1", + ], + visibility = ["//visibility:public"], + ) + """) + assert build_file_expected in build_content + + +def test_shared_libs_and_unix_includes_rpath(): + """ + Testing the RPATH flag is added to the BUILD.bazel file if these conditions + are given: shared library and UNIX systems. + + Issue: https://github.com/conan-io/conan/issues/19190 + Issue: https://github.com/conan-io/conan/issues/19135 + """ + client = TestClient() + csm = textwrap.dedent(""" + import os + from conan import ConanFile + from conan.tools.files import save + class Pkg(ConanFile): + name = "csm" + version = "1.0" + settings = "os" + options = {"shared": [True, False]} + default_options = {"shared": False} + def package(self): + if self.settings.os == "Windows": + save(self, os.path.join(self.package_folder, "bin", "csmapi.dll"), "") + save(self, os.path.join(self.package_folder, "lib", "csmapi.lib"), "") + else: + save(self, os.path.join(self.package_folder, "lib", "libcsmapi.so"), "") + def package_info(self): + self.cpp_info.libs = ["csmapi"] + """) + consumer = textwrap.dedent(""" + [requires] + csm/1.0 + [options] + *:shared=True + """) + client.save({"conanfile.txt": consumer, "csm/conanfile.py": csm}) + client.run("export-pkg csm -o '*:shared=True' -s 'os=Windows'") + client.run("export-pkg csm -o '*:shared=True' -s 'os=Linux'") + client.run("install . -g BazelDeps -s 'os=Windows'") + build_content = load(None, os.path.join(client.current_folder, "csm", "BUILD.bazel")) + assert '"-Wl,-rpath,' not in build_content + client.run("install . -g BazelDeps -s 'os=Linux'") + build_content = load(None, os.path.join(client.current_folder, "csm", "BUILD.bazel")) + assert re.search('"-Wl,-rpath,.*/p/lib"', build_content.replace("\\", "/")) diff --git a/test/integration/toolchains/intel/test_intel_cc.py b/test/integration/toolchains/intel/test_intel_cc.py index 795ee1c094a..5b155437d2e 100644 --- a/test/integration/toolchains/intel/test_intel_cc.py +++ b/test/integration/toolchains/intel/test_intel_cc.py @@ -1,3 +1,4 @@ +import os import platform import textwrap @@ -55,3 +56,16 @@ def test_intel_cc_generator_linux(): conanintelsetvars = get_intel_cc_generator_file(os_, installation_path, "conanintelsetvars.sh") expected = '. "/opt/intel/oneapi/setvars.sh" intel64' assert conanintelsetvars == expected + + +def test_avoid_generation(): + # The empty string for tools.intel:installation_path avoids the generation + profile = intelprofile % ("Linux", "") + client = TestClient() + client.save({ + "conanfile.txt": conanfile, + "intelprofile": profile, + }) + client.run("install . -pr intelprofile") + files = os.listdir(client.current_folder) + assert "conanintelsetvars" not in ",".join(files) diff --git a/test/integration/toolchains/meson/test_mesontoolchain.py b/test/integration/toolchains/meson/test_mesontoolchain.py index 41328438e31..dda6e051f6c 100644 --- a/test/integration/toolchains/meson/test_mesontoolchain.py +++ b/test/integration/toolchains/meson/test_mesontoolchain.py @@ -230,7 +230,6 @@ def generate(self): assert re.search(r"cpp_link_args =.+-mmy-flag.+", content) - def test_linker_scripts_via_conf(): profile = textwrap.dedent(""" [settings] @@ -256,8 +255,10 @@ def test_linker_scripts_via_conf(): t.run("install . -pr:b=profile -pr=profile") content = t.load(MesonToolchain.native_filename) - assert "c_link_args = ['-flag0', '-other=val', '-m64', '-flag5', '-flag6', '-T\"/linker/scripts/flash.ld\"', '-T\"/linker/scripts/extra_data.ld\"']" in content - assert "cpp_link_args = ['-flag0', '-other=val', '-m64', '-flag5', '-flag6', '-T\"/linker/scripts/flash.ld\"', '-T\"/linker/scripts/extra_data.ld\"']" in content + assert ("c_link_args = ['-flag0', '-other=val', '-m64', '-flag5', '-flag6', " + "'-T/linker/scripts/flash.ld', '-T/linker/scripts/extra_data.ld']") in content + assert ("cpp_link_args = ['-flag0', '-other=val', '-m64', '-flag5', '-flag6', " + "'-T/linker/scripts/flash.ld', '-T/linker/scripts/extra_data.ld']") in content def test_correct_quotes(): @@ -689,8 +690,8 @@ def test_compiler_path_with_spaces(): def test_meson_sysroot_app(): """Testing when users pass tools.build:sysroot on the profile with Meson - The generated conan_meson_cross.ini needs to contain both sys_root property to fill the - PKG_CONFIG_PATH and the compiler flags with --sysroot. + * The generated conan_meson_cross.ini does not fill the "sys_root" property (see https://github.com/conan-io/conan/issues/16468) + * It adds the compiler flags with --sysroot. When cross-building, Meson needs both compiler_executables in the config, otherwise it will fail when running setup. @@ -727,7 +728,7 @@ def test_meson_sysroot_app(): client.run("install . -pr:h host -pr:b build") # Check the meson configuration file conan_meson = client.load("conan_meson_cross.ini") - assert f"sys_root = '{sysroot}'\n" in conan_meson + assert f"sys_root = '{sysroot}'\n" not in conan_meson assert re.search(r"c_args =.+--sysroot={}.+".format(sysroot), conan_meson) assert re.search(r"c_link_args =.+--sysroot={}.+".format(sysroot), conan_meson) assert re.search(r"cpp_args =.+--sysroot={}.+".format(sysroot), conan_meson) @@ -824,6 +825,7 @@ def test_conf_extra_apple_flags(): for flags in ["objcpp_args", "objc_args"]: assert f"{flags} = ['-fno-objc-arc', '-m64', '-fvisibility=hidden', '-fvisibility-inlines-hidden']" in tc + @pytest.mark.parametrize( "threads, flags", [("posix", "-pthread"), ("wasm_workers", "-sWASM_WORKERS=1")], @@ -902,3 +904,46 @@ def generate(self): backend = 'xcode' """) assert expected in content + + +def test_needs_exe_wrapper(): + """ + Tests needs_exe_wrapper depends on `can_run()` function instead of + simply checking the cross_building() one. + + Issue: https://github.com/conan-io/conan/issues/19217 + """ + host = textwrap.dedent(""" + [settings] + arch=x86_64 + build_type=Release + compiler=apple-clang + compiler.cppstd=gnu17 + compiler.libcxx=libc++ + compiler.version=16 + os=Macos + [conf] + tools.apple:sdk_path=/my/sdk/path + tools.build.cross_building:can_run=True + """) + build = textwrap.dedent(""" + [settings] + arch=armv8 + build_type=Release + compiler=apple-clang + compiler.cppstd=gnu17 + compiler.libcxx=libc++ + compiler.version=16 + os=Macos + """) + client = TestClient() + client.save({ + "host": host, + "build": build, + "conanfile.py": GenConanfile("pkg", "1.0") + .with_settings("os", "arch", "compiler", "build_type") + .with_generator("MesonToolchain") + }) + client.run("install . -pr:h host -pr:b build") + content = client.load(MesonToolchain.cross_filename) + assert "needs_exe_wrapper = false" in content diff --git a/test/integration/toolchains/microsoft/vcvars_test.py b/test/integration/toolchains/microsoft/vcvars_test.py index d6137847193..73c23cc5498 100644 --- a/test/integration/toolchains/microsoft/vcvars_test.py +++ b/test/integration/toolchains/microsoft/vcvars_test.py @@ -29,7 +29,7 @@ def generate(self): '-s compiler.cppstd=14 -s compiler.runtime=static') assert os.path.exists(os.path.join(client.current_folder, "conanvcvars.bat")) - + assert r"VC\Auxiliary\Build\vcvarsall.bat" in client.load("conanvcvars.bat") bat_contents = client.load("conanbuild.bat") if scope in ("build", None): assert "conanvcvars.bat" in bat_contents @@ -58,9 +58,7 @@ def test_vcvars_generator_skip_on_linux(): Skip creation of conanvcvars.bat on Linux build systems """ client = TestClient() - client.save({"conanfile.py": GenConanfile().with_generator("VCVars") - .with_settings("os", "compiler", - "arch", "build_type")}) + client.save({"conanfile.txt": "[generators]\nVCVars"}) client.run('install . -s os=Windows -s compiler=msvc -s compiler.version=193 ' '-s compiler.runtime=dynamic') @@ -70,14 +68,7 @@ def test_vcvars_generator_skip_on_linux(): @pytest.mark.skipif(platform.system() not in ["Windows"], reason="Requires Windows") def test_vcvars_generator_string(): client = TestClient(path_with_spaces=False) - - conanfile = textwrap.dedent(""" - from conan import ConanFile - class TestConan(ConanFile): - generators = "VCVars" - settings = "os", "compiler", "arch", "build_type" - """) - client.save({"conanfile.py": conanfile}) + client.save({"conanfile.txt": "[generators]\nVCVars"}) client.run('install . -s os=Windows -s compiler="msvc" -s compiler.version=191 ' '-s compiler.cppstd=14 -s compiler.runtime=static') @@ -89,14 +80,7 @@ class TestConan(ConanFile): def test_vcvars_2015_error(): # https://github.com/conan-io/conan/issues/9888 client = TestClient(path_with_spaces=False) - - conanfile = textwrap.dedent(""" - from conan import ConanFile - class TestConan(ConanFile): - generators = "VCVars" - settings = "os", "compiler", "arch", "build_type" - """) - client.save({"conanfile.py": conanfile}) + client.save({"conanfile.txt": "[generators]\nVCVars"}) client.run('install . -s os=Windows -s compiler="msvc" -s compiler.version=190 ' '-s compiler.cppstd=14 -s compiler.runtime=static') @@ -109,14 +93,7 @@ class TestConan(ConanFile): def test_vcvars_platform_x86(): # https://github.com/conan-io/conan/issues/11144 client = TestClient(path_with_spaces=False) - - conanfile = textwrap.dedent(""" - from conan import ConanFile - class TestConan(ConanFile): - generators = "VCVars" - settings = "os", "compiler", "arch", "build_type" - """) - client.save({"conanfile.py": conanfile}) + client.save({"conanfile.txt": "[generators]\nVCVars"}) client.run('install . -s os=Windows -s compiler="msvc" -s compiler.version=193 ' '-s compiler.cppstd=14 -s compiler.runtime=static -s:b arch=x86') @@ -127,14 +104,7 @@ class TestConan(ConanFile): @pytest.mark.skipif(platform.system() != "Windows", reason="Requires Windows") def test_vcvars_winsdk_version(): client = TestClient(path_with_spaces=False) - - conanfile = textwrap.dedent(""" - from conan import ConanFile - class TestConan(ConanFile): - generators = "VCVars" - settings = "os", "compiler", "arch", "build_type" - """) - client.save({"conanfile.py": conanfile}) + client.save({"conanfile.txt": "[generators]\nVCVars"}) client.run('install . -s os=Windows -s compiler=msvc -s compiler.version=193 ' '-s compiler.cppstd=14 -s compiler.runtime=static ' '-c tools.microsoft:winsdk_version=10.0') @@ -146,14 +116,7 @@ class TestConan(ConanFile): @pytest.mark.skipif(platform.system() != "Windows", reason="Requires Windows") def test_vcvars_compiler_update(): client = TestClient(path_with_spaces=False) - - conanfile = textwrap.dedent(""" - from conan import ConanFile - class TestConan(ConanFile): - generators = "VCVars" - settings = "os", "compiler", "arch", "build_type" - """) - client.save({"conanfile.py": conanfile}) + client.save({"conanfile.txt": "[generators]\nVCVars"}) client.run('install . -s os=Windows -s compiler=msvc -s compiler.version=193 ' '-s compiler.cppstd=14 -s compiler.runtime=static ' '-s compiler.update=3') @@ -161,20 +124,28 @@ class TestConan(ConanFile): vcvars = client.load("conanvcvars.bat") assert 'vcvarsall.bat" amd64 -vcvars_ver=14.33' in vcvars + @pytest.mark.skipif(platform.system() != "Windows", reason="Requires Windows") def test_vcvars_armv8_windows_store(): client = TestClient(path_with_spaces=False) - - conanfile = textwrap.dedent(""" - from conan import ConanFile - class TestConan(ConanFile): - generators = "VCVars" - settings = "os", "compiler", "arch", "build_type" - """) - client.save({"conanfile.py": conanfile}) + client.save({"conanfile.txt": "[generators]\nVCVars"}) client.run('install . -s:b os=Windows -s compiler="msvc" -s compiler.version=194 ' '-s compiler.cppstd=14 -s compiler.runtime=static -s:h arch=armv8 ' '-s:h os=WindowsStore -s:h os.version=10.0') vcvars = client.load("conanvcvars.bat") assert 'vcvarsall.bat" amd64_arm64' in vcvars + + +@pytest.mark.skipif(platform.system() != "Windows", reason="Requires Windows") +def test_vcvars_clang_visual2026(): + client = TestClient(path_with_spaces=False) + client.save({"conanfile.txt": "[generators]\nVCVars"}) + client.run('install . -s:b os=Windows -s compiler=clang -s compiler.version=20 ' + '-s compiler.cppstd=14 -s compiler.runtime=static -s arch=x86_64 ' + '-s compiler.runtime_version=v145 ' + # Using a known existing path to avoid auto-detection via vswhere + '-c tools.microsoft.msbuild:installation_path=C:/') + + vcvars = client.load("conanvcvars.bat") + assert '-vcvars_ver=14.5' in vcvars diff --git a/test/integration/toolchains/premake/test_premake.py b/test/integration/toolchains/premake/test_premake.py index ae6b2fea8c7..fcffb951c80 100644 --- a/test/integration/toolchains/premake/test_premake.py +++ b/test/integration/toolchains/premake/test_premake.py @@ -138,4 +138,34 @@ def build(self): tc.save({"conanfile.py": conanfile, "win": windows_profile}) tc.run("build . -pr win") assert 'conanfile.premake5.lua" vs2022 --arch=x86_64!!' in tc.out - assert 'Running msbuild.exe "App.sln" -p:Configuration="Release" -p:Platform=Win64!!' in tc.out + assert 'Running msbuild.exe "App.sln" -p:Configuration="Release" -p:Platform="Win64"!!' in tc.out + + +def test_premake_build_with_custom_configuration(): + tc = TestClient() + conanfile = textwrap.dedent( + """ + from conan import ConanFile + from conan.tools.premake import Premake, PremakeDeps, PremakeToolchain + + class Pkg(ConanFile): + settings = "os", "compiler", "build_type", "arch" + generators = "PremakeToolchain" + def run(self, cmd, *args, **kwargs): + self.output.info(f"Running {cmd}!!") + def generate(self): + deps = PremakeDeps(self) + deps.configuration = "ReleaseDLL" + def build(self): + premake = Premake(self) + premake.configure() + premake.build(workspace="App", configuration="ReleaseDLL") + """ + ) + tc.save({"conanfile.py": conanfile}) + tc.run( + "build . -s compiler=gcc -s compiler.version=13 -s compiler.libcxx=libstdc++ -s arch=armv8 -c tools.build:jobs=4", + ) + assert 'conanfile.premake5.lua" gmake --arch=arm64!!' in tc.out + assert "Running make config=releasedll all -j4!!" in tc.out + diff --git a/test/integration/toolchains/premake/test_premakedeps.py b/test/integration/toolchains/premake/test_premakedeps.py index 4e05f0c065e..436a0208700 100644 --- a/test/integration/toolchains/premake/test_premakedeps.py +++ b/test/integration/toolchains/premake/test_premakedeps.py @@ -137,11 +137,13 @@ def test_premakedeps_link_order(): contents = client.load("consumer/conanconfig_debug_x86_64.premake5.lua") assert 't_conan_deps_order["debug_x86_64"] = {"libc", "libb", "liba"}' in contents + @pytest.mark.parametrize("transitive_headers", [True, False]) @pytest.mark.parametrize("transitive_libs", [True, False]) @pytest.mark.parametrize("brotli_package_type", ["unknown", "static-library", "shared-library"]) @pytest.mark.parametrize("lib_package_type", ["unknown", "static-library", "shared-library"]) -def test_premakedeps_traits(transitive_headers, transitive_libs, brotli_package_type, lib_package_type): +def test_premakedeps_traits(transitive_headers, transitive_libs, brotli_package_type, + lib_package_type): client = TestClient() profile = textwrap.dedent( """ diff --git a/test/integration/toolchains/test_raise_on_universal_binaries.py b/test/integration/toolchains/test_raise_on_universal_binaries.py index b71e1bc0767..6c1f0bfead5 100644 --- a/test/integration/toolchains/test_raise_on_universal_binaries.py +++ b/test/integration/toolchains/test_raise_on_universal_binaries.py @@ -1,15 +1,103 @@ -from parameterized import parameterized +import platform +import textwrap + +import pytest from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient -@parameterized.expand(["AutotoolsToolchain", "MesonToolchain", "BazelToolchain"]) +@pytest.mark.parametrize("toolchain", ["MesonToolchain", "BazelToolchain"]) def test_create_universal_binary(toolchain): client = TestClient() - conanfile = (GenConanfile().with_settings("os", "arch", "compiler", "build_type").with_generator(toolchain)) + conanfile = (GenConanfile().with_settings("os", "arch", "compiler", "build_type") + .with_generator(toolchain)) client.save({"conanfile.py": conanfile}) client.run('create . --name=foo --version=1.0 -s="arch=armv8|armv8.3|x86_64"', assert_error=True) - assert f"Error in generator '{toolchain}': Universal binaries not supported by toolchain." in client.out + assert (f"Error in generator '{toolchain}': " + f"Universal binaries not supported by toolchain.") in client.out + + +@pytest.mark.parametrize("toolchain", ["AutotoolsToolchain", "GnuToolchain"]) +@pytest.mark.skipif(platform.system() != "Darwin", reason="Only OSX") +def test_toolchain_universal_binary_support(toolchain): + """Test that toolchain now supports universal binaries on macOS""" + client = TestClient() + conanfile = (GenConanfile().with_settings("os", "arch", "compiler", "build_type") + .with_generator(toolchain)) + client.save({"conanfile.py": conanfile}) + + client.run('install . --name=foo --version=1.0 -s="arch=armv8|x86_64"') + + script_name = f"conan{toolchain.lower()}.sh" + toolchain_content = client.load(script_name) + assert "-arch arm64" in toolchain_content + assert "-arch x86_64" in toolchain_content + # Verify isysroot flag is NOT present when sdk_path is not configured + assert "-isysroot" not in toolchain_content + + +@pytest.mark.parametrize("toolchain", ["AutotoolsToolchain", "GnuToolchain"]) +def test_toolchain_universal_binary_non_macos(toolchain): + """Test that toolchain still raises error for universal binaries on non-macOS""" + client = TestClient() + conanfile = (GenConanfile().with_settings("os", "arch", "compiler", "build_type") + .with_generator(toolchain)) + client.save({"conanfile.py": conanfile}) + + # This should still raise an error on non-macOS platforms + client.run('create . --name=foo --version=1.0 -s="os=Linux" -s="arch=armv8|x86_64"', + assert_error=True) + assert "Universal arch 'armv8|x86_64' is only supported in Apple OSes" in client.out + + +@pytest.mark.parametrize("toolchain", ["AutotoolsToolchain", "GnuToolchain"]) +@pytest.mark.skipif(platform.system() != "Darwin", reason="Only OSX") +def test_toolchain_universal_binary_with_sdk_path(toolchain): + """Test that toolchain sets isysroot when sdk_path is configured for universal binaries""" + client = TestClient() + conanfile = (GenConanfile().with_settings("os", "arch", "compiler", "build_type") + .with_generator(toolchain)) + client.save({"conanfile.py": conanfile}) + + client.run('install . --name=foo --version=1.0 -s="arch=armv8|x86_64" -c="tools.apple:sdk_path=mysdkpath"') + + script_name = f"conan{toolchain.lower()}.sh" + toolchain_content = client.load(script_name) + + assert "-arch arm64" in toolchain_content + assert "-arch x86_64" in toolchain_content + + assert "-isysroot" in toolchain_content + assert "mysdkpath" in toolchain_content + + +def test_create_universal_binary_test_package_folder(): + # https://github.com/conan-io/conan/issues/18820 + # While multi-arch is Darwin specific, this was a cmake_layout issue, so it can be + # tested in any platform + c = TestClient() + test_conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.cmake import cmake_layout + + class mylibraryTestConan(ConanFile): + settings = "os", "compiler", "build_type", "arch" + + def requirements(self): + self.requires(self.tested_reference_str) + + def layout(self): + cmake_layout(self) + + def test(self): + pass + """) + c.save({"conanfile.py": GenConanfile("pkg", "0.1").with_settings("arch"), + "test_package/conanfile.py": test_conanfile}) + + c.run('create . -s="arch=armv8|x86_64"') + c.run("list *:*") + assert "arch: armv8|x86_64" in c.out diff --git a/test/integration/tools/ros/test_rosenv.py b/test/integration/tools/ros/test_rosenv.py index 987558edc1e..d35fead8173 100644 --- a/test/integration/tools/ros/test_rosenv.py +++ b/test/integration/tools/ros/test_rosenv.py @@ -8,10 +8,10 @@ from conan.test.utils.tools import TestClient -@pytest.mark.skipif(platform.system() == "Windows", reason="Uses UNIX commands") def test_rosenv(): """ - Test that the ROSEnv generator generates the environment files and that the environment variables are correctly set + Test that the ROSEnv generator generates the environment files and that + the environment variables are correctly set. """ client = TestClient() conanfile3 = textwrap.dedent(''' @@ -25,18 +25,37 @@ def test_rosenv(): "conanfile3.txt": conanfile3 }) + if platform.system() == "Windows": + conanrosenv_file = "conanrosenv.bat" + conanrosenv_build = "conanrosenv-build.bat" + build_type = "Release" + cmd_source = "call" + cmd_env = "set" + else: + conanrosenv_file = "conanrosenv.sh" + conanrosenv_build = "conanrosenv-build.sh" + build_type = '"Release"' + cmd_source = "." + cmd_env = "env" + + client.run("install conanfile3.txt --output-folder install/conan") - assert "Generated ROSEnv Conan file: conanrosenv.sh" in client.out + assert f"Generated ROSEnv Conan file: {conanrosenv_file}" in client.out install_folder = os.path.join(client.current_folder, "install", "conan") - conanrosenv_path = os.path.join(install_folder, "conanrosenv.sh") + conanrosenv_path = os.path.join(install_folder, conanrosenv_file) assert os.path.exists(conanrosenv_path) - conanrosenvbuild_path = os.path.join(install_folder, "conanrosenv-build.sh") + conanrosenvbuild_path = os.path.join(install_folder, conanrosenv_build) assert os.path.exists(conanrosenvbuild_path) - toolchain_path = os.path.join(client.current_folder, "install", "conan", "conan_toolchain.cmake") + toolchain_path = os.path.join(client.current_folder, "install", "conan", + "conan_toolchain.cmake") content = client.load(conanrosenvbuild_path) - assert f'CMAKE_TOOLCHAIN_FILE="{toolchain_path}"' in content - assert 'CMAKE_BUILD_TYPE="Release"' in content - client.run_command(f'. "{conanrosenv_path}" && env') + if platform.system() != "Windows": + assert f'CMAKE_TOOLCHAIN_FILE="{toolchain_path}"' in content + else: + assert f'CMAKE_TOOLCHAIN_FILE={toolchain_path}' in content + assert f'CMAKE_BUILD_TYPE={build_type}' in content + + client.run_command(f'{cmd_source} "{conanrosenv_path}" && {cmd_env}') assert f"CMAKE_TOOLCHAIN_FILE={toolchain_path}" in client.out assert "CMAKE_BUILD_TYPE=Release" in client.out diff --git a/test/integration/tools/system/package_manager_test.py b/test/integration/tools/system/package_manager_test.py index 1309469b987..78cc333cde3 100644 --- a/test/integration/tools/system/package_manager_test.py +++ b/test/integration/tools/system/package_manager_test.py @@ -1,7 +1,7 @@ import platform from unittest.mock import MagicMock, patch -import mock +from unittest import mock import pytest from unittest.mock import PropertyMock @@ -12,15 +12,15 @@ from conan.test.utils.mocks import ConanFileMock, MockSettings -@pytest.mark.parametrize("platform, tool", [ +@pytest.mark.parametrize("system, tool", [ ("Linux", "apt-get"), ("Windows", "choco"), ("Darwin", "brew"), ("Solaris", "pkgutil"), ]) @pytest.mark.skipif(platform.system() != "Linux", reason="Only linux") -def test_package_manager_platform(platform, tool): - with mock.patch("platform.system", return_value=platform): +def test_package_manager_platform(system, tool): + with mock.patch("platform.system", return_value=system): with mock.patch("distro.id", return_value=''): with mock.patch('conan.ConanFile.context', new_callable=PropertyMock) as context_mock: context_mock.return_value = "host" @@ -246,6 +246,47 @@ def fake_check(*args, **kwargs): assert tool._conanfile.command == result +@pytest.mark.parametrize("tool_class, arch_host, result", [ + # Install host package and not cross-compile -> do not add host architecture + (Apk, 'x86_64', 'apk add --no-cache package1=0.1 package2=0.2'), + (Apt, 'x86_64', 'apt-get install -y --no-install-recommends package1=0.1 package2=0.2'), + (Yum, 'x86_64', 'yum install -y package1-0.1 package2-0.2'), + (Dnf, 'x86_64', 'dnf install -y package1-0.1 package2-0.2'), + (Brew, 'x86_64', 'brew install package1@0.1 package2@0.2'), + (Pkg, 'x86_64', 'pkg install -y package1-0.1 package2-0.2'), + (PkgUtil, 'x86_64', 'pkgutil --install --yes package1@0.1 package2@0.2'), + (Chocolatey, 'x86_64', 'choco install --yes package1 --version 0.1 package2 --version 0.2'), + (PacMan, 'x86_64', 'pacman -S --noconfirm package1 package2'), + (Zypper, 'x86_64', 'zypper --non-interactive in package1=0.1 package2=0.2'), + # Install host package and cross-compile -> add host architecture + (Apt, 'x86', 'apt-get install -y --no-install-recommends package1:i386=0.1 package2:i386=0.2'), + (Yum, 'x86', 'yum install -y package1-0.1.i?86 package2-0.2.i?86'), + (Dnf, 'x86', 'dnf install -y package1-0.1.i?86 package2-0.2.i?86'), + (Brew, 'x86', 'brew install package1@0.1 package2@0.2'), + (Pkg, 'x86', 'pkg install -y package1-0.1 package2-0.2'), + (PkgUtil, 'x86', 'pkgutil --install --yes package1@0.1 package2@0.2'), + (Chocolatey, 'x86', 'choco install --yes package1 --version 0.1 package2 --version 0.2'), + (PacMan, 'x86', 'pacman -S --noconfirm package1-lib32 package2-lib32'), + (Zypper, 'x86', 'zypper --non-interactive in package1=0.1 package2=0.2'), +]) +def test_tools_install_mode_install_different_archs_with_version(tool_class, arch_host, result): + conanfile = ConanFileMock() + conanfile.settings = MockSettings({"arch": arch_host}) + conanfile.settings_build = MockSettings({"arch": "x86_64"}) + conanfile.conf.define("tools.system.package_manager:tool", tool_class.tool_name) + conanfile.conf.define("tools.system.package_manager:mode", "install") + with mock.patch('conan.ConanFile.context', new_callable=PropertyMock) as context_mock: + context_mock.return_value = "host" + tool = tool_class(conanfile) + + def fake_check(*args, **kwargs): + return ["package1=0.1", "package2=0.2"] + from conan.tools.system.package_manager import _SystemPackageManagerTool + with patch.object(_SystemPackageManagerTool, 'check', MagicMock(side_effect=fake_check)): + tool.install(["package1=0.1", "package2=0.2"]) + assert tool._conanfile.command == result + + @pytest.mark.parametrize("tool_class, arch_host, result", [ # Install build machine package and not cross-compile -> do not add host architecture (Apk, 'x86_64', 'apk add --no-cache package1 package2'), @@ -288,6 +329,48 @@ def fake_check(*args, **kwargs): assert tool._conanfile.command == result +@pytest.mark.parametrize("tool_class, arch_host, result", [ + # Install build machine package and not cross-compile -> do not add host architecture + (Apk, 'x86_64', 'apk add --no-cache package1=0.1 package2=0.2'), + (Apt, 'x86_64', 'apt-get install -y --no-install-recommends package1=0.1 package2=0.2'), + (Yum, 'x86_64', 'yum install -y package1-0.1 package2-0.2'), + (Dnf, 'x86_64', 'dnf install -y package1-0.1 package2-0.2'), + (Brew, 'x86_64', 'brew install package1@0.1 package2@0.2'), + (Pkg, 'x86_64', 'pkg install -y package1-0.1 package2-0.2'), + (PkgUtil, 'x86_64', 'pkgutil --install --yes package1@0.1 package2@0.2'), + (Chocolatey, 'x86_64', 'choco install --yes package1 --version 0.1 package2 --version 0.2'), + (PacMan, 'x86_64', 'pacman -S --noconfirm package1 package2'), + (Zypper, 'x86_64', 'zypper --non-interactive in package1=0.1 package2=0.2'), + # Install build machine package and cross-compile -> do not add host architecture + (Apt, 'x86', 'apt-get install -y --no-install-recommends package1=0.1 package2=0.2'), + (Yum, 'x86', 'yum install -y package1-0.1 package2-0.2'), + (Dnf, 'x86', 'dnf install -y package1-0.1 package2-0.2'), + (Brew, 'x86', 'brew install package1@0.1 package2@0.2'), + (Pkg, 'x86', 'pkg install -y package1-0.1 package2-0.2'), + (PkgUtil, 'x86', 'pkgutil --install --yes package1@0.1 package2@0.2'), + (Chocolatey, 'x86', 'choco install --yes package1 --version 0.1 package2 --version 0.2'), + (PacMan, 'x86', 'pacman -S --noconfirm package1 package2'), + (Zypper, 'x86', 'zypper --non-interactive in package1=0.1 package2=0.2'), +]) +def test_tools_install_mode_install_to_build_machine_arch_with_version(tool_class, arch_host, result): + conanfile = ConanFileMock() + conanfile.settings = MockSettings({"arch": arch_host}) + conanfile.settings_build = MockSettings({"arch": "x86_64"}) + conanfile.conf.define("tools.system.package_manager:tool", tool_class.tool_name) + conanfile.conf.define("tools.system.package_manager:mode", "install") + with mock.patch('conan.ConanFile.context', new_callable=PropertyMock) as context_mock: + context_mock.return_value = "host" + tool = tool_class(conanfile) + + def fake_check(*args, **kwargs): + return ["package1=0.1", "package2=0.2"] + from conan.tools.system.package_manager import _SystemPackageManagerTool + with patch.object(_SystemPackageManagerTool, 'check', MagicMock(side_effect=fake_check)): + tool.install(["package1=0.1", "package2=0.2"], host_package=False) + + assert tool._conanfile.command == result + + @pytest.mark.parametrize("tool_class, result", [ # cross-compile but arch_names=None -> do not add host architecture # https://github.com/conan-io/conan/issues/12320 because the package is archless @@ -315,9 +398,36 @@ def fake_check(*args, **kwargs): assert tool._conanfile.command == result +@pytest.mark.parametrize("tool_class, result", [ + # cross-compile but arch_names=None -> do not add host architecture + # https://github.com/conan-io/conan/issues/12320 because the package is archless + (Apt, 'apt-get install -y --no-install-recommends package1=0.1 package2=0.2'), + (Yum, 'yum install -y package1-0.1 package2-0.2'), + (Dnf, 'dnf install -y package1-0.1 package2-0.2'), + (PacMan, 'pacman -S --noconfirm package1 package2'), +]) +def test_tools_install_archless_with_version(tool_class, result): + conanfile = ConanFileMock() + conanfile.settings = MockSettings({"arch": "x86"}) + conanfile.settings_build = MockSettings({"arch": "x86_64"}) + conanfile.conf.define("tools.system.package_manager:tool", tool_class.tool_name) + conanfile.conf.define("tools.system.package_manager:mode", "install") + with mock.patch('conan.ConanFile.context', new_callable=PropertyMock) as context_mock: + context_mock.return_value = "host" + tool = tool_class(conanfile, arch_names={}) + + def fake_check(*args, **kwargs): + return ["package1=0.1", "package2=0.2"] + from conan.tools.system.package_manager import _SystemPackageManagerTool + with patch.object(_SystemPackageManagerTool, 'check', MagicMock(side_effect=fake_check)): + tool.install(["package1=0.1", "package2=0.2"]) + + assert tool._conanfile.command == result + + @pytest.mark.parametrize("tool_class, result", [ (Apk, 'apk info -e package'), - (Apt, 'dpkg-query -W -f=\'${Status}\' package | grep -q "ok installed"'), + (Apt, r"dpkg-query -W -f='${Architecture}\n' package | grep -qEx '(amd64|all)'"), (Yum, 'rpm -q package'), (Dnf, 'rpm -q package'), (Brew, 'test -n "$(brew ls --versions package)"'), @@ -329,7 +439,7 @@ def fake_check(*args, **kwargs): ]) def test_tools_check(tool_class, result): conanfile = ConanFileMock() - conanfile.settings = Settings() + conanfile.settings = MockSettings({"arch": "x86_64"}) conanfile.conf.define("tools.system.package_manager:tool", tool_class.tool_name) with mock.patch('conan.ConanFile.context', new_callable=PropertyMock) as context_mock: context_mock.return_value = "host" @@ -337,3 +447,67 @@ def test_tools_check(tool_class, result): tool.check(["package"]) assert tool._conanfile.command == result + + +@pytest.mark.parametrize("tool_class, result", [ + (Apk, 'apk info package | grep "0.1"'), + (Apt, r"dpkg-query -W -f='${Architecture} ${Version}\n' package | grep -qEx '(amd64|all) 0.1'"), + (Yum, 'rpm -q package-0.1'), + (Dnf, 'rpm -q package-0.1'), + (Brew, 'brew list --versions package | grep "0.1"'), + (Pkg, 'pkg info package | grep "Version: 0.1"'), + (PkgUtil, 'test -n "`pkgutil --list package`"'), + (Chocolatey, 'choco list --local-only package | findstr /i "0.1"'), + (PacMan, 'pacman -Qi package'), + (Zypper, 'rpm -q package-0.1'), +]) +def test_tools_check_with_version(tool_class, result): + conanfile = ConanFileMock() + conanfile.settings = MockSettings({"arch": "x86_64"}) + conanfile.conf.define("tools.system.package_manager:tool", tool_class.tool_name) + with mock.patch('conan.ConanFile.context', new_callable=PropertyMock) as context_mock: + context_mock.return_value = "host" + tool = tool_class(conanfile) + tool.check(["package=0.1"]) + + assert tool._conanfile.command == result + + +@pytest.mark.parametrize("tool_class, result", [ + (Apt, r"dpkg-query -W -f='${Architecture}\n' package | grep -qEx '(amd64|all)'"), +]) +@pytest.mark.parametrize("arch_host", [ + 'x86_64', + 'x86', +]) +def test_tools_apt_check_install_to_build_machine_arch(tool_class, arch_host, result): + conanfile = ConanFileMock() + conanfile.settings = MockSettings({"arch": arch_host}) + conanfile.settings_build = MockSettings({"arch": "x86_64"}) + conanfile.conf.define("tools.system.package_manager:tool", tool_class.tool_name) + with mock.patch('conan.ConanFile.context', new_callable=PropertyMock) as context_mock: + context_mock.return_value = "host" + tool = tool_class(conanfile) + tool.check(["package"], host_package=False) + + assert tool._conanfile.command == result + + +@pytest.mark.parametrize("tool_class, result", [ + (Apt, r"dpkg-query -W -f='${Architecture} ${Version}\n' package | grep -qEx '(amd64|all) 0.1'"), +]) +@pytest.mark.parametrize("arch_host", [ + 'x86_64', + 'x86', +]) +def test_tools_apt_check_install_to_build_machine_arch_with_version(tool_class, arch_host, result): + conanfile = ConanFileMock() + conanfile.settings = MockSettings({"arch": arch_host}) + conanfile.settings_build = MockSettings({"arch": "x86_64"}) + conanfile.conf.define("tools.system.package_manager:tool", tool_class.tool_name) + with mock.patch('conan.ConanFile.context', new_callable=PropertyMock) as context_mock: + context_mock.return_value = "host" + tool = tool_class(conanfile) + tool.check(["package=0.1"], host_package=False) + + assert tool._conanfile.command == result diff --git a/test/integration/workspace/test_workspace.py b/test/integration/workspace/test_workspace.py index 1ae1863b50a..9c5002921e2 100644 --- a/test/integration/workspace/test_workspace.py +++ b/test/integration/workspace/test_workspace.py @@ -119,6 +119,23 @@ def test_add_from_outside(self): assert "dep1/0.1" in c.out assert "dep2" not in c.out + def test_add_modified(self): + # https://github.com/conan-io/conan/issues/18942 + c = TestClient(light=True) + c.save({"conanws.yml": "", + "dep1/conanfile.py": GenConanfile("dep1", "0.1")}) + c.run("workspace add dep1") + assert "Reference 'dep1/0.1' added to workspace" in c.out + c.run("workspace info") + assert "dep1/0.1" in c.out + c.save({"dep1/conanfile.py": GenConanfile("dep1", "0.2")}) + c.run("workspace add dep1") + assert "Package dep1 already exists, updating its reference" in c.out + assert "Reference 'dep1/0.2' added to workspace" in c.out + c.run("workspace info") + assert "dep1/0.2" in c.out + assert "dep1/0.1" not in c.out + @pytest.mark.parametrize("api", [False, True]) def test_dynamic_editables(self, api): c = TestClient(light=True) @@ -160,7 +177,8 @@ def packages(self): for f in os.listdir(self.folder): if os.path.isdir(os.path.join(self.folder, f)): conanfile = self.load_conanfile(f) - result.append({"path": f, "ref": f"{conanfile.name}/{conanfile.version}"}) + result.append({"path": f, + "ref": f"{conanfile.name}/{conanfile.version}"}) return result """) @@ -216,6 +234,51 @@ def packages(self): c.run("install --requires=pkg/2.1") # it will not fail + def test_api_dynamic_any_version(self): + # For workspace, the version of the conanfile can be ignored if the + # workspace define a different version. + c = TestClient(light=True) + workspace = textwrap.dedent("""\ + import os + from conan import Workspace + + class MyWorkspace(Workspace): + def packages(self): + return [{"path": "dep1", "ref": "pkg/1.2.3"}] + """) + + c.save({"conanws.py": workspace, + "dep1/conanfile.py": GenConanfile("pkg", "0.1")}) + c.run("workspace info --format=json") + info = json.loads(c.stdout) + assert info["packages"] == [{"ref": "pkg/1.2.3", "path": "dep1"}] + c.run("install --requires=pkg/1.2.3") + # it will not fail + + def test_replace_requires(self): + c = TestClient(light=True) + c.save({"conanws.yml": "", + "pkga/conanfile.py": GenConanfile("pkga", "0.1"), + "pkgb/conanfile.py": GenConanfile("pkgb", "0.1").with_requires("pkga/develop"), + "pkgc/conanfile.py": GenConanfile("pkgc", "0.1").with_requires("pkgb/0.1", + "pkga/0.1"), + "myreplaces": "[replace_requires]\npkga/develop: pkga/0.1"}) + c.run("workspace add pkga") + c.run("workspace add pkgb") + c.run("workspace add pkgc") + c.run("workspace info --format=json") + + info = json.loads(c.stdout) + assert info["packages"] == [{'path': 'pkga', 'ref': 'pkga/0.1'}, + {'path': 'pkgb', 'ref': 'pkgb/0.1'}, + {'path': 'pkgc', 'ref': 'pkgc/0.1'}] + c.run("install --requires=pkgc/0.1", assert_error=True) + assert "Version conflict: Conflict between pkga/develop and pkga/0.1 in the graph" in c.out + # it will not fail + c.run("install --requires=pkgc/0.1 -pr=default -pr=myreplaces") + assert "pkga/0.1 - Editable" in c.out + assert "pkga/develop: pkga/0.1" in c.out + def test_error_uppercase(self): c = TestClient(light=True) c.save({"conanws.yml": "", @@ -279,14 +342,20 @@ def remove(self, path, *args, **kwargs): c.save({"conanws.py": workspace, "dep/conanfile.py": GenConanfile("dep", "0.1")}) c.run("workspace add dep") - assert "myws: Adding dep/0.1" in c.out + assert "Workspace 'myws': Adding dep/0.1" in c.out c.run("workspace info") assert "dep/0.1" in c.out c.run("workspace remove dep") - assert "myws: Removing" in c.out + assert "Workspace 'myws': Removing" in c.out c.run("workspace info") assert "dep/0.1" not in c.out + def test_remove_non_existing_error(self): + c = TestClient(light=True) + c.save({"conanws.yml": ""}) + c.run("workspace remove kk", assert_error=True) + assert "ERROR: No editable package to remove from this path: kk" in c.out + class TestOpenAdd: def test_without_git(self): @@ -369,6 +438,77 @@ def test_workspace_build_editables(self): assert "pkgb/0.1: WARN: BUILD PKGB!" in c.out +class TestComplete: + def test_complete_add(self): + c = TestClient(light=True) + c.save({"conanws.yml": "", + "pkga/conanfile.py": GenConanfile("pkga", "0.1"), + "pkgx/conanfile.py": GenConanfile("pkgx", "0.1"), + "pkgb/conanfile.py": GenConanfile("pkgb", "0.1").with_requires("pkga/0.1"), + "pkgc/conanfile.py": GenConanfile("pkgc", "0.1").with_requires("pkga/0.1", + "pkgx/0.1"), + "pkgd/conanfile.py": GenConanfile("pkgd", "0.1").with_requires("pkgb/0.1", + "pkgc/0.1")}) + + c.run("workspace complete") # Does nothing, but it doesn't fail + assert "There are no packages in this workspace, nothing to complete" in c.out + c.run("workspace info") + assert "packages: (empty)" in c.out + + for pkg in ("pkga", "pkgx", "pkgb", "pkgc", "pkgd"): + c.run(f"export {pkg}") + + c.run("workspace add pkgd") + c.run("workspace complete") # Does nothing, but it doesn't fail + assert "There are no intermediate packages to add to the workspace" in c.out + c.run("workspace info") + assert "pkgd/0.1" in c.out + assert "pkga/0.1" not in c.out + + c.run("workspace add pkga") + c.run("workspace install --build=missing", assert_error=True) + assert "Workspace definition error. Package pkgb/0.1 in the Conan cache" in c.out + c.run("workspace info") + assert "pkgb/0.1" not in c.out + assert "pkgc/0.1" not in c.out + + c.run("workspace complete") + c.run("workspace info") + assert "pkgb/0.1" in c.out + assert "pkgc/0.1" in c.out + assert "pkgx/0.1" not in c.out + + def test_complete_open(self): + c = TestClient(light=True) + c.save({"pkgx/conanfile.py": GenConanfile("pkgx", "0.1"), + "pkgb/conanfile.py": GenConanfile("pkgb", "0.1").with_requires("pkga/0.1"), + "pkgc/conanfile.py": GenConanfile("pkgc", "0.1").with_requires("pkga/0.1", + "pkgx/0.1")}) + + for pkg in ("pkgx", "pkgb", "pkgc"): + c.run(f"export {pkg}") + + c.save({"conanws.yml": "", + "pkga/conanfile.py": GenConanfile("pkga", "0.1"), + "pkgd/conanfile.py": GenConanfile("pkgd", "0.1").with_requires("pkgb/0.1", + "pkgc/0.1")}, + clean_first=True) + + c.run("workspace add pkgd") + c.run("workspace add pkga") + c.run("workspace install --build=missing", assert_error=True) + assert "Workspace definition error. Package pkgb/0.1 in the Conan cache" in c.out + c.run("workspace info") + assert "pkgb/0.1" not in c.out + assert "pkgc/0.1" not in c.out + + c.run("workspace complete") + c.run("workspace info") + assert "pkgb/0.1" in c.out + assert "pkgc/0.1" in c.out + assert "pkgx/0.1" not in c.out + + class TestWorkspaceBuild: def test_build(self): @@ -451,6 +591,34 @@ def packages(self): c.run("workspace build") assert "conanfile.py (pkga/0.1): Building pkga AND 0.1!!!" in c.out + def test_build_with_external_editable_python_requires(self): + c = TestClient(light=True) + c.save({"conanws.yml": "", + "ext/conanfile.py": GenConanfile("ext", "0.1").with_package_type("python-require"), + "pkga/conanfile.py": GenConanfile("pkga", "0.1").with_python_requires("ext/0.1") + }) + c.run("editable add ext") + c.run("workspace add pkga") + c.run("workspace build") + c.assert_listed_binary({"pkga/0.1": ("8bca2eae7cd0d6b6da8d14f8c86069d89d265bd4", + "EditableBuild")}) + c.assert_listed_require({"ext/0.1": "Editable"}, python=True) + + def test_build_with_external_editable(self): + c = TestClient(light=True) + c.save({"conanws.yml": "", + "ext/conanfile.py": GenConanfile("ext", "0.1").with_build_msg("BUILD EXT!"), + "pkga/conanfile.py": GenConanfile("pkga", "0.1").with_requires("ext/0.1") + }) + c.run("editable add ext") + c.run("workspace add pkga") + c.run("workspace build") + assert "ext/0.1: WARN: BUILD EXT!" in c.out + c.assert_listed_binary({"pkga/0.1": ("38f8a554b1b3c2cbb44321f0e731b3863359126c", + "EditableBuild"), + "ext/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", + "EditableBuild")}) + class TestNew: def test_new(self): @@ -488,7 +656,9 @@ def test_install(self): clean_first=True) c.run("workspace add liba") c.run("workspace add libb") - c.run("workspace super-install -g CMakeDeps -g CMakeToolchain -of=build --envs-generation=false") + c.run("workspace super-install -g CMakeDeps -g CMakeToolchain -of=build " + "--envs-generation=false") + assert "Packages build order:\n liba/0.1: liba\n libb/0.1: libb" in c.out assert "Workspace conanws.py not found in the workspace folder, using default" in c.out files = os.listdir(os.path.join(c.current_folder, "build")) assert "conan_toolchain.cmake" in files @@ -509,6 +679,9 @@ class MyWs(ConanFile): settings = "arch", "build_type" generators = "MSBuildDeps" + def generate(self): + self.run("echo myhello world!!!!") + class Ws(Workspace): def root_conanfile(self): return MyWs @@ -518,6 +691,7 @@ def root_conanfile(self): "conanws.py": conanfilews}) c.run("workspace add dep") c.run("workspace super-install -of=build") + assert "myhello world!!!!" in c.out files = os.listdir(os.path.join(c.current_folder, "build")) assert "conandeps.props" in files @@ -567,6 +741,228 @@ def test_install_partial(self): assert "dep1-config.cmake" in files assert "dep2-config.cmake" not in files + def test_workspace_options(self): + c = TestClient() + conanfilews = textwrap.dedent(""" + from conan import ConanFile + from conan import Workspace + class MyWs(ConanFile): + settings = "arch", "build_type" + options = {"myoption": [1, 2, 3]} + def generate(self): + self.output.info(f"Generating with my option {self.options.myoption}!!!!") + + class Ws(Workspace): + def root_conanfile(self): + return MyWs + """) + + c.save({"dep/conanfile.py": GenConanfile("dep", "0.1"), + "conanws.py": conanfilews}) + c.run("workspace add dep") + c.run("workspace super-install -of=build") + assert "conanws.py base project Conanfile: Generating with my option None!!!!" in c.out + c.run("workspace super-install -of=build -o *:myoption=1") + assert "conanws.py base project Conanfile: Generating with my option 1!!!!" in c.out + + def test_workspace_common_shared_options(self): + c = TestClient() + conanfilews = textwrap.dedent(""" + from conan import ConanFile + from conan import Workspace + class MyWs(ConanFile): + settings = "os" + options = {"shared": [True, False], + "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + implements = "auto_shared_fpic" + + def generate(self): + self.output.info(f"OS={self.settings.os}!!!!") + self.output.info(f"shared={self.options.shared}!!!!") + self.output.info(f"fPIC={self.options.get_safe('fPIC')}!!!!") + + class Ws(Workspace): + def root_conanfile(self): + return MyWs + """) + + c.save({"dep/conanfile.py": GenConanfile("dep", "0.1"), + "conanws.py": conanfilews}) + c.run("workspace add dep") + c.run("workspace super-install -of=build -s os=Windows") + assert "conanws.py base project Conanfile: OS=Windows!!!!" in c.out + assert "conanws.py base project Conanfile: shared=False!!!!" in c.out + assert "conanws.py base project Conanfile: fPIC=None!!!!" in c.out + c.run("workspace super-install -of=build -s os=Linux -o *:shared=True") + assert "conanws.py base project Conanfile: OS=Linux!!!!" in c.out + assert "conanws.py base project Conanfile: shared=True!!!!" in c.out + assert "conanws.py base project Conanfile: fPIC=None!!!!" in c.out + c.run("workspace super-install -of=build -s os=Linux -o *:shared=False") + assert "conanws.py base project Conanfile: OS=Linux!!!!" in c.out + assert "conanws.py base project Conanfile: shared=False!!!!" in c.out + assert "conanws.py base project Conanfile: fPIC=True!!!!" in c.out + + def test_workspace_pkg_options(self): + c = TestClient() + conanfilews = textwrap.dedent(""" + from conan import ConanFile + from conan import Workspace + + class MyWs(ConanFile): + settings = "arch", "build_type" + def generate(self): + for pkg, dep in self.workspace_packages.items(): + for k, v in dep.options.items(): + self.output.info(f"Generating with opt {pkg}:{k}={v}!!!!") + + class Ws(Workspace): + def root_conanfile(self): + return MyWs + """) + + c.save({"dep/conanfile.py": GenConanfile("dep", "0.1").with_option("myoption", [1, 2, 3]), + "conanws.py": conanfilews}) + c.run("workspace add dep") + c.run("workspace super-install -of=build") + assert "project Conanfile: Generating with opt dep/0.1:myoption=None!!!!" in c.out + c.run("workspace super-install -of=build -o *:myoption=1") + assert "project Conanfile: Generating with opt dep/0.1:myoption=1!!!!" in c.out + + def test_workspace_pkg_definitions(self): + c = TestClient() + conanfilews = textwrap.dedent(""" + from conan import ConanFile + from conan import Workspace + from conan.tools.cmake import CMakeToolchain + + class MyWs(ConanFile): + settings = "arch", "build_type" + def generate(self): + tc = CMakeToolchain(self) + for ref, dep in self.workspace_packages.items(): + dep.configure_toolchain(tc) + tc.generate() + + class Ws(Workspace): + def root_conanfile(self): + return MyWs + """) + dep = textwrap.dedent(""" + from conan import ConanFile + class Dep(ConanFile): + name = "dep" + version = "1.2.3" + + def configure_toolchain(self, tc): + tc.preprocessor_definitions["DEP_SOME_DEFINITION"] = self.version + + def generate(self): + tc = CMakeToolchain(self) + self.configure_toolchain(tc) + tc.generate() + """) + pkg = textwrap.dedent(""" + from conan import ConanFile + class Dep(ConanFile): + name = "pkg" + version = "2.3.4" + + def configure_toolchain(self, tc): + tc.preprocessor_definitions["SOME_PKG_DEFINE"] = self.version + + def generate(self): + tc = CMakeToolchain(self) + self.configure_toolchain(tc) + tc.generate() + """) + c.save({"dep/conanfile.py": dep, + "pkg/conanfile.py": pkg, + "conanws.py": conanfilews}) + c.run("workspace add dep") + c.run("workspace add pkg") + c.run("workspace super-install") + tc = c.load("conan_toolchain.cmake") + assert 'add_compile_definitions("SOME_PKG_DEFINE=2.3.4")' in tc + assert 'add_compile_definitions("DEP_SOME_DEFINITION=1.2.3")' in tc + + def test_intermediate_non_editable(self): + c = TestClient(light=True) + + c.save({"liba/conanfile.py": GenConanfile("liba", "0.1"), + "libb/conanfile.py": GenConanfile("libb", "0.1").with_requires("liba/0.1"), + "libc/conanfile.py": GenConanfile("libc", "0.1").with_requires("libb/0.1")}) + + c.run("workspace init") + c.run("workspace add liba") + c.run("export libb") + c.run("workspace add libc") + c.run("workspace super-install", assert_error=True) + assert ("Workspace definition error. Package libb/0.1 in the Conan cache " + "has dependencies to packages in the workspace: [liba/0.1]") in c.out + + def test_deployers_json(self): + c = TestClient() + dep = textwrap.dedent(""" + import os + from conan.tools.files import save + from conan import ConanFile + class Dep(ConanFile): + name = "dep" + version = "1.0" + + def package(self): + save(self, os.path.join(self.package_folder, "myfile.txt"), "content") + """) + pkg = textwrap.dedent(""" + import os + from conan.tools.files import save + from conan import ConanFile + class Dep(ConanFile): + name = "pkg" + version = "1.0" + requires = "dep/1.0" + + def package(self): + save(self, os.path.join(self.package_folder, "myfile.txt"), "content") + """) + c.save({"dep/conanfile.py": dep, + "pkg/conanfile.py": pkg}) + c.run("workspace init") + c.run("create dep") + c.run("workspace add pkg") + c.run("workspace super-install --format=json --deployer=full_deploy") + graph = json.loads(c.stdout) + # Only 1 node, pkg is not a node in the graph! + assert graph["graph"]["nodes"]["1"]["name"] == "dep" + deploy_folder = os.path.join(c.current_folder, "full_deploy", "host") + folders = os.listdir(deploy_folder) + assert "dep" in folders + assert "pkg" not in folders + assert os.path.isfile(os.path.join(deploy_folder, "dep", "1.0", "myfile.txt")) + + def test_customize_generator_per_package(self): + # https://github.com/conan-io/conan/issues/19326 + c = TestClient(light=True) + + pkg = GenConanfile("pkg", "0.1") + workspace = textwrap.dedent("""\ + from conan import Workspace, ConanFile + + class MyWs(ConanFile): + pass + + class Ws(Workspace): + def root_conanfile(self): + return MyWs + """) + c.save({"pkg/conanfile.py": pkg, + "conanws.py": workspace}) + c.run("workspace add pkg") + c.run("workspace super-install -c acme*:tools.cmake.cmaketoolchain:generator=Ninja") + # It no longer crashes + assert "Install finished successfully" in c.out + def test_workspace_with_local_recipes_index(): c3i_folder = temp_folder() @@ -597,7 +993,7 @@ def test_clean(self): c.save({"conanws.yml": "name: my_workspace"}) pkga = GenConanfile("pkga", "0.1").with_settings("build_type") pkgb = GenConanfile("pkgb", "0.1").with_requires("pkga/0.1").with_settings("build_type") - c.save({"pkga/conanfile.py": pkga , + c.save({"pkga/conanfile.py": pkga, "pkgb/conanfile.py": pkgb, "pkgc/conanfile.py": GenConanfile("pkgc", "0.1").with_requires("pkgb/0.1")}) c.run("workspace add pkga -of=build/pkga") @@ -607,8 +1003,8 @@ def test_clean(self): assert os.path.exists(os.path.join(c.current_folder, "build", "pkga")) assert os.path.exists(os.path.join(c.current_folder, "build", "pkgb")) c.run("workspace clean") - assert "my_workspace: Removing pkga/0.1 output folder" in c.out - assert "my_workspace: Removing pkgb/0.1 output folder" in c.out + assert "Workspace 'my_workspace': Removing pkga/0.1 output folder" in c.out + assert "Workspace 'my_workspace': Removing pkgb/0.1 output folder" in c.out assert "Editable pkgc/0.1 doesn't have an output_folder defined" in c.out assert not os.path.exists(os.path.join(c.current_folder, "build", "pkga")) assert not os.path.exists(os.path.join(c.current_folder, "build", "pkgb")) @@ -626,7 +1022,7 @@ def clean(self): c = TestClient() c.save({"conanws.py": conanfilews}) c.run("workspace clean") - assert "my_workspace: MY CLEAN!!!" in c.out + assert "Workspace 'my_workspace': MY CLEAN!!!" in c.out def test_relative_paths(): @@ -781,7 +1177,6 @@ def test_create(self): assert "WARN: BUILD ZLIB!" not in c.out assert "pkgc/0.1: WARN: BUILD PKGC!" in c.out - def test_create_dynamic_name(self): workspace = textwrap.dedent("""\ import os @@ -803,7 +1198,6 @@ def packages(self): c.run("workspace create") assert "Workspace create pkga/0.1" in c.out - def test_host_build_require(self): c = TestClient() protobuf = textwrap.dedent("""\ @@ -968,12 +1362,17 @@ def test_workspace_defining_only_paths(): assert "liba/0.1@myuser/mychannel - Editable" in c.out assert "libb/0.1 - Editable" in c.out + c.run("workspace super-install") + assert "app1/0.1 - Editable" in c.out + assert "liba/0.1@myuser/mychannel - Editable" in c.out + assert "libb/0.1 - Editable" in c.out + def test_workspace_defining_duplicate_references(): """ Testing duplicate references but different paths """ - c = TestClient() + c = TestClient(light=True) conanws_with_labels = textwrap.dedent("""\ packages: - path: liba @@ -987,11 +1386,11 @@ def test_workspace_defining_duplicate_references(): "liba2/conanfile.py": GenConanfile(name="liba", version="0.1"), }) c.run("workspace install", assert_error=True) - assert "Workspace editable reference 'liba/0.1' already exists." in c.out + assert "Workspace package 'liba/0.1' already exists." in c.out def test_workspace_reference_error(): - c = TestClient() + c = TestClient(light=True) conanws_with_labels = textwrap.dedent("""\ packages: - path: libx @@ -999,5 +1398,24 @@ def test_workspace_reference_error(): c.save({"conanws.yml": conanws_with_labels, "libx/conanfile.py": ""}) c.run("workspace install", assert_error=True) - assert ("Workspace editable reference could not be deduced by libx/conanfile.py or it is not" - " correctly defined in the conanws.yml file.") in c.out + assert ("Workspace package reference could not be deduced by libx/conanfile.py or it is not" + " correctly defined in the conanws.yml file") in c.out + + +def test_workspace_python_error(): + c = TestClient() + workspace = textwrap.dedent("""\ + from conan import Workspace + + class MyWorkspace(Workspace): + def packages(self): + os.listdir(self.folder) + """) + c.save({"conanws.yml": "", + "conanws.py": "bad"}) + c.run("workspace info", assert_error=True) + assert "ERROR: Error loading conanws.py at" in c.out + c.save({"conanws.yml": "", + "conanws.py": workspace}) + c.run("workspace info", assert_error=True) + assert "ERROR: Workspace conanws.py file: Error in packages() method, line 5" in c.out diff --git a/test/performance/test_compatibility_performance.py b/test/performance/test_compatibility_performance.py new file mode 100644 index 00000000000..59540b5e683 --- /dev/null +++ b/test/performance/test_compatibility_performance.py @@ -0,0 +1,129 @@ +import textwrap +import time + +import pytest + +from conan.test.utils.tools import TestClient, GenConanfile + +RUN_COUNT = 2 + + +@pytest.mark.skip(reason="Compatibility performance test") +class TestCompatibilityPerformance: + + results = [] + + @pytest.fixture(scope="module", autouse=True) + def timings(self): + results = [] + yield results + + import matplotlib.pyplot as plt + + fig = plt.figure() + ax = fig.add_subplot() + + ax.set_title("Compatibility performance in real Artifactory server (last match, unoptimized)") + + #ax_tot = ax.twinx() + + # Plot a basic wireframe. + + grouping = {} + x, y, y_tot_opt, y_tot_non = [], [], [], [] + stderr_opt, stderr_non = [], [] + for factors, run, with_optimization, duration in results: + group = grouping.setdefault(factors, {"opt": [], "non": []}) + if with_optimization: + group["opt"].append(duration) + else: + group["non"].append(duration) + + def ssd(values, mean): + return sum((v - mean) ** 2 for v in values) / (len(values) - 1) + + for factors, durations in sorted(grouping.items()): + x.append(factors) + y.append(0) + + mean_opt = sum(durations["opt"]) / RUN_COUNT + mean_non = sum(durations["non"]) / RUN_COUNT + y_tot_opt.append(mean_opt) + y_tot_non.append(mean_non) + + # ssd_opt = ssd(durations["opt"], mean_opt) + # ssd_non = ssd(durations["non"], mean_non) + # stderr_opt.append(ssd_opt / (RUN_COUNT ** 0.5)) + # stderr_non.append(ssd_non / (RUN_COUNT ** 0.5)) + + max_diff_y = max(y) + min_diff_y = min(min(y), 0) + # ax_tot.set_ylim(ymin=min_diff_y-0.1*min_diff_y, ymax=max_diff_y+0.1*max_diff_y) + + max_tot_y = max(max(y_tot_non), max(y_tot_opt)) + ax.set_ylim(ymin=0, ymax=max_tot_y+0.1*max_tot_y) + + ax.grid(visible=True, which="major", axis="y", linestyle="--") + # ax_tot.set_xticks(x) + + ax.set_xticks(x) + ax.plot(x, y_tot_non, 'red', marker='x', label="Without optimization", linestyle=":") + # ax.errorbar(x, y_tot_non, yerr=stderr_non) + ax.plot(x, y_tot_opt, 'blue', marker='x', label="With optimization", linestyle=":") + # ax.errorbar(x, y_tot_opt, yerr=stderr_opt) + + # ax_tot.plot(x, y, 'gray', marker='o', linestyle="--", label="Difference total") + # ax_tot.set_ylabel("Total time diff (s)") + + ax.set_xlabel("Number of compatible configurations in server") + ax.set_ylabel("Total time (s)") + ax.legend(loc="upper left") + + # ax_tot.legend(loc="upper right") + + plt.show() + + @pytest.mark.parametrize("runs", range(RUN_COUNT)) + @pytest.mark.parametrize("with_optimization", [True, False]) + @pytest.mark.parametrize("factors", [pow(2, n) for n in range(1, 6)]) + def test_list_only_compatibility(self, timings, with_optimization, factors, runs): + tc = TestClient(light=True) + tc.run("remote add default https://conandev.jfrog.io/artifactory/api/conan/abril-tests") + tc.run("remote login default abril") + tc.run("remove * -c -r=default") + cppstds = [7 + i*3 for i in range(factors+1)] + compiler_settings = textwrap.dedent(f""" + compiler: + foo: + version: [1] + cppstd: {cppstds}""") + tc.run("version") + tc.save_home({"settings_user.yml": compiler_settings}) + compat = tc.load_home("extensions/plugins/compatibility/compatibility.py") + compat = compat.replace("cppstd_possible_values = supported_cppstd(conanfile)", + f"cppstd_possible_values = {cppstds}") + tc.save_home({"extensions/plugins/compatibility/compatibility.py": compat}) + threshold = -1 if with_optimization else 1 + tc.save_home({"global.conf": f"user.graph:compatibility_optimization_threshold={factors + threshold}"}) + + compiler_args = "-s compiler=foo -s compiler.version=1" + pkg_name = str(time.time()) + tc.save({"conanfile.py": GenConanfile(pkg_name, "0.1").with_settings("compiler")}) + + if with_optimization: + for i in range(factors-1): + tc.run(f"create . {compiler_args} -s compiler.cppstd={7 + i*3} -nr") + tc.run(f"create . {compiler_args} -s compiler.cppstd={7 + (factors-1)*3} -nr") + + tc.run("upload * -r=default -c") + tc.run("remove *:* -c") + + start_time = time.time() + tc.run(f"install --requires={pkg_name}/0.1 {compiler_args} -s compiler.cppstd={7 + factors*3}") + end_time = time.time() + if with_optimization: + assert f"Found {factors} compatible configurations in remotes" in tc.out + else: + assert f"compatible configurations in remotes" not in tc.out + duration = end_time - start_time + timings.append((factors, runs, with_optimization, duration)) diff --git a/test/performance/test_db_performance.py b/test/performance/test_db_performance.py new file mode 100644 index 00000000000..abc228e26a9 --- /dev/null +++ b/test/performance/test_db_performance.py @@ -0,0 +1,74 @@ +import os +import time + +import pytest + +from conan.api.model import RecipeReference +from conan.internal.cache.db.cache_database import CacheDatabase +from conan.test.utils.test_files import temp_folder + + +@pytest.mark.skip(reason="This is a performance test, skip for normal runs") +def test_db_performance(): + f = temp_folder() + # f = r"C:\conan_tests\tmp_o0846cmconans\path with spaces" + print("Tempt folder: ", f) + db = CacheDatabase(os.path.join(f, "mytest.sqlite")) + + num_refs = 1000 + splits = 10 + for num_split in range(10): + init = time.time() + for i in range(int(num_refs / splits)): + index = num_split * int(num_refs / splits) + i + ref = RecipeReference.loads(f"pkg/1.{index}#rev1%1") + path = os.path.join(f, f"folder{index}") + db.create_recipe(path, ref) + creation_time = time.time() - init + print(f"Creation time {num_split}:", creation_time) + print(" Avg:", creation_time/num_refs) + + experiments = 10 + texp = time.time() + for experiment in range(experiments): + ret = db.list_references() + assert len(ret) == num_refs + exp_time = time.time() - texp + print("SEARCH RECIPES time:", exp_time) + print(" Avg:", exp_time / experiments) + + texp = time.time() + specific_ref = RecipeReference.loads(f"pkg/1.1#rev1%1") + for experiment in range(experiments): + db.get_recipe(specific_ref) + exp_time = time.time() - texp + print("GET RECIPE time:", exp_time) + print(" Avg:", exp_time / experiments) + + texp = time.time() + specific_ref = RecipeReference.loads(f"pkg/1.1#rev1%1") + for experiment in range(experiments): + db.get_latest_recipe(specific_ref) + exp_time = time.time() - texp + print("GET LATEST RECIPE time:", exp_time) + print(" Avg:", exp_time / experiments) + + texp = time.time() + for experiment in range(experiments): + db.update_recipes_lru([specific_ref]) + exp_time = time.time() - texp + print("UPDATE LRU:", exp_time) + print(" Avg:", exp_time / experiments) + + updates = 50 + texp = time.time() + for experiment in range(experiments): + refs = [RecipeReference.loads(f"pkg/1.{index}#rev1%1") for index in range(updates)] + db.update_recipes_lru(refs) + exp_time = time.time() - texp + print("UPDATE LRU BATCH:", exp_time) + print(" Avg:", exp_time / experiments) + print(" Avg:", exp_time / (experiments * updates)) + + + diff --git a/test/unittests/cli/test_cli_ref_matching.py b/test/unittests/cli/test_cli_ref_matching.py index 03724398133..538bc1a1f72 100644 --- a/test/unittests/cli/test_cli_ref_matching.py +++ b/test/unittests/cli/test_cli_ref_matching.py @@ -1,6 +1,7 @@ import pytest from conan.api.model import ListPattern +from conan.errors import ConanException @pytest.mark.parametrize("pattern, result", @@ -11,3 +12,9 @@ def test_cli_pattern_matching(pattern, result): pattern = ListPattern(pattern) assert result == (pattern.ref, pattern.rrev, pattern.package_id, pattern.prev) + + +def test_list_pattern(): + with pytest.raises(ConanException) as e: + ListPattern("*:*", only_recipe=True) + assert "Do not specify 'package_id' with 'only-recipe'" in str(e.value) diff --git a/test/unittests/client/build/c_std_flags_test.py b/test/unittests/client/build/c_std_flags_test.py index b5d2e585766..f72a40b054c 100644 --- a/test/unittests/client/build/c_std_flags_test.py +++ b/test/unittests/client/build/c_std_flags_test.py @@ -1,7 +1,7 @@ from conan.internal.api.detect.detect_api import default_cstd -from conan.tools.build.flags import cstd_flag from conan.internal.model.version import Version -from conan.test.utils.mocks import MockSettings, ConanFileMock +from conan.test.utils.mocks import ConanFileMock, MockSettings +from conan.tools.build.flags import cstd_flag def _make_cstd_flag(compiler, compiler_version, cstd=None): @@ -12,44 +12,55 @@ def _make_cstd_flag(compiler, compiler_version, cstd=None): return cstd_flag(conanfile) +def test_gcc_cstd_flags(): + assert _make_cstd_flag("gcc", "4.2", "99") == "-std=c99" + assert _make_cstd_flag("clang", "4.2", "99") == "-std=c99" + assert _make_cstd_flag("apple-clang", "4.2", "99") == "-std=c99" + assert _make_cstd_flag("msvc", "192", "11") == "/std:c11" + assert _make_cstd_flag("msvc", "192", "17") == "/std:c17" + assert _make_cstd_flag("msvc", "191", "11") is None + assert _make_cstd_flag("msvc", "191", "17") is None + + def _make_cstd_default(compiler, compiler_version): return default_cstd(compiler, Version(compiler_version)) def test_gcc_cstd_defaults(): - assert _make_cstd_default("gcc", "4")== "gnu99" - assert _make_cstd_default("gcc", "5")== "gnu11" - assert _make_cstd_default("gcc", "6")== "gnu11" - assert _make_cstd_default("gcc", "6.1")== "gnu11" - assert _make_cstd_default("gcc", "7.3")== "gnu11" - assert _make_cstd_default("gcc", "8.1")== "gnu17" - assert _make_cstd_default("gcc", "11")== "gnu17" - assert _make_cstd_default("gcc", "11.1")== "gnu17" - assert _make_cstd_default("gcc", "15.1")== "gnu23" + assert _make_cstd_default("gcc", "4") == "gnu99" + assert _make_cstd_default("gcc", "5") == "gnu11" + assert _make_cstd_default("gcc", "6") == "gnu11" + assert _make_cstd_default("gcc", "6.1") == "gnu11" + assert _make_cstd_default("gcc", "7.3") == "gnu11" + assert _make_cstd_default("gcc", "8.1") == "gnu17" + assert _make_cstd_default("gcc", "11") == "gnu17" + assert _make_cstd_default("gcc", "11.1") == "gnu17" + assert _make_cstd_default("gcc", "15.1") == "gnu23" + def test_clang_cstd_defaults(): - assert _make_cstd_default("clang", "2")== "gnu99" - assert _make_cstd_default("clang", "2.1")== "gnu99" - assert _make_cstd_default("clang", "3.0")== "gnu99" - assert _make_cstd_default("clang", "3.1")== "gnu99" - assert _make_cstd_default("clang", "3.4")== "gnu99" - assert _make_cstd_default("clang", "3.5")== "gnu99" - assert _make_cstd_default("clang", "5")== "gnu11" - assert _make_cstd_default("clang", "5.1")== "gnu11" - assert _make_cstd_default("clang", "6")== "gnu11" - assert _make_cstd_default("clang", "7")== "gnu11" - assert _make_cstd_default("clang", "8")== "gnu11" - assert _make_cstd_default("clang", "9")== "gnu11" - assert _make_cstd_default("clang", "10")== "gnu11" - assert _make_cstd_default("clang", "11")== "gnu17" - assert _make_cstd_default("clang", "12")== "gnu17" - assert _make_cstd_default("clang", "13")== "gnu17" - assert _make_cstd_default("clang", "14")== "gnu17" - assert _make_cstd_default("clang", "15")== "gnu17" - assert _make_cstd_default("clang", "16")== "gnu17" - assert _make_cstd_default("clang", "17")== "gnu17" - assert _make_cstd_default("clang", "18")== "gnu17" - assert _make_cstd_default("clang", "19")== "gnu17" + assert _make_cstd_default("clang", "2") == "gnu99" + assert _make_cstd_default("clang", "2.1") == "gnu99" + assert _make_cstd_default("clang", "3.0") == "gnu99" + assert _make_cstd_default("clang", "3.1") == "gnu99" + assert _make_cstd_default("clang", "3.4") == "gnu99" + assert _make_cstd_default("clang", "3.5") == "gnu99" + assert _make_cstd_default("clang", "5") == "gnu11" + assert _make_cstd_default("clang", "5.1") == "gnu11" + assert _make_cstd_default("clang", "6") == "gnu11" + assert _make_cstd_default("clang", "7") == "gnu11" + assert _make_cstd_default("clang", "8") == "gnu11" + assert _make_cstd_default("clang", "9") == "gnu11" + assert _make_cstd_default("clang", "10") == "gnu11" + assert _make_cstd_default("clang", "11") == "gnu17" + assert _make_cstd_default("clang", "12") == "gnu17" + assert _make_cstd_default("clang", "13") == "gnu17" + assert _make_cstd_default("clang", "14") == "gnu17" + assert _make_cstd_default("clang", "15") == "gnu17" + assert _make_cstd_default("clang", "16") == "gnu17" + assert _make_cstd_default("clang", "17") == "gnu17" + assert _make_cstd_default("clang", "18") == "gnu17" + assert _make_cstd_default("clang", "19") == "gnu17" assert _make_cstd_default("clang", "20"), "gnu17" diff --git a/test/unittests/client/build/compiler_flags_test.py b/test/unittests/client/build/compiler_flags_test.py index 49c7720318b..ff1463cc116 100644 --- a/test/unittests/client/build/compiler_flags_test.py +++ b/test/unittests/client/build/compiler_flags_test.py @@ -1,33 +1,35 @@ import pytest +from conan.tools.build import default_cppstd, default_cstd from conan.tools.build.flags import architecture_flag, build_type_flags, threads_flags from conan.test.utils.mocks import MockSettings, ConanFileMock class TestCompilerFlags: - @pytest.mark.parametrize("compiler,arch,the_os,flag", [("gcc", "x86", None, "-m32"), - ("clang", "x86", None, "-m32"), - ("sun-cc", "x86", None, "-m32"), - ("gcc", "x86_64", None, "-m64"), - ("clang", "x86_64", None, "-m64"), - ("sun-cc", "x86_64", None, "-m64"), - ("sun-cc", "sparc", None, "-m32"), - ("sun-cc", "sparcv9", None, "-m64"), - ("gcc", "armv7", None, ""), - ("clang", "armv7", None, ""), - ("sun-cc", "armv7", None, ""), - ("gcc", "s390", None, "-m31"), - ("clang", "s390", None, "-m31"), - ("sun-cc", "s390", None, "-m31"), - ("gcc", "s390x", None, "-m64"), - ("clang", "s390x", None, "-m64"), - ("sun-cc", "s390x", None, "-m64"), - ("msvc", "x86", None, ""), - ("msvc", "x86_64", None, ""), - ("gcc", "ppc32", "AIX", "-maix32"), - ("gcc", "ppc64", "AIX", "-maix64"), - ]) + @pytest.mark.parametrize("compiler,arch,the_os,flag", + [("gcc", "x86", None, "-m32"), + ("clang", "x86", None, "-m32"), + ("sun-cc", "x86", None, "-m32"), + ("gcc", "x86_64", None, "-m64"), + ("clang", "x86_64", None, "-m64"), + ("sun-cc", "x86_64", None, "-m64"), + ("sun-cc", "sparc", None, "-m32"), + ("sun-cc", "sparcv9", None, "-m64"), + ("gcc", "armv7", None, ""), + ("clang", "armv7", None, ""), + ("sun-cc", "armv7", None, ""), + ("gcc", "s390", None, "-m31"), + ("clang", "s390", None, "-m31"), + ("sun-cc", "s390", None, "-m31"), + ("gcc", "s390x", None, "-m64"), + ("clang", "s390x", None, "-m64"), + ("sun-cc", "s390x", None, "-m64"), + ("msvc", "x86", None, ""), + ("msvc", "x86_64", None, ""), + ("gcc", "ppc32", "AIX", "-maix32"), + ("gcc", "ppc64", "AIX", "-maix64"), + ]) def test_arch_flag(self, compiler, arch, the_os, flag): settings = MockSettings({"compiler": compiler, "arch": arch, @@ -36,12 +38,12 @@ def test_arch_flag(self, compiler, arch, the_os, flag): conanfile.settings = settings assert architecture_flag(conanfile) == flag - - @pytest.mark.parametrize("compiler,threads,flag", [("clang", None, []), - ("emcc", None, []), - ("emcc", "posix", ["-pthread"]), - ("emcc", "wasm_workers", ["-sWASM_WORKERS=1"]), - ]) + @pytest.mark.parametrize("compiler,threads,flag", + [("clang", None, []), + ("emcc", None, []), + ("emcc", "posix", ["-pthread"]), + ("emcc", "wasm_workers", ["-sWASM_WORKERS=1"]), + ]) def test_threads_flag(self, compiler, threads, flag): settings = MockSettings({"compiler": compiler, "compiler.threads": threads}) @@ -49,9 +51,10 @@ def test_threads_flag(self, compiler, threads, flag): conanfile.settings = settings assert threads_flags(conanfile) == flag - @pytest.mark.parametrize("compiler,arch,the_os,flag", [("clang", "x86", "Windows", ""), - ("clang", "x86_64", "Windows", "") - ]) + @pytest.mark.parametrize("compiler,arch,the_os,flag", + [("clang", "x86", "Windows", ""), + ("clang", "x86_64", "Windows", "") + ]) def test_arch_flag_clangcl(self, compiler, arch, the_os, flag): settings = MockSettings({"compiler": compiler, "arch": arch, @@ -80,11 +83,12 @@ def test_catalyst(self): conanfile.settings = settings assert architecture_flag(conanfile) == "--target=arm64-apple-ios13.1-macabi" - @pytest.mark.parametrize("os_,arch,flag", [("Linux", "x86", "-m32"), - ("Linux", "x86_64", "-m64"), - ("Windows", "x86", "/Qm32"), - ("Windows", "x86_64", "/Qm64"), - ]) + @pytest.mark.parametrize("os_,arch,flag", + [("Linux", "x86", "-m32"), + ("Linux", "x86_64", "-m64"), + ("Windows", "x86", "/Qm32"), + ("Windows", "x86_64", "/Qm64"), + ]) def test_arch_flag_intel(self, os_, arch, flag): settings = MockSettings({"compiler": "intel-cc", "os": os_, @@ -93,13 +97,14 @@ def test_arch_flag_intel(self, os_, arch, flag): conanfile.settings = settings assert architecture_flag(conanfile) == flag - @pytest.mark.parametrize("arch,flag", [("e2k-v2", "-march=elbrus-v2"), - ("e2k-v3", "-march=elbrus-v3"), - ("e2k-v4", "-march=elbrus-v4"), - ("e2k-v5", "-march=elbrus-v5"), - ("e2k-v6", "-march=elbrus-v6"), - ("e2k-v7", "-march=elbrus-v7"), - ]) + @pytest.mark.parametrize("arch,flag", + [("e2k-v2", "-march=elbrus-v2"), + ("e2k-v3", "-march=elbrus-v3"), + ("e2k-v4", "-march=elbrus-v4"), + ("e2k-v5", "-march=elbrus-v5"), + ("e2k-v6", "-march=elbrus-v6"), + ("e2k-v7", "-march=elbrus-v7"), + ]) def test_arch_flag_mcst_lcc(self, arch, flag): conanfile = ConanFileMock() settings = MockSettings({"compiler": "mcst-lcc", @@ -107,31 +112,34 @@ def test_arch_flag_mcst_lcc(self, arch, flag): conanfile.settings = settings assert architecture_flag(conanfile) == flag - @pytest.mark.parametrize("compiler,build_type,vs_toolset,flags", [("msvc", "Debug", None, "-Zi -Ob0 -Od"), - ("msvc", "Release", None, "-O2 -Ob2"), - ("msvc", "RelWithDebInfo", None, "-Zi -O2 -Ob1"), - ("msvc", "MinSizeRel", None, "-O1 -Ob1"), - ("msvc", "Debug", "v140_clang_c2", "-gline-tables-only -fno-inline -O0"), - ("msvc", "Release", "v140_clang_c2", "-O2"), - ("msvc", "RelWithDebInfo", "v140_clang_c2", "-gline-tables-only -O2 -fno-inline"), - ("msvc", "MinSizeRel", "v140_clang_c2", ""), - ("gcc", "Debug", None, "-g"), - ("gcc", "Release", None, "-O3"), - ("gcc", "RelWithDebInfo", None, "-O2 -g"), - ("gcc", "MinSizeRel", None, "-Os"), - ("clang", "Debug", None, "-g"), - ("clang", "Release", None, "-O3"), - ("clang", "RelWithDebInfo", None, "-O2 -g"), - ("clang", "MinSizeRel", None, "-Os"), - ("apple-clang", "Debug", None, "-g"), - ("apple-clang", "Release", None, "-O3"), - ("apple-clang", "RelWithDebInfo", None, "-O2 -g"), - ("apple-clang", "MinSizeRel", None, "-Os"), - ("sun-cc", "Debug", None, "-g"), - ("sun-cc", "Release", None, "-xO3"), - ("sun-cc", "RelWithDebInfo", None, "-xO2 -g"), - ("sun-cc", "MinSizeRel", None, "-xO2 -xspace"), - ]) + @pytest.mark.parametrize("compiler,build_type,vs_toolset,flags", + [("msvc", "Debug", None, "-Zi -Ob0 -Od"), + ("msvc", "Release", None, "-O2 -Ob2"), + ("msvc", "RelWithDebInfo", None, "-Zi -O2 -Ob1"), + ("msvc", "MinSizeRel", None, "-O1 -Ob1"), + ("msvc", "Debug", "v140_clang_c2", + "-gline-tables-only -fno-inline -O0"), + ("msvc", "Release", "v140_clang_c2", "-O2"), + ("msvc", "RelWithDebInfo", "v140_clang_c2", + "-gline-tables-only -O2 -fno-inline"), + ("msvc", "MinSizeRel", "v140_clang_c2", ""), + ("gcc", "Debug", None, "-g"), + ("gcc", "Release", None, "-O3"), + ("gcc", "RelWithDebInfo", None, "-O2 -g"), + ("gcc", "MinSizeRel", None, "-Os"), + ("clang", "Debug", None, "-g"), + ("clang", "Release", None, "-O3"), + ("clang", "RelWithDebInfo", None, "-O2 -g"), + ("clang", "MinSizeRel", None, "-Os"), + ("apple-clang", "Debug", None, "-g"), + ("apple-clang", "Release", None, "-O3"), + ("apple-clang", "RelWithDebInfo", None, "-O2 -g"), + ("apple-clang", "MinSizeRel", None, "-Os"), + ("sun-cc", "Debug", None, "-g"), + ("sun-cc", "Release", None, "-xO3"), + ("sun-cc", "RelWithDebInfo", None, "-xO2 -g"), + ("sun-cc", "MinSizeRel", None, "-xO2 -xspace"), + ]) def test_build_type_flags(self, compiler, build_type, vs_toolset, flags): settings = MockSettings({"compiler": compiler, "build_type": build_type, @@ -140,11 +148,12 @@ def test_build_type_flags(self, compiler, build_type, vs_toolset, flags): conanfile.settings = settings assert ' '.join(build_type_flags(conanfile)) == flags - @pytest.mark.parametrize("compiler,build_type,flags", [("clang", "Debug", "-Zi -Ob0 -Od"), - ("clang", "Release", "-O2 -Ob2"), - ("clang", "RelWithDebInfo", "-Zi -O2 -Ob1"), - ("clang", "MinSizeRel", "-O1 -Ob1"), - ]) + @pytest.mark.parametrize("compiler,build_type,flags", + [("clang", "Debug", "-Zi -Ob0 -Od"), + ("clang", "Release", "-O2 -Ob2"), + ("clang", "RelWithDebInfo", "-Zi -O2 -Ob1"), + ("clang", "MinSizeRel", "-O1 -Ob1"), + ]) def test_build_type_flags_clangcl(self, compiler, build_type, flags): settings = MockSettings({"compiler": compiler, "build_type": build_type}) @@ -152,3 +161,25 @@ def test_build_type_flags_clangcl(self, compiler, build_type, flags): conanfile.conf.define("tools.build:compiler_executables", {"c": "clang-cl"}) conanfile.settings = settings assert ' '.join(build_type_flags(conanfile)) == flags + + @pytest.mark.parametrize("compiler,version,flags", + [("clang", "8", "gnu14"), + ("gcc", "15", "gnu17") + ]) + def test_default_cppstd(self, compiler, version, flags): + settings = MockSettings({"compiler": compiler, + "compiler.version": version}) + conanfile = ConanFileMock() + conanfile.settings = settings + assert default_cppstd(conanfile) == flags + + @pytest.mark.parametrize("compiler,version,flags", + [("clang", "8", "gnu11"), + ("gcc", "15", "gnu23") + ]) + def test_default_cstd(self, compiler, version, flags): + settings = MockSettings({"compiler": compiler, + "compiler.version": version}) + conanfile = ConanFileMock() + conanfile.settings = settings + assert default_cstd(conanfile) == flags diff --git a/test/unittests/client/build/cpp_std_flags_test.py b/test/unittests/client/build/cpp_std_flags_test.py index 14eb4ddf80a..800f269cbd0 100644 --- a/test/unittests/client/build/cpp_std_flags_test.py +++ b/test/unittests/client/build/cpp_std_flags_test.py @@ -1,5 +1,3 @@ -import pytest - from conan.internal.api.detect.detect_api import default_cppstd from conan.tools.build import cppstd_flag from conan.internal.model.version import Version @@ -9,8 +7,8 @@ def _make_cppstd_flag(compiler, compiler_version, cppstd=None): conanfile = ConanFileMock() conanfile.settings = MockSettings({"compiler": compiler, - "compiler.version": compiler_version, - "compiler.cppstd": cppstd}) + "compiler.version": compiler_version, + "compiler.cppstd": cppstd}) return cppstd_flag(conanfile) @@ -23,27 +21,27 @@ class TestCompilerFlags: def test_gcc_cppstd_flags(self): assert _make_cppstd_flag("gcc", "4.2", "98") == "-std=c++98" assert _make_cppstd_flag("gcc", "4.2", "gnu98") == "-std=gnu++98" - assert _make_cppstd_flag("gcc", "4.2", "11") == None - assert _make_cppstd_flag("gcc", "4.2", "14") == None + assert _make_cppstd_flag("gcc", "4.2", "11") is None + assert _make_cppstd_flag("gcc", "4.2", "14") is None assert _make_cppstd_flag("gcc", "4.3", "98") == "-std=c++98" assert _make_cppstd_flag("gcc", "4.3", "gnu98") == "-std=gnu++98" assert _make_cppstd_flag("gcc", "4.3", "11") == "-std=c++0x" - assert _make_cppstd_flag("gcc", "4.3", "14") == None + assert _make_cppstd_flag("gcc", "4.3", "14") is None assert _make_cppstd_flag("gcc", "4.6", "11") == '-std=c++0x' - assert _make_cppstd_flag("gcc", "4.6", "14") == None + assert _make_cppstd_flag("gcc", "4.6", "14") is None assert _make_cppstd_flag("gcc", "4.7", "11") == '-std=c++11' - assert _make_cppstd_flag("gcc", "4.7", "14") == None + assert _make_cppstd_flag("gcc", "4.7", "14") is None assert _make_cppstd_flag("gcc", "4.8", "11") == '-std=c++11' assert _make_cppstd_flag("gcc", "4.8", "14") == '-std=c++1y' - assert _make_cppstd_flag("gcc", "4.8", "17") == None + assert _make_cppstd_flag("gcc", "4.8", "17") is None assert _make_cppstd_flag("gcc", "4.9", "11") == '-std=c++11' assert _make_cppstd_flag("gcc", "4.9", "14") == '-std=c++14' - assert _make_cppstd_flag("gcc", "4.9", "17") == None + assert _make_cppstd_flag("gcc", "4.9", "17") is None assert _make_cppstd_flag("gcc", "5", "11") == '-std=c++11' assert _make_cppstd_flag("gcc", "5", "14") == '-std=c++14' @@ -63,14 +61,14 @@ def test_gcc_cppstd_flags(self): assert _make_cppstd_flag("gcc", "8", "14") == '-std=c++14' assert _make_cppstd_flag("gcc", "8", "17") == '-std=c++17' assert _make_cppstd_flag("gcc", "8", "20") == '-std=c++2a' - assert _make_cppstd_flag("gcc", "8", "23") == None + assert _make_cppstd_flag("gcc", "8", "23") is None assert _make_cppstd_flag("gcc", "11", "11") == '-std=c++11' assert _make_cppstd_flag("gcc", "11", "14") == '-std=c++14' assert _make_cppstd_flag("gcc", "11", "17") == '-std=c++17' assert _make_cppstd_flag("gcc", "11", "20") == '-std=c++20' assert _make_cppstd_flag("gcc", "11", "23") == '-std=c++23' - assert _make_cppstd_flag("gcc", "11", "26") == None + assert _make_cppstd_flag("gcc", "11", "26") is None assert _make_cppstd_flag("gcc", "14", "11") == '-std=c++11' assert _make_cppstd_flag("gcc", "14", "14") == '-std=c++14' @@ -98,25 +96,25 @@ def test_gcc_cppstd_defaults(self): assert _make_cppstd_default("gcc", "15.1") == "gnu17" def test_clang_cppstd_flags(self): - assert _make_cppstd_flag("clang", "2.0", "98") == None - assert _make_cppstd_flag("clang", "2.0", "gnu98") == None - assert _make_cppstd_flag("clang", "2.0", "11") == None - assert _make_cppstd_flag("clang", "2.0", "14") == None + assert _make_cppstd_flag("clang", "2.0", "98") is None + assert _make_cppstd_flag("clang", "2.0", "gnu98") is None + assert _make_cppstd_flag("clang", "2.0", "11") is None + assert _make_cppstd_flag("clang", "2.0", "14") is None assert _make_cppstd_flag("clang", "2.1", "98") == "-std=c++98" assert _make_cppstd_flag("clang", "2.1", "gnu98") == "-std=gnu++98" assert _make_cppstd_flag("clang", "2.1", "11") == "-std=c++0x" - assert _make_cppstd_flag("clang", "2.1", "14") == None + assert _make_cppstd_flag("clang", "2.1", "14") is None assert _make_cppstd_flag("clang", "3.0", "11") == '-std=c++0x' - assert _make_cppstd_flag("clang", "3.0", "14") == None + assert _make_cppstd_flag("clang", "3.0", "14") is None assert _make_cppstd_flag("clang", "3.1", "11") == '-std=c++11' - assert _make_cppstd_flag("clang", "3.1", "14") == None + assert _make_cppstd_flag("clang", "3.1", "14") is None assert _make_cppstd_flag("clang", "3.4", "11") == '-std=c++11' assert _make_cppstd_flag("clang", "3.4", "14") == '-std=c++1y' - assert _make_cppstd_flag("clang", "3.4", "17") == None + assert _make_cppstd_flag("clang", "3.4", "17") is None assert _make_cppstd_flag("clang", "3.5", "11") == '-std=c++11' assert _make_cppstd_flag("clang", "3.5", "14") == '-std=c++14' @@ -136,14 +134,14 @@ def test_clang_cppstd_flags(self): assert _make_cppstd_flag("clang", version, "14") == '-std=c++14' assert _make_cppstd_flag("clang", version, "17") == '-std=c++17' assert _make_cppstd_flag("clang", version, "20") == '-std=c++2a' - assert _make_cppstd_flag("clang", version, "23") == None + assert _make_cppstd_flag("clang", version, "23") is None assert _make_cppstd_flag("clang", "12", "11") == '-std=c++11' assert _make_cppstd_flag("clang", "12", "14") == '-std=c++14' assert _make_cppstd_flag("clang", "12", "17") == '-std=c++17' assert _make_cppstd_flag("clang", "12", "20") == '-std=c++20' assert _make_cppstd_flag("clang", "12", "23") == '-std=c++2b' - assert _make_cppstd_flag("clang", "12", "26") == None + assert _make_cppstd_flag("clang", "12", "26") is None assert _make_cppstd_flag("clang", "17", "11") == '-std=c++11' assert _make_cppstd_flag("clang", "17", "14") == '-std=c++14' @@ -174,20 +172,20 @@ def test_clang_cppstd_defaults(self): assert _make_cppstd_default("clang", "16") == "gnu17" def test_apple_clang_cppstd_flags(self): - assert _make_cppstd_flag("apple-clang", "3.9", "98") == None - assert _make_cppstd_flag("apple-clang", "3.9", "gnu98") == None - assert _make_cppstd_flag("apple-clang", "3.9", "11") == None - assert _make_cppstd_flag("apple-clang", "3.9", "14") == None + assert _make_cppstd_flag("apple-clang", "3.9", "98") is None + assert _make_cppstd_flag("apple-clang", "3.9", "gnu98") is None + assert _make_cppstd_flag("apple-clang", "3.9", "11") is None + assert _make_cppstd_flag("apple-clang", "3.9", "14") is None assert _make_cppstd_flag("apple-clang", "4.0", "98") == "-std=c++98" assert _make_cppstd_flag("apple-clang", "4.0", "gnu98") == "-std=gnu++98" assert _make_cppstd_flag("apple-clang", "4.0", "11") == "-std=c++11" - assert _make_cppstd_flag("apple-clang", "4.0", "14") == None + assert _make_cppstd_flag("apple-clang", "4.0", "14") is None assert _make_cppstd_flag("apple-clang", "5.0", "98") == "-std=c++98" assert _make_cppstd_flag("apple-clang", "5.0", "gnu98") == "-std=gnu++98" assert _make_cppstd_flag("apple-clang", "5.0", "11") == "-std=c++11" - assert _make_cppstd_flag("apple-clang", "5.0", "14") == None + assert _make_cppstd_flag("apple-clang", "5.0", "14") is None assert _make_cppstd_flag("apple-clang", "5.1", "98") == "-std=c++98" assert _make_cppstd_flag("apple-clang", "5.1", "gnu98") == "-std=gnu++98" @@ -213,7 +211,7 @@ def test_apple_clang_cppstd_flags(self): assert _make_cppstd_flag("apple-clang", "9.1", "11") == '-std=c++11' assert _make_cppstd_flag("apple-clang", "9.1", "14") == '-std=c++14' assert _make_cppstd_flag("apple-clang", "9.1", "17") == "-std=c++17" - assert _make_cppstd_flag("apple-clang", "9.1", "20") == None + assert _make_cppstd_flag("apple-clang", "9.1", "20") is None assert _make_cppstd_flag("apple-clang", "10.0", "17") == "-std=c++17" assert _make_cppstd_flag("apple-clang", "10.0", "20") == "-std=c++2a" @@ -222,7 +220,7 @@ def test_apple_clang_cppstd_flags(self): assert _make_cppstd_flag("apple-clang", "12.0", "17") == "-std=c++17" assert _make_cppstd_flag("apple-clang", "12.0", "20") == "-std=c++2a" - assert _make_cppstd_flag("apple-clang", "12.0", "23") == None + assert _make_cppstd_flag("apple-clang", "12.0", "23") is None assert _make_cppstd_flag("apple-clang", "13.0", "17") == "-std=c++17" assert _make_cppstd_flag("apple-clang", "13.0", "gnu17") == "-std=gnu++17" @@ -244,7 +242,7 @@ def test_apple_clang_cppstd_flags(self): assert _make_cppstd_flag("apple-clang", "15.0", "gnu20") == "-std=gnu++20" assert _make_cppstd_flag("apple-clang", "15.0", "23") == "-std=c++2b" assert _make_cppstd_flag("apple-clang", "15.0", "gnu23") == "-std=gnu++2b" - assert _make_cppstd_flag("apple-clang", "15.0", "26") == None + assert _make_cppstd_flag("apple-clang", "15.0", "26") is None assert _make_cppstd_flag("apple-clang", "16.0", "17") == "-std=c++17" assert _make_cppstd_flag("apple-clang", "16.0", "gnu17") == "-std=gnu++17" @@ -283,16 +281,16 @@ def test_apple_clang_cppstd_defaults(self): assert _make_cppstd_default("apple-clang", "17") == "gnu14" def test_visual_cppstd_flags(self): - assert _make_cppstd_flag("msvc", "170", "11") == None - assert _make_cppstd_flag("msvc", "170", "14") == None - assert _make_cppstd_flag("msvc", "170", "17") == None + assert _make_cppstd_flag("msvc", "170", "11") is None + assert _make_cppstd_flag("msvc", "170", "14") is None + assert _make_cppstd_flag("msvc", "170", "17") is None - assert _make_cppstd_flag("msvc", "180", "11") == None + assert _make_cppstd_flag("msvc", "180", "11") is None assert _make_cppstd_flag("msvc", "190", "14") == '/std:c++14' assert _make_cppstd_flag("msvc", "190", "17") == '/std:c++latest' - assert _make_cppstd_flag("msvc", "191", "11") == None + assert _make_cppstd_flag("msvc", "191", "11") is None assert _make_cppstd_flag("msvc", "191", "14") == '/std:c++14' assert _make_cppstd_flag("msvc", "191", "17") == '/std:c++17' assert _make_cppstd_flag("msvc", "191", "20") == '/std:c++latest' @@ -304,8 +302,8 @@ def test_visual_cppstd_flags(self): assert _make_cppstd_flag("msvc", "193", "23") == '/std:c++latest' def test_visual_cppstd_defaults(self): - assert _make_cppstd_default("msvc", "170") == None - assert _make_cppstd_default("msvc", "180") == None + assert _make_cppstd_default("msvc", "170") is None + assert _make_cppstd_default("msvc", "180") is None assert _make_cppstd_default("msvc", "190") == "14" assert _make_cppstd_default("msvc", "191") == "14" assert _make_cppstd_default("msvc", "192") == "14" @@ -333,40 +331,40 @@ def test_mcst_lcc_cppstd_defaults(self): def test_mcst_lcc_cppstd_flag(self): assert _make_cppstd_flag("mcst-lcc", "1.19", "98") == "-std=c++98" - assert _make_cppstd_flag("mcst-lcc", "1.19", "11") == None - assert _make_cppstd_flag("mcst-lcc", "1.19", "14") == None - assert _make_cppstd_flag("mcst-lcc", "1.19", "17") == None - assert _make_cppstd_flag("mcst-lcc", "1.19", "20") == None + assert _make_cppstd_flag("mcst-lcc", "1.19", "11") is None + assert _make_cppstd_flag("mcst-lcc", "1.19", "14") is None + assert _make_cppstd_flag("mcst-lcc", "1.19", "17") is None + assert _make_cppstd_flag("mcst-lcc", "1.19", "20") is None assert _make_cppstd_flag("mcst-lcc", "1.20", "98") == "-std=c++98" - assert _make_cppstd_flag("mcst-lcc", "1.20", "11") == None - assert _make_cppstd_flag("mcst-lcc", "1.20", "14") == None - assert _make_cppstd_flag("mcst-lcc", "1.20", "17") == None - assert _make_cppstd_flag("mcst-lcc", "1.20", "20") == None + assert _make_cppstd_flag("mcst-lcc", "1.20", "11") is None + assert _make_cppstd_flag("mcst-lcc", "1.20", "14") is None + assert _make_cppstd_flag("mcst-lcc", "1.20", "17") is None + assert _make_cppstd_flag("mcst-lcc", "1.20", "20") is None assert _make_cppstd_flag("mcst-lcc", "1.21", "98") == "-std=c++98" assert _make_cppstd_flag("mcst-lcc", "1.21", "11") == "-std=c++11" assert _make_cppstd_flag("mcst-lcc", "1.21", "14") == "-std=c++14" - assert _make_cppstd_flag("mcst-lcc", "1.21", "17") == None - assert _make_cppstd_flag("mcst-lcc", "1.21", "20") == None + assert _make_cppstd_flag("mcst-lcc", "1.21", "17") is None + assert _make_cppstd_flag("mcst-lcc", "1.21", "20") is None assert _make_cppstd_flag("mcst-lcc", "1.22", "98") == "-std=c++98" assert _make_cppstd_flag("mcst-lcc", "1.22", "11") == "-std=c++11" assert _make_cppstd_flag("mcst-lcc", "1.22", "14") == "-std=c++14" - assert _make_cppstd_flag("mcst-lcc", "1.22", "17") == None - assert _make_cppstd_flag("mcst-lcc", "1.22", "20") == None + assert _make_cppstd_flag("mcst-lcc", "1.22", "17") is None + assert _make_cppstd_flag("mcst-lcc", "1.22", "20") is None assert _make_cppstd_flag("mcst-lcc", "1.23", "98") == "-std=c++98" assert _make_cppstd_flag("mcst-lcc", "1.23", "11") == "-std=c++11" assert _make_cppstd_flag("mcst-lcc", "1.23", "14") == "-std=c++14" - assert _make_cppstd_flag("mcst-lcc", "1.23", "17") == None - assert _make_cppstd_flag("mcst-lcc", "1.23", "20") == None + assert _make_cppstd_flag("mcst-lcc", "1.23", "17") is None + assert _make_cppstd_flag("mcst-lcc", "1.23", "20") is None assert _make_cppstd_flag("mcst-lcc", "1.24", "98") == "-std=c++98" assert _make_cppstd_flag("mcst-lcc", "1.24", "11") == "-std=c++11" assert _make_cppstd_flag("mcst-lcc", "1.24", "14") == "-std=c++14" assert _make_cppstd_flag("mcst-lcc", "1.24", "17") == "-std=c++17" - assert _make_cppstd_flag("mcst-lcc", "1.24", "20") == None + assert _make_cppstd_flag("mcst-lcc", "1.24", "20") is None assert _make_cppstd_flag("mcst-lcc", "1.25", "98") == "-std=c++98" assert _make_cppstd_flag("mcst-lcc", "1.25", "11") == "-std=c++11" diff --git a/test/unittests/client/command/parse_arguments_test.py b/test/unittests/client/command/parse_arguments_test.py index fe2c12c391a..60e48bc3516 100644 --- a/test/unittests/client/command/parse_arguments_test.py +++ b/test/unittests/client/command/parse_arguments_test.py @@ -3,6 +3,7 @@ from conan.cli.args import add_profiles_args + @pytest.mark.parametrize("argument", [ ["options", "-o", "--options"], ["profile", "-pr", "--profile"], diff --git a/test/unittests/client/conanfile_loader_test.py b/test/unittests/client/conanfile_loader_test.py index 1403117fb0c..f85df462811 100644 --- a/test/unittests/client/conanfile_loader_test.py +++ b/test/unittests/client/conanfile_loader_test.py @@ -3,7 +3,6 @@ import textwrap import pytest -from parameterized import parameterized from conan.internal.loader import ConanFileLoader, ConanFileTextLoader, load_python_file from conan.errors import ConanException diff --git a/test/unittests/client/conf/detect/test_gcc_compiler.py b/test/unittests/client/conf/detect/test_gcc_compiler.py index ae961c6c0b4..e0067612866 100644 --- a/test/unittests/client/conf/detect/test_gcc_compiler.py +++ b/test/unittests/client/conf/detect/test_gcc_compiler.py @@ -1,5 +1,5 @@ -import mock +from unittest import mock import pytest from conan.internal.api.detect.detect_api import detect_gcc_compiler diff --git a/test/unittests/client/graph/deps_graph_test.py b/test/unittests/client/graph/deps_graph_test.py index 45b9360fd44..34db4234d5b 100644 --- a/test/unittests/client/graph/deps_graph_test.py +++ b/test/unittests/client/graph/deps_graph_test.py @@ -1,4 +1,4 @@ -from mock import Mock +from unittest.mock import Mock from conan.internal.graph.graph import CONTEXT_HOST from conan.internal.graph.graph_builder import DepsGraph, Node diff --git a/test/unittests/client/remote_manager_test.py b/test/unittests/client/remote_manager_test.py index cbad3cba042..60f01384ea9 100644 --- a/test/unittests/client/remote_manager_test.py +++ b/test/unittests/client/remote_manager_test.py @@ -1,15 +1,32 @@ import os +import sys + import pytest from conan.internal.api.uploader import compress_files -from conan.internal.paths import PACKAGE_TGZ_NAME +from conan.internal.rest.remote_manager import uncompress_file from conan.test.utils.test_files import temp_folder from conan.internal.util.files import save class TestRemoteManager: - def test_compress_files(self): + def test_compress_files_tgz(self): + folder = temp_folder() + save(os.path.join(folder, "one_file.txt"), "The contents") + save(os.path.join(folder, "Two_file.txt"), "Two contents") + + files = { + "one_file.txt": os.path.join(folder, "one_file.txt"), + "Two_file.txt": os.path.join(folder, "Two_file.txt"), + } + + path = compress_files(files, "conan_package.tgz", dest_dir=folder) + assert os.path.exists(path) + expected_path = os.path.join(folder, "conan_package.tgz") + assert path == expected_path + + def test_compress_and_uncompress_xz_files(self): folder = temp_folder() save(os.path.join(folder, "one_file.txt"), "The contents") save(os.path.join(folder, "Two_file.txt"), "Two contents") @@ -19,7 +36,47 @@ def test_compress_files(self): "Two_file.txt": os.path.join(folder, "Two_file.txt"), } - path = compress_files(files, PACKAGE_TGZ_NAME, dest_dir=folder) + path = compress_files(files, "conan_package.txz", dest_dir=folder) assert os.path.exists(path) - expected_path = os.path.join(folder, PACKAGE_TGZ_NAME) + expected_path = os.path.join(folder, "conan_package.txz") assert path == expected_path + + extract_dir = os.path.join(folder, "extracted") + uncompress_file(path, extract_dir) + + extract_files = list(sorted(os.listdir(extract_dir))) + expected_files = sorted(files.keys()) + assert extract_files == expected_files + + for name, path in files.items(): + extract_path = os.path.join(extract_dir, name) + with open(path, "r") as f1, open(extract_path, "r") as f2: + assert f1.read() == f2.read() + + @pytest.mark.skipif(sys.version_info.minor < 14, reason="zstd needs Python >= 3.14") + def test_compress_and_uncompress_zst_files(self): + folder = temp_folder() + save(os.path.join(folder, "one_file.txt"), "The contents") + save(os.path.join(folder, "Two_file.txt"), "Two contents") + + files = { + "one_file.txt": os.path.join(folder, "one_file.txt"), + "Two_file.txt": os.path.join(folder, "Two_file.txt"), + } + + path = compress_files(files, "conan_package.tzst", dest_dir=folder) + assert os.path.exists(path) + expected_path = os.path.join(folder, "conan_package.tzst") + assert path == expected_path + + extract_dir = os.path.join(folder, "extracted") + uncompress_file(path, extract_dir) + + extract_files = list(sorted(os.listdir(extract_dir))) + expected_files = sorted(files.keys()) + assert extract_files == expected_files + + for name, path in sorted(files.items()): + extract_path = os.path.join(extract_dir, name) + with open(path, "r") as f1, open(extract_path, "r") as f2: + assert f1.read() == f2.read() diff --git a/test/unittests/client/rest/requester_test.py b/test/unittests/client/rest/requester_test.py index dd97d8cdf38..0e6b2c41627 100644 --- a/test/unittests/client/rest/requester_test.py +++ b/test/unittests/client/rest/requester_test.py @@ -14,4 +14,4 @@ def test_requester_reinit(): conan_api = ConanAPI(tmp_folder) save(os.path.join(home_path.global_conf_path), "core.net.http:timeout=(10, 20)") conan_api.reinit() - assert conan_api.remotes.requester._timeout == (10, 20) + assert conan_api._api_helpers.requester._timeout == (10, 20) diff --git a/test/unittests/client/rest/response_test.py b/test/unittests/client/rest/response_test.py index 0c35a2c4abb..056371ae193 100644 --- a/test/unittests/client/rest/response_test.py +++ b/test/unittests/client/rest/response_test.py @@ -5,7 +5,7 @@ class TestRestString: def _get_html_response(self): - headers = {"content-type": "text/html;charset=utf-8",} + headers = {"content-type": "text/html;charset=utf-8"} return namedtuple("Response", "status_code headers reason content")( 404, headers, diff --git a/test/unittests/client/rest/rest_client_v2/rest_client_v2_test.py b/test/unittests/client/rest/rest_client_v2/rest_client_v2_test.py index 96f009d7468..ad53f0b87e1 100644 --- a/test/unittests/client/rest/rest_client_v2/rest_client_v2_test.py +++ b/test/unittests/client/rest/rest_client_v2/rest_client_v2_test.py @@ -1,30 +1,26 @@ from conan.internal.rest.rest_client_v2 import RestV2Methods -from mock import MagicMock, patch +from unittest.mock import MagicMock, patch @patch("conan.internal.rest.conan_requester.ConanRequester") -def test_check_credentials_set_token_to_unset_with_force_auth_true_and_token_none( - conanRequesterMock, -): +def test_check_credentials_set_token_to_unset_with_force_auth_true_and_token_none(requester_mock): return_status = MagicMock() return_status.status_code = 200 - conanRequesterMock.get = MagicMock(return_value=return_status) + requester_mock.get = MagicMock(return_value=return_status) - rest_v2_methods = RestV2Methods("foo", None, conanRequesterMock, None, True) + rest_v2_methods = RestV2Methods("foo", None, requester_mock, None, True) rest_v2_methods.check_credentials(True) - assert conanRequesterMock.get.call_args[1]["auth"].bearer == "Bearer unset" + assert requester_mock.get.call_args[1]["auth"].bearer == "Bearer unset" @patch("conan.internal.rest.conan_requester.ConanRequester") -def test_check_credentials_keep_token_none_with_force_auth_false_and_token_none( - conanRequesterMock, -): +def test_check_credentials_keep_token_none_with_force_auth_false_and_token_none(requester_mock): return_status = MagicMock() return_status.status_code = 200 - conanRequesterMock.get = MagicMock(return_value=return_status) + requester_mock.get = MagicMock(return_value=return_status) - rest_v2_methods = RestV2Methods("foo", None, conanRequesterMock, None, True) + rest_v2_methods = RestV2Methods("foo", None, requester_mock, None, True) rest_v2_methods.check_credentials(False) - assert conanRequesterMock.get.call_args[1]["auth"].bearer is None + assert requester_mock.get.call_args[1]["auth"].bearer is None diff --git a/test/unittests/client/toolchain/autotools/autotools_test.py b/test/unittests/client/toolchain/autotools/autotools_test.py deleted file mode 100644 index 58388c4c959..00000000000 --- a/test/unittests/client/toolchain/autotools/autotools_test.py +++ /dev/null @@ -1,77 +0,0 @@ -import os - -from conan.tools.build import save_toolchain_args -from conan.tools.gnu import Autotools -from conan import ConanFile -from conan.internal.model.conf import Conf -from test.unittests.util.tools_test import RunnerMock -from conan.test.utils.mocks import MockSettings -from conan.test.utils.test_files import temp_folder - - -def test_configure_arguments(): - tmp = temp_folder() - os.chdir(tmp) - save_toolchain_args({ - "configure_args": "my_configure_args", - "make_args": "my_make_args"} - ) - runner = RunnerMock() - conanfile = ConanFile() - conanfile.run = runner - conanfile.settings = MockSettings({"os": "Linux"}) - conanfile.settings_build = conanfile.settings - conanfile.folders.set_base_source(tmp) - conanfile.conf = Conf() - conanfile.conf.define("tools.gnu:make_program", "my_make") - conanfile.conf.define("tools.build:jobs", 23) - ab = Autotools(conanfile) - ab.configure() - assert "configure\" my_configure_args" in runner.command_called - - ab = Autotools(conanfile) - ab.make() - assert "my_make my_make_args -j23" == runner.command_called - - # test install target argument - - ab.install() - assert 'my_make install my_make_args DESTDIR=None -j23' == runner.command_called - - ab.install(target="install_other") - assert 'my_make install_other my_make_args DESTDIR=None -j23' == runner.command_called - - for make_args in ["my_make_args", ""]: - - save_toolchain_args({ - "configure_args": "my_configure_args", - "make_args": f"{make_args}"} - ) - - ab = Autotools(conanfile) - - make_args_str = f" {make_args}" if make_args else "" - - ab.make(args=["-j1"]) - assert "-j23" not in runner.command_called - assert f"my_make{make_args_str} -j1" == runner.command_called - - ab.install(args=["-j1"]) - assert "-j23" not in runner.command_called - assert f"my_make install{make_args_str} DESTDIR=None -j1" == runner.command_called - - ab.install(args=["DESTDIR=whatever", "-j1"]) - assert "-j23" not in runner.command_called - assert f"my_make install{make_args_str} DESTDIR=whatever -j1" == runner.command_called - - ab.install(args=["DESTDIR=whatever", "-arg1 -j1 -arg2"]) - assert "-j23" not in runner.command_called - assert f"my_make install{make_args_str} DESTDIR=whatever -arg1 -j1 -arg2" == runner.command_called - - # check that we don't detect -j in an argument as number of jobs - ab.install(args=["DESTDIR=/user/smith-john/whatever"]) - assert f"my_make install{make_args_str} DESTDIR=/user/smith-john/whatever -j23" == runner.command_called - - # check that we don't detect -j in an argument as number of jobs - ab.install(args=["DESTDIR=/user/smith-j47/whatever"]) - assert f"my_make install{make_args_str} DESTDIR=/user/smith-j47/whatever -j23" == runner.command_called diff --git a/test/unittests/client/tools/cppstd_required_test.py b/test/unittests/client/tools/cppstd_required_test.py index c919a9c82eb..ae09b556e5a 100644 --- a/test/unittests/client/tools/cppstd_required_test.py +++ b/test/unittests/client/tools/cppstd_required_test.py @@ -1,5 +1,5 @@ import pytest -from mock import mock +from unittest import mock from conan.tools.build import check_max_cppstd, check_min_cppstd, valid_max_cppstd, valid_min_cppstd, \ valid_min_cstd, valid_max_cstd diff --git a/test/unittests/client/tools/files/rename_test.py b/test/unittests/client/tools/files/rename_test.py index 4dff04a895d..d30c4b64a15 100644 --- a/test/unittests/client/tools/files/rename_test.py +++ b/test/unittests/client/tools/files/rename_test.py @@ -6,34 +6,13 @@ class TestRename: - def verify_dir(self, basedir): - assert os.path.isdir(basedir) - - assert os.path.isfile(os.path.join(basedir, "1.txt")) - assert os.path.isfile(os.path.join(basedir, "1.pdb")) - assert os.path.isfile(os.path.join(basedir, "1.pdb1")) - - assert os.path.isdir(os.path.join(basedir, "dir.pdb")) - assert os.path.isfile(os.path.join(basedir, "dir.pdb", "2.txt")) - assert os.path.isfile(os.path.join(basedir, "dir.pdb", "2.pdb")) - assert os.path.isfile(os.path.join(basedir, "dir.pdb", "2.pdb1")) - - assert os.path.isdir(os.path.join(basedir, "middir")) - assert os.path.isfile(os.path.join(basedir, "middir", "3.txt")) - assert os.path.isfile(os.path.join(basedir, "middir", "3.pdb")) - assert os.path.isfile(os.path.join(basedir, "middir", "3.pdb1")) - - assert os.path.isdir(os.path.join(basedir, "middir", "deepdir")) - assert os.path.isfile(os.path.join(basedir, "middir", "deepdir", "4.txt")) - assert os.path.isfile(os.path.join(basedir, "middir", "deepdir", "4.pdb")) - assert os.path.isfile(os.path.join(basedir, "middir", "deepdir", "4.pdb1")) def test_rename(self): client = TestClient() - tmpdir = client.current_folder + folder = client.current_folder sub_space_dir = "sub dir" - with chdir(tmpdir): + with chdir(folder): os.makedirs(sub_space_dir) os.makedirs(os.path.join(sub_space_dir, "dir.pdb")) os.makedirs(os.path.join(sub_space_dir, "middir")) @@ -51,19 +30,42 @@ def test_rename(self): os.path.join(sub_space_dir, "middir", "deepdir", "4.txt"): "", os.path.join(sub_space_dir, "middir", "deepdir", "4.pdb"): "", os.path.join(sub_space_dir, "middir", "deepdir", "4.pdb1"): "" - }) - self.verify_dir(os.path.join(tmpdir, sub_space_dir)) + }) + + def verify_dir(basedir): + assert os.path.isdir(basedir) + + assert os.path.isfile(os.path.join(basedir, "1.txt")) + assert os.path.isfile(os.path.join(basedir, "1.pdb")) + assert os.path.isfile(os.path.join(basedir, "1.pdb1")) + + assert os.path.isdir(os.path.join(basedir, "dir.pdb")) + assert os.path.isfile(os.path.join(basedir, "dir.pdb", "2.txt")) + assert os.path.isfile(os.path.join(basedir, "dir.pdb", "2.pdb")) + assert os.path.isfile(os.path.join(basedir, "dir.pdb", "2.pdb1")) + + assert os.path.isdir(os.path.join(basedir, "middir")) + assert os.path.isfile(os.path.join(basedir, "middir", "3.txt")) + assert os.path.isfile(os.path.join(basedir, "middir", "3.pdb")) + assert os.path.isfile(os.path.join(basedir, "middir", "3.pdb1")) + + assert os.path.isdir(os.path.join(basedir, "middir", "deepdir")) + assert os.path.isfile(os.path.join(basedir, "middir", "deepdir", "4.txt")) + assert os.path.isfile(os.path.join(basedir, "middir", "deepdir", "4.pdb")) + assert os.path.isfile(os.path.join(basedir, "middir", "deepdir", "4.pdb1")) + + verify_dir(os.path.join(folder, sub_space_dir)) - with chdir(tmpdir): + with chdir(folder): rename(None, sub_space_dir, "dst dir") - self.verify_dir(os.path.join(tmpdir, "dst dir")) + verify_dir(os.path.join(folder, "dst dir")) - rename(None,"dst dir", "subdir") - self.verify_dir(os.path.join(tmpdir, "subdir")) + rename(None, "dst dir", "subdir") + verify_dir(os.path.join(folder, "subdir")) rename(None, os.path.join("subdir", "1.txt"), "t.txt") - assert os.path.isfile(os.path.join(tmpdir, "t.txt")) - assert not os.path.isfile(os.path.join(tmpdir, "subdir", "1.txt")) + assert os.path.isfile(os.path.join(folder, "t.txt")) + assert not os.path.isfile(os.path.join(folder, "subdir", "1.txt")) def test_rename_empty_folder(self): client = TestClient() diff --git a/test/unittests/client/tools/test_env.py b/test/unittests/client/tools/test_env.py index dd310468ace..3d0248c2a7b 100644 --- a/test/unittests/client/tools/test_env.py +++ b/test/unittests/client/tools/test_env.py @@ -1,5 +1,5 @@ import os -import mock +from unittest import mock import conan.test.utils.env diff --git a/test/unittests/client/userio_test.py b/test/unittests/client/userio_test.py index 105ee3f772d..d6f5e668a9a 100644 --- a/test/unittests/client/userio_test.py +++ b/test/unittests/client/userio_test.py @@ -1,4 +1,3 @@ -import pytest from unittest.mock import patch from conan.api.input import UserInput diff --git a/test/unittests/model/build_info/components_test.py b/test/unittests/model/build_info/components_test.py index 500cb9bff5a..7efe505079d 100644 --- a/test/unittests/model/build_info/components_test.py +++ b/test/unittests/model/build_info/components_test.py @@ -15,7 +15,7 @@ def test_components_set(self): import pytest with pytest.raises(AttributeError): - cpp_info.required_components = ["liba::liba"] + cpp_info.required_components = ["liba::liba"] # noqa def test_no_components_inside_components(self): cpp_info = CppInfo() diff --git a/test/unittests/model/build_info/new_build_info_test.py b/test/unittests/model/build_info/new_build_info_test.py index c554de8d048..31baafd9006 100644 --- a/test/unittests/model/build_info/new_build_info_test.py +++ b/test/unittests/model/build_info/new_build_info_test.py @@ -64,7 +64,7 @@ def test_component_aggregation(): # for example, cannot be aggregated. But "cmake_build_modules" is aggregated. assert ret.get_property("my_foo") is None assert ret.get_property("my_foo2") is None - assert ret.get_property("cmake_build_modules") == None + assert ret.get_property("cmake_build_modules") is None # If we change the internal graph the order is different cppinfo.components["c1"].requires = [] @@ -136,4 +136,4 @@ def test_cpp_info_merge_aggregating_components_first(aggregate_first): "jar_{}_2".format(n)] assert getattr(cppinfo.components["boo2"], n) == ["jar2_{}_1".format(n), "jar2_{}_2".format(n)] - assert getattr(cppinfo, n) == None + assert getattr(cppinfo, n) is None diff --git a/test/unittests/model/options_test.py b/test/unittests/model/options_test.py index f8cf27e069c..3ec7a53b697 100644 --- a/test/unittests/model/options_test.py +++ b/test/unittests/model/options_test.py @@ -17,8 +17,8 @@ def _setup(self): def test_booleans(self): assert self.sut.static - assert self.sut.static == True - assert self.sut.static != False + assert self.sut.static == True # noqa + assert self.sut.static != False # noqa assert bool(self.sut.static) assert self.sut.optimized @@ -117,17 +117,17 @@ def test_items(self): assert self.sut.items() == [("optimized", "3"), ("path", "mypath"), ("static", "True")] def test_get_safe_options(self): - assert True == self.sut.get_safe("static") + assert True == self.sut.get_safe("static") # noqa assert 3 == self.sut.get_safe("optimized") assert "mypath" == self.sut.get_safe("path") - assert None == self.sut.get_safe("unknown") + assert None == self.sut.get_safe("unknown") # noqa self.sut.path = "None" self.sut.static = False - assert False == self.sut.get_safe("static") + assert False == self.sut.get_safe("static") # noqa assert "None" == self.sut.get_safe("path") - assert False == self.sut.get_safe("static", True) + assert False == self.sut.get_safe("static", True) # noqa assert "None" == self.sut.get_safe("path", True) - assert True == self.sut.get_safe("unknown", True) + assert True == self.sut.get_safe("unknown", True) # noqa class TestOptionsLoad: @@ -183,28 +183,28 @@ def _setup(self): self.sut = Options(options) def test_booleans(self): - assert self.sut.static == None + assert self.sut.static == None # noqa assert not self.sut.static assert self.sut.static != 1 assert self.sut.static != 2 with pytest.raises(ConanException) as e: - self.sut.static == 3 + self.sut.static == 3 # noqa assert "'3' is not a valid 'options.static' value" in str(e.value) with pytest.raises(ConanException) as e: - self.sut.static == "None" + self.sut.static == "None" # noqa assert "'None' is not a valid 'options.static' value" in str(e.value) - assert self.sut.other == None + assert self.sut.other == None # noqa assert self.sut.other != "whatever" # dont raise, ANY self.sut.other = None - assert self.sut.other == None + assert self.sut.other == None # noqa assert not self.sut.more - assert self.sut.more == None + assert self.sut.more == None # noqa assert self.sut.more != 1 with pytest.raises(ConanException) as e: - self.sut.more == 2 + self.sut.more == 2 # noqa assert "'2' is not a valid 'options.more' value" in str(e.value) with pytest.raises(ConanException) as e: self.sut.more = None @@ -212,7 +212,7 @@ def test_booleans(self): self.sut.more = "None" assert not self.sut.more # This is still evaluated to false, like OFF, 0, FALSE, etc assert self.sut.more == "None" - assert self.sut.more != None + assert self.sut.more != None # noqa def test_assign(self): self.sut.static = 1 @@ -228,14 +228,14 @@ def test_boolean_none(self): assert not (options.static == 1) assert options.static != "None" assert not (options.static == "None") - assert options.static == None - assert not (options.static != None) + assert options.static == None # noqa + assert not (options.static != None) # noqa options.static = "None" assert options.static == "None" assert not (options.static != "None") - assert not (options.static == None) - assert options.static != None + assert not (options.static == None) # noqa + assert options.static != None # noqa def test_undefined_value(self): """ Not assigning a value to options will raise an error at validate() step diff --git a/test/unittests/model/other_settings_test.py b/test/unittests/model/other_settings_test.py index 32e40b28688..0f3c71b4916 100644 --- a/test/unittests/model/other_settings_test.py +++ b/test/unittests/model/other_settings_test.py @@ -4,59 +4,51 @@ from conan.internal.model.info import load_binary_info from conan.api.model import RecipeReference from conan.internal.paths import CONANFILE +from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient from conan.internal.util.files import load, save class TestSettings: - def _get_conaninfo(self, reference, client): - ref = client.cache.get_latest_recipe_reference(RecipeReference.loads(reference)) + @staticmethod + def _get_conaninfo(reference, client): + ref = client.cache.get_latest_recipe_revision(RecipeReference.loads(reference)) pkg_ids = client.cache.get_package_references(ref) - pref = client.cache.get_latest_package_reference(pkg_ids[0]) + pref = client.cache.get_latest_package_revision(pkg_ids[0]) pkg_folder = client.cache.pkg_layout(pref).package() return load_binary_info(client.load(os.path.join(pkg_folder, "conaninfo.txt"))) def test_wrong_settings(self): - settings = """os: - null: - subsystem: [null, msys] -""" + settings = textwrap.dedent("""\ + os: + null: + subsystem: [null, msys] + """) client = TestClient() save(client.paths.settings_path, settings) client.save_home({"profiles/default": ""}) - conanfile = """from conan import ConanFile -class Pkg(ConanFile): - settings = "os", "compiler" -""" - client.save({"conanfile.py": conanfile}) - client.run("create . --name=pkg --version=0.1 --user=lasote --channel=testing", assert_error=True) + client.save({"conanfile.py": GenConanfile().with_settings("os", "compiler")}) + client.run("create . --name=pkg --version=0.1", assert_error=True) assert "ERROR: settings.yml: null setting can't have subsettings" in client.out def test_settings_constraint_error_type(self): # https://github.com/conan-io/conan/issues/3022 - conanfile = """from conan import ConanFile -class Test(ConanFile): - settings = "os" - def build(self): - self.output.info("OS!!: %s" % self.settings.os) - """ + conanfile = textwrap.dedent(""" + from conan import ConanFile + class Test(ConanFile): + settings = "os" + def build(self): + self.output.info("OS!!: %s" % self.settings.os) + """) client = TestClient() client.save({"conanfile.py": conanfile}) client.run("create . --name=pkg --version=0.1 --user=user --channel=testing -s os=Linux") assert "pkg/0.1@user/testing: OS!!: Linux" in client.out def test_settings_as_a_str(self): - content = """ -from conan import ConanFile - -class SayConan(ConanFile): - name = "say" - version = "0.1" - settings = "os" -""" client = TestClient() - client.save({CONANFILE: content}) + client.save({CONANFILE: GenConanfile("say", "0.1").with_settings("os")}) client.run("create . -s os=Windows --build missing") # Now read the conaninfo and verify that settings applied is only os and value is windows conan_info = self._get_conaninfo("say/0.1@", client) @@ -70,31 +62,22 @@ class SayConan(ConanFile): def test_settings_as_a_list_conanfile(self): # Now with conanfile as a list - content = """ -from conan import ConanFile - -class SayConan(ConanFile): - name = "say" - version = "0.1" - settings = "os", "arch" -""" client = TestClient() - client.save({CONANFILE: content}) + client.save({CONANFILE: GenConanfile("say", "0.1").with_settings("os", "arch")}) client.run("create . -s os=Windows --build missing") conan_info = self._get_conaninfo("say/0.1@", client) assert conan_info["settings"]["os"] == "Windows" def test_settings_as_a_dict_conanfile(self): - # Now with conanfile as a dict - # XXX: this test only works on machines w default arch "x86", "x86_64", "sparc" or "sparcv9" - content = """ -from conan import ConanFile - -class SayConan(ConanFile): - name = "say" - version = "0.1" - settings = {"os", "arch"} -""" + # Now with conanfile as a set + content = textwrap.dedent(""" + from conan import ConanFile + + class SayConan(ConanFile): + name = "say" + version = "0.1" + settings = {"os", "arch"} + """) client = TestClient() client.save({CONANFILE: content}) client.run("create . -s os=Windows --build missing") @@ -103,31 +86,15 @@ class SayConan(ConanFile): def test_invalid_settings3(self): client = TestClient() - # Test wrong settings in conanfile - content = textwrap.dedent(""" - from conan import ConanFile - - class SayConan(ConanFile): - settings = "invalid" - """) - - client.save({CONANFILE: content}) + client.save({CONANFILE: GenConanfile().with_settings("invalid")}) client.run("install . --build missing", assert_error=True) assert "'settings.invalid' doesn't exist" in client.out # Test wrong values in conanfile def test_invalid_settings4(self): - content = """ -from conan import ConanFile - -class SayConan(ConanFile): - name = "say" - version = "0.1" - settings = "os" -""" client = TestClient() - client.save({CONANFILE: content}) + client.save({CONANFILE: GenConanfile("say", "0.1").with_settings("os")}) client.run("create . -s os=ChromeOS --build missing", assert_error=True) assert "ERROR: Invalid setting 'ChromeOS' is not a valid 'settings.os' value." in client.out assert "Possible values are ['Windows', 'WindowsStore', 'WindowsCE', 'Linux'" in client.out @@ -141,32 +108,32 @@ class SayConan(ConanFile): client.run("create . -s os=ChromeOS --build missing") # Settings is None - content = """ -from conan import ConanFile - -class SayConan(ConanFile): - name = "say" - version = "0.1" - settings = None -""" + content = textwrap.dedent(""" + from conan import ConanFile + + class SayConan(ConanFile): + name = "say" + version = "0.1" + settings = None + """) client.save({CONANFILE: content}) client.run("remove say/0.1 -c") client.run("create . --build missing") conan_info = self._get_conaninfo("say/0.1", client) - assert conan_info.get("settings") == None + assert conan_info.get("settings") == None # noqa # Settings is {} - content = """ -from conan import ConanFile - -class SayConan(ConanFile): - name = "say" - version = "0.1" - settings = {} -""" + content = textwrap.dedent(""" + from conan import ConanFile + + class SayConan(ConanFile): + name = "say" + version = "0.1" + settings = {} + """) client.save({CONANFILE: content}) client.run("remove say/0.1 -c") client.run("create . --build missing") conan_info = self._get_conaninfo("say/0.1", client) - assert conan_info.get("settings") == None + assert conan_info.get("settings") == None # noqa diff --git a/test/unittests/model/settings_test.py b/test/unittests/model/settings_test.py index 226e384c681..c881b047c48 100644 --- a/test/unittests/model/settings_test.py +++ b/test/unittests/model/settings_test.py @@ -263,7 +263,7 @@ def test_constraint(self): s2 = ["os"] self.sut.constrained(s2) with pytest.raises(ConanException) as cm: - self.sut.compiler + self.sut.compiler # noqa assert str(cm.value) == str(undefined_field("settings", "compiler", ["os"], "settings")) self.sut.os = "Windows" self.sut.os = "Linux" @@ -272,16 +272,17 @@ def test_constraint2(self): s2 = ["os2"] with pytest.raises(ConanException) as cm: self.sut.constrained(s2) - assert str(cm.value) == str(undefined_field("settings", "os2", ["compiler", "os"], "settings")) + assert str(cm.value) == str(undefined_field("settings", "os2", ["compiler", "os"], + "settings")) def test_constraint6(self): s2 = {"os", "compiler"} self.sut.constrained(s2) self.sut.compiler = "Visual Studio" with pytest.raises(ConanException) as cm: - self.sut.compiler.arch + self.sut.compiler.arch # noqa assert str(cm.value) == str(undefined_field("settings.compiler", "arch", - ['runtime', 'version'], "Visual Studio")) + ['runtime', 'version'], "Visual Studio")) self.sut.os = "Windows" self.sut.compiler.version = "11" self.sut.compiler.version = "12" @@ -309,10 +310,10 @@ def test_validate(self): self.sut.os = "Windows" self.sut.validate() assert self.sut.values_list == [("compiler", "gcc"), - ("compiler.arch", "x86"), - ("compiler.arch.speed", "A"), - ("compiler.version", "4.8"), - ("os", "Windows")] + ("compiler.arch", "x86"), + ("compiler.arch.speed", "A"), + ("compiler.version", "4.8"), + ("os", "Windows")] def test_validate2(self): self.sut.os = "Windows" @@ -351,11 +352,11 @@ def test_my(self): assert self.sut.compiler == "Visual Studio" with pytest.raises(ConanException) as cm: - self.sut.compiler.kk - assert str(cm.value) == str(undefined_field("settings.compiler", "kk", ['runtime', 'version'], - "Visual Studio")) + self.sut.compiler.kk # noqa + assert str(cm.value) == str(undefined_field("settings.compiler", "kk", + ['runtime', 'version'], "Visual Studio")) - assert self.sut.compiler.version == None + assert self.sut.compiler.version == None # noqa with pytest.raises(ConanException) as cm: self.sut.compiler.version = "123" @@ -381,9 +382,9 @@ def test_my(self): self.sut.compiler = "gcc" with pytest.raises(ConanException) as cm: - self.sut.compiler.runtime - assert str(cm.value) == str(undefined_field("settings.compiler", "runtime", "['arch', 'version']", - "gcc")) + self.sut.compiler.runtime # noqa + assert str(cm.value) == str(undefined_field("settings.compiler", "runtime", + "['arch', 'version']", "gcc")) self.sut.compiler.arch = "x86" self.sut.compiler.arch.speed = "A" diff --git a/test/unittests/model/test_list.py b/test/unittests/model/test_list.py new file mode 100644 index 00000000000..ae68ce70564 --- /dev/null +++ b/test/unittests/model/test_list.py @@ -0,0 +1,49 @@ +from conan.api.model import PackagesList + + +def test_package_list_only_recipes(): + data = { + "foobar/0.1.0": {'revisions': + {'85eb0587a3c12b90216c72070e9eef3e': + {'timestamp': 1740151190.975, + 'packages': {'c29f32beda3c361f536ba35aae5e876faf970169': + {'revisions': { + '9a0569c04839fff6a6e8f40e10434bc6': + {'timestamp': 1742417567.684486}}, + 'info': {}}}}}}, + "qux/0.2.1": {'revisions': + {'71c3c11b98a6f2ae11f0f391f5e62e2b': + {'timestamp': 1740151186.976, 'packages': + {'13be611585c95453f1cbbd053cea04b3e64470ca': + {'revisions': {'5c7e0c0788cd4d89249ad251f0868787': + {'timestamp': 1740152032.884}}, + 'info': + {'settings': + {'arch': 'x86_64', + 'build_type': 'Release', + 'compiler': 'gcc', + 'compiler.cppstd': '17', + 'compiler.libcxx': 'libstdc++11', + 'compiler.version': '11', 'os': 'Linux'}, + 'options': {'fPIC': 'True', 'shared': 'False'}}}, + '30652ea35b512fa3fe0f4dcd39ad217e4e60bd01': + {'revisions': {'8f363f1d95ad766ac400e1f1f5406599': + {'timestamp': 1741861639.2198145}}, + 'info': {}}, + 'fc491156b442836722612d1aa8a8c57e406447b6': + {'revisions': {'62e9e2659596c97cd1dd313d4394e947': + {'timestamp': 1740151900.394}}, + 'info': { + 'settings': + {'arch': 'x86_64', 'build_type': 'Release', + 'compiler': 'gcc', 'compiler.cppstd': '17', + 'compiler.libcxx': 'libstdc++11', + 'compiler.version': '11', 'os': 'Linux'}, + 'options': {'shared': 'True'}}}}}}} + } + pl = PackagesList.deserialize(data) + pl.only_recipes() + assert pl.serialize() == {'foobar/0.1.0': { + 'revisions': {'85eb0587a3c12b90216c72070e9eef3e': {'timestamp': 1740151190.975}}}, + 'qux/0.2.1': {'revisions': { + '71c3c11b98a6f2ae11f0f391f5e62e2b': {'timestamp': 1740151186.976}}}} diff --git a/test/unittests/model/version/test_version_comparison.py b/test/unittests/model/version/test_version_comparison.py index 0b8e475e04e..84602739565 100644 --- a/test/unittests/model/version/test_version_comparison.py +++ b/test/unittests/model/version/test_version_comparison.py @@ -2,7 +2,7 @@ from conan.internal.model.version import Version -v = [("1", "2"), +d = [("1", "2"), ("1.0", "1.1"), ("1.0.2", "1.1.0"), ("1.3", "1.22"), @@ -71,7 +71,7 @@ ] -@pytest.mark.parametrize("v1, v2", v) +@pytest.mark.parametrize("v1, v2", d) def test_comparison(v1, v2): v1 = Version(v1) v2 = Version(v2) @@ -156,6 +156,7 @@ def test_elem_comparison(): assert micro < 5 assert micro == 4 + def test_version_in_range(): v = Version("1.0") assert not v.in_range(">1.0 <2") diff --git a/test/unittests/model/version/test_version_range.py b/test/unittests/model/version/test_version_range.py index a32da8ae3ae..e783bba7774 100644 --- a/test/unittests/model/version/test_version_range.py +++ b/test/unittests/model/version/test_version_range.py @@ -5,6 +5,7 @@ from conan.errors import ConanException from conan.internal.model.version import Version from conan.internal.model.version_range import VersionRange +from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient values = [ @@ -171,3 +172,19 @@ class Pkg(ConanFile): c.run("export . --name=mypkg --version=0.1") c.run("install --requires=mypkg/0.1", assert_error=True) assert "Recipe 'mypkg/0.1' requires 'mydep/[>1.0 < 3]' version-range definition error" in c.out + + +def test_version_range_with_pinned_revision_warn(): + tc = TestClient() + tc.save({"conanfile.py": GenConanfile("pkg", "1.0")}) + tc.run("export .") + rrev1 = tc.exported_recipe_revision() + tc.save({"conanfile.py": GenConanfile("pkg", "1.0").with_class_attribute("# a comment")}) + tc.run("export .") + rrev2 = tc.exported_recipe_revision() + assert rrev1 != rrev2 + + tc.run(f"install --requires=pkg/[>=1.0]#{rrev1} --build=missing -vv") + assert f"WARN: risk: Specifying a revision for requirement 'pkg/[>=1.0]#{rrev1}' together with a version range has no effect" in tc.out + # Note, rrev2 instead of the requested rrev1 + assert f"pkg/1.0#{rrev2} - Cache" in tc.out diff --git a/test/unittests/model/versionrepr_test.py b/test/unittests/model/versionrepr_test.py index 082894a81e9..03ecb968a20 100644 --- a/test/unittests/model/versionrepr_test.py +++ b/test/unittests/model/versionrepr_test.py @@ -1,5 +1,3 @@ -import pytest - from conan.internal.model.info import _VersionRepr from conan.internal.model.version import Version diff --git a/test/unittests/search/search_query_parse_test.py b/test/unittests/search/search_query_parse_test.py index b799fd83ef4..a72a229bf87 100644 --- a/test/unittests/search/search_query_parse_test.py +++ b/test/unittests/search/search_query_parse_test.py @@ -22,7 +22,7 @@ def test_get_postfix(self): import pytest with pytest.raises(Exception, match="Invalid expression: 2"): - r = _infix_to_postfix("a= 2 OR b=3") + _infix_to_postfix("a= 2 OR b=3") def test_evaluate_postfix(self): diff --git a/test/unittests/server/conan_server_config_parser_test.py b/test/unittests/server/conan_server_config_parser_test.py index 56c6e49411c..ffc475f3089 100644 --- a/test/unittests/server/conan_server_config_parser_test.py +++ b/test/unittests/server/conan_server_config_parser_test.py @@ -38,19 +38,23 @@ def test_not_allowed_encoding_password(self): save(conf_path, server_conf % "cönan") server_config = ConanServerConfigParser(tmp_dir) - with pytest.raises(ConanException, match="Password contains invalid characters. Only ASCII encoding is supported"): - server_config.users + with pytest.raises(ConanException, match="Password contains invalid characters. " + "Only ASCII encoding is supported"): + server_config.users # noqa save(conf_path, server_conf % "manol ito!@") server_config = ConanServerConfigParser(tmp_dir) assert server_config.users == {"demo": "manol ito!@"} # Now test from ENV - server_config = ConanServerConfigParser(tmp_dir, environment={"CONAN_SERVER_USERS": "demo: cönan"}) - with pytest.raises(ConanException, match="Password contains invalid characters. Only ASCII encoding is supported"): - server_config.users - - server_config = ConanServerConfigParser(tmp_dir, environment={"CONAN_SERVER_USERS": "demo:manolito!@"}) + server_config = ConanServerConfigParser(tmp_dir, environment={"CONAN_SERVER_USERS": + "demo: cönan"}) + with pytest.raises(ConanException, match="Password contains invalid characters. " + "Only ASCII encoding is supported"): + server_config.users # noqa + + server_config = ConanServerConfigParser(tmp_dir, environment={"CONAN_SERVER_USERS": + "demo:manolito!@"}) assert server_config.users == {"demo": "manolito!@"} def test_relative_public_url(self): diff --git a/test/unittests/server/service/service_test.py b/test/unittests/server/service/service_test.py index ddf7378bd7a..b141c91a3f8 100644 --- a/test/unittests/server/service/service_test.py +++ b/test/unittests/server/service/service_test.py @@ -95,6 +95,12 @@ def test_search(self): assert info == {'77777777777': {'content': '\n[options]\n use_Qt=True\n'} } + info = self.search_service.search_packages(ref2, list_only=True) + assert info == {'12345587754': {}} + + info = self.search_service.search_packages(ref3, list_only=True) + assert info == {'77777777777': {}} + def test_remove(self): ref2 = RecipeReference("OpenCV", "3.0", "lasote", "stable", DEFAULT_REVISION) ref3 = RecipeReference("Assimp", "1.10", "lasote", "stable", DEFAULT_REVISION) diff --git a/test/unittests/tools/apple/test_apple_tools.py b/test/unittests/tools/apple/test_apple_tools.py index a54350f8a23..c26b8a1ac86 100644 --- a/test/unittests/tools/apple/test_apple_tools.py +++ b/test/unittests/tools/apple/test_apple_tools.py @@ -1,4 +1,4 @@ -import mock +from unittest import mock import pytest import textwrap diff --git a/test/unittests/client/tools/apple/test_xcodebuild.py b/test/unittests/tools/apple/test_xcodebuild.py similarity index 100% rename from test/unittests/client/tools/apple/test_xcodebuild.py rename to test/unittests/tools/apple/test_xcodebuild.py diff --git a/test/unittests/tools/build/test_can_run.py b/test/unittests/tools/build/test_can_run.py index d8a193b364c..e2330fb43ea 100644 --- a/test/unittests/tools/build/test_can_run.py +++ b/test/unittests/tools/build/test_can_run.py @@ -8,7 +8,7 @@ class ConfigMock: def __init__(self, run=None): self.can_run = run - def get(self, conf_name, default=None, check_type=None): + def get(self, conf_name, default=None, check_type=None): # noqa return self.can_run @@ -33,9 +33,9 @@ def test_can_run_cross_building(arch, expected): """ config = ConfigMock(None) settings_build = MockSettings({"os": "Macos", - "compiler": "apple-clang", - "compiler.version": "11.0", - "arch": "armv8"}) + "compiler": "apple-clang", + "compiler.version": "11.0", + "arch": "armv8"}) settings = MockSettings({"os": "Macos", "compiler": "apple-clang", "compiler.version": "11.0", @@ -52,9 +52,9 @@ def test_can_run_cross_building_with_explicit(): """ config = ConfigMock(True) settings_build = MockSettings({"os": "Macos", - "compiler": "apple-clang", - "compiler.version": "11.0", - "arch": "armv8"}) + "compiler": "apple-clang", + "compiler.version": "11.0", + "arch": "armv8"}) settings = MockSettings({"os": "Macos", "compiler": "apple-clang", "compiler.version": "11.0", diff --git a/test/unittests/tools/build/test_compiler.py b/test/unittests/tools/build/test_compiler.py new file mode 100644 index 00000000000..3246b4c036e --- /dev/null +++ b/test/unittests/tools/build/test_compiler.py @@ -0,0 +1,34 @@ +import pytest + +from conan.tools.build import check_min_compiler_version +from conan.errors import ConanException, ConanInvalidConfiguration +from conan.test.utils.mocks import MockSettings, ConanFileMock + + +@pytest.mark.parametrize("compiler,compiler_version,restrictions,should_raise", [ + ("clang", "14", (("clang", "14", "coroutines"), ("gcc", "13", "alignas")), False), + ("gcc", "13", (("clang", "14", "coroutines"), ("gcc", "13", "alignas")), False), + ("gcc", "14", (("clang", "14", "coroutines"), ("gcc", "13", "alignas")), False), + ("msvc", "192", (("msvc", "192", "reason"), ("clang", "14", "coroutines")), False), + + ("clang", "13", (("clang", "14", "coroutines"), ("gcc", "13", "alignas")), True), + ("gcc", "12", (("clang", "14", "coroutines"), ("gcc", "13", "alignas")), True), + ("msvc", "191", (("msvc", "192", "reason"), ("clang", "14", "coroutines")), True), + + ("emcc", "1.0", (("gcc", "13", "alignas"),), False), + (None, "12", (("gcc", "13", "alignas"),), True), + ("gcc", None, (("gcc", "13", "alignas"),), True), +]) +def test_check_min_compiler_version(compiler, compiler_version, restrictions, should_raise): + settings = MockSettings({"compiler": compiler, "compiler.version": compiler_version}) + conanfile = ConanFileMock(settings) + + if should_raise: + if compiler is None or compiler_version is None: + with pytest.raises(ConanException): + check_min_compiler_version(conanfile, restrictions) + else: + with pytest.raises(ConanInvalidConfiguration): + check_min_compiler_version(conanfile, restrictions) + else: + check_min_compiler_version(conanfile, restrictions) diff --git a/test/unittests/tools/build/test_cppstd.py b/test/unittests/tools/build/test_cppstd.py index e3c418af9f9..e5c054cc635 100644 --- a/test/unittests/tools/build/test_cppstd.py +++ b/test/unittests/tools/build/test_cppstd.py @@ -137,6 +137,23 @@ def test_supported_cppstd_qcc(compiler, compiler_version, values): assert sot == values +@pytest.mark.parametrize("compiler,compiler_version,values", [ + ("emcc", "3", ["98", "gnu98", "11", "gnu11", "14", "gnu14", "17", "gnu17", + "20", "gnu20", "23", "gnu23"]), + ("emcc", "3.1", ["98", "gnu98", "11", "gnu11", "14", "gnu14", "17", "gnu17", + "20", "gnu20", "23", "gnu23", "26", "gnu26"]), + ("emcc", "3.3", ["98", "gnu98", "11", "gnu11", "14", "gnu14", "17", "gnu17", + "20", "gnu20", "23", "gnu23", "26", "gnu26"]), + ("emcc", "4.1", ["98", "gnu98", "11", "gnu11", "14", "gnu14", "17", "gnu17", + "20", "gnu20", "23", "gnu23", "26", "gnu26"]), +]) +def test_supported_cppstd_emcc(compiler, compiler_version, values): + settings = MockSettings({"compiler": compiler, "compiler.version": compiler_version}) + conanfile = ConanFileMock(settings) + sot = supported_cppstd(conanfile) + assert sot == values + + def test_check_cppstd_type(): """ cppstd must be a number """ diff --git a/test/unittests/tools/build/test_cstd.py b/test/unittests/tools/build/test_cstd.py index 6ebf41622eb..d0622ac92c8 100644 --- a/test/unittests/tools/build/test_cstd.py +++ b/test/unittests/tools/build/test_cstd.py @@ -57,6 +57,19 @@ def test_supported_cstd_msvc(compiler, compiler_version, values): assert sot == values +@pytest.mark.parametrize("compiler,compiler_version,values", [ + ("emcc", "3", ["99", "gnu99", "11", "gnu11", "17", "gnu17"]), + ("emcc", "3.1", ["99", "gnu99", "11", "gnu11", "17", "gnu17", "23", "gnu23"]), + ("emcc", "3.3", ["99", "gnu99", "11", "gnu11", "17", "gnu17", "23", "gnu23"]), + ("emcc", "4.1", ["99", "gnu99", "11", "gnu11", "17", "gnu17", "23", "gnu23"]), +]) +def test_supported_cstd_emcc(compiler, compiler_version, values): + settings = MockSettings({"compiler": compiler, "compiler.version": compiler_version}) + conanfile = ConanFileMock(settings) + sot = supported_cstd(conanfile) + assert sot == values + + def test_check_cstd_type(): """ cppstd must be a number """ diff --git a/test/unittests/tools/cmake/test_cmake_install.py b/test/unittests/tools/cmake/test_cmake_install.py index 9b7f51bba84..2203070dbe9 100644 --- a/test/unittests/tools/cmake/test_cmake_install.py +++ b/test/unittests/tools/cmake/test_cmake_install.py @@ -10,7 +10,6 @@ from conan.test.utils.tools import redirect_output - def test_run_install_component(): """ Testing that the proper component is installed. @@ -80,6 +79,7 @@ def test_run_install_strip(config, deprecated): assert "tools.cmake:install_strip" not in stderr assert "--strip" in conanfile.command + def test_run_install_cli_args(): """ Testing that the passing cli_args to install works diff --git a/test/unittests/tools/cmake/test_cmake_presets_definitions.py b/test/unittests/tools/cmake/test_cmake_presets_definitions.py index 2b474c8fd09..46e7ef30649 100644 --- a/test/unittests/tools/cmake/test_cmake_presets_definitions.py +++ b/test/unittests/tools/cmake/test_cmake_presets_definitions.py @@ -1,5 +1,5 @@ import pytest -from mock import mock +from unittest import mock from conan.tools.cmake import CMake from conan.tools.cmake.presets import write_cmake_presets diff --git a/test/unittests/tools/cmake/test_cmake_test.py b/test/unittests/tools/cmake/test_cmake_test.py index 8caf8603a66..f29f1ef0f51 100644 --- a/test/unittests/tools/cmake/test_cmake_test.py +++ b/test/unittests/tools/cmake/test_cmake_test.py @@ -58,3 +58,27 @@ def test_cli_args_configure(): cmake = CMake(conanfile) cmake.configure(cli_args=["--graphviz=foo.dot"]) assert "--graphviz=foo.dot" in conanfile.command + + +def test_run_ctest(): + settings = Settings.loads(default_settings_yml) + settings.os = "Windows" + settings.arch = "x86" + settings.build_type = "Release" + settings.compiler = "msvc" + settings.compiler.runtime = "dynamic" + settings.compiler.version = "193" + + conanfile = ConanFileMock() + conanfile.conf = Conf() + conanfile.conf.define("tools.cmake:ctest_args", ["--debug", "--output-junit myfile"]) + conanfile.conf.define("tools.build:verbosity", "verbose") + + conanfile.folders.generators = "." + conanfile.folders.set_base_generators(temp_folder()) + conanfile.settings = settings + + write_cmake_presets(conanfile, "toolchain", "Ninja", {}) + cmake = CMake(conanfile) + cmake.ctest(cli_args=["--schedule-random", "--quiet"]) + assert "--schedule-random --quiet --verbose --debug --output-junit myfile" in conanfile.command diff --git a/test/unittests/tools/cmake/test_cmaketoolchain.py b/test/unittests/tools/cmake/test_cmaketoolchain.py index 2850f1fb46b..8a4c77ead06 100644 --- a/test/unittests/tools/cmake/test_cmaketoolchain.py +++ b/test/unittests/tools/cmake/test_cmaketoolchain.py @@ -1,7 +1,7 @@ import types import pytest -from mock import Mock +from unittest.mock import Mock from conan import ConanFile from conan.internal.default_settings import default_settings_yml @@ -14,12 +14,9 @@ @pytest.fixture -def conanfile(): +def base_conanfile(): c = ConanFile() - settings = Settings({"os": ["Windows"], - "compiler": {"clang": {"libcxx": ["libstdc++"]}}, - "build_type": ["Release"], - "arch": ["x86"]}) + settings = Settings.loads(default_settings_yml) c.settings = settings c.settings.build_type = "Release" c.settings.arch = "x86" @@ -28,657 +25,507 @@ def conanfile(): c.settings_build = c.settings c.settings.os = "Windows" c.conf = Conf() - c.conf.define("tools.cmake.cmaketoolchain:system_name", "potato") - c.folders.set_base_generators("/some/abs/path") # non-existing to not relativize - c._conan_node = Mock() - c._conan_node.transitive_deps = {} - c._conan_node.replaced_requires = {} + c.folders.set_base_generators(".") # non-existing to not relativize + conan_node = Mock() + conan_node.dependencies = [] + conan_node.transitive_deps = {} + conan_node.replaced_requires = {} + c._conan_node = conan_node return c -def test_cmake_toolchain(conanfile): - toolchain = CMakeToolchain(conanfile) - content = toolchain.content - assert 'set(CMAKE_SYSTEM_NAME potato)' in content - - -def test_remove(conanfile): - toolchain = CMakeToolchain(conanfile) - toolchain.blocks.remove("generic_system") - content = toolchain.content - assert 'CMAKE_SYSTEM_NAME' not in content - assert "CMAKE_CXX_FLAGS_INIT" in content - assert "_CMAKE_IN_TRY_COMPILE" in content - - # remove multiple - toolchain = CMakeToolchain(conanfile) - toolchain.blocks.remove("generic_system", "cmake_flags_init") - content = toolchain.content - assert 'CMAKE_SYSTEM_NAME' not in content - assert "CMAKE_CXX_FLAGS_INIT" not in content - assert "_CMAKE_IN_TRY_COMPILE" in content - - -def test_select_blocks(conanfile): - toolchain = CMakeToolchain(conanfile) - toolchain.blocks.select("generic_system") - content = toolchain.content - assert "########## 'generic_system' block #############" in content - assert "########## 'cmake_flags_init' block #############" not in content - assert "########## 'libcxx' block #############" not in content - # These are not removed by default, to not break behavior - assert "########## 'variables' block #############" in content - assert "########## 'preprocessor' block #############" in content - assert 'CMAKE_SYSTEM_NAME' in content - assert "CMAKE_CXX_FLAGS_INIT" not in content - assert "_CMAKE_IN_TRY_COMPILE" not in content - - # remove multiple - toolchain = CMakeToolchain(conanfile) - toolchain.blocks.select("generic_system", "cmake_flags_init") - content = toolchain.content - assert "########## 'generic_system' block #############" in content - assert "########## 'cmake_flags_init' block #############" in content - assert "########## 'libcxx' block #############" not in content - # These are not removed by default, to not break behavior - assert "########## 'variables' block #############" in content - assert "########## 'preprocessor' block #############" in content - assert 'CMAKE_SYSTEM_NAME' in content - assert "CMAKE_CXX_FLAGS_INIT" in content - assert "_CMAKE_IN_TRY_COMPILE" not in content - - # remove multiple - toolchain = CMakeToolchain(conanfile) - toolchain.blocks.enabled("generic_system") - content = toolchain.content - assert "########## 'generic_system' block #############" in content - assert "########## 'variables' block #############" not in content - assert "########## 'preprocessor' block #############" not in content - - -def test_enabled_blocks_conf(conanfile): - conanfile.conf.define("tools.cmake.cmaketoolchain:enabled_blocks", ["generic_system"]) - toolchain = CMakeToolchain(conanfile) - content = toolchain.content - assert "########## 'generic_system' block #############" in content - assert "########## 'cmake_flags_init' block #############" not in content - assert "########## 'libcxx' block #############" not in content - assert "########## 'variables' block #############" not in content - assert "########## 'preprocessor' block #############" not in content - - # remove multiple - conanfile.conf.define("tools.cmake.cmaketoolchain:enabled_blocks", - ["generic_system", "cmake_flags_init"]) - toolchain = CMakeToolchain(conanfile) - content = toolchain.content - assert "########## 'generic_system' block #############" in content - assert "########## 'cmake_flags_init' block #############" in content - assert "########## 'libcxx' block #############" not in content - assert "########## 'variables' block #############" not in content - assert "########## 'preprocessor' block #############" not in content - - conanfile.conf.define("tools.cmake.cmaketoolchain:enabled_blocks", ["potato"]) - toolchain = CMakeToolchain(conanfile) - with pytest.raises(ConanException) as e: - _ = toolchain.content - assert "Block 'potato' defined in tools.cmake.cmaketoolchain:enabled_blocks doesn't" in str(e) - - -def test_dict_keys(conanfile): - toolchain = CMakeToolchain(conanfile) - assert "generic_system" in toolchain.blocks.keys() - items = dict(toolchain.blocks.items()) - assert "generic_system" in items - - -def test_template_remove(conanfile): - toolchain = CMakeToolchain(conanfile) - toolchain.blocks["generic_system"].template = "" - content = toolchain.content - assert 'CMAKE_SYSTEM_NAME' not in content - - -def test_template_change(conanfile): - toolchain = CMakeToolchain(conanfile) - tmp = toolchain.blocks["generic_system"].template - toolchain.blocks["generic_system"].template = tmp.replace("CMAKE_SYSTEM_NAME", "OTHER_THING") - content = toolchain.content - assert 'set(OTHER_THING potato)' in content - - -def test_context_change(conanfile): - toolchain = CMakeToolchain(conanfile) - tmp = toolchain.blocks["generic_system"] - - def context(self): - assert self - return {"cmake_system_name": None} - - tmp.context = types.MethodType(context, tmp) - content = toolchain.content - assert 'CMAKE_SYSTEM_NAME' not in content - - -def test_context_update(conanfile): - toolchain = CMakeToolchain(conanfile) - cmake_system_name = toolchain.blocks["generic_system"].values["cmake_system_name"] - toolchain.blocks["generic_system"].values["cmake_system_name"] = "Super" + cmake_system_name - content = toolchain.content - assert 'set(CMAKE_SYSTEM_NAME Superpotato)' in content - - -def test_context_replace(conanfile): - toolchain = CMakeToolchain(conanfile) - toolchain.blocks["generic_system"].values = {"cmake_system_name": "SuperPotato"} - content = toolchain.content - assert 'set(CMAKE_SYSTEM_NAME SuperPotato)' in content - - -def test_replace_block(conanfile): - toolchain = CMakeToolchain(conanfile) - - class MyBlock(Block): - template = "HelloWorld" - - def context(self): - return {} - - toolchain.blocks["generic_system"] = MyBlock - content = toolchain.content - assert 'HelloWorld' in content - assert 'set(CMAKE_SYSTEM_NAME potato)' not in content - - -def test_add_new_block(conanfile): - toolchain = CMakeToolchain(conanfile) - - class MyBlock(Block): - template = "Hello {{myvar}}!!!" - - def context(self): - return {"myvar": "World"} - - toolchain.blocks["mynewblock"] = MyBlock - content = toolchain.content - assert 'Hello World!!!' in content - assert 'set(CMAKE_SYSTEM_NAME potato)' in content +@pytest.fixture +def conanfile(base_conanfile): + c = base_conanfile + c.conf.define("tools.cmake.cmaketoolchain:system_name", "potato") + return c -def test_user_toolchain(conanfile): - toolchain = CMakeToolchain(conanfile) - toolchain.blocks["user_toolchain"].values["paths"] = ["myowntoolchain.cmake"] - content = toolchain.content - assert 'include("myowntoolchain.cmake")' in content +class TestGenericBlocks: + """ Tests general toolchain blocks operations, enabling, disabling, customizing + """ + def test_remove(self, conanfile): + toolchain = CMakeToolchain(conanfile) + toolchain.blocks.remove("generic_system") + content = toolchain.content + assert 'CMAKE_SYSTEM_NAME' not in content + assert "CMAKE_CXX_FLAGS_INIT" in content + assert "_CMAKE_IN_TRY_COMPILE" in content + # remove multiple + toolchain = CMakeToolchain(conanfile) + toolchain.blocks.remove("generic_system", "cmake_flags_init") + content = toolchain.content + assert 'CMAKE_SYSTEM_NAME' not in content + assert "CMAKE_CXX_FLAGS_INIT" not in content + assert "_CMAKE_IN_TRY_COMPILE" in content -@pytest.fixture -def conanfile_apple(): - c = ConanFile(None) - c.settings = Settings({"os": {"Macos": {"version": ["10.15"]}}, - "compiler": {"apple-clang": {"libcxx": ["libc++"]}}, - "build_type": ["Release"], - "arch": ["x86"]}) - c.settings.build_type = "Release" - c.settings.arch = "x86" - c.settings.compiler = "apple-clang" - c.settings.compiler.libcxx = "libc++" - c.settings.os = "Macos" - c.settings.os.version = "10.15" - c.settings_build = c.settings - c.conf = Conf() - c.folders.set_base_generators("/some/abs/path") # non-existing to not relativize - c._conan_node = Mock() - c._conan_node.dependencies = [] - c._conan_node.transitive_deps = {} - # FIXME: Repeated fix pattern - c._conan_node.replaced_requires = {} - return c + def test_select_blocks(self, conanfile): + toolchain = CMakeToolchain(conanfile) + toolchain.blocks.select("generic_system") + content = toolchain.content + assert "########## 'generic_system' block #############" in content + assert "########## 'cmake_flags_init' block #############" not in content + assert "########## 'libcxx' block #############" not in content + # These are not removed by default, to not break behavior + assert "########## 'variables' block #############" in content + assert "########## 'preprocessor' block #############" in content + assert 'CMAKE_SYSTEM_NAME' in content + assert "CMAKE_CXX_FLAGS_INIT" not in content + assert "_CMAKE_IN_TRY_COMPILE" not in content + + # remove multiple + toolchain = CMakeToolchain(conanfile) + toolchain.blocks.select("generic_system", "cmake_flags_init") + content = toolchain.content + assert "########## 'generic_system' block #############" in content + assert "########## 'cmake_flags_init' block #############" in content + assert "########## 'libcxx' block #############" not in content + # These are not removed by default, to not break behavior + assert "########## 'variables' block #############" in content + assert "########## 'preprocessor' block #############" in content + assert 'CMAKE_SYSTEM_NAME' in content + assert "CMAKE_CXX_FLAGS_INIT" in content + assert "_CMAKE_IN_TRY_COMPILE" not in content + + # remove multiple + toolchain = CMakeToolchain(conanfile) + toolchain.blocks.enabled("generic_system") + content = toolchain.content + assert "########## 'generic_system' block #############" in content + assert "########## 'variables' block #############" not in content + assert "########## 'preprocessor' block #############" not in content + def test_enabled_blocks_conf(self, conanfile): + conanfile.conf.define("tools.cmake.cmaketoolchain:enabled_blocks", ["generic_system"]) + toolchain = CMakeToolchain(conanfile) + content = toolchain.content + assert "########## 'generic_system' block #############" in content + assert "########## 'cmake_flags_init' block #############" not in content + assert "########## 'libcxx' block #############" not in content + assert "########## 'variables' block #############" not in content + assert "########## 'preprocessor' block #############" not in content + + # remove multiple + conanfile.conf.define("tools.cmake.cmaketoolchain:enabled_blocks", + ["generic_system", "cmake_flags_init"]) + toolchain = CMakeToolchain(conanfile) + content = toolchain.content + assert "########## 'generic_system' block #############" in content + assert "########## 'cmake_flags_init' block #############" in content + assert "########## 'libcxx' block #############" not in content + assert "########## 'variables' block #############" not in content + assert "########## 'preprocessor' block #############" not in content + + conanfile.conf.define("tools.cmake.cmaketoolchain:enabled_blocks", ["potato"]) + toolchain = CMakeToolchain(conanfile) + with pytest.raises(ConanException) as e: + _ = toolchain.content + assert "Block 'potato' defined in tools.cmake.cmaketoolchain:enabled_blocks" in str(e) + + def test_dict_keys(self, conanfile): + toolchain = CMakeToolchain(conanfile) + assert "generic_system" in toolchain.blocks.keys() + items = dict(toolchain.blocks.items()) + assert "generic_system" in items + + def test_template_remove(self, conanfile): + toolchain = CMakeToolchain(conanfile) + toolchain.blocks["generic_system"].template = "" + content = toolchain.content + assert 'CMAKE_SYSTEM_NAME' not in content -def test_osx_deployment_target(conanfile_apple): - toolchain = CMakeToolchain(conanfile_apple) - content = toolchain.content - assert 'set(CMAKE_OSX_DEPLOYMENT_TARGET "10.15" CACHE STRING "")' in content + def test_template_change(self, conanfile): + toolchain = CMakeToolchain(conanfile) + tmp = toolchain.blocks["generic_system"].template + toolchain.blocks["generic_system"].template = tmp.replace("CMAKE_SYSTEM_NAME", "OTHER_THING") + content = toolchain.content + assert 'set(OTHER_THING potato)' in content + def test_context_change(self, conanfile): + toolchain = CMakeToolchain(conanfile) + tmp = toolchain.blocks["generic_system"] -@pytest.fixture -def conanfile_msvc(): - c = ConanFile(None) - c.settings = Settings({"os": ["Windows"], - "compiler": {"msvc": {"version": ["193", "194"], "cppstd": ["20"], - "update": [None, 8, 9]}}, - "build_type": ["Release"], - "arch": ["x86"]}) - c.settings.build_type = "Release" - c.settings.arch = "x86" - c.settings.compiler = "msvc" - c.settings.compiler.version = "194" - c.settings.compiler.cppstd = "20" - c.settings.os = "Windows" - c.settings_build = c.settings - c.conf = Conf() - c.folders.set_base_generators(".") - c._conan_node = Mock() - c._conan_node.dependencies = [] - c._conan_node.transitive_deps = {} - c._conan_node.replaced_requires = {} - return c + def context(self_): + assert self_ + return {"cmake_system_name": None} + tmp.context = types.MethodType(context, tmp) + content = toolchain.content + assert 'CMAKE_SYSTEM_NAME' not in content -def test_toolset(conanfile_msvc): - toolchain = CMakeToolchain(conanfile_msvc) - assert 'set(CMAKE_GENERATOR_TOOLSET "v143" CACHE STRING "" FORCE)' in toolchain.content - assert 'Visual Studio 17 2022' in toolchain.generator - assert 'CMAKE_CXX_STANDARD 20' in toolchain.content + def test_context_update(self, conanfile): + toolchain = CMakeToolchain(conanfile) + cmake_system_name = toolchain.blocks["generic_system"].values["cmake_system_name"] + toolchain.blocks["generic_system"].values["cmake_system_name"] = "Super" + cmake_system_name + content = toolchain.content + assert 'set(CMAKE_SYSTEM_NAME Superpotato)' in content + def test_context_replace(self, conanfile): + toolchain = CMakeToolchain(conanfile) + toolchain.blocks["generic_system"].values = {"cmake_system_name": "SuperPotato"} + content = toolchain.content + assert 'set(CMAKE_SYSTEM_NAME SuperPotato)' in content -def test_toolset_update_version(conanfile_msvc): - conanfile_msvc.settings.compiler.version = "193" - conanfile_msvc.settings.compiler.update = "8" - toolchain = CMakeToolchain(conanfile_msvc) - assert 'set(CMAKE_GENERATOR_TOOLSET "v143,version=14.38" CACHE STRING "" FORCE)' in toolchain.content + def test_replace_block(self, conanfile): + toolchain = CMakeToolchain(conanfile) + class MyBlock(Block): + template = "HelloWorld" -def test_toolset_update_version_conf(conanfile_msvc): - conanfile_msvc.settings.compiler.version = "193" - conanfile_msvc.conf.define("tools.microsoft:msvc_update", "7") - toolchain = CMakeToolchain(conanfile_msvc) - assert 'set(CMAKE_GENERATOR_TOOLSET "v143,version=14.37" CACHE STRING "" FORCE)' in toolchain.content + def context(self): + return {} + toolchain.blocks["generic_system"] = MyBlock + content = toolchain.content + assert 'HelloWorld' in content + assert 'set(CMAKE_SYSTEM_NAME potato)' not in content -def test_toolset_update_version_forced_conf(conanfile_msvc): - conanfile_msvc.settings.compiler.version = "193" - conanfile_msvc.settings.compiler.update = "8" - conanfile_msvc.conf.define("tools.microsoft:msvc_update", "7") - toolchain = CMakeToolchain(conanfile_msvc) - assert 'set(CMAKE_GENERATOR_TOOLSET "v143,version=14.37" CACHE STRING "" FORCE)' in toolchain.content + def test_add_new_block(self, conanfile): + toolchain = CMakeToolchain(conanfile) + class MyBlock(Block): + template = "Hello {{myvar}}!!!" -def test_toolset_update_version_overflow(conanfile_msvc): - # https://github.com/conan-io/conan/issues/15583 - conanfile_msvc.settings.compiler.version = "194" - conanfile_msvc.settings.compiler.update = "8" - toolchain = CMakeToolchain(conanfile_msvc) - assert 'set(CMAKE_GENERATOR_TOOLSET "v143,version=14.48" CACHE STRING "" FORCE)' in toolchain.content + def context(self): + return {"myvar": "World"} + toolchain.blocks["mynewblock"] = MyBlock + content = toolchain.content + assert 'Hello World!!!' in content + assert 'set(CMAKE_SYSTEM_NAME potato)' in content -def test_toolset_x64(conanfile_msvc): - # https://github.com/conan-io/conan/issues/11144 - conanfile_msvc.conf.define("tools.cmake.cmaketoolchain:toolset_arch", "x64") - toolchain = CMakeToolchain(conanfile_msvc) - assert 'set(CMAKE_GENERATOR_TOOLSET "v143,host=x64" CACHE STRING "" FORCE)' in toolchain.content - assert 'Visual Studio 17 2022' in toolchain.generator - assert 'CMAKE_CXX_STANDARD 20' in toolchain.content + def test_user_toolchain(self, conanfile): + toolchain = CMakeToolchain(conanfile) + toolchain.blocks["user_toolchain"].values["paths"] = ["myowntoolchain.cmake"] + content = toolchain.content + assert 'include("myowntoolchain.cmake")' in content -def test_toolset_cuda(conanfile_msvc): - conanfile_msvc.conf.define("tools.cmake.cmaketoolchain:toolset_cuda", "C:/Path/To/CUDA") - toolchain = CMakeToolchain(conanfile_msvc) - assert 'set(CMAKE_GENERATOR_TOOLSET "v143,cuda=C:/Path/To/CUDA" CACHE STRING "" FORCE)' in toolchain.content +class TestBlocksFunctionality: + def test_cmake_toolchain(self, conanfile): + toolchain = CMakeToolchain(conanfile) + content = toolchain.content + assert 'set(CMAKE_SYSTEM_NAME potato)' in content + + def test_compilers_block(self, conanfile): + cmake_mapping = {"c": "C", "cuda": "CUDA", "cpp": "CXX", "objc": "OBJC", + "objcpp": "OBJCXX", "rc": "RC", 'fortran': "Fortran", 'asm': "ASM", + "hip": "HIP", "ispc": "ISPC"} + compilers = {"c": "path_to_c", "cuda": "path_to_cuda", "cpp": "path_to_cpp", + "objc": "path_to_objc", "objcpp": "path_to_objcpp", "rc": "path_to_rc", + 'fortran': "path_to_fortran", 'asm': "path_to_asm", "hip": "path_to_hip", + "ispc": "path_to_ispc"} + conanfile.conf.define("tools.build:compiler_executables", compilers) + toolchain = CMakeToolchain(conanfile) + content = toolchain.content + for compiler, lang in cmake_mapping.items(): + assert f'set(CMAKE_{lang}_COMPILER "path_to_{compiler}")' in content + def test_linker_scripts_block(self, conanfile): + conanfile.conf.define("tools.build:linker_scripts", + ["path_to_first_linker_script", "path_to_second_linker_script"]) + toolchain = CMakeToolchain(conanfile) + content = toolchain.content + assert r'string(APPEND CONAN_EXE_LINKER_FLAGS " -T\"path_to_first_linker_script\" ' \ + r'-T\"path_to_second_linker_script\"")' in content -def test_older_msvc_toolset(): - c = ConanFile(None) - c.settings = Settings({"os": ["Windows"], - "compiler": {"msvc": {"version": ["170"], "update": [None], - "cppstd": ["98"]}}, - "build_type": ["Release"], - "arch": ["x86"]}) - c.settings.build_type = "Release" - c.settings.arch = "x86" - c.settings.compiler = "msvc" - c.settings.compiler.version = "170" - c.settings.compiler.cppstd = "98" - c.settings.os = "Windows" - c.settings_build = c.settings - c.conf = Conf() - c.folders.set_base_generators(".") - c._conan_node = Mock() - c._conan_node.dependencies = [] - c._conan_node.transitive_deps = {} - c._conan_node.replaced_requires = {} - toolchain = CMakeToolchain(c) - content = toolchain.content - assert 'CMAKE_GENERATOR_TOOLSET "v110"' in content - # As by the CMake docs, this has no effect for VS < 2015 - assert 'CMAKE_CXX_STANDARD 98' in content - - -def test_older_msvc_toolset_update(): - # https://github.com/conan-io/conan/issues/15787 - c = ConanFile(None) - c.settings = Settings({"os": ["Windows"], - "compiler": {"msvc": {"version": ["192"], "update": [8, 9], - "cppstd": ["14"]}}, - "build_type": ["Release"], - "arch": ["x86_64"]}) - c.settings.build_type = "Release" - c.settings.arch = "x86_64" - c.settings.compiler = "msvc" - c.settings.compiler.version = "192" - c.settings.compiler.update = 9 - c.settings.compiler.cppstd = "14" - c.settings.os = "Windows" - c.settings_build = c.settings - c.conf = Conf() - c.folders.set_base_generators(".") - c._conan_node = Mock() - c._conan_node.dependencies = [] - c._conan_node.transitive_deps = {} - c._conan_node.replaced_requires = {} - toolchain = CMakeToolchain(c) - content = toolchain.content - assert 'CMAKE_GENERATOR_TOOLSET "v142,version=14.29"' in content - - -def test_msvc_xp_toolsets(): - c = ConanFile(None) - c.settings = Settings({"os": ["Windows"], - "compiler": {"msvc": {"version": ["170"], "update": [None], - "cppstd": ["98"], "toolset": [None, "v110_xp"]}}, - "build_type": ["Release"], - "arch": ["x86"]}) - c.settings.build_type = "Release" - c.settings.arch = "x86" - c.settings.compiler = "msvc" - c.settings.compiler.version = "170" - c.settings.compiler.toolset = "v110_xp" - c.settings.compiler.cppstd = "98" - c.settings.os = "Windows" - c.settings_build = c.settings - c.conf = Conf() - c.folders.set_base_generators(".") - c._conan_node = Mock() - c._conan_node.dependencies = [] - c._conan_node.transitive_deps = {} - c._conan_node.replaced_requires = {} - toolchain = CMakeToolchain(c) - content = toolchain.content - assert 'CMAKE_GENERATOR_TOOLSET "v110_xp"' in content - # As by the CMake docs, this has no effect for VS < 2015 - assert 'CMAKE_CXX_STANDARD 98' in content + def test_libcxx_abi_flag(self, base_conanfile): + c = base_conanfile + c.settings.build_type = "Release" + c.settings.arch = "x86_64" + c.settings.compiler = "gcc" + c.settings.compiler.version = "11" + c.settings.compiler.cppstd = "20" + c.settings.compiler.libcxx = "libstdc++" + c.settings.os = "Linux" + toolchain = CMakeToolchain(c) + content = toolchain.content + assert '_GLIBCXX_USE_CXX11_ABI=0' in content + c.settings.compiler.libcxx = "libstdc++11" + toolchain = CMakeToolchain(c) + content = toolchain.content + # by default, no flag is output anymore, it is assumed the compiler default + assert 'GLIBCXX_USE_CXX11_ABI' not in content + # recipe workaround for older distros + toolchain.blocks["libcxx"].values["glibcxx"] = "_GLIBCXX_USE_CXX11_ABI=1" + content = toolchain.content + assert '_GLIBCXX_USE_CXX11_ABI=1' in content -@pytest.fixture -def conanfile_linux(): - c = ConanFile() - c.settings = Settings({"os": ["Linux"], - "compiler": {"gcc": {"version": ["11"], "cppstd": ["20"]}}, - "build_type": ["Release"], - "arch": ["x86_64"]}) - c.settings.build_type = "Release" - c.settings.arch = "x86_64" - c.settings.compiler = "gcc" - c.settings.compiler.version = "11" - c.settings.compiler.cppstd = "20" - c.settings.os = "Linux" - c.settings_build = c.settings - c.conf = Conf() - c.folders.set_base_generators(".") - c._conan_node = Mock() - c._conan_node.dependencies = [] - c._conan_node.transitive_deps = {} - c._conan_node.replaced_requires = {} - return c + # but maybe the conf is better + c.conf.define("tools.gnu:define_libcxx11_abi", True) + toolchain = CMakeToolchain(c) + content = toolchain.content + assert '_GLIBCXX_USE_CXX11_ABI=1' in content -def test_no_fpic_when_not_an_option(conanfile_linux): - toolchain = CMakeToolchain(conanfile_linux) - content = toolchain.content - assert 'set(CMAKE_POSITION_INDEPENDENT_CODE' not in content +class TestMSVCToolset: + @pytest.fixture + def conanfile_msvc(self, base_conanfile): + c = base_conanfile + c.settings.build_type = "Release" + c.settings.arch = "x86" + c.settings.compiler = "msvc" + c.settings.compiler.version = "194" + c.settings.compiler.cppstd = "20" + c.settings.os = "Windows" + return c + def test_toolset(self, conanfile_msvc): + toolchain = CMakeToolchain(conanfile_msvc) + assert 'set(CMAKE_GENERATOR_TOOLSET "v143" CACHE STRING "" FORCE)' in toolchain.content + assert 'Visual Studio 17 2022' in toolchain.generator + assert 'CMAKE_CXX_STANDARD 20' in toolchain.content + + def test_toolset_latest_generator(self, conanfile_msvc): + conanfile_msvc.settings.compiler.version = "195" + toolchain = CMakeToolchain(conanfile_msvc) + assert 'set(CMAKE_GENERATOR_TOOLSET "v145" CACHE STRING "" FORCE)' in toolchain.content + assert 'Visual Studio 18 2026' in toolchain.generator + + def test_toolset_update_version(self, conanfile_msvc): + conanfile_msvc.settings.compiler.version = "193" + conanfile_msvc.settings.compiler.update = "8" + c = CMakeToolchain(conanfile_msvc) + assert 'set(CMAKE_GENERATOR_TOOLSET "v143,version=14.38" CACHE STRING "" FORCE)' in c.content + + def test_toolset_update_version_conf(self, conanfile_msvc): + conanfile_msvc.settings.compiler.version = "193" + conanfile_msvc.conf.define("tools.microsoft:msvc_update", "7") + c = CMakeToolchain(conanfile_msvc) + assert 'set(CMAKE_GENERATOR_TOOLSET "v143,version=14.37" CACHE STRING "" FORCE)' in c.content + + def test_toolset_update_version_forced_conf(self, conanfile_msvc): + conanfile_msvc.settings.compiler.version = "193" + conanfile_msvc.settings.compiler.update = "8" + conanfile_msvc.conf.define("tools.microsoft:msvc_update", "7") + c = CMakeToolchain(conanfile_msvc) + assert 'set(CMAKE_GENERATOR_TOOLSET "v143,version=14.37" CACHE STRING "" FORCE)' in c.content + + def test_toolset_update_version_overflow(self, conanfile_msvc): + # https://github.com/conan-io/conan/issues/15583 + conanfile_msvc.settings.compiler.version = "194" + conanfile_msvc.settings.compiler.update = "8" + c = CMakeToolchain(conanfile_msvc) + assert 'set(CMAKE_GENERATOR_TOOLSET "v143,version=14.48" CACHE STRING "" FORCE)' in c.content + + def test_toolset_x64(self, conanfile_msvc): + # https://github.com/conan-io/conan/issues/11144 + conanfile_msvc.conf.define("tools.cmake.cmaketoolchain:toolset_arch", "x64") + c = CMakeToolchain(conanfile_msvc) + assert 'set(CMAKE_GENERATOR_TOOLSET "v143,host=x64" CACHE STRING "" FORCE)' in c.content + assert 'Visual Studio 17 2022' in c.generator + assert 'CMAKE_CXX_STANDARD 20' in c.content + + def test_toolset_cuda(self, conanfile_msvc): + conanfile_msvc.conf.define("tools.cmake.cmaketoolchain:toolset_cuda", "C:/Path/To/CUDA") + c = CMakeToolchain(conanfile_msvc) + assert ('set(CMAKE_GENERATOR_TOOLSET "v143,cuda=C:/Path/To/CUDA" ' + 'CACHE STRING "" FORCE)') in c.content + + def test_older_msvc_toolset(self, conanfile_msvc): + c = conanfile_msvc + c.settings.compiler.version = "170" + c.settings.compiler.cppstd = "14" + toolchain = CMakeToolchain(c) + content = toolchain.content + assert 'CMAKE_GENERATOR_TOOLSET "v110"' in content + # As by the CMake docs, this has no effect for VS < 2015 + assert 'CMAKE_CXX_STANDARD 14' in content -@pytest.fixture -def conanfile_linux_shared(): - c = ConanFile() - c.options = Options({"fPIC": [True, False], - "shared": [True, False]}, - {"fPIC": False, "shared": True, }) - c.settings = Settings({"os": ["Linux"], - "compiler": {"gcc": {"version": ["11"], "cppstd": ["20"]}}, - "build_type": ["Release"], - "arch": ["x86_64"]}) - c.settings.build_type = "Release" - c.settings.arch = "x86_64" - c.settings.compiler = "gcc" - c.settings.compiler.version = "11" - c.settings.compiler.cppstd = "20" - c.settings.os = "Linux" - c.settings_build = c.settings - c.conf = Conf() - c.folders.set_base_generators(".") - c._conan_node = Mock() - c._conan_node.dependencies = [] - c._conan_node.transitive_deps = {} - c._conan_node.replaced_requires = {} - return c + def test_vs_clangcl(self, conanfile_msvc): + c = conanfile_msvc + c.settings.build_type = "Release" + c.settings.arch = "x86_64" + c.settings.compiler = "clang" + c.settings.compiler.runtime_version = "v145" + c.settings.compiler.version = "20" + c.settings.compiler.cppstd = "14" + c.settings.os = "Windows" + c.conf.define("tools.cmake.cmaketoolchain:generator", "Visual Studio 18") + toolchain = CMakeToolchain(c) + content = toolchain.content + assert 'CMAKE_GENERATOR_TOOLSET "ClangCL"' in content + assert 'CMAKE_CXX_STANDARD 14' in content + + def test_older_msvc_toolset_update(self, conanfile_msvc): + # https://github.com/conan-io/conan/issues/15787 + c = conanfile_msvc + c.settings.arch = "x86_64" + c.settings.compiler = "msvc" + c.settings.compiler.version = "192" + c.settings.compiler.update = "9" + c.settings.compiler.cppstd = "14" + c.settings.os = "Windows" + toolchain = CMakeToolchain(c) + content = toolchain.content + assert 'CMAKE_GENERATOR_TOOLSET "v142,version=14.29"' in content + def test_msvc_xp_toolsets(self, conanfile_msvc): + c = conanfile_msvc + c.settings.build_type = "Release" + c.settings.arch = "x86" + c.settings.compiler = "msvc" + c.settings.compiler.version = "170" + c.settings.compiler.toolset = "v110_xp" + c.settings.compiler.cppstd = "14" + toolchain = CMakeToolchain(c) + content = toolchain.content + assert 'CMAKE_GENERATOR_TOOLSET "v110_xp"' in content + # As by the CMake docs, this has no effect for VS < 2015 + assert 'CMAKE_CXX_STANDARD 14' in content -@pytest.mark.parametrize("fPIC", [True, False]) -def test_fpic_when_shared_true(conanfile_linux_shared, fPIC): - conanfile_linux_shared.options.fPIC = fPIC - toolchain = CMakeToolchain(conanfile_linux_shared) - cmake_value = 'ON' if fPIC else 'OFF' - content = toolchain.content - assert 'set(CMAKE_POSITION_INDEPENDENT_CODE {} CACHE BOOL'.format(cmake_value) in content +class TestFpic: + @pytest.fixture + def conanfile_linux(self, base_conanfile): + c = base_conanfile + c.settings.build_type = "Release" + c.settings.arch = "x86_64" + c.settings.compiler = "gcc" + c.settings.compiler.version = "11" + c.settings.compiler.cppstd = "20" + c.settings.os = "Linux" + return c -def test_fpic_when_not_shared(conanfile_linux_shared): - conanfile_linux_shared.options.shared = False - toolchain = CMakeToolchain(conanfile_linux_shared) - content = toolchain.content - assert 'set(CMAKE_POSITION_INDEPENDENT_CODE' in content + def test_no_fpic_when_not_an_option(self, conanfile_linux): + toolchain = CMakeToolchain(conanfile_linux) + content = toolchain.content + assert 'set(CMAKE_POSITION_INDEPENDENT_CODE' not in content + @pytest.fixture + def conanfile_linux_shared(self, conanfile_linux): + c = conanfile_linux + c.options = Options({"fPIC": [True, False], + "shared": [True, False]}, + {"fPIC": False, "shared": True, }) + return c -@pytest.fixture -def conanfile_windows_fpic(): - c = ConanFile() - c.settings = "os", "compiler", "build_type", "arch" - c.options = Options({"fPIC": [True, False]}, - {"fPIC": True}) - c.settings = Settings({"os": ["Windows"], - "compiler": {"gcc": {"libcxx": ["libstdc++"]}}, - "build_type": ["Release"], - "arch": ["x86"]}) - c.settings.build_type = "Release" - c.settings.arch = "x86" - c.settings.compiler = "gcc" - c.settings.compiler.libcxx = "libstdc++" - c.settings.os = "Windows" - c.settings_build = c.settings - c.conf = Conf() - c.folders.set_base_generators(".") - c._conan_node = Mock() - c._conan_node.dependencies = [] - c._conan_node.transitive_deps = {} - c._conan_node.replaced_requires = {} - return c + @pytest.mark.parametrize("fpic", [True, False]) + def test_fpic_when_shared_true(self, conanfile_linux_shared, fpic): + conanfile_linux_shared.options.fPIC = fpic + toolchain = CMakeToolchain(conanfile_linux_shared) + cmake_value = 'ON' if fpic else 'OFF' + content = toolchain.content + assert 'set(CMAKE_POSITION_INDEPENDENT_CODE {} CACHE BOOL'.format(cmake_value) in content + def test_fpic_when_not_shared(self, conanfile_linux_shared): + conanfile_linux_shared.options.shared = False + toolchain = CMakeToolchain(conanfile_linux_shared) + content = toolchain.content + assert 'set(CMAKE_POSITION_INDEPENDENT_CODE' in content -def test_no_fpic_on_windows(conanfile_windows_fpic): - toolchain = CMakeToolchain(conanfile_windows_fpic) - content = toolchain.content - assert 'set(CMAKE_POSITION_INDEPENDENT_CODE' not in content + @pytest.fixture + def conanfile_windows_fpic(self, base_conanfile): + c = base_conanfile + c.options = Options({"fPIC": [True, False]}, + {"fPIC": True}) + c.settings.build_type = "Release" + c.settings.arch = "x86" + c.settings.compiler = "gcc" + c.settings.compiler.libcxx = "libstdc++" + c.settings.os = "Windows" + return c + def test_no_fpic_on_windows(self, conanfile_windows_fpic): + toolchain = CMakeToolchain(conanfile_windows_fpic) + content = toolchain.content + assert 'set(CMAKE_POSITION_INDEPENDENT_CODE' not in content -@pytest.fixture -def conanfile_linux_fpic(): - c = ConanFile() - c.settings = "os", "compiler", "build_type", "arch" - c.options = Options({"fPIC": [True, False]}, - {"fPIC": False,}) - c.settings = Settings({"os": ["Linux"], - "compiler": {"gcc": {"version": ["11"], "cppstd": ["20"]}}, - "build_type": ["Release"], - "arch": ["x86_64"]}) - c.settings.build_type = "Release" - c.settings.arch = "x86_64" - c.settings.compiler = "gcc" - c.settings.compiler.version = "11" - c.settings.compiler.cppstd = "20" - c.settings.os = "Linux" - c.settings_build = c.settings - c.conf = Conf() - c.folders.set_base_generators(".") - c._conan_node = Mock() - c._conan_node.dependencies = [] - c._conan_node.transitive_deps = {} - c._conan_node.replaced_requires = {} - return c + @pytest.fixture + def conanfile_linux_fpic(self, base_conanfile): + c = base_conanfile + c.options = Options({"fPIC": [True, False]}, + {"fPIC": False}) + c.settings.build_type = "Release" + c.settings.arch = "x86_64" + c.settings.compiler = "gcc" + c.settings.compiler.version = "11" + c.settings.compiler.cppstd = "20" + c.settings.os = "Linux" + return c + def test_fpic_disabled(self, conanfile_linux_fpic): + conanfile_linux_fpic.options.fPIC = False + toolchain = CMakeToolchain(conanfile_linux_fpic) + content = toolchain.content + assert 'set(CMAKE_POSITION_INDEPENDENT_CODE OFF' in content -def test_fpic_disabled(conanfile_linux_fpic): - conanfile_linux_fpic.options.fPIC = False - toolchain = CMakeToolchain(conanfile_linux_fpic) - content = toolchain.content - assert 'set(CMAKE_POSITION_INDEPENDENT_CODE OFF' in content + def test_fpic_enabled(self, conanfile_linux_fpic): + conanfile_linux_fpic.options.fPIC = True + toolchain = CMakeToolchain(conanfile_linux_fpic) + content = toolchain.content + assert 'set(CMAKE_POSITION_INDEPENDENT_CODE ON' in content -def test_fpic_enabled(conanfile_linux_fpic): - conanfile_linux_fpic.options.fPIC = True - toolchain = CMakeToolchain(conanfile_linux_fpic) - content = toolchain.content - assert 'set(CMAKE_POSITION_INDEPENDENT_CODE ON' in content +class TestApple: + def test_osx_deployment_target(self, base_conanfile): + c = base_conanfile + c.settings.build_type = "Release" + c.settings.arch = "x86" + c.settings.compiler = "apple-clang" + c.settings.compiler.libcxx = "libc++" + c.settings.os = "Macos" + c.settings.os.version = "10.15" + toolchain = CMakeToolchain(c) + content = toolchain.content + assert 'set(CMAKE_OSX_DEPLOYMENT_TARGET "10.15" CACHE STRING "")' in content + + @pytest.mark.parametrize("os,os_sdk,arch,expected_sdk", [ + ("Macos", None, "x86_64", "macosx"), + ("Macos", None, "armv7", "macosx"), + ("iOS", "iphonesimulator", "armv8", "iphonesimulator"), + ("watchOS", "watchsimulator", "armv8", "watchsimulator") + ]) + def test_apple_cmake_osx_sysroot(self, base_conanfile, os, os_sdk, arch, expected_sdk): + """ + Testing if CMAKE_OSX_SYSROOT is correctly set. + Issue related: https://github.com/conan-io/conan/issues/10275 + """ + c = base_conanfile + c.settings.os = os + if os_sdk: + c.settings.os.sdk = os_sdk + c.settings.build_type = "Release" + c.settings.arch = arch + c.settings.compiler = "apple-clang" + c.settings.compiler.version = "13.0" + c.settings.compiler.libcxx = "libc++" + c.settings.compiler.cppstd = "17" + toolchain = CMakeToolchain(c) + content = toolchain.content + assert 'set(CMAKE_OSX_SYSROOT %s CACHE STRING "" FORCE)' % expected_sdk in content + + @pytest.mark.parametrize("os,arch,expected_sdk", [ + ("iOS", "x86_64", ""), + ("watchOS", "armv8", ""), + ("tvOS", "x86_64", "") + ]) + def test_apple_cmake_osx_sysroot_sdk_mandatory(self, base_conanfile, os, arch, expected_sdk): + """ + Testing if CMAKE_OSX_SYSROOT is correctly set. + Issue related: https://github.com/conan-io/conan/issues/10275 + """ + c = base_conanfile + c.settings.os = os + c.settings.build_type = "Release" + c.settings.arch = arch + c.settings.compiler = "apple-clang" + c.settings.compiler.version = "13.0" + c.settings.compiler.libcxx = "libc++" + c.settings.compiler.cppstd = "17" -def test_libcxx_abi_flag(): - c = ConanFile() - c.settings = "os", "compiler", "build_type", "arch" - c.settings = Settings.loads(default_settings_yml) - c.settings.build_type = "Release" - c.settings.arch = "x86_64" - c.settings.compiler = "gcc" - c.settings.compiler.version = "11" - c.settings.compiler.cppstd = "20" - c.settings.compiler.libcxx = "libstdc++" - c.settings.os = "Linux" - c.settings_build = c.settings - c.conf = Conf() - c.folders.set_base_generators(".") - c._conan_node = Mock() - c._conan_node.dependencies = [] - c._conan_node.transitive_deps = {} - c._conan_node.replaced_requires = {} - - toolchain = CMakeToolchain(c) - content = toolchain.content - assert '_GLIBCXX_USE_CXX11_ABI=0' in content - c.settings.compiler.libcxx = "libstdc++11" - toolchain = CMakeToolchain(c) - content = toolchain.content - # by default, no flag is output anymore, it is assumed the compiler default - assert 'GLIBCXX_USE_CXX11_ABI' not in content - # recipe workaround for older distros - toolchain.blocks["libcxx"].values["glibcxx"] = "_GLIBCXX_USE_CXX11_ABI=1" - content = toolchain.content - assert '_GLIBCXX_USE_CXX11_ABI=1' in content - - # but maybe the conf is better - c.conf.define("tools.gnu:define_libcxx11_abi", True) - toolchain = CMakeToolchain(c) - content = toolchain.content - assert '_GLIBCXX_USE_CXX11_ABI=1' in content - - -@pytest.mark.parametrize("os,os_sdk,arch,expected_sdk", [ - ("Macos", None, "x86_64", "macosx"), - ("Macos", None, "armv7", "macosx"), - ("iOS", "iphonesimulator", "armv8", "iphonesimulator"), - ("watchOS", "watchsimulator", "armv8", "watchsimulator") -]) -def test_apple_cmake_osx_sysroot(os, os_sdk, arch, expected_sdk): - """ - Testing if CMAKE_OSX_SYSROOT is correctly set. - Issue related: https://github.com/conan-io/conan/issues/10275 - """ - c = ConanFile() - c.settings = "os", "compiler", "build_type", "arch" - c.settings = Settings.loads(default_settings_yml) - c.settings.os = os - if os_sdk: - c.settings.os.sdk = os_sdk - c.settings.build_type = "Release" - c.settings.arch = arch - c.settings.compiler = "apple-clang" - c.settings.compiler.version = "13.0" - c.settings.compiler.libcxx = "libc++" - c.settings.compiler.cppstd = "17" - c.settings_build = c.settings - c.conf = Conf() - c.folders.set_base_generators(".") - c._conan_node = Mock() - c._conan_node.dependencies = [] - c._conan_node.transitive_deps = {} - c._conan_node.replaced_requires = {} - - toolchain = CMakeToolchain(c) - content = toolchain.content - assert 'set(CMAKE_OSX_SYSROOT %s CACHE STRING "" FORCE)' % expected_sdk in content - - -@pytest.mark.parametrize("os,arch,expected_sdk", [ - ("iOS", "x86_64", ""), - ("watchOS", "armv8", ""), - ("tvOS", "x86_64", "") -]) -def test_apple_cmake_osx_sysroot_sdk_mandatory(os, arch, expected_sdk): - """ - Testing if CMAKE_OSX_SYSROOT is correctly set. - Issue related: https://github.com/conan-io/conan/issues/10275 - """ - c = ConanFile() - c.settings = "os", "compiler", "build_type", "arch" - c.settings = Settings.loads(default_settings_yml) - c.settings.os = os - c.settings.build_type = "Release" - c.settings.arch = arch - c.settings.compiler = "apple-clang" - c.settings.compiler.version = "13.0" - c.settings.compiler.libcxx = "libc++" - c.settings.compiler.cppstd = "17" - c.settings_build = c.settings - c.conf = Conf() - c.folders.set_base_generators(".") - c._conan_node = Mock() - c._conan_node.dependencies = [] - - with pytest.raises(ConanException) as excinfo: - CMakeToolchain(c).content() - assert "Please, specify a suitable value for os.sdk." in str(excinfo.value) - - -def test_compilers_block(conanfile): - cmake_mapping = {"c": "C", "cuda": "CUDA", "cpp": "CXX", "objc": "OBJC", - "objcpp": "OBJCXX", "rc": "RC", 'fortran': "Fortran", 'asm': "ASM", - "hip": "HIP", "ispc": "ISPC"} - compilers = {"c": "path_to_c", "cuda": "path_to_cuda", "cpp": "path_to_cpp", - "objc": "path_to_objc", "objcpp": "path_to_objcpp", "rc": "path_to_rc", - 'fortran': "path_to_fortran", 'asm': "path_to_asm", "hip": "path_to_hip", - "ispc": "path_to_ispc"} - conanfile.conf.define("tools.build:compiler_executables", compilers) - toolchain = CMakeToolchain(conanfile) - content = toolchain.content - for compiler, lang in cmake_mapping.items(): - assert f'set(CMAKE_{lang}_COMPILER "path_to_{compiler}")' in content - - -def test_linker_scripts_block(conanfile): - conanfile.conf.define("tools.build:linker_scripts", - ["path_to_first_linker_script", "path_to_second_linker_script"]) - toolchain = CMakeToolchain(conanfile) - content = toolchain.content - assert r'string(APPEND CONAN_EXE_LINKER_FLAGS " -T\"path_to_first_linker_script\" ' \ - r'-T\"path_to_second_linker_script\"")' in content + with pytest.raises(ConanException) as excinfo: + _ = CMakeToolchain(c).content + assert "Please, specify a suitable value for os.sdk." in str(excinfo.value) class TestCrossBuild: @pytest.fixture - def conanfile_cross(self): - c = ConanFile() - c.settings = Settings({"os": ["baremetal", "Linux"], - "compiler": {"gcc": {"version": ["11"], "cppstd": ["20"]}}, - "build_type": ["Release"], - "arch": ["armv8", "x86_64"]}) + def conanfile_cross(self, base_conanfile): + c = base_conanfile c.settings.build_type = "Release" c.settings.arch = "armv8" c.settings.compiler = "gcc" @@ -688,12 +535,6 @@ def conanfile_cross(self): c.settings_build = c.settings.copy() c.settings_build.os = "Linux" c.settings_build.arch = "x86_64" - c.conf = Conf() - c.folders.set_base_generators(".") - c._conan_node = Mock() - c._conan_node.dependencies = [] - c._conan_node.transitive_deps = {} - c._conan_node.replaced_requires = {} return c def test_cmake_system_name(self, conanfile_cross): diff --git a/test/unittests/tools/env/test_env.py b/test/unittests/tools/env/test_env.py index 53411087eb2..282a5f40831 100644 --- a/test/unittests/tools/env/test_env.py +++ b/test/unittests/tools/env/test_env.py @@ -232,7 +232,8 @@ def test_windows_case_insensitive_bat(envvars): @pytest.mark.skipif(platform.system() != "Windows", reason="Requires Windows") -def test_windows_case_insensitive_ps1(envvars): +@pytest.mark.parametrize("deactivation_mode", ["function", None],) +def test_windows_case_insensitive_ps1(envvars, deactivation_mode): display = textwrap.dedent("""\ echo "MyVar=$env:MyVar!!" echo "MyVar1=$env:MyVar1!!" @@ -245,9 +246,11 @@ def test_windows_case_insensitive_ps1(envvars): prevenv.update(dict(os.environ.copy())) with chdir(temp_folder()): + envvars._deactivation_mode = deactivation_mode envvars.save_ps1("test.ps1") save("display.ps1", display) - cmd = "powershell.exe .\\test.ps1 ; .\\display.ps1 ; .\\deactivate_test.ps1 ; .\\display.ps1" + deactivate_cmd = "deactivate_test" if deactivation_mode else ".\\deactivate_test.ps1" + cmd = f"powershell.exe .\\test.ps1 ; .\\display.ps1 ; {deactivate_cmd} ; .\\display.ps1" check_command_output(cmd, prevenv) diff --git a/test/unittests/tools/env/test_env_files.py b/test/unittests/tools/env/test_env_files.py index 852ae42317a..02d8101b9ec 100644 --- a/test/unittests/tools/env/test_env_files.py +++ b/test/unittests/tools/env/test_env_files.py @@ -103,7 +103,8 @@ def test_env_files_bat(env, prevenv): @pytest.mark.skipif(platform.system() != "Windows", reason="Requires Windows") -def test_env_files_ps1(env, prevenv): +@pytest.mark.parametrize("deactivation_mode", ["function", None]) +def test_env_files_ps1(env, prevenv, deactivation_mode): prevenv.update(dict(os.environ.copy())) display = textwrap.dedent("""\ @@ -121,16 +122,21 @@ def test_env_files_ps1(env, prevenv): """) with chdir(temp_folder()): - env = env.vars(ConanFileMock()) + conanfile = ConanFileMock() + if deactivation_mode: + conanfile.conf.define("tools.env:deactivation_mode", deactivation_mode) + env = env.vars(conanfile) env._subsystem = WINDOWS env.save_ps1("test.ps1") save("display.ps1", display) - cmd = "powershell.exe .\\test.ps1 ; .\\display.ps1 ; .\\deactivate_test.ps1 ; .\\display.ps1" + deactivate_cmd = "deactivate_test" if deactivation_mode else ".\\deactivate_test.ps1" + cmd = f"powershell.exe .\\test.ps1 ; .\\display.ps1 ; {deactivate_cmd} ; .\\display.ps1" check_env_files_output(cmd, prevenv) @pytest.mark.skipif(platform.system() == "Windows", reason="Not in Windows") -def test_env_files_sh(env, prevenv): +@pytest.mark.parametrize("deactivation_mode", ["function", None]) +def test_env_files_sh(env, prevenv, deactivation_mode): display = textwrap.dedent("""\ echo MyVar=$MyVar!! echo MyVar1=$MyVar1!! @@ -146,12 +152,16 @@ def test_env_files_sh(env, prevenv): """) with chdir(temp_folder()): - env = env.vars(ConanFileMock()) + conanfile = ConanFileMock() + if deactivation_mode: + conanfile.conf.define("tools.env:deactivation_mode", deactivation_mode) + env = env.vars(conanfile) env.save_sh("test.sh") save("display.sh", display) os.chmod("display.sh", 0o777) # We include the "set -e" to test it is robust against errors - cmd = 'set -e && . ./test.sh && ./display.sh && . ./deactivate_test.sh && ./display.sh' + deactivate_cmd = "deactivate_test" if deactivation_mode else ". ./deactivate_test.sh" + cmd = f'set -e && . ./test.sh && ./display.sh && {deactivate_cmd} && ./display.sh' check_env_files_output(cmd, prevenv) @@ -199,3 +209,30 @@ def test_relative_paths(): shell=True).communicate() out = result.decode() assert 'Hello MyWorld!!!' in out + + +@pytest.mark.parametrize("values, expected", + [("myscripts", '"$script_folder/myscripts"'), + ("../myscripts", '"$script_folder/../myscripts"'), + ("../my scripts", '"$script_folder/../my scripts"'), + (["../my scripts", "path/other"], + '"$script_folder/../my scripts:$script_folder/path/other"')]) +def test_relativize(values, expected): + folder = os.path.join(temp_folder(), "subfolder") + os.makedirs(folder) + if isinstance(values, str): + value = os.path.join(folder, values) if not os.path.isabs(values) else values + else: + value = [os.path.join(folder, v) for v in values] + myenv = Environment() + myenv.define_path("PATH", value) + myenv.prepend_path("OTHER", value) + conanfile = ConanFileMock() + conanfile.folders._base_generators = folder + myenv = myenv.vars(conanfile) + with chdir(folder): + myenv.save_sh("test.sh") + content = load("test.sh") + content = content.replace("\\", "/") + assert f'export PATH={expected}' in content + assert f'export OTHER="{expected}:$OTHER"' diff --git a/test/unittests/tools/files/test_downloads.py b/test/unittests/tools/files/test_downloads.py index 8d84c36d19f..f6002cd2ea5 100644 --- a/test/unittests/tools/files/test_downloads.py +++ b/test/unittests/tools/files/test_downloads.py @@ -156,6 +156,7 @@ def test_download_localfile(self): file_location = os.path.join(temp_folder(), "file.txt") save(file_location, "this is some content") + file_location = file_location.lstrip("/") file_url = f"file:///{file_location}" file_md5 = "736db904ad222bf88ee6b8d103fceb8e" diff --git a/test/unittests/tools/files/test_patches.py b/test/unittests/tools/files/test_patches.py index 34708b39da4..844164788da 100644 --- a/test/unittests/tools/files/test_patches.py +++ b/test/unittests/tools/files/test_patches.py @@ -151,7 +151,7 @@ def test_single_no_patchset(monkeypatch): def test_single_apply_fail(monkeypatch): class MockedApply: - def apply(self, *args, **kwargs): + def apply(self, *args, **kwargs): # noqa return False monkeypatch.setattr(patch_ng, "fromfile", lambda _: MockedApply()) diff --git a/test/unittests/tools/files/test_rm.py b/test/unittests/tools/files/test_rm.py index ab0fcbde4ad..44c9a8691b2 100644 --- a/test/unittests/tools/files/test_rm.py +++ b/test/unittests/tools/files/test_rm.py @@ -8,14 +8,14 @@ def test_remove_files_by_mask_recursively(): - tmpdir = temp_folder() + temp_dir = temp_folder() - with chdir(None, tmpdir): + with chdir(None, temp_dir): os.makedirs("subdir") os.makedirs("dir.pdb") os.makedirs(os.path.join("subdir", "deepdir")) - save_files(tmpdir, {"1.txt": "", + save_files(temp_dir, {"1.txt": "", "1.pdb": "", "1.pdb1": "", os.path.join("subdir", "2.txt"): "", @@ -25,45 +25,45 @@ def test_remove_files_by_mask_recursively(): os.path.join("subdir", "deepdir", "3.pdb"): "", os.path.join("subdir", "deepdir", "3.pdb1"): ""}) - rm(None, "*.sh", tmpdir, recursive=True) + rm(None, "*.sh", temp_dir, recursive=True) - removed_files = rm(None, "*.pdb", tmpdir, recursive=True) + rm(None, "*.pdb", temp_dir, recursive=True) - assert os.path.isdir(os.path.join(tmpdir, "dir.pdb")) + assert os.path.isdir(os.path.join(temp_dir, "dir.pdb")) - assert os.path.isfile(os.path.join(tmpdir, "1.txt")) - assert not os.path.isfile(os.path.join(tmpdir, "1.pdb")) - assert os.path.isfile(os.path.join(tmpdir, "1.pdb1")) + assert os.path.isfile(os.path.join(temp_dir, "1.txt")) + assert not os.path.isfile(os.path.join(temp_dir, "1.pdb")) + assert os.path.isfile(os.path.join(temp_dir, "1.pdb1")) - assert os.path.isfile(os.path.join(tmpdir, "subdir", "2.txt")) - assert not os.path.isfile(os.path.join(tmpdir, "subdir", "2.pdb")) - assert os.path.isfile(os.path.join(tmpdir, "subdir", "2.pdb1")) + assert os.path.isfile(os.path.join(temp_dir, "subdir", "2.txt")) + assert not os.path.isfile(os.path.join(temp_dir, "subdir", "2.pdb")) + assert os.path.isfile(os.path.join(temp_dir, "subdir", "2.pdb1")) - assert os.path.isfile(os.path.join(tmpdir, "subdir", "deepdir", "3.txt")) - assert not os.path.isfile(os.path.join(tmpdir, "subdir", "deepdir", "3.pdb")) - assert os.path.isfile(os.path.join(tmpdir, "subdir", "deepdir", "3.pdb1")) + assert os.path.isfile(os.path.join(temp_dir, "subdir", "deepdir", "3.txt")) + assert not os.path.isfile(os.path.join(temp_dir, "subdir", "deepdir", "3.pdb")) + assert os.path.isfile(os.path.join(temp_dir, "subdir", "deepdir", "3.pdb1")) - rm(None, "*.pdb", tmpdir, recursive=True) + rm(None, "*.pdb", temp_dir, recursive=True) def test_remove_files_by_mask_non_recursively(): - tmpdir = temp_folder() - with chdir(None, tmpdir): + temp_dir = temp_folder() + with chdir(None, temp_dir): os.makedirs("subdir") - save_files(tmpdir, {"1.txt": "", + save_files(temp_dir, {"1.txt": "", "1.pdb": "", "1.pdb1": "", os.path.join("subdir", "2.txt"): "", os.path.join("subdir", "2.pdb"): "", os.path.join("subdir", "2.pdb1"): ""}) - rm(None, "*.pdb", tmpdir) - assert not os.path.exists(os.path.join(tmpdir, "1.pdb")) - assert os.path.exists(os.path.join(tmpdir, "subdir", "2.pdb")) + rm(None, "*.pdb", temp_dir) + assert not os.path.exists(os.path.join(temp_dir, "1.pdb")) + assert os.path.exists(os.path.join(temp_dir, "subdir", "2.pdb")) - assert os.path.exists(os.path.join(tmpdir, "1.txt")) - assert os.path.exists(os.path.join(tmpdir, "subdir", "2.txt")) + assert os.path.exists(os.path.join(temp_dir, "1.txt")) + assert os.path.exists(os.path.join(temp_dir, "subdir", "2.txt")) @pytest.mark.parametrize("recursive", [False, True]) diff --git a/test/unittests/tools/files/test_tool_copy.py b/test/unittests/tools/files/test_tool_copy.py index aacbe5605b4..1b2e2293285 100644 --- a/test/unittests/tools/files/test_tool_copy.py +++ b/test/unittests/tools/files/test_tool_copy.py @@ -1,4 +1,4 @@ -import mock +from unittest import mock import os import platform import pytest @@ -54,8 +54,8 @@ def test_symlinks_folder_behavior(self): save(test2, "") gen_folder = os.path.join(build_folder, "gen") mkdir(gen_folder) - bin = os.path.join(gen_folder, "test.bin") - save(bin, "") + binfile = os.path.join(gen_folder, "test.bin") + save(binfile, "") sym_folder = os.path.join(build_folder, "sym") os.symlink(gen_folder, sym_folder) diff --git a/test/unittests/tools/files/test_zipping.py b/test/unittests/tools/files/test_zipping.py index 1d24632d51d..c18c86f47c8 100644 --- a/test/unittests/tools/files/test_zipping.py +++ b/test/unittests/tools/files/test_zipping.py @@ -11,11 +11,6 @@ from conan.errors import ConanException -def test_impossible_to_import_untargz(): - with pytest.raises(ImportError) as exc: - from conan.tools.files import untargz - - def create_example_zip(root_file=True, subfolder=False): tmp_dir = temp_folder() archive = join(tmp_dir, "zipfile.zip") @@ -53,6 +48,16 @@ def test_unzip_with_pattern(): assert not exists(join(dest_dir, "src", "bar.txt")) +def test_unzip_with_exclude_pattern(): + archive = create_example_zip(subfolder=True) + conanfile = ConanFileMock({}) + + dest_dir = temp_folder() + unzip(conanfile, archive, dest_dir, excludes=["src/*"]) + assert exists(join(dest_dir, "foo.txt")) + assert not exists(join(dest_dir, "src", "bar.txt")) + + def test_unzip_with_strip_root(): archive = create_example_zip(root_file=False, subfolder=True) conanfile = ConanFileMock({}) @@ -126,6 +131,16 @@ def test_untargz_with_pattern(): assert not exists(join(dest_dir, "src", "bar.txt")) +def test_untargz_with_exclude_pattern(): + archive = create_example_tar(subfolder=True) + conanfile = ConanFileMock({}) + + dest_dir = temp_folder() + unzip(conanfile, archive, dest_dir, excludes=["src/*"]) + assert exists(join(dest_dir, "foo.txt")) + assert not exists(join(dest_dir, "src", "bar.txt")) + + def test_untargz_with_strip_root(): archive = create_example_tar(root_file=False, subfolder=True) conanfile = ConanFileMock({}) @@ -142,9 +157,9 @@ def test_untargz_with_strip_root_fails(): # Unzip and check permissions are kept dest_dir = temp_folder() - with pytest.raises(ConanException) as error: + with pytest.raises(ConanException) as e: unzip(conanfile, archive, dest_dir, strip_root=True) - assert "Can't untar a tgz containing files in the root with strip_root enabled" in str(error.value) + assert "Can't untar a tgz containing files in the root with strip_root enabled" in str(e.value) def test_untargz_with_strip_root_and_pattern(): diff --git a/test/unittests/tools/gnu/autotools_test.py b/test/unittests/tools/gnu/autotools_test.py index 97b1a22130d..2708e4418e3 100644 --- a/test/unittests/tools/gnu/autotools_test.py +++ b/test/unittests/tools/gnu/autotools_test.py @@ -1,9 +1,10 @@ +import pytest import os -from mock import patch +from unittest.mock import patch from conan.tools.build import save_toolchain_args from conan.tools.gnu import Autotools -from conan.test.utils.mocks import ConanFileMock +from conan.test.utils.mocks import ConanFileMock, MockSettings from conan.test.utils.test_files import temp_folder @@ -28,8 +29,110 @@ def test_source_folder_works(chdir_mock): assert conanfile.command.replace("\\", "/") == '"/path/to/sources/configure" -foo bar ' autotools.autoreconf(build_script_folder="subfolder") - chdir_mock.assert_called_with(autotools, os.path.normpath(os.path.join("/path/to/sources", "subfolder"))) + chdir_mock.assert_called_with(autotools, + os.path.normpath(os.path.join("/path/to/sources", "subfolder"))) autotools.autoreconf() chdir_mock.assert_called_with(autotools, os.path.normpath("/path/to/sources")) assert conanfile.command == 'autoreconf -bar foo' + + +@pytest.mark.parametrize("install_strip", [False, True]) +def test_install_strip(install_strip): + """ + When the configuration `tools.build:install_strip` is set to True, + the Autotools install command should invoke the `install-strip` target. + """ + folder = temp_folder() + os.chdir(folder) + save_toolchain_args({}) + settings = MockSettings({"build_type": "Release", + "compiler": "gcc", + "compiler.version": "7", + "os": "Linux", + "arch": "x86_64"}) + conanfile = ConanFileMock() + conanfile.settings = settings + conanfile.conf.define("tools.build:install_strip", install_strip) + conanfile.folders.generators = "." + + autotools = Autotools(conanfile) + autotools.install() + + assert ('install-strip' in str(conanfile.command)) == install_strip + + +def test_configure_arguments(): + tmp = temp_folder() + os.chdir(tmp) + save_toolchain_args({ + "configure_args": "my_configure_args", + "make_args": "my_make_args"} + ) + + conanfile = ConanFileMock() + conanfile.settings = MockSettings({"os": "Linux"}) + conanfile.folders.set_base_source(tmp) + conanfile.conf.define("tools.gnu:make_program", "my_make") + conanfile.conf.define("tools.build:jobs", 23) + + ab = Autotools(conanfile) + ab.configure() + assert "configure\" my_configure_args" in str(conanfile.command) + + ab = Autotools(conanfile) + ab.make() + assert "my_make my_make_args -j23" == str(conanfile.command) + + # test install target argument + + ab.install() + assert 'my_make install my_make_args DESTDIR=None -j23' == str(conanfile.command) + + ab.install(target="install_other") + assert 'my_make install_other my_make_args DESTDIR=None -j23' == str(conanfile.command) + + +@pytest.mark.parametrize("make_args", ["my_make_args", None]) +def test_configure_install_arguments(make_args): + tmp = temp_folder() + os.chdir(tmp) + + conanfile = ConanFileMock() + conanfile.settings = MockSettings({"os": "Linux"}) + conanfile.settings_build = conanfile.settings + conanfile.folders.set_base_source(tmp) + conanfile.conf.define("tools.gnu:make_program", "my_make") + conanfile.conf.define("tools.build:jobs", 23) + save_toolchain_args({ + "configure_args": "my_configure_args", + "make_args": f"{make_args or ''}"} + ) + + ab = Autotools(conanfile) + + make_args = f" {make_args}" if make_args else "" + + ab.make(args=["-j1"]) + assert "-j23" not in str(conanfile.command) + assert f"my_make{make_args} -j1" == str(conanfile.command) + + ab.install(args=["-j1"]) + assert "-j23" not in str(conanfile.command) + assert f"my_make install{make_args} DESTDIR=None -j1" == str(conanfile.command) + + ab.install(args=["DESTDIR=whatever", "-j1"]) + assert "-j23" not in str(conanfile.command) + assert f"my_make install{make_args} DESTDIR=whatever -j1" == str(conanfile.command) + + ab.install(args=["DESTDIR=whatever", "-arg1 -j1 -arg2"]) + assert "-j23" not in str(conanfile.command) + assert f"my_make install{make_args} DESTDIR=whatever -arg1 -j1 -arg2" == str(conanfile.command) + + # check that we don't detect -j in an argument as number of jobs + ab.install(args=["DESTDIR=/user/smith-john/what"]) + assert f"my_make install{make_args} DESTDIR=/user/smith-john/what -j23" == str(conanfile.command) + + # check that we don't detect -j in an argument as number of jobs + ab.install(args=["DESTDIR=/user/smith-j47/what"]) + assert f"my_make install{make_args} DESTDIR=/user/smith-j47/what -j23" == str(conanfile.command) diff --git a/test/unittests/client/toolchain/autotools/autotools_toolchain_test.py b/test/unittests/tools/gnu/autotools_toolchain_test.py similarity index 90% rename from test/unittests/client/toolchain/autotools/autotools_toolchain_test.py rename to test/unittests/tools/gnu/autotools_toolchain_test.py index ca6dcde783d..8b470ed4fc9 100644 --- a/test/unittests/client/toolchain/autotools/autotools_toolchain_test.py +++ b/test/unittests/tools/gnu/autotools_toolchain_test.py @@ -223,6 +223,51 @@ def test_libcxx(config): assert expected_flag in env["CXXFLAGS"] +def test_disable_libcxx(): + conanfile = ConanFileMock() + conanfile.settings = MockSettings( + {"os": "Linux", + "build_type": "Release", + "arch": "x86", + "compiler": "clang", + "compiler.libcxx": "libc++", + "compiler.version": "7.1", + "compiler.cppstd": "17", + "compiler.cstd": "99"}) + conanfile.settings_build = conanfile.settings + # Disable just one + conanfile.conf.define("tools.gnu:disable_flags", ["libcxx"]) + be = AutotoolsToolchain(conanfile) + assert be.libcxx is None + assert be.arch_flag == "-m32" + assert be.build_type_flags == ['-O3'] + assert be.cppstd == "-std=c++17" + assert be.cstd == "-std=c99" + # Disable two + conanfile.conf.define("tools.gnu:disable_flags", ["libcxx", "cppstd"]) + be = AutotoolsToolchain(conanfile) + assert be.libcxx is None + assert be.arch_flag == "-m32" + assert be.build_type_flags == ['-O3'] + assert be.cppstd == "" + assert be.cstd == "-std=c99" + # Error value + conanfile.conf.define("tools.gnu:disable_flags", ["other"]) + try: + AutotoolsToolchain(conanfile) + except ConanException as e: + assert "tools.gnu:disable_flags value 'other', must be one of:" in str(e) + conanfile.conf.define("tools.gnu:disable_flags", + ["arch", "arch_link", "libcxx", "build_type", + "build_type_link", "threads", "cppstd", "cstd"]) + be = AutotoolsToolchain(conanfile) + assert be.libcxx is None + assert be.arch_flag == "" + assert be.build_type_flags == [] + assert be.cppstd == "" + assert be.cstd == "" + + def test_cxx11_abi_define(): conanfile = ConanFileMock() conanfile.settings = MockSettings( @@ -285,6 +330,7 @@ def test_architecture_flag(config): assert expected in env["LDFLAGS"] assert "-debug" not in env["LDFLAGS"] + @pytest.mark.parametrize("config", [ ("gcc", "x86_64", ""), ("emcc", "wasm", ""), @@ -398,10 +444,10 @@ def test_crossbuild_to_android(build_env_mock): """ Issue related: https://github.com/conan-io/conan/issues/17441 """ - vars = MagicMock() + buildvars = MagicMock() # VirtualBuildEnv defines these variables - vars.vars.return_value = {"CC": "my-clang", "CXX": "my-clang++"} - build_env_mock.return_value = vars + buildvars.vars.return_value = {"CC": "my-clang", "CXX": "my-clang++"} + build_env_mock.return_value = buildvars conanfile = ConanFileMock() conanfile.settings = MockSettings({"os": "Android", "arch": "armv8"}) @@ -485,6 +531,21 @@ def test_sysrootflag(): assert expected in env["LDFLAGS"] +def test_sysrootflag_qnx(): + """Even when no cross building it is adjusted because it could target a Mac version""" + conanfile = ConanFileMock() + conanfile.conf.define("tools.build:sysroot", "/path/to/sysroot") + conanfile.settings = MockSettings( + {"compiler": "qcc", + "build_type": "Debug", + "os": "Linux", + "arch": "x86_64"}) + conanfile.settings_build = conanfile.settings + be = AutotoolsToolchain(conanfile) + expected = "-Wc,-isysroot,/path/to/sysroot" + assert be.sysroot_flag == expected + + def test_custom_defines(): conanfile = ConanFileMock() conanfile.conf.define("tools.apple:sdk_path", "/path/to/sdk") diff --git a/test/unittests/tools/gnu/gnudepsflags_test.py b/test/unittests/tools/gnu/gnudepsflags_test.py index 3e62b9a0f97..70e67cfde6a 100644 --- a/test/unittests/tools/gnu/gnudepsflags_test.py +++ b/test/unittests/tools/gnu/gnudepsflags_test.py @@ -1,5 +1,5 @@ import pytest -from mock import MagicMock +from unittest.mock import MagicMock from conan.tools.apple.apple import is_apple_os from conan.tools.gnu.gnudeps_flags import GnuDepsFlags diff --git a/test/unittests/tools/gnu/test_gnutoolchain.py b/test/unittests/tools/gnu/test_gnutoolchain.py index 650179cbf34..4b19e261fdd 100644 --- a/test/unittests/tools/gnu/test_gnutoolchain.py +++ b/test/unittests/tools/gnu/test_gnutoolchain.py @@ -166,6 +166,21 @@ def test_linker_scripts(): assert "-T'path_to_second_linker_script'" in env["LDFLAGS"] +def test_sysrootflag_qnx(): + """Even when no cross building it is adjusted because it could target a Mac version""" + conanfile = ConanFileMock() + conanfile.conf.define("tools.build:sysroot", "/path/to/sysroot") + conanfile.settings = MockSettings( + {"compiler": "qcc", + "build_type": "Debug", + "os": "Linux", + "arch": "x86_64"}) + conanfile.settings_build = conanfile.settings + be = GnuToolchain(conanfile) + expected = "-Wc,-isysroot,/path/to/sysroot" + assert be.sysroot_flag == expected + + def test_update_or_prune_any_args(cross_building_conanfile): # Issue: https://github.com/conan-io/conan/issues/12642 at = GnuToolchain(cross_building_conanfile) @@ -207,10 +222,10 @@ def test_crossbuild_to_android(build_env_mock): """ Issue related: https://github.com/conan-io/conan/issues/17441 """ - vars = MagicMock() + buildvars = MagicMock() # VirtualBuildEnv defines these variables - vars.vars.return_value = {"CC": "my-clang", "CXX": "my-clang++"} - build_env_mock.return_value = vars + buildvars.vars.return_value = {"CC": "my-clang", "CXX": "my-clang++"} + build_env_mock.return_value = buildvars conanfile = ConanFileMock() conanfile.settings = MockSettings({"os": "Android", "arch": "armv8", "os.api_level": "26r"}) diff --git a/test/unittests/tools/intel/test_intel_cc.py b/test/unittests/tools/intel/test_intel_cc.py index 6fcc8e3e179..4ae8fd70c75 100644 --- a/test/unittests/tools/intel/test_intel_cc.py +++ b/test/unittests/tools/intel/test_intel_cc.py @@ -2,7 +2,7 @@ import textwrap import pytest -from mock import patch +from unittest.mock import patch from conan.tools.build.flags import architecture_flag, cppstd_flag from conan.tools.intel import IntelCC diff --git a/test/unittests/tools/meson/test_meson.py b/test/unittests/tools/meson/test_meson.py index 7b4bc800eed..b0cf0e82895 100644 --- a/test/unittests/tools/meson/test_meson.py +++ b/test/unittests/tools/meson/test_meson.py @@ -50,7 +50,7 @@ def test_meson_subsystem_helper(apple_sdk, subsystem): (None, None) ]) def test_meson_to_cstd_flag(cstd, expected): - assert to_cstd_flag(cstd) == expected + assert to_cstd_flag(ConanFileMock(), cstd) == expected @pytest.mark.parametrize("compiler, compiler_version, cppstd, expected", [ @@ -64,7 +64,7 @@ def test_meson_to_cstd_flag(cstd, expected): (None, None, None, None) ]) def test_meson_to_cppstd_flag(compiler, compiler_version, cppstd, expected): - assert to_cppstd_flag(compiler, compiler_version, cppstd) == expected + assert to_cppstd_flag(ConanFileMock(), compiler, compiler_version, cppstd) == expected def test_meson_install_strip(): @@ -90,3 +90,25 @@ def test_meson_install_strip(): meson.install() assert '--strip' in str(conanfile.command) + +def test_meson_install_cli_args(): + """When the `cli_args` are provided, the Meson install command should include them. + """ + c = ConfDefinition() + + settings = MockSettings({"build_type": "Release", + "compiler": "gcc", + "compiler.version": "7", + "os": "Linux", + "arch": "x86_64"}) + conanfile = ConanFileMock() + conanfile.settings = settings + conanfile.conf = c.get_conanfile_conf(None) + conanfile.folders.generators = "." + conanfile.folders.set_base_generators(temp_folder()) + conanfile.folders.set_base_package(temp_folder()) + + meson = Meson(conanfile) + meson.install(cli_args=["--no-rebuild"]) + + assert '--no-rebuild' in str(conanfile.command) diff --git a/test/unittests/tools/microsoft/test_msbuild.py b/test/unittests/tools/microsoft/test_msbuild.py index 7253ea12362..57819abd124 100644 --- a/test/unittests/tools/microsoft/test_msbuild.py +++ b/test/unittests/tools/microsoft/test_msbuild.py @@ -25,7 +25,7 @@ def test_msbuild_targets(): msbuild = MSBuild(conanfile) cmd = msbuild.command('project.sln', targets=["static", "shared"]) - assert '-target:static;shared' in cmd + assert '-target:"static;shared"' in cmd def test_msbuild_cpu_count(): @@ -40,12 +40,12 @@ def test_msbuild_cpu_count(): msbuild = MSBuild(conanfile) cmd = msbuild.command('project.sln') - assert 'msbuild.exe "project.sln" -p:Configuration="Release" -p:Platform=x64 -m:23' == cmd + assert 'msbuild.exe "project.sln" -p:Configuration="Release" -p:Platform="x64" -m:"23"' == cmd c.loads("tools.microsoft.msbuild:max_cpu_count=0") conanfile.conf = c.get_conanfile_conf(None) cmd = msbuild.command('project.sln') - assert 'msbuild.exe "project.sln" -p:Configuration="Release" -p:Platform=x64 -m' == cmd + assert 'msbuild.exe "project.sln" -p:Configuration="Release" -p:Platform="x64" -m' == cmd def test_msbuild_toolset(): @@ -54,6 +54,7 @@ def test_msbuild_toolset(): "os": ["Windows"], "arch": ["x86_64"]}) conanfile = ConanFile(None) + conanfile.conf = Conf() conanfile.settings = "os", "compiler", "build_type", "arch" conanfile.settings = settings conanfile.settings.build_type = "Release" @@ -77,6 +78,7 @@ def test_msbuild_toolset(): ]) def test_msbuild_toolset_for_intel_cc(mode, expected_toolset): conanfile = ConanFile() + conanfile.conf = Conf() conanfile.settings = "os", "compiler", "build_type", "arch" conanfile.settings = Settings({"build_type": ["Release"], "compiler": {"intel-cc": {"version": ["2021.3"], "mode": [mode]}, diff --git a/test/unittests/tools/microsoft/test_msvs_toolset.py b/test/unittests/tools/microsoft/test_msvs_toolset.py index e1c7f3098f7..5f4f9b9eb2e 100644 --- a/test/unittests/tools/microsoft/test_msvs_toolset.py +++ b/test/unittests/tools/microsoft/test_msvs_toolset.py @@ -15,6 +15,7 @@ def test_invalid_compiler(): @pytest.mark.parametrize("compiler_version,expected_toolset", [ + ("195", "v145"), ("194", "v143"), ("193", "v143"), ("192", "v142"), diff --git a/test/unittests/tools/system/__init__.py b/test/unittests/tools/system/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/unittests/tools/system/pip_manager_test.py b/test/unittests/tools/system/pip_manager_test.py new file mode 100644 index 00000000000..20fcaf42ff0 --- /dev/null +++ b/test/unittests/tools/system/pip_manager_test.py @@ -0,0 +1,49 @@ +from conan.tools.system import PipEnv +from unittest.mock import patch +import pytest +from conan.errors import ConanException +from conan.internal.model.settings import Settings +from conan.test.utils.mocks import ConanFileMock + + +@patch('shutil.which') +def test_pipenv_conf(mock_shutil_which): + conanfile = ConanFileMock() + conanfile.settings = Settings() + conanfile.conf.define("tools.system.pipenv:python_interpreter", + "/python/interpreter/from/config") + result = "/python/interpreter/from/config -m venv" + PipEnv(conanfile, "testenv") + + def fake_run(command, win_bash=False, subsystem=None, env=None, ignore_errors=False, # noqa + quiet=False): # noqa + assert result in command + return 100 + conanfile.run = fake_run + mock_shutil_which.assert_not_called() + + +@patch('shutil.which') +def test_pipenv_error_message(mock_shutil_which): + conanfile = ConanFileMock() + conanfile.settings = Settings() + mock_shutil_which.return_value = None + with pytest.raises(ConanException) as exc_info: + PipEnv(conanfile, "testenv") + assert ("install Python system-wide or set the 'tools.system.pipenv:python_interpreter' " + "conf") in exc_info.value.args[0] + + +def test_pipenv_creation_error_message(): + conanfile = ConanFileMock() + conanfile.settings = Settings() + conanfile.conf.define("tools.system.pipenv:python_interpreter", + "/python/interpreter/from/config") + + def fake_run(command, win_bash=False, subsystem=None, env=None, ignore_errors=False, # noqa + quiet=False): # noqa + raise ConanException("fake error message") + conanfile.run = fake_run + with pytest.raises(ConanException) as exc_info: + PipEnv(conanfile, "testenv") + assert "using '/python/interpreter/from/config': fake error message" in exc_info.value.args[0] diff --git a/test/unittests/util/conanfile_tools_test.py b/test/unittests/util/conanfile_tools_test.py index 1056f213777..f616b451c02 100644 --- a/test/unittests/util/conanfile_tools_test.py +++ b/test/unittests/util/conanfile_tools_test.py @@ -1,5 +1,9 @@ import os import stat +import tarfile +import textwrap + +import pytest from conan.tools.files import replace_in_file, unzip from conan.test.utils.mocks import ConanFileMock @@ -14,12 +18,13 @@ class TestConanfileTools: def test_save_append(self): # https://github.com/conan-io/conan/issues/2841 (regression) client = TestClient() - conanfile = """from conan import ConanFile -from conan.tools.files import save -class Pkg(ConanFile): - def source(self): - save(self, "myfile.txt", "Hello", append=True) -""" + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.files import save + class Pkg(ConanFile): + def source(self): + save(self, "myfile.txt", "Hello", append=True) + """) client.save({"conanfile.py": conanfile, "myfile.txt": "World"}) client.run("source .") @@ -30,15 +35,10 @@ def test_untar(self): file_path = os.path.join(tmp_dir, "example.txt") save(file_path, "Hello world!") tar_path = os.path.join(tmp_dir, "sample.tar") - try: - old_path = os.getcwd() - os.chdir(tmp_dir) - import tarfile - tar = tarfile.open(tar_path, "w") - tar.add("example.txt") - tar.close() - finally: - os.chdir(old_path) + + with tarfile.open(tar_path, "w") as tar: + tar.add(file_path, "example.txt") + output_dir = os.path.join(tmp_dir, "output_dir") unzip(ConanFileMock(), tar_path, output_dir) content = load(os.path.join(output_dir, "example.txt")) @@ -62,8 +62,5 @@ def test_replace_in_file_readonly(self): os.chmod(text_file, os.stat(text_file).st_mode & ~(stat.S_IWRITE | stat.S_IWGRP | stat.S_IWOTH)) - try: + with pytest.raises(PermissionError): replace_in_file(ConanFileMock(), text_file, "ONE TWO THREE", "FOUR FIVE SIX") - assert False, "Expected PermissionError" - except PermissionError: - pass diff --git a/test/unittests/util/detect_libc_test.py b/test/unittests/util/detect_libc_test.py index c0a8160a03a..074f805df6f 100644 --- a/test/unittests/util/detect_libc_test.py +++ b/test/unittests/util/detect_libc_test.py @@ -1,10 +1,10 @@ -from parameterized import parameterized +import pytest from conan.internal.api.detect.detect_api import _parse_gnu_libc, _parse_musl_libc class TestDetectLibc: - @parameterized.expand( + @pytest.mark.parametrize("ldd_output,expected_glibc_version", [ [ """ldd (Debian GLIBC 2.36-9+rpt2+deb12u4) 2.36 @@ -43,7 +43,7 @@ def test_parse_gnu_libc(self, ldd_output, expected_glibc_version): parsed_glibc_version = _parse_gnu_libc(ldd_output) assert expected_glibc_version == parsed_glibc_version - @parameterized.expand( + @pytest.mark.parametrize("ldd_output,expected_musl_libc_version", [ [ """musl libc (x86_64) diff --git a/test/unittests/util/detect_test.py b/test/unittests/util/detect_test.py index be71c00de2d..2e2d1ea9ed6 100644 --- a/test/unittests/util/detect_test.py +++ b/test/unittests/util/detect_test.py @@ -1,9 +1,9 @@ -import mock +from unittest import mock from unittest.mock import patch import pytest -from conan.internal.api.detect.detect_api import _cc_compiler, detect_suncc_compiler, \ - detect_intel_compiler +from conan.internal.api.detect.detect_api import detect_suncc_compiler, \ + detect_intel_compiler, detect_default_compiler from conan.internal.api.profile.detect import detect_defaults_settings from conan.internal.model.version import Version from conan.test.utils.mocks import RedirectedTestOutput @@ -27,7 +27,8 @@ def test_detect_aix(self, processor, bitness, version, expected_arch): with mock.patch("platform.machine", mock.MagicMock(return_value='XXXXXXXXXXXX')), \ mock.patch("platform.processor", mock.MagicMock(return_value=processor)), \ mock.patch("platform.system", mock.MagicMock(return_value='AIX')), \ - mock.patch("conan.internal.api.detect.detect_api._get_aix_conf", mock.MagicMock(return_value=bitness)), \ + mock.patch("conan.internal.api.detect.detect_api._get_aix_conf", + mock.MagicMock(return_value=bitness)), \ mock.patch('subprocess.check_output', mock.MagicMock(return_value=version)): result = detect_defaults_settings() result = dict(result) @@ -54,7 +55,7 @@ def test_detect_arch(self, machine, expected_arch): def test_detect_clang_gcc_toolchain(self, _): output = RedirectedTestOutput() with redirect_output(output): - with environment_update({"CC": "clang-9 --gcc-toolchain=/usr/lib/gcc/x86_64-linux-gnu/9"}): + with environment_update({"CC": "clang-9 --gcc-toolchain=/usr/lib/gcc/x86_64-linux-gnu"}): detect_defaults_settings() assert "CC and CXX: clang-9 --gcc-toolchain" in output @@ -67,9 +68,9 @@ def test_detect_clang_gcc_toolchain(self, _): ["gcc (crosstool-NG 1.27.0) 13.3.0", "13.3.0"], ["gcc (crosstool-NG 1.27.0) 13.3", "13.3"], ["gcc (GCC) 4.8.1", "4.8.1"], - ["clang version 18.1.0rc (https://github.com/llvm/llvm-project.git 461274b81d8641eab64d494accddc81d7db8a09e)", "18.1.0"], + ["clang version 18.1.0rc (https://github.com/llvm/llvm-project.git 4612...a09e)", "18.1.0"], ["cc.exe (Rev3, Built by MSYS2 project) 14.0", "14.0"], - ["clang version 18 (https://github.com/llvm/llvm-project.git 461274b81d8641eab64d494accddc81d7db8a09e)", "18"], + ["clang version 18 (https://github.com/llvm/llvm-project.git 4612...8a09e)", "18"], ["cc.exe (Rev3, Built by MSYS2 project) 14", "14"], ["cc.exe 14.2", "14.2"], ["cc1 14.2", "14.2"], @@ -77,16 +78,22 @@ def test_detect_clang_gcc_toolchain(self, _): @patch("conan.internal.api.detect.detect_api.detect_runner") def test_detect_cc_versioning(detect_runner_mock, version_return, expected_version): detect_runner_mock.return_value = 0, version_return - compiler, installed_version, compiler_exe = _cc_compiler() + with environment_update({"CC": "/usr/bin/cc"}): + compiler, installed_version, compiler_exe = detect_default_compiler() # Using private _value attribute to compare the str that reaches it, so no space issues arise assert installed_version._value == Version(expected_version)._value assert installed_version == Version(expected_version) + @pytest.mark.parametrize("function,version_return,expected_version", [ [detect_suncc_compiler, "Sun C 5.13", ('sun-cc', Version("5.13"), 'cc')], [detect_intel_compiler, "Intel C++ Compiler 2025.0", ('intel-cc', Version("2025.0"), 'icx')], ]) def test_detect_compiler(function, version_return, expected_version): - with mock.patch("conan.internal.api.detect.detect_api.detect_runner", mock.MagicMock(return_value=(0, version_return))): + """ + :type function: Callable + """ + with mock.patch("conan.internal.api.detect.detect_api.detect_runner", + mock.MagicMock(return_value=(0, version_return))): ret = function() assert ret == expected_version diff --git a/test/unittests/util/detected_architecture_test.py b/test/unittests/util/detected_architecture_test.py index f05693014b9..a3bb04f5ca1 100644 --- a/test/unittests/util/detected_architecture_test.py +++ b/test/unittests/util/detected_architecture_test.py @@ -1,4 +1,4 @@ -import mock +from unittest import mock import pytest from conan.internal.api.detect.detect_api import detect_arch diff --git a/test/unittests/util/files/strip_root_extract_test.py b/test/unittests/util/files/strip_root_extract_test.py index 77322bd6db7..33c375765c9 100644 --- a/test/unittests/util/files/strip_root_extract_test.py +++ b/test/unittests/util/files/strip_root_extract_test.py @@ -21,7 +21,8 @@ class TestZipExtractPlain: - def _zipdir(self, path, zipfilename, folder_entry=None): + @staticmethod + def _zipdir(path, zipfilename, folder_entry=None): with zipfile.ZipFile(zipfilename, 'w', zipfile.ZIP_DEFLATED) as z: if folder_entry: zif = zipfile.ZipInfo(folder_entry + "/") @@ -116,7 +117,8 @@ def test_invalid_flat_single_file(self): class TestTarExtractPlain: - def _compress_folder(self, folder, tgz_path, folder_entry=None): + @staticmethod + def _compress_folder(folder, tgz_path, folder_entry=None): # Create a tar.gz file with the files in the folder and an additional TarInfo entry # for the folder_entry (the gather files doesn't return empty dirs) with open(tgz_path, "wb") as tgz_handle: diff --git a/test/unittests/util/files/tar_extract_test.py b/test/unittests/util/files/tar_extract_test.py index 8951d031b0b..d5d83154248 100644 --- a/test/unittests/util/files/tar_extract_test.py +++ b/test/unittests/util/files/tar_extract_test.py @@ -41,10 +41,10 @@ def test_link_folder(self, setup_files): tmp_folder, tgz_file = setup_files # Once unpackaged, this is the content of the destination directory - def check_files(destination_dir): - d = sorted(os.listdir(destination_dir)) + def check_files(dest_dir): + d = sorted(os.listdir(dest_dir)) assert d == ["file1", "folder"] - d_folder = os.listdir(os.path.join(destination_dir, "folder")) + d_folder = os.listdir(os.path.join(dest_dir, "folder")) assert d_folder == ["file2"] working_dir = temp_folder() diff --git a/test/unittests/util/test_encrypt.py b/test/unittests/util/test_encrypt.py index f48293762ca..7348010392c 100644 --- a/test/unittests/util/test_encrypt.py +++ b/test/unittests/util/test_encrypt.py @@ -10,12 +10,9 @@ def test_encryp_basic(): message = 'simple data ascii string' data = encrypt.encode(message, key) - assert type(message) == type(data) - assert message != data - assert message != data + assert isinstance(data, str) decoded = encrypt.decode(data, key) - assert type(message) == type(data) assert message == decoded @@ -25,11 +22,9 @@ def test_encrypt_unicode(): message = message_enc.decode('utf-8') data = encrypt.encode(message, key) - assert type(message) == type(data) - assert message != data + assert isinstance(data, str) decoded = encrypt.decode(data, key) - assert type(message) == type(data) assert message == decoded @@ -38,11 +33,9 @@ def test_key_unicode(): message = 'the message' data = encrypt.encode(message, key) - assert type(message) == type(data) - assert message != data + assert isinstance(data, str) decoded = encrypt.decode(data, key) - assert type(message) == type(data) assert message == decoded From 67a40b22ccf892acb6af146187ae1212f6c3c83d Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 13 Jan 2026 20:23:10 +0100 Subject: [PATCH 026/110] review --- conan/api/conan_api.py | 2 + conan/api/subapi/cache.py | 29 +- conan/internal/api/uploader.py | 37 +- conan/internal/conan_app.py | 7 +- conan/internal/rest/remote_manager.py | 39 +- .../rest/rest_client_local_recipe_index.py | 4 +- conan/internal/util/files.py | 46 +-- .../command/cache/test_cache_save_restore.py | 2 +- .../extensions/test_compression_plugin.py | 349 +++++++----------- 9 files changed, 191 insertions(+), 324 deletions(-) diff --git a/conan/api/conan_api.py b/conan/api/conan_api.py index a3d9225f751..7132c193d5f 100644 --- a/conan/api/conan_api.py +++ b/conan/api/conan_api.py @@ -122,6 +122,8 @@ def __init__(self, conan_api): @property def compression_plugin(self): + # NOTE: Users cannot store a reference to this plugin, otherwise it will not + # be updated after a "conan config install/install-pkg" if self._compression_plugin is None: compression_plugin_path = HomePaths( self._conan_api.home_folder).compression_plugin_path diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index da86417e0e1..f92c5d9ab4f 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -1,6 +1,7 @@ import json import os import shutil +import tarfile import tempfile from conan.api.model import PackagesList @@ -12,7 +13,6 @@ DOWNLOAD_EXPORT_FOLDER) from conan.internal.cache.home_paths import HomePaths from conan.internal.cache.integrity_check import IntegrityChecker -from conan.internal.loader import load_python_file from conan.internal.paths import COMPRESSIONS from conan.internal.rest.download_cache import DownloadCache from conan.errors import ConanException @@ -29,7 +29,6 @@ class CacheAPI: def __init__(self, conan_api, api_helpers): self._conan_api = conan_api self._api_helpers = api_helpers - self._compression_plugin = None def export_path(self, ref: RecipeReference): cache = PkgCache(self._conan_api.cache_folder, self._api_helpers.global_conf) @@ -197,8 +196,10 @@ def save(self, package_list: PackagesList, tgz_path, no_source=False) -> None: pkglist_path = os.path.join(tempfile.gettempdir(), "pkglist.json") save(pkglist_path, serialized) tar_files["pkglist.json"] = pkglist_path + plugin = self._conan_api._api_helpers.compression_plugin # noqa + compress_plugin = getattr(plugin, "tar_compress", None) if plugin else None compress_files(tar_files, tgz_name, os.path.dirname(tgz_path), compresslevel, - recursive=True, compression_plugin=self.compression_plugin) + recursive=True, compress_plugin=compress_plugin) remove(pkglist_path) ConanOutput().success(f"Created cache save file: {tgz_path}") @@ -209,13 +210,21 @@ def restore(self, path) -> PackagesList: cache = PkgCache(self._conan_api.cache_folder, self._api_helpers.global_conf) cache_folder = cache.store # Note, this is not the home, but the actual package cache - with open(path, mode="rb") as file_handler: - tar_extract( - fileobj=file_handler, - destination_dir=cache_folder, - compression_plugin=self.compression_plugin, - conf=self._conan_api.config._helpers.global_conf, - ) + plugin = self._conan_api._api_helpers.compression_plugin # noqa + extract_plugin = getattr(plugin, "tar_extract", None) if plugin else None + extracted = False + if extract_plugin: + extracted = extract_plugin(path, cache_folder, scope="Restore") + # If the plugin returns false, fallback to Conan extraction + + if extracted is False: + with open(path, mode='rb') as file_handler: + the_tar = tarfile.open(fileobj=file_handler) + fileobj = the_tar.extractfile("pkglist.json") + pkglist = fileobj.read() + the_tar.extraction_filter = (lambda member, _: member) # fully_trusted (Py 3.14) + the_tar.extractall(path=cache_folder) + the_tar.close() # Retrieve the package list from the already extracted archive pkglist_path = os.path.join(cache_folder, "pkglist.json") diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 3c93efc1b93..6efd13c17df 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -112,6 +112,8 @@ def __init__(self, app: ConanApp, global_conf): compresslevel = get_compress_level(compressformat, global_conf) self._compressformat = compressformat self._compresslevel = compresslevel + plugin = app.conan_api._api_helpers.compression_plugin # noqa + self._compress_plugin = getattr(plugin, "tar_compress", None) if plugin else None def prepare(self, pkg_list, enabled_remotes): local_url = self._global_conf.get("core.scm:local_url", choices=["allow", "block"]) @@ -221,9 +223,9 @@ def _compressed_file(self, filename, files, download_folder, ref): file_name = filename + self._compressformat package_file = os.path.join(download_folder, file_name) - compression_plugin = self._app.conan_api.cache.compression_plugin compressed_path = compress_files(files, file_name, download_folder, - compresslevel=self._compresslevel, ref=ref, plugin=compression_plugin) + compresslevel=self._compresslevel, ref=ref, + compress_plugin=self._compress_plugin) assert compressed_path == package_file assert os.path.exists(package_file) return file_name @@ -316,26 +318,23 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): return t -def compress_files(files, name, dest_dir, conf=None, ref=None, recursive=False, - compression_plugin=None): - tgz_path = os.path.join(dest_dir, name) +def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, + compress_plugin=None): t1 = time.time() + tgz_path = os.path.join(dest_dir, name) + ConanOutput(scope=str(ref) if ref else None).info(f"Compressing {name}") - ConanOutput(scope=str(ref or "")).info(f"Compressing {name}") - - if ref: - ConanOutput(scope=str(ref) if ref else None).info(f"Compressing {name}") - - if compression_plugin: + if compress_plugin is not None: ConanOutput(scope=str(ref or "")).info(f"Compressing {name} using compression plugin") - compressed_extension = compression_plugin.tar_compress( - archive_path=abs_path_without_extension, + compressed = compress_plugin( + archive_path=tgz_path, files=files, recursive=recursive, - conf=conf, - ref=ref, + compresslevel=compresslevel, + scope=str(ref or "") ) - return tgz_path + if compressed is not False: + return tgz_path if name.endswith("zst"): with tarfile.open(tgz_path, "w:zst", level=compresslevel) as tar: # noqa Py314 only @@ -353,7 +352,6 @@ def compress_files(files, name, dest_dir, conf=None, ref=None, recursive=False, return tgz_path with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: - compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) for filename, abs_path in sorted(files.items()): # recursive is False by default in case it is a symlink to a folder @@ -363,11 +361,6 @@ def compress_files(files, name, dest_dir, conf=None, ref=None, recursive=False, ConanOutput().debug(f"{name} compressed in {time.time() - t1} time") return tgz_path -def _compress_files_with_plugin(files, name, dest_dir, conf, ref, recursive, compression_plugin): - t1 = time.time() - abs_path_without_extension = os.path.join(dest_dir, COMPRESSED_PLUGIN_TAR_NAME) - - def _total_size(cache_files): total_size = 0 diff --git a/conan/internal/conan_app.py b/conan/internal/conan_app.py index 8cb9fa2a4cb..77f458f07a2 100644 --- a/conan/internal/conan_app.py +++ b/conan/internal/conan_app.py @@ -54,7 +54,8 @@ def __init__(self, conan_api): requester = conan_api._api_helpers.requester # noqa auth_manager = ConanApiAuthManager(requester, cache_folder, localdb, global_conf) # Handle remote connections - self.remote_manager = RemoteManager(self.cache, auth_manager, cache_folder, self.conan_api) + compress = conan_api._api_helpers.compression_plugin # noqa + self.remote_manager = RemoteManager(self.cache, auth_manager, cache_folder, compress) global_editables = conan_api.local.editable_packages ws_editables = conan_api.workspace.packages() self.editable_packages = global_editables.update_copy(ws_editables) @@ -84,10 +85,10 @@ class LocalRecipesIndexApp: - loader (for the export phase of local-recipes-index) The others are internally use by other collaborators """ - def __init__(self, cache_folder, conan_api): + def __init__(self, cache_folder): self.global_conf = ConfDefinition() self.cache = PkgCache(cache_folder, self.global_conf) - self.remote_manager = RemoteManager(self.cache, auth_manager=None, home_folder=cache_folder, conan_api=conan_api) + self.remote_manager = RemoteManager(self.cache, auth_manager=None, home_folder=cache_folder) editable_packages = EditablePackages() self.proxy = ConanProxy(self, editable_packages) self.range_resolver = RangeResolver(self, self.global_conf, editable_packages) diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index 78e8b47c06e..ab5bc3fa40e 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -28,16 +28,17 @@ class RemoteManager: _ErrorMsg = namedtuple("ErrorMsg", ["message"]) - def __init__(self, cache, auth_manager, home_folder, conan_api): + def __init__(self, cache, auth_manager, home_folder, compression_plugin=None): self._cache = cache self._auth_manager = auth_manager self._signer = PkgSignaturesPlugin(cache, home_folder) self._home_folder = home_folder - self._conan_api = conan_api + self._extract_plugin = getattr(compression_plugin, "tar_extract", None) \ + if compression_plugin else None def _local_folder_remote(self, remote): if remote.remote_type == LOCAL_RECIPES_INDEX: - return RestApiClientLocalRecipesIndex(remote, self._home_folder, self._conan_api) + return RestApiClientLocalRecipesIndex(remote, self._home_folder) def check_credentials(self, remote, force_auth=False): self._call_remote(remote, "check_credentials", force_auth) @@ -93,7 +94,8 @@ def get_recipe(self, ref, remote, metadata=None): tgz_file = zipped_files.pop(export_file, None) if tgz_file: - uncompress_file(tgz_file, export_folder, scope=str(ref), conan_api=self._conan_api) + uncompress_file(tgz_file, export_folder, scope=str(ref), + extract_plugin=self._extract_plugin) mkdir(export_folder) for file_name, file_path in zipped_files.items(): # copy CONANFILE shutil.move(file_path, os.path.join(export_folder, file_name)) @@ -136,14 +138,10 @@ def get_recipe_sources(self, ref, layout, remote): return self._signer.verify(ref, download_folder, files=zipped_files) -<<<<<<< HEAD - tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME] - uncompress_file(tgz_file, export_sources_folder, scope=str(ref), conan_api=self._conan_api) -======= # Only 1 file is guaranteed tgz_file = next(iter(zipped_files.values())) - uncompress_file(tgz_file, export_sources_folder, scope=str(ref)) ->>>>>>> feature/builtin_compression + uncompress_file(tgz_file, export_sources_folder, scope=str(ref), + extract_plugin=self._extract_plugin) def get_package(self, pref, remote, metadata=None): output = ConanOutput(scope=str(pref.ref)) @@ -198,7 +196,8 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata): tgz_file = zipped_files.pop(package_file) package_folder = layout.package() - uncompress_file(tgz_file, package_folder, scope=str(pref.ref), conan_api=self._conan_api) + uncompress_file(tgz_file, package_folder, scope=str(pref.ref), + extract_plugin=self._extract_plugin) mkdir(package_folder) # Just in case it doesn't exist, because uncompress did nothing for file_name, file_path in zipped_files.items(): # copy CONANINFO and CONANMANIFEST shutil.move(file_path, os.path.join(package_folder, file_name)) @@ -349,14 +348,11 @@ def _call_remote(self, remote, method, *args, **kwargs): raise ConanException(exc, remote=remote) -<<<<<<< HEAD -def uncompress_file(src_path, dest_folder, scope="", conan_api=None): -======= -def uncompress_file(src_path, dest_folder, scope=None): - if sys.version_info.minor < 14 and src_path.endswith("zst"): +def uncompress_file(src_path, dest_folder, scope=None, extract_plugin=None): + if sys.version_info.minor < 14 and src_path.endswith("zst") and extract_plugin is None: raise ConanException(f"File {os.path.basename(src_path)} compressed with 'zst', " f"unsupported for Python<3.14 ") ->>>>>>> feature/builtin_compression + try: filesize = os.path.getsize(src_path) big_file = filesize > 10000000 # 10 MB @@ -364,10 +360,13 @@ def uncompress_file(src_path, dest_folder, scope=None): hs = human_size(filesize) ConanOutput(scope=scope).info(f"Decompressing {hs} {os.path.basename(src_path)}") - compression_plugin=conan_api.cache.compression_plugin if conan_api and conan_api.cache.compression_plugin else None - conf=conan_api.config._helpers.global_conf if conan_api else None + if extract_plugin is not None: + if extract_plugin(src_path, dest_folder, scope) is not False: + # If the plugin returns false, fallback to Conan extraction + return + with open(src_path, mode='rb') as file_handler: - tar_extract(fileobj=file_handler, destination_dir=dest_folder, compression_plugin=compression_plugin, conf=conf) + tar_extract(fileobj=file_handler, destination_dir=dest_folder) except Exception as e: error_msg = "Error while extracting downloaded file '%s' to %s\n%s\n"\ % (src_path, dest_folder, str(e)) diff --git a/conan/internal/rest/rest_client_local_recipe_index.py b/conan/internal/rest/rest_client_local_recipe_index.py index 509378fefa8..2f1817d6267 100644 --- a/conan/internal/rest/rest_client_local_recipe_index.py +++ b/conan/internal/rest/rest_client_local_recipe_index.py @@ -58,14 +58,14 @@ class RestApiClientLocalRecipesIndex: a local folder assuming the conan-center-index repo layout """ - def __init__(self, remote, home_folder, conan_api): + def __init__(self, remote, home_folder): self._remote = remote local_recipes_index_path = HomePaths(home_folder).local_recipes_index_path local_recipes_index_path = os.path.join(local_recipes_index_path, remote.name, ".conan") repo_folder = self._remote.url from conan.internal.conan_app import LocalRecipesIndexApp - self._app = LocalRecipesIndexApp(local_recipes_index_path, conan_api) + self._app = LocalRecipesIndexApp(local_recipes_index_path) self._hook_manager = HookManager(HomePaths(local_recipes_index_path).hooks_path) self._layout = _LocalRecipesIndexLayout(repo_folder) diff --git a/conan/internal/util/files.py b/conan/internal/util/files.py index 76596ca055c..0884ccf2fac 100644 --- a/conan/internal/util/files.py +++ b/conan/internal/util/files.py @@ -1,10 +1,4 @@ import errno -<<<<<<< HEAD -from pathlib import Path -import tempfile -import gzip -======= ->>>>>>> feature/builtin_compression import hashlib import os import platform @@ -16,13 +10,10 @@ from contextlib import contextmanager -from conan.api.output import ConanOutput from conan.errors import ConanException _DIRTY_FOLDER = ".dirty" -# Name (without extension) of the tar file to be created by the compression plugin -COMPRESSED_PLUGIN_TAR_NAME = "__conan_plugin_compressed_contents__" def set_dirty(folder): dirty_file = os.path.normpath(folder) + _DIRTY_FOLDER @@ -264,11 +255,7 @@ def mkdir(path): os.makedirs(path) -def tar_extract(fileobj, destination_dir, compression_plugin=None, conf=None): - if compression_plugin: - _tar_extract_with_plugin(fileobj, destination_dir, compression_plugin, conf) - return - +def tar_extract(fileobj, destination_dir): the_tar = tarfile.open(fileobj=fileobj) # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to # "could not change modification time", with time=0 @@ -276,39 +263,8 @@ def tar_extract(fileobj, destination_dir, compression_plugin=None, conf=None): the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break the_tar.extractall(path=destination_dir) the_tar.close() - if list(Path(destination_dir).glob(f"{COMPRESSED_PLUGIN_TAR_NAME}.*")): - raise ConanException(f"Error while extracting {os.path.basename(fileobj.name)}.\n" - "This file has been compressed using a `compression` plugin.\n" - "If your organization uses this plugin, ensure it is correctly installed on your environment.") -def _tar_extract_with_plugin(fileobj, destination_dir, compression_plugin, conf): - """First remove tar.gz wrapper and then call the plugin to extract""" - t1 = time.time() - the_tar = tarfile.open(fileobj=fileobj) - the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break - the_tar.extractall(path=destination_dir) - extracted_files = the_tar.getnames() - the_tar.close() - # Check if the tar was compressed with the compression plugin by checking the existence of - # our constant COMPRESSED_PLUGIN_TAR_NAME (without extension as extension is added by the plugin) - ConanOutput().success(f"{time.time() - t1}") - for path in extracted_files: - if os.path.basename(path).startswith(COMPRESSED_PLUGIN_TAR_NAME): - # Extract the actual contents from the plugin tar (ignore other files present). - ConanOutput().debug(f"Unwrapped in {time.time() - t1}") - t1 = time.time() - compression_plugin.tar_extract( - archive_path=os.path.join(destination_dir, path), - dest_dir=destination_dir, - conf=conf, - ) - # Remove extracted files from tar - for f in extracted_files: - remove(os.path.join(destination_dir, f)) - break - ConanOutput().debug(f"Extracted in {time.time() - t1}") - def merge_directories(src, dst): from conan.tools.files import copy copy(None, pattern="*", src=src, dst=dst) diff --git a/test/integration/command/cache/test_cache_save_restore.py b/test/integration/command/cache/test_cache_save_restore.py index 77c4774d3cd..edcc5d59cec 100644 --- a/test/integration/command/cache/test_cache_save_restore.py +++ b/test/integration/command/cache/test_cache_save_restore.py @@ -314,7 +314,7 @@ def test_cache_save_restore_compressions(compress): client.run("create . --name=pkg --version=1.0 -s os=Linux") client.run(f"cache save pkg/*:* --file=mysave.t{compress}") if compress in ("xz", "zst"): - assert f"WARN: risk: The '{compress}' compression is experimental." in client.out + assert f"WARN: experimental: The '{compress}' compression is experimental." in client.out cache_path = os.path.join(client.current_folder, f"mysave.t{compress}") assert os.path.exists(cache_path) diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py index 0090d03fd34..c7c1bd96d21 100644 --- a/test/integration/extensions/test_compression_plugin.py +++ b/test/integration/extensions/test_compression_plugin.py @@ -1,65 +1,67 @@ import os import textwrap -import tarfile -from conan.internal.util.files import COMPRESSED_PLUGIN_TAR_NAME, mkdir + +from conan.internal.util.files import load from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient def test_compression_plugin_not_valid(): """Test an error is raised if the compression plugin is not valid""" + c = TestClient(light=True) + c.save_home({"extensions/plugins/compression.py": ""}) + c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) + c.run("create .", assert_error=True) + assert "ERROR: The 'compression.py' plugin does not contain" in c.out + + +def test_compression_plugin_fallbacks(): + """If the plugin methods returns False, fallback to Conan behavior""" + + c = TestClient(default_server_user=True, light=True) + compression_plugin = textwrap.dedent("""\ + import os + from conan.api.output import ConanOutput + + def tar_compress(archive_path, files, recursive, scope=None, compresslevel=None, + *args, **kwargs): + name = os.path.basename(archive_path) + ConanOutput(scope=scope).info(f"Falling back to compress {name} with Conan!") + return False + + def tar_extract(archive_path, dest_dir, scope=None, *args, **kwargs): + name = os.path.basename(archive_path) + ConanOutput(scope=scope).info(f"Falling back to extract {name} with Conan!") + return False + """) + c.save_home({"extensions/plugins/compression.py": compression_plugin}) + c.save({"conanfile.py": GenConanfile("pkg", "1.0").with_exports("*.txt") + .with_package_file("some.lib", "lib"), + "myexportfile.txt": "something"}) - c = TestClient() - compression_plugin = textwrap.dedent( - """ - def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): - pass - """ - ) - - c.save( - { - os.path.join( - c.cache_folder, "extensions", "plugins", "compression.py" - ): compression_plugin, - "conanfile.py": GenConanfile("pkg", "1.0"), - } - ) - c.run("create .") - c.run("cache save 'pkg/*:*'", assert_error=True) - assert ( - "ERROR: The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions" - in c.out - ) - -def test_compression_plugin_returning_invalid_path(): - """Test an error is raised if the compression plugin does not return expected path""" - - c = TestClient() - compression_plugin = textwrap.dedent( - """ - def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): - return - def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): - pass - """ - ) - - c.save( - { - os.path.join( - c.cache_folder, "extensions", "plugins", "compression.py" - ): compression_plugin, - "conanfile.py": GenConanfile("pkg", "1.0"), - } - ) c.run("create .") - c.run("cache save 'pkg/*:*'", assert_error=True) - assert ( - "ERROR: The 'compression.py' did not return the compressed extension." - in c.out - ) + c.run("upload * -r=default -c") + assert "pkg/1.0: Falling back to compress conan_export.tgz with Conan!" in c.out + assert "Falling back to compress conan_package.tgz with Conan!" in c.out + c.run("remove * -c") + c.run("install --requires=pkg/1.0") + assert "pkg/1.0: Falling back to extract conan_export.tgz with Conan!" in c.out + assert "pkg/1.0: Falling back to extract conan_package.tgz with Conan!" in c.out + + # same for cache save/restore + c.run("cache save *:*") + assert "Falling back to compress conan_cache_save.tgz with Conan!" in c.out + + c.run("remove * -c") + c.run("cache restore conan_cache_save.tgz") + assert "Restore: Falling back to extract conan_cache_save.tgz with Conan!" in c.out + c.run("cache path pkg/1.0") + path = str(c.stdout).strip() + assert load(os.path.join(path, "myexportfile.txt")) == "something" + c.run(f"cache path pkg/1.0:da39a3ee5e6b4b0d3255bfef95601890afd80709") + path = str(c.stdout).strip() + assert load(os.path.join(path, "some.lib")) == "lib" def test_compression_plugin_correctly_load(): @@ -67,203 +69,108 @@ def test_compression_plugin_correctly_load(): - cache save/restore - remote upload/download """ - c = TestClient(default_server_user=True) + c = TestClient(default_server_user=True, light=True) - compression_plugin = textwrap.dedent( - """ - import os - import tarfile + compression_plugin = textwrap.dedent("""\ + import os, tarfile from conan.api.output import ConanOutput - # xz compression - def tar_compress(archive_path, files, recursive, conf=None, ref=None, *args, **kwargs): - extension = ".xz" - archive_path += extension + def tar_compress(archive_path, files, recursive, scope=None, compresslevel=None, + *args, **kwargs): name = os.path.basename(archive_path) - ConanOutput(scope=ref).info(f"Compressing {name} using compression plugin (xz)") - compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None - kwargs = {"preset": compresslevel} if compresslevel else {} - with tarfile.open(archive_path, f"w:xz", **kwargs) as tgz: + assert name.endswith("xz") + ConanOutput(scope=scope).info(f"Compressing {name} with my Plugin!") + with tarfile.open(archive_path, f"w:xz", preset=compresslevel, + format=tarfile.PAX_FORMAT) as tgz: for filename, abs_path in sorted(files.items()): - tgz.add(abs_path, filename, recursive=True) - return extension + tgz.add(abs_path, filename, recursive=recursive) - def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): - ConanOutput().info(f"Decompressing {os.path.basename(archive_path)} using compression plugin (xz)") + def tar_extract(archive_path, dest_dir, scope=None, *args, **kwargs): + name = os.path.basename(archive_path) + ConanOutput(scope=scope).info(f"Extracting {name} with my Plugin!") with open(archive_path, mode='rb') as file_handler: the_tar = tarfile.open(fileobj=file_handler) the_tar.extraction_filter = (lambda member, path: member) the_tar.extractall(path=dest_dir) the_tar.close() - """ - ) - - c.save( - { - os.path.join( - c.cache_folder, "extensions", "plugins", "compression.py" - ): compression_plugin, - "conanfile.py": GenConanfile("pkg", "1.0"), - } - ) - c.run("create .") - c.run("cache save 'pkg/*:*'") - assert f"Compressing {COMPRESSED_PLUGIN_TAR_NAME}.xz using compression plugin (xz)" in c.out - c.run("remove pkg/* -c") - c.run("cache restore conan_cache_save.tgz") - assert f"Decompressing {COMPRESSED_PLUGIN_TAR_NAME}.xz using compression plugin (xz)" in c.out - c.run("list pkg/1.0") - assert "Found 1 pkg/version recipes matching pkg/1.0 in local cache" in c.out + """) + + c.save_home({"extensions/plugins/compression.py": compression_plugin}) + c.save({"conanfile.py": GenConanfile("pkg", "1.0").with_exports("*.txt") + .with_package_file("some.lib", "lib"), + "myexportfile.txt": "something"}) - # Remove pre existing tgz to force a recompression - c.run("remove pkg/* -c") c.run("create .") - # Check the plugin is also used on remote interactions - c.run("upload *:* -r=default -c") - assert f"Compressing {COMPRESSED_PLUGIN_TAR_NAME}.xz using compression plugin (xz)" in c.out - assert "pkg/1.0: Uploading recipe" in c.out - c.run("remove pkg/* -c") - c.run("download 'pkg/*' -r=default") - assert f"Decompressing {COMPRESSED_PLUGIN_TAR_NAME}.xz using compression plugin (xz)" in c.out + c.run("upload * -r=default -c -cc core.upload:compression_format=xz") + assert "pkg/1.0: Compressing conan_export.txz with my Plugin!" in c.out + assert "Compressing conan_package.txz with my Plugin!" in c.out + c.run("remove * -c") + c.run("install --requires=pkg/1.0") + assert "pkg/1.0: Extracting conan_export.txz with my Plugin!" in c.out + assert "pkg/1.0: Extracting conan_package.txz with my Plugin!" in c.out -def test_compression_plugin_tar_not_compatible_with_builtin(): - """ - Test that built in tar_extract function fails when uncompressing a non compatible file (a file - which has been compressed using the compression plugin with a different algorithm than the built-in one). - """ - c = TestClient(default_server_user=True) + # same for cache save/restore + c.run("cache save *:* --file=save.txz") + assert "Compressing save.txz with my Plugin!" in c.out + + c.run("remove * -c") + c.run("cache restore save.txz") + assert "Restore: Extracting save.txz with my Plugin!" in c.out + + c.run("cache path pkg/1.0") + path = str(c.stdout).strip() + assert load(os.path.join(path, "myexportfile.txt")) == "something" + c.run(f"cache path pkg/1.0:da39a3ee5e6b4b0d3255bfef95601890afd80709") + path = str(c.stdout).strip() + assert load(os.path.join(path, "some.lib")) == "lib" - compression_plugin = textwrap.dedent( - """ - import os - import zipfile - from conan.api.output import ConanOutput - # zip compression - def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): - # compress files using zipfile library taking into account recursive - extension = ".zip" - archive_path += extension - name = os.path.basename(archive_path) - compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None - ConanOutput().info(f"Compressing {name} using compression plugin (zip)") - with zipfile.ZipFile(archive_path, 'w', zipfile.ZIP_DEFLATED, compresslevel=compresslevel) as zipf: - for filename, abs_path in sorted(files.items()): - if recursive: - arcname = os.path.relpath(abs_path, start=os.path.dirname(abs_path)) - zipf.write(abs_path, arcname) - else: - zipf.write(abs_path, filename) - return extension - - def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): - # extract tar using zipfile library - ConanOutput().info(f"Decompressing {os.path.basename(archive_path)} using compression plugin (zip)") - with zipfile.ZipFile(archive_path, 'r') as zip_ref: - zip_ref.extractall(dest_dir) - """ - ) - - c.save( - { - os.path.join( - c.cache_folder, "extensions", "plugins", "compression.py" - ): compression_plugin, - "conanfile.py": GenConanfile("pkg", "1.0"), - } - ) - c.run("create .") - c.run("cache save 'pkg/*:*'") - c.run("remove pkg/* -c") - os.unlink(os.path.join(c.cache_folder, "extensions", "plugins", "compression.py")) - c.run("cache restore conan_cache_save.tgz", assert_error=True) - assert ( - "Error while extracting conan_cache_save.tgz.\n" - "This file has been compressed using a `compression` plugin.\n" - "If your organization uses this plugin, ensure it is correctly installed on your environment." - ) in c.out - - -# https://github.com/conan-io/conan/issues/18259 def test_compress_in_subdirectory(): - c = TestClient(default_server_user=True) - compression_plugin = textwrap.dedent( - """ - import os - import tarfile + # https://github.com/conan-io/conan/issues/18259 + c = TestClient(light=True) + + compression_plugin = textwrap.dedent("""\ + import os, tarfile from conan.api.output import ConanOutput - def tar_compress(archive_path, files, recursive, *args, **kwargs): - # compress files using tarfile putting all content in a `conan/` subfolder - extension = ".tgz" - archive_path += extension + + def tar_compress(archive_path, files, recursive, scope=None, compresslevel=None, + *args, **kwargs): name = os.path.basename(archive_path) - ConanOutput().info(f"Compressing {os.path.basename(name)} in conan subfolder") - with open(archive_path, "wb") as tgz_handle: - tgz = tarfile.open(name, "w", fileobj=tgz_handle) + assert name.endswith("xz") + ConanOutput(scope=scope).info(f"Compressing {name} with my Plugin!") + with tarfile.open(archive_path, f"w:xz", preset=compresslevel, + format=tarfile.PAX_FORMAT) as tgz: for filename, abs_path in sorted(files.items()): tgz.add(abs_path, os.path.join("conan", filename), recursive=recursive) - tgz.close() - return extension - def tar_extract(archive_path, dest_dir, *args, **kwargs): - ConanOutput().info(f"Decompressing {archive_path} in conan subfolder") - with open(archive_path, mode="rb") as file_handler: + def tar_extract(archive_path, dest_dir, scope=None, *args, **kwargs): + name = os.path.basename(archive_path) + ConanOutput(scope=scope).info(f"Extracting {name} with my Plugin!") + with open(archive_path, mode='rb') as file_handler: the_tar = tarfile.open(fileobj=file_handler) the_tar.extraction_filter = (lambda member, path: member) + for member in the_tar.getmembers(): if member.name.startswith("conan/"): member.name = member.name[len("conan/"):] # Strip 'conan/' prefix - the_tar.extract(member, path=dest_dir) + the_tar.extract(member, path=dest_dir) the_tar.close() - """ - ) - c.save( - { - os.path.join( - c.cache_folder, "extensions", "plugins", "compression.py" - ): compression_plugin, - "conanfile.py": GenConanfile("pkg", "1.0"), - } - ) + """) + + c.save_home({"extensions/plugins/compression.py": compression_plugin}) + c.save({"conanfile.py": GenConanfile("pkg", "1.0").with_exports("*.txt"), + "myexportfile.txt": "something"}) + c.run("create .") - c.run("cache save 'pkg/*:*'") - tgz = os.path.join(c.current_folder, "conan_cache_save.tgz") - assert os.path.exists(tgz) - - mkdir("extract_folder") - destination_dir = os.path.join(c.current_folder, "extract_folder") - extracted_files = _tar_extract(tgz, destination_dir) - assert extracted_files == [COMPRESSED_PLUGIN_TAR_NAME + ".tgz"] - # Create example files - c.save({os.path.join(destination_dir, "README.md"): "This is a readme file.", - os.path.join(destination_dir, "Cache-contents-graph-report.html"): "Cache contents graph report", - os.path.join(destination_dir, "Cache-contents-graph-report.exe"): "executable file...", - os.path.join(destination_dir, "conan_cache_save.tgz"): "this should also be ignored", - }) - # Recompress the file with metadata - _tar_compress(os.path.join(c.current_folder, "conan_cache_save_rearchived.tgz"), destination_dir) - - c.run("remove pkg/* -c") - c.run("cache restore conan_cache_save_rearchived.tgz") - - # Check any of the metadata are present in the cache - assert any(item not in os.listdir(os.path.join(c.cache_folder, "p")) for item in ("README.md", - "Cache-contents-graph-report.html", - "Cache-contents-graph-report.exe")) - - -def _tar_extract(tgz_path, destination_dir): - with open(tgz_path, "rb") as fileobj: - the_tar = tarfile.open(fileobj=fileobj) - the_tar.extraction_filter = (lambda member, path: member) - the_tar.extractall(path=destination_dir) - return the_tar.getnames() - - -def _tar_compress(archive_path, folder): - with open(archive_path, "wb") as tgz_handle: - tgz = tarfile.open(os.path.basename(archive_path), "w", fileobj=tgz_handle) - for filename in os.listdir(folder): - tgz.add(os.path.join(folder, filename), filename, recursive=True) - tgz.close() + + c.run("cache save *:* --file=save.txz") + assert "Compressing save.txz with my Plugin!" in c.out + + c.run("remove * -c") + c.run("cache restore save.txz") + assert "Restore: Extracting save.txz with my Plugin!" in c.out + + c.run("cache path pkg/1.0") + path = str(c.stdout).strip() + assert load(os.path.join(path, "myexportfile.txt")) == "something" From 4cea425b2b156760f2d3b08550ffac64bc5026fb Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 14 Jan 2026 01:41:25 +0100 Subject: [PATCH 027/110] wip --- conan/api/subapi/cache.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index f92c5d9ab4f..365d5a804a8 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -220,8 +220,6 @@ def restore(self, path) -> PackagesList: if extracted is False: with open(path, mode='rb') as file_handler: the_tar = tarfile.open(fileobj=file_handler) - fileobj = the_tar.extractfile("pkglist.json") - pkglist = fileobj.read() the_tar.extraction_filter = (lambda member, _: member) # fully_trusted (Py 3.14) the_tar.extractall(path=cache_folder) the_tar.close() From a6106b3185319f06b218ba42168ee3b90aecf8a4 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 25 Feb 2026 12:38:19 +0100 Subject: [PATCH 028/110] fix CMakeConfigDeps casing in-package (#19669) --- .../cmake/cmakeconfigdeps/cmakeconfigdeps.py | 13 ++++----- .../cmakeconfigdeps/test_cmakeconfigdeps.py | 29 +++++++++++++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/conan/tools/cmake/cmakeconfigdeps/cmakeconfigdeps.py b/conan/tools/cmake/cmakeconfigdeps/cmakeconfigdeps.py index e900b364718..cc15cf8e98d 100644 --- a/conan/tools/cmake/cmakeconfigdeps/cmakeconfigdeps.py +++ b/conan/tools/cmake/cmakeconfigdeps/cmakeconfigdeps.py @@ -341,13 +341,12 @@ def generate(self): build_dir = dep.package_folder pkg_folder = build_dir.replace("\\", "/") if build_dir else None if pkg_folder: - f = self._cmakedeps.get_cmake_filename(dep) - for filename in (f"{f}-config.cmake", f"{f}Config.cmake"): - if os.path.isfile(os.path.join(pkg_folder, filename)): - relative_path = relativize_path(pkg_folder, self._conanfile, - "${CMAKE_CURRENT_LIST_DIR}") - for pkg_name in pkg_names: - pkg_paths[pkg_name] = relative_path + if any(os.path.isfile(os.path.join(pkg_folder, f + ext)) for f in pkg_names + for ext in ("-config.cmake", "Config.cmake")): + relative_path = relativize_path(pkg_folder, self._conanfile, + "${CMAKE_CURRENT_LIST_DIR}") + for pkg_name in pkg_names: + pkg_paths[pkg_name] = relative_path for pkg_name in pkg_names: existing_paths = pkg_paths_multi.setdefault(pkg_name, []) diff --git a/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps.py b/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps.py index 797dbb08c5f..ebb98ba3813 100644 --- a/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps.py +++ b/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps.py @@ -917,6 +917,35 @@ def package_info(self): assert paths_content.count("list(APPEND CONAN_HellO_DIR_MULTI") == 2 assert paths_content.count("list(APPEND CONAN_HELLO_DIR_MULTI") == 2 + def test_find_file_in_package(self): + tc = TestClient() + conanfile = textwrap.dedent(""" + import os + from conan import ConanFile + from conan.tools.files import save + + class HelloConan(ConanFile): + name = "hello" + version = "1.0" + settings = "build_type" + + def package(self): + save(self, os.path.join(self.package_folder, "HellOConfig.cmake"), "") + + def package_info(self): + self.cpp_info.builddirs = ["."] + self.cpp_info.set_property("cmake_find_mode", "none") + self.cpp_info.set_property("cmake_file_name_variants", ["HellO", "HELLO"]) + """) + tc.save({"conanfile.py": conanfile}) + tc.run("create") + tc.run("create -s=build_type=Debug") + tc.run("install --requires=hello/1.0 -g CMakeConfigDeps") + paths_content = tc.load("conan_cmakedeps_paths.cmake") + assert "set(hello_DIR" in paths_content + assert "set(HellO_DIR" in paths_content + assert "set(HELLO_DIR" in paths_content + def test_requires_only_component_target_generation(): tc = TestClient() From 2a77a33ee9e60607f8f62ed0c4c1f94d85e684b9 Mon Sep 17 00:00:00 2001 From: Carlos Zoido Date: Wed, 25 Feb 2026 13:40:50 +0100 Subject: [PATCH 029/110] Use lazy imports to avoid circular dependency so PyInstaller bundles `conan.tools.system` (#19670) * use lazy import * wip --- conan/internal/api/install/generators.py | 2 +- conan/tools/env/environment.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conan/internal/api/install/generators.py b/conan/internal/api/install/generators.py index 8ce37361257..6241789a536 100644 --- a/conan/internal/api/install/generators.py +++ b/conan/internal/api/install/generators.py @@ -7,7 +7,6 @@ from conan.internal.cache.home_paths import HomePaths from conan.internal.errors import conanfile_exception_formatter from conan.internal.util.files import mkdir, chdir -from conan.tools.env.environment import generate_aggregated_env _generators = {"CMakeToolchain": "conan.tools.cmake", @@ -142,6 +141,7 @@ def write_generators(conanfile, hook_manager, home_folder, envs_generation=None) env = VirtualRunEnv(conanfile) env.generate() + from conan.tools.env.environment import generate_aggregated_env generate_aggregated_env(conanfile) hook_manager.execute("post_generate", conanfile=conanfile) diff --git a/conan/tools/env/environment.py b/conan/tools/env/environment.py index 1b207a83ee7..41ed9c8f7ac 100644 --- a/conan/tools/env/environment.py +++ b/conan/tools/env/environment.py @@ -9,7 +9,6 @@ from conan.errors import ConanException from conan.internal.model.recipe_ref import ref_matches from conan.internal.util.files import save -from conan.tools.microsoft.visual import CONAN_VCVARS class _EnvVarPlaceHolder: @@ -910,6 +909,7 @@ def bat_content(files): content = ["@echo off"] if deactivation_mode == "function": + from conan.tools.microsoft.visual import CONAN_VCVARS deactivates_var = f"_CONAN_{group}_DEACTIVATES_DIR" content += [ f'set "{deactivates_var}=%TEMP%\\conan_{group}_%RANDOM%"', From 90bd6c03503dca0597da990eea93307b10590914 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20S=C3=A1nchez=20Falero?= Date: Wed, 25 Feb 2026 17:29:01 +0100 Subject: [PATCH 030/110] Expose PyEnv root and python executable (#19628) * python_hints added * wip * wip * wip * wip * wip * wip * wip * wip * wip * Apply suggestion from @czoido Co-authored-by: Carlos Zoido * wip --------- Co-authored-by: Carlos Zoido --- conan/tools/system/python_manager.py | 42 ++++++++++-------- .../tools/system/python_manager_test.py | 43 ++++++++++++++++++- 2 files changed, 67 insertions(+), 18 deletions(-) diff --git a/conan/tools/system/python_manager.py b/conan/tools/system/python_manager.py index 2856c3b95ee..62b6ac93da8 100644 --- a/conan/tools/system/python_manager.py +++ b/conan/tools/system/python_manager.py @@ -16,9 +16,9 @@ class PyEnv: def __init__(self, conanfile, folder=None, name="", py_version=None): """ - :param conanfile: The current conanfile "self" - :param folder: Optional folder, by default the "build_folder" - :param name: Optional name for the virtualenv, by default "conan_pyenv" + :param conanfile: The current conanfile ``self`` + :param folder: Optional folder, by default the ``build_folder`` + :param name: Optional name for the virtualenv, by default ``conan_pyenv`` :param py_version: Optional python version to create the virtualenv using UV """ if sys.version_info.minor < 8 and py_version: @@ -43,15 +43,11 @@ def __init__(self, conanfile, folder=None, name="", py_version=None): "Python system-wide or set the " "'tools.system.pyenv:python_interpreter' " "conf to the full path of a Python executable") - self.env_name = f"conan_pyenv{f'_{name}' if name else ''}" + self._env_name = f"conan_pyenv{f'_{name}' if name else ''}" if py_version: - self.env_name += f'_{py_version.replace(".", "_")}' + self._env_name += f'_{py_version.replace(".", "_")}' base_env_dir = os.path.abspath(folder or conanfile.build_folder) - self._env_dir = os.path.join(base_env_dir, self.env_name) - bins = "Scripts" if platform.system() == "Windows" else "bin" - self.bin_dir = os.path.join(self._env_dir, bins) - pyexe = "python.exe" if platform.system() == "Windows" else "python" - self._python_exe = os.path.join(self.bin_dir, pyexe) + self._env_dir = os.path.join(base_env_dir, self._env_name) if not os.path.exists(self._env_dir): if py_version: self._create_uv_venv(base_env_dir, py_version) @@ -59,8 +55,20 @@ def __init__(self, conanfile, folder=None, name="", py_version=None): self._create_venv() @property - def python(self): - return self._get_env_python(self._env_dir) + def env_dir(self): + """Root directory of the virtual environment.""" + return self._env_dir.replace("\\", "/") + + @property + def env_exe(self): + """Path to the Python executable inside the virtual environment.""" + return self._get_env_python(self._env_dir).replace("\\", "/") + + @property + def bin_path(self): + """Path to the bin or Scripts directory inside the virtual environment.""" + bins = "Scripts" if platform.system() == "Windows" else "bin" + return os.path.join(self._env_dir, bins).replace("\\", "/") @staticmethod def _get_env_python(env_dir): @@ -72,11 +80,11 @@ def generate(self): Create a conan environment to use the python venv in the next steps of the conanfile. """ env = Environment() - env.prepend_path("PATH", self.bin_dir) - env.vars(self._conanfile).save_script(self.env_name) + env.prepend_path("PATH", self.bin_path) + env.vars(self._conanfile).save_script(self._env_name) def run(self, args): - return self._conanfile.run(cmd_args_to_string([self.python] + list(args))) + return self._conanfile.run(cmd_args_to_string([self.env_exe] + list(args))) def install(self, packages, pip_args=None): """ @@ -88,7 +96,7 @@ def install(self, packages, pip_args=None): Defaults to ``None``. :return: the return code of the executed pip command. """ - args = [self.python, "-m", "pip", "install", "--disable-pip-version-check"] + args = [self.env_exe, "-m", "pip", "install", "--disable-pip-version-check"] if pip_args: args.extend(pip_args) args += [f'"{p}"' for p in packages] @@ -110,7 +118,7 @@ def _create_uv_venv(self, base_env_dir, py_version): if uv_path: uv_cmd = [uv_path] else: - uv_env_dir = os.path.join(base_env_dir, f"uv_{self.env_name}") + uv_env_dir = os.path.join(base_env_dir, f"uv_{self._env_name}") self._conanfile.run(cmd_args_to_string( [self._default_python, '-m', 'venv', uv_env_dir]) ) diff --git a/test/functional/tools/system/python_manager_test.py b/test/functional/tools/system/python_manager_test.py index ecb10407aeb..030ef5cb0be 100644 --- a/test/functional/tools/system/python_manager_test.py +++ b/test/functional/tools/system/python_manager_test.py @@ -1,3 +1,4 @@ +import json import sys import textwrap import platform @@ -48,6 +49,9 @@ def build(self): assert "colorama" not in c.out assert "Jinja2" not in c.out assert "PyJWT" not in c.out + ext = ".bat" if platform.system() == "Windows" else ".sh" + script = c.load(f"conan_pyenv{ext}") + assert "conan_pyenv" in script def test_build_py_manager(): @@ -141,7 +145,7 @@ def finalize(self): PyEnv(self, self.package_folder).install(["{pip_package_folder}"]) def package_info(self): - python_env_bin = PyEnv(self, self.package_folder).bin_dir + python_env_bin = PyEnv(self, self.package_folder).bin_path self.buildenv_info.prepend_path("PATH", python_env_bin) """) @@ -337,3 +341,40 @@ def build(self): assert "WARN: deprecated: 'PipEnv()' is deprecated, use 'PyEnv()'" in client.out assert "RUN: hello-world" in client.out assert "Hello Test World!" in client.out + + +def test_cmake_toolchain_configure_find_python(): + client = TestClient(path_with_spaces=False) + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.cmake import CMakeToolchain + from conan.tools.system import PyEnv + + class Pkg(ConanFile): + settings = "os", "arch", "compiler", "build_type" + + def generate(self): + pyenv = PyEnv(self) + pyenv.generate() + tc = CMakeToolchain(self) + tc.cache_variables["Python_ROOT_DIR"] = pyenv.env_dir + tc.cache_variables["Python_EXECUTABLE"] = pyenv.env_exe + tc.cache_variables["Python_FIND_UNVERSIONED_NAMES"] = "FIRST" + tc.cache_variables["Python_FIND_STRATEGY"] = "LOCATION" + tc.cache_variables["Python_FIND_VIRTUALENV"] = "STANDARD" + tc.cache_variables["Python_FIND_REGISTRY"] = "NEVER" + tc.generate() + """) + client.save({"conanfile.py": conanfile}) + client.run("install .") + presets = json.loads(client.load("CMakePresets.json")) + cv = presets["configurePresets"][0]["cacheVariables"] + assert "Python_ROOT_DIR" in cv + assert "Python_EXECUTABLE" in cv + assert "\\" not in cv["Python_ROOT_DIR"] + assert "\\" not in cv["Python_EXECUTABLE"] + assert "conan_pyenv" in cv["Python_ROOT_DIR"] + assert cv["Python_FIND_UNVERSIONED_NAMES"] == "FIRST" + assert cv["Python_FIND_STRATEGY"] == "LOCATION" + assert cv["Python_FIND_VIRTUALENV"] == "STANDARD" + assert cv["Python_FIND_REGISTRY"] == "NEVER" From 0a713f9860bda09adaa9b10438dc050b987ee22a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= <5364255+AbrilRBS@users.noreply.github.com> Date: Fri, 27 Feb 2026 10:08:18 +0100 Subject: [PATCH 031/110] Avoid missing binaries due to default platform requires revision (#19680) * Avoid missing binaries due to default platform requires revision --------- Co-authored-by: James * Different files! --------- Co-authored-by: James --- conan/internal/graph/graph_builder.py | 1 + conan/internal/model/info.py | 4 +- .../graph/test_platform_requires.py | 82 +++++++++++++------ test/integration/graph/test_system_tools.py | 33 ++++---- 4 files changed, 78 insertions(+), 42 deletions(-) diff --git a/conan/internal/graph/graph_builder.py b/conan/internal/graph/graph_builder.py index b7949d062dd..c3638a8bd11 100644 --- a/conan/internal/graph/graph_builder.py +++ b/conan/internal/graph/graph_builder.py @@ -299,6 +299,7 @@ def _resolved_system(node, require, profile_build, profile_host, resolve_prerele if version_range: if version_range.contains(d.version, resolve_prereleases): require.ref.version = d.version # resolved range is replaced by exact + require.ref.revision = d.revision or "platform" layout = BasicLayout(require.ref, None) return layout, ConanFile(str(d)), RECIPE_PLATFORM, None elif require.ref.version == d.version: diff --git a/conan/internal/model/info.py b/conan/internal/model/info.py index b7d30660d47..12260cd7df2 100644 --- a/conan/internal/model/info.py +++ b/conan/internal/model/info.py @@ -155,7 +155,7 @@ def revision_mode(self): self.user = self._ref.user self.channel = self._ref.channel self.package_id = None - self.recipe_revision = self._ref.revision + self.recipe_revision = self._ref.revision if self._ref.revision != "platform" else None def full_mode(self): self.name = self._ref.name @@ -163,7 +163,7 @@ def full_mode(self): self.user = self._ref.user self.channel = self._ref.channel self.package_id = self._package_id - self.recipe_revision = self._ref.revision + self.recipe_revision = self._ref.revision if self._ref.revision != "platform" else None full_recipe_mode = full_version_mode recipe_revision_mode = full_mode # to not break everything and help in upgrade diff --git a/test/integration/graph/test_platform_requires.py b/test/integration/graph/test_platform_requires.py index 78db3dd4561..81a08592c6c 100644 --- a/test/integration/graph/test_platform_requires.py +++ b/test/integration/graph/test_platform_requires.py @@ -9,12 +9,13 @@ class TestPlatformRequires: - def test_platform_requires(self): + @pytest.mark.parametrize("revision", ["", "#myrev"]) + def test_platform_requires(self, revision): client = TestClient(light=True) client.save({"conanfile.py": GenConanfile("pkg", "1.0").with_requires("dep/1.0"), - "profile": "[platform_requires]\ndep/1.0"}) + "profile": f"[platform_requires]\ndep/1.0{revision}"}) client.run("create . -pr=profile") - assert "dep/1.0 - Platform" in client.out + assert f"dep/1.0{revision or '#platform'} - Platform" in client.out def test_platform_requires_non_matching(self): """ if what is specified in [platform_requires] doesn't match what the recipe requires, then @@ -28,12 +29,13 @@ def test_platform_requires_non_matching(self): client.run("create . -pr=profile") assert "dep/1.0#6a99f55e933fb6feeb96df134c33af44 - Cache" in client.out - def test_platform_requires_range(self): + @pytest.mark.parametrize("revision", ["", "#myrev"]) + def test_platform_requires_range(self, revision): client = TestClient(light=True) client.save({"conanfile.py": GenConanfile("pkg", "1.0").with_requires("dep/[>=1.0]"), - "profile": "[platform_requires]\ndep/1.1"}) + "profile": f"[platform_requires]\ndep/1.1{revision}"}) client.run("create . -pr=profile") - assert "dep/1.1 - Platform" in client.out + assert f"dep/1.1{revision or '#platform'} - Platform" in client.out def test_platform_requires_range_non_matching(self): """ if what is specified in [platform_requires] doesn't match what the recipe requires, then @@ -78,7 +80,7 @@ def test_graph_info_platform_requires_range(self): client.save({"conanfile.py": GenConanfile("pkg", "1.0").with_requires("dep/[>=1.0]"), "profile": "[platform_requires]\ndep/1.1"}) client.run("graph info . -pr=profile") - assert "dep/1.1 - Platform" in client.out + assert "dep/1.1#platform - Platform" in client.out def test_consumer_resolved_version(self): client = TestClient(light=True) @@ -94,10 +96,11 @@ def generate(self): client.save({"conanfile.py": conanfile, "profile": "[platform_requires]\ndep/1.1"}) client.run("install . -pr=profile") - assert "dep/1.1 - Platform" in client.out + assert "dep/1.1#platform - Platform" in client.out assert "conanfile.py: DEPENDENCY dep/1.1" in client.out - def test_consumer_resolved_revision(self): + @pytest.mark.parametrize("revision", ["", "#rev1"]) + def test_consumer_resolved_revision(self, revision): client = TestClient(light=True) conanfile = textwrap.dedent(""" from conan import ConanFile @@ -109,10 +112,10 @@ def generate(self): self.output.info(f"DEPENDENCY {repr(r.ref)}") """) client.save({"conanfile.py": conanfile, - "profile": "[platform_requires]\ndep/1.1#rev1"}) + "profile": f"[platform_requires]\ndep/1.1{revision}"}) client.run("install . -pr=profile") assert "dep/1.1 - Platform" in client.out - assert "conanfile.py: DEPENDENCY dep/1.1#rev1" in client.out + assert f"conanfile.py: DEPENDENCY dep/1.1{revision or '#platform'}" in client.out conanfile = textwrap.dedent(""" from conan import ConanFile @@ -126,7 +129,7 @@ def generate(self): client.save({"conanfile.py": conanfile}) client.run("install . -pr=profile") assert "dep/1.1 - Platform" in client.out - assert "conanfile.py: DEPENDENCY dep/1.1#rev1" in client.out + assert f"conanfile.py: DEPENDENCY dep/1.1{revision or '#platform'}" in client.out def test_consumer_unresolved_revision(self): """ if a recipe specifies an exact revision and so does the profile @@ -175,24 +178,25 @@ def test_platform_requires(self): class TestPlatformRequiresLock: - def test_platform_requires_range(self): + @pytest.mark.parametrize("revision", ["", "#rev1"]) + def test_platform_requires_range(self, revision): c = TestClient(light=True) c.save({"conanfile.py": GenConanfile("pkg", "1.0").with_requires("dep/[>=1.0]"), - "profile": "[platform_requires]\ndep/1.1"}) + "profile": f"[platform_requires]\ndep/1.1{revision}"}) c.run("lock create . -pr=profile") assert "dep/1.1 - Platform" in c.out lock = json.loads(c.load("conan.lock")) - assert lock["requires"] == ["dep/1.1"] + assert lock["requires"] == [f"dep/1.1{revision or '#platform'}"] c.run("install .", assert_error=True) assert "Package 'dep/1.1' not resolved: No remote defined" in c.out c.run("install . -pr=profile") - assert "dep/1.1 - Platform" in c.out + assert f"dep/1.1{revision or '#platform'} - Platform" in c.out # if the profile points to another version it is an error, not in the lockfile - c.save({"profile": "[platform_requires]\ndep/1.2"}) + c.save({"profile": f"[platform_requires]\ndep/1.2{revision}"}) c.run("install . -pr=profile", assert_error=True) - assert "ERROR: Requirement 'dep/1.2' not in lockfile" in c.out + assert f"ERROR: Requirement 'dep/1.2{revision or '#platform'}' not in lockfile" in c.out @pytest.mark.parametrize("platform_rev", [None, "myrev"]) @pytest.mark.parametrize("is_tool_platform", [True, False]) @@ -276,15 +280,14 @@ class TestPackageID: @pytest.mark.parametrize("package_id_mode", ["recipe_revision_mode", "full_package_mode"]) def test_package_id_modes(self, package_id_mode): """ this test validates that the computation of the downstream consumers package_id - doesn't break even if it depends on fields not existing in upstream platform_requires, like revision - or package_id + doesn't break even if it depends on fields not existing in upstream platform_requires, like package_id """ client = TestClient(light=True) client.save_home({"global.conf": f"core.package_id:default_unknown_mode={package_id_mode}"}) client.save({"conanfile.py": GenConanfile("pkg", "1.0").with_requires("dep/1.0"), "profile": "[platform_requires]\ndep/1.0"}) client.run("create . -pr=profile") - assert "dep/1.0 - Platform" in client.out + assert "dep/1.0#platform - Platform" in client.out def test_package_id_explicit_revision(self): """ @@ -307,21 +310,50 @@ def test_package_id_explicit_revision(self): assert "pkg/1.0#7ed9bbd2a7c3c4381438c163c93a9f21:" \ "2f6bc9cf5015a7210181592d454f36687791a941 - Build" in client.out - def test_package_id_full_mode(self): + @pytest.mark.parametrize("revision", ["", "#myrev1"]) + def test_package_id_full_package_mode(self, revision): """ platform_requires do not have settings or package_id, so it is ignored """ client = TestClient() client.save_home({"global.conf": "core.package_id:default_unknown_mode=full_package_mode"}) client.save({"conanfile.py": GenConanfile("pkg", "1.0").with_requires("dep/1.0"), - "profile": "[platform_requires]\ndep/1.0"}) + "profile": f"[platform_requires]\ndep/1.0{revision}"}) client.run("create . -pr=profile -s os=Linux") - assert "dep/1.0 - Platform" in client.out + assert f"dep/1.0{revision or '#platform'} - Platform" in client.out assert "pkg/1.0#7ed9bbd2a7c3c4381438c163c93a9f21:" \ "f2cfe57716d0a3320019f058edcd728d3379ab32 - Build" in client.out + # The default revision is not taken into account at all + assert f"pkg/1.0: requires: dep/1.0:da39a3ee5e6b4b0d3255bfef95601890afd80709" in client.out client.run("create . -pr=profile -s os=Windows") # pkg gets exactly same package_id, changing the settings, do not affect plaform package-id - assert "dep/1.0 - Platform" in client.out + assert f"dep/1.0{revision or '#platform'} - Platform" in client.out assert "pkg/1.0#7ed9bbd2a7c3c4381438c163c93a9f21:" \ "f2cfe57716d0a3320019f058edcd728d3379ab32 - Build" in client.out + + @pytest.mark.parametrize("revision, package_id", [ + ("", "f2cfe57716d0a3320019f058edcd728d3379ab32"), + ("#myrev1", "bc8dc9f0bfb1e1d217f2ba7cccf545cdfdb382f6") + ]) + def test_package_id_full_mode(self, revision, package_id): + """ + platform_requires do not have settings or package_id, so it is ignored + """ + client = TestClient() + client.save_home({"global.conf": "core.package_id:default_unknown_mode=full_mode"}) + client.save({"conanfile.py": GenConanfile("pkg", "1.0").with_requires("dep/1.0"), + "profile": f"[platform_requires]\ndep/1.0{revision}"}) + client.run("create . -pr=profile -s os=Linux") + assert f"dep/1.0{revision or '#platform'} - Platform" in client.out + assert "pkg/1.0#7ed9bbd2a7c3c4381438c163c93a9f21:" \ + f"{package_id} - Build" in client.out + # The default revision is not taken into account for package id calculation, + # but if the user defines it, it is + assert f"pkg/1.0: requires: dep/1.0{revision or ''}:da39a3ee5e6b4b0d3255bfef95601890afd80709" in client.out + + client.run("create . -pr=profile -s os=Windows") + # pkg gets exactly same package_id, changing the settings, do not affect plaform package-id + assert f"dep/1.0{revision or '#platform'} - Platform" in client.out + assert "pkg/1.0#7ed9bbd2a7c3c4381438c163c93a9f21:" \ + f"{package_id} - Build" in client.out diff --git a/test/integration/graph/test_system_tools.py b/test/integration/graph/test_system_tools.py index c66494d3654..f24baf89070 100644 --- a/test/integration/graph/test_system_tools.py +++ b/test/integration/graph/test_system_tools.py @@ -9,12 +9,13 @@ class TestToolRequires: - def test_system_tool_require(self): + @pytest.mark.parametrize("revision", ["", "#myrev"]) + def test_system_tool_require(self, revision): client = TestClient(light=True) client.save({"conanfile.py": GenConanfile("pkg", "1.0").with_tool_requires("tool/1.0"), - "profile": "[platform_tool_requires]\ntool/1.0"}) + "profile": f"[platform_tool_requires]\ntool/1.0{revision}"}) client.run("create . -pr=profile") - assert "tool/1.0 - Platform" in client.out + assert f"tool/1.0{revision or '#platform'} - Platform" in client.out def test_system_tool_require_non_matching(self): """ if what is specified in [system_tool_require] doesn't match what the recipe requires, then @@ -29,12 +30,13 @@ def test_system_tool_require_non_matching(self): assert "WARN: Profile [system_tools] is deprecated" in client.out assert "tool/1.0#60ed6e65eae112df86da7f6d790887fd - Cache" in client.out - def test_system_tool_require_range(self): + @pytest.mark.parametrize("revision", ["", "#myrev"]) + def test_system_tool_require_range(self, revision): client = TestClient(light=True) client.save({"conanfile.py": GenConanfile("pkg", "1.0").with_tool_requires("tool/[>=1.0]"), - "profile": "[platform_tool_requires]\ntool/1.1"}) + "profile": f"[platform_tool_requires]\ntool/1.1{revision}"}) client.run("create . -pr=profile") - assert "tool/1.1 - Platform" in client.out + assert f"tool/1.1{revision or '#platform'} - Platform" in client.out def test_system_tool_require_range_non_matching(self): """ if what is specified in [system_tool_require] doesn't match what the recipe requires, then @@ -82,7 +84,7 @@ def generate(self): client.save({"conanfile.py": conanfile, "profile": "[platform_tool_requires]\ntool/1.1"}) client.run("install . -pr=profile") - assert "tool/1.1 - Platform" in client.out + assert "tool/1.1#platform - Platform" in client.out assert "conanfile.py: DEPENDENCY tool/1.1" in client.out def test_consumer_resolved_revision(self): @@ -113,7 +115,7 @@ def generate(self): """) client.save({"conanfile.py": conanfile}) client.run("install . -pr=profile") - assert "tool/1.1 - Platform" in client.out + assert "tool/1.1#rev1 - Platform" in client.out assert "conanfile.py: DEPENDENCY tool/1.1#rev1" in client.out def test_consumer_unresolved_revision(self): @@ -158,32 +160,33 @@ def test_require_build_context(self): class TestToolRequiresLock: - def test_system_tool_require_range(self): + @pytest.mark.parametrize("revision", ["", "#myrev"]) + def test_system_tool_require_range(self, revision): c = TestClient(light=True) c.save({"conanfile.py": GenConanfile("pkg", "1.0").with_tool_requires("tool/[>=1.0]"), - "profile": "[platform_tool_requires]\ntool/1.1"}) + "profile": f"[platform_tool_requires]\ntool/1.1{revision}"}) c.run("lock create . -pr=profile") assert "tool/1.1 - Platform" in c.out lock = json.loads(c.load("conan.lock")) - assert lock["build_requires"] == ["tool/1.1"] + assert lock["build_requires"] == [f"tool/1.1{revision or '#platform'}"] c.run("install .", assert_error=True) assert "Package 'tool/1.1' not resolved: No remote defined" in c.out c.run("install . -pr=profile") - assert "tool/1.1 - Platform" in c.out + assert f"tool/1.1{revision or '#platform'} - Platform" in c.out # even if we create a version within the range, it will error if not matching the profile c.save({"tool/conanfile.py": GenConanfile("tool", "1.1")}) c.run("create tool") # if the profile points to another version it is an error, not in the lockfile - c.save({"profile": "[platform_tool_requires]\ntool/1.2"}) + c.save({"profile": f"[platform_tool_requires]\ntool/1.2{revision}"}) c.run("install . -pr=profile", assert_error=True) - assert "ERROR: Requirement 'tool/1.2' not in lockfile" in c.out + assert f"ERROR: Requirement 'tool/1.2{revision or '#platform'}' not in lockfile" in c.out # if we relax the lockfile, we can still resolve to the platform_tool_requires # specified by the profile c.run("install . -pr=profile --lockfile-partial") - assert "tool/1.2 - Platform" in c.out + assert f"tool/1.2{revision or '#platform'} - Platform" in c.out class TestGenerators: From 5ac1cc4dfecf4422c0d1bdb4a5c4cc71542352a9 Mon Sep 17 00:00:00 2001 From: Noah Massey Date: Mon, 2 Mar 2026 06:30:17 -0500 Subject: [PATCH 032/110] Correct encoding of make_program path for AutoTools + msys2 (#15047) * Add autotools win_bash test for configured make_program * AutoTools.make(): encode path to make_program appropriately Autotools expects to be run in a unix-like environment. So the correct encoding for paths to execute is unix-like. * simplified test * Convert make_program path appropriately for selected subsystem --------- Co-authored-by: memsharded --- conan/tools/gnu/autotools.py | 2 + .../toolchains/gnu/autotools/test_win_bash.py | 59 ++++++++++++++++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/conan/tools/gnu/autotools.py b/conan/tools/gnu/autotools.py index bf3cf257f6e..a8aa7757951 100644 --- a/conan/tools/gnu/autotools.py +++ b/conan/tools/gnu/autotools.py @@ -71,6 +71,8 @@ def make(self, target=None, args=None, makefile=None): make_program = self._conanfile.conf.get("tools.gnu:make_program", default="mingw32-make" if self._use_win_mingw() else "make") + subsystem = deduce_subsystem(self._conanfile, scope="build") + make_program = subsystem_path(subsystem, make_program) str_args = self._make_args str_extra_args = " ".join(args) if args is not None else "" jobs = "" diff --git a/test/functional/toolchains/gnu/autotools/test_win_bash.py b/test/functional/toolchains/gnu/autotools/test_win_bash.py index fff3b4ed1e0..9d1a9ac8c15 100644 --- a/test/functional/toolchains/gnu/autotools/test_win_bash.py +++ b/test/functional/toolchains/gnu/autotools/test_win_bash.py @@ -4,7 +4,7 @@ import pytest -from conan.test.assets.autotools import gen_makefile_am, gen_configure_ac +from conan.test.assets.autotools import gen_makefile_am, gen_configure_ac, gen_makefile from conan.test.assets.genconanfile import GenConanfile from conan.test.assets.sources import gen_function_cpp from test.conftest import tools_locations @@ -338,3 +338,60 @@ def build(self): bat_contents = client.load("conanbuild.bat") assert "conanvcvars.bat" in bat_contents + + +@pytest.mark.skipif(platform.system() != "Windows", reason="Requires Windows") +def test_autotools_support_custom_make(): + """ Check that the conf setting `tools.gnu:make_program` works when set with + windows native paths. For example, when set programatically by a package + """ + client = TestClient(path_with_spaces=False) + bash_path = None + make_path = None + try: + bash_path = tools_locations["msys2"]["system"]["path"]["Windows"] + "/bash.exe" + make_path = tools_locations["msys2"]["system"]["path"]["Windows"] + "/make.exe" + except KeyError: + pytest.skip("msys2 path not defined") + if not os.path.exists(make_path): + pytest.skip("msys2 make not installed") + + make_path = make_path.replace("/", "\\") + assert os.path.exists(make_path) + + profile = textwrap.dedent(f""" + include(default) + + [conf] + tools.microsoft.bash:subsystem=msys2 + tools.microsoft.bash:path={bash_path} + tools.gnu:make_program={make_path} + tools.build:compiler_executables={{"c": "cl", "cpp": "cl"}} + """) + + # The autotools support for "cl" compiler (VS) is very limited, linking with deps doesn't + # work but building a simple app do + makefile = gen_makefile() + + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.gnu import Autotools + + class TestConan(ConanFile): + settings = "os", "compiler", "arch", "build_type" + generators = "AutotoolsToolchain" + win_bash = True + + def build(self): + # These commands will run in bash activating first the vcvars and + # then inside the bash activating the + autotools = Autotools(self) + autotools.make() + """) + + client.save({"conanfile.py": conanfile, + "Makefile": makefile, + "profile": profile}) + client.run("build . -pr=profile") + # This used to crash, because ``make_program`` was not unix_path + assert "conanfile.py: Calling build()" in client.out From af38e1310f888e96573324e4aec9eb09d0da57c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= <5364255+AbrilRBS@users.noreply.github.com> Date: Mon, 2 Mar 2026 14:30:08 +0100 Subject: [PATCH 033/110] Add support for Apple IS 26.3 (#19691) Darwin new os releases --- conan/internal/default_settings.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/conan/internal/default_settings.py b/conan/internal/default_settings.py index 16ee28d3c83..23da5230177 100644 --- a/conan/internal/default_settings.py +++ b/conan/internal/default_settings.py @@ -24,7 +24,7 @@ "16.0", "16.1", "16.2", "16.3", "16.4", "16.5", "16.6", "16.7", "17.0", "17.1", "17.2", "17.3", "17.4", "17.5", "17.6", "17.7", "17.8", "18.0", "18.1", "18.2", "18.3", "18.4", "18.5", "18.6", "18.7", - "26.0", "26.1", "26.2"] + "26.0", "26.1", "26.2", "26.3"] sdk: ["iphoneos", "iphonesimulator"] sdk_version: [null, "11.3", "11.4", "12.0", "12.1", "12.2", "12.4", "13.0", "13.1", "13.2", "13.3", "13.4", "13.5", "13.6", "13.7", @@ -39,7 +39,7 @@ "9.0","9.1", "9.2", "9.3", "9.4", "9.5", "9.6", "10.0", "10.1", "10.2", "10.3", "10.4", "10.5", "10.6", "11.0", "11.1", "11.2", "11.3", "11.4", "11.5", "11.6", - "26.0", "26.1", "26.2"] + "26.0", "26.1", "26.2", "26.3"] sdk: ["watchos", "watchsimulator"] sdk_version: [null, "4.3", "5.0", "5.1", "5.2", "5.3", "6.0", "6.1", "6.2", "7.0", "7.1", "7.2", "7.3", "7.4", "8.0", "8.0.1", "8.3", "8.5", "9.0", "9.1", @@ -55,7 +55,7 @@ "16.0", "16.1", "16.2", "16.3", "16.4", "16.5", "16.6", "17.0", "17.1", "17.2", "17.3", "17.4", "17.5", "17.6", "18.0", "18.1", "18.2", "18.3", "18.4", "18.5", "18.6", - "26.0", "26.1", "26.2"] + "26.0", "26.1", "26.2", "26.3"] sdk: ["appletvos", "appletvsimulator"] sdk_version: [null, "11.3", "11.4", "12.0", "12.1", "12.2", "12.4", "13.0", "13.2", "13.3", "13.4", "14.0", "14.2", "14.3", "14.4", "14.5", "15.0", @@ -64,7 +64,7 @@ "26.0", "26.1", "26.2"] visionOS: version: ["1.0", "1.1", "1.2", "1.3", "2.0", "2.1", "2.2", "2.3", "2.4", "2.5", "2.6", - "26.0", "26.1", "26.2"] + "26.0", "26.1", "26.2", "26.3"] sdk: ["xros", "xrsimulator"] sdk_version: [null, "1.0", "1.1", "1.2", "1.3", "2.0", "2.1", "2.2", "2.4", "2.5", "26.0", "26.1", "26.2"] @@ -75,7 +75,7 @@ "13.0", "13.1", "13.2", "13.3", "13.4", "13.5", "13.6", "13.7", "14.0", "14.1", "14.2", "14.3", "14.4", "14.5", "14.6", "14.7", "15.0", "15.1", "15.2", "15.3", "15.4", "15.5", "15.6", "15.7", - "26.0", "26.1", "26.2"] + "26.0", "26.1", "26.2", "26.3"] sdk_version: [null, "10.13", "10.14", "10.15", "11.0", "11.1", "11.2", "11.3", "12.0", "12.1", "12.3", "12.4", "13.0", "13.1", "13.3", "14.0", "14.2", "14.4", "14.5", "15.0", "15.1", "15.2", "15.4", "15.5", From efba35cf3d10352d04fb662a1910b380d202d1c4 Mon Sep 17 00:00:00 2001 From: Carlos Zoido Date: Tue, 3 Mar 2026 14:36:40 +0100 Subject: [PATCH 034/110] Create stubs correctly for user CMake presets when user_presets_path specified. (#19251) * add test * wip * wip * wip * wip * refactor * move tests * use cmake * wip * wip * wip * wip * wip * wip * wip * wip * simplifu * add assert --- conan/tools/cmake/presets.py | 54 ++++++-- .../toolchains/cmake/test_presets_inherit.py | 122 +++++++++++------- 2 files changed, 113 insertions(+), 63 deletions(-) diff --git a/conan/tools/cmake/presets.py b/conan/tools/cmake/presets.py index 4b702665631..73e9e3b95ec 100644 --- a/conan/tools/cmake/presets.py +++ b/conan/tools/cmake/presets.py @@ -296,18 +296,12 @@ def generate(conanfile, preset_path, user_presets_path, preset_prefix, preset_da if not os.path.exists(user_presets_path): data = {"version": 4, "vendor": {"conan": dict()}} - for preset, inherits in inherited_user.items(): - for i in inherits: - data.setdefault(preset, []).append({"name": i}) else: data = json.loads(load(user_presets_path)) if "conan" not in data.get("vendor", {}): # The file is not ours, we cannot overwrite it return - if inherited_user: - _IncludingPresets._clean_user_inherits(data, preset_data) - if not absolute_paths: try: # Make it relative to the CMakeUserPresets.json if possible preset_path = os.path.relpath(preset_path, output_dir) @@ -318,10 +312,50 @@ def generate(conanfile, preset_path, user_presets_path, preset_prefix, preset_da pass data = _IncludingPresets._append_user_preset_path(data, preset_path, output_dir) + if inherited_user: + data = _IncludingPresets._update_stubs(data, inherited_user, output_dir, absolute_paths) + data = json.dumps(data, indent=4) ConanOutput(str(conanfile)).info(f"CMakeToolchain generated: {user_presets_path}") save(user_presets_path, data) + @staticmethod + def _update_stubs(data, inherited_user, output_dir, absolute_paths): + """ + Set configurePresets/buildPresets/testPresets to stubs for conan-* presets + that the user inherits but that don't have a real preset of the same type in the includes. + """ + real_preset_names_by_type = { + "configurePresets": set(), + "buildPresets": set(), + "testPresets": set(), + } + for inc in data.get("include", []): + inc_path = os.path.join(output_dir, inc) if not absolute_paths else inc + assert os.path.exists(inc_path), f"Presets include must point to an existing file: '{inc_path}'" + try: + inc_json = json.loads(load(inc_path)) + except Exception: + continue + for preset_type in ("configurePresets", "buildPresets", "testPresets"): + for p in inc_json.get(preset_type, []): + name = p.get("name") + if name: + real_preset_names_by_type[preset_type].add(name) + + for preset_type in ("configurePresets", "buildPresets", "testPresets"): + real_names = real_preset_names_by_type[preset_type] + stubs = [] + for name in inherited_user.get(preset_type, []): + if name not in real_names: + stub = {"name": name} + if preset_type in ("buildPresets", "testPresets"): + stub["configurePreset"] = name + stubs.append(stub) + data[preset_type] = stubs + + return data + @staticmethod def _collect_user_inherits(output_dir, preset_prefix): # Collect all the existing targets in the user files, to create empty conan- presets @@ -346,14 +380,6 @@ def _collect_user_inherits(output_dir, preset_prefix): return collected_targets - @staticmethod - def _clean_user_inherits(data, preset_data): - for preset_type in "configurePresets", "buildPresets", "testPresets": - presets = preset_data.get(preset_type, []) - presets_names = [p["name"] for p in presets] - other = data.get(preset_type, []) - other[:] = [p for p in other if p["name"] not in presets_names] - @staticmethod def _append_user_preset_path(data, preset_path, output_dir): """ - Appends a 'include' to preset_path if the schema supports it. diff --git a/test/functional/toolchains/cmake/test_presets_inherit.py b/test/functional/toolchains/cmake/test_presets_inherit.py index 3678ca4f77d..a0ea6d3b971 100644 --- a/test/functional/toolchains/cmake/test_presets_inherit.py +++ b/test/functional/toolchains/cmake/test_presets_inherit.py @@ -1,10 +1,45 @@ +import json +import os import platform import textwrap +from shutil import rmtree import pytest from conan.test.utils.tools import TestClient +# Shared CMakePresets.json for tests that use user_presets_path + ConanPresets.json +_CMAKE_PRESETS_FILE = textwrap.dedent(""" + { + "version": 4, + "include": ["./ConanPresets.json"], + "configurePresets": [ + {"name": "default", "displayName": "multi config", "inherits": "conan-default"}, + {"name": "release", "displayName": "release single config", "inherits": "conan-release"}, + {"name": "debug", "displayName": "debug single config", "inherits": "conan-debug"} + ], + "buildPresets": [ + {"name": "multi-release", "configurePreset": "default", "configuration": "Release", "inherits": "conan-release"}, + {"name": "multi-debug", "configurePreset": "default", "configuration": "Debug", "inherits": "conan-debug"}, + {"name": "release", "configurePreset": "release", "configuration": "Release", "inherits": "conan-release"}, + {"name": "debug", "configurePreset": "debug", "configuration": "Debug", "inherits": "conan-debug"} + ] + } +""") + + +def _client_with_user_presets(): + """TestClient with cmake_exe, user_presets_path and standard CMakePresets.json.""" + c = TestClient() + c.run("new cmake_exe -d name=foo -d version=1.0") + conanfile = c.load("conanfile.py") + conanfile = conanfile.replace( + "tc = CMakeToolchain(self)", + "tc = CMakeToolchain(self)\n tc.user_presets_path = 'ConanPresets.json'", + ) + c.save({"conanfile.py": conanfile, "CMakePresets.json": _CMAKE_PRESETS_FILE}) + return c + @pytest.mark.tool("cmake", "3.23") def test_cmake_presets_with_user_presets_file(): @@ -17,56 +52,8 @@ def test_cmake_presets_with_user_presets_file(): "tc = CMakeToolchain(self)\n" " tc.user_presets_path = 'ConanPresets.json'\n" " tc.presets_prefix = 'conan'\n") - cmake_presets = textwrap.dedent(""" - { - "version": 4, - "include": ["./ConanPresets.json"], - "configurePresets": [ - { - "name": "default", - "displayName": "multi config", - "inherits": "conan-default" - }, - { - "name": "release", - "displayName": "release single config", - "inherits": "conan-release" - }, - { - "name": "debug", - "displayName": "debug single config", - "inherits": "conan-debug" - } - ], - "buildPresets": [ - { - "name": "multi-release", - "configurePreset": "default", - "configuration": "Release", - "inherits": "conan-release" - }, - { - "name": "multi-debug", - "configurePreset": "default", - "configuration": "Debug", - "inherits": "conan-debug" - }, - { - "name": "release", - "configurePreset": "release", - "configuration": "Release", - "inherits": "conan-release" - }, - { - "name": "debug", - "configurePreset": "debug", - "configuration": "Debug", - "inherits": "conan-debug" - } - ] - }""") c.save({"conanfile.py": conanfile, - "CMakePresets.json": cmake_presets}) + "CMakePresets.json": _CMAKE_PRESETS_FILE}) c.run(f"install . ") c.run(f"install . -s build_type=Debug") @@ -91,3 +78,40 @@ def test_cmake_presets_with_user_presets_file(): c.run_command("build\\Release\\foo") assert "Hello World Release!" in c.out + + +@pytest.mark.tool("cmake", "3.23") +def test_cmake_presets_build_preset_stub_needs_configure_preset(): + """Reproduce issue #19180: buildPresets stubs in ConanPresets.json must include + 'configurePreset' field for cmake --list-presets to succeed (single-config generators). + """ + c = _client_with_user_presets() + c.run("install .") + + conan_presets = json.loads(c.load("ConanPresets.json")) + for stub in conan_presets.get("buildPresets", []): + assert "configurePreset" in stub + + c.run_command("cmake --list-presets") + assert "Invalid preset" not in c.out, f"cmake --list-presets failed: {c.out}" + + +@pytest.mark.tool("cmake", "3.23") +def test_cmake_presets_stubs_restored_after_build_folder_deleted(): + """Reproduce issue #19173: after deleting build/ and reinstalling one config, + ConanPresets.json must still contain stubs for presets inherited by user (e.g. conan-release) + so cmake --list-presets does not fail. + """ + c = _client_with_user_presets() + c.run("install . -s build_type=Debug") + c.run("install . -s build_type=Release") + + rmtree(os.path.join(c.current_folder, "build")) + c.run("install . -s build_type=Debug") + + conan_presets = json.loads(c.load("ConanPresets.json")) + stub_names = {s["name"] for s in conan_presets.get("configurePresets", [])} + assert "conan-release" in stub_names + + c.run_command("cmake --list-presets") + assert "Invalid preset" not in c.out, f"cmake --list-presets failed: {c.out}" From 20d0fa56056bd44cfd992638c05a07a9209bc977 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 4 Mar 2026 07:13:50 +0100 Subject: [PATCH 035/110] fix win_bash_true with scope=run (#19703) --- conan/internal/model/conan_file.py | 3 ++- .../toolchains/gnu/autotools/test_win_bash.py | 15 ++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/conan/internal/model/conan_file.py b/conan/internal/model/conan_file.py index 37bc95e9c7b..eff7c2cd842 100644 --- a/conan/internal/model/conan_file.py +++ b/conan/internal/model/conan_file.py @@ -384,7 +384,8 @@ def run(self, command: str, stdout=None, cwd=None, ignore_errors=False, env="", env = [env] if env and isinstance(env, str) else (env or []) assert isinstance(env, list), "env argument to ConanFile.run() should be a list" envfiles_folder = self.generators_folder or os.getcwd() - wrapped_cmd = command_env_wrapper(self, command, env, envfiles_folder=envfiles_folder) + wrapped_cmd = command_env_wrapper(self, command, env, envfiles_folder=envfiles_folder, + scope=scope) from conan.internal.util.runners import conan_run if not quiet: ConanOutput().info(f"{self.display_name}: RUN: {command}", fg=Color.BRIGHT_BLUE) diff --git a/test/functional/toolchains/gnu/autotools/test_win_bash.py b/test/functional/toolchains/gnu/autotools/test_win_bash.py index 9d1a9ac8c15..455acc79b3e 100644 --- a/test/functional/toolchains/gnu/autotools/test_win_bash.py +++ b/test/functional/toolchains/gnu/autotools/test_win_bash.py @@ -144,8 +144,9 @@ def build(self): check_vs_runtime("main.exe", client, "17", build_type=build_type, static_runtime=static_runtime) +@pytest.mark.parametrize("scope", ["build", "run"]) @pytest.mark.skipif(platform.system() != "Windows", reason="Requires Windows") -def test_add_msys2_path_automatically(): +def test_add_msys2_path_automatically(scope): """ Check that commands like ar, autoconf, etc, that are in the /usr/bin folder together with the bash.exe, can be automaticallly used when running in windows bash, without user extra addition to [buildenv] of that msys64/usr/bin path @@ -164,21 +165,25 @@ def test_add_msys2_path_automatically(): tools.microsoft.bash:path={} """.format(bash_path))}) - conanfile = textwrap.dedent(""" + conanfile = textwrap.dedent(f""" from conan import ConanFile class HelloConan(ConanFile): name = "hello" version = "0.1" - win_bash = True + def configure(self): + if "{scope}" == "build": + self.win_bash = True + else: + self.win_bash_run = True def build(self): - self.run("ar -h") + self.run("ar -h", scope="{scope}") """) client.save({"conanfile.py": conanfile}) - client.run("create .") + client.run("build .") assert "ar.exe" in client.out From fa8a6e664e38da0d9c86d2496e71b2f1399797b1 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 4 Mar 2026 07:30:00 +0100 Subject: [PATCH 036/110] implicit C linkage support by CXX projects in CMakeConfigDeps (#19704) --- .../cmakeconfigdeps/target_configuration.py | 6 + .../test_cmakeconfigdeps_new_cpp_linkage.py | 115 +++++++++++++++++- 2 files changed, 118 insertions(+), 3 deletions(-) diff --git a/conan/tools/cmake/cmakeconfigdeps/target_configuration.py b/conan/tools/cmake/cmakeconfigdeps/target_configuration.py index 739ebbfc6c5..29f19d95da1 100644 --- a/conan/tools/cmake/cmakeconfigdeps/target_configuration.py +++ b/conan/tools/cmake/cmakeconfigdeps/target_configuration.py @@ -408,6 +408,12 @@ def _template(self): {% if lib_info.get("link_languages") %} get_property(_languages GLOBAL PROPERTY ENABLED_LANGUAGES) + if("CXX" IN_LIST _languages) + list(APPEND _languages "C") + endif() + if("CUDA" IN_LIST _languages) + list(APPEND _languages "C" "CXX") + endif() {% for lang in lib_info["link_languages"] %} if(NOT "{{lang}}" IN_LIST _languages) message(SEND_ERROR diff --git a/test/functional/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_new_cpp_linkage.py b/test/functional/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_new_cpp_linkage.py index 0932a77fa35..e29e11cba1b 100644 --- a/test/functional/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_new_cpp_linkage.py +++ b/test/functional/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_new_cpp_linkage.py @@ -2,8 +2,117 @@ import textwrap import pytest +from conan.test.assets.sources import gen_function_c, gen_function_h +from conan.test.utils.tools import TestClient -new_value = "will_break_next" + +@pytest.mark.tool("cmake") +def test_cxx_only_project_links_c_library(): + """ + When the consumer enables only CXX (project(myapp CXX)), the generated CMake config + must not SEND_ERROR for a dependency with C linkage. CXX implies C linkage support, + so the template adds "C" to the enabled languages check when "CXX" is present. + """ + c = TestClient() + # C library package: builds a static C lib, declares languages = "C" so link_languages = ["C"] + hello_c = gen_function_c(name="hello") + hello_h = gen_function_h(name="hello") + hello_cmake = textwrap.dedent(""" + cmake_minimum_required(VERSION 3.15) + project(hello C) + add_library(hello STATIC src/hello.c) + target_include_directories(hello PUBLIC + $ + $) + install(TARGETS hello + ARCHIVE DESTINATION lib + INCLUDES DESTINATION include) + install(FILES include/hello.h DESTINATION include) + """) + hello_conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.cmake import CMake, cmake_layout + + class Clib(ConanFile): + name = "hello" + version = "0.1" + settings = "os", "compiler", "build_type", "arch" + package_type = "static-library" + generators = "CMakeToolchain" + exports_sources = "CMakeLists.txt", "src/*", "include/*" + languages = "C" + + def layout(self): + cmake_layout(self) + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def package(self): + cmake = CMake(self) + cmake.install() + + def package_info(self): + self.cpp_info.libs = ["hello"] + """) + c.save({ + "hello/conanfile.py": hello_conanfile, + "hello/CMakeLists.txt": hello_cmake, + "hello/src/hello.c": hello_c, + "hello/include/hello.h": hello_h, + }) + c.run("create hello") + + # Consumer: project(CXX) only - no C in project(). Links to C library. + consumer_cmake = textwrap.dedent(""" + cmake_minimum_required(VERSION 3.15) + project(myapp CXX) + + find_package(hello CONFIG REQUIRED) + + add_executable(app main.cpp) + target_link_libraries(app PRIVATE hello::hello) + """) + main_cpp = textwrap.dedent(""" + extern "C" { + #include "hello.h" + } + int main() { + hello(); + } + """) + consumer_conanfile = textwrap.dedent("""\ + import os + from conan import ConanFile + from conan.tools.cmake import CMake, cmake_layout + + class Recipe(ConanFile): + settings = "os", "compiler", "build_type", "arch" + package_type = "application" + generators = "CMakeToolchain", "CMakeConfigDeps" + requires = "hello/0.1" + + def layout(self): + cmake_layout(self) + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + self.run(os.path.join(self.cpp.build.bindir, "app"), env="conanrun") + """) + c.save({ + "consumer/conanfile.py": consumer_conanfile, + "consumer/CMakeLists.txt": consumer_cmake, + "consumer/main.cpp": main_cpp, + }) + c.run("build consumer") + + # If the fix were missing, CMake configure would SEND_ERROR: "Target hello::hello has C + # linkage but C not enabled in project()". So reaching here and running the app proves the fix. + assert "hello: Release!" in c.out @pytest.mark.skipif(platform.system() == "Windows", reason="Windows doesn't fail to link") @@ -28,7 +137,7 @@ def test_auto_cppstd(matrix_c_interface_client): class Recipe(ConanFile): settings = "os", "compiler", "build_type", "arch" package_type = "application" - generators = "CMakeToolchain", "CMakeDeps" + generators = "CMakeToolchain", "CMakeConfigDeps" requires = "matrix/0.1" def layout(self): @@ -50,5 +159,5 @@ def build(self): c.save({"conanfile.py": conanfile, "CMakeLists.txt": consumer, "app.c": app}, clean_first=True) - c.run(f"build . -c tools.cmake.cmakedeps:new={new_value}") + c.run(f"build .") assert "Hello Matrix!" in c.out From fba035123b4bf9ab6cd4e7d99689e2dd660bc703 Mon Sep 17 00:00:00 2001 From: Ernesto de Gracia Herranz Date: Wed, 4 Mar 2026 18:03:21 +0100 Subject: [PATCH 037/110] Representation of a cycle/loop in the graph info with a red arrow (#19694) * loop representation in conan graph info * fix test * fix * remove use of level --- conan/cli/formatters/graph/info_graph_html.py | 13 +++++++++++++ conan/internal/graph/graph_error.py | 7 +++++++ .../command/info/test_graph_info_graphical.py | 14 ++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/conan/cli/formatters/graph/info_graph_html.py b/conan/cli/formatters/graph/info_graph_html.py index 1d0160f71d0..8d69450afdb 100644 --- a/conan/cli/formatters/graph/info_graph_html.py +++ b/conan/cli/formatters/graph/info_graph_html.py @@ -89,12 +89,15 @@ let conflict=null; let provide_conflict=null; let missing_error=null; + let loop_error=null; if (graph_data["error"] && graph_data["error"]["type"] == "conflict") conflict = graph_data["error"]; else if (graph_data["error"] && graph_data["error"]["type"] == "provide_conflict") provide_conflict = graph_data["error"]; else if (graph_data["error"] && graph_data["error"]["type"] == "missing") missing_error = graph_data["error"]; + else if (graph_data["error"] && graph_data["error"]["type"] == "loop") + loop_error = [graph_data["error"]['node']['label'], graph_data["error"]['require']['name']]; for (const [node_id, node] of Object.entries(graph_data["nodes"])) { if (node.context == "build" && hide_build) continue; if (node.test && hide_test) continue; @@ -183,6 +186,16 @@ color: {color: "SkyBlue", highlight: "Blue"}}); global_edges[edge_counter++] = dep; } + if (loop_error && loop_error[1] == node["name"] && loop_error[0] == dep["ref"]) { + let target_id = targets[dep_id] || dep_id; + edges.push({id: edge_counter, from: node_id, to: target_id, + color: {color: "Red", highlight: "Red"}, + smooth: { enabled: true, type: 'curvedCW', roundness: 0.4 }, + arrows: "from", + label: "loop", + title: "loop"}); + global_edges[edge_counter++] = dep; + } } } if (conflict) { diff --git a/conan/internal/graph/graph_error.py b/conan/internal/graph/graph_error.py index e864cd240c0..c3db7c6a2bc 100644 --- a/conan/internal/graph/graph_error.py +++ b/conan/internal/graph/graph_error.py @@ -43,6 +43,13 @@ def __init__(self, node, require, ancestor): self.require = require self.ancestor = ancestor + def serialize(self): + return {"type": "loop", + "require": {**self.require.serialize(), "name": str(self.require.ref).split("/")[0]}, + "node": self.node.serialize(), + "ancestor": self.ancestor.serialize() + } + def __str__(self): return "There is a cycle/loop in the graph:\n" \ f" Initial ancestor: {self.ancestor}\n" \ diff --git a/test/integration/command/info/test_graph_info_graphical.py b/test/integration/command/info/test_graph_info_graphical.py index 27aa57bfd1a..f757368c01f 100644 --- a/test/integration/command/info/test_graph_info_graphical.py +++ b/test/integration/command/info/test_graph_info_graphical.py @@ -161,6 +161,20 @@ def test_graph_conflict_diamond(): assert "ERROR: Version conflict: Conflict between math/1.0.1 and math/1.0 in the graph." in c.out +def test_graph_conflict_loop(): + c = TestClient(light=True) + c.save({"lib_a/conanfile.py": GenConanfile("lib_a", "1.0").with_requires("lib_b/1.0"), + "lib_b/conanfile.py": GenConanfile("lib_b", "1.0").with_requires("lib_c/1.0"), + "lib_c/conanfile.py": GenConanfile("lib_c", "1.0").with_requires("lib_a/1.0"), + "lib_x/conanfile.py": GenConanfile("lib_x", "1.0").with_requires("lib_a/1.0"), + }) + c.run("export lib_a") + c.run("export lib_b") + c.run("export lib_c") + c.run("graph info lib_x --format=html", assert_error=True, redirect_stdout="graph.html") + # checked manually + #c.open("graph.html") + def test_graph_missing_error(): c = TestClient() c.save({"engine/conanfile.py": GenConanfile("engine", "1.0").with_requires("math/1.0"), From 0485a86328068f51f34c631bd78609ed27baf36d Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Wed, 4 Mar 2026 23:08:33 +0000 Subject: [PATCH 038/110] Add support for Clang version 22 (#19709) --- conan/internal/default_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conan/internal/default_settings.py b/conan/internal/default_settings.py index 23da5230177..a8aebe4b99c 100644 --- a/conan/internal/default_settings.py +++ b/conan/internal/default_settings.py @@ -141,7 +141,7 @@ version: ["3.3", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9", "4.0", "5.0", "6.0", "7.0", "7.1", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", - "18", "19", "20", "21"] + "18", "19", "20", "21", "22"] libcxx: [null, libstdc++, libstdc++11, libc++, c++_shared, c++_static] cppstd: [null, 98, gnu98, 11, gnu11, 14, gnu14, 17, gnu17, 20, gnu20, 23, gnu23, 26, gnu26] runtime: [null, static, dynamic] From 2bdab344529ee468802a6aed35b8f2774faaf746 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 5 Mar 2026 07:24:31 +0100 Subject: [PATCH 039/110] Fix CMake escape error (#19706) * fix cmake escape error * remove patch * revert * revert cmake_escape_value() --- conan/tools/cmake/toolchain/blocks.py | 7 ++----- conan/tools/cmake/utils.py | 17 +-------------- .../toolchains/cmake/test_cmaketoolchain.py | 10 ++++++--- test/unittests/tools/cmake/test_cmake_test.py | 21 ------------------- 4 files changed, 10 insertions(+), 45 deletions(-) diff --git a/conan/tools/cmake/toolchain/blocks.py b/conan/tools/cmake/toolchain/blocks.py index 9d13cbc7115..d798e9a8753 100644 --- a/conan/tools/cmake/toolchain/blocks.py +++ b/conan/tools/cmake/toolchain/blocks.py @@ -1331,11 +1331,8 @@ def template(self): """) def context(self): - variables = {k: cmake_escape_value(v) for k, v in self._toolchain.variables.items()} - vars_config = {k: [(config, cmake_escape_value(v)) for (config, v) in configs] - for k, configs in self._toolchain.variables.configuration_types.items()} - return {"variables": variables, - "variables_config": vars_config} + return {"variables": self._toolchain.variables, + "variables_config": self._toolchain.variables.configuration_types} class PreprocessorBlock(Block): diff --git a/conan/tools/cmake/utils.py b/conan/tools/cmake/utils.py index 0df8e5ff38b..757a63a3568 100644 --- a/conan/tools/cmake/utils.py +++ b/conan/tools/cmake/utils.py @@ -42,19 +42,4 @@ def parse_extra_variable(source, key, value): def cmake_escape_value(v): - if not isinstance(v, str): - return v - - def _stream_chars(s): - iterable = iter(enumerate(s)) - for i, char in iterable: - # If we see a backslash, check the next char immediately - if char == '\\' and i + 1 < len(s) and s[i+1] in r'\$"': - yield s[i:i+2] # Yield the existing pair - next(iterable) # Skip the next char in the loop - elif char in r'\$"': - yield '\\' + char # Escape the lonely char - else: - yield char - - return "".join(_stream_chars(v)) + return v.replace('\\', '\\\\').replace('$', '\\$').replace('"', '\\"') diff --git a/test/integration/toolchains/cmake/test_cmaketoolchain.py b/test/integration/toolchains/cmake/test_cmaketoolchain.py index 67e71c5db68..dccd2708243 100644 --- a/test/integration/toolchains/cmake/test_cmaketoolchain.py +++ b/test/integration/toolchains/cmake/test_cmaketoolchain.py @@ -841,6 +841,7 @@ def generate(self): def test_variables_escaping(): # https://github.com/conan-io/conan/issues/19638 client = TestClient() + # NOTE: Users need to do explicit escaping conanfile = textwrap.dedent(r""" from conan import ConanFile from conan.tools.cmake import CMakeToolchain @@ -849,14 +850,17 @@ class Conan(ConanFile): settings = "os", "arch", "compiler", "build_type" def generate(self): toolchain = CMakeToolchain(self) - toolchain.variables["FOO"] = r"D:\new\thing\path" - toolchain.variables.release["BAR"] = r"C:\new\thing\path" + toolchain.variables["FOO"] = r"D:\new\thing\path".replace("\\", "\\\\") + toolchain.variables["CMAKE_Fortran_FLAGS_INIT"] = "${CMAKE_C_FLAGS_INIT}" + toolchain.variables.release["BAR"] = r"C:\new\thing\path".replace("\\", "\\\\") toolchain.generate() """) + client.save({"conanfile.py": conanfile}) - client.run("install . --name=mylib --version=1.0") + client.run("install .") toolchain = client.load("conan_toolchain.cmake") + assert 'set(CMAKE_Fortran_FLAGS_INIT "${CMAKE_C_FLAGS_INIT}"' in toolchain assert r'set(FOO "D:\\new\\thing\\path" CACHE STRING' in toolchain assert r'set(CONAN_DEF_releaseBAR "C:\\new\\thing\\path")' in toolchain diff --git a/test/unittests/tools/cmake/test_cmake_test.py b/test/unittests/tools/cmake/test_cmake_test.py index d507557f3a2..8ed861fd1aa 100644 --- a/test/unittests/tools/cmake/test_cmake_test.py +++ b/test/unittests/tools/cmake/test_cmake_test.py @@ -83,24 +83,3 @@ def test_run_ctest(): cmake = CMake(conanfile) cmake.ctest(cli_args=["--schedule-random", "--quiet"]) assert "--schedule-random --quiet --verbose --debug --output-junit myfile" in conanfile.command - - -@pytest.mark.parametrize("input_str, expected", [ - (r"PlainString", r"PlainString"), # Case 1: Plain strings (No change) - # Case 2: Individual characters (First-time escape) - (r"C:\Path", r"C:\\Path"), - (r'He said "Hi"', r'He said \"Hi\"'), - (r"Cost is $10", r"Cost is \$10"), - # Case 3: Complex mixed strings - (r'Mixed \path and "quote" with $VAR', r'Mixed \\path and \"quote\" with \$VAR'), - # Case 4: Partial escapes (Only unescaped parts get fixed) - (r'\"Already" and \$Already$', r'\"Already\" and \$Already\$'), - # Case 5: Edge cases - (r'TrailingSlash\\', r'TrailingSlash\\'), - (r"Double\\Slash", r"Double\\Slash"), - (r"", r""), # Empty string -]) -def test_cmake_escape_correctness(input_str, expected): - escaped = cmake_escape_value(input_str) - assert escaped == expected - assert cmake_escape_value(escaped) == expected From f68019c138dd6b2cd2f7101f716d022a838e9f77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= <5364255+AbrilRBS@users.noreply.github.com> Date: Thu, 5 Mar 2026 07:50:45 +0100 Subject: [PATCH 040/110] Fix exception with corrupted recipes (#19713) --- conan/internal/cache/integrity_check.py | 2 +- test/integration/command/cache/test_cache_integrity.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/conan/internal/cache/integrity_check.py b/conan/internal/cache/integrity_check.py index f5b22690c13..44e41a1eda0 100644 --- a/conan/internal/cache/integrity_check.py +++ b/conan/internal/cache/integrity_check.py @@ -52,7 +52,7 @@ def _recipe_corrupted(self, ref: RecipeReference): if not k.startswith("export_source")} if read_manifest != expected_manifest: - output_lines = ["", "Manifest mismatch", f" Folder: {layout.package()}"] + output_lines = ["", "Manifest mismatch", f" Folder: {layout.export()}"] diff = read_manifest.difference(expected_manifest) for fname, (h1, h2) in diff.items(): output_lines.append(f" {fname} (manifest: {h1}, file: {h2})") diff --git a/test/integration/command/cache/test_cache_integrity.py b/test/integration/command/cache/test_cache_integrity.py index a40cf817445..67ebddc1bc9 100644 --- a/test/integration/command/cache/test_cache_integrity.py +++ b/test/integration/command/cache/test_cache_integrity.py @@ -10,7 +10,7 @@ @pytest.mark.parametrize("use_pkglist", [True, False]) @pytest.mark.parametrize("output_pkglist", [True, False]) def test_cache_integrity(use_pkglist, output_pkglist): - t = TestClient() + t = TestClient(light=True) t.save({"conanfile.py": GenConanfile()}) t.run("create . --name pkg1 --version 1.0") t.run("create . --name pkg2 --version=2.0") @@ -25,6 +25,9 @@ def test_cache_integrity(use_pkglist, output_pkglist): layout = t.created_layout() conaninfo = os.path.join(layout.package(), "conaninfo.txt") save(conaninfo, "[settings]") + t.run("create . --name pkg5 --version=5.0") + layout = t.exported_layout() + save(layout.conanfile(), "empty") if use_pkglist: t.run("list *:*#* -f=json", redirect_stdout="pkglist.json") @@ -47,6 +50,7 @@ def test_cache_integrity(use_pkglist, output_pkglist): "#0ba8627bd47edc3a501e8f0eb9a79e5e: ERROR: \nManifest mismatch" in t.out assert "pkg4/4.0#4d670581ccb765839f2239cc8dff8fbd:da39a3ee5e6b4b0d3255bfef95601890afd80709" \ "#0ba8627bd47edc3a501e8f0eb9a79e5e: ERROR: \nManifest mismatch" in t.out + assert "pkg5/5.0#4d670581ccb765839f2239cc8dff8fbd: ERROR: \nManifest mismatch" in t.out if output_pkglist: t.run("remove --list=pkglist.json -c") @@ -54,6 +58,7 @@ def test_cache_integrity(use_pkglist, output_pkglist): t.run("remove pkg2/2.0:da39a3ee5e6b4b0d3255bfef95601890afd80709 -c") t.run("remove pkg3/3.0:da39a3ee5e6b4b0d3255bfef95601890afd80709 -c") t.run("remove pkg4/4.0:da39a3ee5e6b4b0d3255bfef95601890afd80709 -c") + t.run("remove pkg5/5.0 -c") t.run("cache check-integrity *") assert "pkg1/1.0#4d670581ccb765839f2239cc8dff8fbd: Integrity check: ok" in t.out assert "pkg2/2.0#4d670581ccb765839f2239cc8dff8fbd: Integrity check: ok" in t.out From 8338a01d2915846d1db8eb8fc1404aeb1e52cacd Mon Sep 17 00:00:00 2001 From: James Date: Mon, 9 Mar 2026 10:34:02 +0100 Subject: [PATCH 041/110] show transitive html (#19725) --- conan/cli/formatters/graph/info_graph_html.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/conan/cli/formatters/graph/info_graph_html.py b/conan/cli/formatters/graph/info_graph_html.py index 8d69450afdb..bf67480a5c0 100644 --- a/conan/cli/formatters/graph/info_graph_html.py +++ b/conan/cli/formatters/graph/info_graph_html.py @@ -35,6 +35,10 @@
    +
    + + +
    @@ -63,6 +67,7 @@ const graph_data = {{ deps_graph | tojson }}; let hide_build = false; let hide_test = false; + let show_transitive = false; let search_pkgs = null; let focus_search = false; let excluded_pkgs = null; @@ -186,6 +191,13 @@ color: {color: "SkyBlue", highlight: "Blue"}}); global_edges[edge_counter++] = dep; } + if (show_transitive && dep.direct === false){ + let target_id = targets[dep_id] || dep_id; + edges.push({id: edge_counter, from: node_id, to: target_id, + color: {color: "LightGray", highlight: "Gray"}, + dashes: true}); + global_edges[edge_counter++] = dep; + } if (loop_error && loop_error[1] == node["name"] && loop_error[0] == dep["ref"]) { let target_id = targets[dep_id] || dep_id; edges.push({id: edge_counter, from: node_id, to: target_id, @@ -381,6 +393,10 @@ hide_test = !hide_test; draw(); } + function switchTransitive() { + show_transitive = !show_transitive; + draw(); + } function collapsePackages() { collapse_packages = !collapse_packages; draw(); From 5d2f00d7b3e6a79140cc357365cf18a114b97f25 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 9 Mar 2026 10:34:52 +0100 Subject: [PATCH 042/110] add trace to DB timeout (#19728) add trace --- conan/internal/cache/db/table.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conan/internal/cache/db/table.py b/conan/internal/cache/db/table.py index eb77a64d726..71e0ebf5813 100644 --- a/conan/internal/cache/db/table.py +++ b/conan/internal/cache/db/table.py @@ -1,9 +1,11 @@ import sqlite3 import threading +import traceback from collections import defaultdict, namedtuple from contextlib import contextmanager from typing import Tuple, List +from conan.api.output import ConanOutput from conan.errors import ConanException @@ -26,6 +28,8 @@ def __init__(self, filename): @contextmanager def db_connection(self): if not self._lock.acquire(timeout=20): + m = traceback.format_exc() + "\n" + ConanOutput().error(m) raise ConanException("Conan failed to acquire database lock in 20s. Maybe the system is " "under very heavy load. Please report it to Github tickets") # isolation_level=None, puts it in regular SQLITE autocommit mode, every From ba2d6e7a88e05cbf2e6ea43c1874761321def626 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 9 Mar 2026 11:23:01 +0100 Subject: [PATCH 043/110] Tests/improves (#19727) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fake_runs * simplified dead attrs * wip * add workspace check * wip * fix * Rename cmakedeps2 tests folder --------- Co-authored-by: Abril Rincón Blanco <5364255+AbrilRBS@users.noreply.github.com> --- conan/internal/rest/rest_client_v2.py | 28 +++----- test/functional/sbom/test_cyclonedx.py | 71 +++++++++---------- .../apple/test_xcodebuild_targets.py | 2 - .../__init__.py | 0 .../test_cmakeconfigdeps_aliases.py | 0 .../test_cmakeconfigdeps_frameworks.py | 0 .../test_cmakeconfigdeps_new.py | 0 .../test_cmakeconfigdeps_new_cpp_linkage.py | 0 .../test_cmakeconfigdeps_new_paths.py | 0 .../test_cmakeconfigdeps_sources.py | 0 .../cmake/cmakedeps/test_apple_frameworks.py | 18 ++--- .../functional/toolchains/cmake/test_cmake.py | 20 +++--- .../toolchains/cmake/test_cmake_multi.py | 26 +++---- .../cmake/test_cmake_transitive_rpath.py | 34 +++++---- .../env/test_virtualenv_powershell.py | 1 - .../toolchains/scons/test_sconsdeps.py | 2 - test/functional/toolchains/test_premake.py | 12 ++-- .../build_requires/build_requires_test.py | 2 - .../cache/test_home_special_char.py | 1 - test/integration/command/test_package_test.py | 7 -- .../integration/editable/editable_add_test.py | 1 - test/integration/environment/test_env.py | 1 - .../graph/test_replace_requires.py | 16 ++--- .../lockfile/test_lock_requires.py | 3 +- .../package_id_requires_modes_test.py | 2 + .../python_requires_package_id_test.py | 32 +++++++++ test/integration/remote/server_error_test.py | 16 +++++ test/integration/sbom/test_cyclonedx.py | 16 ++--- test/integration/workspace/test_workspace.py | 4 +- .../tools/system/python_manager_test.py | 14 +++- 30 files changed, 176 insertions(+), 153 deletions(-) rename test/functional/toolchains/cmake/{cmakedeps2 => cmakeconfigdeps}/__init__.py (100%) rename test/functional/toolchains/cmake/{cmakedeps2 => cmakeconfigdeps}/test_cmakeconfigdeps_aliases.py (100%) rename test/functional/toolchains/cmake/{cmakedeps2 => cmakeconfigdeps}/test_cmakeconfigdeps_frameworks.py (100%) rename test/functional/toolchains/cmake/{cmakedeps2 => cmakeconfigdeps}/test_cmakeconfigdeps_new.py (100%) rename test/functional/toolchains/cmake/{cmakedeps2 => cmakeconfigdeps}/test_cmakeconfigdeps_new_cpp_linkage.py (100%) rename test/functional/toolchains/cmake/{cmakedeps2 => cmakeconfigdeps}/test_cmakeconfigdeps_new_paths.py (100%) rename test/functional/toolchains/cmake/{cmakedeps2 => cmakeconfigdeps}/test_cmakeconfigdeps_sources.py (100%) diff --git a/conan/internal/rest/rest_client_v2.py b/conan/internal/rest/rest_client_v2.py index ddcee217f13..aabd00536d1 100644 --- a/conan/internal/rest/rest_client_v2.py +++ b/conan/internal/rest/rest_client_v2.py @@ -52,19 +52,13 @@ def __call__(self, request): return request -def get_exception_from_error(error_code): +def _raise_exception_from_error(error_code, text): tmp = {v: k for k, v in EXCEPTION_CODE_MAPPING.items() # All except NotFound if k not in (RecipeNotFoundException, PackageNotFoundException)} - if error_code in tmp: - # logger.debug("REST ERROR: %s" % str(tmp[error_code])) - return tmp[error_code] - else: - base_error = int(str(error_code)[0] + "00") - # logger.debug("REST ERROR: %s" % str(base_error)) - try: - return tmp[base_error] - except KeyError: - return None + try: + raise tmp[error_code](text) + except KeyError: + raise ConanException(f"Server exception {error_code}: {text}") def _get_mac_digest(): # To avoid re-hashing all the time the same mac @@ -126,7 +120,7 @@ def check_credentials(self, force_auth=False): if ret.status_code != 200: ret.charset = "utf-8" # To be able to access ret.text (ret.content are bytes) text = ret.text if ret.status_code != 404 else "404 Not found" - raise get_exception_from_error(ret.status_code)(text) + _raise_exception_from_error(ret.status_code, text) return ret.content.decode() def server_capabilities(self): @@ -139,7 +133,7 @@ def server_capabilities(self): if not server_capabilities and not ret.ok: # Old Artifactory might return 401/403 without capabilities, we don't want # to cache them #5687, so raise the exception and force authentication - raise get_exception_from_error(ret.status_code)(response_to_str(ret)) + _raise_exception_from_error(ret.status_code, response_to_str(ret)) if server_capabilities is None: # Some servers returning 200-ok, even if not valid repo raise ConanException(f"Remote {self.remote_url} doesn't seem like a valid Conan remote") @@ -154,7 +148,7 @@ def _get_json(self, url, headers=None): if response.status_code != 200: # Error message is text response.charset = "utf-8" # To be able to access ret.text (ret.content are bytes) - raise get_exception_from_error(response.status_code)(response_to_str(response)) + _raise_exception_from_error(response.status_code, response_to_str(response)) content = response.content.decode() content_type = response.headers.get("Content-Type") @@ -358,7 +352,7 @@ def remove_all_packages(self, ref): if response.status_code != 200: # Error message is text # To be able to access ret.text (ret.content are bytes) response.charset = "utf-8" - raise get_exception_from_error(response.status_code)(response.text) + _raise_exception_from_error(response.status_code, response.text) def remove_packages(self, prefs): self.check_credentials() @@ -376,7 +370,7 @@ def remove_packages(self, prefs): if response.status_code != 200: # Error message is text # To be able to access ret.text (ret.content are bytes) response.charset = "utf-8" - raise get_exception_from_error(response.status_code)(response.text) + _raise_exception_from_error(response.status_code, response.text) def remove_recipe(self, ref): """ Remove a recipe and packages """ @@ -396,7 +390,7 @@ def remove_recipe(self, ref): if response.status_code != 200: # Error message is text # To be able to access ret.text (ret.content are bytes) response.charset = "utf-8" - raise get_exception_from_error(response.status_code)(response.text) + _raise_exception_from_error(response.status_code, response.text) def get_recipe_revision_reference(self, ref): # FIXME: implement this new endpoint in the remotes? diff --git a/test/functional/sbom/test_cyclonedx.py b/test/functional/sbom/test_cyclonedx.py index 528d413fff2..2b357ce8c6f 100644 --- a/test/functional/sbom/test_cyclonedx.py +++ b/test/functional/sbom/test_cyclonedx.py @@ -1,11 +1,11 @@ import json +import os import pytest +from conan.internal.util.files import save from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient -from conan.internal.util.files import save -import os # Using the sbom tool with "conan create" sbom_hook_post_package = """ @@ -13,73 +13,72 @@ import os from conan.errors import ConanException from conan.api.output import ConanOutput -from conan.tools.sbom import {cyclone_version} +from conan.tools.sbom import cyclonedx_1_4, cyclonedx_1_6 def post_package(conanfile): - sbom_{cyclone_version} = {cyclone_version}(conanfile, add_build={add_build}, add_tests={add_tests}) - metadata_folder = conanfile.package_metadata_folder - file_name = "sbom.cdx.json" - with open(os.path.join(metadata_folder, file_name), 'w') as f: - json.dump(sbom_{cyclone_version}, f, indent=4) - ConanOutput().success(f"CYCLONEDX CREATED - {{conanfile.package_metadata_folder}}") + sbom_cyclonedx_1_4 = cyclonedx_1_4(conanfile, add_build=True, add_tests=True) + sbom_cyclonedx_1_6 = cyclonedx_1_6(conanfile, add_build=True, add_tests=True) + with open(os.path.join(conanfile.package_metadata_folder, "sbom14.cdx.json"), 'w') as f: + json.dump(sbom_cyclonedx_1_4, f, indent=4) + with open(os.path.join(conanfile.package_metadata_folder, "sbom16.cdx.json"), 'w') as f: + json.dump(sbom_cyclonedx_1_6, f, indent=4) """ -@pytest.mark.parametrize("cyclone_version", ["cyclonedx_1_4", "cyclonedx_1_6"]) class TestCyclonedx: @pytest.fixture() - def hook_setup_post_package(self, cyclone_version): - tc = TestClient() - hook_path = os.path.join(tc.paths.hooks_path, "hook_sbom.py") - save(hook_path, sbom_hook_post_package.format(cyclone_version=cyclone_version, - add_build=True, add_tests=True)) - return tc - - @pytest.fixture() - def hook_setup_post_package_tl(self, cyclone_version, transitive_libraries): + def hook_setup_post_package_tl(self, transitive_libraries): tc = transitive_libraries hook_path = os.path.join(tc.paths.hooks_path, "hook_sbom.py") - save(hook_path, sbom_hook_post_package.format(cyclone_version=cyclone_version, - add_build=True, add_tests=True)) + save(hook_path, sbom_hook_post_package) return tc @pytest.mark.tool("cmake") def test_sbom_generation_create(self, hook_setup_post_package_tl): + # TODO This doesn't need to be a functional test, check why tc = hook_setup_post_package_tl tc.run("new cmake_lib -d name=bar -d version=1.0 -d requires=engine/1.0 -f") # bar -> engine/1.0 -> matrix/1.0 tc.run("create . -tf=") bar_layout = tc.created_layout() - assert os.path.exists(os.path.join(bar_layout.metadata(), "sbom.cdx.json")) + assert os.path.exists(os.path.join(bar_layout.metadata(), "sbom14.cdx.json")) + assert os.path.exists(os.path.join(bar_layout.metadata(), "sbom16.cdx.json")) @pytest.mark.tool("cmake") @pytest.mark.parametrize("user, channel, user_dep, channel_dep", [("user", None, "user_dep", None), ("user", "channel", "user_dep", "channel_dep")]) - def test_sbom_user_path(self, hook_setup_post_package_tl, user, channel, user_dep, channel_dep): - tc = hook_setup_post_package_tl + def test_sbom_user_path(self, user, channel, user_dep, channel_dep): + tc = TestClient(light=True) + hook_path = os.path.join(tc.paths.hooks_path, "hook_sbom.py") + save(hook_path, sbom_hook_post_package) channel_ref = f"/{channel_dep}" if channel_dep else "" tc.save({"dep/conanfile.py": GenConanfile("dep", "1.0"), "conanfile.py": GenConanfile("main", "1.0").with_requires( f"dep/1.0@{user_dep}{channel_ref}")}) command = "create dep" - if user: command += f" --user={user_dep}" - if channel: command += f" --channel={channel_dep}" + if user: + command += f" --user={user_dep}" + if channel: + command += f" --channel={channel_dep}" tc.run(command) command = "create ." - if user: command += f" --user={user}" - if channel: command += f" --channel={channel}" + if user: + command += f" --user={user}" + if channel: + command += f" --channel={channel}" tc.run(command) - create_layout = tc.created_layout() - cyclone_path = os.path.join(create_layout.metadata(), "sbom.cdx.json") - content = tc.load(cyclone_path) - content_json = json.loads(content) + for version in ("14", "16"): + create_layout = tc.created_layout() + cyclone_path = os.path.join(create_layout.metadata(), f"sbom{version}.cdx.json") + content = tc.load(cyclone_path) + content_json = json.loads(content) - assert content_json["components"][0]["bom-ref"].split("&user=")[ - 1] == f"{user}&channel={channel}" if channel else user - assert content_json["dependencies"][0]["dependsOn"][0].split("&user=")[ - 1] == f"{user_dep}&channel={channel_dep}" if channel_dep else user_dep + assert content_json["components"][0]["bom-ref"].split("&user=")[ + 1] == f"{user}&channel={channel}" if channel else user + assert content_json["dependencies"][0]["dependsOn"][0].split("&user=")[ + 1] == f"{user_dep}&channel={channel_dep}" if channel_dep else user_dep diff --git a/test/functional/toolchains/apple/test_xcodebuild_targets.py b/test/functional/toolchains/apple/test_xcodebuild_targets.py index 8c6767dedf8..4b759d06531 100644 --- a/test/functional/toolchains/apple/test_xcodebuild_targets.py +++ b/test/functional/toolchains/apple/test_xcodebuild_targets.py @@ -60,8 +60,6 @@ class HelloTestConan(ConanFile): settings = "os", "compiler", "build_type", "arch" generators = "CMakeDeps", "CMakeToolchain", "VirtualBuildEnv", "VirtualRunEnv" - apply_env = False - test_type = "explicit" options = {"shared": [True, False], "fPIC": [True, False]} default_options = {"shared": False, "fPIC": True} diff --git a/test/functional/toolchains/cmake/cmakedeps2/__init__.py b/test/functional/toolchains/cmake/cmakeconfigdeps/__init__.py similarity index 100% rename from test/functional/toolchains/cmake/cmakedeps2/__init__.py rename to test/functional/toolchains/cmake/cmakeconfigdeps/__init__.py diff --git a/test/functional/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_aliases.py b/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_aliases.py similarity index 100% rename from test/functional/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_aliases.py rename to test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_aliases.py diff --git a/test/functional/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_frameworks.py b/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_frameworks.py similarity index 100% rename from test/functional/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_frameworks.py rename to test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_frameworks.py diff --git a/test/functional/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_new.py b/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_new.py similarity index 100% rename from test/functional/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_new.py rename to test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_new.py diff --git a/test/functional/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_new_cpp_linkage.py b/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_new_cpp_linkage.py similarity index 100% rename from test/functional/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_new_cpp_linkage.py rename to test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_new_cpp_linkage.py diff --git a/test/functional/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_new_paths.py b/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_new_paths.py similarity index 100% rename from test/functional/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_new_paths.py rename to test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_new_paths.py diff --git a/test/functional/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_sources.py b/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_sources.py similarity index 100% rename from test/functional/toolchains/cmake/cmakedeps2/test_cmakeconfigdeps_sources.py rename to test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_sources.py diff --git a/test/functional/toolchains/cmake/cmakedeps/test_apple_frameworks.py b/test/functional/toolchains/cmake/cmakedeps/test_apple_frameworks.py index e00f857d02f..7c3e3956cc4 100644 --- a/test/functional/toolchains/cmake/cmakedeps/test_apple_frameworks.py +++ b/test/functional/toolchains/cmake/cmakedeps/test_apple_frameworks.py @@ -196,9 +196,9 @@ class HELLO_EXPORT Hello @pytest.mark.tool("cmake") @pytest.mark.skipif(platform.system() != "Darwin", reason="Only OSX") @pytest.mark.parametrize("settings", - [('',), - ('-pr:b default -s os=iOS -s os.sdk=iphoneos -s os.version=10.0 -s arch=armv8',), - ("-pr:b default -s os=tvOS -s os.sdk=appletvos -s os.version=11.0 -s arch=armv8",)]) + ['', + '-s os=iOS -s os.sdk=iphoneos -s os.version=10.0 -s arch=armv8', + "-s os=tvOS -s os.sdk=appletvos -s os.version=11.0 -s arch=armv8"]) def test_apple_own_framework_cross_build(settings): client = TestClient() @@ -429,7 +429,7 @@ def package_info(self): self.cpp_info.components["libhello"].frameworks.extend(["CoreFoundation"]) """) - hello_cpp = textwrap.dedent(""" + hello_cpp_core = textwrap.dedent(""" #include void hello_api() @@ -441,7 +441,7 @@ def package_info(self): CFRelease(dict); } """) - hello_h = textwrap.dedent(""" + hello_h_core = textwrap.dedent(""" void hello_api(); """) cmakelists_txt = textwrap.dedent(""" @@ -511,8 +511,8 @@ def test(self): """) t = TestClient() t.save({'conanfile.py': conanfile_py, - 'hello.cpp': hello_cpp, - 'hello.h': hello_h, + 'hello.cpp': hello_cpp_core, + 'hello.h': hello_h_core, 'CMakeLists.txt': cmakelists_txt, 'test_package/conanfile.py': test_conanfile_py, 'test_package/CMakeLists.txt': test_cmakelists_txt, @@ -554,7 +554,7 @@ def test_iphoneos_crossbuild(): target_link_libraries(main hello::hello) """) - conanfile = textwrap.dedent(""" + hello = textwrap.dedent(""" from conan import ConanFile from conan.tools.cmake import CMake @@ -570,7 +570,7 @@ def build(self): cmake.build() """) - client.save({"conanfile.py": conanfile, + client.save({"conanfile.py": hello, "CMakeLists.txt": cmakelists, "main.cpp": main, "ios-armv8": profile}, clean_first=True) diff --git a/test/functional/toolchains/cmake/test_cmake.py b/test/functional/toolchains/cmake/test_cmake.py index cc01c55ed64..f7caf919625 100644 --- a/test/functional/toolchains/cmake/test_cmake.py +++ b/test/functional/toolchains/cmake/test_cmake.py @@ -428,18 +428,16 @@ def test_toolchain_apple(self, build_type, cppstd, shared): "CMAKE_INSTALL_NAME_DIR": "" } + arch_flags = { + "CMAKE_C_FLAGS": "-m64", + "CMAKE_CXX_FLAGS": "-m64 -stdlib=libc++", + "CMAKE_SHARED_LINKER_FLAGS": "-m64", + "CMAKE_EXE_LINKER_FLAGS": "-m64", + } host_profile = self.client.get_default_host_profile() - if host_profile.settings.get("arch") == "x86_64": - vals.update({ - "CMAKE_C_FLAGS": "-m64", - "CMAKE_CXX_FLAGS": "-m64 -stdlib=libc++", - "CMAKE_SHARED_LINKER_FLAGS": "-m64", - "CMAKE_EXE_LINKER_FLAGS": "-m64", - }) - else: - vals.update({ - "CMAKE_CXX_FLAGS": "-stdlib=libc++", - }) + if host_profile.settings.get("arch") != "x86_64": + arch_flags = {"CMAKE_CXX_FLAGS": "-stdlib=libc++"} + vals.update(arch_flags) def _verify_out(marker=">>"): if shared: diff --git a/test/functional/toolchains/cmake/test_cmake_multi.py b/test/functional/toolchains/cmake/test_cmake_multi.py index 0407765420b..bff9cd45f66 100644 --- a/test/functional/toolchains/cmake/test_cmake_multi.py +++ b/test/functional/toolchains/cmake/test_cmake_multi.py @@ -8,21 +8,19 @@ def test_multi_cmake(): conanfile = textwrap.dedent(""" from conan import ConanFile - from conan.tools.cmake import CMakeToolchain, CMake, cmake_layout, CMakeDeps + from conan.tools.cmake import CMakeToolchain, CMake, cmake_layout import os - class multiRecipe(ConanFile): settings = "os", "compiler", "build_type", "arch" - exports_sources = "cmake_one/CMakeLists.txt", "cmake_two/CMakeLists.txt", "src_one/*", "src_two/*" + exports_sources = ("cmake_one/CMakeLists.txt", "cmake_two/CMakeLists.txt", + "src_one/*", "src_two/*") def layout(self): cmake_layout(self) def generate(self): - deps = CMakeDeps(self) - deps.generate() tc = CMakeToolchain(self) tc.generate() @@ -30,11 +28,9 @@ def build_one(self): cmake = CMake(self) cmake.configure(build_script_folder="cmake_one", subfolder="one") cmake.build(subfolder="one") - # cmake.install(subfolder="one") def build_two(self): cmake = CMake(self) - # CMAKE_PREFIX_PATH cmake.configure(build_script_folder="cmake_two", subfolder="two") cmake.build(subfolder="two") @@ -62,12 +58,8 @@ def package_info(self): """) hello_h = textwrap.dedent(""" - #ifndef HELLO_{name}_H - #define HELLO_{name}_H - + #pragma once void hello_{name}(); - - #endif """) cmakelist = textwrap.dedent(""" @@ -91,13 +83,13 @@ def package_info(self): "src_two/hello_two.cpp": hello_cpp.format(name="two")}) client.run("create . --name=multi --version=0.1") - file_ext = '.a' if platform.system() != "Windows" else '.lib' - lib_prefix = 'lib' if platform.system() != "Windows" else '' + ext = '.a' if platform.system() != "Windows" else '.lib' + prefix = 'lib' if platform.system() != "Windows" else '' assert "multi/0.1: package(): Packaged 1 '.h' file: hello_two.h" in client.out - assert f"multi/0.1: package(): Packaged 1 '{file_ext}' file: {lib_prefix}hello_two{file_ext}" in client.out + assert f"multi/0.1: package(): Packaged 1 '{ext}' file: {prefix}hello_two{ext}" in client.out package_folder = client.created_layout().package() assert not os.path.exists(os.path.join(package_folder, "one", "include", "hello_one.h")) - assert not os.path.exists(os.path.join(package_folder, "one", "lib", f"{lib_prefix}hello_one{file_ext}")) + assert not os.path.exists(os.path.join(package_folder, "one", "lib", f"{prefix}hello_one{ext}")) assert os.path.exists(os.path.join(package_folder, "two", "include", "hello_two.h")) - assert os.path.exists(os.path.join(package_folder, "two", "lib", f"{lib_prefix}hello_two{file_ext}")) + assert os.path.exists(os.path.join(package_folder, "two", "lib", f"{prefix}hello_two{ext}")) diff --git a/test/functional/toolchains/cmake/test_cmake_transitive_rpath.py b/test/functional/toolchains/cmake/test_cmake_transitive_rpath.py index 4eb5381896a..f649cff6251 100644 --- a/test/functional/toolchains/cmake/test_cmake_transitive_rpath.py +++ b/test/functional/toolchains/cmake/test_cmake_transitive_rpath.py @@ -2,18 +2,16 @@ import textwrap import pytest -from conan.test.utils.mocks import ConanFileMock -from conan.tools.env.environment import environment_wrap_command -from conan.test.utils.test_files import temp_folder from conan.test.utils.tools import TestClient -@pytest.mark.skipif(platform.system() != "Linux", reason="Linux/gcc required for -rpath/-rpath-link testing") + +@pytest.mark.skipif(platform.system() != "Linux", + reason="Linux/gcc required for -rpath/-rpath-link testing") @pytest.mark.tool("cmake", "3.27") @pytest.mark.parametrize("use_cmake_config_deps", [True, False]) def test_cmake_sysroot_transitive_rpath(use_cmake_config_deps): c = TestClient() - extra_profile = textwrap.dedent(""" [conf] tools.build:sysroot=/path/to/nowhere @@ -78,8 +76,8 @@ def test_cmake_sysroot_transitive_rpath(use_cmake_config_deps): c.run(f"create . -o '*:shared=True' -pr=default -pr=../extra_profile {extra_conf}") - -@pytest.mark.skipif(platform.system() != "Linux", reason="Linux/gcc required for -rpath/-rpath-link testing") +@pytest.mark.skipif(platform.system() != "Linux", + reason="Linux/gcc required for -rpath/-rpath-link testing") @pytest.mark.tool("cmake", "3.27") @pytest.mark.parametrize("use_cmake_config_deps", [True, False]) def test_cmake_transitive_rpath_private_internal(use_cmake_config_deps): @@ -123,7 +121,8 @@ def test_cmake_transitive_rpath_private_internal(use_cmake_config_deps): install(TARGETS foo bar) """) - foobar_conanfile = textwrap.dedent(""" + cmake_deps_gen = "CMakeConfigDeps" if use_cmake_config_deps else "CMakeDeps" + foobar_conanfile = textwrap.dedent(f""" from conan import ConanFile from conan.tools.cmake import CMake, cmake_layout @@ -133,12 +132,12 @@ class foobarRecipe(ConanFile): version = "1.0" package_type = "library" settings = "os", "compiler", "build_type", "arch" - options = {"shared": [True, False]} - default_options = {"shared": True} + options = {{"shared": [True, False]}} + default_options = {{"shared": True}} exports_sources = "CMakeLists.txt", "src/*", "include/*" - generators = "CMakeDeps", "CMakeToolchain" + generators = "{cmake_deps_gen}", "CMakeToolchain" def layout(self): cmake_layout(self) @@ -158,7 +157,7 @@ def package_info(self): self.cpp_info.components["bar"].requires = ["foo"] """) - consumer_conanfile = textwrap.dedent(""" + consumer_conanfile = textwrap.dedent(f""" from conan import ConanFile from conan.tools.cmake import CMake, cmake_layout @@ -167,14 +166,14 @@ class consumerRecipe(ConanFile): version = "1.0" package_type = "library" settings = "os", "compiler", "build_type", "arch" - options = {"shared": [True, False]} - default_options = {"shared": True} - generators = "CMakeDeps", "CMakeToolchain" + options = {{"shared": [True, False]}} + default_options = {{"shared": True}} + generators = "{cmake_deps_gen}", "CMakeToolchain" exports_sources = "CMakeLists.txt", "src/*", "include/*" def layout(self): cmake_layout(self) - + def requirements(self): self.requires("foobar/1.0") @@ -223,8 +222,7 @@ def package_info(self): int main() { return consumer(2, 3) == 20 ? 0 : 1; } """) - extra_conf = "-c tools.cmake.cmakedeps:new=will_break_next" if use_cmake_config_deps else "" - extra_conf += " -c tools.build:add_rpath_link=True" # removing this should break the test + extra_conf = "-c tools.build:add_rpath_link=True" # removing this should break the test with c.chdir("foobar"): c.save({"include/foo.h": foo_h, diff --git a/test/functional/toolchains/env/test_virtualenv_powershell.py b/test/functional/toolchains/env/test_virtualenv_powershell.py index 25119d30501..15d192a53ee 100644 --- a/test/functional/toolchains/env/test_virtualenv_powershell.py +++ b/test/functional/toolchains/env/test_virtualenv_powershell.py @@ -43,7 +43,6 @@ class ConanFileToolsTest(ConanFile): name = "app" version = "0.1" requires = "pkg/0.1" - apply_env = False def build(self): self.output.info("----------BUILD----------------") diff --git a/test/functional/toolchains/scons/test_sconsdeps.py b/test/functional/toolchains/scons/test_sconsdeps.py index f7d9760fdd3..e30f64fc013 100644 --- a/test/functional/toolchains/scons/test_sconsdeps.py +++ b/test/functional/toolchains/scons/test_sconsdeps.py @@ -136,8 +136,6 @@ def package_info(self): class helloTestConan(ConanFile): settings = "os", "compiler", "build_type", "arch" generators = "SConsDeps" - apply_env = False - test_type = "explicit" def requirements(self): self.requires(self.tested_reference_str) diff --git a/test/functional/toolchains/test_premake.py b/test/functional/toolchains/test_premake.py index 85921c6eb9a..e9022008090 100644 --- a/test/functional/toolchains/test_premake.py +++ b/test/functional/toolchains/test_premake.py @@ -11,7 +11,7 @@ from conan.test.assets.sources import gen_function_cpp -@pytest.mark.skipif(platform.machine() != "x86_64", +@pytest.mark.skipif(platform.machine() not in ("x86_64", "AMD64"), reason="Premake Legacy generator only supports x86_64 machines") @pytest.mark.tool("premake") def test_premake_legacy(matrix_client): @@ -71,10 +71,7 @@ def build(self): c.run("build .") assert "main: Release!" in c.out assert "matrix/1.0: Hello World Release!" in c.out - if platform.system() == "Windows": - assert "main _M_X64 defined" in c.out - else: - assert "main __x86_64__ defined" in c.out + assert "main _M_X64 defined" in c.out or "main __x86_64__ defined" in c.out c.run("build . -s build_type=Debug --build=missing") assert "main: Debug!" in c.out assert "matrix/1.0: Hello World Debug!" in c.out @@ -257,6 +254,7 @@ def build(self): c.run("build .", assert_error=True) # Error should be about not finding matrix + @pytest.mark.tool("premake") def test_premake_custom_configuration(transitive_libraries): c = transitive_libraries @@ -307,7 +305,7 @@ def build(self): ], configurations=["Debug", "Release", "DebugSanitizer", "ReleaseSanitizer"], ), - "conanfile.py": conanfile, - }) + "conanfile.py": conanfile + }) # Test it builds successfully with the custom configuration c.run("build . -o sanitizer=True") diff --git a/test/integration/build_requires/build_requires_test.py b/test/integration/build_requires/build_requires_test.py index fbf394fdde6..36c54e177fe 100644 --- a/test/integration/build_requires/build_requires_test.py +++ b/test/integration/build_requires/build_requires_test.py @@ -464,8 +464,6 @@ def build_requirements(self): test = textwrap.dedent(""" from conan import ConanFile class Test(ConanFile): - test_type = "explicit" - def build_requirements(self): self.tool_requires(self.tested_reference_str) def test(self): diff --git a/test/integration/cache/test_home_special_char.py b/test/integration/cache/test_home_special_char.py index 0ec98ebcba7..b6234c304cd 100644 --- a/test/integration/cache/test_home_special_char.py +++ b/test/integration/cache/test_home_special_char.py @@ -46,7 +46,6 @@ class App(ConanFile): settings = 'os', 'arch', 'compiler', 'build_type' generators = "VirtualBuildEnv" tool_requires = "mytool/1.0" - apply_env = False # SUPER IMPORTANT, DO NOT REMOVE def build(self): mycmd = "mytool.bat" if platform.system() == "Windows" else "mytool.sh" diff --git a/test/integration/command/test_package_test.py b/test/integration/command/test_package_test.py index b9b3ea7a8d9..a68cf3b1b43 100644 --- a/test/integration/command/test_package_test.py +++ b/test/integration/command/test_package_test.py @@ -323,13 +323,6 @@ def test_tested_reference_str(): """ At the test_package/conanfile the variable `self.tested_reference_str` is injected with the str of the reference being tested. It is available in all the methods. - - Compatibility with Conan 2.0: - If the 'test_type' is set to "explicit" the require won't be automatically injected and has to - be the user the one injecting the require or the build require using the - `self.tested_reference_str`. This 'test_type' can be removed in 2.0 if we consider it has - to be always explicit. The recipes will still work in Conan 2.0 because the 'test_type' will be - ignored. """ client = TestClient() test_conanfile = textwrap.dedent(""" diff --git a/test/integration/editable/editable_add_test.py b/test/integration/editable/editable_add_test.py index cb418090c27..0c6225ae95e 100644 --- a/test/integration/editable/editable_add_test.py +++ b/test/integration/editable/editable_add_test.py @@ -50,7 +50,6 @@ def test_editable_no_name_version_test_package(): tc = TestClient() tc.save({"conanfile.py": GenConanfile(), "test_package/conanfile.py": GenConanfile("test_package") - .with_class_attribute("test_type = 'explicit'") .with_test("self.output.info('Testing the package')")}) tc.run("editable add . --name=foo", assert_error=True) assert "ERROR: Editable package recipe should declare its name and version" in tc.out diff --git a/test/integration/environment/test_env.py b/test/integration/environment/test_env.py index 03bc0f2ea65..8a3aa3aedab 100644 --- a/test/integration/environment/test_env.py +++ b/test/integration/environment/test_env.py @@ -895,7 +895,6 @@ def package_info(self): from conan import ConanFile class TestTool(ConanFile): settings = "build_type" - test_type = "explicit" generators = "VirtualBuildEnv" def build_requirements(self): self.tool_requires(self.tested_reference_str) diff --git a/test/integration/graph/test_replace_requires.py b/test/integration/graph/test_replace_requires.py index 3197ba2bf95..d4499254f10 100644 --- a/test/integration/graph/test_replace_requires.py +++ b/test/integration/graph/test_replace_requires.py @@ -425,7 +425,7 @@ class App(ConanFile): settings = "build_type", "arch" requires = "openssl/0.1", {zlib} package_type = "application" - generators = "CMakeDeps", "PkgConfigDeps", "MSBuildDeps" + generators = "CMakeConfigDeps", "PkgConfigDeps", "MSBuildDeps" """) profile = textwrap.dedent(""" [settings] @@ -442,7 +442,7 @@ class App(ConanFile): c.run("create zlibng") c.run("create openssl -pr=profile") - c.run("install app -pr=profile -c tools.cmake.cmakedeps:new=will_break_next") + c.run("install app -pr=profile") assert "zlib/0.1: zlib-ng/0.1" in c.out pc_content = c.load("app/ZLIB.pc") @@ -505,7 +505,7 @@ class App(ConanFile): settings = "build_type", "arch" requires = "openssl/0.1", {zlib} package_type = "application" - generators = "CMakeDeps", "PkgConfigDeps", "MSBuildDeps" + generators = "CMakeConfigDeps", "PkgConfigDeps", "MSBuildDeps" """) profile = textwrap.dedent(""" [settings] @@ -522,7 +522,7 @@ class App(ConanFile): c.run("create zlibng") c.run("create openssl -pr=profile") - c.run("install app -pr=profile -c tools.cmake.cmakedeps:new=will_break_next") + c.run("install app -pr=profile") assert "zlib/0.1: zlib-ng/0.1" in c.out pc_content = c.load("app/ZLIB.pc") @@ -589,7 +589,7 @@ class App(ConanFile): settings = "build_type", "arch" requires = "openssl/0.1", {zlib} package_type = "application" - generators = "CMakeDeps", "PkgConfigDeps", "MSBuildDeps" + generators = "CMakeConfigDeps", "PkgConfigDeps", "MSBuildDeps" """) profile = textwrap.dedent(""" [settings] @@ -606,7 +606,7 @@ class App(ConanFile): c.run("create zlibng") c.run("create openssl -pr=profile") - c.run("install app -pr=profile -c tools.cmake.cmakedeps:new=will_break_next") + c.run("install app -pr=profile") assert "zlib/0.1: zlib-ng/0.1" in c.out pc_content = c.load("app/zlib-ng.pc") @@ -679,7 +679,7 @@ class App(ConanFile): settings = "build_type", "arch" requires = "openssl/0.1", {zlib} package_type = "application" - generators = "CMakeDeps", "PkgConfigDeps", "MSBuildDeps" + generators = "CMakeConfigDeps", "PkgConfigDeps", "MSBuildDeps" """) profile = textwrap.dedent(""" [settings] @@ -696,7 +696,7 @@ class App(ConanFile): c.run("create zlibng") c.run("create openssl -pr=profile") - c.run("install app -pr=profile -c tools.cmake.cmakedeps:new=will_break_next") + c.run("install app -pr=profile") assert "zlib/0.1: zlib-ng/0.1" in c.out pc_content = c.load("app/zlib-ng.pc") diff --git a/test/integration/lockfile/test_lock_requires.py b/test/integration/lockfile/test_lock_requires.py index 1fb8ff353de..6df9bcc119f 100644 --- a/test/integration/lockfile/test_lock_requires.py +++ b/test/integration/lockfile/test_lock_requires.py @@ -665,7 +665,7 @@ def test_conanfile_txt_deps_ranges(self, requires): def test_error_test_explicit(): # https://github.com/conan-io/conan/issues/14833 client = TestClient(light=True) - test = GenConanfile().with_test("pass").with_class_attribute("test_type = 'explicit'") + test = GenConanfile().with_test("pass") client.save({"conanfile.py": GenConanfile("pkg", "0.1"), "test_package/conanfile.py": test}) client.run("lock create conanfile.py --lockfile-out=my.lock") @@ -718,7 +718,6 @@ def requirements(self): class TestPackageConan(ConanFile): settings = "build_type" - test_type = "explicit" def requirements(self): self.requires(self.tested_reference_str) diff --git a/test/integration/package_id/package_id_requires_modes_test.py b/test/integration/package_id/package_id_requires_modes_test.py index d01fa89f657..dbffa8ad608 100644 --- a/test/integration/package_id/package_id_requires_modes_test.py +++ b/test/integration/package_id/package_id_requires_modes_test.py @@ -41,6 +41,8 @@ def test(self, mode, accepted_version, rejected_version, pattern): ("major_mode", "6ac597ffb99c3747ed78699f206dc1041537a8df"), # This is equal to semver_mode for 0.X.Y.Z.. ("full_version_mode", "13b9e753af3958dd1b2d4b3f935b04b8fb6b6760"), + ("full_recipe_mode", "13b9e753af3958dd1b2d4b3f935b04b8fb6b6760"), + ("full_package_mode", "19906d8a245d9f466d7d7f697c666222a9854a1a"), ("revision_mode", "ae5b9eeb74880aeb1cfa3db7f84c007a05ce3a76"), ("full_mode", "d1b2a9538cd69363b4bae7e66c9f900b8f4c58bb")]) def test_modes(mode, pkg_id): diff --git a/test/integration/package_id/python_requires_package_id_test.py b/test/integration/package_id/python_requires_package_id_test.py index 0ad771ae905..0bfabef1fc3 100644 --- a/test/integration/package_id/python_requires_package_id_test.py +++ b/test/integration/package_id/python_requires_package_id_test.py @@ -167,6 +167,9 @@ class Pkg(ConanFile): ("minor_mode", "c53bd9e48dd09ceeaa1bb425830490d8b243e39c"), ("major_mode", "331c17383dcdf37f79bc2b86fa55ac56afdc6fec"), ("full_version_mode", "1f0070b00ccebfec93dc90854a163c7af229f587"), + ("full_recipe_mode", "1f0070b00ccebfec93dc90854a163c7af229f587"), + # Doesn't make much sense, but was doable, not worth removing it + ("full_package_mode", "1f0070b00ccebfec93dc90854a163c7af229f587"), ("revision_mode", "0071dd0296afa0db533e21f924273485c87f0d32"), ("full_mode", "0071dd0296afa0db533e21f924273485c87f0d32")]) def test_modes(mode, pkg_id): @@ -178,3 +181,32 @@ def test_modes(mode, pkg_id): c.run("create pkg") pkgid = c.created_package_id("pkg/0.1") assert pkgid == pkg_id + + +@pytest.mark.parametrize("mode, pkg_id", + [("unrelated_mode", "da39a3ee5e6b4b0d3255bfef95601890afd80709"), + ("semver_mode", "1f0070b00ccebfec93dc90854a163c7af229f587"), + ("patch_mode", "b9ca872dfd5b48f5f1f69d66f0950fc35469d0cd"), + ("minor_mode", "c53bd9e48dd09ceeaa1bb425830490d8b243e39c"), + ("major_mode", "331c17383dcdf37f79bc2b86fa55ac56afdc6fec"), + ("full_version_mode", "1f0070b00ccebfec93dc90854a163c7af229f587"), + ("full_recipe_mode", "1f0070b00ccebfec93dc90854a163c7af229f587"), + ("revision_mode", "0071dd0296afa0db533e21f924273485c87f0d32"), + ("full_mode", "0071dd0296afa0db533e21f924273485c87f0d32")]) +def test_modes_recipe(mode, pkg_id): + c = TestClient(light=True) + + conanfile = textwrap.dedent(f""" + from conan import ConanFile + class Pkg(ConanFile): + name = "pkg" + version = "0.1" + python_requires ="dep/[*]" + def package_id(self): + self.info.python_requires.{mode}() + """) + c.save({"dep/conanfile.py": GenConanfile("dep", "0.1.1.1"), + "pkg/conanfile.py": conanfile}) + c.run("create dep") + c.run("create pkg") + c.assert_listed_binary({"pkg/0.1": (pkg_id, "Build")}) diff --git a/test/integration/remote/server_error_test.py b/test/integration/remote/server_error_test.py index e39117a6097..b7a093e9ae6 100644 --- a/test/integration/remote/server_error_test.py +++ b/test/integration/remote/server_error_test.py @@ -1,3 +1,5 @@ +from requests import Response + from conan.internal import REVISIONS from conan.test.utils.tools import TestClient, TestServer, TestRequester from collections import namedtuple @@ -54,3 +56,17 @@ def get(self, *args, **kwargs): # @UnusedVariable client.run("install --requires=pkg/ref@user/testing", assert_error=True) assert "Unexpected server response [1, 2, 3]" in client.out + + +def test_unrecongized_exception(): + class BuggyRequester(TestRequester): + def get(self, *args, **kwargs): + resp = Response() + resp.status_code = 444 + resp._content = 'some 444 error message' + return resp + + c = TestClient(default_server_user=True, requester_class=BuggyRequester) + c.run("install --requires=zlib/1.2 -r=default", assert_error=True) + assert ("ERROR: Package 'zlib/1.2' not resolved: Server exception 444:" + " some 444 error message") in c.out diff --git a/test/integration/sbom/test_cyclonedx.py b/test/integration/sbom/test_cyclonedx.py index d29744b110d..220a3cbc8aa 100644 --- a/test/integration/sbom/test_cyclonedx.py +++ b/test/integration/sbom/test_cyclonedx.py @@ -88,16 +88,16 @@ def test_sbom_generation_skipped_dependencies(self, hook_setup_post_package): # A skipped dependency also shows up in the sbom assert "pkg:conan/dep@1.0?rref=6a99f55e933fb6feeb96df134c33af44" in content - @pytest.mark.parametrize("l, n", [('"simple"', 1), ('"multi1", "multi2"', 2), - ('("tuple1", "tuple2")', 2)]) - def test_multi_license(self, hook_setup_post_package, l, n): + @pytest.mark.parametrize("lic, n", [('"simple"', 1), ('"multi1", "multi2"', 2), + ('("tuple1", "tuple2")', 2)]) + def test_multi_license(self, hook_setup_post_package, lic, n): tc = hook_setup_post_package conanfile = textwrap.dedent(f""" from conan import ConanFile class HelloConan(ConanFile): name = 'foo' version = '1.0' - license = {l} + license = {lic} """) tc.save({"conanfile.py": conanfile}) tc.run("create .") @@ -106,16 +106,16 @@ class HelloConan(ConanFile): content = json.loads(tc.load(cyclone_path)) assert len(content["components"][0]["licenses"]) == n - @pytest.mark.parametrize("l, keys", [('"Mit"', ["id"]), ('"custom_license name"', ["name"]), - ('("mIT", "custom")', ["id", "name"])]) - def test_license_spdx_valid(self, hook_setup_post_package, l, keys): + @pytest.mark.parametrize("lic, keys", [('"Mit"', ["id"]), ('"custom_license name"', ["name"]), + ('("mIT", "custom")', ["id", "name"])]) + def test_license_spdx_valid(self, hook_setup_post_package, lic, keys): tc = hook_setup_post_package conanfile = textwrap.dedent(f""" from conan import ConanFile class HelloConan(ConanFile): name = 'foo' version = '1.0' - license = {l} + license = {lic} """) tc.save({"conanfile.py": conanfile}) tc.run("create .") diff --git a/test/integration/workspace/test_workspace.py b/test/integration/workspace/test_workspace.py index edbeaa6aea0..3bbf6955a34 100644 --- a/test/integration/workspace/test_workspace.py +++ b/test/integration/workspace/test_workspace.py @@ -581,7 +581,8 @@ def packages(self): result = [] for f in os.listdir(self.folder): if os.path.isdir(os.path.join(self.folder, f)): - result.append({"path": f, "ref": f"{f}/0.1"}) + result.append({"path": f, "ref": f"{f}/0.1", + "output_folder": f"{f}/myout"}) return result """) c = TestClient(light=True) @@ -590,6 +591,7 @@ def packages(self): c.run("workspace info") assert "pkga/0.1" in c.out c.run("workspace build") + assert "metadata" in os.listdir(os.path.join(c.current_folder, "pkga", "myout")) assert "conanfile.py (pkga/0.1): Building pkga AND 0.1!!!" in c.out def test_build_with_external_editable_python_requires(self): diff --git a/test/unittests/tools/system/python_manager_test.py b/test/unittests/tools/system/python_manager_test.py index 7b969887171..7c52d081b65 100644 --- a/test/unittests/tools/system/python_manager_test.py +++ b/test/unittests/tools/system/python_manager_test.py @@ -12,8 +12,13 @@ def test_pyenv_conf(mock_shutil_which): conanfile.settings = Settings() conanfile.conf.define("tools.system.pyenv:python_interpreter", "/python/interpreter/from/config") + + def fake_run(command, win_bash=False, subsystem=None, env=None, ignore_errors=False, # noqa + quiet=False): # noqa + assert "/python/interpreter/from/config" in command + + conanfile.run = fake_run PyEnv(conanfile, "testenv") - conanfile.run = None mock_shutil_which.assert_not_called() @@ -23,8 +28,13 @@ def test_pyenv_deprecated_conf(mock_shutil_which): conanfile.settings = Settings() conanfile.conf.define("tools.system.pipenv:python_interpreter", "/python/interpreter/from/config") + + def fake_run(command, win_bash=False, subsystem=None, env=None, ignore_errors=False, # noqa + quiet=False): # noqa + assert "/python/interpreter/from/config" in command + + conanfile.run = fake_run PyEnv(conanfile, "testenv") - conanfile.run = None mock_shutil_which.assert_not_called() From c5e278feee15760a76dcdac320e24f90a2ed226a Mon Sep 17 00:00:00 2001 From: James Date: Tue, 10 Mar 2026 10:00:36 +0100 Subject: [PATCH 044/110] Tests/slow (#19720) * wip * conditional on comment * fix * failing test * wip * wip * more slows --------- Co-authored-by: Carlos Zoido --- .github/workflows/linux-tests.yml | 6 ++++- .github/workflows/main.yml | 26 +++++++++++++++++++ .github/workflows/osx-tests.yml | 6 ++++- .github/workflows/win-tests.yml | 6 ++++- pytest.ini | 1 + .../toolchains/android/test_using_cmake.py | 1 + .../toolchains/emscripten/test_emcc.py | 4 +++ .../toolchains/gnu/autotools/test_win_bash.py | 1 + .../gnu/test_v2_autotools_template.py | 1 + .../toolchains/google/test_bazel.py | 7 ++++- .../test_bazeltoolchain_cross_compilation.py | 1 + .../toolchains/ios/test_using_cmake.py | 1 + .../toolchains/test_nmake_toolchain.py | 2 ++ test/functional/toolchains/test_premake.py | 6 +++++ 14 files changed, 65 insertions(+), 4 deletions(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 95b9fd36366..a95669f2300 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -6,6 +6,10 @@ on: python-versions: required: true type: string + run-slow-tests: + required: false + default: 'false' + type: string jobs: build_container: @@ -91,7 +95,7 @@ jobs: with: python-version: ${{ matrix.python-version }} test-type: ${{ matrix.test-type }} - tests: test/${{ matrix.test-type }} + tests: ${{ matrix.test-type == 'functional' && (inputs.run-slow-tests == 'true' && 'test/functional' || 'test/functional -m "not slow"') || format('test/{0}', matrix.test-type) }} linux_runner_tests: needs: build_container diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1e8abfe2e16..811d24649fe 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,8 +35,22 @@ jobs: outputs: python_versions_linux_windows: ${{ steps.set_versions.outputs.python_versions_linux_windows }} python_versions_macos: ${{ steps.set_versions.outputs.python_versions_macos }} + run_slow_tests: ${{ steps.set_versions.outputs.run_slow_tests }} name: Determine Python versions steps: + - name: Fetch current PR body and check for slow-tests marker + if: github.event_name == 'pull_request' + id: fetch_pr + run: | + BODY=$(curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}" \ + | jq -r '.body // ""') + if echo "$BODY" | grep -Fq '$runslowtests'; then + echo "run_slow_tests=true" >> $GITHUB_OUTPUT + else + echo "run_slow_tests=false" >> $GITHUB_OUTPUT + fi + - name: Determine Python versions id: set_versions run: | @@ -48,12 +62,22 @@ jobs: echo "python_versions_macos=['3.10']" >> $GITHUB_OUTPUT fi + # Run slow tests on develop2, or when PR description contains $runslowtests (checked in fetch_pr step) + if [[ "${{ github.ref }}" == "refs/heads/develop2" ]]; then + echo "run_slow_tests=true" >> $GITHUB_OUTPUT + elif [[ "${{ github.event_name }}" == "pull_request" ]]; then + echo "run_slow_tests=${{ steps.fetch_pr.outputs.run_slow_tests }}" >> $GITHUB_OUTPUT + else + echo "run_slow_tests=false" >> $GITHUB_OUTPUT + fi + linux_suite: needs: [ensure_latest_tag_merged, set_python_versions] uses: ./.github/workflows/linux-tests.yml name: Linux test suite with: python-versions: ${{ needs.set_python_versions.outputs.python_versions_linux_windows }} + run-slow-tests: ${{ needs.set_python_versions.outputs.run_slow_tests }} osx_suite: needs: [ensure_latest_tag_merged, set_python_versions] @@ -61,6 +85,7 @@ jobs: name: OSX test suite with: python-versions: ${{ needs.set_python_versions.outputs.python_versions_macos }} + run-slow-tests: ${{ needs.set_python_versions.outputs.run_slow_tests }} windows_suite: needs: [ensure_latest_tag_merged, set_python_versions] @@ -68,6 +93,7 @@ jobs: name: Windows test suite with: python-versions: ${{ needs.set_python_versions.outputs.python_versions_linux_windows }} + run-slow-tests: ${{ needs.set_python_versions.outputs.run_slow_tests }} code_coverage: runs-on: ubuntu-latest diff --git a/.github/workflows/osx-tests.yml b/.github/workflows/osx-tests.yml index b98cb026f2b..e44c3c90eba 100644 --- a/.github/workflows/osx-tests.yml +++ b/.github/workflows/osx-tests.yml @@ -6,6 +6,10 @@ on: python-versions: required: true type: string + run-slow-tests: + required: false + default: 'false' + type: string jobs: osx_setup: @@ -177,4 +181,4 @@ jobs: with: python-version: ${{ matrix.python-version }} test-type: ${{ matrix.test-type }} - tests: test/${{ matrix.test-type }} + tests: ${{ matrix.test-type == 'functional' && (inputs.run-slow-tests == 'true' && 'test/functional' || 'test/functional -m "not slow"') || format('test/{0}', matrix.test-type) }} diff --git a/.github/workflows/win-tests.yml b/.github/workflows/win-tests.yml index 0cf044b2522..9b76a55cdba 100644 --- a/.github/workflows/win-tests.yml +++ b/.github/workflows/win-tests.yml @@ -6,6 +6,10 @@ on: python-versions: required: true type: string + run-slow-tests: + required: false + default: 'false' + type: string jobs: unit_integration_tests: @@ -255,4 +259,4 @@ jobs: with: python-version: ${{ matrix.python-version }} test-type: functional - tests: test/functional + tests: ${{ inputs.run-slow-tests == 'true' && 'test/functional' || 'test/functional -m "not slow"' }} diff --git a/pytest.ini b/pytest.ini index bf8bb6be5d0..275c77374c0 100644 --- a/pytest.ini +++ b/pytest.ini @@ -4,5 +4,6 @@ testpaths = 'test' markers = docker_runner: Mark tests that require Docker to run. artifactory_ready: These tests can be run against a full Artifactory + slow: These tests can executed more occasionally, slow, but unlikely to fail in normal PRs filterwarnings = ignore:'cgi' is deprecated:DeprecationWarning diff --git a/test/functional/toolchains/android/test_using_cmake.py b/test/functional/toolchains/android/test_using_cmake.py index 3b09d62c827..bf650e1289f 100644 --- a/test/functional/toolchains/android/test_using_cmake.py +++ b/test/functional/toolchains/android/test_using_cmake.py @@ -8,6 +8,7 @@ from conan.test.utils.tools import TestClient +@pytest.mark.slow @pytest.mark.tool("cmake", "3.23") # Android complains if <3.19 @pytest.mark.tool("ninja") # so it easily works in Windows too @pytest.mark.tool("android_ndk") diff --git a/test/functional/toolchains/emscripten/test_emcc.py b/test/functional/toolchains/emscripten/test_emcc.py index f68646fdc54..62402d53cc0 100644 --- a/test/functional/toolchains/emscripten/test_emcc.py +++ b/test/functional/toolchains/emscripten/test_emcc.py @@ -79,6 +79,7 @@ ) +@pytest.mark.slow @pytest.mark.tool("cmake") @pytest.mark.tool("emcc") @pytest.mark.tool("node") @@ -106,6 +107,7 @@ def test_cmake_emscripten(): assert "Hello World Release!" in client.out +@pytest.mark.slow @pytest.mark.tool("meson") @pytest.mark.tool("emcc") @pytest.mark.tool("node") @@ -138,6 +140,7 @@ def test_meson_emscripten(): assert "Hello World Release!" in client.out +@pytest.mark.slow @pytest.mark.tool("autotools") @pytest.mark.tool("emcc") @pytest.mark.tool("node") @@ -168,6 +171,7 @@ def test_autotools_emscripten(): assert "Hello World Release!" in client.out +@pytest.mark.slow @pytest.mark.tool("premake") @pytest.mark.tool("emcc") @pytest.mark.tool("node") diff --git a/test/functional/toolchains/gnu/autotools/test_win_bash.py b/test/functional/toolchains/gnu/autotools/test_win_bash.py index 455acc79b3e..e98059317a4 100644 --- a/test/functional/toolchains/gnu/autotools/test_win_bash.py +++ b/test/functional/toolchains/gnu/autotools/test_win_bash.py @@ -64,6 +64,7 @@ def build(self): assert "conanvcvars.bat" in bat_contents +@pytest.mark.slow @pytest.mark.skipif(platform.system() != "Windows", reason="Requires Windows") @pytest.mark.tool("msys2") @pytest.mark.tool("clang", "20") diff --git a/test/functional/toolchains/gnu/test_v2_autotools_template.py b/test/functional/toolchains/gnu/test_v2_autotools_template.py index 0f6c3092451..e2db1fc6be3 100644 --- a/test/functional/toolchains/gnu/test_v2_autotools_template.py +++ b/test/functional/toolchains/gnu/test_v2_autotools_template.py @@ -347,6 +347,7 @@ def package_info(self): assert "Bye, bye!" in client.out +@pytest.mark.slow @pytest.mark.skipif(platform.system() != "Windows", reason="Using msys2") @pytest.mark.tool("msys2") class TestAutotoolsTemplateWindows: diff --git a/test/functional/toolchains/google/test_bazel.py b/test/functional/toolchains/google/test_bazel.py index c9d28c47220..e349a5b77ba 100644 --- a/test/functional/toolchains/google/test_bazel.py +++ b/test/functional/toolchains/google/test_bazel.py @@ -38,6 +38,7 @@ def base_profile(): """) +@pytest.mark.slow @pytest.mark.parametrize("build_type", ["Debug", "Release", "RelWithDebInfo", "MinSizeRel"]) @pytest.mark.tool("bazel", "6.x") def test_basic_exe_6x(bazelrc, build_type, base_profile, bazel_output_root_dir): @@ -56,6 +57,7 @@ def test_basic_exe_6x(bazelrc, build_type, base_profile, bazel_output_root_dir): assert "myapp/1.0: Hello World Debug!" in client.out +@pytest.mark.slow @pytest.mark.parametrize("build_type", ["Debug", "Release", "RelWithDebInfo", "MinSizeRel"]) @pytest.mark.tool("bazel", "7.x") def test_basic_exe(bazelrc, build_type, base_profile, bazel_output_root_dir): @@ -74,6 +76,7 @@ def test_basic_exe(bazelrc, build_type, base_profile, bazel_output_root_dir): assert "myapp/1.0: Hello World Debug!" in client.out +@pytest.mark.slow @pytest.mark.tool("bazel", "8.x") def test_basic_lib(bazelrc, base_profile, bazel_output_root_dir): """ @@ -84,7 +87,7 @@ def test_basic_lib(bazelrc, base_profile, bazel_output_root_dir): client.run("create .") assert "mylib/1.0: Hello World Release!" in client.out - +@pytest.mark.slow @pytest.mark.parametrize("shared", [False, True]) @pytest.mark.tool("bazel", "6.x") def test_transitive_libs_consuming_6x(shared, bazel_output_root_dir): @@ -211,6 +214,7 @@ def test_transitive_libs_consuming_6x(shared, bazel_output_root_dir): assert "myfirstlib/1.2.11: Hello World Release!" +@pytest.mark.slow @pytest.mark.parametrize("shared", [False, True]) @pytest.mark.tool("bazel", "7.x") @pytest.mark.skipif(platform.system() == "Linux", @@ -344,6 +348,7 @@ def test_transitive_libs_consuming_7x(shared, bazel_output_root_dir): assert "myfirstlib/1.2.11: Hello World Release!" +@pytest.mark.slow @pytest.mark.tool("bazel", "8.x") def test_empty_bazel_query(): """ diff --git a/test/functional/toolchains/google/test_bazeltoolchain_cross_compilation.py b/test/functional/toolchains/google/test_bazeltoolchain_cross_compilation.py index e848e82e5be..95feba6363e 100644 --- a/test/functional/toolchains/google/test_bazeltoolchain_cross_compilation.py +++ b/test/functional/toolchains/google/test_bazeltoolchain_cross_compilation.py @@ -9,6 +9,7 @@ from conan.test.utils.tools import TestClient +@pytest.mark.slow @pytest.mark.skipif(platform.system() != "Darwin", reason="Only for Darwin") @pytest.mark.tool("bazel", "6.x") # not working for Bazel 7.x def test_bazel_simple_cross_compilation(): diff --git a/test/functional/toolchains/ios/test_using_cmake.py b/test/functional/toolchains/ios/test_using_cmake.py index 203d85f43c8..de78d4601f9 100644 --- a/test/functional/toolchains/ios/test_using_cmake.py +++ b/test/functional/toolchains/ios/test_using_cmake.py @@ -8,6 +8,7 @@ from ._utils import create_library +@pytest.mark.slow @pytest.mark.skipif(platform.system() != "Darwin", reason="Requires XCode") @pytest.mark.tool("cmake", "3.19") def test_xcode_ios_generator(): diff --git a/test/functional/toolchains/test_nmake_toolchain.py b/test/functional/toolchains/test_nmake_toolchain.py index b1267378c57..3d4e32acadc 100644 --- a/test/functional/toolchains/test_nmake_toolchain.py +++ b/test/functional/toolchains/test_nmake_toolchain.py @@ -8,6 +8,7 @@ from conan.test.utils.tools import TestClient +@pytest.mark.slow @pytest.mark.parametrize( "compiler, version, runtime, cppstd, build_type, defines, cflags, cxxflags, sharedlinkflags, exelinkflags", [ @@ -89,6 +90,7 @@ def build(self): conf_preprocessors) +@pytest.mark.slow @pytest.mark.skipif(platform.system() != "Windows", reason="Requires Windows") @pytest.mark.tool("cmake", "3.23") # This test uses clang inside Visual Studio, not managed by mark.tool diff --git a/test/functional/toolchains/test_premake.py b/test/functional/toolchains/test_premake.py index e9022008090..8f8155fe455 100644 --- a/test/functional/toolchains/test_premake.py +++ b/test/functional/toolchains/test_premake.py @@ -11,6 +11,7 @@ from conan.test.assets.sources import gen_function_cpp +@pytest.mark.slow @pytest.mark.skipif(platform.machine() not in ("x86_64", "AMD64"), reason="Premake Legacy generator only supports x86_64 machines") @pytest.mark.tool("premake") @@ -77,6 +78,7 @@ def build(self): assert "matrix/1.0: Hello World Debug!" in c.out +@pytest.mark.slow @pytest.mark.tool("premake") def test_premake_new_generator(): c = TestClient() @@ -90,6 +92,7 @@ def test_premake_new_generator(): assert "example/1.0: Hello World Release!" in c.out +@pytest.mark.slow @pytest.mark.tool("premake") def test_premake_shared_lib(): c = TestClient() @@ -99,6 +102,7 @@ def test_premake_shared_lib(): assert "lib/0.1: package(): Packaged 1 '.a' file: liblib.a" not in c.out +@pytest.mark.slow @pytest.mark.tool("premake") @pytest.mark.parametrize("transitive_libs", [True, False]) def test_premake_components(transitive_libs): @@ -202,6 +206,7 @@ def package_info(self): c.run("build consumer", assert_error=not transitive_libs) +@pytest.mark.slow @pytest.mark.tool("premake") def test_transitive_headers_not_public(transitive_libraries): c = transitive_libraries @@ -255,6 +260,7 @@ def build(self): # Error should be about not finding matrix +@pytest.mark.slow @pytest.mark.tool("premake") def test_premake_custom_configuration(transitive_libraries): c = transitive_libraries From aa4addca641ca8408ae1ac024b3945a94cd168f1 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 11 Mar 2026 07:31:36 +0100 Subject: [PATCH 045/110] Fix/source creds msg (#19737) * improve source creds msg * improve source creds msg --- conan/api/subapi/upload.py | 4 ++-- conan/internal/rest/caching_file_downloader.py | 6 +++--- test/integration/cache/backup_sources_test.py | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/conan/api/subapi/upload.py b/conan/api/subapi/upload.py index 752d8c68c77..53eeb38d7fc 100644 --- a/conan/api/subapi/upload.py +++ b/conan/api/subapi/upload.py @@ -183,7 +183,7 @@ def upload_backup_sources(self, files: List) -> None: "Skipping updating file but continuing with upload. " f"Missing permissions?: {e}") else: - raise ConanException(f"The source backup server '{url}' needs authentication" - f"/permissions, please provide 'source_credentials.json': {e}") + raise ConanException(f"Authentication to source backup server '{url}' failed, " + f"please check your 'source_credentials.json': {e}") output.success("Upload backup sources complete\n") diff --git a/conan/internal/rest/caching_file_downloader.py b/conan/internal/rest/caching_file_downloader.py index b2b50a8522b..297a0698d21 100644 --- a/conan/internal/rest/caching_file_downloader.py +++ b/conan/internal/rest/caching_file_downloader.py @@ -121,9 +121,9 @@ def _backup_download(self, backup_url, backups_urls, sha256, cached_path, urls, else: self._output.warning(f"File {urls} not found in {backup_url}") except (AuthenticationException, ForbiddenException) as e: - raise ConanException(f"The source backup server '{backup_url}' " - f"needs authentication: {e}. " - f"Please provide 'source_credentials.json'") + raise ConanException(f"Authentication to source backup server '{backup_url}' " + f"failed: {e}. " + f"Please check your 'source_credentials.json'") def _download_from_urls(self, urls, file_path, retry, retry_wait, verify_ssl, auth, headers, md5, sha1, sha256): diff --git a/test/integration/cache/backup_sources_test.py b/test/integration/cache/backup_sources_test.py index c0b6f66e44c..c00c7de750f 100644 --- a/test/integration/cache/backup_sources_test.py +++ b/test/integration/cache/backup_sources_test.py @@ -414,8 +414,8 @@ def source(self): client.save({"conanfile.py": conanfile}) client.run("create .", assert_error=True) - assert f"ConanException: The source backup server '{http_server.fake_url}" \ - f"/downloader/' needs authentication" in client.out + assert f"ConanException: Authentication to source backup server '{http_server.fake_url}" \ + f"/downloader/' failed" in client.out content = {"credentials": [ {"url": f"{http_server.fake_url}", "token": "mytoken"} ]} @@ -424,8 +424,8 @@ def source(self): client.run("create .") assert "CONTENT: Hello, world!" in client.out client.run("upload * -c -r=default", assert_error=True) - assert f"The source backup server '{http_server.fake_url}" \ - f"/uploader/' needs authentication" in client.out + assert f"Authentication to source backup server '{http_server.fake_url}" \ + f"/uploader/' failed" in client.out content = {"credentials": [ {"url": f"{http_server.fake_url}", "token": "myuploadtoken"} ]} From 488308aa84343c9cc7b3fec47a5a32198bd55830 Mon Sep 17 00:00:00 2001 From: stevenwdv Date: Wed, 11 Mar 2026 10:34:56 +0100 Subject: [PATCH 046/110] Fix detect_emcc_compiler (#19735) on first run & on Windows See #19677 --- conan/internal/api/detect/detect_api.py | 25 +++++++++++-------------- test/unittests/util/detect_test.py | 4 ++++ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/conan/internal/api/detect/detect_api.py b/conan/internal/api/detect/detect_api.py index 6148d9cbdb4..e39b81132ac 100644 --- a/conan/internal/api/detect/detect_api.py +++ b/conan/internal/api/detect/detect_api.py @@ -616,21 +616,18 @@ def detect_cl_compiler(compiler_exe="cl"): def detect_emcc_compiler(compiler_exe="emcc"): - try: - ret, out = detect_runner(f'"{compiler_exe}" --version') - if ret != 0: - return None, None, None - first_line = out.splitlines()[0] - if "Emscripten" not in first_line: - return None, None, None - compiler = "emcc" - version_match = re.search(r"[0-9]+\.[0-9]+\.[0-9]+", first_line) - if version_match: - version = version_match.group() - ConanOutput(scope="detect_api").info("Found %s %s" % (compiler, version)) - return compiler, Version(version), compiler_exe - except (Exception,): # to disable broad-except + ret, out = detect_runner(f'"{compiler_exe}" --version') + if ret != 0: + return None, None, None + if "Emscripten" not in out: + return None, None, None + compiler = "emcc" + version_match = re.search(r"[0-9]+\.[0-9]+\.[0-9]+", out) + if not version_match: return None, None, None + version = version_match.group() + ConanOutput(scope="detect_api").info("Found %s %s" % (compiler, version)) + return compiler, Version(version), compiler_exe def default_compiler_version(compiler, version): diff --git a/test/unittests/util/detect_test.py b/test/unittests/util/detect_test.py index 62cb4aac291..17d478ef5d2 100644 --- a/test/unittests/util/detect_test.py +++ b/test/unittests/util/detect_test.py @@ -91,6 +91,10 @@ def test_detect_cc_versioning(detect_runner_mock, version_return, expected_versi [detect_emcc_compiler, "emcc (Emscripten gcc/clang-like replacement + linker emulating GNU ld) 4.0.22 (0f3d2e62bccf8e14497ff19e05a1202c51eb0c65)", ('emcc', Version("4.0.22"), 'emcc')], + [detect_emcc_compiler, + "shared:INFO: (Emscripten: Running sanity checks)\n" + + "emcc (Emscripten gcc/clang-like replacement + linker emulating GNU ld) 4.0.22 (0f3d2e62bccf8e14497ff19e05a1202c51eb0c65)", + ('emcc', Version("4.0.22"), 'emcc')], ]) def test_detect_compiler(function, version_return, expected_version): """ From 37b9e8201bd421992c9455b8794614565c3e9395 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 11 Mar 2026 14:31:55 +0100 Subject: [PATCH 047/110] remove alias support (#19740) --- conan/internal/graph/graph_builder.py | 4 ++++ test/functional/revisions_test.py | 4 ++++ test/integration/command/alias_test.py | 3 +++ test/integration/graph/core/test_alias.py | 3 +++ test/integration/lockfile/test_lock_alias.py | 3 +++ 5 files changed, 17 insertions(+) diff --git a/conan/internal/graph/graph_builder.py b/conan/internal/graph/graph_builder.py index c3638a8bd11..993a4e2fd27 100644 --- a/conan/internal/graph/graph_builder.py +++ b/conan/internal/graph/graph_builder.py @@ -1,3 +1,4 @@ +import os from collections import deque from conan.internal.cache.conan_reference_layout import BasicLayout @@ -20,6 +21,7 @@ class DepsGraphBuilder: + ALLOW_ALIAS = False def __init__(self, proxy, loader, resolver, cache, remotes, update, check_update, global_conf): self._proxy = proxy @@ -219,6 +221,8 @@ def _initialize_requires(self, node, graph, graph_lock, profile_build, profile_h result.append(require) alias = require.alias # alias needs to be processed this early if alias is not None: + if not DepsGraphBuilder.ALLOW_ALIAS and os.getenv("CONAN_ALLOW_ALIAS") != "will_break_next": + raise ConanException(f"Alias requirements have been removed: '{node}' requiring: '{alias}'") resolved = False if graph_lock is not None: resolved = graph_lock.replace_alias(require, alias) diff --git a/test/functional/revisions_test.py b/test/functional/revisions_test.py index 8b699af35f6..6c8bc4d9adb 100644 --- a/test/functional/revisions_test.py +++ b/test/functional/revisions_test.py @@ -5,6 +5,7 @@ import pytest from unittest.mock import patch +from conan.internal.graph.graph_builder import DepsGraphBuilder from conan.test.utils.env import environment_update from conan.internal.errors import RecipeNotFoundException from conan.api.model import RecipeReference @@ -24,6 +25,9 @@ def _create(c_v2, ref, conanfile=None, args=None, assert_error=False): return pref +DepsGraphBuilder.ALLOW_ALIAS = True + + @pytest.mark.artifactory_ready class TestInstallingPackagesWithRevisions: diff --git a/test/integration/command/alias_test.py b/test/integration/command/alias_test.py index c43a707e917..2f94df5e65f 100644 --- a/test/integration/command/alias_test.py +++ b/test/integration/command/alias_test.py @@ -2,8 +2,11 @@ import textwrap from conan.api.model import RecipeReference +from conan.internal.graph.graph_builder import DepsGraphBuilder from conan.test.utils.tools import TestClient, GenConanfile +DepsGraphBuilder.ALLOW_ALIAS = True + class TestConanAlias: diff --git a/test/integration/graph/core/test_alias.py b/test/integration/graph/core/test_alias.py index bf7eb87b0cf..4c1df85fa6d 100644 --- a/test/integration/graph/core/test_alias.py +++ b/test/integration/graph/core/test_alias.py @@ -1,8 +1,11 @@ +from conan.internal.graph.graph_builder import DepsGraphBuilder from conan.test.assets.genconanfile import GenConanfile from test.integration.graph.core.graph_manager_base import GraphManagerTest from test.integration.graph.core.graph_manager_test import _check_transitive from conan.test.utils.tools import TestClient +DepsGraphBuilder.ALLOW_ALIAS = True + class TestAlias(GraphManagerTest): diff --git a/test/integration/lockfile/test_lock_alias.py b/test/integration/lockfile/test_lock_alias.py index 0b2752a0064..c37d0746896 100644 --- a/test/integration/lockfile/test_lock_alias.py +++ b/test/integration/lockfile/test_lock_alias.py @@ -2,9 +2,12 @@ import pytest +from conan.internal.graph.graph_builder import DepsGraphBuilder from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient +DepsGraphBuilder.ALLOW_ALIAS = True + @pytest.mark.parametrize("requires", ["requires", "tool_requires"]) def test_conanfile_txt_deps_ranges(requires): From 0115fa1906930b88199b266f6bf10c3deb7a54e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20S=C3=A1nchez=20Falero?= Date: Fri, 13 Mar 2026 12:29:59 +0100 Subject: [PATCH 048/110] PyEnv output based on verbosity level (#19731) * PyEnv output based on verbosity level * wip * wip * wip * wip --- conan/tools/system/python_manager.py | 51 +++++++++++++++++-- .../tools/system/python_manager_test.py | 22 ++++++++ .../tools/system/python_manager_test.py | 47 +++++++++++++++++ 3 files changed, 115 insertions(+), 5 deletions(-) diff --git a/conan/tools/system/python_manager.py b/conan/tools/system/python_manager.py index 62b6ac93da8..1cccf1359e0 100644 --- a/conan/tools/system/python_manager.py +++ b/conan/tools/system/python_manager.py @@ -7,11 +7,42 @@ from conan.tools.env.environment import Environment from conan.errors import ConanException -from conan.api.output import ConanOutput +from conan.api.output import ( + ConanOutput, + LEVEL_QUIET, + LEVEL_ERROR, + LEVEL_WARNING, + LEVEL_STATUS, + LEVEL_VERBOSE, + LEVEL_DEBUG, + LEVEL_TRACE, +) from conan.internal.util.files import rmdir +def _get_pip_verbosity(): + return { + LEVEL_QUIET: "-qqq", + LEVEL_ERROR: "-qq", + LEVEL_WARNING: "-q", + LEVEL_VERBOSE: "-v", + LEVEL_DEBUG: "-vv", + LEVEL_TRACE: "-vvv", + }.get(ConanOutput.get_output_level(), "") + + +def _get_uv_verbosity(): + return { + LEVEL_QUIET: "-qq", + LEVEL_ERROR: "-qq", + LEVEL_WARNING: "-q", + LEVEL_VERBOSE: "--verbose", + LEVEL_DEBUG: "--verbose", + LEVEL_TRACE: "--verbose", + }.get(ConanOutput.get_output_level(), "") + + class PyEnv: def __init__(self, conanfile, folder=None, name="", py_version=None): @@ -97,6 +128,9 @@ def install(self, packages, pip_args=None): :return: the return code of the executed pip command. """ args = [self.env_exe, "-m", "pip", "install", "--disable-pip-version-check"] + pip_verbosity = _get_pip_verbosity() + if pip_verbosity: + args.append(pip_verbosity) if pip_args: args.extend(pip_args) args += [f'"{p}"' for p in packages] @@ -124,12 +158,19 @@ def _create_uv_venv(self, base_env_dir, py_version): ) python_exe = self._get_env_python(uv_env_dir) - self._conanfile.run(cmd_args_to_string( - [python_exe, "-m", "pip", "install", "--disable-pip-version-check", "uv"]) - ) + pip_args = [python_exe, "-m", "pip", "install", "--disable-pip-version-check"] + pip_verbosity = _get_pip_verbosity() + if pip_verbosity: + pip_args.append(pip_verbosity) + pip_args.append("uv") + self._conanfile.run(cmd_args_to_string(pip_args)) uv_cmd = [python_exe, "-m", "uv"] - self._conanfile.run(cmd_args_to_string(uv_cmd + ['venv', '--seed', '--python', py_version, self._env_dir])) + uv_venv_args = uv_cmd + ['venv', '--seed', '--python', py_version, self._env_dir] + uv_verbosity = _get_uv_verbosity() + if uv_verbosity: + uv_venv_args.append(uv_verbosity) + self._conanfile.run(cmd_args_to_string(uv_venv_args)) self._conanfile.output.info(f"Virtual environment for Python " f"{py_version} created successfully using UV.") except Exception as e: diff --git a/test/functional/tools/system/python_manager_test.py b/test/functional/tools/system/python_manager_test.py index 030ef5cb0be..21997af7af6 100644 --- a/test/functional/tools/system/python_manager_test.py +++ b/test/functional/tools/system/python_manager_test.py @@ -89,6 +89,9 @@ def build(self): assert "Found existing installation: hello 0.1.0" in client.out assert "Hello Test World!" in client.out + client.run("build pip -verror") + assert "Found existing installation" not in client.out + def test_install_version_range(): c = TestClient(path_with_spaces=False) @@ -343,6 +346,25 @@ def build(self): assert "Hello Test World!" in client.out +@pytest.mark.parametrize("verbosity", ["-verror", "-vstatus"]) +def test_pyenv_install_error_always_shown(verbosity): + conanfile_pyenv = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.system import PyEnv + + class PyenvPackage(ConanFile): + def generate(self): + pyenv = PyEnv(self) + pyenv.install(["package_does_not_exist"]) + """) + + client = TestClient(path_with_spaces=False) + client.save({"conanfile.py": conanfile_pyenv}) + client.run(f"build . {verbosity}", assert_error=True) + assert "package_does_not_exist" in client.out + assert "ERROR" in client.out + + def test_cmake_toolchain_configure_find_python(): client = TestClient(path_with_spaces=False) conanfile = textwrap.dedent(""" diff --git a/test/unittests/tools/system/python_manager_test.py b/test/unittests/tools/system/python_manager_test.py index 7c52d081b65..a0719308fd1 100644 --- a/test/unittests/tools/system/python_manager_test.py +++ b/test/unittests/tools/system/python_manager_test.py @@ -1,6 +1,8 @@ from conan.tools.system import PyEnv from unittest.mock import patch import pytest +from conan.api.output import ConanOutput, LEVEL_QUIET, LEVEL_ERROR, LEVEL_WARNING, \ + LEVEL_STATUS, LEVEL_VERBOSE, LEVEL_DEBUG, LEVEL_TRACE from conan.errors import ConanException from conan.internal.model.settings import Settings from conan.test.utils.mocks import ConanFileMock @@ -62,3 +64,48 @@ def fake_run(command, win_bash=False, subsystem=None, env=None, ignore_errors=Fa with pytest.raises(ConanException) as exc_info: PyEnv(conanfile, "testenv") assert "using '/python/interpreter/from/config': fake error message" in exc_info.value.args[0] + + +@pytest.mark.parametrize("level, expected_pip_flag", [ + (LEVEL_QUIET, "-qqq"), + (LEVEL_ERROR, "-qq"), + (LEVEL_WARNING, "-q"), + (LEVEL_STATUS, None), + (LEVEL_VERBOSE, "-v"), + (LEVEL_DEBUG, "-vv"), + (LEVEL_TRACE, "-vvv"), +]) +def test_pyenv_pip_verbosity(level, expected_pip_flag): + """ + https://github.com/conan-io/conan/issues/19729 + PyEnv.install() should map Conan verbosity levels to pip's native -q/-v flags. + """ + conanfile = ConanFileMock() + conanfile.settings = Settings() + conanfile.conf.define("tools.system.pyenv:python_interpreter", + "/python/interpreter/from/config") + + calls = [] + + def fake_run(command, win_bash=False, subsystem=None, env=None, ignore_errors=False, # noqa + quiet=False): # noqa + calls.append(command) + + conanfile.run = fake_run + + old_level = ConanOutput.get_output_level() + try: + ConanOutput.set_output_level(level) + pyenv = PyEnv(conanfile, f"testenv_{level}") + calls.clear() + + pyenv.install(["some_package"]) + assert len(calls) == 1 + assert "pip install" in calls[0] + if expected_pip_flag: + assert f" {expected_pip_flag} " in calls[0] + else: + assert " -q" not in calls[0] + assert " -v " not in calls[0] + finally: + ConanOutput.set_output_level(old_level) From 995eadcea80b346bf2717fc909823e43e375c6be Mon Sep 17 00:00:00 2001 From: James Date: Fri, 13 Mar 2026 12:31:48 +0100 Subject: [PATCH 049/110] Feature/build arg context (pure refactor) (#19746) * build arg context * wip * wip --- conan/internal/graph/build_mode.py | 41 +++++++++++-------- conan/internal/graph/graph_binaries.py | 5 --- .../command/install/install_cascade_test.py | 1 + .../unittests/client/graph/build_mode_test.py | 17 -------- 4 files changed, 26 insertions(+), 38 deletions(-) diff --git a/conan/internal/graph/build_mode.py b/conan/internal/graph/build_mode.py index 042652007f8..6d964d2fcca 100644 --- a/conan/internal/graph/build_mode.py +++ b/conan/internal/graph/build_mode.py @@ -1,3 +1,4 @@ +from conan.api.output import ConanOutput from conan.errors import ConanException from conan.internal.model.recipe_ref import ref_matches @@ -9,12 +10,12 @@ class BuildMode: => ["!foo"] or ["~foo"] means exclude when building all from sources """ def __init__(self, params): - self.missing = False - self.never = False + self._missing = False + self._never = False self.cascade = False - self.editable = False - self.patterns = [] - self.build_missing_patterns = [] + self._editable = False + self._patterns = [] + self._build_missing_patterns = [] self._build_missing_excluded = [] self._build_compatible_patterns = [] self._build_compatible_excluded = [] @@ -27,20 +28,23 @@ def __init__(self, params): for param in params: if param == "missing": - self.missing = True + self._missing = True elif param == "editable": - self.editable = True + self._editable = True elif param == "never": - self.never = True + self._never = True elif param == "cascade": self.cascade = True + ConanOutput().warning("Using build-mode 'cascade' is generally inefficient and it " + "shouldn't be used. Use 'package_id' and 'package_id_modes' " + "for more efficient re-builds") else: if param.startswith("missing:"): clean_pattern = param[len("missing:"):] if clean_pattern and clean_pattern[0] in ["!", "~"]: self._build_missing_excluded.append(clean_pattern[1:]) else: - self.build_missing_patterns.append(clean_pattern) + self._build_missing_patterns.append(clean_pattern) elif param == "compatible": self._build_compatible_patterns = ["*"] elif param.startswith("compatible:"): @@ -54,11 +58,16 @@ def __init__(self, params): if clean_pattern and clean_pattern[0] in ["!", "~"]: self._excluded_patterns.append(clean_pattern[1:]) else: - self.patterns.append(clean_pattern) + self._patterns.append(clean_pattern) - if self.never and (self.missing or self.patterns or self.cascade): + if self._never and (self._missing or self._patterns or self.cascade): raise ConanException("--build=never not compatible with other options") + @property + def editable(self): + # we can make this conditional on the context in the future + return self._editable + def forced(self, conan_file, ref, with_deps_to_build=False): # TODO: ref can be obtained from conan_file @@ -70,7 +79,7 @@ def forced(self, conan_file, ref, with_deps_to_build=False): if conan_file.build_policy == "never": # this package has been export-pkg return False - if self.never: + if self._never: return False if conan_file.build_policy == "always": @@ -81,15 +90,15 @@ def forced(self, conan_file, ref, with_deps_to_build=False): return True # Patterns to match, if package matches pattern, build is forced - for pattern in self.patterns: + for pattern in self._patterns: if ref_matches(ref, pattern, is_consumer=conan_file._conan_is_consumer): # noqa return True return False def allowed(self, conan_file): - if self.never or conan_file.build_policy == "never": # this package has been export-pkg + if self._never or conan_file.build_policy == "never": # this package has been export-pkg return False - if self.missing: + if self._missing: return True if conan_file.build_policy == "missing": conan_file.output.info("Building package from source as defined by " @@ -119,6 +128,6 @@ def should_build_missing(self, conanfile): return False return True # If it has not been excluded by the negated patterns, it is included - for pattern in self.build_missing_patterns: + for pattern in self._build_missing_patterns: if ref_matches(conanfile.ref, pattern, is_consumer=conanfile._conan_is_consumer): # noqa return True diff --git a/conan/internal/graph/graph_binaries.py b/conan/internal/graph/graph_binaries.py index ffd361a2e84..afa14528557 100644 --- a/conan/internal/graph/graph_binaries.py +++ b/conan/internal/graph/graph_binaries.py @@ -476,11 +476,6 @@ def evaluate_graph(self, deps_graph, build_mode, lockfile, remotes, update, buil mainprefs = [str(n.pref) for n in tested_graph.nodes if n.recipe not in (RECIPE_CONSUMER, RECIPE_VIRTUAL)] - if main_mode.cascade: - ConanOutput().warning("Using build-mode 'cascade' is generally inefficient and it " - "shouldn't be used. Use 'package_id' and 'package_id_modes' for" - "more efficient re-builds") - def _evaluate_single(n): mode = main_mode if mainprefs is None or str(n.pref) in mainprefs else test_mode if lockfile: diff --git a/test/integration/command/install/install_cascade_test.py b/test/integration/command/install/install_cascade_test.py index 04114740b5e..f4f6445a900 100644 --- a/test/integration/command/install/install_cascade_test.py +++ b/test/integration/command/install/install_cascade_test.py @@ -31,6 +31,7 @@ def _assert_built(refs): # Building A everything is built c.run("install app --build=liba* --build cascade") + assert "Using build-mode 'cascade' is generally inefficient" in c.out _assert_built(["liba/1.0", "libb/1.0", "libc/1.0", "libd/1.0", "libe/1.0", "libf/1.0"]) c.run("install app --build=libd* --build cascade") diff --git a/test/unittests/client/graph/build_mode_test.py b/test/unittests/client/graph/build_mode_test.py index 7ecf1e47163..72893698940 100644 --- a/test/unittests/client/graph/build_mode_test.py +++ b/test/unittests/client/graph/build_mode_test.py @@ -26,23 +26,6 @@ def test_skip_package(conanfile): assert not build_mode.forced(conanfile, RecipeReference.loads("other/1.2")) -def test_valid_params(): - build_mode = BuildMode(["missing"]) - assert build_mode.missing is True - assert build_mode.never is False - assert build_mode.cascade is False - - build_mode = BuildMode(["never"]) - assert build_mode.missing is False - assert build_mode.never is True - assert build_mode.cascade is False - - build_mode = BuildMode(["cascade"]) - assert build_mode.missing is False - assert build_mode.never is False - assert build_mode.cascade is True - - def test_invalid_configuration(): for mode in ["missing", "cascade"]: with pytest.raises(ConanException, match=r"--build=never not compatible " From e5b01b26d10bec75ba361e4401a42bc2730feb54 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 13 Mar 2026 12:37:16 +0100 Subject: [PATCH 050/110] fix platform_requires definition (#19750) --- conan/internal/api/profile/profile_loader.py | 12 ++++++++++-- test/integration/graph/test_system_tools.py | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/conan/internal/api/profile/profile_loader.py b/conan/internal/api/profile/profile_loader.py index 61342c293e2..5de78063094 100644 --- a/conan/internal/api/profile/profile_loader.py +++ b/conan/internal/api/profile/profile_loader.py @@ -245,8 +245,16 @@ def get_profile(profile_text, base_profile=None): if doc.system_tools: ConanOutput().warning("Profile [system_tools] is deprecated," " please use [platform_tool_requires]") - platform_tool_requires = [RecipeReference.loads(r) for r in doc_platform_tool_requires.splitlines()] - platform_requires = [RecipeReference.loads(r) for r in doc_platform_requires.splitlines()] + + def parse_replaces(replaces): + result = [RecipeReference.loads(r) for r in replaces.splitlines()] + errors = [r for r in result if str(r.version).startswith("[")] + if errors: + raise ConanException("Profile [platform_requires]/[platform_tool_requires] must " + f"be exact versions, not version ranges: {errors}") + return result + platform_tool_requires = parse_replaces(doc_platform_tool_requires) + platform_requires = parse_replaces(doc_platform_requires) def load_replace(doc_replace_requires): result = {} diff --git a/test/integration/graph/test_system_tools.py b/test/integration/graph/test_system_tools.py index f24baf89070..e72b1ab88e3 100644 --- a/test/integration/graph/test_system_tools.py +++ b/test/integration/graph/test_system_tools.py @@ -157,6 +157,26 @@ def test_require_build_context(self): c.run("install app -pr:b=profile_build --build=missing") assert "Install finished successfully" in c.out + def test_platform_requires_error(self): + """ + https://github.com/conan-io/conan/issues/19745 + """ + c = TestClient(light=True) + profile = textwrap.dedent("""\ + include(default) + + [platform_tool_requires] + cmake/[*] + """) + c.save({"conanfile.py": GenConanfile("catch2").with_tool_requires("cmake/[*]"), + "profile": profile}) + + c.run("create --version=3.6.0 -pr=profile", assert_error=True) + assert "AssertionError" not in c.out + assert ("ERROR: Error reading 'profile' profile: Profile [platform_requires]/" + "[platform_tool_requires] must be exact versions, " + "not version ranges: [cmake/[*]]") in c.out + class TestToolRequiresLock: From b498010ed6411a167fafc5d00ffb2ae8fd7f53fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= <5364255+AbrilRBS@users.noreply.github.com> Date: Fri, 13 Mar 2026 13:41:08 +0100 Subject: [PATCH 051/110] Only group build packages in `conan graph info .. -f=html` (#19744) * Only group build packages * Group per context --- conan/cli/formatters/graph/info_graph_html.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conan/cli/formatters/graph/info_graph_html.py b/conan/cli/formatters/graph/info_graph_html.py index bf67480a5c0..e2466d225b9 100644 --- a/conan/cli/formatters/graph/info_graph_html.py +++ b/conan/cli/formatters/graph/info_graph_html.py @@ -87,7 +87,7 @@ function define_data(){ let nodes = []; let edges = []; - let collapsed_packages = {}; + let collapsed_packages = {"build": {}, "host": {}}; let targets = {}; global_edges = {}; let edge_counter = 0; @@ -115,10 +115,10 @@ else label = node.recipe == "Consumer"? "conanfile": "CLI"; if (collapse_packages) { - let existing = collapsed_packages[label]; + let existing = collapsed_packages[node.context][label]; targets[node_id] = existing; if (existing) continue; - collapsed_packages[label] = node_id; + collapsed_packages[node.context][label] = node_id; } if (excluded_pkgs) { let patterns = excluded_pkgs.split(',') From fe60b2047ad3cdf8c0d075d14718ae9bbd99ed0c Mon Sep 17 00:00:00 2001 From: James Date: Sun, 15 Mar 2026 13:55:01 +0100 Subject: [PATCH 052/110] fix CMakeConfigDeps .set_property context (#19760) --- .../cmake/cmakeconfigdeps/cmakeconfigdeps.py | 6 +++- .../cmakeconfigdeps/test_cmakeconfigdeps.py | 28 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/conan/tools/cmake/cmakeconfigdeps/cmakeconfigdeps.py b/conan/tools/cmake/cmakeconfigdeps/cmakeconfigdeps.py index cc15cf8e98d..1a483bf0405 100644 --- a/conan/tools/cmake/cmakeconfigdeps/cmakeconfigdeps.py +++ b/conan/tools/cmake/cmakeconfigdeps/cmakeconfigdeps.py @@ -165,7 +165,11 @@ def set_property(self, dep, prop, value, build_context=False): def get_property(self, prop, dep, comp_name=None, check_type=None): dep_name = dep.ref.name - build_suffix = "&build" if dep.context == "build" else "" + # Find the requirement that points to this "dep". + # TODO: It would probably be more explicit if it was an argument as "dep", but to keep + # diff minimal + require = next(iter(r for r, d in self._conanfile.dependencies.items() if d is dep)) + build_suffix = "&build" if require.build else "" dep_comp = f"{str(dep_name)}::{comp_name}" if comp_name else f"{str(dep_name)}" try: value = self._properties[f"{dep_comp}{build_suffix}"][prop] diff --git a/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps.py b/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps.py index ebb98ba3813..9e6d913c2f6 100644 --- a/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps.py +++ b/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps.py @@ -802,6 +802,34 @@ def test_legacy_defines(): assert 'set(mypkg_DEFINITIONS "-DMY_DEFINE;-DMYVAR=1" )' in mypkg_config +class TestPropertiesBuildContext: + def test_property_build_context(self): + c = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.cmake import CMakeConfigDeps + + class PackageConan(ConanFile): + name = "package" + settings = "os", "arch", "compiler", "build_type" + + def requirements(self): + self.requires("zlib/1.3.1") + + def generate(self): + deps = CMakeConfigDeps(self) + deps.set_property("zlib", "cmake_file_name", "MyZlibName") + deps.generate() + """) + c.save({"zlib/conanfile.py": GenConanfile("zlib", "1.3.1"), + "pkg/conanfile.py": conanfile}) + c.run("create zlib") + c.run("install pkg --build-require") + assert "find_package(MyZlibName)" in c.out + config = c.load("pkg/MyZlibNameConfig.cmake") + assert 'set(MyZlibName_VERSION_STRING "1.3.1")' in config + + class TestExtraFindExtraVariants: def test_generated_dir_entries(self): tc = TestClient() From 41a4d71d48ce21744aaf8ef7073ebf385a00e83b Mon Sep 17 00:00:00 2001 From: James Date: Mon, 16 Mar 2026 08:00:25 +0100 Subject: [PATCH 053/110] Fix/overrides lockfiles (#19739) * fix lockfile overrides * wip * fix test --- conan/internal/graph/graph_builder.py | 2 +- conan/internal/model/lockfile.py | 5 +- .../lockfile/test_graph_overrides.py | 54 ++++++++++++++++++- 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/conan/internal/graph/graph_builder.py b/conan/internal/graph/graph_builder.py index 993a4e2fd27..2e2c416771f 100644 --- a/conan/internal/graph/graph_builder.py +++ b/conan/internal/graph/graph_builder.py @@ -231,7 +231,7 @@ def _initialize_requires(self, node, graph, graph_lock, profile_build, profile_h self._resolve_alias(node, require, alias, graph) self._resolve_replace_requires(node, require, profile_build, profile_host, graph) if graph_lock: - graph_lock.resolve_overrides(require) + graph_lock.resolve_overrides(require, node.context) node.transitive_deps[require] = TransitiveRequirement(require, node=None) return result diff --git a/conan/internal/model/lockfile.py b/conan/internal/model/lockfile.py index c66a6f2e09a..f6811344af8 100644 --- a/conan/internal/model/lockfile.py +++ b/conan/internal/model/lockfile.py @@ -293,7 +293,7 @@ def resolve_locked(self, node, require, resolve_prereleases): ConanOutput().error(msg, error_type="exception") raise - def resolve_overrides(self, require): + def resolve_overrides(self, require, context): """ The lockfile contains the overrides to be able to inject them when the lockfile is applied to upstream dependencies, that have the overrides downstream """ @@ -303,6 +303,9 @@ def resolve_overrides(self, require): overriden = self._overrides.get(require.ref) if overriden and len(overriden) == 1: override_ref = next(iter(overriden)) + locked_refs = self._build_requires.refs() if context == "build" else self._requires.refs() + if override_ref not in locked_refs: + return # The override came from the other context require.overriden_ref = require.overriden_ref or require.ref.copy() require.override_ref = override_ref require.ref = override_ref diff --git a/test/integration/lockfile/test_graph_overrides.py b/test/integration/lockfile/test_graph_overrides.py index 2212e298921..3010ebf143b 100644 --- a/test/integration/lockfile/test_graph_overrides.py +++ b/test/integration/lockfile/test_graph_overrides.py @@ -65,6 +65,8 @@ def test_overrides_half_diamond_ranges(override, force): c.run("graph info pkgc --lockfile=pkgc/conan.lock") assert "pkga/0.2" in c.out assert "pkga/0.1" not in c.out + c.run("graph info pkgb --lockfile=pkgc/conan.lock") + # should work @pytest.mark.parametrize("override, force", [(True, False), (False, True)]) @@ -94,6 +96,8 @@ def test_overrides_half_diamond_ranges_inverted(override, force): c.run("graph info pkgc --lockfile=pkgc/conan.lock") assert "pkga/0.1" in c.out assert "pkga/0.2" not in c.out + c.run("graph info pkgb --lockfile=pkgc/conan.lock") + # should work @pytest.mark.parametrize("override, force", [(True, False), (False, True)]) @@ -224,6 +228,8 @@ def test_overrides_multiple(override1, force1, override2, force2): assert "pkga/0.3" in c.out assert "pkga/0.2#" not in c.out assert "pkga/0.1#" not in c.out # appears in override information + c.run("graph info pkgb --lockfile=pkgd/conan.lock") + # should work def test_graph_different_overrides(): @@ -351,8 +357,12 @@ def test_command_line_lockfile_overrides(): c.run('install pkgc --lockfile-overrides="{\'pkga/0.1\': [\'pkga/0.2\']}"', assert_error=True) assert "Cannot define overrides without a lockfile" in c.out c.run('lock create pkgc') - c.run('install pkgc --lockfile-overrides="{\'pkga/0.1\': [\'pkga/0.2\']}"', assert_error=True) - assert "Requirement 'pkga/0.2' not in lockfile" in c.out + c.run('install pkgc --lockfile-overrides="{\'pkga/0.1\': [\'pkga/0.2\']}"') + # From https://github.com/conan-io/conan/issues/19738 it is simply ignored + # Not a hard "protection" error, still users can't inject a dependency to pkga/0.2 if not + # in the lockfile already + assert "pka/0.2" not in c.out + assert "pkga/0.1" in c.out def test_consecutive_installs(): @@ -371,3 +381,43 @@ def test_consecutive_installs(): # This used to crash when overrides were not managed c.run("install pkgc --build=missing --lockfile=conan.lock --lockfile-out=conan.lock") c.assert_overrides({"pkga/0.1": ["pkga/0.2"]}) + + +class TestOverrideContextError: + # https://github.com/conan-io/conan/issues/19738 + def test_error_lockfile_override_build_require(self): + # The override that comes from the host context is breaking the build locking + c = TestClient(light=True) + c.save({"abseil/conanfile.py": GenConanfile("abseil"), + "protobuf/conanfile.py": GenConanfile("protobuf", "0.1").with_requires("abseil/[*]"), + "app/conanfile.py": GenConanfile("pkgd", "0.1").with_requirement("protobuf/0.1") + .with_requirement("abseil/0.1", + override=True) + .with_tool_requires("protobuf/0.1") + }) + c.run("create abseil --version=0.1") + c.run("create abseil --version=0.2") + c.run("create protobuf") + c.run("lock create app") + c.run("install app --build=missing --lockfile=app/conan.lock") + # It doesnt fail + + def test_error_lockfile_override_build_require_build(self): + # Same as the above, but now the override is in the "build" context, affecting the + # host one that shouldn't be overriden + c = TestClient(light=True) + c.save({"pkga/conanfile.py": GenConanfile("pkga"), + "pkgb/conanfile.py": GenConanfile("pkgb", "0.1").with_requires("pkga/[*]"), + "toolb/conanfile.py": GenConanfile("toolb", "0.1").with_requirement("pkgb/0.1") + .with_requirement("pkga/0.1", + override=True), + "app/conanfile.py": GenConanfile("pkgd", "0.1").with_requirement("pkgb/0.1") + .with_tool_requires("toolb/0.1") + }) + c.run("create pkga --version=0.1") + c.run("create pkga --version=0.2") + c.run("create pkgb") + c.run("create toolb --build=missing") + c.run("lock create app") + c.run("install app --lockfile=app/conan.lock") + # It doesnt fail From f253d3c2eab75b493cabeda7d4fd6607e80ca1bd Mon Sep 17 00:00:00 2001 From: James Date: Mon, 16 Mar 2026 16:04:16 +0100 Subject: [PATCH 054/110] fix issue with --build=editable with download source (#19758) --- conan/internal/graph/installer.py | 2 +- .../command/test_forced_download_source.py | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/conan/internal/graph/installer.py b/conan/internal/graph/installer.py index 5748e4dd611..230f61e668f 100644 --- a/conan/internal/graph/installer.py +++ b/conan/internal/graph/installer.py @@ -182,7 +182,7 @@ def _install_source(self, node, remotes, need_conf=False): return conanfile = node.conanfile - if node.binary == BINARY_EDITABLE: + if node.binary in (BINARY_EDITABLE, BINARY_EDITABLE_BUILD): return recipe_layout = self._cache.recipe_layout(node.ref) diff --git a/test/integration/command/test_forced_download_source.py b/test/integration/command/test_forced_download_source.py index cc06ceae63c..bd48ff76fad 100644 --- a/test/integration/command/test_forced_download_source.py +++ b/test/integration/command/test_forced_download_source.py @@ -75,3 +75,32 @@ def source(self): assert "RUNNING SOURCE" not in c.out c.run("graph info --requires=dep/0.1 -c tools.build:download_source=True") assert "RUNNING SOURCE" not in c.out # BUT it doesn't crash, it used to crash + + +def test_build_editable_with_download_source(): + """ conan build with -b=editable and tools.build:download_source=True should not crash + # https://github.com/conan-io/conan/issues/19757 + """ + c = TestClient() + liba = textwrap.dedent(""" + from conan import ConanFile + + class LibA(ConanFile): + name = "liba" + version = "0.1" + + def source(self): + self.output.info("RUNNING SOURCE!!") + """) + consumer = textwrap.dedent(""" + from conan import ConanFile + + class Consumer(ConanFile): + requires = "liba/0.1" + """) + + c.save({"liba/conanfile.py": liba, + "consumer/conanfile.py": consumer}) + c.run("editable add liba") + c.run("build consumer -b=editable -c tools.build:download_source=True") + assert "RUNNING SOURCE" not in c.out From b254f3c9ae7d509fd71bc34c30ca3a381d107828 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 19 Mar 2026 11:14:23 +0100 Subject: [PATCH 055/110] moving collaborators from ConanApp to ApiHelpers (#19755) * moving collaborators from ConanApp to ApiHelpers * wip * wip * fix test --- conan/api/conan_api.py | 30 +++++++++-- conan/api/subapi/cache.py | 4 +- conan/api/subapi/config.py | 3 +- conan/api/subapi/download.py | 31 ++++++------ conan/api/subapi/export.py | 2 +- conan/api/subapi/graph.py | 11 ++-- conan/api/subapi/install.py | 6 +-- conan/api/subapi/list.py | 53 +++++++++---------- conan/api/subapi/remotes.py | 7 +-- conan/api/subapi/remove.py | 34 ++++++------- conan/api/subapi/report.py | 3 +- conan/api/subapi/upload.py | 15 +++--- conan/api/subapi/workspace.py | 3 +- conan/internal/api/uploader.py | 28 ++++++----- conan/internal/conan_app.py | 56 +++------------------ conan/internal/graph/graph_binaries.py | 10 ++-- conan/internal/graph/installer.py | 21 ++++---- conan/internal/graph/proxy.py | 6 +-- conan/internal/graph/python_requires.py | 6 +-- conan/internal/graph/range_resolver.py | 6 +-- conan/internal/model/workspace.py | 2 +- conan/internal/rest/remote_manager.py | 2 +- conan/test/utils/mocks.py | 2 +- test/integration/cache/storage_path_test.py | 2 +- 24 files changed, 163 insertions(+), 180 deletions(-) diff --git a/conan/api/conan_api.py b/conan/api/conan_api.py index 6ba8b30ff18..85d7a616856 100644 --- a/conan/api/conan_api.py +++ b/conan/api/conan_api.py @@ -21,6 +21,8 @@ from conan.api.subapi.remove import RemoveAPI from conan.api.subapi.upload import UploadAPI from conan.errors import ConanException +from conan.internal.api.remotes.localdb import LocalDB +from conan.internal.cache.cache import PkgCache from conan.internal.cache.home_paths import HomePaths from conan.internal.hook_manager import HookManager from conan.internal.loader import load_python_file @@ -29,7 +31,9 @@ from conan.internal.paths import get_conan_user_home from conan.internal.api.migrations import ClientMigrator from conan.internal.model.version_range import validate_conan_version +from conan.internal.rest.auth_manager import ConanApiAuthManager from conan.internal.rest.conan_requester import ConanRequester +from conan.internal.rest.remote_manager import RemoteManager class ConanAPI: @@ -63,19 +67,19 @@ def __init__(self, cache_folder=None): self.remotes: RemotesAPI = RemotesAPI(self, self._api_helpers) self.command = CommandAPI(self) #: Used to get latest refs and list refs of recipes and packages - self.list: ListAPI = ListAPI(self) + self.list: ListAPI = ListAPI(self, self._api_helpers) self.profiles = ProfilesAPI(self, self._api_helpers) #: Used to install binaries, sources, deploy packages and more self.install: InstallAPI = InstallAPI(self, self._api_helpers) self.graph = GraphAPI(self, self._api_helpers) #: Used to export recipes and pre-compiled package binaries to the Conan cache self.export: ExportAPI = ExportAPI(self, self._api_helpers) - self.remove = RemoveAPI(self) + self.remove = RemoveAPI(self, self._api_helpers) self.new = NewAPI(self) #: Used to upload recipes and packages to remotes self.upload: UploadAPI = UploadAPI(self, self._api_helpers) #: Used to download recipes and packages from remotes - self.download: DownloadAPI = DownloadAPI(self) + self.download: DownloadAPI = DownloadAPI(self, self._api_helpers) #: Used to interact wit the packages storage cache self.cache: CacheAPI = CacheAPI(self, self._api_helpers) #: Used to read and manage lockfile files @@ -108,18 +112,22 @@ def migrate(self): # Migration system # TODO: A prettier refactoring of migrators would be nice from conan import conan_version - migrator = ClientMigrator(self.cache_folder, conan_version) + migrator = ClientMigrator(self._home_folder, conan_version) migrator.migrate() class _ApiHelpers: + # This is an internal implementation detail of Conan, DO NOT USE def __init__(self, conan_api): self._conan_api = conan_api self._cli_core_confs = None self._init_global_conf() + # TODO: Make uniform lazy vs non lazy collaborators self.hook_manager = HookManager(HomePaths(self._conan_api.home_folder).hooks_path) # Wraps an http_requester to inject proxies, certs, etc self._requester = ConanRequester(self.global_conf, self._conan_api.home_folder) + self.cache = PkgCache(self._conan_api.home_folder, self.global_conf) self._settings_yml = None + self._remote_manager = None self._compression_plugin = None @property @@ -165,6 +173,8 @@ def reinit(self): self.hook_manager.reinit() self._requester = ConanRequester(self.global_conf, self._conan_api.home_folder) self._settings_yml = None + self.cache = PkgCache(self._conan_api.home_folder, self.global_conf) + self._remote_manager = None self._compression_plugin = None @property @@ -173,6 +183,18 @@ def settings_yml(self): self._settings_yml = load_settings_yml(self._conan_api.home_folder) return self._settings_yml + @property + def remote_manager(self): + if self._remote_manager is None: + home_folder = self._conan_api.home_folder + localdb = LocalDB(home_folder) + requester = self._conan_api._api_helpers.requester # noqa + auth_manager = ConanApiAuthManager(requester, self._conan_api.home_folder, localdb, + self.global_conf) + self._remote_manager = RemoteManager(self.cache, auth_manager, home_folder, + self.compression_plugin) + return self._remote_manager + @property def requester(self): return self._requester diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index bad3f9473ba..f2bf74e136c 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -195,7 +195,9 @@ def sign(self, package_list): "https://docs.conan.io/2/reference/extensions/package_signing.html.") app = ConanApp(self._conan_api) - preparator = PackagePreparator(app, self._api_helpers.global_conf) + preparator = PackagePreparator(app, self._api_helpers.cache, + self._api_helpers.remote_manager, + self._api_helpers.global_conf) # Some packages can have missing sources/exports_sources enabled_remotes = self._conan_api.remotes.list() preparator.prepare(package_list, enabled_remotes, None, force=True) diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index bcc252832a8..eca212d3534 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -213,6 +213,7 @@ def fetch_packages(self, requires, lockfile=None, remotes=None, profile=None): profile_host = profile_build = profile or conan_api.profiles.get_profile([]) app = ConanApp(self._conan_api) + cache = self._helpers.cache ConanOutput().title("Fetching requested configuration packages") result = [] @@ -225,7 +226,7 @@ def fetch_packages(self, requires, lockfile=None, remotes=None, profile=None): recipe=RECIPE_VIRTUAL) root_node.is_conf = True update = ["*"] - builder = DepsGraphBuilder(app.proxy, app.loader, app.range_resolver, app.cache, remotes, + builder = DepsGraphBuilder(app.proxy, app.loader, app.range_resolver, cache, remotes, update, update, self._helpers.global_conf) deps_graph = builder.load_graph(root_node, profile_host, profile_build, lockfile) diff --git a/conan/api/subapi/download.py b/conan/api/subapi/download.py index d67ff123720..b136a307aec 100644 --- a/conan/api/subapi/download.py +++ b/conan/api/subapi/download.py @@ -4,7 +4,6 @@ from conan.api.model import Remote, PackagesList from conan.api.output import ConanOutput -from conan.internal.conan_app import ConanBasicApp from conan.errors import ConanException from conan.api.model import PkgReference from conan.api.model import RecipeReference @@ -13,38 +12,39 @@ class DownloadAPI: """ This API is used to download recipes and packages from a remote server.""" - def __init__(self, conan_api): + def __init__(self, conan_api, api_helpers): self._conan_api = conan_api + self._api_helpers = api_helpers def recipe(self, ref: RecipeReference, remote: Remote, metadata: Optional[List[str]] = None): """Download the recipe specified in the ref from the remote. If the recipe is already in the cache it will be skipped, but the specified metadata will be downloaded.""" output = ConanOutput() - app = ConanBasicApp(self._conan_api) assert ref.revision, f"Reference '{ref}' must have revision" try: - recipe_layout = app.cache.recipe_layout(ref) # raises if not found + recipe_layout = self._api_helpers.cache.recipe_layout(ref) # raises if not found except ConanException: pass else: output.info(f"Skip recipe {ref.repr_notime()} download, already in cache") if metadata: - app.remote_manager.get_recipe_metadata(recipe_layout, ref, remote, metadata) + self._api_helpers.remote_manager.get_recipe_metadata(recipe_layout, ref, remote, + metadata) return False output.info(f"Downloading recipe '{ref.repr_notime()}'") if ref.timestamp is None: # we didnt obtain the timestamp before (in general it should be) # Respect the timestamp of the server, the ``get_recipe()`` doesn't do it internally # Best would be that ``get_recipe()`` returns the timestamp in the same call - server_ref = app.remote_manager.get_recipe_revision(ref, remote) + server_ref = self._api_helpers.remote_manager.get_recipe_revision(ref, remote) assert server_ref == ref ref.timestamp = server_ref.timestamp - recipe_layout = app.remote_manager.get_recipe(ref, remote, metadata) + recipe_layout = self._api_helpers.remote_manager.get_recipe(ref, remote, metadata) # Download the sources too, don't be lazy output.info(f"Downloading '{str(ref)}' sources") - app.remote_manager.get_recipe_sources(ref, recipe_layout, remote) + self._api_helpers.remote_manager.get_recipe_sources(ref, recipe_layout, remote) return True def package(self, pref: PkgReference, remote: Remote, metadata: Optional[List[str]] = None): @@ -53,28 +53,28 @@ def package(self, pref: PkgReference, remote: Remote, metadata: Optional[List[st If the package is already in the cache it will be skipped, but the specified metadata will be downloaded.""" output = ConanOutput() - app = ConanBasicApp(self._conan_api) + try: - app.cache.recipe_layout(pref.ref) # raises if not found + self._api_helpers.cache.recipe_layout(pref.ref) # raises if not found except ConanException: raise ConanException("The recipe of the specified package " "doesn't exist, download it first") - skip_download = app.cache.exists_prev(pref) + skip_download = self._api_helpers.cache.exists_prev(pref) if skip_download: output.info(f"Skip package {pref.repr_notime()} download, already in cache") if metadata: - app.remote_manager.get_package_metadata(pref, remote, metadata) + self._api_helpers.remote_manager.get_package_metadata(pref, remote, metadata) return False if pref.timestamp is None: # we didn't obtain the timestamp before (in general it should be) # Respect the timestamp of the server - server_pref = app.remote_manager.get_package_revision(pref, remote) + server_pref = self._api_helpers.remote_manager.get_package_revision(pref, remote) assert server_pref == pref pref.timestamp = server_pref.timestamp output.info(f"Downloading package '{pref.repr_notime()}'") - app.remote_manager.get_package(pref, remote, metadata) + self._api_helpers.remote_manager.get_package(pref, remote, metadata) return True def download_full(self, package_list: PackagesList, remote: Remote, @@ -94,7 +94,8 @@ def _download_pkglist(pkglist): pkg_dict.pop("upload-urls", None) t = time.time() - parallel = self._conan_api.config.get("core.download:parallel", default=1, check_type=int) + parallel = self._api_helpers.global_conf.get("core.download:parallel", default=1, + check_type=int) thread_pool = ThreadPool(parallel) if parallel > 1 else None if not thread_pool or len(package_list._data) <= 1: # FIXME: Iteration when multiple rrevs _download_pkglist(package_list) diff --git a/conan/api/subapi/export.py b/conan/api/subapi/export.py index 797364c1574..3901c3c0b8c 100644 --- a/conan/api/subapi/export.py +++ b/conan/api/subapi/export.py @@ -46,7 +46,7 @@ def export(self, path, name: str = None, version: str = None, user: str = None, ConanOutput().title("Exporting recipe to the cache") app = ConanApp(self._conan_api) hook_manager = self._helpers.hook_manager - return cmd_export(app.loader, app.cache, hook_manager, self._helpers.global_conf, path, + return cmd_export(app.loader,self._helpers.cache, hook_manager, self._helpers.global_conf, path, name, version, user, channel, graph_lock=lockfile, remotes=remotes) def export_pkg_graph(self, path, ref: RecipeReference, profile_host, profile_build, diff --git a/conan/api/subapi/graph.py b/conan/api/subapi/graph.py index 3fb81c3621a..24200b455ba 100644 --- a/conan/api/subapi/graph.py +++ b/conan/api/subapi/graph.py @@ -1,5 +1,5 @@ from conan.api.output import ConanOutput -from conan.internal.conan_app import ConanApp, ConanBasicApp +from conan.internal.conan_app import ConanApp from conan.internal.model.recipe_ref import ref_matches from conan.internal.graph.graph import Node, RECIPE_CONSUMER, CONTEXT_HOST, RECIPE_VIRTUAL, \ CONTEXT_BUILD, BINARY_MISSING, DepsGraph @@ -188,7 +188,8 @@ def load_graph(self, root_node, profile_host, profile_build, lockfile=None, remo assert profile_build is not None remotes = remotes or [] - builder = DepsGraphBuilder(app.proxy, app.loader, app.range_resolver, app.cache, remotes, + cache = self._conan_api._api_helpers.cache # noqa + builder = DepsGraphBuilder(app.proxy, app.loader, app.range_resolver, cache, remotes, update, check_update, self._conan_api._api_helpers.global_conf) deps_graph = builder.load_graph(root_node, profile_host, profile_build, lockfile) return deps_graph @@ -213,8 +214,10 @@ def analyze_binaries(self, graph, build_mode=None, remotes=None, update=None, :param tested_graph: In case of a "test_package", the graph being tested """ ConanOutput().title("Computing necessary packages") - conan_app = ConanBasicApp(self._conan_api) - binaries_analyzer = GraphBinariesAnalyzer(conan_app, self._conan_api._api_helpers.global_conf, + binaries_analyzer = GraphBinariesAnalyzer(self._helpers.cache, + self._helpers.remote_manager, + self._conan_api.home_folder, + self._helpers.global_conf, self._helpers.hook_manager) binaries_analyzer.evaluate_graph(graph, build_mode, lockfile, remotes, update, build_modes_test, tested_graph) diff --git a/conan/api/subapi/install.py b/conan/api/subapi/install.py index cf4cb60cc3e..806ec3f9bb5 100644 --- a/conan/api/subapi/install.py +++ b/conan/api/subapi/install.py @@ -40,7 +40,7 @@ def install_binaries(self, deps_graph, remotes: List[Remote] = None, return_inst :param return_install_error: If ``True``, do not raise an exception, but return it """ app = ConanBasicApp(self._conan_api) - installer = BinaryInstaller(app, self._helpers.global_conf, app.editable_packages, + installer = BinaryInstaller(self._conan_api, self._helpers.global_conf, app.editable_packages, self._helpers.hook_manager) install_graph = InstallGraph(deps_graph) install_graph.raise_errors() @@ -69,7 +69,7 @@ def install_system_requires(self, graph, only_info=False): :param only_info: If ``True``, only reporting and checking of whether the system requirements are installed is performed. """ app = ConanBasicApp(self._conan_api) - installer = BinaryInstaller(app, self._helpers.global_conf, app.editable_packages, + installer = BinaryInstaller(self._conan_api, self._helpers.global_conf, app.editable_packages, self._helpers.hook_manager) installer.install_system_requires(graph, only_info) @@ -90,7 +90,7 @@ def install_sources(self, graph, remotes: List[Remote]): :param graph: Dependency graph to download sources from """ app = ConanBasicApp(self._conan_api) - installer = BinaryInstaller(app, self._helpers.global_conf, app.editable_packages, + installer = BinaryInstaller(self._conan_api, self._helpers.global_conf, app.editable_packages, self._helpers.hook_manager) installer.install_sources(graph, remotes) diff --git a/conan/api/subapi/list.py b/conan/api/subapi/list.py index 1d3a0ddf8bb..724c449d0a4 100644 --- a/conan/api/subapi/list.py +++ b/conan/api/subapi/list.py @@ -5,7 +5,6 @@ from conan.api.model import PackagesList, MultiPackagesList, ListPattern, Remote from conan.api.output import ConanOutput, TimedOutput from conan.internal.api.list.query_parse import filter_package_configs -from conan.internal.conan_app import ConanBasicApp from conan.internal.model.recipe_ref import ref_matches from conan.internal.paths import CONANINFO from conan.internal.errors import NotFoundException @@ -47,18 +46,18 @@ class ListAPI: """ Get references from the recipes and packages in the cache or a remote """ - def __init__(self, conan_api): + def __init__(self, conan_api, api_helpers): self._conan_api = conan_api + self._api_helpers = api_helpers def latest_recipe_revision(self, ref: RecipeReference, remote: Remote = None): """ For a given recipe reference, return the latest revision of the recipe in the remote, or in the local cache if no remote is specified, or ``None`` if the recipe does not exist.""" assert ref.revision is None, "latest_recipe_revision: ref already have a revision" - app = ConanBasicApp(self._conan_api) if remote: - ret = app.remote_manager.get_latest_recipe_revision(ref, remote=remote) + ret = self._api_helpers.remote_manager.get_latest_recipe_revision(ref, remote=remote) else: - ret = app.cache.get_latest_recipe_revision(ref) + ret = self._api_helpers.cache.get_latest_recipe_revision(ref) return ret @@ -66,11 +65,10 @@ def recipe_revisions(self, ref: RecipeReference, remote: Remote = None): """ For a given recipe reference, return all the revisions of the recipe in the remote, or in the local cache if no remote is specified""" assert ref.revision is None, "recipe_revisions: ref already have a revision" - app = ConanBasicApp(self._conan_api) if remote: - results = app.remote_manager.get_recipe_revisions(ref, remote=remote) + results = self._api_helpers.remote_manager.get_recipe_revisions(ref, remote=remote) else: - results = app.cache.get_recipe_revisions(ref) + results = self._api_helpers.cache.get_recipe_revisions(ref) return results @@ -80,33 +78,30 @@ def latest_package_revision(self, pref: PkgReference, remote=None): # is used as an "exists" check too in other places, lets respect the None return assert pref.revision is None, "latest_package_revision: ref already have a revision" assert pref.package_id is not None, "package_id must be defined" - app = ConanBasicApp(self._conan_api) if remote: - ret = app.remote_manager.get_latest_package_revision(pref, remote=remote) + ret = self._api_helpers.remote_manager.get_latest_package_revision(pref, remote=remote) else: - ret = app.cache.get_latest_package_revision(pref) + ret = self._api_helpers.cache.get_latest_package_revision(pref) return ret def package_revisions(self, pref: PkgReference, remote=None): assert pref.ref.revision is not None, "package_revisions requires a recipe revision, " \ "check latest first if needed" - app = ConanBasicApp(self._conan_api) if remote: - results = app.remote_manager.get_package_revisions(pref, remote=remote) + results = self._api_helpers.remote_manager.get_package_revisions(pref, remote=remote) else: - results = app.cache.get_package_revisions(pref) + results = self._api_helpers.cache.get_package_revisions(pref) return results def _packages_configurations(self, ref: RecipeReference, remote=None) -> Dict[PkgReference, dict]: assert ref.revision is not None and ref.revision != "latest", \ "packages: ref should have a revision. Check latest if needed." - app = ConanBasicApp(self._conan_api) if not remote: - prefs = app.cache.get_package_references(ref) - packages = _get_cache_packages_binary_info(app.cache, prefs) + prefs = self._api_helpers.cache.get_package_references(ref) + packages = _get_cache_packages_binary_info(self._api_helpers.cache, prefs) else: - packages = app.remote_manager.search_packages(remote, ref) + packages = self._api_helpers.remote_manager.search_packages(remote, ref) return packages @staticmethod @@ -182,13 +177,13 @@ def select(self, pattern: ListPattern, package_query=None, remote: Remote = None select_bundle = PackagesList() # Avoid doing a ``search`` of recipes if it is an exact ref and it will be used later search_ref = pattern.search_ref - app = ConanBasicApp(self._conan_api) + cache = self._api_helpers.cache limit_time = _timelimit(lru) if lru else None out = ConanOutput() remote_name = "local cache" if not remote else remote.name if search_ref: - refs = _search_recipes(app, search_ref, remote=remote) - global_conf = self._conan_api._api_helpers.global_conf # noqa + refs = _search_recipes(self._api_helpers, search_ref, remote=remote) + global_conf = self._api_helpers.global_conf # noqa resolve_prereleases = global_conf.get("core.version_ranges:resolve_prereleases") refs = pattern.filter_versions(refs, resolve_prereleases) pattern.check_refs(refs) @@ -219,7 +214,7 @@ def msg_format(msg, item, total): rrevs = list(reversed(rrevs)) # Order older revisions first if lru and pattern.package_id is None: # Filter LRUs - rrevs = [r for r in rrevs if app.cache.get_recipe_lru(r) < limit_time] + rrevs = [r for r in rrevs if cache.get_recipe_lru(r) < limit_time] for rr in rrevs: select_bundle.add_ref(rr) @@ -261,7 +256,7 @@ def msg_format(msg, item, total): prefs = new_prefs if lru: # Filter LRUs - prefs = [r for r in prefs if app.cache.get_package_lru(r) < limit_time] + prefs = [r for r in prefs if cache.get_package_lru(r) < limit_time] # Packages dict has been listed, even if empty select_bundle.recipe_dict(rrev)["packages"] = {} @@ -317,13 +312,13 @@ def find_remotes(self, package_list, remotes): (Experimental) Find the remotes where the current package lists can be found """ result = MultiPackagesList() - app = ConanBasicApp(self._conan_api) + remote_manager = self._api_helpers.remote_manager for r in remotes: result_pkg_list = PackagesList() for ref, ref_contents in package_list.serialize().items(): ref = RecipeReference.loads(ref) try: - remote_rrevs = app.remote_manager.get_recipe_revisions(ref, remote=r) + remote_rrevs = remote_manager.get_recipe_revisions(ref, remote=r) except NotFoundException: continue revisions = ref_contents.get("revisions") @@ -345,7 +340,7 @@ def find_remotes(self, package_list, remotes): for pkgid, pkgcontent in packages.items(): pref = PkgReference(ref, pkgid) try: - remote_prefs = app.remote_manager.get_package_revisions(pref, remote=r) + remote_prefs = remote_manager.get_package_revisions(pref, remote=r) except NotFoundException: continue pkg_revisions = pkgcontent.get("revisions") @@ -551,16 +546,16 @@ def _get_cache_packages_binary_info(cache, prefs) -> Dict[PkgReference, dict]: return result -def _search_recipes(app, query: str, remote=None): +def _search_recipes(api_helpers, query: str, remote=None): only_none_user_channel = False if query and query.endswith("@"): only_none_user_channel = True query = query[:-1] if remote: - refs = app.remote_manager.search_recipes(remote, query) + refs = api_helpers.remote_manager.search_recipes(remote, query) else: - refs = app.cache.search_recipes(query) + refs = api_helpers.cache.search_recipes(query) ret = [] for r in refs: if not only_none_user_channel or (r.user is None and r.channel is None): diff --git a/conan/api/subapi/remotes.py b/conan/api/subapi/remotes.py index 30e754b0694..51dd9f5be94 100644 --- a/conan/api/subapi/remotes.py +++ b/conan/api/subapi/remotes.py @@ -7,7 +7,6 @@ from conan.api.model import Remote, LOCAL_RECIPES_INDEX from conan.api.output import ConanOutput from conan.internal.cache.home_paths import HomePaths -from conan.internal.conan_app import ConanBasicApp from conan.internal.rest.remote_credentials import RemoteCredentials from conan.internal.rest.rest_client_local_recipe_index import add_local_recipes_index_remote, \ remove_local_recipes_index_remote @@ -228,8 +227,7 @@ def user_login(self, remote: Remote, username: str, password: str): :param username: the user login as ``str`` :param password: password ``str`` """ - app = ConanBasicApp(self._conan_api) - app.remote_manager.authenticate(remote, username, password) + self._api_helpers.remote_manager.authenticate(remote, username, password) def login(self, remotes, username=None, password=None): creds = RemoteCredentials(self._conan_api.cache_folder, self._api_helpers.global_conf) @@ -271,7 +269,6 @@ def user_set(self, remote: Remote, username): def user_auth(self, remote: Remote, with_user=False, force=False): # TODO: Review localdb = LocalDB(self._home_folder) - app = ConanBasicApp(self._conan_api) if with_user: user, token, _ = localdb.get_login(remote.url) if not user: @@ -279,7 +276,7 @@ def user_auth(self, remote: Remote, with_user=False, force=False): user = os.getenv(var_name, None) or os.getenv("CONAN_LOGIN_USERNAME", None) if not user: return - app.remote_manager.check_credentials(remote, force) + self._api_helpers.remote_manager.check_credentials(remote, force) user, token, _ = localdb.get_login(remote.url) return user diff --git a/conan/api/subapi/remove.py b/conan/api/subapi/remove.py index 1abde464766..89f5132f2aa 100644 --- a/conan/api/subapi/remove.py +++ b/conan/api/subapi/remove.py @@ -1,53 +1,51 @@ from typing import Optional from conan.api.model import Remote -from conan.internal.conan_app import ConanBasicApp from conan.api.model import PkgReference from conan.api.model import RecipeReference class RemoveAPI: - def __init__(self, conan_api): + def __init__(self, conan_api, api_helpers): self._conan_api = conan_api + self._api_helpers = api_helpers def recipe(self, ref: RecipeReference, remote: Optional[Remote] = None): assert ref.revision, "Recipe revision cannot be None to remove a recipe" """Removes the recipe (or recipe revision if present) and all the packages (with all prev)""" - app = ConanBasicApp(self._conan_api) + if remote: - app.remote_manager.remove_recipe(ref, remote) + self._api_helpers.remote_manager.remove_recipe(ref, remote) else: self.all_recipe_packages(ref) - recipe_layout = app.cache.recipe_layout(ref) - app.cache.remove_recipe_layout(recipe_layout) + recipe_layout = self._api_helpers.cache.recipe_layout(ref) + self._api_helpers.cache.remove_recipe_layout(recipe_layout) def all_recipe_packages(self, ref: RecipeReference, remote: Optional[Remote] = None): assert ref.revision, "Recipe revision cannot be None to remove a recipe" """Removes all the packages from the provided reference""" - app = ConanBasicApp(self._conan_api) + if remote: - app.remote_manager.remove_all_packages(ref, remote) + self._api_helpers.remote_manager.remove_all_packages(ref, remote) else: # Remove all the prefs with all the prevs - self._remove_all_local_packages(app, ref) + self._remove_all_local_packages(ref) - @staticmethod - def _remove_all_local_packages(app, ref): + def _remove_all_local_packages(self, ref): # Get all the prefs and all the prevs - pkg_ids = app.cache.get_package_references(ref, only_latest_prev=False) + pkg_ids = self._api_helpers.cache.get_package_references(ref, only_latest_prev=False) for pref in pkg_ids: - package_layout = app.cache.pkg_layout(pref) - app.cache.remove_package_layout(package_layout) + package_layout = self._api_helpers.cache.pkg_layout(pref) + self._api_helpers.cache.remove_package_layout(package_layout) def package(self, pref: PkgReference, remote: Optional[Remote]): assert pref.ref.revision, "Recipe revision cannot be None to remove a package" assert pref.revision, "Package revision cannot be None to remove a package" - app = ConanBasicApp(self._conan_api) if remote: # FIXME: Create a "packages" method to optimize remote remove? - app.remote_manager.remove_packages([pref], remote) + self._api_helpers.remote_manager.remove_packages([pref], remote) else: - package_layout = app.cache.pkg_layout(pref) - app.cache.remove_package_layout(package_layout) + package_layout = self._api_helpers.cache.pkg_layout(pref) + self._api_helpers.cache.remove_package_layout(package_layout) diff --git a/conan/api/subapi/report.py b/conan/api/subapi/report.py index 134a3f479dc..7fe903411f2 100644 --- a/conan/api/subapi/report.py +++ b/conan/api/subapi/report.py @@ -98,7 +98,8 @@ def _configure_source(conan_api, hook_manager, conanfile_path, ref, remotes): with conanfile_exception_formatter(conanfile, "layout"): conanfile.layout() - recipe_layout = app.cache.recipe_layout(ref) + cache = conan_api._api_helpers.cache # noqa + recipe_layout = cache.recipe_layout(ref) export_source_folder = recipe_layout.export_sources() source_folder = recipe_layout.source() diff --git a/conan/api/subapi/upload.py b/conan/api/subapi/upload.py index 53eeb38d7fc..1fc18bcd593 100644 --- a/conan/api/subapi/upload.py +++ b/conan/api/subapi/upload.py @@ -38,7 +38,7 @@ def check_upstream(self, package_list: PackagesList, remote: Remote, """ app = ConanApp(self._conan_api) for ref, _ in package_list.items(): - layout = app.cache.recipe_layout(ref) + layout = self._api_helpers.cache.recipe_layout(ref) conanfile_path = layout.conanfile() conanfile = app.loader.load_basic(conanfile_path, remotes=enabled_remotes) if conanfile.upload_policy == "skip": @@ -46,7 +46,7 @@ def check_upstream(self, package_list: PackagesList, remote: Remote, "because upload_policy='skip'") package_list.recipe_dict(ref)["packages"] = {} - UploadUpstreamChecker(app).check(package_list, remote, force) + UploadUpstreamChecker(self._api_helpers.remote_manager).check(package_list, remote, force) def prepare(self, package_list: PackagesList, enabled_remotes: List[Remote], metadata: List[str] = None): @@ -64,18 +64,19 @@ def prepare(self, package_list: PackagesList, enabled_remotes: List[Remote], if metadata and metadata != [''] and '' in metadata: raise ConanException("Empty string and patterns can not be mixed for metadata.") app = ConanApp(self._conan_api) - preparator = PackagePreparator(app, self._api_helpers.global_conf) + preparator = PackagePreparator(app, self._api_helpers.cache, + self._api_helpers.remote_manager, + self._api_helpers.global_conf) preparator.prepare(package_list, enabled_remotes, metadata) - signer = PkgSignaturesPlugin(app.cache, app.cache_folder) + signer = PkgSignaturesPlugin(self._api_helpers.cache, self._conan_api.home_folder) if signer.is_sign_configured: ConanOutput().warning("[Package sign] Implicitly signing packages in the upload " "command has been removed. Use 'conan cache sign' command before " "uploading instead.", warn_tag="deprecated") def _upload(self, package_list, remote): - app = ConanApp(self._conan_api) - app.remote_manager.check_credentials(remote) - executor = UploadExecutor(app) + self._api_helpers.remote_manager.check_credentials(remote) + executor = UploadExecutor(self._api_helpers.remote_manager) executor.upload(package_list, remote) def upload_full(self, package_list: PackagesList, remote: Remote, enabled_remotes: List[Remote], diff --git a/conan/api/subapi/workspace.py b/conan/api/subapi/workspace.py index d2f56bac545..0394f1829cd 100644 --- a/conan/api/subapi/workspace.py +++ b/conan/api/subapi/workspace.py @@ -159,7 +159,8 @@ def open(self, ref, remotes, cwd=None): conanfile.output.warning("conandata doesn't contain 'scm' information\n" "doing a local copy!!!") shutil.copytree(layout.export(), dst_path) - retrieve_exports_sources(app.remote_manager, layout, conanfile, ref, remotes) + remote_manager = self._conan_api._api_helpers.remote_manager # noqa + retrieve_exports_sources(remote_manager, layout, conanfile, ref, remotes) export_sources = layout.export_sources() if os.path.exists(export_sources): conanfile.output.warning("There are export-sources, copying them, but the location" diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 6186a31f4fa..a5b70603224 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -26,8 +26,8 @@ class UploadUpstreamChecker: This is completely irrespective of the actual package contents, it only uses the local computed revision and the remote one """ - def __init__(self, app: ConanApp): - self._app = app + def __init__(self, remote_manager): + self._remote_manager = remote_manager def check(self, package_list, remote, force): for ref, packages in package_list.items(): @@ -43,7 +43,7 @@ def _check_upstream_recipe(self, ref, ref_bundle, remote, force): try: assert ref.revision # TODO: It is a bit ugly, interface-wise to ask for revisions to check existence - server_ref = self._app.remote_manager.get_recipe_revision(ref, remote) + server_ref = self._remote_manager.get_recipe_revision(ref, remote) assert server_ref # If successful (not raising NotFoundException), this will exist except NotFoundException: ref_bundle["force_upload"] = False @@ -64,7 +64,7 @@ def _check_upstream_package(self, pref, prev_bundle, remote, force): try: # TODO: It is a bit ugly, interface-wise to ask for revisions to check existence - server_revisions = self._app.remote_manager.get_package_revision(pref, remote) + server_revisions = self._remote_manager.get_package_revision(pref, remote) assert server_revisions except NotFoundException: prev_bundle["force_upload"] = False @@ -104,8 +104,10 @@ def get_compress_level(compressformat, global_conf): class PackagePreparator: - def __init__(self, app: ConanApp, global_conf): + def __init__(self, app: ConanApp, cache, remote_manager, global_conf): self._app = app + self._remote_manager = remote_manager + self._cache = cache self._global_conf = global_conf compressformat = self._global_conf.get("core.upload:compression_format", default="gz", choices=COMPRESSIONS) @@ -118,7 +120,7 @@ def __init__(self, app: ConanApp, global_conf): def prepare(self, pkg_list, enabled_remotes, metadata, force=False): local_url = self._global_conf.get("core.scm:local_url", choices=["allow", "block"]) for ref, packages in pkg_list.items(): - recipe_layout = self._app.cache.recipe_layout(ref) + recipe_layout = self._cache.recipe_layout(ref) conanfile_path = recipe_layout.conanfile() conanfile = self._app.loader.load_basic(conanfile_path) url = conanfile.conan_data.get("scm", {}).get("url") if conanfile.conan_data else None @@ -156,7 +158,7 @@ def _prepare_recipe(self, recipe_layout, ref, ref_bundle, conanfile, remotes): - compress the artifacts in conan_export.tgz and conan_export_sources.tgz """ try: - retrieve_exports_sources(self._app.remote_manager, recipe_layout, conanfile, ref, + retrieve_exports_sources(self._remote_manager, recipe_layout, conanfile, ref, remotes) cache_files = self._compress_recipe_files(recipe_layout, ref) ref_bundle["files"] = cache_files @@ -201,7 +203,7 @@ def _compress_recipe_files(self, layout, ref): def _prepare_package(self, pref, prev_bundle, metadata, force=False): pkg_layout = None if prev_bundle.get("upload") or force: - pkg_layout = self._app.cache.pkg_layout(pref) + pkg_layout = self._cache.pkg_layout(pref) if pkg_layout.package_is_dirty(): raise ConanException(f"Package {pref} is corrupted, aborting upload.\n" f"Remove it with 'conan remove {pref}'") @@ -210,7 +212,7 @@ def _prepare_package(self, pref, prev_bundle, metadata, force=False): # Package metadata files too if metadata != [""] and (metadata or prev_bundle.get("upload")): - pkg_layout = pkg_layout or self._app.cache.pkg_layout(pref) + pkg_layout = pkg_layout or self._cache.pkg_layout(pref) metadata_folder = pkg_layout.metadata() files = _metadata_files(metadata_folder, metadata) if files: @@ -283,8 +285,8 @@ class UploadExecutor: been computed and are passed in the ``upload_data`` parameter, so this executor is also agnostic about which files are transferred """ - def __init__(self, app: ConanApp): - self._app = app + def __init__(self, remote_manager): + self._remote_manager = remote_manager def upload(self, upload_data, remote): for ref, packages in upload_data.items(): @@ -303,7 +305,7 @@ def upload_recipe(self, ref, bundle, remote): output.info(f"Uploading recipe '{ref.repr_notime()}' ({_total_size(cache_files)})") t1 = time.time() - self._app.remote_manager.upload_recipe(ref, cache_files, remote) + self._remote_manager.upload_recipe(ref, cache_files, remote) duration = time.time() - t1 output.debug(f"Upload {ref} in {duration} time") @@ -318,7 +320,7 @@ def upload_package(self, pref, prev_bundle, remote): output.info(f"Uploading package '{pref.repr_notime()}' ({_total_size(cache_files)})") t1 = time.time() - self._app.remote_manager.upload_package(pref, cache_files, remote) + self._remote_manager.upload_package(pref, cache_files, remote) duration = time.time() - t1 output.debug(f"Upload {pref} in {duration} time") diff --git a/conan/internal/conan_app.py b/conan/internal/conan_app.py index 77f458f07a2..3dba2e69b31 100644 --- a/conan/internal/conan_app.py +++ b/conan/internal/conan_app.py @@ -4,13 +4,11 @@ from conan.internal.cache.cache import PkgCache from conan.internal.cache.home_paths import HomePaths from conan.internal.model.conf import ConfDefinition -from conan.internal.rest.auth_manager import ConanApiAuthManager from conan.internal.graph.proxy import ConanProxy from conan.internal.graph.python_requires import PyRequireLoader from conan.internal.graph.range_resolver import RangeResolver from conan.internal.loader import ConanFileLoader, load_python_file from conan.internal.rest.remote_manager import RemoteManager -from conan.internal.api.remotes.localdb import LocalDB class CmdWrapper: @@ -28,54 +26,13 @@ def wrap(self, cmd, conanfile, **kwargs): class ConanFileHelpers: - def __init__(self, requester, cmd_wrapper, global_conf, cache, home_folder): + def __init__(self, requester, cmd_wrapper, global_conf, cache, home_folder, conan_api): self.requester = requester self.cmd_wrapper = cmd_wrapper self.global_conf = global_conf self.cache = cache self.home_folder = home_folder - - -class ConanBasicApp: - def __init__(self, conan_api): - """ Needs: - - Global configuration - - Cache home folder - """ - # TODO: Remove this global_conf from here - global_conf = conan_api._api_helpers.global_conf # noqa - self._global_conf = global_conf - self.conan_api = conan_api - cache_folder = conan_api.home_folder - self.cache_folder = cache_folder - self.cache = PkgCache(self.cache_folder, global_conf) - # Wraps RestApiClient to add authentication support (same interface) - localdb = LocalDB(cache_folder) - requester = conan_api._api_helpers.requester # noqa - auth_manager = ConanApiAuthManager(requester, cache_folder, localdb, global_conf) - # Handle remote connections - compress = conan_api._api_helpers.compression_plugin # noqa - self.remote_manager = RemoteManager(self.cache, auth_manager, cache_folder, compress) - global_editables = conan_api.local.editable_packages - ws_editables = conan_api.workspace.packages() - self.editable_packages = global_editables.update_copy(ws_editables) - - -class ConanApp(ConanBasicApp): - def __init__(self, conan_api): - """ Needs: - - LocalAPI to read editable packages - """ - super().__init__(conan_api) - legacy_update = self._global_conf.get("core:update_policy", choices=["legacy"]) - self.proxy = ConanProxy(self, self.editable_packages, legacy_update=legacy_update) - self.range_resolver = RangeResolver(self, self._global_conf, self.editable_packages) - - self.pyreq_loader = PyRequireLoader(self, self._global_conf) - cmd_wrap = CmdWrapper(HomePaths(self.cache_folder).wrapper_path) - requester = conan_api._api_helpers.requester # noqa - conanfile_helpers = ConanFileHelpers(requester, cmd_wrap, self._global_conf, self.cache, self.cache_folder) - self.loader = ConanFileLoader(self.pyreq_loader, conanfile_helpers) + self.conan_api = conan_api # Might be None for local-recipes-index class LocalRecipesIndexApp: @@ -90,8 +47,9 @@ def __init__(self, cache_folder): self.cache = PkgCache(cache_folder, self.global_conf) self.remote_manager = RemoteManager(self.cache, auth_manager=None, home_folder=cache_folder) editable_packages = EditablePackages() - self.proxy = ConanProxy(self, editable_packages) - self.range_resolver = RangeResolver(self, self.global_conf, editable_packages) - pyreq_loader = PyRequireLoader(self, self.global_conf) - helpers = ConanFileHelpers(None, CmdWrapper(""), self.global_conf, self.cache, cache_folder) + self.proxy = ConanProxy(self.cache, self.remote_manager, editable_packages) + self.range_resolver = RangeResolver(self.cache, self.remote_manager, self.global_conf, + editable_packages) + pyreq_loader = PyRequireLoader(self.proxy, self.range_resolver, self.global_conf) + helpers = ConanFileHelpers(None, CmdWrapper(""), self.global_conf, self.cache, cache_folder, None) self.loader = ConanFileLoader(pyreq_loader, helpers) diff --git a/conan/internal/graph/graph_binaries.py b/conan/internal/graph/graph_binaries.py index afa14528557..09889f638f6 100644 --- a/conan/internal/graph/graph_binaries.py +++ b/conan/internal/graph/graph_binaries.py @@ -21,15 +21,15 @@ class GraphBinariesAnalyzer: - def __init__(self, conan_app, global_conf, hook_manager): - self._cache = conan_app.cache - self._home_folder = conan_app.cache_folder + def __init__(self, cache, remote_manager, home_folder, global_conf, hook_manager): + self._cache = cache + self._home_folder = home_folder self._global_conf = global_conf - self._remote_manager = conan_app.remote_manager + self._remote_manager = remote_manager self._hook_manager = hook_manager # These are the nodes with pref (not including PREV) that have been evaluated self._evaluated = {} # {pref: [nodes]} - compat_folder = HomePaths(conan_app.cache_folder).compatibility_plugin_path + compat_folder = HomePaths(home_folder).compatibility_plugin_path self._compatibility = BinaryCompatibility(compat_folder, hook_manager) unknown_mode = global_conf.get("core.package_id:default_unknown_mode", default="semver_mode") non_embed = global_conf.get("core.package_id:default_non_embed_mode", default="minor_mode") diff --git a/conan/internal/graph/installer.py b/conan/internal/graph/installer.py index 230f61e668f..384519d4449 100644 --- a/conan/internal/graph/installer.py +++ b/conan/internal/graph/installer.py @@ -34,11 +34,11 @@ def build_id(conan_file): class _PackageBuilder: - def __init__(self, app, hook_manager): - self._cache = app.cache + def __init__(self, cache, remote_manager, cache_folder, hook_manager): + self._cache = cache self._hook_manager = hook_manager - self._remote_manager = app.remote_manager - self._home_folder = app.cache_folder + self._remote_manager = remote_manager + self._home_folder = cache_folder def _get_build_folder(self, conanfile, package_layout): # Build folder can use a different package_ID if build_id() is defined. @@ -165,14 +165,14 @@ class BinaryInstaller: locally in case they are not found in remotes """ - def __init__(self, app, global_conf, editable_packages, hook_manager): - self._app = app + def __init__(self, api, global_conf, editable_packages, hook_manager): + helpers = api._api_helpers # noqa self._editable_packages = editable_packages - self._cache = app.cache - self._remote_manager = app.remote_manager + self._cache = helpers.cache + self._remote_manager = helpers.remote_manager self._hook_manager = hook_manager self._global_conf = global_conf - self._home_folder = app.cache_folder + self._home_folder = api.home_folder def _install_source(self, node, remotes, need_conf=False): conanfile = node.conanfile @@ -394,7 +394,8 @@ def _handle_node_build(self, package, recipe_layout, pkg_layout): with pkg_layout.package_lock(): pkg_layout.package_remove() with pkg_layout.set_dirty_context_manager(): - builder = _PackageBuilder(self._app, self._hook_manager) + builder = _PackageBuilder(self._cache, self._remote_manager, self._home_folder, + self._hook_manager) pref = builder.build_package(node, recipe_layout, pkg_layout) assert node.prev, "Node PREV shouldn't be empty" assert node.pref.revision, "Node PREF revision shouldn't be empty" diff --git a/conan/internal/graph/proxy.py b/conan/internal/graph/proxy.py index f47c11f23e7..2fa1fcb3b73 100644 --- a/conan/internal/graph/proxy.py +++ b/conan/internal/graph/proxy.py @@ -8,11 +8,11 @@ class ConanProxy: - def __init__(self, conan_app, editable_packages, legacy_update=None): + def __init__(self, cache, remote_manager, editable_packages, legacy_update=None): # collaborators self._editable_packages = editable_packages - self._cache = conan_app.cache - self._remote_manager = conan_app.remote_manager + self._cache = cache + self._remote_manager = remote_manager self._resolved = {} # Cache of the requested recipes to optimize calls self._legacy_update = legacy_update diff --git a/conan/internal/graph/python_requires.py b/conan/internal/graph/python_requires.py index 689bbb9f868..517f6c1ce1a 100644 --- a/conan/internal/graph/python_requires.py +++ b/conan/internal/graph/python_requires.py @@ -65,9 +65,9 @@ def add_pyrequire(self, py_require): class PyRequireLoader: - def __init__(self, conan_app, global_conf): - self._proxy = conan_app.proxy - self._range_resolver = conan_app.range_resolver + def __init__(self, proxy, range_resolver, global_conf): + self._proxy = proxy + self._range_resolver = range_resolver self._cached_py_requires = {} self._resolve_prereleases = global_conf.get("core.version_ranges:resolve_prereleases") diff --git a/conan/internal/graph/range_resolver.py b/conan/internal/graph/range_resolver.py index f6d3f5795df..f5938d489f5 100644 --- a/conan/internal/graph/range_resolver.py +++ b/conan/internal/graph/range_resolver.py @@ -7,10 +7,10 @@ class RangeResolver: - def __init__(self, conan_app, global_conf, editable_packages): - self._cache = conan_app.cache + def __init__(self, cache, remote_manager, global_conf, editable_packages): + self._cache = cache self._editable_packages = editable_packages - self._remote_manager = conan_app.remote_manager + self._remote_manager = remote_manager self._cached_cache = {} # Cache caching of search result, so invariant wrt installations self._cached_remote_found = {} # dict {ref (pkg/*): {remote_name: results (pkg/1, pkg/2)}} self.resolved_ranges = {} diff --git a/conan/internal/model/workspace.py b/conan/internal/model/workspace.py index 41c793ee4c0..80b96f64978 100644 --- a/conan/internal/model/workspace.py +++ b/conan/internal/model/workspace.py @@ -118,7 +118,7 @@ def load_conanfile(self, conanfile_path): from conan.internal.conan_app import ConanFileHelpers, CmdWrapper cmd_wrap = CmdWrapper(HomePaths(self._conan_api.home_folder).wrapper_path) helpers = ConanFileHelpers(None, cmd_wrap, self._conan_api._api_helpers.global_conf, - cache=None, home_folder=self._conan_api.home_folder) + cache=None, home_folder=self._conan_api.home_folder, conan_api=self._conan_api) loader = ConanFileLoader(pyreq_loader=None, conanfile_helpers=helpers) conanfile = loader.load_named(conanfile_path, name=None, version=None, user=None, channel=None, remotes=None, graph_lock=None) diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index 16ec33ab22f..c0cb085374b 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -12,7 +12,7 @@ from conan.internal.rest.rest_client_local_recipe_index import RestApiClientLocalRecipesIndex from conan.api.model import Remote from conan.api.output import ConanOutput -from conan.internal.cache.conan_reference_layout import METADATA, PackageLayout +from conan.internal.cache.conan_reference_layout import METADATA from conan.internal.rest.pkg_sign import PkgSignaturesPlugin from conan.internal.errors import ConanConnectionError, NotFoundException, PackageNotFoundException from conan.errors import ConanException diff --git a/conan/test/utils/mocks.py b/conan/test/utils/mocks.py index 0c16a7c4ecd..8d83b13720d 100644 --- a/conan/test/utils/mocks.py +++ b/conan/test/utils/mocks.py @@ -72,7 +72,7 @@ def __init__(self, settings=None, options=None, runner=None, display_name=""): self.win_bash = None self.command = None self._commands = [] - self._conan_helpers = ConanFileHelpers(None, None, self.conf, None, None) + self._conan_helpers = ConanFileHelpers(None, None, self.conf, None, None, None) def run(self, *args, **kwargs): self.command = args[0] diff --git a/test/integration/cache/storage_path_test.py b/test/integration/cache/storage_path_test.py index 2cac6d7b2ee..f326aab50e2 100644 --- a/test/integration/cache/storage_path_test.py +++ b/test/integration/cache/storage_path_test.py @@ -22,7 +22,7 @@ def test_storage_path(): def test_wrong_home_error(): client = TestClient(light=True) client.save_home({"global.conf": "core.cache:storage_path=//"}) - client.run("list *") + client.run("list *", assert_error=True) assert "Couldn't initialize storage in" in client.out From 462a5236b4dd6a8da20dd5e00121644c4a9c65f5 Mon Sep 17 00:00:00 2001 From: rikka0612 <104909326+ReinerBRO@users.noreply.github.com> Date: Fri, 20 Mar 2026 18:01:11 +0800 Subject: [PATCH 056/110] CMakeToolchain: disable CMake package registry exports (#19766) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * CMakeToolchain: disable package registry exports * Limit CMake package registry override to supported variable * Apply suggestions from code review Co-authored-by: Abril Rincón Blanco <5364255+AbrilRBS@users.noreply.github.com> * Move package registry handling to output dirs --------- Co-authored-by: Abril Rincón Blanco <5364255+AbrilRBS@users.noreply.github.com> --- conan/tools/cmake/toolchain/blocks.py | 7 +++++++ .../toolchains/cmake/test_cmaketoolchain.py | 12 ++++++++++++ 2 files changed, 19 insertions(+) diff --git a/conan/tools/cmake/toolchain/blocks.py b/conan/tools/cmake/toolchain/blocks.py index d798e9a8753..c10ff941305 100644 --- a/conan/tools/cmake/toolchain/blocks.py +++ b/conan/tools/cmake/toolchain/blocks.py @@ -1260,6 +1260,13 @@ def template(self): return textwrap.dedent("""\ # Definition of CMAKE_INSTALL_XXX folders + # Ensure export(PACKAGE) honors CMAKE_EXPORT_PACKAGE_REGISTRY even if the + # project sets cmake_minimum_required() lower than 3.15. + cmake_policy(SET CMP0090 NEW) + if(NOT DEFINED CMAKE_EXPORT_PACKAGE_REGISTRY) + set(CMAKE_EXPORT_PACKAGE_REGISTRY OFF) + endif() + {% if package_folder %} set(CMAKE_INSTALL_PREFIX "{{package_folder}}") {% endif %} diff --git a/test/integration/toolchains/cmake/test_cmaketoolchain.py b/test/integration/toolchains/cmake/test_cmaketoolchain.py index dccd2708243..94af6901812 100644 --- a/test/integration/toolchains/cmake/test_cmaketoolchain.py +++ b/test/integration/toolchains/cmake/test_cmaketoolchain.py @@ -427,6 +427,18 @@ def test_runtime_lib_dirs_multiconf(lib_dir_setup): assert "" in runtime_lib_dirs +def test_disable_package_registry(): + # https://github.com/conan-io/conan/issues/19749 + client = TestClient(light=True) + client.save({"conanfile.txt": "[generators]\nCMakeToolchain"}) + client.run("install .") + toolchain = client.load("conan_toolchain.cmake") + before_output_dirs, output_dirs_block = toolchain.split("########## 'output_dirs' block #############\n", 1) + assert "CMAKE_EXPORT_PACKAGE_REGISTRY" not in before_output_dirs + assert "cmake_policy(SET CMP0090 NEW)" in output_dirs_block + assert "set(CMAKE_EXPORT_PACKAGE_REGISTRY OFF)" in toolchain + + @pytest.mark.skipif(platform.system() != "Darwin", reason="Only OSX") def test_cmaketoolchain_cmake_system_processor_cross_apple(): """ From 217fc0011940272abdfa7f9de4d624207e6040f4 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 20 Mar 2026 11:07:19 +0100 Subject: [PATCH 057/110] adding rcflags (#19693) --- conan/internal/model/conf.py | 1 + conan/tools/cmake/toolchain/blocks.py | 11 ++++++ conan/tools/gnu/autotoolstoolchain.py | 7 ++++ conan/tools/gnu/gnutoolchain.py | 7 ++++ conan/tools/microsoft/nmaketoolchain.py | 7 ++++ conan/tools/microsoft/toolchain.py | 10 ++++-- conan/tools/premake/toolchain.py | 8 +++++ conan/tools/qbs/qbsprofile.py | 1 + .../toolchains/cmake/test_cmaketoolchain.py | 25 +++++++++++++ .../toolchains/gnu/test_autotoolstoolchain.py | 27 ++++++++++++++ .../toolchains/gnu/test_gnutoolchain.py | 25 +++++++++++++ .../microsoft/test_msbuildtoolchain.py | 19 ++++++++++ .../microsoft/test_nmaketoolchain.py | 28 +++++++++++++++ .../premake/test_premaketoolchain.py | 26 ++++++++++++++ .../toolchains/qbs/test_qbsprofile.py | 35 +++++++++++++++++++ 15 files changed, 235 insertions(+), 2 deletions(-) create mode 100644 test/integration/toolchains/microsoft/test_nmaketoolchain.py create mode 100644 test/integration/toolchains/qbs/test_qbsprofile.py diff --git a/conan/internal/model/conf.py b/conan/internal/model/conf.py index 4aa6ff6d365..0ca7848a434 100644 --- a/conan/internal/model/conf.py +++ b/conan/internal/model/conf.py @@ -154,6 +154,7 @@ "tools.build:defines": "List of extra definition flags used by different toolchains like CMakeToolchain, AutotoolsToolchain and MesonToolchain", "tools.build:sharedlinkflags": "List of extra flags used by different toolchains like CMakeToolchain, AutotoolsToolchain and MesonToolchain", "tools.build:exelinkflags": "List of extra flags used by different toolchains like CMakeToolchain, AutotoolsToolchain and MesonToolchain", + "tools.build:rcflags": "List of extra RC (resource compiler) flags used by different toolchains like CMakeToolchain, MSBuildToolchain and MesonToolchain", "tools.build:linker_scripts": "List of linker script files to pass to the linker used by different toolchains like CMakeToolchain, AutotoolsToolchain, and MesonToolchain", # Toolchain installation "tools.build:install_strip": "(boolean) Strip the binaries when installing them with CMake, Meson and Autotools", diff --git a/conan/tools/cmake/toolchain/blocks.py b/conan/tools/cmake/toolchain/blocks.py index c10ff941305..357e8f5d7c4 100644 --- a/conan/tools/cmake/toolchain/blocks.py +++ b/conan/tools/cmake/toolchain/blocks.py @@ -787,6 +787,9 @@ class ExtraFlagsBlock(Block): {% if exelinkflags %} string(APPEND CONAN_EXE_LINKER_FLAGS{{suffix}} "{% for exelinkflag in exelinkflags %} {{ exelinkflag }}{% endfor %}") {% endif %} + {% if rcflags %} + string(APPEND CONAN_RC_FLAGS{{suffix}} "{% for rcflag in rcflags %} {{ rcflag }}{% endfor %}") + {% endif %} {% if defines %} {% if config %} {% for define in defines %} @@ -837,6 +840,7 @@ def context(self): cflags = self._toolchain.extra_cflags + self._conanfile.conf.get("tools.build:cflags", default=[], check_type=list) sharedlinkflags = self._toolchain.extra_sharedlinkflags + self._conanfile.conf.get("tools.build:sharedlinkflags", default=[], check_type=list) exelinkflags = self._toolchain.extra_exelinkflags + self._conanfile.conf.get("tools.build:exelinkflags", default=[], check_type=list) + rcflags = self._conanfile.conf.get("tools.build:rcflags", default=[], check_type=list) defines = self._conanfile.conf.get("tools.build:defines", default=[], check_type=list) # See https://github.com/conan-io/conan/issues/13374 @@ -859,6 +863,7 @@ def context(self): "cflags": cflags, "sharedlinkflags": sharedlinkflags, "exelinkflags": exelinkflags, + "rcflags": rcflags, "defines": [define.replace('"', '\\"') for define in defines], } @@ -881,6 +886,9 @@ class CMakeFlagsInitBlock(Block): if(DEFINED CONAN_EXE_LINKER_FLAGS_${config}) string(APPEND CMAKE_EXE_LINKER_FLAGS_${config}_INIT " ${CONAN_EXE_LINKER_FLAGS_${config}}") endif() + if(DEFINED CONAN_RC_FLAGS_${config}) + string(APPEND CMAKE_RC_FLAGS_${config}_INIT " ${CONAN_RC_FLAGS_${config}}") + endif() endforeach() if(DEFINED CONAN_CXX_FLAGS) @@ -895,6 +903,9 @@ class CMakeFlagsInitBlock(Block): if(DEFINED CONAN_EXE_LINKER_FLAGS) string(APPEND CMAKE_EXE_LINKER_FLAGS_INIT " ${CONAN_EXE_LINKER_FLAGS}") endif() + if(DEFINED CONAN_RC_FLAGS) + string(APPEND CMAKE_RC_FLAGS_INIT " ${CONAN_RC_FLAGS}") + endif() if(DEFINED CONAN_OBJCXX_FLAGS) string(APPEND CMAKE_OBJCXX_FLAGS_INIT " ${CONAN_OBJCXX_FLAGS}") endif() diff --git a/conan/tools/gnu/autotoolstoolchain.py b/conan/tools/gnu/autotoolstoolchain.py index 52b644382a9..4f8b3899c03 100644 --- a/conan/tools/gnu/autotoolstoolchain.py +++ b/conan/tools/gnu/autotoolstoolchain.py @@ -258,6 +258,11 @@ def defines(self): ret = [self.ndebug, self.gcc_cxx11_abi] + self.extra_defines + conf_flags return self._filter_list_empty_fields(ret) + @property + def rcflags(self): + conf_flags = self._conanfile.conf.get("tools.build:rcflags", default=[], check_type=list) + return self._filter_list_empty_fields(conf_flags) + def _include_obj_arc_flags(self, env): enable_arc = self._conanfile.conf.get("tools.apple:enable_arc", check_type=bool) fobj_arc = "" @@ -301,6 +306,8 @@ def environment(self): env.append("CXXFLAGS", self.cxxflags) env.append("CFLAGS", self.cflags) env.append("LDFLAGS", self.ldflags) + if self.rcflags: + env.append("RCFLAGS", self.rcflags) env.prepend_path("PKG_CONFIG_PATH", self._conanfile.generators_folder) # Objective C/C++ self._include_obj_arc_flags(env) diff --git a/conan/tools/gnu/gnutoolchain.py b/conan/tools/gnu/gnutoolchain.py index 5a1701d9b58..78607ad19d2 100644 --- a/conan/tools/gnu/gnutoolchain.py +++ b/conan/tools/gnu/gnutoolchain.py @@ -325,6 +325,11 @@ def defines(self): ret = [self.ndebug, self.gcc_cxx11_abi] + self.extra_defines + conf_flags return self._filter_list_empty_fields(ret) + @property + def rcflags(self): + conf_flags = self._conanfile.conf.get("tools.build:rcflags", default=[], check_type=list) + return self._filter_list_empty_fields(conf_flags) + def _get_default_configure_shared_flags(self): args = {} # Just add these flags if there's a shared option defined (never add to exe's) @@ -373,6 +378,8 @@ def _environment(self): env.append("CXXFLAGS", self.cxxflags) env.append("CFLAGS", self.cflags) env.append("LDFLAGS", self.ldflags) + if self.rcflags: + env.append("RCFLAGS", self.rcflags) env.prepend_path("PKG_CONFIG_PATH", self._conanfile.generators_folder) # Objective C/C++ self._include_obj_arc_flags(env) diff --git a/conan/tools/microsoft/nmaketoolchain.py b/conan/tools/microsoft/nmaketoolchain.py index c20b575f11a..ff205af8411 100644 --- a/conan/tools/microsoft/nmaketoolchain.py +++ b/conan/tools/microsoft/nmaketoolchain.py @@ -88,6 +88,11 @@ def _link(self): return ["/nologo"] + self._format_options(ldflags) + @property + def _rcflags(self): + rcflags = self._conanfile.conf.get("tools.build:rcflags", default=[], check_type=list) + return self._format_options(rcflags) if rcflags else [] + def environment(self): env = Environment() # Injection of compile flags in CL env-var: @@ -96,6 +101,8 @@ def environment(self): # Injection of link flags in _LINK_ env-var: # https://learn.microsoft.com/en-us/cpp/build/reference/linking env.append("_LINK_", self._link) + if self._rcflags: + env.append("RCFLAGS", self._rcflags) # Also define some special env-vars which can override special NMake macros: # https://learn.microsoft.com/en-us/cpp/build/reference/special-nmake-macros conf_compilers = self._conanfile.conf.get("tools.build:compiler_executables", default={}, diff --git a/conan/tools/microsoft/toolchain.py b/conan/tools/microsoft/toolchain.py index e1f58bcf283..1086f4062d7 100644 --- a/conan/tools/microsoft/toolchain.py +++ b/conan/tools/microsoft/toolchain.py @@ -40,6 +40,7 @@ class MSBuildToolchain: {{ defines }}%(PreprocessorDefinitions) + {% if rc_flags %}{{ rc_flags }} %(AdditionalOptions){% endif %} @@ -69,6 +70,8 @@ def __init__(self, conanfile): self.cflags = [] #: List of all the LD linker flags self.ldflags = [] + #: List of all the RC (resource compiler) flags + self.rcflags = [] #: The build type. By default, the ``conanfile.settings.build_type`` value self.configuration = conanfile.settings.build_type #: The runtime flag. By default, it'll be based on the `compiler.runtime` setting. @@ -123,13 +126,14 @@ def context_config_toolchain(self): def format_macro(key, value): return '%s=%s' % (key, value) if value is not None else key - cxxflags, cflags, defines, sharedlinkflags, exelinkflags = self._get_extra_flags() + cxxflags, cflags, defines, sharedlinkflags, exelinkflags, rcflags = self._get_extra_flags() preprocessor_definitions = "".join(["%s;" % format_macro(k, v) for k, v in self.preprocessor_definitions.items()]) defines = preprocessor_definitions + "".join("%s;" % d for d in defines) self.cxxflags.extend(cxxflags) self.cflags.extend(cflags) self.ldflags.extend(sharedlinkflags + exelinkflags) + self.rcflags.extend(rcflags) cppstd = "stdcpp%s" % self.cppstd if self.cppstd else "" cstd = f"stdc{self.cstd}" if self.cstd else "" @@ -154,6 +158,7 @@ def format_macro(key, value): 'defines': defines, 'compiler_flags': " ".join(self.cxxflags + self.cflags), 'linker_flags': " ".join(self.ldflags), + 'rc_flags': " ".join(self.rcflags), "cppstd": cppstd, "cstd": cstd, "runtime_library": runtime_library, @@ -223,8 +228,9 @@ def _get_extra_flags(self): check_type=list) exelinkflags = self._conanfile.conf.get("tools.build:exelinkflags", default=[], check_type=list) + rcflags = self._conanfile.conf.get("tools.build:rcflags", default=[], check_type=list) defines = self._conanfile.conf.get("tools.build:defines", default=[], check_type=list) - return cxxflags, cflags, defines, sharedlinkflags, exelinkflags + return cxxflags, cflags, defines, sharedlinkflags, exelinkflags, rcflags def _get_toolset_props(conanfile): diff --git a/conan/tools/premake/toolchain.py b/conan/tools/premake/toolchain.py index 0131cf6dae8..5905d3fa614 100644 --- a/conan/tools/premake/toolchain.py +++ b/conan/tools/premake/toolchain.py @@ -31,6 +31,12 @@ def _generate_flags(self, conanfile): -- Link flags retrieved from LDFLAGS environment, conan.conf(tools.build:sharedlinkflags), conan.conf(tools.build:exelinkflags), extra_cxxflags and compiler settings linkoptions { {{ extra_ldflags }} } {% endif %} + {% if extra_rcflags %} + -- RC flags retrieved from conan.conf(tools.build:rcflags) + filter { files { "**.rc" } } + buildoptions { {{ extra_rcflags }} } + filter {} + {% endif %} {% if extra_defines %} -- Defines retrieved from DEFINES environment, conan.conf(tools.build:defines) and extra_defines defines { {{ extra_defines }} } @@ -75,6 +81,7 @@ def to_list(value): + arch_link_flags + thread_flags_list ) + extra_rc_flags = format_list(conanfile.conf.get("tools.build:rcflags", default=[], check_type=list)) return ( Template(template, trim_blocks=True, lstrip_blocks=True) @@ -83,6 +90,7 @@ def to_list(value): extra_cflags=extra_c_flags, extra_cxxflags=extra_cxx_flags, extra_ldflags=extra_ld_flags, + extra_rcflags=extra_rc_flags, ) .strip() ) diff --git a/conan/tools/qbs/qbsprofile.py b/conan/tools/qbs/qbsprofile.py index 8ad984a5380..a1c0abd3364 100644 --- a/conan/tools/qbs/qbsprofile.py +++ b/conan/tools/qbs/qbsprofile.py @@ -287,6 +287,7 @@ def map_list_property(key, qbs_property, extra): map_list_property("tools.build:cflags", "cpp.cFlags", self.extra_cflags) map_list_property("tools.build:cxxflags", "cpp.cxxFlags", self.extra_cxxflags) + map_list_property("tools.build:rcflags", "cpp.rcFlags", []) map_list_property("tools.build:defines", "cpp.defines", self.extra_defines) def ldflags(): diff --git a/test/integration/toolchains/cmake/test_cmaketoolchain.py b/test/integration/toolchains/cmake/test_cmaketoolchain.py index 94af6901812..4768835d989 100644 --- a/test/integration/toolchains/cmake/test_cmaketoolchain.py +++ b/test/integration/toolchains/cmake/test_cmaketoolchain.py @@ -535,6 +535,31 @@ def test_extra_flags_via_conf(): assert 'add_compile_definitions( "D1" "D2")' in toolchain +def test_cmaketoolchain_rcflags(): + """Test that tools.build:rcflags is applied to CONAN_RC_FLAGS and CMAKE_RC_FLAGS_INIT""" + profile = textwrap.dedent(""" + [settings] + os=Linux + arch=x86_64 + compiler=gcc + compiler.version=6 + compiler.libcxx=libstdc++11 + build_type=Release + + [conf] + tools.build:rcflags=["/nologo", "/flag-rc"] + """) + + client = TestClient() + conanfile = GenConanfile().with_settings("os", "arch", "compiler", "build_type")\ + .with_generator("CMakeToolchain") + client.save({"conanfile.py": conanfile, "profile": profile}) + client.run("install . --profile:host=profile") + toolchain = client.load("conan_toolchain.cmake") + assert 'string(APPEND CONAN_RC_FLAGS " /nologo /flag-rc")' in toolchain + assert 'string(APPEND CMAKE_RC_FLAGS_INIT " ${CONAN_RC_FLAGS}")' in toolchain + + def test_bitcode_enable_flag(): profile = textwrap.dedent(""" [settings] diff --git a/test/integration/toolchains/gnu/test_autotoolstoolchain.py b/test/integration/toolchains/gnu/test_autotoolstoolchain.py index 4e2acb5df3b..a87e1ac38f0 100644 --- a/test/integration/toolchains/gnu/test_autotoolstoolchain.py +++ b/test/integration/toolchains/gnu/test_autotoolstoolchain.py @@ -84,6 +84,33 @@ def generate(self): assert 'extra_ldflags sharedlinkflags exelinkflags' in toolchain +def test_autotoolstoolchain_rcflags(): + """Test that tools.build:rcflags is applied to RCFLAGS in the generated script.""" + os_ = platform.system() + os_ = "Macos" if os_ == "Darwin" else os_ + profile = textwrap.dedent(""" + [settings] + os=%s + arch=x86_64 + compiler=gcc + compiler.version=6 + compiler.libcxx=libstdc++11 + build_type=Release + + [conf] + tools.build:rcflags=["--rcflag1", "--rcflag2"] + """ % os_) + client = TestClient() + conanfile = GenConanfile().with_settings("os", "arch", "compiler", "build_type").with_generator("AutotoolsToolchain") + client.save({"conanfile.py": conanfile, "profile": profile}) + client.run("install . --profile:build=profile --profile:host=profile") + ext = ".bat" if os_ == "Windows" else ".sh" + toolchain = client.load("conanautotoolstoolchain{}".format(ext)) + assert "RCFLAGS" in toolchain + assert "--rcflag1" in toolchain + assert "--rcflag2" in toolchain + + def test_autotools_custom_environment(): client = TestClient() conanfile = textwrap.dedent(""" diff --git a/test/integration/toolchains/gnu/test_gnutoolchain.py b/test/integration/toolchains/gnu/test_gnutoolchain.py index 312c7c978b7..155a02c2661 100644 --- a/test/integration/toolchains/gnu/test_gnutoolchain.py +++ b/test/integration/toolchains/gnu/test_gnutoolchain.py @@ -92,6 +92,31 @@ def generate(self): assert 'extra_ldflags sharedlinkflags exelinkflags' in toolchain +def test_gnutoolchain_rcflags(): + """Test that tools.build:rcflags is applied to RCFLAGS in the generated script.""" + profile = textwrap.dedent(""" + [settings] + os=Linux + arch=x86_64 + compiler=gcc + compiler.version=6 + compiler.libcxx=libstdc++11 + build_type=Release + + [conf] + tools.build:rcflags=["--rcflag1", "--rcflag2"] + """) + client = TestClient() + conanfile = (GenConanfile().with_settings("os", "arch", "compiler", "build_type") + .with_generator("GnuToolchain")) + client.save({"conanfile.py": conanfile, "profile": profile}) + client.run("install . --profile:build=profile --profile:host=profile") + toolchain = client.load("conangnutoolchain.sh") + assert "RCFLAGS" in toolchain + assert "--rcflag1" in toolchain + assert "--rcflag2" in toolchain + + def test_autotools_custom_environment(): client = TestClient() conanfile = textwrap.dedent(""" diff --git a/test/integration/toolchains/microsoft/test_msbuildtoolchain.py b/test/integration/toolchains/microsoft/test_msbuildtoolchain.py index 327e3ac1c01..f725c906599 100644 --- a/test/integration/toolchains/microsoft/test_msbuildtoolchain.py +++ b/test/integration/toolchains/microsoft/test_msbuildtoolchain.py @@ -43,3 +43,22 @@ def test_msbuildtoolchain_props_with_extra_flags(): assert expected_cl_compile in toolchain assert expected_link in toolchain assert expected_resource_compile in toolchain + + +def test_msbuildtoolchain_rcflags(): + """Test that tools.build:rcflags is applied to ResourceCompile AdditionalOptions""" + profile = textwrap.dedent("""\ + include(default) + [settings] + arch=x86_64 + [conf] + tools.build:rcflags=["/flag-rc1", "/flag-rc2"] + """) + client = TestClient() + client.run("new msbuild_lib -d name=hello -d version=0.1") + client.save({"myprofile": profile}) + client.run("install . -pr myprofile") + toolchain = client.load(os.path.join("conan", "conantoolchain_release_x64.props")) + expected_resource_compile = ("/flag-rc1 /flag-rc2 %(AdditionalOptions)" + "") + assert expected_resource_compile in toolchain diff --git a/test/integration/toolchains/microsoft/test_nmaketoolchain.py b/test/integration/toolchains/microsoft/test_nmaketoolchain.py new file mode 100644 index 00000000000..0cf6722b9b8 --- /dev/null +++ b/test/integration/toolchains/microsoft/test_nmaketoolchain.py @@ -0,0 +1,28 @@ +import os +import platform +import textwrap + +import pytest + +from conan.test.assets.genconanfile import GenConanfile +from conan.test.utils.tools import TestClient + + +@pytest.mark.skipif(platform.system() != "Windows", reason="NMake toolchain is Windows-only") +def test_nmaketoolchain_rcflags(): + """Test that tools.build:rcflags is applied to RCFLAGS in the NMake toolchain environment.""" + profile = textwrap.dedent("""\ + include(default) + [settings] + arch=x86_64 + [conf] + tools.build:rcflags=["/nologo", "/flag-rc1"] + """) + client = TestClient() + conanfile = GenConanfile().with_settings("os", "arch", "compiler", "build_type").with_generator("NMakeToolchain") + client.save({"conanfile.py": conanfile, "profile": profile}) + client.run("install . -pr profile") + script = client.load("conannmaketoolchain.bat") + assert "RCFLAGS" in script + assert "/nologo" in script or "nologo" in script + assert "/flag-rc1" in script or "flag-rc1" in script diff --git a/test/integration/toolchains/premake/test_premaketoolchain.py b/test/integration/toolchains/premake/test_premaketoolchain.py index b764579a021..498893e5c8a 100644 --- a/test/integration/toolchains/premake/test_premaketoolchain.py +++ b/test/integration/toolchains/premake/test_premaketoolchain.py @@ -60,6 +60,32 @@ def test_extra_flags_via_conf(): assert 'defines { "define1=0", "_GLIBCXX_USE_CXX11_ABI=0" }' in content +def test_premaketoolchain_rcflags(): + """Test that tools.build:rcflags is applied to buildoptions for **.rc files.""" + profile = textwrap.dedent(""" + [settings] + os=Linux + arch=x86_64 + compiler=gcc + compiler.version=9 + compiler.libcxx=libstdc++ + build_type=Release + + [conf] + tools.build:rcflags=["-rcflag1", "-rcflag2"] + """) + tc = TestClient() + tc.save({ + "conanfile.txt": "[generators]\nPremakeToolchain", + "profile": profile, + }) + tc.run("install . -pr:a=profile") + content = tc.load(PremakeToolchain.filename) + assert '**.rc' in content + assert "-rcflag1" in content + assert "-rcflag2" in content + + def test_project_configuration(): tc = TestClient(path_with_spaces=False) profile = textwrap.dedent( diff --git a/test/integration/toolchains/qbs/test_qbsprofile.py b/test/integration/toolchains/qbs/test_qbsprofile.py new file mode 100644 index 00000000000..485f8d9276c --- /dev/null +++ b/test/integration/toolchains/qbs/test_qbsprofile.py @@ -0,0 +1,35 @@ +import os +import platform +import textwrap + +import pytest + +from conan.test.assets.genconanfile import GenConanfile +from conan.test.utils.tools import TestClient + + +@pytest.mark.skipif(platform.system() == "Windows", reason="QbsProfile requires host compiler (gcc) to be found") +def test_qbsprofile_rcflags(): + """Test that tools.build:rcflags is applied to cpp.rcFlags in qbs_settings.txt.""" + profile = textwrap.dedent(""" + [settings] + os=Linux + arch=x86_64 + compiler=gcc + compiler.version=9 + compiler.libcxx=libstdc++ + build_type=Release + + [conf] + tools.build:rcflags=["-rcflag1", "-rcflag2"] + """) + client = TestClient() + conanfile = GenConanfile().with_settings("os", "arch", "compiler", "build_type").with_generator("QbsProfile") + client.save({"conanfile.py": conanfile, "profile": profile}) + client.run("install . --profile:build=profile --profile:host=profile") + settings_path = os.path.join(client.current_folder, "qbs_settings.txt") + assert os.path.exists(settings_path) + content = client.load("qbs_settings.txt") + assert "cpp.rcFlags" in content + assert "-rcflag1" in content + assert "-rcflag2" in content From a5cd2f578c614e9a2e64f1bff7cb34e42fb8813f Mon Sep 17 00:00:00 2001 From: James Date: Mon, 23 Mar 2026 14:55:06 +0100 Subject: [PATCH 058/110] CMake.configure() extra args, via CMakePresets (#19639) * CMake.configure() extra args, via CMakePresets * review * improve conf description --- conan/internal/model/conf.py | 3 ++- conan/tools/cmake/cmake.py | 4 ++++ test/unittests/tools/cmake/test_cmake_test.py | 17 +++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/conan/internal/model/conf.py b/conan/internal/model/conf.py index 0ca7848a434..84a41389f4f 100644 --- a/conan/internal/model/conf.py +++ b/conan/internal/model/conf.py @@ -100,7 +100,8 @@ "tools.cmake.cmake_layout:test_folder": "(Experimental) Allow configuring the base folder of the build for test_package", "tools.cmake:cmake_program": "Path to CMake executable", "tools.cmake.cmakedeps:new": "Use the new CMakeDeps generator", - "tools.cmake:ctest_args": "To inject list of arguments to CMake.ctest() runner", + "tools.cmake:ctest_args": "Add extra arguments to CMake.ctest() runner command line", + "tools.cmake:configure_args": "Add extra arguments to CMake.configure() command line ", "tools.cmake:install_strip": "(Deprecated) Add --strip to cmake.install(). Use tools.build:install_strip instead", "tools.deployer:symlinks": "Set to False to disable deployers copying symlinks", "tools.files.download:retry": "Number of retries in case of failure when downloading", diff --git a/conan/tools/cmake/cmake.py b/conan/tools/cmake/cmake.py index 1d99223d068..46e8fc8bc94 100644 --- a/conan/tools/cmake/cmake.py +++ b/conan/tools/cmake/cmake.py @@ -121,6 +121,10 @@ def configure(self, variables=None, build_script_folder=None, cli_args=None, arg_list.append('"{}"'.format(cmakelist_folder)) + extra_args = self._conanfile.conf.get("tools.cmake:configure_args", check_type=list, + default=[]) + arg_list.extend(extra_args) + if not cli_args or ("--log-level" not in cli_args and "--loglevel" not in cli_args): arg_list.extend(self._cmake_log_levels_args) diff --git a/test/unittests/tools/cmake/test_cmake_test.py b/test/unittests/tools/cmake/test_cmake_test.py index 8ed861fd1aa..2be03004f2d 100644 --- a/test/unittests/tools/cmake/test_cmake_test.py +++ b/test/unittests/tools/cmake/test_cmake_test.py @@ -61,6 +61,23 @@ def test_cli_args_configure(): assert "--graphviz=foo.dot" in conanfile.command +def test_cli_args_configure_extra_args(): + settings = Settings.loads(default_settings_yml) + + conanfile = ConanFileMock() + conanfile.conf = Conf() + conanfile.conf.define("tools.cmake:configure_args", ["-DCMAKE_PROJECT_INCLUDE_BEFORE=MyFile", + "--fresh"]) + conanfile.folders.generators = "." + conanfile.folders.set_base_generators(temp_folder()) + conanfile.settings = settings + + write_cmake_presets(conanfile, "toolchain", "Unix Makefiles", {}) + cmake = CMake(conanfile) + cmake.configure() + assert '-DCMAKE_PROJECT_INCLUDE_BEFORE=MyFile --fresh' in conanfile.command + + def test_run_ctest(): settings = Settings.loads(default_settings_yml) settings.os = "Windows" From 15148c93a6eeb99d526e1d212f59e0d0835467f0 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 23 Mar 2026 15:32:44 +0100 Subject: [PATCH 059/110] Minor improvements (#19778) * fix minor issues * wip * cache RemoteCredentials * fixing tests * new test --- conan/internal/api/profile/profile_loader.py | 2 +- conan/internal/cache/db/table.py | 2 +- conan/internal/rest/auth_manager.py | 9 +++++- conan/tools/cmake/toolchain/blocks.py | 2 +- .../test_cmakeconfigdeps_new_paths.py | 32 +++++++++---------- .../test_cmakeconfigdeps_sources.py | 6 ++-- .../configuration/test_auth_remote_plugin.py | 21 +++++++++++- test/integration/graph/test_system_tools.py | 2 +- 8 files changed, 49 insertions(+), 27 deletions(-) diff --git a/conan/internal/api/profile/profile_loader.py b/conan/internal/api/profile/profile_loader.py index 5de78063094..addf1f71b06 100644 --- a/conan/internal/api/profile/profile_loader.py +++ b/conan/internal/api/profile/profile_loader.py @@ -244,7 +244,7 @@ def get_profile(profile_text, base_profile=None): doc_platform_tool_requires = doc.platform_tool_requires or doc.system_tools or "" if doc.system_tools: ConanOutput().warning("Profile [system_tools] is deprecated," - " please use [platform_tool_requires]") + " please use [platform_tool_requires]", warn_tag="deprecated") def parse_replaces(replaces): result = [RecipeReference.loads(r) for r in replaces.splitlines()] diff --git a/conan/internal/cache/db/table.py b/conan/internal/cache/db/table.py index 71e0ebf5813..0ff12443d08 100644 --- a/conan/internal/cache/db/table.py +++ b/conan/internal/cache/db/table.py @@ -29,7 +29,7 @@ def __init__(self, filename): def db_connection(self): if not self._lock.acquire(timeout=20): m = traceback.format_exc() + "\n" - ConanOutput().error(m) + ConanOutput().error("Error while acquiring lock for DB: " + m) raise ConanException("Conan failed to acquire database lock in 20s. Maybe the system is " "under very heavy load. Please report it to Github tickets") # isolation_level=None, puts it in regular SQLITE autocommit mode, every diff --git a/conan/internal/rest/auth_manager.py b/conan/internal/rest/auth_manager.py index 299274bb942..486fc82ebcf 100644 --- a/conan/internal/rest/auth_manager.py +++ b/conan/internal/rest/auth_manager.py @@ -45,6 +45,7 @@ class ConanApiAuthManager: def __init__(self, requester, cache_folder, localdb, global_conf): self._requester = requester self._creds = _RemoteCreds(localdb) + self._remote_creds = None self._global_conf = global_conf self._cache_folder = cache_folder @@ -69,11 +70,17 @@ def call_rest_api_method(self, remote, method_name, *args, **kwargs): if self._get_credentials_and_authenticate(rest_client, user, remote): return self.call_rest_api_method(remote, method_name, *args, **kwargs) + def _get_remote_creds(self): + if self._remote_creds is None: + self._remote_creds = RemoteCredentials(self._cache_folder, self._global_conf) + return self._remote_creds + def _get_credentials_and_authenticate(self, rest_client, user, remote): """Try LOGIN_RETRIES to obtain a password from user input for which we can get a valid token from api_client. If a token is returned, credentials are stored in localdb and rest method is called""" - creds = RemoteCredentials(self._cache_folder, self._global_conf) + creds = self._get_remote_creds() + for _ in range(LOGIN_RETRIES): input_user, input_password, interactive = creds.auth(remote) try: diff --git a/conan/tools/cmake/toolchain/blocks.py b/conan/tools/cmake/toolchain/blocks.py index 357e8f5d7c4..00e06db1746 100644 --- a/conan/tools/cmake/toolchain/blocks.py +++ b/conan/tools/cmake/toolchain/blocks.py @@ -568,7 +568,7 @@ class FindFiles(Block): template = textwrap.dedent("""\ # Define paths to find packages, programs, libraries, etc. if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/conan_cmakedeps_paths.cmake") - message(STATUS "Conan toolchain: Including CMakeDeps generated conan_cmakedeps_paths.cmake") + message(STATUS "Conan toolchain: Including CMakeConfigDeps generated conan_cmakedeps_paths.cmake") include("${CMAKE_CURRENT_LIST_DIR}/conan_cmakedeps_paths.cmake") else() diff --git a/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_new_paths.py b/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_new_paths.py index 5432437a9ba..67b585ef63e 100644 --- a/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_new_paths.py +++ b/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_new_paths.py @@ -6,8 +6,6 @@ from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient -new_value = "will_break_next" - @pytest.fixture def client(): @@ -19,7 +17,7 @@ def client(): class Pkg(ConanFile): settings = "build_type", "os", "arch", "compiler" requires = "dep/0.1" - generators = "CMakeDeps", "CMakeToolchain" + generators = "CMakeConfigDeps", "CMakeToolchain" def layout(self): # Necessary to force config files in another location cmake_layout(self) def build(self): @@ -41,8 +39,8 @@ def build(self): def test_cmake_generated(client): c = client c.run("create dep") - c.run(f"build pkg -c tools.cmake.cmakedeps:new={new_value}") - assert "Conan toolchain: Including CMakeDeps generated conan_cmakedeps_paths.cmake" in c.out + c.run(f"build pkg") + assert "Conan toolchain: Including CMakeConfigDeps generated conan_cmakedeps_paths.cmake" in c.out assert "Conan: Target declared imported INTERFACE library 'dep::dep'" in c.out @@ -70,8 +68,8 @@ def package_info(self): c.save({"dep/conanfile.py": dep}) c.run("create dep") - c.run(f"build pkg -c tools.cmake.cmakedeps:new={new_value}") - assert "Conan toolchain: Including CMakeDeps generated conan_cmakedeps_paths.cmake" in c.out + c.run(f"build pkg") + assert "Conan toolchain: Including CMakeConfigDeps generated conan_cmakedeps_paths.cmake" in c.out assert "Hello from dep dep-Config.cmake!!!!!" in c.out @@ -79,7 +77,7 @@ class TestRuntimeDirs: def test_runtime_lib_dirs_multiconf(self): client = TestClient() - app = GenConanfile().with_requires("dep/1.0").with_generator("CMakeDeps")\ + app = GenConanfile().with_requires("dep/1.0").with_generator("CMakeConfigDeps")\ .with_settings("build_type") client.save({"lib/conanfile.py": GenConanfile(), "dep/conanfile.py": GenConanfile("dep").with_requires("onelib/1.0", @@ -89,8 +87,8 @@ def test_runtime_lib_dirs_multiconf(self): client.run("create lib --name=twolib --version=1.0") client.run("create dep --version=1.0") - client.run(f'install app -s build_type=Release -c tools.cmake.cmakedeps:new={new_value}') - client.run(f'install app -s build_type=Debug -c tools.cmake.cmakedeps:new={new_value}') + client.run(f'install app -s build_type=Release') + client.run(f'install app -s build_type=Debug') contents = client.load("app/conan_cmakedeps_paths.cmake") pattern_lib_dirs = r"set\(CONAN_RUNTIME_LIB_DIRS ([^)]*)\)" @@ -102,7 +100,7 @@ def test_runtime_lib_dirs_multiconf(self): @pytest.mark.tool("cmake") -class TestCMakeDepsPaths: +class TestCMakeConfigDepsPaths: @pytest.mark.parametrize("requires, tool_requires", [(True, False), (False, True), (True, True)]) def test_find_program_path(self, requires, tool_requires): @@ -134,7 +132,7 @@ class PkgConan(ConanFile): {requires} {tool_requires} settings = "os", "arch", "compiler", "build_type" - generators = "CMakeToolchain", "CMakeDeps" + generators = "CMakeToolchain", "CMakeConfigDeps" def build(self): cmake = CMake(self) cmake.configure() @@ -148,7 +146,7 @@ def build(self): endif() """) c.save({"conanfile.py": conanfile, "CMakeLists.txt": consumer}, clean_first=True) - c.run(f"build . -c tools.cmake.cmakedeps:new={new_value}") + c.run(f"build .") assert "Found hello prog" in c.out if requires and tool_requires: assert "There is already a 'tool/1.0' package contributing to CMAKE_PROGRAM_PATH" in c.out @@ -180,7 +178,7 @@ def package(self): class PkgConan(ConanFile): requires = "hello/1.0" settings = "os", "arch", "compiler", "build_type" - generators = "CMakeToolchain", "CMakeDeps" + generators = "CMakeToolchain", "CMakeConfigDeps" def build(self): cmake = CMake(self) cmake.configure() @@ -198,7 +196,7 @@ def build(self): endif() """) c.save({"conanfile.py": conanfile, "CMakeLists.txt": consumer}, clean_first=True) - c.run(f"build . -c tools.cmake.cmakedeps:new={new_value}") + c.run(f"build .") assert "Found hello header" in c.out assert "Found hello lib" in c.out @@ -242,7 +240,7 @@ def build(self): """) c.save({"conanfile.py": conanfile, "CMakeLists.txt": consumer}, clean_first=True) - c.run(f"build . -c tools.cmake.cmakedeps:new={new_value}") + c.run(f"build .") assert "MYOWNCMAKE FROM hello!" in c.out def test_include_modules_both_build_host(self): @@ -282,7 +280,7 @@ def build(self): """) c.save({"conanfile.py": conanfile, "CMakeLists.txt": consumer}, clean_first=True) - c.run(f"build . -c tools.cmake.cmakedeps:new={new_value}") + c.run(f"build .") assert "conanfile.py: There is already a 'hello/0.1' package " \ "contributing to CMAKE_MODULE_PATH" in c.out assert "MYOWNCMAKE FROM hello!" in c.out diff --git a/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_sources.py b/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_sources.py index 7e22f6f814c..40eb384df8c 100644 --- a/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_sources.py +++ b/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_sources.py @@ -38,8 +38,7 @@ def package_info(self): # Check that the hello library builds in test_package c.run(f"create . -c tools.cmake.cmakedeps:new={new_value}") # Check content of the generated files - c.run(f"install --requires=hello/1.0 -g=CMakeConfigDeps " - f"-c tools.cmake.cmakedeps:new={new_value}") + c.run(f"install --requires=hello/1.0 -g=CMakeConfigDeps") cmake = c.load("hello-Targets-release.cmake") assert "add_library(hello::hello INTERFACE IMPORTED)" in cmake assert "set_property(TARGET hello::hello APPEND PROPERTY INTERFACE_SOURCES\n"\ @@ -81,8 +80,7 @@ def package_info(self): # Check that the hello library builds in test_package c.run(f"create . -c tools.cmake.cmakedeps:new={new_value}") # Check the content of the generated files - c.run(f"install --requires=hello/1.0 -g=CMakeConfigDeps " - f"-c tools.cmake.cmakedeps:new={new_value}") + c.run(f"install --requires=hello/1.0 -g=CMakeConfigDeps") cmake = c.load("hello-Targets-release.cmake") assert "add_library(hello::hello INTERFACE IMPORTED)" in cmake assert "add_library(hello::my_comp INTERFACE IMPORTED)" in cmake diff --git a/test/integration/configuration/test_auth_remote_plugin.py b/test/integration/configuration/test_auth_remote_plugin.py index 8bc4ce07838..ffc57402daa 100644 --- a/test/integration/configuration/test_auth_remote_plugin.py +++ b/test/integration/configuration/test_auth_remote_plugin.py @@ -2,7 +2,7 @@ import pytest -from conan.test.utils.tools import TestClient +from conan.test.utils.tools import TestClient, TestServer class TestAuthRemotePlugin: @@ -57,3 +57,22 @@ def auth_remote_plugin(remote, user=None): # the input methods in this case the stdin provided by TestClient. assert ("Changed user of remote 'default' from 'None' (anonymous) to " "'admin' (authenticated)") in c.out + + def test_creds_caching_multiple_remotes(self): + """ The auth plugin can cache partial results and credentials to avoid repeated + multiple interactive requests, reusing the same inputs for all remotes + https://github.com/conan-io/conan/issues/19772 + """ + c = TestClient(servers={"server1": TestServer(users={"admin1": "passwd"}), + "server2": TestServer(users={"admin2": "passwd"})}) + auth_plugin = textwrap.dedent("""\ + count = 0 + def auth_remote_plugin(remote, user=None): + global count + count = count + 1 + return f"admin{count}", "passwd" + """) + c.save_home({"extensions/plugins/auth_remote.py": auth_plugin}) + c.run("remote auth *") # Triggers all remotes + assert "Authenticated in remote 'server1' with user 'admin1'" in c.out + assert "Authenticated in remote 'server2' with user 'admin2'" in c.out diff --git a/test/integration/graph/test_system_tools.py b/test/integration/graph/test_system_tools.py index e72b1ab88e3..5526bb9c267 100644 --- a/test/integration/graph/test_system_tools.py +++ b/test/integration/graph/test_system_tools.py @@ -27,7 +27,7 @@ def test_system_tool_require_non_matching(self): "profile": "[system_tools]\ntool/1.1"}) client.run("create tool") client.run("create . -pr=profile") - assert "WARN: Profile [system_tools] is deprecated" in client.out + assert "WARN: deprecated: Profile [system_tools] is deprecated" in client.out assert "tool/1.0#60ed6e65eae112df86da7f6d790887fd - Cache" in client.out @pytest.mark.parametrize("revision", ["", "#myrev"]) From 768531b456fcc06c384d038f9646e6aec9984339 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 23 Mar 2026 16:27:31 +0100 Subject: [PATCH 060/110] Fix NMake integrations to handle hexadecimal values in defines (#19779) * wip * wip --- conan/tools/microsoft/nmakedeps.py | 35 +++++++++++++------ conan/tools/microsoft/nmaketoolchain.py | 17 ++------- .../toolchains/test_nmake_toolchain.py | 5 ++- .../toolchains/microsoft/test_nmakedeps.py | 14 ++++---- 4 files changed, 38 insertions(+), 33 deletions(-) diff --git a/conan/tools/microsoft/nmakedeps.py b/conan/tools/microsoft/nmakedeps.py index e2af344b441..fe416885e01 100644 --- a/conan/tools/microsoft/nmakedeps.py +++ b/conan/tools/microsoft/nmakedeps.py @@ -5,6 +5,29 @@ from conan.tools.env import Environment +def format_defines(defines): + def is_hex_or_numeric(s): + try: + # Check for Hexadecimal (base 16) + int(s, 16) + return True + except ValueError: + return False + + formated_defines = [] + for define in defines: + if "=" in define: + # CL env-var can't accept '=' sign in /D option, it can be replaced by '#' sign: + # https://learn.microsoft.com/en-us/cpp/build/reference/cl-environment-variables + macro, value = define.split("=", 1) + if value and not is_hex_or_numeric(value): + # value quotes are escaped + value = f'\\"{value}\\"' + define = f"{macro}#{value}" + formated_defines.append(f'/D"{define}"') + return formated_defines + + class NMakeDeps: def __init__(self, conanfile): @@ -45,20 +68,10 @@ def format_lib(lib): ret.extend([format_lib(lib) for lib in cpp_info.system_libs or []]) link_args = " ".join(ret) - def format_define(define): - if "=" in define: - # CL env-var can't accept '=' sign in /D option, it can be replaced by '#' sign: - # https://learn.microsoft.com/en-us/cpp/build/reference/cl-environment-variables - macro, value = define.split("=", 1) - if value and not value.isnumeric(): - value = f'\"{value}\"' - define = f"{macro}#{value}" - return f"/D\"{define}\"" - cl_flags = [f'-I"{p}"' for p in cpp_info.includedirs or []] cl_flags.extend(cpp_info.cflags or []) cl_flags.extend(cpp_info.cxxflags or []) - cl_flags.extend([format_define(define) for define in cpp_info.defines or []]) + cl_flags.extend(format_defines(cpp_info.defines or [])) env = Environment() env.append("CL", " ".join(cl_flags)) diff --git a/conan/tools/microsoft/nmaketoolchain.py b/conan/tools/microsoft/nmaketoolchain.py index ff205af8411..c016291547e 100644 --- a/conan/tools/microsoft/nmaketoolchain.py +++ b/conan/tools/microsoft/nmaketoolchain.py @@ -2,6 +2,7 @@ from conan.internal import check_duplicated_generator from conan.tools.build.flags import build_type_flags, cppstd_flag, build_type_link_flags from conan.tools.env import Environment +from conan.tools.microsoft.nmakedeps import format_defines from conan.tools.microsoft.visual import msvc_runtime_flag, VCVars @@ -29,20 +30,6 @@ def __init__(self, conanfile): def _format_options(options): return [f"{opt[0].replace('-', '/')}{opt[1:]}" for opt in options if len(opt) > 1] - @staticmethod - def _format_defines(defines): - formated_defines = [] - for define in defines: - if "=" in define: - # CL env-var can't accept '=' sign in /D option, it can be replaced by '#' sign: - # https://learn.microsoft.com/en-us/cpp/build/reference/cl-environment-variables - macro, value = define.split("=", 1) - if value and not value.isnumeric(): - value = f'\\"{value}\\"' - define = f"{macro}#{value}" - formated_defines.append(f"/D\"{define}\"") - return formated_defines - @property def _cl(self): bt_flags = build_type_flags(self._conanfile) @@ -71,7 +58,7 @@ def _cl(self): defines.extend(self.extra_defines) return (["/nologo"] + self._format_options(bt_flags + rt_flags + cflags + cxxflags) + - self._format_defines(defines)) + format_defines(defines)) @property def _link(self): diff --git a/test/functional/toolchains/test_nmake_toolchain.py b/test/functional/toolchains/test_nmake_toolchain.py index 3d4e32acadc..96dc5fdf50a 100644 --- a/test/functional/toolchains/test_nmake_toolchain.py +++ b/test/functional/toolchains/test_nmake_toolchain.py @@ -15,7 +15,7 @@ ("msvc", "191", "dynamic", "14", "Release", [], [], [], [], []), ("msvc", "191", "dynamic", "14", "Release", ["TEST_DEFINITION1", "TEST_DEFINITION2=0", "TEST_DEFINITION3=", "TEST_DEFINITION4=TestPpdValue4", - "TEST_DEFINITION5=__declspec(dllexport)", "TEST_DEFINITION6=foo bar"], + "TEST_DEFINITION5=__declspec(dllexport)", "TEST_DEFINITION6=foo bar", "TEST_WINVER=0x0601"], ["/GL"], ["/GL"], ["/LTCG"], ["/LTCG"]), ("msvc", "191", "static", "17", "Debug", [], [], [], [], []), ], @@ -86,6 +86,9 @@ def build(self): client.run(f"build . {settings} {conf}") client.run_command("simple.exe") assert "dep/1.0" in client.out + # It is printed as an integer number by default + if "TEST_WINVER" in conf_preprocessors: + conf_preprocessors["TEST_WINVER"] = 1537 check_exe_run(client.out, "main", "msvc", version, build_type, "x86_64", cppstd, conf_preprocessors) diff --git a/test/integration/toolchains/microsoft/test_nmakedeps.py b/test/integration/toolchains/microsoft/test_nmakedeps.py index 4eee33c80b2..abed9daf82e 100644 --- a/test/integration/toolchains/microsoft/test_nmakedeps.py +++ b/test/integration/toolchains/microsoft/test_nmakedeps.py @@ -31,7 +31,8 @@ def package_info(self): self.cpp_info.components["pkg-4"].defines = ["TEST_DEFINITION4=foo", "TEST_DEFINITION5=__declspec(dllexport)", "TEST_DEFINITION6=foo bar", - "TEST_DEFINITION7=7"] + "TEST_DEFINITION7=7", + "TEST_WINVER=0x0601"] """) client.save({"conanfile.py": conanfile}) client.run("create . -s arch=x86_64") @@ -42,12 +43,13 @@ def package_info(self): # Checking that defines are added to CL for flag in ( r'/D"TEST_DEFINITION1"', '/D"TEST_DEFINITION2#0"', - r'/D"TEST_DEFINITION3#"', '/D"TEST_DEFINITION4#"foo""', - r'/D"TEST_DEFINITION5#"__declspec\(dllexport\)""', - r'/D"TEST_DEFINITION6#"foo bar""', - r'/D"TEST_DEFINITION7#7"' + r'/D"TEST_DEFINITION3#"', r'/D"TEST_DEFINITION4#\"foo\""', + r'/D"TEST_DEFINITION5#\"__declspec(dllexport)\""', + r'/D"TEST_DEFINITION6#\"foo bar\""', + r'/D"TEST_DEFINITION7#7"', + r'/D"TEST_WINVER#0x0601"' ): - assert re.search(fr'set "CL=%CL%.*\s{flag}(?:\s|")', bat_file) + assert flag in bat_file # Checking that libs and system libs are added to _LINK_ for flag in (r"pkg-1\.lib", r"pkg-2\.lib", r"pkg-3\.lib", r"pkg-4\.lib", r"ws2_32\.lib"): assert re.search(fr'set "_LINK_=%_LINK_%.*\s{flag}(?:\s|")', bat_file) From 4e0bf67c1361a7ad55d4cd0cf907de8bfb3d1ced Mon Sep 17 00:00:00 2001 From: James Date: Mon, 23 Mar 2026 16:31:10 +0100 Subject: [PATCH 061/110] fix DB connection timeout msg (#19781) --- conan/internal/cache/db/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conan/internal/cache/db/table.py b/conan/internal/cache/db/table.py index 0ff12443d08..2b5d6b0b0b2 100644 --- a/conan/internal/cache/db/table.py +++ b/conan/internal/cache/db/table.py @@ -28,7 +28,7 @@ def __init__(self, filename): @contextmanager def db_connection(self): if not self._lock.acquire(timeout=20): - m = traceback.format_exc() + "\n" + m = "".join(traceback.format_stack()) + "\n" ConanOutput().error("Error while acquiring lock for DB: " + m) raise ConanException("Conan failed to acquire database lock in 20s. Maybe the system is " "under very heavy load. Please report it to Github tickets") From a5d453a5198dd3f9f28015525450840c8524a4e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= <5364255+AbrilRBS@users.noreply.github.com> Date: Tue, 24 Mar 2026 11:13:28 +0100 Subject: [PATCH 062/110] Fix missing libraries in legacy `_LIBRARIES` variable definition (#19724) * Missing libraries in legacy variable * SHow all libraries in legacy libraries version * Dont remove old test * Typo * Styling * Fix --- conan/tools/cmake/cmakeconfigdeps/config.py | 13 ++++- .../cmakeconfigdeps/test_cmakeconfigdeps.py | 47 +++++++++++++++---- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/conan/tools/cmake/cmakeconfigdeps/config.py b/conan/tools/cmake/cmakeconfigdeps/config.py index fd53c71e30c..b0cc4928749 100644 --- a/conan/tools/cmake/cmakeconfigdeps/config.py +++ b/conan/tools/cmake/cmakeconfigdeps/config.py @@ -99,9 +99,18 @@ def _get_legacy_vars(self): for i in incdirs] include_dirs = ";".join(incdirs) definitions = ";".join("-D" + cmake_escape_value(d) for d in aggregated_cppinfo.defines) - root_target_name = self._cmakedeps.get_property("cmake_target_name", self._conanfile) - libraries = root_target_name or f"{pkg_name}::{pkg_name}" + libraries = [] + if self._full_cpp_info.has_components: + for component in self._full_cpp_info.components.keys(): + root_target_name = self._cmakedeps.get_property("cmake_target_name", + self._conanfile, + comp_name=component) + libraries.append(root_target_name or f"{pkg_name}::{component}") + else: + root_target_name = self._cmakedeps.get_property("cmake_target_name", self._conanfile) + libraries.append(root_target_name or f"{pkg_name}::{pkg_name}") + libraries = " ".join(libraries) if libraries else "" return {"additional_variables_prefixes": prefixes, "version": self._conanfile.ref.version, "include_dirs": include_dirs, diff --git a/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps.py b/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps.py index 9e6d913c2f6..fa2afa42f40 100644 --- a/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps.py +++ b/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps.py @@ -790,16 +790,43 @@ def package_info(self): assert "# Requirement dep::dep -> pkg::compA (Full link: True)\n# Link feature: MYFET" in targets -def test_legacy_defines(): - # We used not to populate this. - # We do for backward compatibility with old check_symbol_exists and similar CMake code - tc = TestClient() - tc.save({"conanfile.py": GenConanfile("mypkg", "1.0") - .with_package_info({"defines": ["MY_DEFINE", "MYVAR=1"]})}) - tc.run("create") - tc.run("install --requires=mypkg/1.0 -g CMakeConfigDeps") - mypkg_config = tc.load("mypkg-config.cmake") - assert 'set(mypkg_DEFINITIONS "-DMY_DEFINE;-DMYVAR=1" )' in mypkg_config +class TestLegacyVariables: + def test_legacy_defines(self): + # We used not to populate this. + # We do for backward compatibility with old check_symbol_exists and similar CMake code + tc = TestClient() + tc.save({"conanfile.py": GenConanfile("mypkg", "1.0") + .with_package_info({"defines": ["MY_DEFINE", "MYVAR=1"]})}) + tc.run("create") + tc.run("install --requires=mypkg/1.0 -g CMakeConfigDeps") + mypkg_config = tc.load("mypkg-config.cmake") + assert 'set(mypkg_DEFINITIONS "-DMY_DEFINE;-DMYVAR=1" )' in mypkg_config + + def test_legacy_defines_multiple_components(self): + tc = TestClient() + tc.save({"conanfile.py": GenConanfile("mypkg", "1.0") + .with_package_info({"components": {"mypkg": {"defines": ["MY_DEFINE", "MYVAR=1"]}, + "lib2": {"defines": ["MY_DEFINE2", "MYVAR2=1"]}}}) + }) + tc.run("create") + tc.run("install --requires=mypkg/1.0 -g CMakeConfigDeps") + mypkg_config = tc.load("mypkg-config.cmake") + assert 'set(mypkg_DEFINITIONS "-DMY_DEFINE2;-DMYVAR2=1;-DMY_DEFINE;-DMYVAR=1" )' in mypkg_config + + def test_legacy_libraries(self): + tc = TestClient() + tc.save({"conanfile.py": GenConanfile("mypkg", "1.0") + .with_package_file("lib/mylib1.a", "library") + .with_package_file("lib/mylib2.a", "library") + .with_package_info({"components": {"mypkg": {"libs": ["mylib1"]}, + "lib2": {"libs": ["mylib2"]}}}) + }) + tc.run("create") + tc.run("install --requires=mypkg/1.0 -g CMakeConfigDeps") + mypkg_config = tc.load("mypkg-config.cmake") + # If there's no interface global target + # mypkg::lib2 is not added to the list of libraries + assert "set(mypkg_LIBRARIES mypkg::mypkg mypkg::lib2 )" in mypkg_config class TestPropertiesBuildContext: From d820ef12773bb28fb4139f57c703cca6b4ed3c9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Ram=C3=ADrez?= Date: Tue, 24 Mar 2026 16:39:16 +0100 Subject: [PATCH 063/110] [GH] Update clang major version (#19785) Update clang major version --- .../toolchains/cmake/test_cmake_toolchain_win_clang.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/functional/toolchains/cmake/test_cmake_toolchain_win_clang.py b/test/functional/toolchains/cmake/test_cmake_toolchain_win_clang.py index 638301e3350..00ef0d1feac 100644 --- a/test/functional/toolchains/cmake/test_cmake_toolchain_win_clang.py +++ b/test/functional/toolchains/cmake/test_cmake_toolchain_win_clang.py @@ -215,7 +215,7 @@ def test_msys2_clang(self, client): # clang compilations in Windows will use MinGW Makefiles by default assert 'cmake -G "MinGW Makefiles"' in client.out # TODO: Version is still not controlled - assert "main __clang_major__21" in client.out + assert "main __clang_major__22" in client.out # Not using libstdc++ assert "_GLIBCXX_USE_CXX11_ABI" not in client.out assert "main __cplusplus2014" in client.out @@ -274,7 +274,7 @@ def test_clang_pure_c(self, client): client.run(f"create . --name=pkg --version=0.1 -pr=clang") # clang compilations in Windows will use MinGW Makefiles by default assert 'cmake -G "MinGW Makefiles"' in client.out - assert "main __clang_major__21" in client.out + assert "main __clang_major__22" in client.out assert "GLIBCXX" not in client.out assert "cplusplus" not in client.out assert "main __GNUC__" in client.out From 49810d32a68f21d7aec4bd31674ef79d46d86c49 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 25 Mar 2026 10:57:09 +0100 Subject: [PATCH 064/110] allowing tool-requires negated or (#19780) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * allowing tool-requires negated or * fix * new proposal * review * format --------- Co-authored-by: Francisco Ramírez --- conan/internal/graph/graph_builder.py | 11 +++++- .../profile_build_requires_test.py | 39 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/conan/internal/graph/graph_builder.py b/conan/internal/graph/graph_builder.py index 2e2c416771f..9be4e23901a 100644 --- a/conan/internal/graph/graph_builder.py +++ b/conan/internal/graph/graph_builder.py @@ -191,8 +191,17 @@ def _prepare_node(node, profile_host, profile_build, down_options, define_consum # Apply build_tools_requires from profile, overriding the declared ones profile = profile_host if node.context == CONTEXT_HOST else profile_build + for pattern, tool_requires in profile.tool_requires.items(): - if ref_matches(ref, pattern, is_consumer=conanfile._conan_is_consumer): # noqa + is_consumer = conanfile._conan_is_consumer # noqa + if pattern[0] in "!~" and pattern[1] == "(": # This is a negated OR operation + assert pattern[-1] == ")", (f"Malformed profile OR expression without" + f" closing parenthesis: {pattern}") + parts = pattern[2:-1].split("|") + matched = not any(ref_matches(ref, p, is_consumer=is_consumer) for p in parts) + else: + matched = ref_matches(ref, pattern, is_consumer=is_consumer) + if matched: for tool_require in tool_requires: # Do the override # Check if it is a self-loop of build-requires in build context and avoid it if ref and tool_require.name == ref.name and tool_require.user == ref.user and \ diff --git a/test/integration/build_requires/profile_build_requires_test.py b/test/integration/build_requires/profile_build_requires_test.py index f293f7b4e1d..955aaef90b3 100644 --- a/test/integration/build_requires/profile_build_requires_test.py +++ b/test/integration/build_requires/profile_build_requires_test.py @@ -1,3 +1,4 @@ +import json import os import platform import textwrap @@ -222,3 +223,41 @@ def test_tool_requires_version_range_loop(): c.run("create tool") c.run("install app -pr:b=build_profile") assert "tool/1.1" in c.out # It is skipped + + +def test_profile_tool_requires_negated_or_patterns(): + """Negated [tool_requires] patterns may use | so the rule applies + if the ref matches none of the branches.""" + c = TestClient(light=True) + profile_build = textwrap.dedent("""\ + [settings] + os=Linux + + [tool_requires] + mold/1.0 + !(zlib*|mold*):cmake/1.0 + """) + profile = textwrap.dedent("""\ + [tool_requires] + mold/1.0 + cmake/1.0 + gcc/1.0 + """) + c.save({"mold/conanfile.py": GenConanfile("mold", "1.0"), + "zlib/conanfile.py": GenConanfile("zlib", "1.0"), + "cmake/conanfile.py": GenConanfile("cmake", "1.0").with_tool_requires("zlib/1.0"), + "gcc/conanfile.py": GenConanfile("gcc", "1.0").with_tool_requires("zlib/1.0"), + "app/conanfile.py": GenConanfile("app", "1.0"), + "profile_build": profile_build, + "profile": profile}) + c.run("create mold") + c.run("create zlib") + c.run("create cmake") + c.run("create gcc") + c.run("graph info app -pr=profile -pr:b=profile_build --format=json") + graph = json.loads(c.stdout) + assert len(graph["graph"]["nodes"]) == 14 + c.assert_listed_require({"cmake/1.0": "Cache", + "gcc/1.0": "Cache", + "mold/1.0": "Cache", + "zlib/1.0": "Cache"}, build=True) From 3004d93c16600068c8e16adb52e54e886a5a5ae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= <5364255+AbrilRBS@users.noreply.github.com> Date: Wed, 25 Mar 2026 13:02:38 +0100 Subject: [PATCH 065/110] Add cve version info in `conan audit` results (#19774) * Add cve version info * v4 has priority for sorting * Review * Scores for text * Testing * Testing * Update txt output, improve tests --- conan/cli/commands/audit.py | 4 +- conan/cli/formatters/audit/vulnerabilities.py | 48 ++++++++--- conan/internal/api/audit/providers.py | 6 ++ test/integration/command/test_audit.py | 86 ++++++++++++++++++- 4 files changed, 128 insertions(+), 16 deletions(-) diff --git a/conan/cli/commands/audit.py b/conan/cli/commands/audit.py index c5f08802d99..32654e1dd66 100644 --- a/conan/cli/commands/audit.py +++ b/conan/cli/commands/audit.py @@ -14,7 +14,6 @@ from conan.cli.printers import print_profiles from conan.cli.printers.graph import print_graph_basic from conan.errors import ConanException -from conan.internal.util.files import load def _add_provider_arg(subparser): @@ -134,7 +133,8 @@ def audit_list(conan_api: ConanAPI, parser, subparser, *args): ConanOutput().warning("Nothing to list, package list does not contain recipe revisions") elif args.sbom: sbom_file = make_abs_path(args.sbom) - sbom = json.loads(load(sbom_file)) + with open(sbom_file, 'r') as f: + sbom = json.load(f) if sbom.get("bomFormat") != "CycloneDX": raise ConanException(f"Unsupported SBOM format, only CycloneDX is supported.") purls = [component["purl"] for component in sbom["components"]] diff --git a/conan/cli/formatters/audit/vulnerabilities.py b/conan/cli/formatters/audit/vulnerabilities.py index 468f5d8f21d..1dbd6b5b818 100644 --- a/conan/cli/formatters/audit/vulnerabilities.py +++ b/conan/cli/formatters/audit/vulnerabilities.py @@ -72,8 +72,9 @@ def wrap_and_indent(txt, limit=80, indent=2): name = node["name"] sev = node.get("severity", "Medium") sev_color = severity_colors.get(sev, Color.BRIGHT_YELLOW) - score = node.get("cvss", {}).get("preferredBaseScore") - score_txt = f", CVSS: {score}" if score else "" + cvss = node.get("cvss", {}) + preferred_score = cvss.get("preferredBaseScore") + score_txt = f" - {preferred_score}" if preferred_score else "" desc = node.get("description", "") desc = (desc[:240] + "...") if len(desc) > 240 else desc desc_wrapped = wrap_and_indent(desc) @@ -127,6 +128,17 @@ def wrap_and_indent(txt, limit=80, indent=2): if fixVersions: cli_out_write(f" fixed in version(s): ", endline="", fg=Color.BRIGHT_BLUE) cli_out_write(', '.join(fixVersions)) + + if "v3" in cvss and cvss["v3"].get("baseScore", 0) > 0: + score_v3 = cvss["v3"].get("baseScore", 0) + if score_v3: + cli_out_write(f" CVSS v3: ", endline="", fg=Color.BRIGHT_BLUE) + cli_out_write(score_v3) + if "v4" in cvss and cvss["v4"].get("baseScore", 0) > 0: + score_v4 = cvss["v4"].get("baseScore", 0) + if score_v4: + cli_out_write(f" CVSS v4: ", endline="", fg=Color.BRIGHT_BLUE) + cli_out_write(score_v4) cli_out_write("") color_for_total = Color.BRIGHT_RED if total_vulns else Color.BRIGHT_GREEN @@ -237,18 +249,20 @@ def _render_vulns(vulns, template): {{ vuln.package }}
    PackageIDSeverityScoreInfo Description
    {{ vuln.package }}{{ vuln.vuln_id }} - {% if severity_id == 'N/A' %}-1{% else %}{{ severity_id }}{% endif %} + {{ vuln.package }} + + {{ vuln.score }} + {% if vuln.withdrawn %} + [WITHDRAWN]
    + {% endif %} + {{ vuln.vuln_id }} +
    {% if vuln.severity not in ['N/A', ''] %} {{ severity_label }} {% else %} {{ vuln.severity }} {% endif %} + {{ vuln.score }}     		{{ vuln.score }} + {% for research in vuln.advisories %} + {% if research.shortDescription %} +
    + Summary provided by JFrog Research ({{ research.name }}) +
    + Short description: {{ research.shortDescription }}
    + {% if research.severity %} + Impact severity: {{ research.severity }}
    + {% if research.impactReasons %} + Impact reasons: +
      + {% for reason in research.impactReasons %} +
    • {{ reason.name }}
      • + {% endfor %} +
    + {% endif %} + {% endif %} + {% if vuln.provider_url %} + {% set expected_url = vuln.provider_url.rstrip('/') + '/ui/catalog/vulnerabilities/details/' + research.name %} + More info available in: {{ expected_url }}
    + {% endif %} +
    +
    + {% endif %} + {% endfor %} + Description: +
    {{ vuln.description }} + {% if vuln.publishedAt %} +
    +
    + Published at: {{ vuln.publishedAt }} + {% endif %} + {% if vuln.fixVersions %} +
    +
    + Fixed in version(s): +
    + {% for version in vuln.fixVersions %} + {{ version }} + {% endfor %} +
    + {% endif %} {% if vuln.references %} -

    References: +
    References:
      {% for ref in vuln.references %}
    • {{ ref }}
      • @@ -212,7 +303,7 @@ def _render_vulns(vulns, template):
    {% endif %} {% if vuln.aliases %} -

    Aliases: {{ ', '.join(vuln.aliases) }} +
    Aliases: {{ ', '.join(vuln.aliases) }} {% endif %}
    - {{ vuln.score }} - {% if vuln.withdrawn %} - [WITHDRAWN]
    - {% endif %} - {{ vuln.vuln_id }} -
    + {{ vuln.preferred_score }} {% if vuln.severity not in ['N/A', ''] %} - {{ severity_label }} + {{ severity_label }} {% if vuln.preferred_score %}({{ vuln.preferred_score }}){% endif %} {% else %} {{ vuln.severity }} {% endif %} - {{ vuln.score }} +
    +
    + {% if vuln.withdrawn %} + [WITHDRAWN]
    + {% endif %} + {{ vuln.vuln_id }} + {% if vuln.score_v3 %}
    CVSS v3: {{ vuln.score_v3 }}{% endif %} + {% if vuln.score_v4 %}
    CVSS v4: {{ vuln.score_v4 }}{% endif %}     		{% for research in vuln.advisories %} @@ -349,8 +363,14 @@ def html_vuln_formatter(result): name = node.get("name") sev = node.get("severity", "Medium") sev = f"{severity_order.get(sev, 2)} - {sev}" - score = node.get("cvss", {}).get("preferredBaseScore") - score_txt = f"CVSS: {score}" if score else "-" + cvss = node.get("cvss", {}) + preferred_score = cvss.get("preferredBaseScore") + score_v3 = 0 + score_v4 = 0 + if "v3" in cvss and cvss["v3"].get("baseScore", 0) > 0: + score_v3 = cvss["v3"].get("baseScore", 0) + if "v4" in cvss and cvss["v4"].get("baseScore", 0) > 0: + score_v4 = cvss["v4"].get("baseScore", 0) aliases = node.get("aliases", []) references = node.get("references", []) desc = node.get("description", "") @@ -366,7 +386,9 @@ def html_vuln_formatter(result): "vuln_id": name, "aliases": aliases, "severity": sev, - "score": score_txt, + "preferred_score": preferred_score, + "score_v3": score_v3, + "score_v4": score_v4, "description": desc, "references": references, "withdrawn": withdrawn, diff --git a/conan/internal/api/audit/providers.py b/conan/internal/api/audit/providers.py index ebf5017718e..1bc1bafe464 100644 --- a/conan/internal/api/audit/providers.py +++ b/conan/internal/api/audit/providers.py @@ -148,6 +148,12 @@ def _build_query(ref): severity cvss {{ preferredBaseScore + v3 {{ + baseScore + }} + v4 {{ + baseScore + }} }} aliases withdrawn diff --git a/test/integration/command/test_audit.py b/test/integration/command/test_audit.py index eb2e0980594..554cd5cc53a 100644 --- a/test/integration/command/test_audit.py +++ b/test/integration/command/test_audit.py @@ -479,7 +479,7 @@ def test_audit_scan_threshold_error(severity_level, threshold, should_fail): severity_param = "" if threshold is None else f"-sl {threshold}" tc.run(f"audit scan --requires=foobar/0.1.0 {severity_param}", assert_error=should_fail) assert "foobar/0.1.0 1 vulnerability found" in tc.out - assert f"CVSS: {severity_level}" in tc.out + assert f"Critical - {severity_level}" in tc.out if should_fail: if threshold is None: threshold = "9.0" @@ -530,6 +530,90 @@ def test_audit_scan_context_filter(package_context, filter_context): assert "Requesting vulnerability info for: zlib/1.2.11" not in tc.out +@pytest.mark.parametrize("score", [ + { + "preferredBaseScore": 8.9, + }, + { + "preferredBaseScore": 8.9, + "v3": { + "baseScore": 8.9 + } + }, + { + "preferredBaseScore": 8.9, + "v3": { + "baseScore": 8.9 + }, + "v4": { + "baseScore": 5.6 + } + }, +]) +@pytest.mark.parametrize("out", ["html", "text"]) +def test_audit_cvss_versions(score, out): + """In case the severity level is equal or higher than the found for a CVE, + the command should output the information as usual, and exit with non-success code error. + """ + successful_response = { + "data": { + "query": { + "vulnerabilities": { + "totalCount": 1, + "edges": [ + { + "node": { + "name": "CVE-2023-45853", + "description": "Zip vulnerability", + "severity": "Critical", + "cvss": score, + "aliases": [ + "CVE-2023-45853", + "JFSA-2023-000272529" + ], + "advisories": [ + { + "name": "CVE-2023-45853" + }, + { + "name": "JFSA-2023-000272529" + } + ], + "references": [ + "https://pypi.org/project/pyminizip/#history", + ] + } + } + ] + } + } + } + } + + tc = TestClient(light=True) + + tc.save({"conanfile.py": GenConanfile("foobar", "0.1.0")}) + tc.run("export .") + tc.run("audit provider auth conancenter --token=valid_token") + + with proxy_response(200, successful_response): + tc.run(f"audit scan --requires=foobar/0.1.0 -f={out}") + if out == "text": + assert "foobar/0.1.0 1 vulnerability found" in tc.out + assert f"Critical - {score['preferredBaseScore']}" in tc.out + if "v4" in score: + assert f'CVSS v4: {score["v4"]["baseScore"]}' in tc.out + if "v3" in score: + assert f'CVSS v3: {score["v3"]["baseScore"]}' in tc.out + else: + assert "CVE-2023-45853" in tc.out + assert f"Critical ({score['preferredBaseScore']})" in tc.out + if "v4" in score: + assert f'CVSS v4: {score["v4"]["baseScore"]}' in tc.out + if "v3" in score: + assert f'CVSS v3: {score["v3"]["baseScore"]}' in tc.out + + class TestAuditApiBranchouts: def test_audit_load_provider_default(self): tc = TestClient(light=True) From 3500931911bf61674c0c1912813e0bcd098a6014 Mon Sep 17 00:00:00 2001 From: Carlos Zoido Date: Wed, 25 Mar 2026 13:04:25 +0100 Subject: [PATCH 066/110] Conan 2.27.0 --- conan/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conan/__init__.py b/conan/__init__.py index 415d5f586e3..f0b3b7346ed 100644 --- a/conan/__init__.py +++ b/conan/__init__.py @@ -2,5 +2,5 @@ from conan.internal.model.workspace import Workspace from conan.internal.model.version import Version -__version__ = '2.27.0-dev' +__version__ = '2.27.0' conan_version = Version(__version__) From a4a7f77f3ea79f39e1a085b925c26ff72b5a7ae2 Mon Sep 17 00:00:00 2001 From: Carlos Zoido Date: Wed, 25 Mar 2026 15:52:35 +0100 Subject: [PATCH 067/110] Conan 2.28.0 dev --- conan/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conan/__init__.py b/conan/__init__.py index f0b3b7346ed..9933544736f 100644 --- a/conan/__init__.py +++ b/conan/__init__.py @@ -2,5 +2,5 @@ from conan.internal.model.workspace import Workspace from conan.internal.model.version import Version -__version__ = '2.27.0' +__version__ = '2.28.0-dev' conan_version = Version(__version__) From 80210c043416b411af5da087f27ac3f2520e5caf Mon Sep 17 00:00:00 2001 From: Linus van Elswijk Date: Thu, 26 Mar 2026 09:55:27 +0100 Subject: [PATCH 068/110] Feature/cachyos package manager (#19788) Add support for CachyOS in package_manager - Register pacman as the default package_manager for CachyOS - Cover this in `package_manager_test.py` --- conan/tools/system/package_manager.py | 2 +- test/integration/tools/system/package_manager_test.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/conan/tools/system/package_manager.py b/conan/tools/system/package_manager.py index eeec1856a08..f1fcfae57f5 100644 --- a/conan/tools/system/package_manager.py +++ b/conan/tools/system/package_manager.py @@ -49,7 +49,7 @@ def get_default_tool(self): "dnf": ["fedora", "rhel", "centos", "mageia", "nobara", "almalinux", "rocky", "oracle"], "brew": ["Darwin"], - "pacman": ["arch", "manjaro", "msys2", "endeavouros"], + "pacman": ["arch", "manjaro", "msys2", "endeavouros", "cachyos"], "choco": ["Windows"], "zypper": ["opensuse", "sles"], "pkg": ["freebsd"], diff --git a/test/integration/tools/system/package_manager_test.py b/test/integration/tools/system/package_manager_test.py index 643f05727ea..6632c1ee683 100644 --- a/test/integration/tools/system/package_manager_test.py +++ b/test/integration/tools/system/package_manager_test.py @@ -58,6 +58,7 @@ def test_msys2(): ("fedora", "dnf"), ("nobara", "dnf"), ("arch", "pacman"), + ("cachyos", "pacman"), ("opensuse", "zypper"), ("sles", "zypper"), ("opensuse", "zypper"), From 13a89a245f2a869898f802ffa86407d4e8eb224e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= <5364255+AbrilRBS@users.noreply.github.com> Date: Thu, 26 Mar 2026 11:32:32 +0100 Subject: [PATCH 069/110] Add missing `CMAKE_CXX_COMPILER_WORKS` and family (#19708) * Add some shortcuts for cmake * Only where necessary * Improvements --- test/functional/command/test_new.py | 2 ++ .../cmake/test_cmake_extra_variables.py | 2 ++ .../toolchains/cmake/test_cmake_find_none.py | 4 ++++ .../toolchains/cmake/test_cmake_multi.py | 2 ++ test/functional/toolchains/cmake/test_cps.py | 18 ++++++++++++++++++ test/functional/toolchains/ios/_utils.py | 2 ++ .../toolchains/meson/test_subproject.py | 2 ++ .../functional/util/test_cmd_args_to_string.py | 2 ++ 8 files changed, 34 insertions(+) diff --git a/test/functional/command/test_new.py b/test/functional/command/test_new.py index d9cbf58781b..dee307e3622 100644 --- a/test/functional/command/test_new.py +++ b/test/functional/command/test_new.py @@ -23,6 +23,8 @@ def test_conan_new_empty(): c = TestClient() c.run("new") cmakelists = textwrap.dedent(""" + set(CMAKE_CXX_COMPILER_WORKS 1) + set(CMAKE_CXX_ABI_COMPILED 1) cmake_minimum_required(VERSION 3.15) project(PackageTest CXX) add_executable(example main.cpp) diff --git a/test/functional/toolchains/cmake/test_cmake_extra_variables.py b/test/functional/toolchains/cmake/test_cmake_extra_variables.py index 2a9de2caf03..f2d1de7b560 100644 --- a/test/functional/toolchains/cmake/test_cmake_extra_variables.py +++ b/test/functional/toolchains/cmake/test_cmake_extra_variables.py @@ -27,6 +27,8 @@ def package_info(self): client.run("create dep") cmakelists = textwrap.dedent(""" + set(CMAKE_CXX_COMPILER_WORKS 1) + set(CMAKE_CXX_ABI_COMPILED 1) cmake_minimum_required(VERSION 3.27) project(myproject CXX) find_package(dep CONFIG REQUIRED) diff --git a/test/functional/toolchains/cmake/test_cmake_find_none.py b/test/functional/toolchains/cmake/test_cmake_find_none.py index 08245cf5efc..f801e358169 100644 --- a/test/functional/toolchains/cmake/test_cmake_find_none.py +++ b/test/functional/toolchains/cmake/test_cmake_find_none.py @@ -115,6 +115,8 @@ def test_cmake_find_none_relocation(): conanfile = conanfile + '\n self.cpp_info.builddirs = ["pkg/cmake"]' cmake_export = textwrap.dedent(""" + set(CMAKE_CXX_COMPILER_WORKS 1) + set(CMAKE_CXX_ABI_COMPILED 1) cmake_minimum_required(VERSION 3.15) project(MyHello CXX) @@ -220,6 +222,8 @@ def package_info(self): """) cmake_export = textwrap.dedent(""" + set(CMAKE_CXX_COMPILER_WORKS 1) + set(CMAKE_CXX_ABI_COMPILED 1) cmake_minimum_required(VERSION 3.15) project(MyHello CXX) diff --git a/test/functional/toolchains/cmake/test_cmake_multi.py b/test/functional/toolchains/cmake/test_cmake_multi.py index bff9cd45f66..16cf34bd560 100644 --- a/test/functional/toolchains/cmake/test_cmake_multi.py +++ b/test/functional/toolchains/cmake/test_cmake_multi.py @@ -63,6 +63,8 @@ def package_info(self): """) cmakelist = textwrap.dedent(""" + set(CMAKE_CXX_COMPILER_WORKS 1) + set(CMAKE_CXX_ABI_COMPILED 1) cmake_minimum_required(VERSION 3.15) project(hello_{name} LANGUAGES CXX) diff --git a/test/functional/toolchains/cmake/test_cps.py b/test/functional/toolchains/cmake/test_cps.py index 681acaecf24..84e9ad028b0 100644 --- a/test/functional/toolchains/cmake/test_cps.py +++ b/test/functional/toolchains/cmake/test_cps.py @@ -52,6 +52,8 @@ def package_info(self): """) cmake = textwrap.dedent("""\ + set(CMAKE_CXX_COMPILER_WORKS 1) + set(CMAKE_CXX_ABI_COMPILED 1) cmake_minimum_required(VERSION 4.2) project(mypkg CXX) @@ -80,6 +82,8 @@ def package_info(self): # Lets consume directly with CPS test_cmake = textwrap.dedent("""\ + set(CMAKE_CXX_COMPILER_WORKS 1) + set(CMAKE_CXX_ABI_COMPILED 1) cmake_minimum_required(VERSION 4.2) project(PackageTest CXX) @@ -182,6 +186,8 @@ def package_info(self): """) cmake = textwrap.dedent("""\ + set(CMAKE_CXX_COMPILER_WORKS 1) + set(CMAKE_CXX_ABI_COMPILED 1) cmake_minimum_required(VERSION 4.2) project(mypkg CXX) @@ -214,6 +220,8 @@ def package_info(self): # Create test_package files for the two components test_package_cmake = textwrap.dedent("""\ + set(CMAKE_CXX_COMPILER_WORKS 1) + set(CMAKE_CXX_ABI_COMPILED 1) cmake_minimum_required(VERSION 3.15) project(PackageTest CXX) @@ -342,6 +350,8 @@ def package_info(self): lib_type = "PUBLIC" if "public" in kind else "PRIVATE" cmake = textwrap.dedent("""\ + set(CMAKE_CXX_COMPILER_WORKS 1) + set(CMAKE_CXX_ABI_COMPILED 1) cmake_minimum_required(VERSION 4.2) project({name} CXX) @@ -386,6 +396,8 @@ def package_info(self): # Create test_package files for the two components test_package_cmake = textwrap.dedent("""\ + set(CMAKE_CXX_COMPILER_WORKS 1) + set(CMAKE_CXX_ABI_COMPILED 1) cmake_minimum_required(VERSION 3.15) project(PackageTest CXX) @@ -512,6 +524,8 @@ def test_pure_cmake_shared(): c = TestClient() cmake = textwrap.dedent("""\ + set(CMAKE_CXX_COMPILER_WORKS 1) + set(CMAKE_CXX_ABI_COMPILED 1) cmake_minimum_required(VERSION 4.2) project(myproj CXX) @@ -609,6 +623,8 @@ def package_info(self): """) cmake = textwrap.dedent("""\ + set(CMAKE_CXX_COMPILER_WORKS 1) + set(CMAKE_CXX_ABI_COMPILED 1) cmake_minimum_required(VERSION 4.2) project(mypkg CXX) @@ -642,6 +658,8 @@ def package_info(self): # Lets consume directly with CPS test_cmake = textwrap.dedent("""\ + set(CMAKE_CXX_COMPILER_WORKS 1) + set(CMAKE_CXX_ABI_COMPILED 1) cmake_minimum_required(VERSION 4.2) project(PackageTest CXX) diff --git a/test/functional/toolchains/ios/_utils.py b/test/functional/toolchains/ios/_utils.py index 10caeec6333..ccc5d0f7140 100644 --- a/test/functional/toolchains/ios/_utils.py +++ b/test/functional/toolchains/ios/_utils.py @@ -23,6 +23,8 @@ class HelloLib { """) cmakelists = textwrap.dedent(""" + set(CMAKE_CXX_COMPILER_WORKS 1) + set(CMAKE_CXX_ABI_COMPILED 1) cmake_minimum_required(VERSION 3.1) project(MyHello CXX) set(SOURCES diff --git a/test/functional/toolchains/meson/test_subproject.py b/test/functional/toolchains/meson/test_subproject.py index 7415bb0388b..668017a39b2 100644 --- a/test/functional/toolchains/meson/test_subproject.py +++ b/test/functional/toolchains/meson/test_subproject.py @@ -150,6 +150,8 @@ def test(self): """) _test_package_cmake_lists = textwrap.dedent(""" + set(CMAKE_C_COMPILER_WORKS 1) + set(CMAKE_C_ABI_COMPILED 1) cmake_minimum_required(VERSION 3.1) project(test_package C) diff --git a/test/functional/util/test_cmd_args_to_string.py b/test/functional/util/test_cmd_args_to_string.py index 8fe570315ab..08158f59332 100644 --- a/test/functional/util/test_cmd_args_to_string.py +++ b/test/functional/util/test_cmd_args_to_string.py @@ -30,6 +30,8 @@ def application_folder(): ) cmake = textwrap.dedent(""" + set(CMAKE_C_COMPILER_WORKS 1) + set(CMAKE_C_ABI_COMPILED 1) cmake_minimum_required(VERSION 3.15) project(arg_printer C) From eef9e937bb162e1501f98456dd9fa407cadc52f6 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 26 Mar 2026 14:12:21 +0100 Subject: [PATCH 070/110] short form for --update-requires (#19791) --- conan/cli/commands/lock.py | 6 +++--- test/integration/lockfile/test_user_overrides.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conan/cli/commands/lock.py b/conan/cli/commands/lock.py index d3cd0ff636a..db27e806810 100644 --- a/conan/cli/commands/lock.py +++ b/conan/cli/commands/lock.py @@ -180,11 +180,11 @@ def lock_upgrade(conan_api, parser, subparser, *args): given a conanfile or a reference. """ common_graph_args(subparser) - subparser.add_argument('--update-requires', action="append", + subparser.add_argument('-ur', '--update-requires', action="append", help='Update requires from lockfile') - subparser.add_argument('--update-build-requires', action="append", + subparser.add_argument('-ubr', '--update-build-requires', action="append", help='Update build-requires from lockfile') - subparser.add_argument('--update-python-requires', action="append", + subparser.add_argument('-upr', '--update-python-requires', action="append", help='Update python-requires from lockfile') subparser.add_argument('--build-require', action='store_true', default=False, help='Whether the provided reference is a build-require') diff --git a/test/integration/lockfile/test_user_overrides.py b/test/integration/lockfile/test_user_overrides.py index 5107fe9d7c0..0cb547f111d 100644 --- a/test/integration/lockfile/test_user_overrides.py +++ b/test/integration/lockfile/test_user_overrides.py @@ -419,7 +419,7 @@ def test_lock_upgrade_path(self): c.run(f"export libb --version=1.2") c.run(f"export libc --version=1.1") c.run(f"export libd --version=1.1") - c.run("lock upgrade . --update-requires=liba/1.0 --update-requires=libb/[*] --update-build-requires=libc/[*] --update-python-requires=libd/1.0") + c.run("lock upgrade . -ur=liba/1.0 -ur=libb/[*] -ubr=libc/[*] -upr=libd/1.0") lock = c.load("conan.lock") assert "liba/1.9" in lock assert "libb/1.1" in lock From 89711b440192d7000ac93a8e56942bb442363d7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Dahlgren?= Date: Fri, 27 Mar 2026 10:48:25 +0100 Subject: [PATCH 071/110] Add support for Apple Clang version 21 (#19795) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add support for Apple Clang version 21 Xcode 26.4 shipped with new Clang 21 * Add missing sdk verisons and apple os versions * Add missing sdk verisons and apple os versions --------- Co-authored-by: Abril Rincón Blanco <5364255+AbrilRBS@users.noreply.github.com> --- conan/internal/default_settings.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/conan/internal/default_settings.py b/conan/internal/default_settings.py index a8aebe4b99c..13b57f83c3d 100644 --- a/conan/internal/default_settings.py +++ b/conan/internal/default_settings.py @@ -24,14 +24,14 @@ "16.0", "16.1", "16.2", "16.3", "16.4", "16.5", "16.6", "16.7", "17.0", "17.1", "17.2", "17.3", "17.4", "17.5", "17.6", "17.7", "17.8", "18.0", "18.1", "18.2", "18.3", "18.4", "18.5", "18.6", "18.7", - "26.0", "26.1", "26.2", "26.3"] + "26.0", "26.1", "26.2", "26.3", "26.4"] sdk: ["iphoneos", "iphonesimulator"] sdk_version: [null, "11.3", "11.4", "12.0", "12.1", "12.2", "12.4", "13.0", "13.1", "13.2", "13.3", "13.4", "13.5", "13.6", "13.7", "14.0", "14.1", "14.2", "14.3", "14.4", "14.5", "15.0", "15.2", "15.4", "15.5", "16.0", "16.1", "16.2", "16.4", "17.0", "17.1", "17.2", "17.4", "17.5", "18.0", "18.1", "18.2", "18.4", "18.5", - "26.0", "26.1", "26.2"] + "26.0", "26.1", "26.2", "26.4"] watchOS: version: ["4.0", "4.1", "4.2", "4.3", "5.0", "5.1", "5.2", "5.3", "6.0", "6.1", "6.2", "6.3", "7.0", "7.1", "7.2", "7.3", "7.4", "7.5", "7.6", @@ -39,13 +39,13 @@ "9.0","9.1", "9.2", "9.3", "9.4", "9.5", "9.6", "10.0", "10.1", "10.2", "10.3", "10.4", "10.5", "10.6", "11.0", "11.1", "11.2", "11.3", "11.4", "11.5", "11.6", - "26.0", "26.1", "26.2", "26.3"] + "26.0", "26.1", "26.2", "26.3", "26.4"] sdk: ["watchos", "watchsimulator"] sdk_version: [null, "4.3", "5.0", "5.1", "5.2", "5.3", "6.0", "6.1", "6.2", "7.0", "7.1", "7.2", "7.3", "7.4", "8.0", "8.0.1", "8.3", "8.5", "9.0", "9.1", "9.4", "10.0", "10.1", "10.2", "10.4", "10.5", "11.0", "11.1", "11.2", "11.4", "11.5", - "26.0", "26.1", "26.2"] + "26.0", "26.1", "26.2", "26.4"] tvOS: version: ["11.0", "11.1", "11.2", "11.3", "11.4", "12.0", "12.1", "12.2", "12.3", "12.4", @@ -55,19 +55,19 @@ "16.0", "16.1", "16.2", "16.3", "16.4", "16.5", "16.6", "17.0", "17.1", "17.2", "17.3", "17.4", "17.5", "17.6", "18.0", "18.1", "18.2", "18.3", "18.4", "18.5", "18.6", - "26.0", "26.1", "26.2", "26.3"] + "26.0", "26.1", "26.2", "26.3", "26.4"] sdk: ["appletvos", "appletvsimulator"] sdk_version: [null, "11.3", "11.4", "12.0", "12.1", "12.2", "12.4", "13.0", "13.2", "13.3", "13.4", "14.0", "14.2", "14.3", "14.4", "14.5", "15.0", "15.2", "15.4", "15.5", "16.0", "16.1", "16.4", "17.0", "17.1", "17.2", "17.4", "17.5", "18.0", "18.1", "18.2", "18.4", "18.5", - "26.0", "26.1", "26.2"] + "26.0", "26.1", "26.2", "26.4"] visionOS: version: ["1.0", "1.1", "1.2", "1.3", "2.0", "2.1", "2.2", "2.3", "2.4", "2.5", "2.6", - "26.0", "26.1", "26.2", "26.3"] + "26.0", "26.1", "26.2", "26.3", "26.4"] sdk: ["xros", "xrsimulator"] sdk_version: [null, "1.0", "1.1", "1.2", "1.3", "2.0", "2.1", "2.2", "2.4", "2.5", - "26.0", "26.1", "26.2"] + "26.0", "26.1", "26.2", "26.4"] Macos: version: [null, "10.6", "10.7", "10.8", "10.9", "10.10", "10.11", "10.12", "10.13", "10.14", "10.15", "11.0", "11.1", "11.2", "11.3", "11.4", "11.5", "11.6", "11.7", @@ -75,7 +75,7 @@ "13.0", "13.1", "13.2", "13.3", "13.4", "13.5", "13.6", "13.7", "14.0", "14.1", "14.2", "14.3", "14.4", "14.5", "14.6", "14.7", "15.0", "15.1", "15.2", "15.3", "15.4", "15.5", "15.6", "15.7", - "26.0", "26.1", "26.2", "26.3"] + "26.0", "26.1", "26.2", "26.3", "26.4"] sdk_version: [null, "10.13", "10.14", "10.15", "11.0", "11.1", "11.2", "11.3", "12.0", "12.1", "12.3", "12.4", "13.0", "13.1", "13.3", "14.0", "14.2", "14.4", "14.5", "15.0", "15.1", "15.2", "15.4", "15.5", @@ -151,7 +151,7 @@ apple-clang: version: ["5.0", "5.1", "6.0", "6.1", "7.0", "7.3", "8.0", "8.1", "9.0", "9.1", "10.0", "11.0", "12.0", "13", "13.0", "13.1", "14", "14.0", "15", "15.0", - "16", "16.0", "17", "17.0"] + "16", "16.0", "17", "17.0", "21", "21.0"] libcxx: [libstdc++, libc++] cppstd: [null, 98, gnu98, 11, gnu11, 14, gnu14, 17, gnu17, 20, gnu20, 23, gnu23, 26, gnu26] cstd: [null, 99, gnu99, 11, gnu11, 17, gnu17, 23, gnu23] From 29a20a715ba02af56e570c66fc8f6c4caec00d76 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 27 Mar 2026 17:34:31 +0100 Subject: [PATCH 072/110] Test improvements (#19800) * wip * wip * wip * wip --- conan/internal/model/profile.py | 2 +- test/functional/revisions_test.py | 13 +- test/functional/sbom/__init__.py | 0 test/functional/sbom/test_cyclonedx.py | 84 --------- test/integration/cache/cache2_update_test.py | 57 +++--- .../command/download/download_test.py | 13 +- .../command/install/install_cascade_test.py | 8 +- .../command/install/install_test.py | 103 +++++------ test/integration/command/list/list_test.py | 6 +- test/integration/command/list/search_test.py | 51 ++---- .../list/test_combined_pkglist_flows.py | 10 +- .../integration/command/remote/remote_test.py | 9 +- .../command/remote/test_remote_users.py | 169 +++++++++--------- test/integration/command/source_test.py | 13 +- test/integration/command/test_outdated.py | 11 +- .../integration/command/upload/upload_test.py | 6 +- .../integration/configuration/profile_test.py | 41 +++-- .../python_requires_test.py | 0 .../graph/core/test_version_ranges.py | 13 +- .../version_ranges_cached_test.py | 42 ++--- .../version_ranges_diamond_test.py | 8 +- .../metadata/test_metadata_deploy.py | 3 +- test/integration/py_requires/__init__.py | 0 .../remote/multi_remote_checks_test.py | 58 +++--- test/integration/remote/multi_remote_test.py | 95 ++++------ .../remote/selected_remotes_test.py | 6 +- test/integration/sbom/test_cyclonedx.py | 78 ++++++++ test/unittests/model/profile_test.py | 84 +++++---- 28 files changed, 444 insertions(+), 539 deletions(-) delete mode 100644 test/functional/sbom/__init__.py delete mode 100644 test/functional/sbom/test_cyclonedx.py rename test/integration/{py_requires => extensions}/python_requires_test.py (100%) delete mode 100644 test/integration/py_requires/__init__.py diff --git a/conan/internal/model/profile.py b/conan/internal/model/profile.py index c9f95b4da97..e452536eb9f 100644 --- a/conan/internal/model/profile.py +++ b/conan/internal/model/profile.py @@ -173,7 +173,7 @@ def update_settings(self, new_settings): """Mix the specified settings with the current profile. Specified settings are prioritized to profile""" - assert isinstance(new_settings, OrderedDict) + assert isinstance(new_settings, (OrderedDict, dict)) # apply the current profile res = copy.copy(self.settings) diff --git a/test/functional/revisions_test.py b/test/functional/revisions_test.py index 6c8bc4d9adb..d069e380de7 100644 --- a/test/functional/revisions_test.py +++ b/test/functional/revisions_test.py @@ -1,6 +1,5 @@ import copy import time -from collections import OrderedDict import pytest from unittest.mock import patch @@ -35,8 +34,7 @@ class TestInstallingPackagesWithRevisions: def setup(self): self.server = TestServer() self.server2 = TestServer() - self.servers = OrderedDict([("default", self.server), - ("remote2", self.server2)]) + self.servers = {"default": self.server, "remote2": self.server2} self.c_v2 = TestClient(servers=self.servers, inputs=2*["admin", "password"]) self.ref = RecipeReference.loads("lib/1.0@conan/testing") @@ -274,7 +272,8 @@ class TestRemoveWithRevisions: @pytest.fixture(autouse=True) def setup(self): self.server = TestServer() - self.c_v2 = TestClient(servers={"default": self.server}, inputs=["admin", "password"]) + self.c_v2 = TestClient(light=True, + servers={"default": self.server}, inputs=["admin", "password"]) self.ref = RecipeReference.loads("lib/1.0@conan/testing") def test_remove_local_recipe(self): @@ -530,7 +529,8 @@ class TestServerRevisionsIndexes: @pytest.fixture(autouse=True) def setup(self): self.server = TestServer() - self.c_v2 = TestClient(servers={"default": self.server}, inputs=["admin", "password"]) + self.c_v2 = TestClient(light=True, + servers={"default": self.server}, inputs=["admin", "password"]) self.ref = RecipeReference.loads("lib/1.0@conan/testing") def test_rotation_deleting_recipe_revisions(self): @@ -660,8 +660,7 @@ def test_deleting_all_prevs(self): def test_touching_other_server(): # https://github.com/conan-io/conan/issues/9333 - servers = OrderedDict([("remote1", TestServer()), - ("remote2", None)]) # None server will crash if touched + servers = {"remote1": TestServer(), "remote2": None} # None server will crash if touched c = TestClient(servers=servers, inputs=["admin", "password"]) c.save({"conanfile.py": GenConanfile().with_settings("os")}) c.run("create . --name=pkg --version=0.1 --user=conan --channel=channel -s os=Windows") diff --git a/test/functional/sbom/__init__.py b/test/functional/sbom/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/test/functional/sbom/test_cyclonedx.py b/test/functional/sbom/test_cyclonedx.py deleted file mode 100644 index 2b357ce8c6f..00000000000 --- a/test/functional/sbom/test_cyclonedx.py +++ /dev/null @@ -1,84 +0,0 @@ -import json -import os - -import pytest - -from conan.internal.util.files import save -from conan.test.assets.genconanfile import GenConanfile -from conan.test.utils.tools import TestClient - -# Using the sbom tool with "conan create" -sbom_hook_post_package = """ -import json -import os -from conan.errors import ConanException -from conan.api.output import ConanOutput -from conan.tools.sbom import cyclonedx_1_4, cyclonedx_1_6 - -def post_package(conanfile): - sbom_cyclonedx_1_4 = cyclonedx_1_4(conanfile, add_build=True, add_tests=True) - sbom_cyclonedx_1_6 = cyclonedx_1_6(conanfile, add_build=True, add_tests=True) - with open(os.path.join(conanfile.package_metadata_folder, "sbom14.cdx.json"), 'w') as f: - json.dump(sbom_cyclonedx_1_4, f, indent=4) - with open(os.path.join(conanfile.package_metadata_folder, "sbom16.cdx.json"), 'w') as f: - json.dump(sbom_cyclonedx_1_6, f, indent=4) -""" - - -class TestCyclonedx: - - @pytest.fixture() - def hook_setup_post_package_tl(self, transitive_libraries): - tc = transitive_libraries - hook_path = os.path.join(tc.paths.hooks_path, "hook_sbom.py") - save(hook_path, sbom_hook_post_package) - return tc - - @pytest.mark.tool("cmake") - def test_sbom_generation_create(self, hook_setup_post_package_tl): - # TODO This doesn't need to be a functional test, check why - tc = hook_setup_post_package_tl - tc.run("new cmake_lib -d name=bar -d version=1.0 -d requires=engine/1.0 -f") - # bar -> engine/1.0 -> matrix/1.0 - tc.run("create . -tf=") - bar_layout = tc.created_layout() - assert os.path.exists(os.path.join(bar_layout.metadata(), "sbom14.cdx.json")) - assert os.path.exists(os.path.join(bar_layout.metadata(), "sbom16.cdx.json")) - - @pytest.mark.tool("cmake") - @pytest.mark.parametrize("user, channel, user_dep, channel_dep", - [("user", None, "user_dep", None), - ("user", "channel", "user_dep", "channel_dep")]) - def test_sbom_user_path(self, user, channel, user_dep, channel_dep): - tc = TestClient(light=True) - hook_path = os.path.join(tc.paths.hooks_path, "hook_sbom.py") - save(hook_path, sbom_hook_post_package) - channel_ref = f"/{channel_dep}" if channel_dep else "" - tc.save({"dep/conanfile.py": GenConanfile("dep", "1.0"), - "conanfile.py": GenConanfile("main", "1.0").with_requires( - f"dep/1.0@{user_dep}{channel_ref}")}) - command = "create dep" - if user: - command += f" --user={user_dep}" - if channel: - command += f" --channel={channel_dep}" - - tc.run(command) - - command = "create ." - if user: - command += f" --user={user}" - if channel: - command += f" --channel={channel}" - tc.run(command) - - for version in ("14", "16"): - create_layout = tc.created_layout() - cyclone_path = os.path.join(create_layout.metadata(), f"sbom{version}.cdx.json") - content = tc.load(cyclone_path) - content_json = json.loads(content) - - assert content_json["components"][0]["bom-ref"].split("&user=")[ - 1] == f"{user}&channel={channel}" if channel else user - assert content_json["dependencies"][0]["dependsOn"][0].split("&user=")[ - 1] == f"{user_dep}&channel={channel_dep}" if channel_dep else user_dep diff --git a/test/integration/cache/cache2_update_test.py b/test/integration/cache/cache2_update_test.py index 4021f07889e..9acb947dd84 100644 --- a/test/integration/cache/cache2_update_test.py +++ b/test/integration/cache/cache2_update_test.py @@ -1,7 +1,6 @@ import copy import json import textwrap -from collections import OrderedDict import pytest from unittest.mock import patch @@ -17,7 +16,7 @@ class TestUpdateFlows: def _setup(self): self.liba = RecipeReference.loads("liba/1.0.0") - servers = OrderedDict() + servers = {} for index in range(3): servers[f"server{index}"] = TestServer([("*/*@*/*", "*")], [("*/*@*/*", "*")], users={"user": "password"}) @@ -442,33 +441,33 @@ def test_version_ranges(self): "from remote 'server2' " in self.client.out -@pytest.mark.parametrize("update,result", [ - # Not a real pattern, works to support legacy syntax - ["*", {"liba/1.1": "Downloaded (default)", - "libb/1.1": "Downloaded (default)"}], - ["libc", {"liba/1.0": "Cache", - "libb/1.0": "Cache"}], - ["liba", {"liba/1.1": "Downloaded (default)", - "libb/1.0": "Cache"}], - ["libb", {"liba/1.0": "Cache", - "libb/1.1": "Downloaded (default)"}], - ["libb/1.0", {"liba/1.0": "Cache", - "libb/1.0": "Cache"}], - ["libb/1.0#7e88fd43dc3c8171b6f38f8d1b139641", - {"liba/1.0": "Cache", - "libb/1.0": "Cache"}], - ["", {"liba/1.0": "Cache", - "libb/1.0": "Cache"}], - # Patterns not supported, only full name match - ["lib*", {"liba/1.0": "Cache", - "libb/1.0": "Cache"}], - ["liba/*", {"liba/1.0": "Cache", - "libb/1.0": "Cache"}], - # None only passes legacy --update without args, - # to ensure it works, it should be the same as passing * - [None, {"liba/1.1": "Downloaded (default)", - "libb/1.1": "Downloaded (default)"}] - ]) +@pytest.mark.parametrize("update,result", + [ + # Not a real pattern, works to support legacy syntax + ["*", {"liba/1.1": "Downloaded (default)", + "libb/1.1": "Downloaded (default)"}], + ["libc", {"liba/1.0": "Cache", + "libb/1.0": "Cache"}], + ["liba", {"liba/1.1": "Downloaded (default)", + "libb/1.0": "Cache"}], + ["libb", {"liba/1.0": "Cache", + "libb/1.1": "Downloaded (default)"}], + ["libb/1.0", {"liba/1.0": "Cache", + "libb/1.0": "Cache"}], + ["libb/1.0#7e88fd43dc3c8171b6f38f8d1b139641", {"liba/1.0": "Cache", + "libb/1.0": "Cache"}], + ["", {"liba/1.0": "Cache", + "libb/1.0": "Cache"}], + # Patterns not supported, only full name match + ["lib*", {"liba/1.0": "Cache", + "libb/1.0": "Cache"}], + ["liba/*", {"liba/1.0": "Cache", + "libb/1.0": "Cache"}], + # None only passes legacy --update without args, + # to ensure it works, it should be the same as passing * + [None, {"liba/1.1": "Downloaded (default)", + "libb/1.1": "Downloaded (default)"}] + ]) def test_muliref_update_pattern(update, result): tc = TestClient(light=True, default_server_user=True) tc.save({"liba/conanfile.py": GenConanfile("liba"), diff --git a/test/integration/command/download/download_test.py b/test/integration/command/download/download_test.py index fb1daa139f4..7b0bb92512a 100644 --- a/test/integration/command/download/download_test.py +++ b/test/integration/command/download/download_test.py @@ -1,6 +1,6 @@ import os import textwrap -from collections import OrderedDict + from unittest import mock from conan.api.model import RecipeReference @@ -35,12 +35,14 @@ def test_no_user_channel(): client.run("remove * -c") client.run("download pkg/1.0:{} -r default".format(NO_SETTINGS_PACKAGE_ID)) - assert f"Downloading package 'pkg/1.0#4d670581ccb765839f2239cc8dff8fbd:{NO_SETTINGS_PACKAGE_ID}" in client.out + assert (f"Downloading package " + f"'pkg/1.0#4d670581ccb765839f2239cc8dff8fbd:{NO_SETTINGS_PACKAGE_ID}") in client.out # All client.run("remove * -c") client.run("download pkg/1.0#*:* -r default") - assert f"Downloading package 'pkg/1.0#4d670581ccb765839f2239cc8dff8fbd:{NO_SETTINGS_PACKAGE_ID}" in client.out + assert (f"Downloading package " + f"'pkg/1.0#4d670581ccb765839f2239cc8dff8fbd:{NO_SETTINGS_PACKAGE_ID}") in client.out def test_download_with_python_requires(): @@ -52,8 +54,7 @@ def test_download_with_python_requires(): really need to load conanfile, so it doesn't fail because of this. """ # https://github.com/conan-io/conan/issues/9548 - servers = OrderedDict([("tools", TestServer()), - ("pkgs", TestServer())]) + servers = {"tools": TestServer(), "pkgs": TestServer()} c = TestClient(servers=servers, inputs=["admin", "password", "admin", "password"]) c.save({"tool/conanfile.py": GenConanfile("tool", "0.1"), @@ -90,7 +91,7 @@ def source(self): did_verify = {} - def custom_download(this, url, filepath, *args, **kwargs): + def custom_download(this, url, filepath, *args, **kwargs): # noqa did_verify[url] = args[2] with mock.patch("conan.internal.rest.file_downloader.FileDownloader.download", diff --git a/test/integration/command/install/install_cascade_test.py b/test/integration/command/install/install_cascade_test.py index f4f6445a900..f2e0b2a3e76 100644 --- a/test/integration/command/install/install_cascade_test.py +++ b/test/integration/command/install/install_cascade_test.py @@ -1,6 +1,4 @@ -from collections import OrderedDict - -from conan.test.utils.tools import TestServer, GenConanfile, TestClient +from conan.test.utils.tools import GenConanfile, TestClient def test_cascade(): @@ -8,9 +6,7 @@ def test_cascade(): app -> E -> D -> B -> A \\-> F -> C -------/ """ - server = TestServer() - servers = OrderedDict([("default", server)]) - c = TestClient(servers=servers) + c = TestClient(default_server_user=True) c.save({"a/conanfile.py": GenConanfile("liba", "1.0"), "b/conanfile.py": GenConanfile("libb", "1.0").with_requires("liba/1.0"), "c/conanfile.py": GenConanfile("libc", "1.0").with_requires("liba/1.0"), diff --git a/test/integration/command/install/install_test.py b/test/integration/command/install/install_test.py index e0c26dd5d75..da3c42b16f9 100644 --- a/test/integration/command/install/install_test.py +++ b/test/integration/command/install/install_test.py @@ -2,7 +2,6 @@ import os import re import textwrap -from collections import OrderedDict import pytest @@ -28,8 +27,10 @@ def test_install_reference_txt(client): def test_install_reference_error(client): # Test to check the "conan install " command argument - client.run("install --requires=pkg/0.1@myuser/testing --user=user --channel=testing", assert_error=True) - assert "ERROR: Can't use --name, --version, --user or --channel arguments with --requires" in client.out + client.run("install --requires=pkg/0.1@myuser/testing --user=user --channel=testing", + assert_error=True) + assert ("ERROR: Can't use --name, --version, " + "--user or --channel arguments with --requires") in client.out client.save({"conanfile.py": GenConanfile("pkg", "1.0")}) client.run("install . --channel=testing", assert_error=True) assert "Can't specify channel 'testing' without user" in client.out @@ -77,67 +78,67 @@ class Pkg(ConanFile): def package_info(self): self.output.info("PKG OPTION: %s" % self.options.shared) """)}) - client.run("create . --name=pkg --version=0.1 --user=user --channel=testing -o shared=True") - assert "pkg/0.1@user/testing: PKG OPTION: True" in client.out + client.run("create . --name=pkg --version=0.1 -o shared=True") + assert "pkg/0.1: PKG OPTION: True" in client.out client.save({"conanfile.py": textwrap.dedent(""" from conan import ConanFile class Pkg(ConanFile): - requires = "pkg/0.1@user/testing" + requires = "pkg/0.1" options = {"shared": [True, False, "header"]} default_options = {"shared": False} def package_info(self): self.output.info("PKG2 OPTION: %s" % self.options.shared) """)}) - client.run("create . --name=pkg2 --version=0.1 --user=user --channel=testing -o *:shared=True") - assert "pkg/0.1@user/testing: PKG OPTION: True" in client.out - assert "pkg2/0.1@user/testing: PKG2 OPTION: True" in client.out - client.run(" install --requires=pkg2/0.1@user/testing -o *:shared=True") - assert "pkg/0.1@user/testing: PKG OPTION: True" in client.out - assert "pkg2/0.1@user/testing: PKG2 OPTION: True" in client.out + client.run("create . --name=pkg2 --version=0.1 -o *:shared=True") + assert "pkg/0.1: PKG OPTION: True" in client.out + assert "pkg2/0.1: PKG2 OPTION: True" in client.out + client.run(" install --requires=pkg2/0.1 -o *:shared=True") + assert "pkg/0.1: PKG OPTION: True" in client.out + assert "pkg2/0.1: PKG2 OPTION: True" in client.out # Priority of non-scoped options - client.run("create . --name=pkg2 --version=0.1 --user=user --channel=testing -o shared=header -o *:shared=True") - assert "pkg/0.1@user/testing: PKG OPTION: True" in client.out - assert "pkg2/0.1@user/testing: PKG2 OPTION: header" in client.out - client.run(" install --requires=pkg2/0.1@user/testing -o shared=header -o *:shared=True") - assert "pkg/0.1@user/testing: PKG OPTION: True" in client.out - assert "pkg2/0.1@user/testing: PKG2 OPTION: header" in client.out + client.run("create . --name=pkg2 --version=0.1 -o shared=header -o *:shared=True") + assert "pkg/0.1: PKG OPTION: True" in client.out + assert "pkg2/0.1: PKG2 OPTION: header" in client.out + client.run(" install --requires=pkg2/0.1 -o shared=header -o *:shared=True") + assert "pkg/0.1: PKG OPTION: True" in client.out + assert "pkg2/0.1: PKG2 OPTION: header" in client.out # Prevalence of exact named option - client.run("create . --name=pkg2 --version=0.1 --user=user --channel=testing -o *:shared=True -o pkg2*:shared=header") - assert "pkg/0.1@user/testing: PKG OPTION: True" in client.out - assert "pkg2/0.1@user/testing: PKG2 OPTION: header" in client.out - client.run(" install --requires=pkg2/0.1@user/testing -o *:shared=True -o pkg2*:shared=header") - assert "pkg/0.1@user/testing: PKG OPTION: True" in client.out - assert "pkg2/0.1@user/testing: PKG2 OPTION: header" in client.out + client.run("create . --name=pkg2 --version=0.1 -o *:shared=True -o pkg2*:shared=header") + assert "pkg/0.1: PKG OPTION: True" in client.out + assert "pkg2/0.1: PKG2 OPTION: header" in client.out + client.run(" install --requires=pkg2/0.1 -o *:shared=True -o pkg2*:shared=header") + assert "pkg/0.1: PKG OPTION: True" in client.out + assert "pkg2/0.1: PKG2 OPTION: header" in client.out # Prevalence of exact named option reverse - client.run("create . --name=pkg2 --version=0.1 --user=user --channel=testing -o *:shared=True -o pkg/*:shared=header " + client.run("create . --name=pkg2 --version=0.1 -o *:shared=True -o pkg/*:shared=header " "--build=missing") - assert "pkg/0.1@user/testing: PKG OPTION: header" in client.out - assert "pkg2/0.1@user/testing: PKG2 OPTION: True" in client.out - client.run(" install --requires=pkg2/0.1@user/testing -o *:shared=True -o pkg/*:shared=header") - assert "pkg/0.1@user/testing: PKG OPTION: header" in client.out - assert "pkg2/0.1@user/testing: PKG2 OPTION: True" in client.out + assert "pkg/0.1: PKG OPTION: header" in client.out + assert "pkg2/0.1: PKG2 OPTION: True" in client.out + client.run(" install --requires=pkg2/0.1 -o *:shared=True -o pkg/*:shared=header") + assert "pkg/0.1: PKG OPTION: header" in client.out + assert "pkg2/0.1: PKG2 OPTION: True" in client.out # Prevalence of alphabetical pattern - client.run("create . --name=pkg2 --version=0.1 --user=user --channel=testing -o *:shared=True -o pkg2*:shared=header") - assert "pkg/0.1@user/testing: PKG OPTION: True" in client.out - assert "pkg2/0.1@user/testing: PKG2 OPTION: header" in client.out - client.run(" install --requires=pkg2/0.1@user/testing -o *:shared=True -o pkg2*:shared=header") - assert "pkg/0.1@user/testing: PKG OPTION: True" in client.out - assert "pkg2/0.1@user/testing: PKG2 OPTION: header" in client.out + client.run("create . --name=pkg2 --version=0.1 -o *:shared=True -o pkg2*:shared=header") + assert "pkg/0.1: PKG OPTION: True" in client.out + assert "pkg2/0.1: PKG2 OPTION: header" in client.out + client.run(" install --requires=pkg2/0.1 -o *:shared=True -o pkg2*:shared=header") + assert "pkg/0.1: PKG OPTION: True" in client.out + assert "pkg2/0.1: PKG2 OPTION: header" in client.out # Prevalence of last match, even first pattern match - client.run("create . --name=pkg2 --version=0.1 --user=user --channel=testing -o pkg2*:shared=header -o *:shared=True") - assert "pkg/0.1@user/testing: PKG OPTION: True" in client.out - assert "pkg2/0.1@user/testing: PKG2 OPTION: True" in client.out - client.run(" install --requires=pkg2/0.1@user/testing -o pkg2*:shared=header -o *:shared=True") - assert "pkg/0.1@user/testing: PKG OPTION: True" in client.out - assert "pkg2/0.1@user/testing: PKG2 OPTION: True" in client.out + client.run("create . --name=pkg2 --version=0.1 -o pkg2*:shared=header -o *:shared=True") + assert "pkg/0.1: PKG OPTION: True" in client.out + assert "pkg2/0.1: PKG2 OPTION: True" in client.out + client.run(" install --requires=pkg2/0.1 -o pkg2*:shared=header -o *:shared=True") + assert "pkg/0.1: PKG OPTION: True" in client.out + assert "pkg2/0.1: PKG2 OPTION: True" in client.out # Prevalence and override of alphabetical pattern - client.run("create . --name=pkg2 --version=0.1 --user=user --channel=testing -o *:shared=True -o pkg*:shared=header") - assert "pkg/0.1@user/testing: PKG OPTION: header" in client.out - assert "pkg2/0.1@user/testing: PKG2 OPTION: header" in client.out - client.run(" install --requires=pkg2/0.1@user/testing -o *:shared=True -o pkg*:shared=header") - assert "pkg/0.1@user/testing: PKG OPTION: header" in client.out - assert "pkg2/0.1@user/testing: PKG2 OPTION: header" in client.out + client.run("create . --name=pkg2 --version=0.1 -o *:shared=True -o pkg*:shared=header") + assert "pkg/0.1: PKG OPTION: header" in client.out + assert "pkg2/0.1: PKG2 OPTION: header" in client.out + client.run(" install --requires=pkg2/0.1 -o *:shared=True -o pkg*:shared=header") + assert "pkg/0.1: PKG OPTION: header" in client.out + assert "pkg2/0.1: PKG2 OPTION: header" in client.out def test_install_package_folder(client): @@ -297,9 +298,9 @@ def test_install_no_remotes(client): def test_install_skip_disabled_remote(): - client = TestClient(servers=OrderedDict({"default": TestServer(), - "server2": TestServer(), - "server3": TestServer()}), + client = TestClient(servers={"default": TestServer(), + "server2": TestServer(), + "server3": TestServer()}, inputs=2*["admin", "password"]) client.save({"conanfile.py": GenConanfile()}) client.run("create . --name=pkg --version=0.1 --user=lasote --channel=testing") diff --git a/test/integration/command/list/list_test.py b/test/integration/command/list/list_test.py index 202408be4d1..579485f4287 100644 --- a/test/integration/command/list/list_test.py +++ b/test/integration/command/list/list_test.py @@ -3,7 +3,6 @@ import re import textwrap import time -from collections import OrderedDict from unittest.mock import patch, Mock import pytest @@ -85,9 +84,8 @@ def test_graph_file_error(self): @pytest.fixture(scope="module") def client(): - servers = OrderedDict([("default", TestServer()), - ("other", TestServer())]) - c = TestClient(servers=servers, inputs=2*["admin", "password"]) + c = TestClient(servers={"default": TestServer(), "other": TestServer()}, + inputs=2*["admin", "password"]) c.save({ "zlib.py": GenConanfile("zlib"), "zlib_ng.py": GenConanfile("zlib_ng", "1.0.0"), diff --git a/test/integration/command/list/search_test.py b/test/integration/command/list/search_test.py index a6ceffacbc5..dc299c2dbf9 100644 --- a/test/integration/command/list/search_test.py +++ b/test/integration/command/list/search_test.py @@ -1,5 +1,5 @@ import textwrap -from collections import OrderedDict + from unittest.mock import patch, Mock import pytest @@ -15,18 +15,14 @@ class TestSearch: @pytest.fixture def remotes(self): - self.servers = OrderedDict() - self.servers["remote1"] = TestServer(server_capabilities=[]) - self.servers["remote2"] = TestServer(server_capabilities=[]) - - self.client = TestClient(servers=self.servers) + self.servers = {"remote1": TestServer(server_capabilities=[]), + "remote2": TestServer(server_capabilities=[])} + self.client = TestClient(light=True, servers=self.servers) def test_search_no_params(self): - self.servers = OrderedDict() - self.client = TestClient(servers=self.servers) - - self.client.run("search", assert_error=True) - assert "error: the following arguments are required: reference" in self.client.out + client = TestClient(light=True, servers={}) + client.run("search", assert_error=True) + assert "error: the following arguments are required: reference" in client.out def test_search_no_matching_recipes(self, remotes): expected_output = ("Connecting to remote 'remote1' anonymously\n" @@ -40,11 +36,9 @@ def test_search_no_matching_recipes(self, remotes): assert expected_output == self.client.out def test_search_no_configured_remotes(self): - self.servers = OrderedDict() - self.client = TestClient(servers=self.servers) - - self.client.run("search whatever", assert_error=True) - assert "There are no remotes to search from" in self.client.out + client = TestClient(light=True) + client.run("search whatever", assert_error=True) + assert "There are no remotes to search from" in client.out def test_search_disabled_remote(self, remotes): self.client.run("remote disable remote1") @@ -57,25 +51,20 @@ class TestRemotes: @pytest.fixture(autouse=True) def _setup(self): - self.servers = OrderedDict() self.users = {} - self.client = TestClient() + self.servers = {} + self.client = TestClient(light=True) def _add_remote(self, remote_name): self.servers[remote_name] = TestServer(users={"user": "passwd"}) self.users[remote_name] = [("user", "passwd")] - self.client = TestClient(servers=self.servers, inputs=["user", "passwd"]) + self.client = TestClient(light=True, servers=self.servers, inputs=["user", "passwd"]) def _add_recipe(self, remote, reference): - conanfile = textwrap.dedent(""" - from conan import ConanFile - class MyLib(ConanFile): - pass - """) - - self.client.save({'conanfile.py': conanfile}) + self.client.save({'conanfile.py': GenConanfile()}) reference = RecipeReference.loads(str(reference)) - self.client.run(f"export . --name={reference.name} --version={reference.version} --user={reference.user} --channel={reference.channel}") + self.client.run(f"export . --name={reference.name} --version={reference.version} " + f"--user={reference.user} --channel={reference.channel}") self.client.run("upload --force -r {} {}".format(remote, reference)) @pytest.mark.parametrize("exc,output", [ @@ -96,11 +85,6 @@ def test_search_remote_errors_but_no_raising_exceptions(self, exc, output): """) assert expected_output == self.client.out - def test_no_remotes(self): - self.client.run("search something", assert_error=True) - expected_output = "There are no remotes to search from" - assert expected_output in self.client.out - def test_search_by_name(self): remote_name = "remote1" recipe_name = "test_recipe/1.0.0@user/channel" @@ -176,7 +160,6 @@ def test_search_in_one_remote(self): assert expected_output in self.client.out def test_search_package_found_in_one_remote(self): - remote1 = "remote1" remote2 = "remote2" @@ -249,7 +232,7 @@ def test_search_wildcard(self): def test_no_user_channel_error(): # https://github.com/conan-io/conan/issues/13170 - c = TestClient(default_server_user=True) + c = TestClient(light=True, default_server_user=True) c.save({"conanfile.py": GenConanfile("pkg")}) c.run("export . --version=1.0") c.run("export . --version=1.0 --user=user --channel=channel") diff --git a/test/integration/command/list/test_combined_pkglist_flows.py b/test/integration/command/list/test_combined_pkglist_flows.py index 906a3d42be9..eac7f51f007 100644 --- a/test/integration/command/list/test_combined_pkglist_flows.py +++ b/test/integration/command/list/test_combined_pkglist_flows.py @@ -1,5 +1,4 @@ import json -from collections import OrderedDict import pytest @@ -179,7 +178,7 @@ class TestPkgListFindRemote: """ we can recover a list of remotes for an already installed graph, for metadata download """ def test_graph_2_pkg_list_remotes(self): - servers = OrderedDict([("default", TestServer()), ("remote2", TestServer())]) + servers = {"default": TestServer(), "remote2": TestServer()} c = TestClient(servers=servers, inputs=2 * ["admin", "password"], light=True) c.save({"zlib/conanfile.py": GenConanfile("zlib", "1.0"), "app/conanfile.py": GenConanfile("app", "1.0").with_requires("zlib/1.0")}) @@ -329,7 +328,7 @@ class TestPkgListMerge: """ deep merge lists """ def test_graph_2_pkg_list_remotes(self): - servers = OrderedDict([("default", TestServer()), ("remote2", TestServer())]) + servers = {"default": TestServer(), "remote2": TestServer()} c = TestClient(servers=servers, inputs=2 * ["admin", "password"]) c.save({"zlib/conanfile.py": GenConanfile("zlib", "1.0").with_settings("build_type"), "bzip2/conanfile.py": GenConanfile("bzip2", "1.0").with_settings("build_type"), @@ -603,9 +602,8 @@ def test_context_only_binary_mismatch(self, context): tc = TestClient(light=True) tc.save({ "protobuf/conanfile.py": GenConanfile("protobuf", "1.0"), - "onnx/conanfile.py": GenConanfile("onnx", "1.0") - .with_requires("protobuf/1.0") - .with_tool_requires("protobuf/1.0")}) + "onnx/conanfile.py": GenConanfile("onnx", "1.0").with_requires("protobuf/1.0") + .with_tool_requires("protobuf/1.0")}) tc.run("create protobuf") tc.run("create onnx") diff --git a/test/integration/command/remote/remote_test.py b/test/integration/command/remote/remote_test.py index 45e57366103..a8173575d1c 100644 --- a/test/integration/command/remote/remote_test.py +++ b/test/integration/command/remote/remote_test.py @@ -1,7 +1,6 @@ import json import os import pytest -from collections import OrderedDict from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient, TestServer @@ -12,12 +11,8 @@ class TestRemoteServer: @pytest.fixture(autouse=True) def setup(self): - self.servers = OrderedDict() - for i in range(3): - test_server = TestServer() - self.servers["remote%d" % i] = test_server - - self.client = TestClient(servers=self.servers, inputs=3 * ["admin", "password"], light=True) + self.client = TestClient(servers={f"remote{i}": TestServer() for i in range(3)}, + inputs=3 * ["admin", "password"], light=True) def test_list_json(self): self.client.run("remote list --format=json") diff --git a/test/integration/command/remote/test_remote_users.py b/test/integration/command/remote/test_remote_users.py index d492d4b4296..3c67a679151 100644 --- a/test/integration/command/remote/test_remote_users.py +++ b/test/integration/command/remote/test_remote_users.py @@ -1,22 +1,22 @@ import json import textwrap import time -from collections import OrderedDict from datetime import timedelta from unittest.mock import patch from conan.internal.api.remotes.localdb import LocalDB +from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient, TestServer from conan.test.utils.env import environment_update -class TestUser: +class TestRemoteLogin: def test_command_user_no_remotes(self): """ Test that proper error is reported when no remotes are defined and conan user is executed """ - client = TestClient() + client = TestClient(light=True) client.run("remote list-users", assert_error=True) assert "ERROR: No remotes defined" in client.out @@ -26,10 +26,8 @@ def test_command_user_no_remotes(self): def test_command_user_list(self): """ Test list of user is reported for all remotes or queried remote """ - servers = OrderedDict() - servers["default"] = TestServer() - servers["test_remote_1"] = TestServer() - client = TestClient(servers=servers) + client = TestClient(light=True, servers={"default": TestServer(), + "test_remote_1": TestServer()}) # Test with wrong remote right error is reported client.run("remote login Test_Wrong_Remote foo", assert_error=True) @@ -45,7 +43,7 @@ def test_command_user_list(self): def test_with_remote_no_connect(self): test_server = TestServer() - client = TestClient(servers={"default": test_server}) + client = TestClient(light=True, servers={"default": test_server}) client.run('remote list-users') assert textwrap.dedent(""" default: @@ -61,7 +59,8 @@ def test_with_remote_no_connect(self): assert ('will', None, None) == localdb.get_login(test_server.fake_url) client.run('remote logout default') - assert "Changed user of remote 'default' from 'will' (anonymous) to 'None' (anonymous)" in client.out + assert ("Changed user of remote 'default' from 'will' (anonymous) to " + "'None' (anonymous)") in client.out assert (None, None, None) == localdb.get_login(test_server.fake_url) def test_command_user_with_password(self): @@ -71,14 +70,15 @@ def test_command_user_with_password(self): """ test_server = TestServer() servers = {"default": test_server} - client = TestClient(servers=servers, inputs=["admin", "password"]) + client = TestClient(light=True, servers=servers, inputs=["admin", "password"]) client.run('remote login default dummy -p ping_pong2', assert_error=True) assert "ERROR: Wrong user or password" in client.out client.run('remote login default admin -p password') assert "ERROR: Wrong user or password" not in client.out assert "Changed user of remote 'default' from 'None' (anonymous) to 'admin'" in client.out client.run('remote logout default') - assert "Changed user of remote 'default' from 'admin' (authenticated) to 'None' (anonymous)" in client.out + assert ("Changed user of remote 'default' from 'admin' (authenticated) to " + "'None' (anonymous)") in client.out localdb = LocalDB(client.cache_folder) assert (None, None, None) == localdb.get_login(test_server.fake_url) client.run('remote list-users') @@ -91,7 +91,7 @@ def test_command_user_with_password_spaces(self): """ test_server = TestServer(users={"lasote": 'my "password'}) servers = {"default": test_server} - client = TestClient(servers=servers, inputs=["lasote", "mypass"]) + client = TestClient(light=True, servers=servers, inputs=["lasote", "mypass"]) client.run(r'remote login default lasote -p="my \"password"') assert "Connecting to remote" not in client.out assert "Changed user of remote 'default' from 'None' (anonymous) to 'lasote'" in client.out @@ -103,22 +103,15 @@ def test_command_user_with_password_spaces(self): def test_clean(self): test_server = TestServer() servers = {"default": test_server} - client = TestClient(servers=servers, inputs=2*["admin", "password"]) - base = ''' -from conan import ConanFile - -class ConanLib(ConanFile): - name = "lib" - version = "0.1" -''' - files = {"conanfile.py": base} - client.save(files) + client = TestClient(light=True, servers=servers, inputs=2*["admin", "password"]) + client.save({"conanfile.py": GenConanfile("lib", "0.1")}) client.run("export . --user=lasote --channel=stable") client.run("upload lib/0.1@lasote/stable -r default --only-recipe") client.run("remote list-users") assert 'default:\n Username: admin\n authenticated: True' in client.out client.run("remote logout default") - assert "Changed user of remote 'default' from 'admin' (authenticated) to 'None' (anonymous)" in client.out + assert ("Changed user of remote 'default' from 'admin' (authenticated) to " + "'None' (anonymous)") in client.out client.run("remote list-users") assert 'default:\n No user' in client.out # --force will force re-authentication, otherwise not necessary to auth @@ -129,16 +122,17 @@ class ConanLib(ConanFile): def test_command_interactive_only(self): test_server = TestServer() servers = {"default": test_server} - client = TestClient(servers=servers, inputs=["password"]) + client = TestClient(light=True, servers=servers, inputs=["password"]) client.run('remote login default admin -p') - assert "Changed user of remote 'default' from 'None' (anonymous) to 'admin' (authenticated)" in client.out + assert ("Changed user of remote 'default' from 'None' (anonymous) to 'admin' " + "(authenticated)") in client.out def test_command_user_with_interactive_password_login_prompt_disabled(self): """ Interactive password should not work. """ test_server = TestServer() servers = {"default": test_server} - client = TestClient(servers=servers, inputs=[]) + client = TestClient(light=True, servers=servers, inputs=[]) conan_conf = "core:non_interactive=True" client.save_home({"global.conf": conan_conf}) client.run('remote login default admin -p', assert_error=True) @@ -149,12 +143,12 @@ def test_command_user_with_interactive_password_login_prompt_disabled(self): def test_authenticated(self): test_server = TestServer(users={"lasote": "mypass", "danimtb": "passpass"}) - servers = OrderedDict() - servers["default"] = test_server - servers["other_server"] = TestServer() - client = TestClient(servers=servers, inputs=["lasote", "mypass", "mypass", "mypass"]) + servers = {"default": test_server, "other_server": TestServer()} + client = TestClient(light=True, servers=servers, + inputs=["lasote", "mypass", "mypass", "mypass"]) client.run("remote logout default") - assert "Changed user of remote 'default' from 'None' (anonymous) to 'None' (anonymous)" in client.out + assert ("Changed user of remote 'default' from 'None' (anonymous) to " + "'None' (anonymous)") in client.out assert "[authenticated]" not in client.out client.run('remote set-user default bad_user') client.run("remote list-users") @@ -167,17 +161,17 @@ def test_authenticated(self): assert 'default:\n Username: lasote\n authenticated: True' in client.out client.run("remote login default danimtb -p passpass") - assert "Changed user of remote 'default' from 'lasote' (authenticated) to 'danimtb' (authenticated)" in client.out + assert ("Changed user of remote 'default' from 'lasote' (authenticated) to " + "'danimtb' (authenticated)") in client.out client.run("remote list-users") assert 'default:\n Username: danimtb\n authenticated: True' in client.out def test_json(self): default_server = TestServer(users={"lasote": "mypass", "danimtb": "passpass"}) other_server = TestServer() - servers = OrderedDict() - servers["default"] = default_server - servers["other_server"] = other_server - client = TestClient(servers=servers, inputs=["lasote", "mypass", "danimtb", "passpass"]) + servers = {"default": default_server, "other_server": other_server} + client = TestClient(light=True, servers=servers, + inputs=["lasote", "mypass", "danimtb", "passpass"]) client.run("remote list-users -f json") info = json.loads(client.stdout) assert info == [ @@ -294,19 +288,20 @@ def test_json(self): def test_skip_auth(self): default_server = TestServer(users={"lasote": "mypass", "danimtb": "passpass"}) - servers = OrderedDict() - servers["default"] = default_server - client = TestClient(servers=servers) + servers = {"default": default_server} + client = TestClient(light=True, servers=servers) # Regular auth client.run("remote login default lasote -p mypass") # Now skip the auth but keeping the same user client.run("remote set-user default lasote") - assert "Changed user of remote 'default' from 'lasote' (authenticated) to 'lasote' (authenticated)" in client.out + assert ("Changed user of remote 'default' from 'lasote' (authenticated) to " + "'lasote' (authenticated)") in client.out # If we change the user the credentials are removed client.run("remote set-user default flanders") - assert "Changed user of remote 'default' from 'lasote' (authenticated) to 'flanders' (anonymous)" in client.out + assert ("Changed user of remote 'default' from 'lasote' (authenticated) to " + "'flanders' (anonymous)") in client.out client.run("remote login default lasote -p BAD_PASS", assert_error=True) assert "Wrong user or password" in client.out @@ -315,10 +310,11 @@ def test_skip_auth(self): client.run("remote login default lasote -p mypass") def test_login_multiremote(self): - servers = OrderedDict() - servers["default"] = TestServer(users={"admin": "password"}) - servers["other"] = TestServer(users={"admin": "password"}) - c = TestClient(servers=servers, inputs=["admin", "password", "wrong", "wrong"]) + servers = { + "default": TestServer(users={"admin": "password"}), + "other": TestServer(users={"admin": "password"}), + } + c = TestClient(light=True, servers=servers, inputs=["admin", "password", "wrong", "wrong"]) # This must fail, not autthenticate in the next remote c.run("remote login *", assert_error=True) assert "ERROR: Wrong user or password. [Remote: other]" in c.out @@ -327,7 +323,7 @@ def test_login_multiremote(self): def test_user_removed_remote_removed(): # Make sure that removing a remote clears the credentials # https://github.com/conan-io/conan/issues/5562 - c = TestClient(default_server_user=True) + c = TestClient(light=True, default_server_user=True) server_url = c.servers["default"].fake_url c.run("remote login default admin -p password") localdb = LocalDB(c.cache_folder) @@ -340,11 +336,12 @@ def test_user_removed_remote_removed(): class TestRemoteAuth: def test_remote_auth(self): - servers = OrderedDict() - servers["default"] = TestServer(users={"lasote": "mypass", "danimtb": "passpass"}) - servers["other_server"] = TestServer(users={"lasote": "mypass"}) - c = TestClient(servers=servers, inputs=["lasote", "mypass", "danimtb", "passpass", - "lasote", "mypass"]) + servers = { + "default": TestServer(users={"lasote": "mypass", "danimtb": "passpass"}), + "other_server": TestServer(users={"lasote": "mypass"}), + } + c = TestClient(light=True, servers=servers, + inputs=["lasote", "mypass", "danimtb", "passpass", "lasote", "mypass"]) c.run("remote auth *") text = textwrap.dedent("""\ default: @@ -358,32 +355,34 @@ def test_remote_auth(self): assert result == {'default': {'user': 'lasote'}, 'other_server': {'user': 'lasote'}} def test_remote_auth_force(self): - servers = OrderedDict() - servers["default"] = TestServer(users={"lasote": "mypass", "danimtb": "passpass"}) - servers["other_server"] = TestServer(users={"lasote": "mypass"}) - c = TestClient(servers=servers, inputs=["lasote", "mypass", "danimtb", "passpass", - "lasote", "mypass"]) - - with patch("conan.internal.rest.rest_client_v2.RestV2Methods.check_credentials") as check_credentials_mock: + servers = { + "default": TestServer(users={"lasote": "mypass", "danimtb": "passpass"}), + "other_server": TestServer(users={"lasote": "mypass"}), + } + c = TestClient(light=True, servers=servers, + inputs=["lasote", "mypass", "danimtb", "passpass", "lasote", "mypass"]) + + with (patch("conan.internal.rest.rest_client_v2.RestV2Methods.check_credentials") + as check_credentials_mock): c.run("remote auth --force *") check_credentials_mock.assert_called_with(True) def test_remote_auth_force_false(self): - servers = OrderedDict() - servers["default"] = TestServer(users={"lasote": "mypass", "danimtb": "passpass"}) - servers["other_server"] = TestServer(users={"lasote": "mypass"}) - c = TestClient(servers=servers, inputs=["lasote", "mypass", "danimtb", "passpass", - "lasote", "mypass"]) - - with patch("conan.internal.rest.rest_client_v2.RestV2Methods.check_credentials") as check_credentials_mock: + servers = { + "default": TestServer(users={"lasote": "mypass", "danimtb": "passpass"}), + "other_server": TestServer(users={"lasote": "mypass"}), + } + c = TestClient(light=True, servers=servers, + inputs=["lasote", "mypass", "danimtb", "passpass", "lasote", "mypass"]) + + with (patch("conan.internal.rest.rest_client_v2.RestV2Methods.check_credentials") + as check_credentials_mock): c.run("remote auth *") check_credentials_mock.assert_called_with(False) def test_remote_auth_with_user(self): - servers = OrderedDict() - servers["default"] = TestServer(users={"lasote": "mypass"}) - servers["other_server"] = TestServer() - c = TestClient(servers=servers, inputs=["lasote", "mypass"]) + servers = {"default": TestServer(users={"lasote": "mypass"}), "other_server": TestServer()} + c = TestClient(light=True, servers=servers, inputs=["lasote", "mypass"]) c.run("remote set-user default lasote") c.run("remote auth * --with-user") text = textwrap.dedent("""\ @@ -394,10 +393,8 @@ def test_remote_auth_with_user(self): assert text in c.out def test_remote_auth_with_user_env_var(self): - servers = OrderedDict() - servers["default"] = TestServer(users={"lasote": "mypass"}) - servers["other_server"] = TestServer() - c = TestClient(servers=servers) + servers = {"default": TestServer(users={"lasote": "mypass"}), "other_server": TestServer()} + c = TestClient(light=True, servers=servers) with environment_update({"CONAN_LOGIN_USERNAME_DEFAULT": "lasote", "CONAN_PASSWORD_DEFAULT": "mypass"}): c.run("remote auth * --with-user") @@ -409,18 +406,19 @@ def test_remote_auth_with_user_env_var(self): assert text in c.out def test_remote_auth_error(self): - servers = OrderedDict() - servers["default"] = TestServer(users={"user": "password"}) - c = TestClient(servers=servers, inputs=["user1", "pass", "user2", "pass", "user3", "pass"]) + servers = {"default": TestServer(users={"user": "password"})} + c = TestClient(light=True, servers=servers, + inputs=["user1", "pass", "user2", "pass", "user3", "pass"]) c.run("remote auth *") assert "error: Too many failed login attempts, bye!" in c.out def test_remote_auth_server_expire_token_secret(self): server = TestServer(users={"myuser": "password", "myotheruser": "otherpass"}) - c = TestClient(servers={"default": server}, inputs=["myuser", "password", - "myotheruser", "otherpass", - "user", "pass", "user", "pass", - "user", "pass"]) + c = TestClient(light=True, servers={"default": server}, + inputs=["myuser", "password", + "myotheruser", "otherpass", + "user", "pass", "user", "pass", + "user", "pass"]) c.run("remote auth *") assert "Remote 'default' needs authentication, obtaining credentials" in c.out assert "user: myuser" in c.out @@ -439,10 +437,11 @@ def test_remote_auth_server_expire_token_secret(self): def test_remote_auth_server_expire_token(self): server = TestServer(users={"myuser": "password", "myotheruser": "otherpass"}) server.test_server.ra.api_v2.credentials_manager.expire_time = timedelta(seconds=2) - c = TestClient(servers={"default": server}, inputs=["myuser", "password", - "myotheruser", "otherpass", - "user", "pass", "user", "pass", - "user", "pass"]) + c = TestClient(light=True, servers={"default": server}, + inputs=["myuser", "password", + "myotheruser", "otherpass", + "user", "pass", "user", "pass", + "user", "pass"]) c.run("remote auth *") assert "user: myuser" in c.out # token not expired yet, should work @@ -459,7 +458,7 @@ def test_remote_auth_server_expire_token(self): def test_auth_after_logout(self): server = TestServer(users={"myuser": "password"}) - c = TestClient(servers={"default": server}, inputs=["myuser", "password"]*2) + c = TestClient(light=True, servers={"default": server}, inputs=["myuser", "password"]*2) c.run("remote auth *") assert "Remote 'default' needs authentication, obtaining credentials" in c.out assert "user: myuser" in c.out diff --git a/test/integration/command/source_test.py b/test/integration/command/source_test.py index f818ec3133a..8c65e139824 100644 --- a/test/integration/command/source_test.py +++ b/test/integration/command/source_test.py @@ -1,7 +1,6 @@ import os import re import textwrap -from collections import OrderedDict import pytest @@ -72,12 +71,8 @@ def source(self): def test_source_warning_os_build(self): # https://github.com/conan-io/conan/issues/2368 - conanfile = '''from conan import ConanFile -class ConanLib(ConanFile): - pass -''' client = TestClient(light=True) - client.save({CONANFILE: conanfile}) + client.save({CONANFILE: GenConanfile()}) client.run("source .") assert "This package defines both 'os' and 'os_build'" not in client.out @@ -87,8 +82,8 @@ def test_source_with_path_errors(self): # Path with conanfile.txt client.run("source conanfile.txt", assert_error=True) - assert "A conanfile.py is needed, %s is not acceptable" \ - % os.path.join(client.current_folder, "conanfile.txt") in client.out + assert ("A conanfile.py is needed, %s is not acceptable" + % os.path.join(client.current_folder, "conanfile.txt") in client.out) def test_source_local_cwd(self): conanfile = ''' @@ -175,7 +170,7 @@ def test_retrieve_exports_sources(self): # For Conan 2.0 if we install a package from a remote and we want to upload to other # remote we need to download the sources, as we consider revisions immutable, let's # iterate through the remotes to get the sources from the first match - servers = OrderedDict() + servers = {} for index in range(2): servers[f"server{index}"] = TestServer([("*/*@*/*", "*")], [("*/*@*/*", "*")], users={"user": "password"}) diff --git a/test/integration/command/test_outdated.py b/test/integration/command/test_outdated.py index 4c94e38bc7a..57b32cd8c60 100644 --- a/test/integration/command/test_outdated.py +++ b/test/integration/command/test_outdated.py @@ -1,5 +1,4 @@ import json -from collections import OrderedDict import pytest @@ -106,11 +105,7 @@ def test_two_remotes(): # remote1: zlib/1.0, libcurl/2.0, foo/1.0 # remote2: zlib/[1.0, 2.0], libcurl/1.0, foo/1.0 # local cache: zlib/1.0, libcurl/1.0, foo/1.0 - servers = OrderedDict() - for i in [1, 2]: - test_server = TestServer() - servers["remote%d" % i] = test_server - + servers = {f"remote{i}": TestServer() for i in (1, 2)} tc = TestClient(servers=servers, inputs=2 * ["admin", "password"], light=True) tc.save({"conanfile.py": GenConanfile()}) @@ -150,7 +145,7 @@ def test_two_remotes(): def test_duplicated_tool_requires(): - tc = TestClient(default_server_user=True) + tc = TestClient(default_server_user=True, light=True) # Create libraries needed to generate the dependency graph tc.save({"conanfile.py": GenConanfile()}) tc.run("create . --name=cmake --version=1.0") @@ -176,7 +171,7 @@ def test_duplicated_tool_requires(): def test_no_outdated_dependencies(): - tc = TestClient(default_server_user=True) + tc = TestClient(default_server_user=True, light=True) tc.save({"conanfile.py": GenConanfile()}) tc.run("create . --name=foo --version=1.0") diff --git a/test/integration/command/upload/upload_test.py b/test/integration/command/upload/upload_test.py index 30de88a034d..94fca4d45bf 100644 --- a/test/integration/command/upload/upload_test.py +++ b/test/integration/command/upload/upload_test.py @@ -3,7 +3,6 @@ import platform import stat import textwrap -from collections import OrderedDict import pytest from unittest.mock import patch @@ -374,10 +373,7 @@ def test_upload_key_error(self): files = {"conanfile.py": GenConanfile("hello0", "1.2.1")} server1 = TestServer([("*/*@*/*", "*")], [("*/*@*/*", "*")], users={"lasote": "mypass"}) server2 = TestServer([("*/*@*/*", "*")], [("*/*@*/*", "*")], users={"lasote": "mypass"}) - servers = OrderedDict() - servers["server1"] = server1 - servers["server2"] = server2 - client = TestClient(servers=servers) + client = TestClient(servers={"server1": server1, "server2": server2}, light=True) client.save(files) client.run("create . --user=user --channel=testing") client.run("remote login server1 lasote -p mypass") diff --git a/test/integration/configuration/profile_test.py b/test/integration/configuration/profile_test.py index ff59ed1ec63..09ac4951e0d 100644 --- a/test/integration/configuration/profile_test.py +++ b/test/integration/configuration/profile_test.py @@ -2,7 +2,6 @@ import os import platform import textwrap -from collections import OrderedDict from textwrap import dedent import pytest @@ -109,10 +108,12 @@ def test_install_with_missing_profile(self, path): @pytest.mark.skipif(platform.system() != "Windows", reason="Windows profiles") def test_install_profile_settings(self): # Create a profile and use it - profile_settings = OrderedDict([("compiler", "msvc"), - ("compiler.version", "191"), - ("compiler.runtime", "dynamic"), - ("arch", "x86")]) + profile_settings = { + "compiler": "msvc", + "compiler.version": "191", + "compiler.runtime": "dynamic", + "arch": "x86", + } create_profile(self.client.paths.profiles_path, "vs_12_86", settings=profile_settings, package_settings={}) @@ -134,10 +135,11 @@ def test_install_profile_settings(self): assert "compiler.version=191" in info # Use package settings in profile - tmp_settings = OrderedDict() - tmp_settings["compiler"] = "gcc" - tmp_settings["compiler.libcxx"] = "libstdc++11" - tmp_settings["compiler.version"] = "4.8" + tmp_settings = { + "compiler": "gcc", + "compiler.libcxx": "libstdc++11", + "compiler.version": "4.8", + } package_settings = {"hello0/*": tmp_settings} create_profile(self.client.paths.profiles_path, "vs_12_86_hello0_gcc", settings=profile_settings, @@ -182,17 +184,20 @@ def configure(self): self.client.save({"conanfile.py": conanfile}) # Create a profile and use it - profile_settings = OrderedDict([("os", "Windows"), - ("compiler", "msvc"), - ("compiler.version", "191"), - ("compiler.runtime", "dynamic"), - ("arch", "x86")]) + profile_settings = { + "os": "Windows", + "compiler": "msvc", + "compiler.version": "191", + "compiler.runtime": "dynamic", + "arch": "x86", + } # Use package settings in profile - tmp_settings = OrderedDict() - tmp_settings["compiler"] = "gcc" - tmp_settings["compiler.libcxx"] = "libstdc++11" - tmp_settings["compiler.version"] = "4.8" + tmp_settings = { + "compiler": "gcc", + "compiler.libcxx": "libstdc++11", + "compiler.version": "4.8", + } package_settings = {"*@lasote/*": tmp_settings} _create_profile(self.client.paths.profiles_path, "myprofile", settings=profile_settings, diff --git a/test/integration/py_requires/python_requires_test.py b/test/integration/extensions/python_requires_test.py similarity index 100% rename from test/integration/py_requires/python_requires_test.py rename to test/integration/extensions/python_requires_test.py diff --git a/test/integration/graph/core/test_version_ranges.py b/test/integration/graph/core/test_version_ranges.py index 4f9a19d65d6..66834ec18e1 100644 --- a/test/integration/graph/core/test_version_ranges.py +++ b/test/integration/graph/core/test_version_ranges.py @@ -1,4 +1,3 @@ -from collections import OrderedDict import pytest from conan.api.model import Remote @@ -278,10 +277,7 @@ def test_transitive_fixed_conflict(self): deps_graph = self.build_consumer(consumer, install=False) assert type(deps_graph.error) is GraphConflictError - assert 2 == len(deps_graph.nodes) - app = deps_graph.root - libb = app.edges[0].dst def test_transitive_fixed_conflict_forced(self): # app ---> libb/0.1 -----------> liba/1.2 @@ -391,18 +387,15 @@ def test_remote_version_ranges(): def test_different_user_channel_resolved_correctly(): - server1 = TestServer() - server2 = TestServer() - servers = OrderedDict([("server1", server1), ("server2", server2)]) - - client = TestClient(servers=servers, inputs=2*["admin", "password"], light=True) + client = TestClient(servers={"server1": TestServer(), "server2": TestServer()}, + inputs=2*["admin", "password"], light=True) client.save({"conanfile.py": GenConanfile()}) client.run("create . --name=lib --version=1.0 --user=conan --channel=stable") client.run("create . --name=lib --version=1.0 --user=conan --channel=testing") client.run("upload lib/1.0@conan/stable -r=server1") client.run("upload lib/1.0@conan/testing -r=server2") - client2 = TestClient(servers=servers, light=True) + client2 = TestClient(servers=client.servers, light=True) client2.run("install --requires=lib/[>=1.0]@conan/testing") assert f"lib/1.0@conan/testing: Retrieving package {NO_SETTINGS_PACKAGE_ID} " \ f"from remote 'server2' " in client2.out diff --git a/test/integration/graph/version_ranges/version_ranges_cached_test.py b/test/integration/graph/version_ranges/version_ranges_cached_test.py index 7aeaa764ae9..2748999de66 100644 --- a/test/integration/graph/version_ranges/version_ranges_cached_test.py +++ b/test/integration/graph/version_ranges/version_ranges_cached_test.py @@ -1,6 +1,3 @@ -from collections import OrderedDict - -import pytest from unittest.mock import patch from conan.internal.rest.remote_manager import RemoteManager @@ -11,27 +8,14 @@ class TestVersionRangesCache: - @pytest.fixture(autouse=True) - def _setup(self): - self.counters = {"server0": 0, "server1": 0} - - def _mocked_search_recipes(self, remote, pattern, ignorecase=True): - packages = { - "server0": [RecipeReference.loads("liba/1.0.0"), - RecipeReference.loads("liba/1.1.0")], - "server1": [RecipeReference.loads("liba/2.0.0"), - RecipeReference.loads("liba/2.1.0")] - } - self.counters[remote.name] = self.counters[remote.name] + 1 - return packages[remote.name] - def test_version_ranges_cached(self): - servers = OrderedDict() + servers = {} for index in range(2): servers[f"server{index}"] = TestServer([("*/*@*/*", "*")], [("*/*@*/*", "*")], users={"user": "password"}) - client = TestClient(light=True, servers=servers, inputs=["user", "password", "user", "password"]) + client = TestClient(light=True, servers=servers, + inputs=["user", "password", "user", "password"]) # server0 does not satisfy range # server1 does @@ -58,12 +42,22 @@ def test_version_ranges_cached(self): .with_requires("libb/1.0", "libc/1.0")}) # should call only once to server0 - self.counters["server0"] = 0 - self.counters["server1"] = 0 - with patch.object(RemoteManager, "search_recipes", new=self._mocked_search_recipes): + counters = {"server0": 0, "server1": 0} + + def _mocked_search_recipes(_, remote, pattern, ignorecase=True): # noqa + packages = { + "server0": [RecipeReference.loads("liba/1.0.0"), + RecipeReference.loads("liba/1.1.0")], + "server1": [RecipeReference.loads("liba/2.0.0"), + RecipeReference.loads("liba/2.1.0")] + } + counters[remote.name] = counters[remote.name] + 1 + return packages[remote.name] + + with patch.object(RemoteManager, "search_recipes", new=_mocked_search_recipes): client.run("create . --update") - assert self.counters["server0"] == 1 - assert self.counters["server1"] == 1 + assert counters["server0"] == 1 + assert counters["server1"] == 1 class TestVersionRangesDiamond: diff --git a/test/integration/graph/version_ranges/version_ranges_diamond_test.py b/test/integration/graph/version_ranges/version_ranges_diamond_test.py index 748ff899196..9c21dc6a4f0 100644 --- a/test/integration/graph/version_ranges/version_ranges_diamond_test.py +++ b/test/integration/graph/version_ranges/version_ranges_diamond_test.py @@ -1,5 +1,3 @@ -from collections import OrderedDict - from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient, TestServer @@ -70,10 +68,8 @@ def test_update(self): class TestVersionRangesMultiRemote: def test_multi_remote(self): - servers = OrderedDict() - servers["default"] = TestServer() - servers["other"] = TestServer() - client = TestClient(light=True, servers=servers, inputs=2*["admin", "password"]) + client = TestClient(light=True, servers={"default": TestServer(), "other": TestServer()}, + inputs=2 * ["admin", "password"]) client.save({"hello0/conanfile.py": GenConanfile("hello0"), "hello1/conanfile.py": GenConanfile("hello1").with_requires("hello0/[*]")}) client.run("export hello0 --version=0.1") diff --git a/test/integration/metadata/test_metadata_deploy.py b/test/integration/metadata/test_metadata_deploy.py index 1737c61aaaf..8fe545e477f 100644 --- a/test/integration/metadata/test_metadata_deploy.py +++ b/test/integration/metadata/test_metadata_deploy.py @@ -1,6 +1,5 @@ import os.path import textwrap -from collections import OrderedDict import pytest @@ -43,7 +42,7 @@ def deploy(graph, output_folder, **kwargs): d.ref.name)) """) - servers = OrderedDict([("default", TestServer()), ("remote2", TestServer())]) + servers = {"default": TestServer(), "remote2": TestServer()} c = TestClient(servers=servers, inputs=2 * ["admin", "password"], light=True) c.save({"conanfile.py": conanfile, "deploy.py": deploy}) diff --git a/test/integration/py_requires/__init__.py b/test/integration/py_requires/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/test/integration/remote/multi_remote_checks_test.py b/test/integration/remote/multi_remote_checks_test.py index c4c9d2783cc..6ae4e0d74ba 100644 --- a/test/integration/remote/multi_remote_checks_test.py +++ b/test/integration/remote/multi_remote_checks_test.py @@ -1,5 +1,3 @@ -from collections import OrderedDict - from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import NO_SETTINGS_PACKAGE_ID, TestClient, TestServer @@ -7,14 +5,9 @@ class TestRemoteChecks: def test_binary_defines_remote(self): - servers = OrderedDict([("server1", TestServer()), - ("server2", TestServer()), - ("server3", TestServer())]) + servers = {"server1": TestServer(), "server2": TestServer(), "server3": TestServer()} client = TestClient(servers=servers, inputs=3*["admin", "password"]) - conanfile = """from conan import ConanFile -class Pkg(ConanFile): - pass""" - client.save({"conanfile.py": conanfile}) + client.save({"conanfile.py": GenConanfile()}) client.run("create . --name=pkg --version=0.1 --user=lasote --channel=testing") client.run("upload pkg* -r=server1 --confirm") client.run("upload pkg* -r=server2 --confirm") @@ -28,7 +21,7 @@ class Pkg(ConanFile): client.assert_listed_binary( {"pkg/0.1@lasote/testing": (NO_SETTINGS_PACKAGE_ID, "Download (server1)")}) assert "pkg/0.1@lasote/testing: Retrieving package " \ - "%s from remote 'server1'" % NO_SETTINGS_PACKAGE_ID in client.out + "%s from remote 'server1'" % NO_SETTINGS_PACKAGE_ID in client.out # Explicit remote also defines the remote client.run("remove * -c") @@ -37,7 +30,7 @@ class Pkg(ConanFile): client.assert_listed_binary( {"pkg/0.1@lasote/testing": (NO_SETTINGS_PACKAGE_ID, "Download (server2)")}) assert "pkg/0.1@lasote/testing: Retrieving package " \ - "%s from remote 'server2'" % NO_SETTINGS_PACKAGE_ID in client.out + "%s from remote 'server2'" % NO_SETTINGS_PACKAGE_ID in client.out # Ordered search of binary works client.run("remove * -c") @@ -47,7 +40,7 @@ class Pkg(ConanFile): client.assert_listed_binary( {"pkg/0.1@lasote/testing": (NO_SETTINGS_PACKAGE_ID, "Download (server2)")}) assert "pkg/0.1@lasote/testing: Retrieving package " \ - "%s from remote 'server2'" % NO_SETTINGS_PACKAGE_ID in client.out + "%s from remote 'server2'" % NO_SETTINGS_PACKAGE_ID in client.out # Download recipe and binary from the remote2 by iterating client.run("remove * -c") @@ -56,42 +49,35 @@ class Pkg(ConanFile): client.assert_listed_binary( {"pkg/0.1@lasote/testing": (NO_SETTINGS_PACKAGE_ID, "Download (server2)")}) assert "pkg/0.1@lasote/testing: Retrieving package " \ - "%s from remote 'server2'" % NO_SETTINGS_PACKAGE_ID in client.out + "%s from remote 'server2'" % NO_SETTINGS_PACKAGE_ID in client.out def test_binaries_from_different_remotes(self): - servers = OrderedDict() - servers["server1"] = TestServer() - servers["server2"] = TestServer() + servers = {"server1": TestServer(), "server2": TestServer()} client = TestClient(servers=servers, inputs=2*["admin", "password"]) - conanfile = """from conan import ConanFile -class Pkg(ConanFile): - options = {"opt": [1, 2, 3]} -""" - client.save({"conanfile.py": conanfile}) - client.run("create . --name=pkg --version=0.1 --user=lasote --channel=testing -o pkg/*:opt=1") + client.save({"conanfile.py": GenConanfile().with_option("opt", [1, 2, 3])}) + client.run("create . --name=pkg --version=0.1 -o pkg/*:opt=1") client.run("upload pkg* -r=server1 --confirm") client.run("remove *:* -c") - client.run("create . --name=pkg --version=0.1 --user=lasote --channel=testing -o pkg/*:opt=2") - package_id2 = client.created_package_id("pkg/0.1@lasote/testing") + client.run("create . --name=pkg --version=0.1 -o pkg/*:opt=2") + package_id2 = client.created_package_id("pkg/0.1") client.run("upload pkg* -r=server2 --confirm") client.run("remove *:* -c") # recipe is cached, takes binary from server2 - client.run("install --requires=pkg/0.1@lasote/testing -o pkg/*:opt=2 -r=server2") - client.assert_listed_binary({"pkg/0.1@lasote/testing": (package_id2, "Download (server2)")}) - assert f"pkg/0.1@lasote/testing: Retrieving package {package_id2} " \ - "from remote 'server2'" in client.out + client.run("install --requires=pkg/0.1 -o pkg/*:opt=2 -r=server2") + client.assert_listed_binary({"pkg/0.1": (package_id2, "Download (server2)")}) + assert f"pkg/0.1: Retrieving package {package_id2} from remote 'server2'" in client.out # Nothing to update - client.run("install --requires=pkg/0.1@lasote/testing -o pkg/*:opt=2 -r=server2 -u") - client.assert_listed_binary({"pkg/0.1@lasote/testing": (package_id2, "Cache")}) + client.run("install --requires=pkg/0.1 -o pkg/*:opt=2 -r=server2 -u") + client.assert_listed_binary({"pkg/0.1": (package_id2, "Cache")}) # Build missing - client.run("install --requires=pkg/0.1@lasote/testing -o pkg/*:opt=3 -r=server2", assert_error=True) - assert "ERROR: Missing prebuilt package for 'pkg/0.1@lasote/testing'" in client.out + client.run("install --requires=pkg/0.1 -o pkg/*:opt=3 -r=server2", assert_error=True) + assert "ERROR: Missing prebuilt package for 'pkg/0.1'" in client.out - client.run("install --requires=pkg/0.1@lasote/testing -o pkg/*:opt=3", assert_error=True) - assert "ERROR: Missing prebuilt package for 'pkg/0.1@lasote/testing'" in client.out + client.run("install --requires=pkg/0.1 -o pkg/*:opt=3", assert_error=True) + assert "ERROR: Missing prebuilt package for 'pkg/0.1'" in client.out def test_version_range_multi_remote(): @@ -100,9 +86,7 @@ def test_version_range_multi_remote(): valid is found in the first server, it is return - Using --update forces to really update to the latest version available, anywhere """ - servers = OrderedDict([("server1", TestServer()), - ("server2", TestServer()), - ("server3", TestServer())]) + servers = {"server1": TestServer(), "server2": TestServer(), "server3": TestServer()} client = TestClient(servers=servers, inputs=3*["admin", "password"]) client.save({"conanfile.py": GenConanfile()}) for i in (1, 2, 3): diff --git a/test/integration/remote/multi_remote_test.py b/test/integration/remote/multi_remote_test.py index 012266f5c95..d426ed2161d 100644 --- a/test/integration/remote/multi_remote_test.py +++ b/test/integration/remote/multi_remote_test.py @@ -1,5 +1,3 @@ -import pytest -from collections import OrderedDict from time import sleep from conan.api.model import RecipeReference @@ -11,47 +9,38 @@ class TestExportsSourcesMissing: def test_exports_sources_missing(self): - client = TestClient(default_server_user=True) + client = TestClient(light=True, default_server_user=True) client.save({"conanfile.py": GenConanfile().with_exports_sources("*"), "source.txt": "somesource"}) client.run("create . --name=pkg --version=0.1 --user=user --channel=testing") client.run("upload pkg/0.1@user/testing -r default") # Failure because remote is removed - servers = OrderedDict(client.servers) + servers = dict(client.servers) servers["new_server"] = TestServer(users={"user": "password"}) - client2 = TestClient(servers=servers, inputs=["user", "password"]) + client2 = TestClient(light=True, servers=servers, inputs=["user", "password"]) client2.run("install --requires=pkg/0.1@user/testing") client2.run("remote remove default") client2.run("upload pkg/0.1@user/testing -r=new_server", assert_error=True) assert "The 'pkg/0.1@user/testing' package has 'exports_sources' but sources " \ - "not found in local cache." in client2.out - assert "Probably it was installed from a remote that is no longer available." \ - in client2.out + "not found in local cache." in client2.out + assert "Probably it was installed from a remote that is no longer available." in client2.out # Failure because remote removed the package - client2 = TestClient(servers=servers, inputs=2*["admin", "password"]) + client2 = TestClient(light=True, servers=servers, inputs=2 * ["admin", "password"]) client2.run("install --requires=pkg/0.1@user/testing") client2.run("remove * -r=default -c") client2.run("upload pkg/0.1@user/testing -r=new_server", assert_error=True) assert "pkg/0.1@user/testing Error while compressing: The 'pkg/0.1@user/testing' " \ - in client2.out - assert "The 'pkg/0.1@user/testing' package has 'exports_sources' but sources " \ - in client2.out - assert "Probably it was installed from a remote that is no longer available." \ - in client2.out + in client2.out + assert "The 'pkg/0.1@user/testing' package has 'exports_sources' but sources " in client2.out + assert "Probably it was installed from a remote that is no longer available." in client2.out class TestMultiRemotes: - @pytest.fixture(autouse=True) - def setup(self): - self.servers = OrderedDict() - self.servers["default"] = TestServer() - self.servers["local"] = TestServer() - @staticmethod - def _create(client, number, version, modifier=""): + def _export(client, number, version, modifier=""): files = {CONANFILE: str(GenConanfile(number, version)) + modifier} client.save(files, clean_first=True) client.run("export . --user=lasote --channel=stable") @@ -60,11 +49,12 @@ def test_conan_install_build_flag(self): """ Checks conan install --update works with different remotes """ - client_a = TestClient(servers=self.servers, inputs=2*["admin", "password"]) - client_b = TestClient(servers=self.servers, inputs=2*["admin", "password"]) + servers = {"default": TestServer(), "local": TestServer()} + client_a = TestClient(light=True, servers=servers, inputs=2*["admin", "password"]) + client_b = TestClient(light=True, servers=servers, inputs=2*["admin", "password"]) # Upload hello0 to local and default from client_a - self._create(client_a, "hello0", "0.0") + self._export(client_a, "hello0", "0.0") client_a.run("upload hello0/0.0@lasote/stable -r local --only-recipe") client_a.run("upload hello0/0.0@lasote/stable -r default --only-recipe") sleep(1) # For timestamp and updates checks @@ -73,7 +63,7 @@ def test_conan_install_build_flag(self): client_b.run("install --requires=hello0/0.0@lasote/stable -r local --build missing") # Update hello0 with client_a and reupload - self._create(client_a, "hello0", "0.0", modifier="\n") + self._export(client_a, "hello0", "0.0", modifier="\n") client_a.run("upload hello0/0.0@lasote/stable -r local --only-recipe") assert "Uploading recipe 'hello0/0.0@lasote/stable" in client_a.out @@ -84,13 +74,12 @@ def test_conan_install_build_flag(self): # Now try to update the package with install -u client_b.run("install --requires=hello0/0.0@lasote/stable -u --build='*'") - assert "hello0/0.0@lasote/stable#64fd8ae21db9eff69c6c681b0e2fc178 - Updated" \ - in client_b.out + assert "hello0/0.0@lasote/stable#64fd8ae21db9eff69c6c681b0e2fc178 - Updated" in client_b.out # Upload a new version from client A, but only to the default server (not the ref-listed) # Upload hello0 to local and default from client_a sleep(1) # For timestamp and updates checks - self._create(client_a, "hello0", "0.0", modifier="\n\n") + self._export(client_a, "hello0", "0.0", modifier="\n\n") client_a.run("upload hello0/0.0@lasote/stable#latest -r default --only-recipe") # Now client_b checks for updates without -r parameter @@ -107,8 +96,7 @@ def test_conan_install_build_flag(self): # Well, now try to update the package with -r default -u client_b.run("install --requires=hello0/0.0@lasote/stable -r default -u --build='*'") - assert "hello0/0.0@lasote/stable: Forced build from source" \ - in str(client_b.out) + assert "hello0/0.0@lasote/stable: Forced build from source" in str(client_b.out) # TODO: cache2.0 conan info not yet implemented with new cache client_b.run("graph info --requires=hello0/0.0@lasote/stable -u") assert "recipe: Cache" in client_b.out @@ -118,15 +106,16 @@ def test_conan_install_update(self): """ Checks conan install --update works only with the remote associated """ - client = TestClient(servers=self.servers, inputs=2*["admin", "password"]) + servers = {"default": TestServer(), "local": TestServer()} + client = TestClient(light=True, servers=servers, inputs=2*["admin", "password"]) - self._create(client, "hello0", "0.0") + self._export(client, "hello0", "0.0") default_remote_rev = client.exported_recipe_revision() client.run("install --requires=hello0/0.0@lasote/stable --build missing") client.run("upload hello0/0.0@lasote/stable -r default") sleep(1) # For timestamp and updates checks - self._create(client, "hello0", "0.0", modifier=" ") + self._export(client, "hello0", "0.0", modifier=" ") local_remote_rev = client.exported_recipe_revision() client.run("install --requires=hello0/0.0@lasote/stable --build missing") @@ -140,13 +129,12 @@ def test_conan_install_update(self): assert f"hello0/0.0@lasote/stable#{local_remote_rev} - Updated" in client.out client.run("install --requires=hello0/0.0@lasote/stable --update -r default") - assert f"hello0/0.0@lasote/stable#{local_remote_rev} - Newer" \ - in client.out + assert f"hello0/0.0@lasote/stable#{local_remote_rev} - Newer" in client.out sleep(1) # For timestamp and updates checks # Check that it really updates in case of newer package uploaded to the associated remote - client_b = TestClient(servers=self.servers, inputs=3*["admin", "password"]) - self._create(client_b, "hello0", "0.0", modifier=" ") + client_b = TestClient(light=True, servers=servers, inputs=3*["admin", "password"]) + self._export(client_b, "hello0", "0.0", modifier=" ") new_local_remote_rev = client_b.exported_recipe_revision() client_b.run("install --requires=hello0/0.0@lasote/stable --build missing") client_b.run("upload hello0/0.0@lasote/stable -r local") @@ -156,52 +144,39 @@ def test_conan_install_update(self): class TestMultiRemote: - @pytest.fixture(autouse=True) - def setup(self): - self.servers = OrderedDict() - self.users = {} - for i in range(3): - test_server = TestServer() - self.servers["remote%d" % i] = test_server - self.users["remote%d" % i] = [("admin", "password")] - - self.client = TestClient(servers=self.servers, inputs=3*["admin", "password"]) - def test_fail_when_not_notfound(self): """ If a remote fails with a 404 it has to keep looking in the next remote, but if it fails by any other reason it has to stop """ - servers = OrderedDict() - servers["s0"] = TestServer() - servers["s1"] = TestServer() - servers["s2"] = TestServer() + servers = {"s0": TestServer(), "s1": TestServer(), "s2": TestServer()} - client = TestClient(servers=servers) + client = TestClient(light=True, servers=servers) client.save({"conanfile.py": GenConanfile("mylib", "0.1")}) client.run("create . --user=lasote --channel=testing") client.run("remote login s1 admin -p password") client.run("upload mylib* -r s1 -c") servers["s1"].fake_url = "http://asdlhaljksdhlajkshdljakhsd.com" # Do not exist - client2 = TestClient(servers=servers) + client2 = TestClient(light=True, servers=servers) client2.run("install --requires=mylib/0.1@conan/testing --build=missing", assert_error=True) assert "mylib/0.1@conan/testing: Checking remote: s0" in client2.out assert "mylib/0.1@conan/testing: Checking remote: s1" in client2.out - assert "Unable to connect to remote s1=http://asdlhaljksdhlajkshdljakhsd.com" \ - in client2.out + assert "Unable to connect to remote s1=http://asdlhaljksdhlajkshdljakhsd.com" in client2.out # s2 is not even tried assert "mylib/0.1@conan/testing: Trying with 's2'..." not in client2.out def test_install_from_remotes(self): + servers = {"remote%d" % i: TestServer() for i in range(3)} + client = TestClient(light=True, servers=servers, inputs=3 * ["admin", "password"]) for i in range(3): ref = RecipeReference.loads("hello%d/0.1@lasote/stable" % i) - self.client.save({"conanfile.py": GenConanfile("hello%d" % i, "0.1")}) - self.client.run("export . --user=lasote --channel=stable") - self.client.run("upload %s -r=remote%d" % (str(ref), i)) + client.save({"conanfile.py": GenConanfile("hello%d" % i, "0.1")}) + client.run("export . --user=lasote --channel=stable") + client.run("upload %s -r=remote%d" % (str(ref), i)) # Now install it in other machine from remote 0 - client2 = TestClient(servers=self.servers) + client2 = TestClient(light=True, servers=servers) refs = ["hello0/0.1@lasote/stable", "hello1/0.1@lasote/stable", "hello2/0.1@lasote/stable"] client2.save({"conanfile.py": GenConanfile("helloX", "0.1").with_requires(*refs)}) diff --git a/test/integration/remote/selected_remotes_test.py b/test/integration/remote/selected_remotes_test.py index 8191d0c44b5..7b7d6a84acf 100644 --- a/test/integration/remote/selected_remotes_test.py +++ b/test/integration/remote/selected_remotes_test.py @@ -1,5 +1,3 @@ -from collections import OrderedDict - import pytest from conan.test.assets.genconanfile import GenConanfile @@ -9,11 +7,11 @@ class TestSelectedRemotesInstall: @pytest.fixture(autouse=True) def _setup(self): - servers = OrderedDict() + servers = {} for index in range(3): servers[f"server{index}"] = TestServer([("*/*@*/*", "*")], [("*/*@*/*", "*")], users={"user": "password"}) - self.client = TestClient(servers=servers, inputs=3 * ["user", "password"]) + self.client = TestClient(light=True, servers=servers, inputs=3 * ["user", "password"]) def test_selected_remotes(self): self.client.save({"conanfile.py": GenConanfile("liba", "1.0").with_build_msg("OLDREV")}) diff --git a/test/integration/sbom/test_cyclonedx.py b/test/integration/sbom/test_cyclonedx.py index 220a3cbc8aa..4964d57fe0b 100644 --- a/test/integration/sbom/test_cyclonedx.py +++ b/test/integration/sbom/test_cyclonedx.py @@ -353,3 +353,81 @@ def test_sbom_deployer(self, cyclone_version, install): }.get(cyclone_version) tc.run(f"install {install} --deployer=cyclone_{method}") assert os.path.exists(os.path.join(tc.current_folder, f"sbom-cyclonedx-{method}.json")) + + +class TestCyclonedx2: + # Using the sbom tool with "conan create" + sbom_hook_post_package = textwrap.dedent(""" + import json + import os + from conan.errors import ConanException + from conan.api.output import ConanOutput + from conan.tools.sbom import cyclonedx_1_4, cyclonedx_1_6 + + def post_package(conanfile): + sbom_cyclonedx_1_4 = cyclonedx_1_4(conanfile, add_build=True, add_tests=True) + sbom_cyclonedx_1_6 = cyclonedx_1_6(conanfile, add_build=True, add_tests=True) + with open(os.path.join(conanfile.package_metadata_folder, "sbom14.cdx.json"), 'w') as f: + json.dump(sbom_cyclonedx_1_4, f, indent=4) + with open(os.path.join(conanfile.package_metadata_folder, "sbom16.cdx.json"), 'w') as f: + json.dump(sbom_cyclonedx_1_6, f, indent=4) + """) + + @pytest.fixture() + def sbom_hook_client(self): + tc = TestClient(light=True) + hook_path = os.path.join(tc.paths.hooks_path, "hook_sbom.py") + save(hook_path, self.sbom_hook_post_package) + return tc + + def test_sbom_generation_create(self, sbom_hook_client): + tc = sbom_hook_client + # bar -> engine/1.0 -> matrix/1.0 (same graph as cmake_lib + transitive_libraries, no CMake) + tc.save({ + "matrix/conanfile.py": GenConanfile("matrix", "1.0"), + "engine/conanfile.py": GenConanfile("engine", "1.0").with_requires("matrix/1.0"), + "conanfile.py": GenConanfile("bar", "1.0").with_requires("engine/1.0"), + }) + tc.run("create matrix") + tc.run("create engine") + tc.run("create . -tf=") + bar_layout = tc.created_layout() + assert os.path.exists(os.path.join(bar_layout.metadata(), "sbom14.cdx.json")) + assert os.path.exists(os.path.join(bar_layout.metadata(), "sbom16.cdx.json")) + + @pytest.mark.parametrize("user, channel, user_dep, channel_dep", + [("user", None, "user_dep", None), + ("user", "channel", "user_dep", "channel_dep")]) + def test_sbom_user_path(self, user, channel, user_dep, channel_dep): + tc = TestClient(light=True) + hook_path = os.path.join(tc.paths.hooks_path, "hook_sbom.py") + save(hook_path, self.sbom_hook_post_package) + channel_ref = f"/{channel_dep}" if channel_dep else "" + tc.save({"dep/conanfile.py": GenConanfile("dep", "1.0"), + "conanfile.py": GenConanfile("main", "1.0").with_requires( + f"dep/1.0@{user_dep}{channel_ref}")}) + command = "create dep" + if user: + command += f" --user={user_dep}" + if channel: + command += f" --channel={channel_dep}" + + tc.run(command) + + command = "create ." + if user: + command += f" --user={user}" + if channel: + command += f" --channel={channel}" + tc.run(command) + + for version in ("14", "16"): + create_layout = tc.created_layout() + cyclone_path = os.path.join(create_layout.metadata(), f"sbom{version}.cdx.json") + content = tc.load(cyclone_path) + content_json = json.loads(content) + + assert content_json["components"][0]["bom-ref"].split("&user=")[ + 1] == f"{user}&channel={channel}" if channel else user + assert content_json["dependencies"][0]["dependsOn"][0].split("&user=")[ + 1] == f"{user_dep}&channel={channel_dep}" if channel_dep else user_dep diff --git a/test/unittests/model/profile_test.py b/test/unittests/model/profile_test.py index 9ba034cfb29..994483a05ff 100644 --- a/test/unittests/model/profile_test.py +++ b/test/unittests/model/profile_test.py @@ -1,4 +1,4 @@ -from collections import OrderedDict +import textwrap from conan.tools.env.environment import ProfileEnvironment from conan.internal.model.profile import Profile @@ -8,38 +8,48 @@ class TestProfile: def test_profile_settings_update(self): new_profile = Profile() - new_profile.update_settings(OrderedDict([("os", "Windows")])) + new_profile.update_settings({"os": "Windows"}) - new_profile.update_settings(OrderedDict([("OTHER", "2")])) - assert new_profile.settings == OrderedDict([("os", "Windows"), ("OTHER", "2")]) + new_profile.update_settings({"OTHER": "2"}) + assert new_profile.settings == {"os": "Windows", "OTHER": "2"} - new_profile.update_settings(OrderedDict([("compiler", "2"), ("compiler.version", "3")])) - assert new_profile.settings == OrderedDict([("os", "Windows"), ("OTHER", "2"), - ("compiler", "2"), ("compiler.version", "3")]) + new_profile.update_settings({"compiler": "2", "compiler.version": "3"}) + assert new_profile.settings == { + "os": "Windows", + "OTHER": "2", + "compiler": "2", + "compiler.version": "3", + } def test_profile_subsettings_update(self): new_profile = Profile() - new_profile.update_settings(OrderedDict([("os", "Windows"), - ("compiler", "Visual Studio"), - ("compiler.runtime", "MT")])) + new_profile.update_settings({ + "os": "Windows", + "compiler": "Visual Studio", + "compiler.runtime": "MT", + }) - new_profile.update_settings(OrderedDict([("compiler", "gcc")])) + new_profile.update_settings({"compiler": "gcc"}) assert dict(new_profile.settings) == {"compiler": "gcc", "os": "Windows"} new_profile = Profile() - new_profile.update_settings(OrderedDict([("os", "Windows"), - ("compiler", "Visual Studio"), - ("compiler.runtime", "MT")])) - - new_profile.update_settings(OrderedDict([("compiler", "Visual Studio"), - ("compiler.subsetting", "3"), - ("other", "value")])) + new_profile.update_settings({ + "os": "Windows", + "compiler": "Visual Studio", + "compiler.runtime": "MT", + }) + + new_profile.update_settings({ + "compiler": "Visual Studio", + "compiler.subsetting": "3", + "other": "value", + }) assert dict(new_profile.settings) == {"compiler": "Visual Studio", - "os": "Windows", - "compiler.runtime": "MT", - "compiler.subsetting": "3", - "other": "value"} + "os": "Windows", + "compiler.runtime": "MT", + "compiler.subsetting": "3", + "other": "value"} def test_package_settings_update(self): np = Profile() @@ -51,7 +61,8 @@ def test_package_settings_update(self): np._package_settings_values = None # invalidate caching np.update_package_settings({"MyPackage": [("compiler", "2"), ("compiler.version", "3")]}) assert np.package_settings_values == {"MyPackage": [("os", "Windows"), ("OTHER", "2"), - ("compiler", "2"), ("compiler.version", "3")]} + ("compiler", "2"), + ("compiler.version", "3")]} def test_profile_dump_order(self): # Settings @@ -61,17 +72,17 @@ def test_profile_dump_order(self): profile.settings["compiler"] = "Visual Studio" profile.settings["compiler.version"] = "12" profile.tool_requires["*"] = ["zlib/1.2.8@lasote/testing"] - profile.tool_requires["zlib/*"] = ["aaaa/1.2.3@lasote/testing", - "bb/1.2@lasote/testing"] - assert """[settings] -arch=x86_64 -compiler=Visual Studio -compiler.version=12 -zlib:compiler=gcc -[tool_requires] -*: zlib/1.2.8@lasote/testing -zlib/*: aaaa/1.2.3@lasote/testing, bb/1.2@lasote/testing -""".splitlines() == profile.dumps().splitlines() + profile.tool_requires["zlib/*"] = ["aaaa/1.2.3@lasote/testing", "bb/1.2@lasote/testing"] + assert textwrap.dedent("""\ + [settings] + arch=x86_64 + compiler=Visual Studio + compiler.version=12 + zlib:compiler=gcc + [tool_requires] + *: zlib/1.2.8@lasote/testing + zlib/*: aaaa/1.2.3@lasote/testing, bb/1.2@lasote/testing + """) == profile.dumps() def test_apply(self): # Settings @@ -80,9 +91,10 @@ def test_apply(self): profile.settings["compiler"] = "Visual Studio" profile.settings["compiler.version"] = "12" - profile.update_settings(OrderedDict([("compiler.version", "14")])) + profile.update_settings({"compiler.version": "14"}) - assert '[settings]\narch=x86_64\ncompiler=Visual Studio\ncompiler.version=14\n' == profile.dumps() + assert ('[settings]\narch=x86_64\n' + 'compiler=Visual Studio\ncompiler.version=14\n') == profile.dumps() def test_update_build_requires(): From f7853ebf3d6f5ca6ca0b6ff62c78e96e22b9edea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= <5364255+AbrilRBS@users.noreply.github.com> Date: Sat, 28 Mar 2026 00:16:29 +0100 Subject: [PATCH 073/110] Fix syntax error in Python <3.9 (#19805) --- test/integration/command/remote/test_remote_users.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/integration/command/remote/test_remote_users.py b/test/integration/command/remote/test_remote_users.py index 3c67a679151..0503097a4a1 100644 --- a/test/integration/command/remote/test_remote_users.py +++ b/test/integration/command/remote/test_remote_users.py @@ -362,8 +362,8 @@ def test_remote_auth_force(self): c = TestClient(light=True, servers=servers, inputs=["lasote", "mypass", "danimtb", "passpass", "lasote", "mypass"]) - with (patch("conan.internal.rest.rest_client_v2.RestV2Methods.check_credentials") - as check_credentials_mock): + with patch("conan.internal.rest.rest_client_v2.RestV2Methods.check_credentials") \ + as check_credentials_mock: c.run("remote auth --force *") check_credentials_mock.assert_called_with(True) @@ -375,8 +375,8 @@ def test_remote_auth_force_false(self): c = TestClient(light=True, servers=servers, inputs=["lasote", "mypass", "danimtb", "passpass", "lasote", "mypass"]) - with (patch("conan.internal.rest.rest_client_v2.RestV2Methods.check_credentials") - as check_credentials_mock): + with patch("conan.internal.rest.rest_client_v2.RestV2Methods.check_credentials") \ + as check_credentials_mock: c.run("remote auth *") check_credentials_mock.assert_called_with(False) From cb806e4858d69eba8716494c8c74789f46963818 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 30 Mar 2026 15:24:49 +0200 Subject: [PATCH 074/110] avoid casing errors while looking for version ranges (#19799) --- conan/internal/cache/cache.py | 4 ++-- test/integration/conanfile/conanfile_errors_test.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/conan/internal/cache/cache.py b/conan/internal/cache/cache.py index c61e6146837..0281702179e 100644 --- a/conan/internal/cache/cache.py +++ b/conan/internal/cache/cache.py @@ -201,13 +201,13 @@ def update_recipe_timestamp(self, ref: RecipeReference): assert ref.timestamp self._db.update_recipe_timestamp(ref) - def search_recipes(self, pattern=None, ignorecase=True): + def search_recipes(self, pattern=None): # Conan references in main storage if pattern: if isinstance(pattern, RecipeReference): pattern = repr(pattern) pattern = translate(pattern) - pattern = re.compile(pattern, re.IGNORECASE if ignorecase else 0) + pattern = re.compile(pattern) return self._db.list_references(pattern) diff --git a/test/integration/conanfile/conanfile_errors_test.py b/test/integration/conanfile/conanfile_errors_test.py index 3fc05e9250c..4ca09a6d52a 100644 --- a/test/integration/conanfile/conanfile_errors_test.py +++ b/test/integration/conanfile/conanfile_errors_test.py @@ -137,6 +137,16 @@ class HelloConan(ConanFile): client.run("export .", assert_error=True) assert "Duplicated requirement" in client.out + @pytest.mark.parametrize("version", ["0.1", "[*]"]) + def test_error_requirement_casing(self, version): + # https://github.com/conan-io/conan/issues/19796 + c = TestClient(light=True) + c.save({"mypkg/conanfile.py": GenConanfile("mypkg", "0.1"), + "app/conanfile.py": GenConanfile("app", "0.1").with_requires(f"myPkg/{version}")}) + c.run("create mypkg") + c.run("install app", assert_error=True) + assert f"ERROR: Package 'myPkg/{version}' not resolved" in c.out + class TestWrongMethods: # https://github.com/conan-io/conan/issues/12961 From 27cb3fe8298e79a3d22e5728090a64d07500e57d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= <5364255+AbrilRBS@users.noreply.github.com> Date: Mon, 6 Apr 2026 15:18:00 +0200 Subject: [PATCH 075/110] Undefined locale for list html (#19828) --- conan/cli/formatters/list/search_table_html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conan/cli/formatters/list/search_table_html.py b/conan/cli/formatters/list/search_table_html.py index 0fa834621a2..50ac39cc48c 100644 --- a/conan/cli/formatters/list/search_table_html.py +++ b/conan/cli/formatters/list/search_table_html.py @@ -50,7 +50,7 @@ second: "2-digit", hour12: false }; - return new Date(timeStamp * 1000).toLocaleDateString('en', options); + return new Date(timeStamp * 1000).toLocaleDateString(undefined, options); } function getInfoFieldsBadges(info) { From dc2cefb2255d25177e3d4b1e8af2916bdf79f924 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Mon, 6 Apr 2026 18:44:29 +0200 Subject: [PATCH 076/110] finalize() output folder should be printed only once (#19834) --- conan/internal/graph/installer.py | 2 +- .../conanfile/test_finalize_method.py | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/conan/internal/graph/installer.py b/conan/internal/graph/installer.py index 384519d4449..b5a75d38280 100644 --- a/conan/internal/graph/installer.py +++ b/conan/internal/graph/installer.py @@ -487,4 +487,4 @@ def _call_finalize_method(conanfile, finalize_folder): 'finalize'): conanfile.finalize() - conanfile.output.success(f"Finalized folder {finalize_folder}") + conanfile.output.success(f"Finalized folder {finalize_folder}") diff --git a/test/integration/conanfile/test_finalize_method.py b/test/integration/conanfile/test_finalize_method.py index 45162fd7bb3..568c9d03af7 100644 --- a/test/integration/conanfile/test_finalize_method.py +++ b/test/integration/conanfile/test_finalize_method.py @@ -319,6 +319,27 @@ def test_test_package_uses_created_tool_which_modifies_pkgfolder(self): assert "There are corrupted artifacts" not in tc.out + def test_multiple_instances_of_finalized_package(self): + tc = TestClient(light=True) + tc.save({"tool/conanfile.py": GenConanfile("tool", "1.0") + .with_package_type("application") + .with_finalize("self.output.info('RUNNING MY FINALIZE')"), + "liba/conanfile.py": GenConanfile("liba", "1.0").with_tool_requires("tool/1.0"), + "libb/conanfile.py": GenConanfile("libb", "1.0").with_tool_requires("tool/1.0"), + "consumer/conanfile.py": GenConanfile("consummer", "1.0") + .with_requires("liba/1.0") + .with_requires("libb/1.0")}) + + tc.run("export tool") + tc.run("export liba") + tc.run("export libb") + tc.run("install consumer -c:a tools.graph:skip_binaries=False -b missing") + + # The finalize() method of the tool should only be called once, even if multiple packages depend on it + assert tc.out.count("RUNNING MY FINALIZE") == 1 + # Finalize folder conan output should be printed only once + assert tc.out.count("Finalized folder ") == 1 + class TestRemoteFlows: @pytest.fixture From ecb8c39d5e0ed6b3b39d76f0140e80c5c072cd71 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 7 Apr 2026 10:41:17 +0200 Subject: [PATCH 077/110] Refactor/ordered dict (#19839) * wip * more rid * wip --- conan/cli/formatters/graph/graph_info_text.py | 3 +-- conan/internal/graph/compatibility.py | 2 +- conan/internal/model/conf.py | 23 ++++--------------- conan/internal/model/lockfile.py | 13 +++++------ conan/tools/apple/xcodedeps.py | 7 +++--- conan/tools/cmake/toolchain/blocks.py | 11 ++++----- .../lockfile/test_lock_packages.py | 7 ++++++ 7 files changed, 27 insertions(+), 39 deletions(-) diff --git a/conan/cli/formatters/graph/graph_info_text.py b/conan/cli/formatters/graph/graph_info_text.py index ac04a056e60..319fe1be509 100644 --- a/conan/cli/formatters/graph/graph_info_text.py +++ b/conan/cli/formatters/graph/graph_info_text.py @@ -1,5 +1,4 @@ import fnmatch -from collections import OrderedDict from conan.api.model import RecipeReference from conan.api.output import ConanOutput, cli_out_write @@ -29,7 +28,7 @@ def _matching(node, pattern): field_filter.append("ref") result = {} for id_, n in graph["nodes"].items(): - new_node = OrderedDict((k, v) for k, v in n.items() if k in field_filter) + new_node = {k: v for k, v in n.items() if k in field_filter} result[id_] = new_node graph["nodes"] = result return graph diff --git a/conan/internal/graph/compatibility.py b/conan/internal/graph/compatibility.py index 2de231caa46..0300f434ffa 100644 --- a/conan/internal/graph/compatibility.py +++ b/conan/internal/graph/compatibility.py @@ -135,7 +135,7 @@ def compatibles(self, conanfile): if not compat_infos: return {} - result = OrderedDict() + result = {} original_info = conanfile.info original_settings = conanfile.settings original_settings_target = conanfile.settings_target diff --git a/conan/internal/model/conf.py b/conan/internal/model/conf.py index 84a41389f4f..459e7457a98 100644 --- a/conan/internal/model/conf.py +++ b/conan/internal/model/conf.py @@ -7,8 +7,6 @@ import fnmatch import textwrap -from collections import OrderedDict - from jinja2 import Environment, FileSystemLoader from conan.errors import ConanException @@ -345,20 +343,11 @@ class Conf: def __init__(self): # It being ordered allows for Windows case-insensitive composition - self._values = OrderedDict() # {var_name: [] of values, including separators} + self._values = {} # {var_name: [] of values, including separators} def __bool__(self): return bool(self._values) - def __repr__(self): - return "Conf: " + repr(self._values) - - def __eq__(self, other): - """ - :type other: Conf - """ - return other._values == self._values - def clear(self): self._values.clear() @@ -429,13 +418,12 @@ def show(self, fnpattern, pattern=""): def copy(self): c = Conf() - c._values = OrderedDict((k, v.copy()) for k, v in self._values.items()) + c._values = {k: v.copy() for k, v in self._values.items()} return c def filter_core(self): c = Conf() - c._values = OrderedDict((k, v.copy()) for k, v in self._values.items() - if not CORE_CONF_PATTERN.match(k)) + c._values = {k: v.copy() for k, v in self._values.items() if not CORE_CONF_PATTERN.match(k)} return c def dumps(self): @@ -600,10 +588,7 @@ class ConfDefinition: ("=!", "unset"), ("*=", "update"), ("=", "define")) def __init__(self): - self._pattern_confs = OrderedDict() - - def __repr__(self): - return "ConfDefinition: " + repr(self._pattern_confs) + self._pattern_confs = {} def __bool__(self): return bool(self._pattern_confs) diff --git a/conan/internal/model/lockfile.py b/conan/internal/model/lockfile.py index f6811344af8..d93a5f7436e 100644 --- a/conan/internal/model/lockfile.py +++ b/conan/internal/model/lockfile.py @@ -1,7 +1,6 @@ import fnmatch import json import os -from collections import OrderedDict from conan.api.output import ConanOutput from conan.internal.graph.graph import RECIPE_VIRTUAL, RECIPE_CONSUMER, CONTEXT_BUILD, Overrides @@ -21,7 +20,7 @@ class _LockRequires: otherwise it could be a bare list """ def __init__(self): - self._requires = OrderedDict() # {require: package_ids} + self._requires = {} # {require: package_ids} def refs(self): return self._requires.keys() @@ -53,9 +52,9 @@ def add(self, ref, package_ids=None): old_package_ids = self._requires.pop(ref, None) # Get existing one if old_package_ids is not None: if package_ids is not None: - package_ids = old_package_ids.update(package_ids) - else: - package_ids = old_package_ids + assert isinstance(old_package_ids, dict) + old_package_ids.update(package_ids) + package_ids = old_package_ids self._requires[ref] = package_ids else: # Manual addition of something without revision existing = {r: r for r in self._requires}.get(ref) @@ -77,7 +76,7 @@ def remove(self, pattern): remove.append(k) else: remove = [k for k in self._requires if k.matches(pattern, False)] - self._requires = OrderedDict((k, v) for k, v in self._requires.items() if k not in remove) + self._requires = {k: v for k, v in self._requires.items() if k not in remove} return remove def update(self, refs, name): @@ -96,7 +95,7 @@ def update(self, refs, name): self.sort() def sort(self): - self._requires = OrderedDict(reversed(sorted(self._requires.items()))) + self._requires = dict(reversed(sorted(self._requires.items()))) def merge(self, other): """ diff --git a/conan/tools/apple/xcodedeps.py b/conan/tools/apple/xcodedeps.py index f78ef8878f6..9cdad6c3b19 100644 --- a/conan/tools/apple/xcodedeps.py +++ b/conan/tools/apple/xcodedeps.py @@ -1,7 +1,6 @@ import os import re import textwrap -from collections import OrderedDict from jinja2 import Template @@ -130,7 +129,7 @@ def _conf_xconfig_file(self, require, pkg_name, comp_name, package_folder, trans """ def _merged_vars(name): merged = [var for cpp_info in transitive_cpp_infos for var in getattr(cpp_info, name)] - return list(OrderedDict.fromkeys(merged).keys()) + return list(dict.fromkeys(merged).keys()) # TODO: Investigate if paths can be made relative to "root" folder fields = { @@ -292,8 +291,8 @@ def _transitive_components(component): _transitive_components(comp_cpp_info) # remove duplicates - transitive_internal = list(OrderedDict.fromkeys(transitive_internal).keys()) - transitive_external = list(OrderedDict.fromkeys(transitive_external).keys()) + transitive_internal = list(dict.fromkeys(transitive_internal).keys()) + transitive_external = list(dict.fromkeys(transitive_external).keys()) # In case dep is editable and package_folder=None pkg_folder = dep.package_folder or dep.recipe_folder diff --git a/conan/tools/cmake/toolchain/blocks.py b/conan/tools/cmake/toolchain/blocks.py index 00e06db1746..4b62064220f 100644 --- a/conan/tools/cmake/toolchain/blocks.py +++ b/conan/tools/cmake/toolchain/blocks.py @@ -1,7 +1,6 @@ import os import re import textwrap -from collections import OrderedDict from jinja2 import Template @@ -13,7 +12,7 @@ from conan.tools.build.flags import architecture_flag, architecture_link_flag, libcxx_flags, threads_flags from conan.tools.build.cross_building import cross_building from conan.tools.cmake.toolchain import CONAN_TOOLCHAIN_FILENAME -from conan.tools.cmake.utils import is_multi_configuration, cmake_escape_value +from conan.tools.cmake.utils import is_multi_configuration from conan.tools.intel import IntelCC from conan.tools.microsoft.visual import msvc_version_to_toolset_version, msvc_platform_from_arch from conan.internal.api.install.generators import relativize_path @@ -1390,7 +1389,7 @@ def context(self): class ToolchainBlocks: def __init__(self, conanfile, toolchain, items=None): - self._blocks = OrderedDict() + self._blocks = {} self._conanfile = conanfile self._toolchain = toolchain if items: @@ -1416,14 +1415,14 @@ def select(self, name, *args): self._conanfile.output.warning("CMakeToolchain.select is deprecated. Use blocks.enabled()" " instead", warn_tag="deprecated") to_keep = [name] + list(args) + ["variables", "preprocessor"] - self._blocks = OrderedDict((k, v) for k, v in self._blocks.items() if k in to_keep) + self._blocks = {k: v for k, v in self._blocks.items() if k in to_keep} def enabled(self, name, *args): """ keep the blocks provided as arguments, remove the others """ to_keep = [name] + list(args) - self._blocks = OrderedDict((k, v) for k, v in self._blocks.items() if k in to_keep) + self._blocks = {k: v for k, v in self._blocks.items() if k in to_keep} def __setitem__(self, name, block_type): # Create a new class inheriting Block with the elements of the provided one @@ -1438,7 +1437,7 @@ def process_blocks(self): check_type=list) if blocks is not None: try: - new_blocks = OrderedDict((b, self._blocks[b]) for b in blocks) + new_blocks = {b: self._blocks[b] for b in blocks} except KeyError as e: raise ConanException(f"Block {e} defined in tools.cmake.cmaketoolchain" f":enabled_blocks doesn't exist in {list(self._blocks.keys())}") diff --git a/test/integration/lockfile/test_lock_packages.py b/test/integration/lockfile/test_lock_packages.py index 5b1d3acbcdd..a648d78b75e 100644 --- a/test/integration/lockfile/test_lock_packages.py +++ b/test/integration/lockfile/test_lock_packages.py @@ -27,6 +27,13 @@ def test_lock_packages(requires): lock = client.load("consumer/conan.lock") assert NO_SETTINGS_PACKAGE_ID in lock + # Just repeat + client.run("lock create consumer/conanfile.txt --lockfile-packages") + assert "ERROR: The --lockfile-packages arg is private and shouldn't be used" in client.out + assert "pkg/0.1#" in client.out + lock = client.load("consumer/conan.lock") + assert NO_SETTINGS_PACKAGE_ID in lock + with environment_update({"MYVAR": "MYVALUE2"}): client.run("create pkg --name=pkg --version=0.1") prev2 = client.created_package_revision("pkg/0.1") From 336b58004029acbc2e910b8ff08555b99c415eff Mon Sep 17 00:00:00 2001 From: James Date: Tue, 7 Apr 2026 10:46:32 +0200 Subject: [PATCH 078/110] fix powershell unset (and other possible unsets too) (#19820) --- conan/internal/model/conf.py | 8 +++++--- .../toolchains/env/test_virtualenv_powershell.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/conan/internal/model/conf.py b/conan/internal/model/conf.py index 459e7457a98..956e5cb278f 100644 --- a/conan/internal/model/conf.py +++ b/conan/internal/model/conf.py @@ -376,7 +376,9 @@ def get(self, conf_name, default=None, check_type=None, choices=None): conf_value = self._values.get(conf_name) if conf_value: v = conf_value.value - if choices is not None and v not in choices and v is not None: + if v is None: # value was unset + return default + if choices is not None and v not in choices: raise ConanException(f"Unknown value '{v}' for '{conf_name}'") # Some smart conversions if check_type is bool and not isinstance(v, bool): @@ -387,9 +389,9 @@ def get(self, conf_name, default=None, check_type=None, choices=None): raise ConanException(f"[conf] {conf_name} must be a boolean-like object " f"(true/false, 1/0, on/off) and value '{v}' does not match it.") elif check_type is str and not isinstance(v, str): + # TODO: this would be converting things like lists to strings without + # proper error, is it worth trying to change it? return str(v) - elif v is None: # value was unset - return default elif (check_type is not None and not isinstance(v, check_type) or check_type is int and isinstance(v, bool)): raise ConanException(f"[conf] {conf_name} must be a " diff --git a/test/functional/toolchains/env/test_virtualenv_powershell.py b/test/functional/toolchains/env/test_virtualenv_powershell.py index 15d192a53ee..3041c3a6fa9 100644 --- a/test/functional/toolchains/env/test_virtualenv_powershell.py +++ b/test/functional/toolchains/env/test_virtualenv_powershell.py @@ -260,3 +260,15 @@ def test_verbosity_flag(): assert "/verbosity:Detailed" in tc.out assert "-verbosity:Detailed" not in tc.out + + +def test_powershell_deactivation(): + # https://github.com/conan-io/conan/issues/19819 + c = TestClient(light=True) + c.save({"conanfile.txt": ""}) + c.run('install . -c tools.env.virtualenv:powershell=!') + files = os.listdir(c.current_folder) + assert "conanrun.ps1" not in files + assert "conanbuild.ps1" not in files + assert "conanrunenv.ps1" not in files + assert "conanbuildenv.ps1" not in files From 57f643d5a8086eb69a75e844211e4d0a346275fb Mon Sep 17 00:00:00 2001 From: James Date: Tue, 7 Apr 2026 13:08:59 +0200 Subject: [PATCH 079/110] doc retry conf and change default (#19830) * doc retry conf and change default * fix tests --- conan/internal/model/conf.py | 12 ++++++------ conan/internal/rest/rest_client_v2.py | 2 +- test/integration/remote/broken_download_test.py | 6 +++--- test/integration/remote/download_retries_test.py | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/conan/internal/model/conf.py b/conan/internal/model/conf.py index 956e5cb278f..620bd4b0f67 100644 --- a/conan/internal/model/conf.py +++ b/conan/internal/model/conf.py @@ -28,12 +28,12 @@ "core:default_build_profile": "Defines the default build profile ('default' by default)", "core:allow_uppercase_pkg_names": "Temporarily (will be removed in 2.X) allow uppercase names", "core.version_ranges:resolve_prereleases": "Whether version ranges can resolve to pre-releases or not", - "core.upload:retry": "Number of retries in case of failure when uploading to Conan server", - "core.upload:retry_wait": "Seconds to wait between upload attempts to Conan server", + "core.upload:retry": "(int, default: 1) Number of retries in case of failure when uploading to Conan server", + "core.upload:retry_wait": "(int, default: 5s) Seconds to wait between upload attempts to Conan server", "core.upload:parallel": "Number of concurrent threads to upload packages", "core.download:parallel": "Number of concurrent threads to download packages", - "core.download:retry": "Number of retries in case of failure when downloading from Conan server", - "core.download:retry_wait": "Seconds to wait between download attempts from Conan server", + "core.download:retry": " (int, default: 2) Number of retries in case of failure when downloading from Conan server", + "core.download:retry_wait": "(int, default: 1s) Seconds to wait between download attempts from Conan server", "core.download:download_cache": "Define path to a file download cache", "core.cache:storage_path": "Absolute path where the packages and database are stored", "core:update_policy": "(Legacy). If equal 'legacy' when multiple remotes, update based on order of remotes, only the timestamp of the first occurrence of each revision counts.", @@ -102,8 +102,8 @@ "tools.cmake:configure_args": "Add extra arguments to CMake.configure() command line ", "tools.cmake:install_strip": "(Deprecated) Add --strip to cmake.install(). Use tools.build:install_strip instead", "tools.deployer:symlinks": "Set to False to disable deployers copying symlinks", - "tools.files.download:retry": "Number of retries in case of failure when downloading", - "tools.files.download:retry_wait": "Seconds to wait between download attempts", + "tools.files.download:retry": "(int, default: 2) Number of retries in case of failure when downloading", + "tools.files.download:retry_wait": "(int, default: 5s) Seconds to wait between download attempts", "tools.files.download:verify": "If set, overrides recipes on whether to perform SSL verification for their downloaded files. Only recommended to be set while testing", "tools.files.unzip:filter": "Define tar extraction filter: 'fully_trusted', 'tar', 'data'", "tools.graph:vendor": "(Experimental) If 'build', enables the computation of dependencies of vendoring packages to build them", diff --git a/conan/internal/rest/rest_client_v2.py b/conan/internal/rest/rest_client_v2.py index aabd00536d1..9033e6d9189 100644 --- a/conan/internal/rest/rest_client_v2.py +++ b/conan/internal/rest/rest_client_v2.py @@ -309,7 +309,7 @@ def _download_and_save_files(self, urls, dest_folder, files, parallel=False, sco # Take advantage of filenames ordering, so that conan_package.tgz and conan_export.tgz # can be < conanfile, conaninfo, and sent always the last, so smaller files go first retry = self._config.get("core.download:retry", check_type=int, default=2) - retry_wait = self._config.get("core.download:retry_wait", check_type=int, default=0) + retry_wait = self._config.get("core.download:retry_wait", check_type=int, default=1) downloader = ConanInternalCacheDownloader(self.requester, self._config, scope=scope) threads = [] diff --git a/test/integration/remote/broken_download_test.py b/test/integration/remote/broken_download_test.py index fff7898910e..44aac5592c0 100644 --- a/test/integration/remote/broken_download_test.py +++ b/test/integration/remote/broken_download_test.py @@ -114,13 +114,13 @@ def DownloadFilesBrokenRequesterTimesOne(*args, **kwargs): # noqa client.run("install --requires=lib/1.0@lasote/stable") assert "WARN: network: Error downloading file" in client.out assert 'Fake connection error exception' in client.out - assert 1 == str(client.out).count("Waiting 0 seconds to retry...") + assert 1 == str(client.out).count("Waiting 1 seconds to retry...") client = TestClient(servers=servers, inputs=["admin", "password"], requester_class=DownloadFilesBrokenRequesterTimesOne) - client.save_home({"global.conf": "core.download:retry_wait=1"}) + client.save_home({"global.conf": "core.download:retry_wait=2"}) client.run("install --requires=lib/1.0@lasote/stable") - assert 1 == str(client.out).count("Waiting 1 seconds to retry...") + assert 1 == str(client.out).count("Waiting 2 seconds to retry...") def DownloadFilesBrokenRequesterTimesTen(*args, **kwargs): # noqa return DownloadFilesBrokenRequester(10, *args, **kwargs) diff --git a/test/integration/remote/download_retries_test.py b/test/integration/remote/download_retries_test.py index bff70264dff..7772978376e 100644 --- a/test/integration/remote/download_retries_test.py +++ b/test/integration/remote/download_retries_test.py @@ -29,5 +29,5 @@ def get(self, *args, **kwargs): client = TestClient(servers={"default": test_server}, inputs=["admin", "password"], requester_class=BuggyRequester) client.run("install --requires=pkg/0.1@lasote/stable", assert_error=True) - assert str(client.out).count("Waiting 0 seconds to retry...") == 2 + assert str(client.out).count("Waiting 1 seconds to retry...") == 2 assert str(client.out).count("Error 200 downloading") == 3 From c6fd6918e5f9c74f2ea8c14df17d29b035fb6b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= <5364255+AbrilRBS@users.noreply.github.com> Date: Tue, 7 Apr 2026 17:41:27 +0200 Subject: [PATCH 080/110] Documentation improvement for items() iteration of packages list (#19829) * Improve items() * Improve items documentation * Fix docstring --- conan/api/model/list.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/conan/api/model/list.py b/conan/api/model/list.py index ea6495d4c28..63572209e0f 100644 --- a/conan/api/model/list.py +++ b/conan/api/model/list.py @@ -312,11 +312,32 @@ def refs(self): return result def items(self) -> Iterable[Tuple[RecipeReference, Dict[PkgReference, Dict]]]: - """ Iterate the contents of the package list. + """Iterate over the contents of the package list. - The first dictionary is the information directly belonging to the recipe-revision. - The second dictionary contains PkgReference as keys, and a dictionary with the values - belonging to that specific package reference (settings, options, etc.). + Yields tuples containing a recipe reference and a dictionary of its + associated package references content. + + Returns: + An iterable of tuples where: + + - The first element is a ``RecipeReference`` (representing the recipe revision). + - The second element is a dictionary mapping a ``PkgReference`` to a nested dictionary of + its specific attributes (e.g., settings, options). + + Warning: + **Missing Revisions Behavior:** + This method filters out results that lack revision information. + + - It will ONLY yield items if they contain at least a **recipe revision**. + - The nested package dictionary will be empty unless it contains a **package revision**. + + **When to use serialize instead:** + If you perform a general search that does not fetch revisions (e.g., running + ``conan list *``), this method will yield nothing because no artifact references + are created. In these cases, use the ``serialize()`` method to access the results. + + To successfully use ``items()``, your query must explicitly request revisions + (e.g., running ``conan list pkg/version#*:*#*``). """ for ref, ref_dict in self._data.items(): for rrev, rrev_dict in ref_dict.get("revisions", {}).items(): From 54720f385124a7a1d4de1ad231ed0fc314ff40a0 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 7 Apr 2026 17:51:00 +0200 Subject: [PATCH 081/110] Poc/editable refactor (#19835) * wip * wip * wip --- conan/api/subapi/install.py | 13 +++---------- conan/internal/api/local/editable.py | 5 ----- conan/internal/cache/conan_reference_layout.py | 5 +++++ conan/internal/graph/graph.py | 1 + conan/internal/graph/graph_builder.py | 2 ++ conan/internal/graph/installer.py | 10 +++------- conan/internal/graph/proxy.py | 8 +++++--- 7 files changed, 19 insertions(+), 25 deletions(-) diff --git a/conan/api/subapi/install.py b/conan/api/subapi/install.py index 806ec3f9bb5..f53abb27f6f 100644 --- a/conan/api/subapi/install.py +++ b/conan/api/subapi/install.py @@ -3,7 +3,6 @@ from conan.api.model import Remote from conan.internal.api.install.generators import write_generators -from conan.internal.conan_app import ConanBasicApp from conan.internal.deploy import do_deploys from conan.internal.graph.install_graph import InstallGraph @@ -39,9 +38,7 @@ def install_binaries(self, deps_graph, remotes: List[Remote] = None, return_inst :param remotes: List of remotes to fetch packages from if necessary. :param return_install_error: If ``True``, do not raise an exception, but return it """ - app = ConanBasicApp(self._conan_api) - installer = BinaryInstaller(self._conan_api, self._helpers.global_conf, app.editable_packages, - self._helpers.hook_manager) + installer = BinaryInstaller(self._conan_api, self._helpers.global_conf, self._helpers.hook_manager) install_graph = InstallGraph(deps_graph) install_graph.raise_errors() install_order = install_graph.install_order() @@ -68,9 +65,7 @@ def install_system_requires(self, graph, only_info=False): :param graph: Dependency graph to install system requirements for :param only_info: If ``True``, only reporting and checking of whether the system requirements are installed is performed. """ - app = ConanBasicApp(self._conan_api) - installer = BinaryInstaller(self._conan_api, self._helpers.global_conf, app.editable_packages, - self._helpers.hook_manager) + installer = BinaryInstaller(self._conan_api, self._helpers.global_conf, self._helpers.hook_manager) installer.install_system_requires(graph, only_info) def install_sources(self, graph, remotes: List[Remote]): @@ -89,9 +84,7 @@ def install_sources(self, graph, remotes: List[Remote]): :param remotes: List of remotes where the ``exports_sources`` of the packages might be located :param graph: Dependency graph to download sources from """ - app = ConanBasicApp(self._conan_api) - installer = BinaryInstaller(self._conan_api, self._helpers.global_conf, app.editable_packages, - self._helpers.hook_manager) + installer = BinaryInstaller(self._conan_api, self._helpers.global_conf, self._helpers.hook_manager) installer.install_sources(graph, remotes) def install_consumer(self, deps_graph, generators: List[str] = None, source_folder=None, diff --git a/conan/internal/api/local/editable.py b/conan/internal/api/local/editable.py index 1d0608af949..6a64dda4119 100644 --- a/conan/internal/api/local/editable.py +++ b/conan/internal/api/local/editable.py @@ -49,11 +49,6 @@ def get(self, ref): _tmp.revision = None return self._edited_refs.get(_tmp) - def get_path(self, ref): - editable = self.get(ref) - if editable is not None: - return editable["path"] - def add(self, ref, path, output_folder=None): assert isinstance(ref, RecipeReference) _tmp = copy.copy(ref) diff --git a/conan/internal/cache/conan_reference_layout.py b/conan/internal/cache/conan_reference_layout.py index b9522ca8975..37ffe9b9dca 100644 --- a/conan/internal/cache/conan_reference_layout.py +++ b/conan/internal/cache/conan_reference_layout.py @@ -32,6 +32,11 @@ def remove(self): class BasicLayout(LayoutBase): # For editables and platform_requires + + def __init__(self, ref, base_folder, editable_output_folder=None): + super().__init__(ref, base_folder) + self.editable_output_folder = editable_output_folder + def conanfile(self): # the full conanfile path (including other other.py names) for editables # None for platform_requires diff --git a/conan/internal/graph/graph.py b/conan/internal/graph/graph.py index 924b7e2248b..9af88d7b07a 100644 --- a/conan/internal/graph/graph.py +++ b/conan/internal/graph/graph.py @@ -71,6 +71,7 @@ def __init__(self, ref, conanfile, context, recipe=None, path=None, test=False): self.is_conf = False self.replaced_requires = {} # To track the replaced requires for self.edges[old-ref] self.skipped_build_requires = False + self.editable_output_folder = None # In case this node is editable @property def dependencies(self): diff --git a/conan/internal/graph/graph_builder.py b/conan/internal/graph/graph_builder.py index 9be4e23901a..73cf3d65a7a 100644 --- a/conan/internal/graph/graph_builder.py +++ b/conan/internal/graph/graph_builder.py @@ -418,6 +418,8 @@ def _create_new_node(self, node, require, graph, profile_host, profile_build, gr new_node = Node(new_ref, dep_conanfile, context=context, test=require.test or node.test) new_node.recipe = recipe_status new_node.remote = remote + if isinstance(layout, BasicLayout): # Store the editable_output_folder for BinaryInstaller + new_node.editable_output_folder = layout.editable_output_folder down_options = self._compute_down_options(node, require, new_ref) diff --git a/conan/internal/graph/installer.py b/conan/internal/graph/installer.py index b5a75d38280..5293ac0ddc5 100644 --- a/conan/internal/graph/installer.py +++ b/conan/internal/graph/installer.py @@ -165,9 +165,8 @@ class BinaryInstaller: locally in case they are not found in remotes """ - def __init__(self, api, global_conf, editable_packages, hook_manager): + def __init__(self, api, global_conf, hook_manager): helpers = api._api_helpers # noqa - self._editable_packages = editable_packages self._cache = helpers.cache self._remote_manager = helpers.remote_manager self._hook_manager = hook_manager @@ -351,12 +350,9 @@ def _handle_node_editable(self, install_node): # It will only run generation node = install_node.nodes[0] conanfile = node.conanfile - ref = node.ref - editable = self._editable_packages.get(ref) - conanfile_path = editable["path"] - output_folder = editable.get("output_folder") - base_path = os.path.dirname(conanfile_path) + output_folder = node.editable_output_folder + base_path = conanfile.recipe_folder conanfile.folders.set_base_folders(base_path, output_folder) diff --git a/conan/internal/graph/proxy.py b/conan/internal/graph/proxy.py index 2fa1fcb3b73..910970932b3 100644 --- a/conan/internal/graph/proxy.py +++ b/conan/internal/graph/proxy.py @@ -32,9 +32,11 @@ def get_recipe(self, ref, remotes, update, check_update): def _get_recipe(self, reference, remotes, update, check_update): output = ConanOutput(scope=str(reference)) - conanfile_path = self._editable_packages.get_path(reference) - if conanfile_path is not None: - return BasicLayout(reference, conanfile_path), RECIPE_EDITABLE, None + editable = self._editable_packages.get(reference) + if editable is not None: + path = editable["path"] + output_folder = editable.get("output_folder") + return BasicLayout(reference, path, output_folder), RECIPE_EDITABLE, None # check if it there's any revision of this recipe in the local cache try: From 683e0753697f62f9912cd2081d67598aacc4897d Mon Sep 17 00:00:00 2001 From: James Date: Wed, 8 Apr 2026 09:37:53 +0200 Subject: [PATCH 082/110] testing msvc minor update via conf (#19843) --- conan/internal/model/conan_file.py | 1 + test/unittests/tools/cmake/test_cmaketoolchain.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/conan/internal/model/conan_file.py b/conan/internal/model/conan_file.py index eff7c2cd842..313060cf40f 100644 --- a/conan/internal/model/conan_file.py +++ b/conan/internal/model/conan_file.py @@ -75,6 +75,7 @@ class ConanFile: buildenv_info = None runenv_info = None conf_info = None + conf = None generator_info = None conan_data = None diff --git a/test/unittests/tools/cmake/test_cmaketoolchain.py b/test/unittests/tools/cmake/test_cmaketoolchain.py index 05d7720bfcb..e1fd31ea41b 100644 --- a/test/unittests/tools/cmake/test_cmaketoolchain.py +++ b/test/unittests/tools/cmake/test_cmaketoolchain.py @@ -314,6 +314,12 @@ def test_toolset_update_version_overflow(self, conanfile_msvc): c = CMakeToolchain(conanfile_msvc) assert 'set(CMAKE_GENERATOR_TOOLSET "v143,version=14.48" CACHE STRING "" FORCE)' in c.content + def test_toolset_exact(self, conanfile_msvc): + conanfile_msvc.settings.compiler.version = "195" + conanfile_msvc.conf.define("tools.microsoft:msvc_update", "0.35717") + c = CMakeToolchain(conanfile_msvc) + assert 'set(CMAKE_GENERATOR_TOOLSET "v145,version=14.50.35717"' in c.content + def test_toolset_x64(self, conanfile_msvc): # https://github.com/conan-io/conan/issues/11144 conanfile_msvc.conf.define("tools.cmake.cmaketoolchain:toolset_arch", "x64") From 035961fc963a98e8afcb8918ed4c8368e3048b22 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Wed, 8 Apr 2026 14:07:36 +0200 Subject: [PATCH 083/110] Avoid detecting default package manager when overrided from profile (#19847) Avoid searching for default package manager when defined in profile --- conan/tools/system/package_manager.py | 2 +- .../integration/tools/system/package_manager_test.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/conan/tools/system/package_manager.py b/conan/tools/system/package_manager.py index f1fcfae57f5..412569ea0f9 100644 --- a/conan/tools/system/package_manager.py +++ b/conan/tools/system/package_manager.py @@ -26,7 +26,7 @@ def __init__(self, conanfile): :param conanfile: The current recipe object. Always use ``self``. """ self._conanfile = conanfile - self._active_tool = self._conanfile.conf.get("tools.system.package_manager:tool", default=self.get_default_tool()) + self._active_tool = self._conanfile.conf.get("tools.system.package_manager:tool") or self.get_default_tool() self._sudo = self._conanfile.conf.get("tools.system.package_manager:sudo", default=False, check_type=bool) self._sudo_askpass = self._conanfile.conf.get("tools.system.package_manager:sudo_askpass", default=False, check_type=bool) self._mode = self._conanfile.conf.get("tools.system.package_manager:mode", default=self.mode_check) diff --git a/test/integration/tools/system/package_manager_test.py b/test/integration/tools/system/package_manager_test.py index 6632c1ee683..883b9bd3e4f 100644 --- a/test/integration/tools/system/package_manager_test.py +++ b/test/integration/tools/system/package_manager_test.py @@ -84,6 +84,18 @@ def test_package_manager_distro(distro, tool): assert tool == manager.get_default_tool() +def test_conf_tool_skips_default_detection_message_on_unknown_distro(): + """If ``tools.system.package_manager:tool`` is set, ``get_default_tool`` shall never be invoked""" + with mock.patch('conan.ConanFile.context', new_callable=PropertyMock) as context_mock: + context_mock.return_value = "host" + conanfile = ConanFileMock() + conanfile.settings = Settings() + conanfile.conf.define("tools.system.package_manager:tool", "apt-get") + with mock.patch.object(_SystemPackageManagerTool, "get_default_tool") as get_default_mock: + Apt(conanfile) + get_default_mock.assert_not_called() + + @pytest.mark.parametrize("sudo, sudo_askpass, expected_str", [ (True, True, "sudo -A "), (True, False, "sudo "), From 6f4cb53e04034eceececc64b2950af68109835e3 Mon Sep 17 00:00:00 2001 From: Carlos Zoido Date: Wed, 8 Apr 2026 19:35:32 +0200 Subject: [PATCH 084/110] Add `--strict` flag to `conan remote auth` (#19848) * add strict * third case --- conan/cli/commands/remote.py | 9 +++++++ .../command/remote/test_remote_users.py | 25 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/conan/cli/commands/remote.py b/conan/cli/commands/remote.py index 43cd7d2d813..96f450572f5 100644 --- a/conan/cli/commands/remote.py +++ b/conan/cli/commands/remote.py @@ -284,6 +284,8 @@ def remote_auth(conan_api, parser, subparser, *args): no CONAN_LOGIN* and CONAN_PASSWORD* variables available which could be used. Usually you'd use this method over conan remote login for scripting which needs to run in CI and locally. + By default, this command returns exit code 0 even if authentication fails for some remotes. + Use --strict to return exit code 1 if authentication fails for any remote. """ subparser.add_argument("remote", help="Pattern or name of the remote/s to authenticate against." " The pattern uses 'fnmatch' style wildcards.") @@ -296,6 +298,8 @@ def remote_auth(conan_api, parser, subparser, *args): "instance has anonymous access enabled and Conan would not ask " "for username and password even for non-anonymous repositories " "if not yet authenticated.") + subparser.add_argument("--strict", action="store_true", + help="Return exit code 1 if authentication fails for any remote.") args = parser.parse_args(*args) remotes = conan_api.remotes.list(pattern=args.remote) if not remotes: @@ -307,6 +311,11 @@ def remote_auth(conan_api, parser, subparser, *args): results[r.name] = {"user": conan_api.remotes.user_auth(r, args.with_user, args.force)} except Exception as e: results[r.name] = {"error": str(e)} + + if args.strict: + failed = [name for name, v in results.items() if "error" in v] + if failed: + raise ConanException("Authentication error in remotes: {}".format(", ".join(failed))) return results diff --git a/test/integration/command/remote/test_remote_users.py b/test/integration/command/remote/test_remote_users.py index 0503097a4a1..dfd42522875 100644 --- a/test/integration/command/remote/test_remote_users.py +++ b/test/integration/command/remote/test_remote_users.py @@ -456,6 +456,31 @@ def test_remote_auth_server_expire_token(self): c.run("remote auth *") assert "error: Too many failed login attempts, bye!" in c.out + def test_remote_auth_strict(self): + servers = { + "good": TestServer(users={"myuser": "mypassword"}), + "bad": TestServer(users={"myuser": "mypassword"}), + } + # All succeed -> exit 0 + c = TestClient(light=True, servers=servers, + inputs=["myuser", "mypassword", "myuser", "mypassword"]) + c.run("remote auth * --strict") + assert "user: myuser" in c.out + + # Partial failure without --strict -> exit 0 + c2 = TestClient(light=True, servers=servers, + inputs=["myuser", "mypassword", + "wrong", "wrong", "wrong", "wrong", "wrong", "wrong"]) + c2.run("remote auth *") + assert "error" in c2.out + + # Partial failure with --strict -> exit != 0 + c3 = TestClient(light=True, servers=servers, + inputs=["myuser", "mypassword", + "wrong", "wrong", "wrong", "wrong", "wrong", "wrong"]) + c3.run("remote auth * --strict", assert_error=True) + assert "Authentication error in remotes: bad" in c3.out + def test_auth_after_logout(self): server = TestServer(users={"myuser": "password"}) c = TestClient(light=True, servers={"default": server}, inputs=["myuser", "password"]*2) From fc22da00955d572011d80bf075d1e0c277353073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= <5364255+AbrilRBS@users.noreply.github.com> Date: Wed, 8 Apr 2026 19:39:43 +0200 Subject: [PATCH 085/110] Propagate build requirement run trait to upstream shared dependency (#19751) * Only propagate shared build requires if the requires itself is run=True * Test for run=False of the shared library * Different versions * Fix tests * experimental * This arcane knowledge shall be kept secret for now --- conan/internal/model/requires.py | 2 +- .../graph/core/graph_manager_test.py | 48 +++++++++++++++++-- .../graph/test_require_same_pkg_versions.py | 23 +++++++++ 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/conan/internal/model/requires.py b/conan/internal/model/requires.py index 9ca88535232..62cbed2c1b3 100644 --- a/conan/internal/model/requires.py +++ b/conan/internal/model/requires.py @@ -289,7 +289,7 @@ def transform_downstream(self, pkg_type, require, dep_pkg_type): # visible=self.visible will further propagate it downstream if dep_pkg_type is PackageType.SHARED or require.run: downstream_require = Requirement(require.ref, headers=False, libs=False, build=True, - run=True, visible=self.visible, direct=False) + run=self.run, visible=self.visible, direct=False) return downstream_require return diff --git a/test/integration/graph/core/graph_manager_test.py b/test/integration/graph/core/graph_manager_test.py index ccca1384154..4146e41d712 100644 --- a/test/integration/graph/core/graph_manager_test.py +++ b/test/integration/graph/core/graph_manager_test.py @@ -14,11 +14,11 @@ def _check_transitive(node, transitive_deps): f"!=\n{transitive_deps}" for v1, v2 in zip(values, transitive_deps): - assert v1.node is v2[0], f"{v1.node}!={v2[0]}" - assert v1.require.headers is v2[1], f"{v1.node}!={v2[0]} headers" - assert v1.require.libs is v2[2], f"{v1.node}!={v2[0]} libs" - assert v1.require.build is v2[3], f"{v1.node}!={v2[0]} build" - assert v1.require.run is v2[4], f"{v1.node}!={v2[0]} run" + assert v1.node is v2[0], f"{v1.node}!=expected {v2[0]}" + assert v1.require.headers is v2[1], f"{v1.node}!=expected {v2[0]} ({v2[1]}) headers" + assert v1.require.libs is v2[2], f"{v1.node}!=expected {v2[0]} ({v2[2]}) libs" + assert v1.require.build is v2[3], f"{v1.node}!=expected {v2[0]} ({v2[3]}) build" + assert v1.require.run is v2[4], f"{v1.node}!=expected {v2[0]} ({v2[4]}) run" assert len(v2) <= 5 @@ -761,6 +761,44 @@ def test_negate_run(self): (libb, False, False, False, True), (liba, False, False, False, False)]) + def test_run_false_multiversion_shared(self): + self.recipe_conanfile("liba/1.0", GenConanfile().with_package_type("shared-library")) + self.recipe_conanfile("liba/2.0", GenConanfile().with_package_type("shared-library")) + self.recipe_conanfile("tool/1.0", GenConanfile().with_package_type("application") + .with_requires("liba/1.0")) + self.recipe_conanfile("tool/2.0", GenConanfile().with_package_type("application") + .with_requires("liba/2.0")) + self.recipe_conanfile("libc/0.1", + GenConanfile().with_package_type("shared-library") + .with_tool_requirement("tool/1.0", run=False) + .with_tool_requirement("tool/2.0", run=False)) + consumer = self.recipe_consumer("app/0.1", ["libc/0.1"]) + + deps_graph = self.build_consumer(consumer) + + assert 6 == len(deps_graph.nodes) + app = deps_graph.root + libc = app.edges[0].dst + tool1 = libc.edges[0].dst + tool2 = libc.edges[1].dst + liba1 = tool1.edges[0].dst + liba2 = tool2.edges[0].dst + + self._check_node(app, "app/0.1", deps=[libc]) + self._check_node(libc, "libc/0.1#123", deps=[tool1, tool2], dependents=[app]) + self._check_node(tool1, "tool/1.0#123", deps=[liba1], dependents=[libc]) + self._check_node(tool2, "tool/2.0#123", deps=[liba2], dependents=[libc]) + + # node, headers, lib, build, run + _check_transitive(libc, [ + (tool1, False, False, True, False), + (liba1, False, False, True, False), + (tool2, False, False, True, False), + (liba2, False, False, True, False) + ]) + + _check_transitive(app, [(libc, True, True, False, True)]) + def test_force_run(self): # app -> libc/0.1 -> libb0.1 -> liba0.1 # even if all are static, there is something in a static lib (files or whatever) that diff --git a/test/integration/graph/test_require_same_pkg_versions.py b/test/integration/graph/test_require_same_pkg_versions.py index c97d9a00a54..399031504cf 100644 --- a/test/integration/graph/test_require_same_pkg_versions.py +++ b/test/integration/graph/test_require_same_pkg_versions.py @@ -419,3 +419,26 @@ def package(self): assert "MYGCC=1.0!!" in c.out assert "wrapperb=1.0!!" in c.out assert "MYGCC=2.0!!" in c.out + + def test_require_different_versions_transitive_conflict_require(self): + c = TestClient(light=True) + c.save({"abseil/conanfile.py": GenConanfile("abseil") + .with_package_type("shared-library"), + "protobuf1/conanfile.py": GenConanfile("protobuf", "1.0") + .with_requirement("abseil/[~1]"), + "protobuf2/conanfile.py": GenConanfile("protobuf", "2.0") + .with_requirement("abseil/[~2]"), + "consumer/conanfile.py": GenConanfile("consumer", "1.0") + .with_tool_requirement("protobuf/1.0", run=False) + .with_tool_requirement("protobuf/2.0", run=False) + }) + + c.run("create abseil --version=1.0") + c.run("create abseil --version=2.0") + c.run("create protobuf1") + c.run("create protobuf2") + c.run("export consumer") + c.run("install consumer") + # No conflict, because both protobufs are run=False, + # so the shared-library abseil is not propagated with run=True in any of the branches + assert "Install finished successfully" in c.out From 24559231f04ef9605046047cae1a9bce18c2a464 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 10 Apr 2026 07:15:14 +0200 Subject: [PATCH 086/110] Refactor/api helpers (#19797) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wip * wip * wip * fix * Undefined locale for list html (#19828) * finalize() output folder should be printed only once (#19834) * Refactor/ordered dict (#19839) * wip * more rid * wip * fix powershell unset (and other possible unsets too) (#19820) * doc retry conf and change default (#19830) * doc retry conf and change default * fix tests * Documentation improvement for items() iteration of packages list (#19829) * Improve items() * Improve items documentation * Fix docstring * wip * review, removed global_editable_packages * fix --------- Co-authored-by: Abril Rincón Blanco <5364255+AbrilRBS@users.noreply.github.com> Co-authored-by: PerseoGI --- conan/api/conan_api.py | 35 ++++++++++++++- conan/api/subapi/cache.py | 5 +-- conan/api/subapi/config.py | 7 ++- conan/api/subapi/export.py | 5 +-- conan/api/subapi/graph.py | 45 ++++++++++---------- conan/api/subapi/install.py | 11 +++-- conan/api/subapi/local.py | 30 ++++++------- conan/api/subapi/report.py | 9 ++-- conan/api/subapi/upload.py | 10 ++--- conan/api/subapi/workspace.py | 14 +++--- conan/internal/api/uploader.py | 7 ++- conan/internal/conan_app.py | 1 - conan/internal/runner/docker.py | 5 +-- test/integration/conan_api/test_local_api.py | 4 +- 14 files changed, 105 insertions(+), 83 deletions(-) diff --git a/conan/api/conan_api.py b/conan/api/conan_api.py index 85d7a616856..c0f2f63458d 100644 --- a/conan/api/conan_api.py +++ b/conan/api/conan_api.py @@ -24,11 +24,17 @@ from conan.internal.api.remotes.localdb import LocalDB from conan.internal.cache.cache import PkgCache from conan.internal.cache.home_paths import HomePaths +from conan.internal.conan_app import ConanFileHelpers, CmdWrapper +from conan.internal.graph.proxy import ConanProxy +from conan.internal.graph.python_requires import PyRequireLoader +from conan.internal.graph.range_resolver import RangeResolver from conan.internal.hook_manager import HookManager from conan.internal.loader import load_python_file +from conan.internal.loader import ConanFileLoader from conan.internal.model.conf import load_global_conf, ConfDefinition, CORE_CONF_PATTERN from conan.internal.model.settings import load_settings_yml from conan.internal.paths import get_conan_user_home +from conan.internal.api.local.editable import EditablePackages from conan.internal.api.migrations import ClientMigrator from conan.internal.model.version_range import validate_conan_version from conan.internal.rest.auth_manager import ConanApiAuthManager @@ -106,7 +112,6 @@ def reinit(self): Reinitialize the Conan API. This is useful when the configuration changes. """ self._api_helpers.reinit() - self.local.reinit() def migrate(self): # Migration system @@ -123,6 +128,7 @@ def __init__(self, conan_api): self._init_global_conf() # TODO: Make uniform lazy vs non lazy collaborators self.hook_manager = HookManager(HomePaths(self._conan_api.home_folder).hooks_path) + self._editable_packages = EditablePackages(self._conan_api.home_folder) # Wraps an http_requester to inject proxies, certs, etc self._requester = ConanRequester(self.global_conf, self._conan_api.home_folder) self.cache = PkgCache(self._conan_api.home_folder, self.global_conf) @@ -175,6 +181,7 @@ def reinit(self): self._settings_yml = None self.cache = PkgCache(self._conan_api.home_folder, self.global_conf) self._remote_manager = None + self._editable_packages = EditablePackages(self._conan_api.home_folder) self._compression_plugin = None @property @@ -198,3 +205,29 @@ def remote_manager(self): @property def requester(self): return self._requester + + @property + def editable_packages(self): + # These are just the global editables, not including workspace ones + return self._editable_packages + + def get_loader(self): + ws_editables = self._conan_api.workspace.packages() + editable_packages = self._editable_packages.update_copy(ws_editables) + + legacy_update = self.global_conf.get("core:update_policy", choices=["legacy"]) + # This proxy is caching information + proxy = ConanProxy(self.cache, self.remote_manager, editable_packages, + legacy_update=legacy_update) + # This is caching too + range_resolver = RangeResolver(self.cache, self.remote_manager, self.global_conf, + editable_packages) + + cmd_wrap = CmdWrapper(HomePaths(self._conan_api.home_folder).wrapper_path) + conanfile_helpers = ConanFileHelpers(self._requester, cmd_wrap, self.global_conf, + self.cache, self._conan_api.home_folder, + self._conan_api) + pyreq_loader = PyRequireLoader(proxy, range_resolver, self.global_conf) + # This is caching too! + loader = ConanFileLoader(pyreq_loader, conanfile_helpers) + return proxy, range_resolver, loader diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index f2bf74e136c..7e43faaff17 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -19,7 +19,6 @@ from conan.api.model import PkgReference from conan.api.model import RecipeReference from conan.internal.api.uploader import PackagePreparator -from conan.internal.conan_app import ConanApp from conan.internal.rest.pkg_sign import PkgSignaturesPlugin from conan.internal.util.dates import revision_timestamp_now from conan.internal.util.files import rmdir, mkdir, remove, save @@ -194,8 +193,8 @@ def sign(self, package_list): "information on how to configure the plugin, please read the documentation at " "https://docs.conan.io/2/reference/extensions/package_signing.html.") - app = ConanApp(self._conan_api) - preparator = PackagePreparator(app, self._api_helpers.cache, + _, _, loader = self._api_helpers.get_loader() + preparator = PackagePreparator(loader, self._api_helpers.cache, self._api_helpers.remote_manager, self._api_helpers.global_conf) # Some packages can have missing sources/exports_sources diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index eca212d3534..d8af964b5a3 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -3,7 +3,6 @@ from conan.api.output import ConanOutput from conan.internal.cache.home_paths import HomePaths -from conan.internal.conan_app import ConanApp from conan.internal.graph.graph import CONTEXT_HOST, RECIPE_VIRTUAL, Node from conan.internal.graph.graph_builder import DepsGraphBuilder from conan.internal.graph.profile_node_definer import consumer_definer @@ -212,7 +211,7 @@ def fetch_packages(self, requires, lockfile=None, remotes=None, profile=None): remotes = conan_api.remotes.list() if remotes is None else remotes profile_host = profile_build = profile or conan_api.profiles.get_profile([]) - app = ConanApp(self._conan_api) + proxy, range_resolver, loader = self._helpers.get_loader() cache = self._helpers.cache ConanOutput().title("Fetching requested configuration packages") @@ -220,13 +219,13 @@ def fetch_packages(self, requires, lockfile=None, remotes=None, profile=None): for ref in requires: # Computation of a very simple graph that requires "ref" # Need to convert input requires to RecipeReference - conanfile = app.loader.load_virtual(requires=[ref]) + conanfile = loader.load_virtual(requires=[ref]) consumer_definer(conanfile, profile_host, profile_build) root_node = Node(ref=None, conanfile=conanfile, context=CONTEXT_HOST, recipe=RECIPE_VIRTUAL) root_node.is_conf = True update = ["*"] - builder = DepsGraphBuilder(app.proxy, app.loader, app.range_resolver, cache, remotes, + builder = DepsGraphBuilder(proxy, loader, range_resolver, cache, remotes, update, update, self._helpers.global_conf) deps_graph = builder.load_graph(root_node, profile_host, profile_build, lockfile) diff --git a/conan/api/subapi/export.py b/conan/api/subapi/export.py index 3901c3c0b8c..ca8a74cbf93 100644 --- a/conan/api/subapi/export.py +++ b/conan/api/subapi/export.py @@ -5,7 +5,6 @@ from conan.api.output import ConanOutput from conan.cli.printers.graph import print_graph_basic from conan.internal.cache.cache import PkgCache -from conan.internal.conan_app import ConanApp from conan.internal.api.export import cmd_export from conan.internal.methods import run_package_method from conan.internal.graph.graph import BINARY_BUILD, RECIPE_INCACHE @@ -44,9 +43,9 @@ def export(self, path, name: str = None, version: str = None, user: str = None, :return: A tuple of the exported RecipeReference and a ConanFile object """ ConanOutput().title("Exporting recipe to the cache") - app = ConanApp(self._conan_api) + _, _, loader = self._helpers.get_loader() hook_manager = self._helpers.hook_manager - return cmd_export(app.loader,self._helpers.cache, hook_manager, self._helpers.global_conf, path, + return cmd_export(loader,self._helpers.cache, hook_manager, self._helpers.global_conf, path, name, version, user, channel, graph_lock=lockfile, remotes=remotes) def export_pkg_graph(self, path, ref: RecipeReference, profile_host, profile_build, diff --git a/conan/api/subapi/graph.py b/conan/api/subapi/graph.py index 24200b455ba..cfa182b0641 100644 --- a/conan/api/subapi/graph.py +++ b/conan/api/subapi/graph.py @@ -1,5 +1,4 @@ from conan.api.output import ConanOutput -from conan.internal.conan_app import ConanApp from conan.internal.model.recipe_ref import ref_matches from conan.internal.graph.graph import Node, RECIPE_CONSUMER, CONTEXT_HOST, RECIPE_VIRTUAL, \ CONTEXT_BUILD, BINARY_MISSING, DepsGraph @@ -21,17 +20,17 @@ def _load_root_consumer_conanfile(self, path, profile_host, profile_build, name=None, version=None, user=None, channel=None, update=None, remotes=None, lockfile=None, is_build_require=False): - app = ConanApp(self._conan_api) + _, _, loader = self._helpers.get_loader() if path.endswith(".py"): - conanfile = app.loader.load_consumer(path, - name=name, - version=version, - user=user, - channel=channel, - graph_lock=lockfile, - remotes=remotes, - update=update) + conanfile = loader.load_consumer(path, + name=name, + version=version, + user=user, + channel=channel, + graph_lock=lockfile, + remotes=remotes, + update=update) ref = RecipeReference(conanfile.name, conanfile.version, conanfile.user, conanfile.channel) context = CONTEXT_BUILD if is_build_require else CONTEXT_HOST @@ -43,7 +42,7 @@ def _load_root_consumer_conanfile(self, path, profile_host, profile_build, root_node = Node(ref, conanfile, context=context, recipe=RECIPE_CONSUMER, path=path) root_node.should_build = True # It is a consumer, this is something we are building else: - conanfile = app.loader.load_conanfile_txt(path) + conanfile = loader.load_conanfile_txt(path) consumer_definer(conanfile, profile_host, profile_build) root_node = Node(None, conanfile, context=CONTEXT_HOST, recipe=RECIPE_CONSUMER, path=path) @@ -65,10 +64,9 @@ def load_root_test_conanfile(self, path, tested_reference, profile_host, profile :return: a graph Node, recipe=RECIPE_CONSUMER """ - app = ConanApp(self._conan_api) # necessary for correct resolution and update of remote python_requires - loader = app.loader + _, _, loader = self._helpers.get_loader() profile_host.options.scope(tested_reference) # do not try apply lock_python_requires for test_package/conanfile.py consumer @@ -92,12 +90,12 @@ def _load_root_virtual_conanfile(self, profile_host, profile_build, requires, to python_requires=None): if not python_requires and not requires and not tool_requires: raise ConanException("Provide requires or tool_requires") - app = ConanApp(self._conan_api) - conanfile = app.loader.load_virtual(requires=requires, - tool_requires=tool_requires, - python_requires=python_requires, - graph_lock=lockfile, remotes=remotes, - update=update, check_updates=check_updates) + _, _, loader = self._helpers.get_loader() + conanfile = loader.load_virtual(requires=requires, + tool_requires=tool_requires, + python_requires=python_requires, + graph_lock=lockfile, remotes=remotes, + update=update, check_updates=check_updates) consumer_definer(conanfile, profile_host, profile_build) root_node = Node(ref=None, conanfile=conanfile, context=CONTEXT_HOST, recipe=RECIPE_VIRTUAL) @@ -182,15 +180,16 @@ def load_graph(self, root_node, profile_host, profile_build, lockfile=None, remo :param check_update: For "graph info" command, check if there are recipe updates """ ConanOutput().title("Computing dependency graph") - app = ConanApp(self._conan_api) assert profile_host is not None assert profile_build is not None + proxy, range_resolver, loader = self._helpers.get_loader() + remotes = remotes or [] - cache = self._conan_api._api_helpers.cache # noqa - builder = DepsGraphBuilder(app.proxy, app.loader, app.range_resolver, cache, remotes, - update, check_update, self._conan_api._api_helpers.global_conf) + cache = self._helpers.cache + builder = DepsGraphBuilder(proxy, loader, range_resolver, cache, remotes, + update, check_update, self._helpers.global_conf) deps_graph = builder.load_graph(root_node, profile_host, profile_build, lockfile) return deps_graph diff --git a/conan/api/subapi/install.py b/conan/api/subapi/install.py index f53abb27f6f..3c9271b7846 100644 --- a/conan/api/subapi/install.py +++ b/conan/api/subapi/install.py @@ -38,7 +38,8 @@ def install_binaries(self, deps_graph, remotes: List[Remote] = None, return_inst :param remotes: List of remotes to fetch packages from if necessary. :param return_install_error: If ``True``, do not raise an exception, but return it """ - installer = BinaryInstaller(self._conan_api, self._helpers.global_conf, self._helpers.hook_manager) + installer = BinaryInstaller(self._conan_api, self._helpers.global_conf, + self._helpers.hook_manager) install_graph = InstallGraph(deps_graph) install_graph.raise_errors() install_order = install_graph.install_order() @@ -65,7 +66,8 @@ def install_system_requires(self, graph, only_info=False): :param graph: Dependency graph to install system requirements for :param only_info: If ``True``, only reporting and checking of whether the system requirements are installed is performed. """ - installer = BinaryInstaller(self._conan_api, self._helpers.global_conf, self._helpers.hook_manager) + installer = BinaryInstaller(self._conan_api, self._helpers.global_conf, + self._helpers.hook_manager) installer.install_system_requires(graph, only_info) def install_sources(self, graph, remotes: List[Remote]): @@ -84,7 +86,8 @@ def install_sources(self, graph, remotes: List[Remote]): :param remotes: List of remotes where the ``exports_sources`` of the packages might be located :param graph: Dependency graph to download sources from """ - installer = BinaryInstaller(self._conan_api, self._helpers.global_conf, self._helpers.hook_manager) + installer = BinaryInstaller(self._conan_api, self._helpers.global_conf, + self._helpers.hook_manager) installer.install_sources(graph, remotes) def install_consumer(self, deps_graph, generators: List[str] = None, source_folder=None, @@ -140,7 +143,7 @@ def install_consumer(self, deps_graph, generators: List[str] = None, source_fold write_generators(conanfile, hook_manager, self._conan_api.home_folder, envs_generation=envs_generation) - def deploy(self, graph, deployer: List[str], deploy_package: List[str]=None, + def deploy(self, graph, deployer: List[str], deploy_package: List[str] = None, deploy_folder=None) -> None: """ Run the given deployer in the dependency graph. diff --git a/conan/api/subapi/local.py b/conan/api/subapi/local.py index 848c33be54c..01c572954ee 100644 --- a/conan/api/subapi/local.py +++ b/conan/api/subapi/local.py @@ -2,8 +2,6 @@ from typing import List from conan.cli import make_abs_path -from conan.internal.conan_app import ConanApp -from conan.internal.api.local.editable import EditablePackages from conan.internal.methods import run_build_method, run_source_method from conan.internal.graph.graph import CONTEXT_HOST from conan.internal.graph.profile_node_definer import initialize_conanfile_profile @@ -21,7 +19,6 @@ class LocalAPI: def __init__(self, conan_api, helpers): self._conan_api = conan_api self._helpers = helpers - self.editable_packages = EditablePackages(conan_api.home_folder) @staticmethod def get_conanfile_path(path, cwd, py): @@ -73,8 +70,8 @@ def editable_add(self, path, name=None, version=None, user=None, channel=None, c :return: RecipeReference of the added package """ path = self.get_conanfile_path(path, cwd, py=True) - app = ConanApp(self._conan_api) - conanfile = app.loader.load_named(path, name, version, user, channel, remotes=remotes) + _, _, loader = self._helpers.get_loader() + conanfile = loader.load_named(path, name, version, user, channel, remotes=remotes) if conanfile.name is None or conanfile.version is None: raise ConanException("Editable package recipe should declare its name and version") ref = RecipeReference(conanfile.name, conanfile.version, conanfile.user, conanfile.channel) @@ -82,7 +79,7 @@ def editable_add(self, path, name=None, version=None, user=None, channel=None, c target_path = self.get_conanfile_path(path=path, cwd=cwd, py=True) output_folder = make_abs_path(output_folder) if output_folder else None # Check the conanfile is there, and name/version matches - self.editable_packages.add(ref, target_path, output_folder=output_folder) + self._helpers.editable_packages.add(ref, target_path, output_folder=output_folder) return ref def editable_remove(self, path=None, requires=None, cwd=None): @@ -99,10 +96,10 @@ def editable_remove(self, path=None, requires=None, cwd=None): if path: path = make_abs_path(path, cwd) path = os.path.join(path, "conanfile.py") - return self.editable_packages.remove(path, requires) + return self._helpers.editable_packages.remove(path, requires) def editable_list(self): - return self.editable_packages.edited_refs + return self._helpers.editable_packages.edited_refs def source(self, path, name=None, version=None, user=None, channel=None, remotes: List[Remote] = None): @@ -118,10 +115,10 @@ def source(self, path, name=None, version=None, user=None, channel=None, :param channel: The channel of the package. If not defined, it is taken from conanfile :param remotes: The remotes to resolve possible ``python-requires`` for this recipe if needed. """ - app = ConanApp(self._conan_api) - conanfile = app.loader.load_consumer(path, name=name, version=version, - user=user, channel=channel, graph_lock=None, - remotes=remotes) + _, _, loader = self._helpers.get_loader() + conanfile = loader.load_consumer(path, name=name, version=version, + user=user, channel=channel, graph_lock=None, + remotes=remotes) # This profile is empty, but with the conf from global.conf profile = self._conan_api.profiles.get_profile([]) initialize_conanfile_profile(conanfile, profile, profile, CONTEXT_HOST, False) @@ -184,10 +181,7 @@ def test(conanfile) -> None: def inspect(self, conanfile_path, remotes, lockfile, name=None, version=None, user=None, channel=None): - app = ConanApp(self._conan_api) - conanfile = app.loader.load_named(conanfile_path, name=name, version=version, user=user, - channel=channel, remotes=remotes, graph_lock=lockfile) + _, _, loader = self._helpers.get_loader() + conanfile = loader.load_named(conanfile_path, name=name, version=version, user=user, + channel=channel, remotes=remotes, graph_lock=lockfile) return conanfile - - def reinit(self): - self.editable_packages = EditablePackages(self._conan_api.home_folder) diff --git a/conan/api/subapi/report.py b/conan/api/subapi/report.py index 7fe903411f2..21dcd960fae 100644 --- a/conan/api/subapi/report.py +++ b/conan/api/subapi/report.py @@ -5,7 +5,6 @@ from conan.api.output import ConanOutput from conan.errors import ConanException from conan.api.model import RecipeReference -from conan.internal.conan_app import ConanApp from conan.internal.errors import conanfile_exception_formatter from conan.internal.graph.graph import CONTEXT_HOST from conan.internal.graph.profile_node_definer import initialize_conanfile_profile @@ -86,10 +85,10 @@ def _source(path_to_conanfile, reference): def _configure_source(conan_api, hook_manager, conanfile_path, ref, remotes): - app = ConanApp(conan_api) - conanfile = app.loader.load_consumer(conanfile_path, name=ref.name, version=str(ref.version), - user=ref.user, channel=ref.channel, graph_lock=None, - remotes=remotes) + _, _, loader = conan_api._api_helpers.get_loader() # noqa + conanfile = loader.load_consumer(conanfile_path, name=ref.name, version=str(ref.version), + user=ref.user, channel=ref.channel, graph_lock=None, + remotes=remotes) # This profile is empty, but with the conf from global.conf profile = conan_api.profiles.get_profile([]) initialize_conanfile_profile(conanfile, profile, profile, CONTEXT_HOST, False) diff --git a/conan/api/subapi/upload.py b/conan/api/subapi/upload.py index 1fc18bcd593..c8243790b45 100644 --- a/conan/api/subapi/upload.py +++ b/conan/api/subapi/upload.py @@ -6,7 +6,6 @@ from conan.api.model import PackagesList, Remote from conan.api.output import ConanOutput from conan.internal.api.upload import add_urls -from conan.internal.conan_app import ConanApp from conan.internal.api.uploader import PackagePreparator, UploadExecutor, UploadUpstreamChecker from conan.internal.rest.pkg_sign import PkgSignaturesPlugin from conan.internal.rest.file_uploader import FileUploader @@ -36,11 +35,11 @@ def check_upstream(self, package_list: PackagesList, remote: Remote, :parameter force: If ``True``, it will skip the check and mark that all items need to be uploaded. A ``force_upload`` key will be added to the entries that will be uploaded. """ - app = ConanApp(self._conan_api) + _, _, loader = self._api_helpers.get_loader() for ref, _ in package_list.items(): layout = self._api_helpers.cache.recipe_layout(ref) conanfile_path = layout.conanfile() - conanfile = app.loader.load_basic(conanfile_path, remotes=enabled_remotes) + conanfile = loader.load_basic(conanfile_path, remotes=enabled_remotes) if conanfile.upload_policy == "skip": ConanOutput().info(f"{ref}: Skipping upload of binaries, " "because upload_policy='skip'") @@ -63,8 +62,9 @@ def prepare(self, package_list: PackagesList, enabled_remotes: List[Remote], it means that no metadata files should be uploaded.""" if metadata and metadata != [''] and '' in metadata: raise ConanException("Empty string and patterns can not be mixed for metadata.") - app = ConanApp(self._conan_api) - preparator = PackagePreparator(app, self._api_helpers.cache, + + _, _, loader = self._api_helpers.get_loader() + preparator = PackagePreparator(loader, self._api_helpers.cache, self._api_helpers.remote_manager, self._api_helpers.global_conf) preparator.prepare(package_list, enabled_remotes, metadata) diff --git a/conan/api/subapi/workspace.py b/conan/api/subapi/workspace.py index 0394f1829cd..00d7adeb912 100644 --- a/conan/api/subapi/workspace.py +++ b/conan/api/subapi/workspace.py @@ -10,7 +10,6 @@ from conan.cli import make_abs_path from conan.cli.printers.graph import print_graph_basic, print_graph_packages from conan.errors import ConanException -from conan.internal.conan_app import ConanApp from conan.internal.errors import conanfile_exception_formatter from conan.internal.graph.install_graph import ProfileArgs from conan.internal.methods import auto_language, auto_shared_fpic_config_options, \ @@ -142,16 +141,16 @@ def packages(self): def open(self, ref, remotes, cwd=None): cwd = cwd or os.getcwd() - app = ConanApp(self._conan_api) + proxy, _, loader = self._conan_api._api_helpers.get_loader() # noqa ref = RecipeReference.loads(ref) if isinstance(ref, str) else ref - recipe = app.proxy.get_recipe(ref, remotes, update=False, check_update=False) + recipe = proxy.get_recipe(ref, remotes, update=False, check_update=False) layout, recipe_status, remote = recipe if recipe_status == RECIPE_EDITABLE: raise ConanException(f"Can't open a dependency that is already an editable: {ref}") ref = layout.reference conanfile_path = layout.conanfile() - conanfile, module = app.loader.load_basic_module(conanfile_path, remotes=remotes) + conanfile, module = loader.load_basic_module(conanfile_path, remotes=remotes) scm = conanfile.conan_data.get("scm") if conanfile.conan_data else None dst_path = os.path.join(cwd, ref.name) @@ -194,8 +193,8 @@ def add(self, path, name=None, version=None, user=None, channel=None, cwd=None, """ self._check_ws() full_path = self._conan_api.local.get_conanfile_path(path, cwd, py=True) - app = ConanApp(self._conan_api) - conanfile = app.loader.load_named(full_path, name, version, user, channel, remotes=remotes) + _, _, loader = self._conan_api._api_helpers.get_loader() # noqa + conanfile = loader.load_named(full_path, name, version, user, channel, remotes=remotes) if conanfile.name is None or conanfile.version is None: raise ConanException("Editable package recipe should declare its name and version") ref = RecipeReference(conanfile.name, conanfile.version, conanfile.user, conanfile.channel) @@ -330,7 +329,8 @@ def find_folder(ref): if root_class is not None: conanfile = root_class(f"{WORKSPACE_PY} base project Conanfile") # To inject things like cmd_wrapper to the consumer conanfile, so self.run() works - helpers = ConanApp(self._conan_api).loader._conanfile_helpers # noqa + _, _, loader = self._conan_api._api_helpers.get_loader() # noqa + helpers = loader._conanfile_helpers # noqa conanfile._conan_helpers = helpers conanfile._conan_is_consumer = True initialize_conanfile_profile(conanfile, profile_build, profile_host, CONTEXT_HOST, diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index a5b70603224..9de51e9232a 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -6,7 +6,6 @@ import tarfile import time -from conan.internal.conan_app import ConanApp from conan.api.output import ConanOutput from conan.internal.source import retrieve_exports_sources from conan.internal.errors import NotFoundException @@ -104,8 +103,8 @@ def get_compress_level(compressformat, global_conf): class PackagePreparator: - def __init__(self, app: ConanApp, cache, remote_manager, global_conf): - self._app = app + def __init__(self, loader, cache, remote_manager, global_conf): + self._loader = loader self._remote_manager = remote_manager self._cache = cache self._global_conf = global_conf @@ -122,7 +121,7 @@ def prepare(self, pkg_list, enabled_remotes, metadata, force=False): for ref, packages in pkg_list.items(): recipe_layout = self._cache.recipe_layout(ref) conanfile_path = recipe_layout.conanfile() - conanfile = self._app.loader.load_basic(conanfile_path) + conanfile = self._loader.load_basic(conanfile_path) url = conanfile.conan_data.get("scm", {}).get("url") if conanfile.conan_data else None if local_url != "allow" and url is not None: if not any(url.startswith(v) for v in ("ssh", "git", "http", "file")): diff --git a/conan/internal/conan_app.py b/conan/internal/conan_app.py index 3dba2e69b31..6abdde4cf47 100644 --- a/conan/internal/conan_app.py +++ b/conan/internal/conan_app.py @@ -2,7 +2,6 @@ from conan.internal.api.local.editable import EditablePackages from conan.internal.cache.cache import PkgCache -from conan.internal.cache.home_paths import HomePaths from conan.internal.model.conf import ConfDefinition from conan.internal.graph.proxy import ConanProxy from conan.internal.graph.python_requires import PyRequireLoader diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index 4a62993c81d..01fab2cc5b9 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -16,7 +16,6 @@ from conan.internal.model.profile import Profile from conan.internal.model.version import Version from conan.internal.runner.output import RunnerOutput -from conan.internal.conan_app import ConanApp class _ContainerConfig(NamedTuple): @@ -245,9 +244,9 @@ def _run_command(self, command: str, workdir: Optional[str] = None, verbose: boo return stdout_log, stderr_log def _get_volumes_and_docker_path(self) -> tuple[dict, str]: - app = ConanApp(self.conan_api) + _, _, loader = self.conan_api._api_helpers.get_loader() # noqa remotes = self.conan_api.remotes.list(self.args.remote) if not self.args.no_remote else [] - conanfile = app.loader.load_consumer(self.abs_host_path / "conanfile.py", remotes=remotes) + conanfile = loader.load_consumer(self.abs_host_path / "conanfile.py", remotes=remotes) abs_docker_base_path = Path('/') / self.docker_user_name / 'conanrunner' # Check if recipe has defined a root folder # In this case, mount the root folder as the base path and update the abs_docker_path to the diff --git a/test/integration/conan_api/test_local_api.py b/test/integration/conan_api/test_local_api.py index ce728fd58c8..078899cb418 100644 --- a/test/integration/conan_api/test_local_api.py +++ b/test/integration/conan_api/test_local_api.py @@ -13,6 +13,6 @@ def test_local_api(): cache_folder = temp_folder() save(os.path.join(current_folder, "conanfile.py"), str(GenConanfile("foo", "1.0"))) api = ConanAPI(cache_folder) - assert api.local.editable_packages.edited_refs == {} + assert api._api_helpers.editable_packages.edited_refs == {} api.local.editable_add(".", cwd=current_folder) - assert list(api.local.editable_packages.edited_refs) == [RecipeReference.loads("foo/1.0")] + assert list(api._api_helpers.editable_packages.edited_refs) == [RecipeReference.loads("foo/1.0")] From 102fa4175ef45badfa069f3536bb5395956efcec Mon Sep 17 00:00:00 2001 From: James Date: Fri, 10 Apr 2026 10:11:20 +0200 Subject: [PATCH 087/110] Internal improvements (#19852) * wip * wip * wip --- conan/internal/api/migrations.py | 84 +++++++++++++++++- conan/internal/rest/download_cache.py | 2 +- conans/migrations.py | 86 ------------------- conans/server/migrations.py | 2 +- .../test_cmakeconfigdeps_aliases.py | 37 ++++---- .../test_cmakeconfigdeps_new.py | 86 +++++++++---------- test/integration/cache/backup_sources_test.py | 50 +++++++---- .../command/cache/test_cache_sign.py | 1 + .../command/info/test_graph_info_graphical.py | 3 +- .../conanfile/test_finalize_method.py | 2 +- test/integration/remote/retry_test.py | 2 +- .../cmakeconfigdeps/test_cmakeconfigdeps.py | 79 ++++++++--------- .../test_cmakeconfigdeps_frameworks.py | 9 +- .../microsoft/test_nmaketoolchain.py | 1 - .../tools/system/package_manager_test.py | 18 ++-- .../client/migrations/test_migrator.py | 2 +- 16 files changed, 230 insertions(+), 234 deletions(-) delete mode 100644 conans/migrations.py diff --git a/conan/internal/api/migrations.py b/conan/internal/api/migrations.py index e1ff0b7abec..cd59605fb29 100644 --- a/conan/internal/api/migrations.py +++ b/conan/internal/api/migrations.py @@ -1,9 +1,12 @@ import os import textwrap +from conan import conan_version from conan.api.output import ConanOutput +from conan.errors import ConanException, ConanMigrationError from conan.internal.default_settings import migrate_settings_file -from conans.migrations import Migrator +from conan.internal.loader import load_python_file +from conan.internal.model.version import Version from conan.internal.util.files import load, save CONAN_GENERATED_COMMENT = "This file was generated by Conan" @@ -32,6 +35,85 @@ def update_file(file_path, new_content): out.success(f"Migration: Successfully updated {file_name}") +CONAN_VERSION = "version.txt" + + +class Migrator: + + def __init__(self, conf_path, current_version): + self.conf_path = conf_path + + self.current_version = current_version + self.file_version_path = os.path.join(self.conf_path, CONAN_VERSION) + + def migrate(self): + try: + old_version = self._load_old_version() + if old_version is None or old_version < self.current_version: + self._apply_migrations(old_version) + self._update_version_file() + elif self.current_version < old_version: # backwards migrations + ConanOutput().warning(f"Downgrading cache from Conan {old_version} to " + f"{self.current_version}") + self._apply_back_migrations() + self._update_version_file() + except Exception as e: + ConanOutput().error(str(e), error_type="exception") + raise ConanMigrationError(e) + + def _update_version_file(self): + try: + save(self.file_version_path, str(self.current_version)) + except Exception as error: + raise ConanException("Can't write version file in '{}': {}" + .format(self.file_version_path, str(error))) + + def _load_old_version(self): + try: + tmp = load(self.file_version_path) + old_version = Version(tmp) + except Exception: + old_version = None + return old_version + + def _apply_migrations(self, old_version): + """ + Apply any migration script. + + :param old_version: ``str`` previous Conan version. + """ + pass + + def _apply_back_migrations(self): + migrations = os.path.join(self.conf_path, "migrations") + if not os.path.exists(migrations): + return + + # Order by versions, and filter only newer than the current version + migration_files = [] + for f in os.listdir(migrations): + if not f.endswith(".py"): + continue + version, remain = f.split("_", 1) + version = Version(version) + if version > conan_version: + migration_files.append((version, remain)) + migration_files = [f"{v}_{r}" for (v, r) in reversed(sorted(migration_files))] + + for migration in migration_files: + ConanOutput().warning(f"Applying downgrade migration {migration}") + migration = os.path.join(migrations, migration) + try: + migrate_module, _ = load_python_file(migration) + migrate_method = migrate_module.migrate + migrate_method(self.conf_path) + except Exception as e: + ConanOutput().error(f"There was an error running downgrade migration: {e}. " + f"Recommended to remove the cache and start from scratch", + error_type="exception") + os.remove(migration) + + class ClientMigrator(Migrator): def __init__(self, cache_folder, current_version): diff --git a/conan/internal/rest/download_cache.py b/conan/internal/rest/download_cache.py index 5e366084470..5aeddb5b72d 100644 --- a/conan/internal/rest/download_cache.py +++ b/conan/internal/rest/download_cache.py @@ -139,7 +139,7 @@ def update_backup_sources_json(cached_path, conanfile, urls): urls = [urls] existing_urls = summary["references"].setdefault(summary_key, []) existing_urls.extend(url for url in urls if url not in existing_urls) - conanfile.output.verbose(f"Updating ${summary_path} summary file") + conanfile.output.verbose(f"Updating {summary_path} summary file") summary_dump = json.dumps(summary) conanfile.output.debug(f"New summary: ${summary_dump}") save(summary_path, json.dumps(summary)) diff --git a/conans/migrations.py b/conans/migrations.py deleted file mode 100644 index 3efa116536c..00000000000 --- a/conans/migrations.py +++ /dev/null @@ -1,86 +0,0 @@ -import os - -from conan import conan_version -from conan.api.output import ConanOutput -from conan.internal.loader import load_python_file -from conan.errors import ConanException, ConanMigrationError -from conan.internal.model.version import Version -from conan.internal.util.files import load, save - -CONAN_VERSION = "version.txt" - - -class Migrator: - - def __init__(self, conf_path, current_version): - self.conf_path = conf_path - - self.current_version = current_version - self.file_version_path = os.path.join(self.conf_path, CONAN_VERSION) - - def migrate(self): - try: - old_version = self._load_old_version() - if old_version is None or old_version < self.current_version: - self._apply_migrations(old_version) - self._update_version_file() - elif self.current_version < old_version: # backwards migrations - ConanOutput().warning(f"Downgrading cache from Conan {old_version} to " - f"{self.current_version}") - self._apply_back_migrations() - self._update_version_file() - except Exception as e: - ConanOutput().error(str(e), error_type="exception") - raise ConanMigrationError(e) - - def _update_version_file(self): - try: - save(self.file_version_path, str(self.current_version)) - except Exception as error: - raise ConanException("Can't write version file in '{}': {}" - .format(self.file_version_path, str(error))) - - def _load_old_version(self): - try: - tmp = load(self.file_version_path) - old_version = Version(tmp) - except Exception: - old_version = None - return old_version - - def _apply_migrations(self, old_version): - """ - Apply any migration script. - - :param old_version: ``str`` previous Conan version. - """ - pass - - def _apply_back_migrations(self): - migrations = os.path.join(self.conf_path, "migrations") - if not os.path.exists(migrations): - return - - # Order by versions, and filter only newer than the current version - migration_files = [] - for f in os.listdir(migrations): - if not f.endswith(".py"): - continue - version, remain = f.split("_", 1) - version = Version(version) - if version > conan_version: - migration_files.append((version, remain)) - migration_files = [f"{v}_{r}" for (v, r) in reversed(sorted(migration_files))] - - for migration in migration_files: - ConanOutput().warning(f"Applying downgrade migration {migration}") - migration = os.path.join(migrations, migration) - try: - migrate_module, _ = load_python_file(migration) - migrate_method = migrate_module.migrate - migrate_method(self.conf_path) - except Exception as e: - ConanOutput().error(f"There was an error running downgrade migration: {e}. " - f"Recommended to remove the cache and start from scratch", - error_type="exception") - os.remove(migration) diff --git a/conans/server/migrations.py b/conans/server/migrations.py index ff0eef353c4..0af576824e2 100644 --- a/conans/server/migrations.py +++ b/conans/server/migrations.py @@ -1,4 +1,4 @@ -from conans.migrations import Migrator +from conan.internal.api.migrations import Migrator class ServerMigrator(Migrator): diff --git a/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_aliases.py b/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_aliases.py index b09dd22071a..1a4b4c8c968 100644 --- a/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_aliases.py +++ b/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_aliases.py @@ -3,7 +3,6 @@ from conan.test.utils.tools import TestClient -new_value = "will_break_next" consumer = textwrap.dedent(""" from conan import ConanFile @@ -13,7 +12,7 @@ class Consumer(ConanFile): name = "consumer" version = "1.0" settings = "os", "compiler", "build_type", "arch" - generators = "CMakeDeps", "CMakeToolchain" + generators = "CMakeConfigDeps", "CMakeToolchain" exports_sources = ["CMakeLists.txt"] requires = "hello/1.0" @@ -50,10 +49,10 @@ def package_info(self): client = TestClient() client.save({"conanfile.py": conanfile}) - client.run(f"create . -c tools.cmake.cmakedeps:new={new_value}") + client.run(f"create .") client.save({"conanfile.py": consumer, "CMakeLists.txt": cmakelists}) - client.run(f"create . -c tools.cmake.cmakedeps:new={new_value}") + client.run(f"create .") assert "hello aliased target: hello::hello" in client.out @@ -84,10 +83,10 @@ def package_info(self): client = TestClient() client.save({"conanfile.py": conanfile}) - client.run(f"create . -c tools.cmake.cmakedeps:new={new_value}") + client.run(f"create .") client.save({"conanfile.py": consumer, "CMakeLists.txt": cmakelists}) - client.run(f"create . -c tools.cmake.cmakedeps:new={new_value}") + client.run(f"create .") assert "hola::adios aliased target: hello::buy" in client.out @@ -119,10 +118,10 @@ def package_info(self): client = TestClient() client.save({"conanfile.py": conanfile}) - client.run(f"create . -c tools.cmake.cmakedeps:new={new_value}") + client.run(f"create .") client.save({"conanfile.py": consumer, "CMakeLists.txt": cmakelists}) - client.run(f"create . -c tools.cmake.cmakedeps:new={new_value}") + client.run(f"create .") assert "hola::hola aliased target: hello::hello" in client.out @@ -153,10 +152,10 @@ def package_info(self): client = TestClient() client.save({"conanfile.py": conanfile}) - client.run(f"create . -c tools.cmake.cmakedeps:new={new_value}") + client.run(f"create .") client.save({"conanfile.py": consumer, "CMakeLists.txt": cmakelists}) - client.run(f"create . -c tools.cmake.cmakedeps:new={new_value}") + client.run(f"create .") assert "hello aliased target: ola::comprar" in client.out @@ -184,10 +183,10 @@ def package_info(self): client = TestClient() client.save({"conanfile.py": conanfile}) - client.run(f"create . -c tools.cmake.cmakedeps:new={new_value}") + client.run(f"create .") client.save({"conanfile.py": consumer, "CMakeLists.txt": cmakelists}) - client.run(f"create . -c tools.cmake.cmakedeps:new={new_value}", assert_error=True) + client.run(f"create .", assert_error=True) assert "Alias 'hello::buy' already defined as a target in hello/1.0" in client.out @@ -216,10 +215,10 @@ def package_info(self): client = TestClient() client.save({"conanfile.py": conanfile}) - client.run(f"create . -c tools.cmake.cmakedeps:new={new_value}") + client.run(f"create .") client.save({"conanfile.py": consumer, "CMakeLists.txt": cmakelists}) - client.run(f"create . -c tools.cmake.cmakedeps:new={new_value}", assert_error=True) + client.run(f"create .", assert_error=True) assert "Alias 'hello::foo' already defined in hello/1.0" in client.out @@ -227,7 +226,8 @@ def package_info(self): @pytest.mark.tool("cmake", "3.27") @pytest.mark.parametrize("root_target", ["hello::custom", None]) def test_skip_global_if_aliased(root_target): - target_line = f'self.cpp_info.set_property("cmake_target_name", "{root_target}")' if root_target else "" + target_line = f'self.cpp_info.set_property("cmake_target_name", "{root_target}")' \ + if root_target else "" target_name = root_target or "hello::hello" conanfile = textwrap.dedent(f""" from conan import ConanFile @@ -253,9 +253,10 @@ def package_info(self): client = TestClient() client.save({"conanfile.py": conanfile}) - client.run(f"create . -c tools.cmake.cmakedeps:new={new_value}") + client.run(f"create .") client.save({"conanfile.py": consumer, "CMakeLists.txt": cmakelists}) - client.run(f"create . -c tools.cmake.cmakedeps:new={new_value}", assert_error=True) + client.run(f"create .", assert_error=True) - assert f"Can't define an alias '{target_name}' for the root target '{target_name}' in hello/1.0" in client.out + assert (f"Can't define an alias '{target_name}' for the root " + f"target '{target_name}' in hello/1.0") in client.out diff --git a/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_new.py b/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_new.py index faaf91787fc..03e8c119942 100644 --- a/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_new.py +++ b/test/functional/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_new.py @@ -71,13 +71,13 @@ def package_info(self): for requires in ("tool_requires", "requires"): consumer = textwrap.dedent(f""" from conan import ConanFile - from conan.tools.cmake import CMakeDeps, CMakeToolchain, CMake, cmake_layout + from conan.tools.cmake import CMakeConfigDeps, CMakeToolchain, CMake, cmake_layout class Consumer(ConanFile): settings = "os", "compiler", "arch", "build_type" {requires} = "mytool/0.1" def generate(self): - deps = CMakeDeps(self) + deps = CMakeConfigDeps(self) deps.generate() tc = CMakeToolchain(self) tc.generate() @@ -101,7 +101,7 @@ def build(self): """) c.save({f"consumer_{requires}/conanfile.py": consumer, f"consumer_{requires}/CMakeLists.txt": cmake}) - c.run(f"build consumer_{requires} -c tools.cmake.cmakedeps:new={new_value}") + c.run(f"build consumer_{requires}") assert "find_package(mytool)" in c.out assert "target_link_libraries(..." not in c.out assert "Conan: Target declared imported executable 'MyTool::myexe'" in c.out @@ -169,13 +169,13 @@ def package_info(self): consumer = textwrap.dedent(""" from conan import ConanFile - from conan.tools.cmake import CMakeDeps, CMakeToolchain, CMake, cmake_layout + from conan.tools.cmake import CMakeConfigDeps, CMakeToolchain, CMake, cmake_layout class Consumer(ConanFile): settings = "os", "compiler", "arch", "build_type" tool_requires = "mytool/0.1" def generate(self): - deps = CMakeDeps(self) + deps = CMakeConfigDeps(self) deps.generate() tc = CMakeToolchain(self) tc.generate() @@ -199,7 +199,7 @@ def build(self): """) c.save({"conanfile.py": consumer, "CMakeLists.txt": cmake}, clean_first=True) - c.run(f"build . -c tools.cmake.cmakedeps:new={new_value}") + c.run(f"build .") assert "Conan: Target declared imported executable 'MyTool::my1exe'" in c.out assert "Mytool1 generating out1.c!!!!!" in c.out assert "Conan: Target declared imported executable 'MyTool::my2exe'" in c.out @@ -324,9 +324,8 @@ def package_info(self): .with_settings("build_type")}) c.run("create dep") - c.run(f"install app -c tools.cmake.cmakedeps:new={new_value} -g CMakeDeps", - assert_error=True) - assert "ERROR: Error in generator 'CMakeDeps': dep/0.1: Cannot obtain 'location' " \ + c.run(f"install app -g CMakeConfigDeps", assert_error=True) + assert "ERROR: Error in generator 'CMakeConfigDeps': dep/0.1: Cannot obtain 'location' " \ "for library 'dep'" in c.out def test_custom_file_targetname(self): @@ -350,7 +349,7 @@ def package_info(self): c.run("create dep") c.run("create pkg") - c.run(f"install app -c tools.cmake.cmakedeps:new={new_value} -g CMakeDeps") + c.run(f"install app -g CMakeConfigDeps") targets_cmake = c.load("app/pkg-Targets-release.cmake") assert "find_dependency(MyDep REQUIRED CONFIG)" in targets_cmake assert 'set_property(TARGET pkg::pkg APPEND PROPERTY INTERFACE_LINK_LIBRARIES\n' \ @@ -772,7 +771,7 @@ class Matrix(ConanFile): generators = "CMakeToolchain" exports_sources = "src/*", "CMakeLists.txt" - generators = "CMakeDeps", "CMakeToolchain" + generators = "CMakeConfigDeps", "CMakeToolchain" def build(self): cmake = CMake(self) @@ -902,7 +901,7 @@ def package_info(self): class EngineHeader(ConanFile): settings = "os", "compiler", "build_type", "arch" requires = "engine/1.0" - generators = "CMakeDeps", "CMakeToolchain" + generators = "CMakeConfigDeps", "CMakeToolchain" def build(self): cmake = CMake(self) cmake.configure() @@ -921,7 +920,7 @@ def build(self): "CMakeLists.txt": cmake, "src/app.cpp": gen_function_cpp(name="main", includes=["engine"], calls=["engine"])}, clean_first=True) - c.run(f"build . -c tools.cmake.cmakedeps:new={new_value}") + c.run(f"build .") assert "Conan: Target declared imported STATIC library 'matrix::matrix'" in c.out assert "Conan: Target declared imported INTERFACE library 'engine::engine'" in c.out @@ -962,7 +961,7 @@ def package_info(self): from conan.tools.cmake import CMake, cmake_layout class EngineHeader(ConanFile): settings = "os", "compiler", "build_type", "arch" - generators = "CMakeDeps", "CMakeToolchain" + generators = "CMakeConfigDeps", "CMakeToolchain" def requirements(self): v = "1_0" if self.settings.build_type == "Debug" else "1_1" self.requires(f"engine/{v}") @@ -987,13 +986,13 @@ def build(self): "CMakeLists.txt": cmake, "src/app.cpp": gen_function_cpp(name="main", includes=["engine"], calls=["engine"])}, clean_first=True) - c.run(f"build . -c tools.cmake.cmakedeps:new={new_value}") + c.run(f"build .") assert "engine/1_1" in c.out assert "engine/1_0" not in c.out assert "Conan: Target declared imported INTERFACE library 'engine::engine'" in c.out assert "Engine 1_1!" in c.out - c.run(f"build . -c tools.cmake.cmakedeps:new={new_value} -s build_type=Debug") + c.run(f"build . -s build_type=Debug") assert "engine/1_1" not in c.out assert "engine/1_0" in c.out assert "Conan: Target declared imported INTERFACE library 'engine::engine'" in c.out @@ -1010,7 +1009,7 @@ def test_tool_requires(self): "pkg/conanfile.py": GenConanfile("pkg", "0.1").with_settings("build_type") .with_tool_requires("tool/0.1")}) c.run("create tool") - c.run(f"install pkg -g CMakeDeps -c tools.cmake.cmakedeps:new={new_value}") + c.run(f"install pkg -g CMakeConfigDeps") assert "find_package(tool) # Optional. This is a tool-require, " \ "can't link its targets" in c.out assert "target_link_libraries" not in c.out @@ -1070,7 +1069,7 @@ def package_info(self): class Conan(ConanFile): settings = "os", "compiler", "build_type", "arch" - generators = "CMakeToolchain", "CMakeDeps" + generators = "CMakeToolchain", "CMakeConfigDeps" {requires} = "myfunctions/1.0" def build(self): @@ -1086,7 +1085,7 @@ def build(self): client.save({"conanfile.py": consumer, "CMakeLists.txt": cmakelists}, clean_first=True) - client.run(f"build . -c tools.cmake.cmakedeps:new={new_value}") + client.run(f"build .") assert "Hello myfunction!!!!" in client.out @@ -1170,7 +1169,7 @@ def package_info(self): class Consumer(ConanFile): settings = "os", "compiler", "arch", "build_type" requires = "protobuf/0.1" - generators = "CMakeToolchain", "CMakeDeps" + generators = "CMakeToolchain", "CMakeConfigDeps" def layout(self): cmake_layout(self) @@ -1209,7 +1208,7 @@ def build(self): def test_requires(self, protobuf): c = protobuf - c.run(f"build . --build=missing -c tools.cmake.cmakedeps:new={new_value}") + c.run(f"build . --build=missing") assert "Conan: Target declared imported STATIC library 'protobuf::protobuf'" in c.out assert "Conan: Target declared imported executable 'Protobuf::Protocompile'" in c.out assert "Protoc RELEASE generating out.c!!!!!" in c.out @@ -1224,7 +1223,7 @@ class Consumer(ConanFile): settings = "os", "compiler", "arch", "build_type" requires = "protobuf/0.1" tool_requires = "protobuf/0.1" - generators = "CMakeToolchain", "CMakeDeps" + generators = "CMakeToolchain", "CMakeConfigDeps" def layout(self): cmake_layout(self) @@ -1235,8 +1234,7 @@ def build(self): """) c = protobuf c.save({"conanfile.py": consumer}) - c.run("build . -s:h build_type=Debug --build=missing " - f"-c tools.cmake.cmakedeps:new={new_value}") + c.run("build . -s:h build_type=Debug --build=missing") assert "Conan: Target declared imported STATIC library 'protobuf::protobuf'" in c.out assert "Conan: Target declared imported executable 'Protobuf::Protocompile'" in c.out @@ -1250,8 +1248,7 @@ def build(self): assert "protobuf: Debug!" in c.out assert "protobuf: Release!" not in c.out - c.run("build . --build=missing " - f"-c tools.cmake.cmakedeps:new={new_value}") + c.run("build . --build=missing") assert "Conan: Target declared imported STATIC library 'protobuf::protobuf'" in c.out assert "Conan: Target declared imported executable 'Protobuf::Protocompile'" in c.out @@ -1393,13 +1390,13 @@ def test_check_c_source_compiles(self, matrix_client): consumer = textwrap.dedent(""" from conan import ConanFile - from conan.tools.cmake import CMakeDeps + from conan.tools.cmake import CMakeConfigDeps class PkgConan(ConanFile): settings = "os", "arch", "compiler", "build_type" requires = "matrix/1.0" generators = "CMakeToolchain", def generate(self): - deps = CMakeDeps(self) + deps = CMakeConfigDeps(self) deps.set_property("matrix", "cmake_additional_variables_prefixes", ["MyMatrix"]) deps.generate() """) @@ -1421,7 +1418,7 @@ def generate(self): c.save({"conanfile.py": consumer, "CMakeLists.txt": cmakelist}, clean_first=True) - c.run(f"install . -c tools.cmake.cmakedeps:new={new_value}") + c.run(f"install .") preset = "conan-default" if platform.system() == "Windows" else "conan-release" c.run_command(f"cmake --preset {preset} ") @@ -1451,7 +1448,7 @@ def package_info(self): class PkgConan(ConanFile): settings = "os", "arch", "compiler", "build_type" requires = "dep/0.1" - generators = "CMakeToolchain", "CMakeDeps" + generators = "CMakeToolchain", "CMakeConfigDeps" def build(self): deps = CMake(self) deps.configure() @@ -1466,7 +1463,7 @@ def build(self): c.save({"conanfile.py": consumer, "CMakeLists.txt": cmakelist}, clean_first=True) - c.run(f"build . -c tools.cmake.cmakedeps:new={new_value}", assert_error=not found) + c.run(f"build .", assert_error=not found) if not found: assert f"Conan: Error: 'dep' required COMPONENT '{components}' not found" in c.out @@ -1486,7 +1483,7 @@ def package_info(self): """) c.save({"conanfile.py": dep}) c.run("create .") - c.run(f"install --requires=dep/0.1 -g CMakeDeps -c tools.cmake.cmakedeps:new={new_value}") + c.run(f"install --requires=dep/0.1 -g CMakeConfigDeps") cmake = c.load("dep-config.cmake") assert 'set(dep_PACKAGE_PROVIDED_COMPONENTS MyC1 MyC2 c3)' in cmake @@ -1505,7 +1502,7 @@ def package_info(self): """) c.save({"conanfile.py": dep}) c.run("create .") - c.run(f"install --requires=dep/0.1 -g CMakeDeps -c tools.cmake.cmakedeps:new={new_value}") + c.run(f"install --requires=dep/0.1 -g CMakeConfigDeps") cmake = c.load("dep-config.cmake") assert 'set(dep_PACKAGE_PROVIDED_COMPONENTS MyCompC1 MyC2 c3)' in cmake @@ -1524,9 +1521,9 @@ def package_info(self): """) c.save({"conanfile.py": dep}) c.run("create .") - args = f"-g CMakeDeps -c tools.cmake.cmakedeps:new={new_value}" - c.run(f"install --requires=dep/0.1 {args}", assert_error=True) - assert "Error in generator 'CMakeDeps': dep/0.1 " 'cpp_info has both .exe and .libs' in c.out + c.run(f"install --requires=dep/0.1 -g CMakeConfigDeps", assert_error=True) + assert ("Error in generator 'CMakeConfigDeps': dep/0.1 " + 'cpp_info has both .exe and .libs' in c.out) def test_exe_no_location(self): c = TestClient() @@ -1540,9 +1537,9 @@ def package_info(self): """) c.save({"conanfile.py": dep}) c.run("create .") - args = f"-g CMakeDeps -c tools.cmake.cmakedeps:new={new_value}" - c.run(f"install --requires=dep/0.1 {args}", assert_error=True) - assert "Error in generator 'CMakeDeps': dep/0.1 cpp_info has .exe and no .location" in c.out + c.run(f"install --requires=dep/0.1 -g CMakeConfigDeps", assert_error=True) + assert ("Error in generator 'CMakeConfigDeps': " + "dep/0.1 cpp_info has .exe and no .location") in c.out def test_check_exe_wrong_type(self): c = TestClient() @@ -1557,8 +1554,7 @@ def package_info(self): """) c.save({"conanfile.py": dep}) c.run("create .") - args = f"-g CMakeDeps -c tools.cmake.cmakedeps:new={new_value}" - c.run(f"install --requires=dep/0.1 {args}", assert_error=True) + c.run(f"install --requires=dep/0.1 -g CMakeConfigDeps", assert_error=True) assert "dep/0.1 cpp_info incorrect .type shared-library for .exe myexe" in c.out @@ -1593,7 +1589,7 @@ def package_info(self): from conan.tools.cmake import CMake class Pkg(ConanFile): requires = "matrix/1.0" - generators = "CMakeToolchain", "CMakeDeps" + generators = "CMakeToolchain", "CMakeConfigDeps" settings = "os", "compiler", "build_type", "arch" def build(self): cmake = CMake(self) @@ -1603,7 +1599,7 @@ def build(self): "CMakeLists.txt": cmake, "subdir/CMakeLists.txt": subcmake}, clean_first=True) - c.run(f"build . -c tools.cmake.cmakedeps:new={new_value}") + c.run(f"build .") assert "find_package(matrix)" in c.out assert "target_link_libraries(... matrix::matrix)" in c.out assert "Conan: Target declared imported INTERFACE library 'matrix::matrix'" in c.out @@ -1630,7 +1626,7 @@ def test_find_package_casing_non_fallback(): from conan.tools.cmake import CMake, cmake_layout class Pkg(ConanFile): requires = "hello/1.0" - generators = "CMakeToolchain", "CMakeDeps" + generators = "CMakeToolchain", "CMakeConfigDeps" settings = "os", "compiler", "build_type", "arch" def layout(self): cmake_layout(self) @@ -1644,7 +1640,7 @@ def build(self): client.run(f"create .") client.save({"conanfile.py": consumer, "CMakeLists.txt": cmakelists}) - client.run(f"build . -c tools.cmake.cmakedeps:new={new_value}", assert_error=True) + client.run(f"build .", assert_error=True) assert 'Could not find a package configuration file provided by "HellO"' in client.out diff --git a/test/integration/cache/backup_sources_test.py b/test/integration/cache/backup_sources_test.py index c00c7de750f..f959b4c4796 100644 --- a/test/integration/cache/backup_sources_test.py +++ b/test/integration/cache/backup_sources_test.py @@ -190,8 +190,10 @@ def source(self): f"core.sources:upload_url={self.file_server.fake_url}/backups/\n" f"core.sources:exclude_urls=['{self.file_server.fake_url}/mycompanystorage/', '{self.file_server.fake_url}/mycompanystorage2/']"}) self.client.run("source .") - assert f"Sources for {self.file_server.fake_url}/internet/myfile.txt found in remote backup" in self.client.out - assert f"File {self.file_server.fake_url}/mycompanystorage/mycompanyfile.txt not found in {self.file_server.fake_url}/backups/" in self.client.out + assert (f"Sources for {self.file_server.fake_url}/internet/myfile.txt found in " + f"remote backup") in self.client.out + assert (f"File {self.file_server.fake_url}/mycompanystorage/mycompanyfile.txt not " + f"found in {self.file_server.fake_url}/backups/") in self.client.out # Ensure defaults backup folder works if it's not set in global.conf # (The rest is needed to exercise the rest of the code) @@ -199,8 +201,10 @@ def source(self): {"global.conf": f"core.sources:download_urls=['{self.file_server.fake_url}/backups/', 'origin']\n" f"core.sources:exclude_urls=['{self.file_server.fake_url}/mycompanystorage/', '{self.file_server.fake_url}/mycompanystorage2/']"}) self.client.run("source .") - assert f"Sources for {self.file_server.fake_url}/internet/myfile.txt found in remote backup" in self.client.out - assert f"File {self.file_server.fake_url}/mycompanystorage/mycompanyfile.txt not found in {self.file_server.fake_url}/backups/" in self.client.out + assert (f"Sources for {self.file_server.fake_url}/internet/myfile.txt found in " + f"remote backup") in self.client.out + assert (f"File {self.file_server.fake_url}/mycompanystorage/mycompanyfile.txt not " + f"found in {self.file_server.fake_url}/backups/") in self.client.out def test_unknown_handling(self): http_server_base_folder_internet = os.path.join(self.file_server.store, "internet") @@ -269,9 +273,11 @@ def source(self): rmdir(self.download_cache_folder) self.client.run("source .") - assert f"Sources for {self.file_server.fake_url}/internet/myfile.txt found in origin" in self.client.out + assert (f"Sources for {self.file_server.fake_url}/internet/myfile.txt found " + f"in origin") in self.client.out self.client.run("source .") - assert f"Source {self.file_server.fake_url}/internet/myfile.txt retrieved from local download cache" in self.client.out + assert (f"Source {self.file_server.fake_url}/internet/myfile.txt retrieved from " + f"local download cache") in self.client.out def test_download_origin_last(self): http_server_base_folder_internet = os.path.join(self.file_server.store, "internet") @@ -297,16 +303,18 @@ def source(self): self.client.save({"conanfile.py": conanfile}) self.client.run("create . -vv") - assert f"WARN: File {self.file_server.fake_url}/internet/myfile.txt not found in {self.file_server.fake_url}/backup/" in self.client.out - assert f"Downloaded {self.file_server.fake_url}/internet/myfile.txt from {self.file_server.fake_url}/internet/myfile.txt" + assert (f"WARN: File {self.file_server.fake_url}/internet/myfile.txt not found " + f"in {self.file_server.fake_url}/backup/") in self.client.out self.client.run("upload * -c -r=default") rmdir(self.download_cache_folder) self.client.run("source .") - assert f"Sources for {self.file_server.fake_url}/internet/myfile.txt found in remote backup" in self.client.out + assert (f"Sources for {self.file_server.fake_url}/internet/myfile.txt found in " + f"remote backup") in self.client.out self.client.run("source .") - assert f"Source {self.file_server.fake_url}/internet/myfile.txt retrieved from local download cache" in self.client.out + assert (f"Source {self.file_server.fake_url}/internet/myfile.txt retrieved from local " + f"download cache") in self.client.out def test_sources_backup_server_error_500(self): conanfile = textwrap.dedent(f""" @@ -498,7 +506,8 @@ def source(self): remove(os.path.join(http_server_base_folder_internet, "myfile.txt")) self.client.run("source .") - assert f"Sources for {self.file_server.fake_url}/internet/myfile.txt found in remote backup {self.file_server.fake_url}/backup/" in self.client.out + assert (f"Sources for {self.file_server.fake_url}/internet/myfile.txt found in remote " + f"backup {self.file_server.fake_url}/backup/") in self.client.out # And if the first one has them, prefer it before others in the list save(os.path.join(http_server_base_folder_downloader, sha256), @@ -507,7 +516,8 @@ def source(self): load(os.path.join(http_server_base_folder_backup, sha256 + ".json"))) rmdir(self.download_cache_folder) self.client.run("source .") - assert f"Sources for {self.file_server.fake_url}/internet/myfile.txt found in remote backup {self.file_server.fake_url}/downloader/" in self.client.out + assert (f"Sources for {self.file_server.fake_url}/internet/myfile.txt found in remote " + f"backup {self.file_server.fake_url}/downloader/") in self.client.out def test_list_urls_miss(self): def custom_download(this, url, *args, **kwargs): # noqa @@ -566,7 +576,8 @@ def source(self): self.client.save({"conanfile.py": conanfile}) self.client.run("create .") - assert f"Sources for {self.file_server.fake_url}/internal_error/myfile.txt found in remote backup {self.file_server.fake_url}/backup2/" in self.client.out + assert (f"Sources for {self.file_server.fake_url}/internal_error/myfile.txt " + f"found in remote backup {self.file_server.fake_url}/backup2/") in self.client.out def test_ok_when_origin_authorization_error(self): client = TestClient(default_server_user=True, light=True) @@ -591,7 +602,7 @@ def __init__(self, store=None): @staticmethod def _attach_to(app, store): # noqa @app.route("/internet/", method=["GET"]) - def get_internet_file(file): + def get_internet_file(_): return HTTPError(401, "You Are Not Allowed Here") @app.route("/downloader1/", method=["GET"]) @@ -631,11 +642,13 @@ def source(self): client.save({"conanfile.py": conanfile}) client.run("create .") - assert f"Sources for {http_server.fake_url}/internet/myfile.txt found in remote backup {http_server.fake_url}/downloader2/" in client.out + assert (f"Sources for {http_server.fake_url}/internet/myfile.txt found in remote" + f" backup {http_server.fake_url}/downloader2/") in client.out # TODO: Check better message with Authentication error message assert "failed in 'origin'" in client.out - # Now try to upload once to the first backup server. It's configured so it has write permissions but no overwrite + # Now try to upload once to the first backup server. It's configured so + # it has write permissions but no overwrite client.run("upload * -c -r=default") upload_server_contents = os.listdir(http_server_base_folder_backup1) assert sha256 in upload_server_contents @@ -674,7 +687,8 @@ def source(self): self.client.save({"conanfile.py": conanfile}) self.client.run("create .") - assert f"Sources for {self.file_server.fake_url}/internet/myfile.txt found in remote backup {self.file_server.fake_url}/backup2/" in self.client.out + assert (f"Sources for {self.file_server.fake_url}/internet/myfile.txt found in remote " + f"backup {self.file_server.fake_url}/backup2/") in self.client.out assert "sha256 hash failed for" in self.client.out def test_export_then_upload_workflow(self): @@ -811,7 +825,7 @@ def source(self): @pytest.mark.parametrize("exception", [Exception, ConanException]) @pytest.mark.parametrize("upload", [True, False]) def test_backup_source_dirty_download_handle(self, exception, upload): - def custom_download(this, *args, **kwargs): + def custom_download(this, *args, **kwargs): # noqa raise exception() http_server_base_folder_internet = os.path.join(self.file_server.store, "internet") diff --git a/test/integration/command/cache/test_cache_sign.py b/test/integration/command/cache/test_cache_sign.py index 24767d3e89b..887e5438aab 100644 --- a/test/integration/command/cache/test_cache_sign.py +++ b/test/integration/command/cache/test_cache_sign.py @@ -20,6 +20,7 @@ def verify(ref, artifacts_folder, signature_folder, files, **kwargs): pass """) + def test_pkg_sign_no_plugin(): c = TestClient() c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) diff --git a/test/integration/command/info/test_graph_info_graphical.py b/test/integration/command/info/test_graph_info_graphical.py index f757368c01f..da75b6decdc 100644 --- a/test/integration/command/info/test_graph_info_graphical.py +++ b/test/integration/command/info/test_graph_info_graphical.py @@ -173,7 +173,8 @@ def test_graph_conflict_loop(): c.run("export lib_c") c.run("graph info lib_x --format=html", assert_error=True, redirect_stdout="graph.html") # checked manually - #c.open("graph.html") + # c.open("graph.html") + def test_graph_missing_error(): c = TestClient() diff --git a/test/integration/conanfile/test_finalize_method.py b/test/integration/conanfile/test_finalize_method.py index 568c9d03af7..a48fdc179ac 100644 --- a/test/integration/conanfile/test_finalize_method.py +++ b/test/integration/conanfile/test_finalize_method.py @@ -318,7 +318,6 @@ def test_test_package_uses_created_tool_which_modifies_pkgfolder(self): tc.run(f"cache check-integrity {app_layout.reference}") assert "There are corrupted artifacts" not in tc.out - def test_multiple_instances_of_finalized_package(self): tc = TestClient(light=True) tc.save({"tool/conanfile.py": GenConanfile("tool", "1.0") @@ -340,6 +339,7 @@ def test_multiple_instances_of_finalized_package(self): # Finalize folder conan output should be printed only once assert tc.out.count("Finalized folder ") == 1 + class TestRemoteFlows: @pytest.fixture diff --git a/test/integration/remote/retry_test.py b/test/integration/remote/retry_test.py index 09316cda7a3..93e9cb358e7 100644 --- a/test/integration/remote/retry_test.py +++ b/test/integration/remote/retry_test.py @@ -33,7 +33,7 @@ def __init__(self, status_code, content): self.retry = 0 self.retry_wait = 0 - def put(self, *args, **kwargs): + def put(self, *args, **kwargs): # noqa return self.response diff --git a/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps.py b/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps.py index fa2afa42f40..b12fffac0b0 100644 --- a/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps.py +++ b/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps.py @@ -4,8 +4,6 @@ from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient -new_value = "will_break_next" - def test_cmakedeps_direct_deps_paths(): c = TestClient() @@ -29,10 +27,10 @@ def package_info(self): class PkgConan(ConanFile): requires = "lib/1.0" settings = "os", "arch", "compiler", "build_type" - generators = "CMakeDeps" + generators = "CMakeConfigDeps" """) c.save({"conanfile.py": conanfile}, clean_first=True) - c.run(f"install . -c tools.cmake.cmakedeps:new={new_value}") + c.run(f"install .") cmake_paths = c.load("conan_cmakedeps_paths.cmake") assert "set(CMAKE_FIND_PACKAGE_PREFER_CONFIG ON)" in cmake_paths assert re.search(r"list\(PREPEND CMAKE_PROGRAM_PATH \".*/bin\"", cmake_paths) # default @@ -77,10 +75,10 @@ def package_info(self): class PkgConan(ConanFile): requires = "libb/1.0" settings = "os", "arch", "compiler", "build_type" - generators = "CMakeDeps" + generators = "CMakeConfigDeps" """) c.save({"conanfile.py": conanfile}, clean_first=True) - c.run(f"install . -c tools.cmake.cmakedeps:new={new_value}") + c.run(f"install .") cmake_paths = c.load("conan_cmakedeps_paths.cmake") assert re.search(r"list\(PREPEND CMAKE_PROGRAM_PATH \".*/libb.*/p/binb\"\)", cmake_paths) assert not re.search(r"list\(PREPEND CMAKE_PROGRAM_PATH /bina\"", cmake_paths) @@ -132,12 +130,12 @@ def package_info(self): class PkgConan(ConanFile): requires = "liba/1.0", "libb/1.0" settings = "os", "arch", "compiler", "build_type" - generators = "CMakeDeps" + generators = "CMakeConfigDeps" """) c.save({"conanfile.py": conanfile}, clean_first=True) # Now with a deployment - c.run(f"install . -c tools.cmake.cmakedeps:new={new_value} --deployer=full_deploy") + c.run(f"install . --deployer=full_deploy") cmake_paths = c.load("conan_cmakedeps_paths.cmake") assert 'set(libb_DIR "${CMAKE_CURRENT_LIST_DIR}/full_deploy/host/libb/1.0")' in cmake_paths assert ('set(CONAN_RUNTIME_LIB_DIRS "$<$:${CMAKE_CURRENT_LIST_DIR}' @@ -214,8 +212,7 @@ def package_info(self): c.save({"conanfile.py": conanfile}) c.run("create .") - c.run(f"install --requires=lib/system -g CMakeConfigDeps " - f"-c tools.cmake.cmakedeps:new={new_value}") + c.run(f"install --requires=lib/system -g CMakeConfigDeps") cmake = c.load("lib-Targets-release.cmake") assert "add_library(lib::lib INTERFACE IMPORTED)" in cmake assert "set_property(TARGET lib::lib APPEND PROPERTY INTERFACE_LINK_LIBRARIES\n" \ @@ -234,10 +231,10 @@ def package_info(self): c.save({"conanfile.py": conanfile, "test_package/conanfile.py": GenConanfile().with_test("pass") .with_settings("build_type") - .with_generator("CMakeDeps")}) + .with_generator("CMakeConfigDeps")}) c.run("create . --name=pkg --version=0.1") - assert "CMakeDeps: cmake_set_interface_link_directories is legacy, not necessary" in c.out - c.run(f"create . --name=pkg --version=0.1 -c tools.cmake.cmakedeps:new={new_value}") + assert "CMakeConfigDeps: cmake_set_interface_link_directories deprecated" in c.out + c.run(f"create . --name=pkg --version=0.1") assert "CMakeConfigDeps: cmake_set_interface_link_directories deprecated and invalid. " \ "The package 'package_info()' must correctly define the (CPS) information" in c.out @@ -255,8 +252,8 @@ def package_info(self): c.save({"conanfile.py": conanfile, "test_package/conanfile.py": GenConanfile().with_settings("build_type") .with_test("pass") - .with_generator("CMakeDeps")}) - c.run(f"create . --name=pkg --version=0.1 -c tools.cmake.cmakedeps:new={new_value}") + .with_generator("CMakeConfigDeps")}) + c.run(f"create . --name=pkg --version=0.1") # it doesn't break assert "find_package(pkg)" in c.out @@ -283,9 +280,9 @@ def package_info(self): "pkg/conanfile.py": conanfile, "pkg/test_package/conanfile.py": GenConanfile().with_settings("build_type") .with_test("pass") - .with_generator("CMakeDeps")}) + .with_generator("CMakeConfigDeps")}) c.run("create dep") - c.run(f"create pkg --name=pkg --version=0.1 -c tools.cmake.cmakedeps:new={new_value}") + c.run(f"create pkg --name=pkg --version=0.1") # it doesn't break assert "find_package(pkg)" in c.out @@ -303,12 +300,11 @@ def package_info(self): c.save({"dep/conanfile.py": GenConanfile("dep", "0.1"), "pkg/conanfile.py": conanfile, "pkg/test_package/conanfile.py": GenConanfile().with_settings("build_type") - .with_generator("CMakeDeps") + .with_generator("CMakeConfigDeps") .with_test("pass")}) c.run("create dep") - c.run(f"create pkg --name=pkg --version=0.1 -c tools.cmake.cmakedeps:new={new_value}", - assert_error=True) - assert ("ERROR: Error in generator 'CMakeDeps': pkg/0.1 recipe cpp_info did .requires to " + c.run(f"create pkg --name=pkg --version=0.1", assert_error=True) + assert ("ERROR: Error in generator 'CMakeConfigDeps': pkg/0.1 recipe cpp_info did .requires to " "'dep::lib' but component 'lib' not found in dep") in c.out @@ -335,7 +331,7 @@ def package_info(self): from conan import ConanFile class TestPkg(ConanFile): settings = "os", "compiler", "arch", "build_type" - generators = "VirtualRunEnv", "CMakeDeps" + generators = "VirtualRunEnv", "CMakeConfigDeps" def requirements(self): self.requires(self.tested_reference_str) @@ -346,9 +342,8 @@ def test(self): c.save({"dependent/conanfile.py": dependent_conanfile, "main/conanfile.py": conanfile, "main/test_package/conanfile.py": test_package}) - c.run("create ./dependent/ --name=dependent --version=0.1 " - f"-c tools.cmake.cmakedeps:new={new_value}") - c.run(f"create ./main/ --name=pkg --version=0.1 -c tools.cmake.cmakedeps:new={new_value}") + c.run("create ./dependent/ --name=dependent --version=0.1") + c.run(f"create ./main/ --name=pkg --version=0.1") def test_cmake_find_mode_deprecated(): @@ -365,8 +360,7 @@ def package_info(self): """) tc.save({"conanfile.py": dep}) tc.run("create .") - args = f"-g CMakeDeps -c tools.cmake.cmakedeps:new={new_value}" - tc.run(f"install --requires=dep/0.1 {args}") + tc.run(f"install --requires=dep/0.1 -g CMakeConfigDeps") assert "CMakeConfigDeps does not support module find mode" @@ -406,8 +400,7 @@ def package_info(self): """) tc.save({"conanfile.py": dep}) tc.run("create .") - args = f"-g CMakeDeps -c tools.cmake.cmakedeps:new={new_value}" - tc.run(f"install --requires=dep/0.1 {args}") + tc.run(f"install --requires=dep/0.1 -g CMakeConfigDeps") dep = tc.load("dep-Targets-release.cmake") assert "find_dependency(MyOpenMPI REQUIRED )" in dep assert "set_property(TARGET dep::dep APPEND PROPERTY INTERFACE_LINK_LIBRARIES\n" \ @@ -442,8 +435,7 @@ def package_info(self): """) tc.save({"conanfile.py": dep}) tc.run("create .") - args = f"-g CMakeDeps -c tools.cmake.cmakedeps:new={new_value}" - tc.run(f"install --requires=dep/0.1 {args}") + tc.run(f"install --requires=dep/0.1 -g CMakeConfigDeps") dep = tc.load("dep-Targets-release.cmake") assert "find_dependency(MyOpenMPI REQUIRED )" in dep assert "set_property(TARGET dep::mycomp APPEND PROPERTY INTERFACE_LINK_LIBRARIES\n" \ @@ -462,7 +454,7 @@ def test_requires_to_application(self): class TestPkg(ConanFile): settings = "os", "compiler", "arch", "build_type" - generators = "CMakeDeps", "CMakeToolchain" + generators = "CMakeConfigDeps", "CMakeToolchain" def requirements(self): self.requires(self.tested_reference_str) @@ -475,7 +467,7 @@ def test(self): "libtool/conanfile.py": conanfile, "libtool/test_package/conanfile.py": test_package}) c.run("create automake") - c.run(f"create libtool -c tools.cmake.cmakedeps:new={new_value}") + c.run(f"create libtool") targets = c.load("libtool/test_package/libtool-Targets-release.cmake") # The libtool shouldn't depend on the automake::automake target assert "automake::automake" not in targets @@ -513,8 +505,7 @@ def package_info(self): c.run("create automake") c.run("create libtool") - c.run("install --requires=libtool/0.1 -g CMakeDeps " - f"-c tools.cmake.cmakedeps:new={new_value}") + c.run("install --requires=libtool/0.1 -g CMakeConfigDeps") targets = c.load("libtool-Targets-release.cmake") # The libtool shouldn't depend on the automake::automake target assert "automake::automake" not in targets @@ -541,8 +532,7 @@ def package_info(self): "libtool/conanfile.py": conanfile}) c.run("create automake") c.run("create libtool") - c.run("install --requires=libtool/0.1 -g CMakeDeps " - f"-c tools.cmake.cmakedeps:new={new_value}") + c.run("install --requires=libtool/0.1 -g CMakeConfigDeps") targets = c.load("libtool-Targets-release.cmake") # The libtool shouldn't depend on the automake::automake target assert "automake::automake" not in targets @@ -577,8 +567,7 @@ def package_info(self): "libtool/conanfile.py": conanfile}) c.run("create automake") c.run("create libtool") - c.run("install --requires=libtool/0.1 -g CMakeDeps " - f"-c tools.cmake.cmakedeps:new={new_value}") + c.run("install --requires=libtool/0.1 -g CMakeConfigDeps") targets = c.load("libtool-Targets-release.cmake") # The libtool shouldn't depend on the automake::automake target assert "automake::myapp" not in targets @@ -600,7 +589,7 @@ def package_info(self): """), "conanfile.py": textwrap.dedent(""" from conan import ConanFile - from conan.tools.cmake import CMakeDeps, CMake + from conan.tools.cmake import CMakeConfigDeps, CMake class Pkg(ConanFile): name = "pkg" version = "1.0" @@ -609,14 +598,14 @@ class Pkg(ConanFile): requires = "dep/1.0" def generate(self): - deps = CMakeDeps(self) + deps = CMakeConfigDeps(self) deps.set_property("dep", "cmake_target_aliases", ["alias", "dep::other_name"]) deps.set_property("dep::mycomp", "cmake_target_aliases", ["component_alias", "dep::my_aliased_component"]) deps.generate() """)}) tc.run("create dep") - tc.run(f"install . -c tools.cmake.cmakedeps:new={new_value}") + tc.run(f"install .") targets_data = tc.load('dep-Targets-release.cmake') assert "add_library(dep::dep" in targets_data assert "add_library(alias" in targets_data @@ -649,7 +638,7 @@ def package_info(self): client.save({"conanfile.py": conanfile}) client.run("create .") - client.run(f"install --requires=pkg/0.1 -g CMakeDeps -c tools.cmake.cmakedeps:new={new_value} " + client.run(f"install --requires=pkg/0.1 -g CMakeConfigDeps " """-c tools.cmake.cmaketoolchain:extra_variables="{'BAR': 9}" """) target = client.load("pkg-config.cmake") assert 'set(BAR' not in target @@ -676,7 +665,7 @@ def package_info(self): client.save({"conanfile.py": conanfile}) client.run("create .") - client.run(f"install --requires=pkg/0.1 -g CMakeDeps -c tools.cmake.cmakedeps:new={new_value}") + client.run(f"install --requires=pkg/0.1 -g CMakeConfigDeps") target = client.load("pkg-Targets-release.cmake") assert 'add_library(pkg::base INTERFACE IMPORTED)' in target assert "# Requirement pkg::comp -> pkg::base (Full link: True)" in target @@ -820,7 +809,7 @@ def test_legacy_libraries(self): .with_package_file("lib/mylib2.a", "library") .with_package_info({"components": {"mypkg": {"libs": ["mylib1"]}, "lib2": {"libs": ["mylib2"]}}}) - }) + }) tc.run("create") tc.run("install --requires=mypkg/1.0 -g CMakeConfigDeps") mypkg_config = tc.load("mypkg-config.cmake") diff --git a/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_frameworks.py b/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_frameworks.py index f21e110c256..be5d443e639 100644 --- a/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_frameworks.py +++ b/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps_frameworks.py @@ -5,8 +5,6 @@ from conan.test.utils.tools import TestClient -new_value = "will_break_next" - @pytest.mark.skipif(platform.system() != "Darwin", reason="Only OSX") def test_package_framework_needs_location(): @@ -40,8 +38,9 @@ def requirements(self): 'test_package/conanfile.py': test_conanfile, 'conanfile.py': conanfile }) - client.run(f"create . -c tools.cmake.cmakedeps:new={new_value}", assert_error=True) - assert "Error in generator 'CMakeConfigDeps': cpp_info.location missing for framework MyFramework" in client.out + client.run(f"create .", assert_error=True) + assert ("Error in generator 'CMakeConfigDeps': cpp_info.location" + " missing for framework MyFramework") in client.out def test_framework_only_component_generates_target(): @@ -63,6 +62,6 @@ def package_info(self): tc = TestClient() tc.save({"conanfile.py": conanfile}) tc.run("create .") - tc.run(f"install --requires=frame/1.0 -g CMakeConfigDeps -c=tools.cmake.cmakedeps:new={new_value}") + tc.run(f"install --requires=frame/1.0 -g CMakeConfigDeps") targets = tc.load("frame-Targets-release.cmake") assert "add_library(frame::frame INTERFACE IMPORTED)" in targets diff --git a/test/integration/toolchains/microsoft/test_nmaketoolchain.py b/test/integration/toolchains/microsoft/test_nmaketoolchain.py index 0cf6722b9b8..420cbde4d57 100644 --- a/test/integration/toolchains/microsoft/test_nmaketoolchain.py +++ b/test/integration/toolchains/microsoft/test_nmaketoolchain.py @@ -1,4 +1,3 @@ -import os import platform import textwrap diff --git a/test/integration/tools/system/package_manager_test.py b/test/integration/tools/system/package_manager_test.py index 883b9bd3e4f..292038d4fca 100644 --- a/test/integration/tools/system/package_manager_test.py +++ b/test/integration/tools/system/package_manager_test.py @@ -139,7 +139,7 @@ def test_tools_install_mode_check(tool_class): context_mock.return_value = "host" tool = tool_class(conanfile) with pytest.raises(ConanException) as exc_info: - def fake_check(*args, **kwargs): + def fake_check(*args, **kwargs): # noqa return ["package1", "package2"] from conan.tools.system.package_manager import _SystemPackageManagerTool with patch.object(_SystemPackageManagerTool, 'check', MagicMock(side_effect=fake_check)): @@ -196,8 +196,8 @@ def test_dnf_yum_return_code_100(tool_class, result): context_mock.return_value = "host" tool = tool_class(conanfile) - def fake_run(command, win_bash=False, subsystem=None, env=None, ignore_errors=False, - quiet=False): + def fake_run(command, win_bash=False, subsystem=None, env=None, ignore_errors=False, # noqa + quiet=False): # noqa assert command == result return 100 if "check-update" in command else 0 @@ -252,7 +252,7 @@ def test_tools_install_mode_install_different_archs(tool_class, arch_host, resul context_mock.return_value = "host" tool = tool_class(conanfile) - def fake_check(*args, **kwargs): + def fake_check(*args, **kwargs): # noqa return ["package1", "package2"] from conan.tools.system.package_manager import _SystemPackageManagerTool with patch.object(_SystemPackageManagerTool, 'check', MagicMock(side_effect=fake_check)): @@ -294,7 +294,7 @@ def test_tools_install_mode_install_different_archs_with_version(tool_class, arc context_mock.return_value = "host" tool = tool_class(conanfile) - def fake_check(*args, **kwargs): + def fake_check(*args, **kwargs): # noqa return ["package1=0.1", "package2=0.2"] from conan.tools.system.package_manager import _SystemPackageManagerTool with patch.object(_SystemPackageManagerTool, 'check', MagicMock(side_effect=fake_check)): @@ -335,7 +335,7 @@ def test_tools_install_mode_install_to_build_machine_arch(tool_class, arch_host, context_mock.return_value = "host" tool = tool_class(conanfile) - def fake_check(*args, **kwargs): + def fake_check(*args, **kwargs): # noqa return ["package1", "package2"] from conan.tools.system.package_manager import _SystemPackageManagerTool with patch.object(_SystemPackageManagerTool, 'check', MagicMock(side_effect=fake_check)): @@ -377,7 +377,7 @@ def test_tools_install_mode_install_to_build_machine_arch_with_version(tool_clas context_mock.return_value = "host" tool = tool_class(conanfile) - def fake_check(*args, **kwargs): + def fake_check(*args, **kwargs): # noqa return ["package1=0.1", "package2=0.2"] from conan.tools.system.package_manager import _SystemPackageManagerTool with patch.object(_SystemPackageManagerTool, 'check', MagicMock(side_effect=fake_check)): @@ -404,7 +404,7 @@ def test_tools_install_archless(tool_class, result): context_mock.return_value = "host" tool = tool_class(conanfile, arch_names={}) - def fake_check(*args, **kwargs): + def fake_check(*args, **kwargs): # noqa return ["package1", "package2"] from conan.tools.system.package_manager import _SystemPackageManagerTool with patch.object(_SystemPackageManagerTool, 'check', MagicMock(side_effect=fake_check)): @@ -431,7 +431,7 @@ def test_tools_install_archless_with_version(tool_class, result): context_mock.return_value = "host" tool = tool_class(conanfile, arch_names={}) - def fake_check(*args, **kwargs): + def fake_check(*args, **kwargs): # noqa return ["package1=0.1", "package2=0.2"] from conan.tools.system.package_manager import _SystemPackageManagerTool with patch.object(_SystemPackageManagerTool, 'check', MagicMock(side_effect=fake_check)): diff --git a/test/unittests/client/migrations/test_migrator.py b/test/unittests/client/migrations/test_migrator.py index f8224cff5a5..6c05cd1f0de 100644 --- a/test/unittests/client/migrations/test_migrator.py +++ b/test/unittests/client/migrations/test_migrator.py @@ -4,7 +4,7 @@ import pytest from conan.errors import ConanMigrationError -from conans.migrations import Migrator +from conan.internal.api.migrations import Migrator from conan.test.utils.test_files import temp_folder From 4f5febbdc29f2714cee875fd39c054decb289f2e Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Mon, 13 Apr 2026 09:43:11 +0200 Subject: [PATCH 088/110] Forward underlying system package manager error messages (#19858) * Check package manager in path * Simplification: just raise the shell message error to user * Single run invokation * Fix lint warnings * Improve output --- conan/tools/system/package_manager.py | 13 +++++-- .../tools/system/package_manager_test.py | 35 +++++++++++++++++-- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/conan/tools/system/package_manager.py b/conan/tools/system/package_manager.py index 412569ea0f9..b57a0d0df06 100644 --- a/conan/tools/system/package_manager.py +++ b/conan/tools/system/package_manager.py @@ -1,4 +1,5 @@ import platform +from io import StringIO from conan.tools.build import cross_building from conan.internal.graph.graph import CONTEXT_BUILD @@ -108,9 +109,17 @@ def run(self, method, *args, **kwargs): def _conanfile_run(self, command, accepted_returns, quiet=True): # When checking multiple packages, this is too noisy - ret = self._conanfile.run(command, ignore_errors=True, quiet=quiet) + # Capture output and show it only on failure. + stdout_buf = StringIO() if quiet else None + stderr_buf = StringIO() if quiet else None + ret = self._conanfile.run(command, ignore_errors=True, quiet=quiet, stdout=stdout_buf, stderr=stderr_buf) if ret not in accepted_returns: - raise ConanException("Command '%s' failed" % command) + msg = f"Command '{command}' failed with exit code {ret}" + if stderr_buf is not None and stderr_buf.getvalue(): + msg += f"\nstderr: {stderr_buf.getvalue().strip()}" + if stdout_buf is not None and stdout_buf.getvalue(): + msg += f"\nstdout: {stdout_buf.getvalue().strip()}" + raise ConanException(msg) return ret def install_substitutes(self, *args, **kwargs): diff --git a/test/integration/tools/system/package_manager_test.py b/test/integration/tools/system/package_manager_test.py index 292038d4fca..5e29e45776e 100644 --- a/test/integration/tools/system/package_manager_test.py +++ b/test/integration/tools/system/package_manager_test.py @@ -95,6 +95,35 @@ def test_conf_tool_skips_default_detection_message_on_unknown_distro(): Apt(conanfile) get_default_mock.assert_not_called() +@pytest.mark.parametrize("use_quiet_check", [True, False]) +def test_package_manager_not_found(use_quiet_check): + """Failed runs surface the shell's error (e.g. command not found) when output is captured.""" + conanfile = ConanFileMock() + conanfile.settings = Settings() + conanfile.conf.define("tools.system.package_manager:tool", "apt-get") + conanfile.conf.define("tools.system.package_manager:mode", "install") + + def fake_run(command, stdout=None, stderr=None, ignore_errors=False, env="", quiet=False, **kwargs): + if quiet and stderr is not None: + stderr.write("sh: apt-get: command not found\n") + return 127 + + conanfile.run = fake_run + with mock.patch('conan.ConanFile.context', new_callable=PropertyMock) as context_mock: + context_mock.return_value = "host" + tool = Apt(conanfile) + with pytest.raises(ConanException) as exc_info: + if use_quiet_check: + tool.check(["pkg"]) + else: + tool.install(["pkg"], check=False) + + msg = str(exc_info.value) + assert "failed with exit code 127" in msg + if use_quiet_check: + assert "stderr:" in msg + assert "sh: apt-get: command not found" in msg + @pytest.mark.parametrize("sudo, sudo_askpass, expected_str", [ (True, True, "sudo -A "), @@ -197,7 +226,7 @@ def test_dnf_yum_return_code_100(tool_class, result): tool = tool_class(conanfile) def fake_run(command, win_bash=False, subsystem=None, env=None, ignore_errors=False, # noqa - quiet=False): # noqa + quiet=False, **kwargs): # noqa assert command == result return 100 if "check-update" in command else 0 @@ -210,13 +239,13 @@ def fake_run(command, win_bash=False, subsystem=None, env=None, ignore_errors=Fa tool = tool_class(conanfile) def fake_run(command, win_bash=False, subsystem=None, env=None, ignore_errors=False, - quiet=False): + quiet=False, **kwargs): return 55 if "check-update" in command else 0 conanfile.run = fake_run with pytest.raises(ConanException) as exc_info: tool.update() - assert f"Command '{result}' failed" == str(exc_info.value) + assert f"Command '{result}' failed with exit code 55" == str(exc_info.value) @pytest.mark.parametrize("tool_class, arch_host, result", [ From e3c6499e28f73ff21b9bed74d1aba84068254bb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= <5364255+AbrilRBS@users.noreply.github.com> Date: Mon, 13 Apr 2026 11:32:41 +0200 Subject: [PATCH 089/110] Revert quote changes for nmake defines (#19859) * Revert quote changes for nmake defines * Bump patch * Fix tests --- conan/tools/microsoft/nmakedeps.py | 4 ++-- conan/tools/microsoft/nmaketoolchain.py | 2 +- test/integration/toolchains/microsoft/test_nmakedeps.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/conan/tools/microsoft/nmakedeps.py b/conan/tools/microsoft/nmakedeps.py index fe416885e01..b96fb064722 100644 --- a/conan/tools/microsoft/nmakedeps.py +++ b/conan/tools/microsoft/nmakedeps.py @@ -5,7 +5,7 @@ from conan.tools.env import Environment -def format_defines(defines): +def format_defines(defines, toolchain=False): def is_hex_or_numeric(s): try: # Check for Hexadecimal (base 16) @@ -22,7 +22,7 @@ def is_hex_or_numeric(s): macro, value = define.split("=", 1) if value and not is_hex_or_numeric(value): # value quotes are escaped - value = f'\\"{value}\\"' + value = f'\\"{value}\\"' if toolchain else f'\"{value}\"' define = f"{macro}#{value}" formated_defines.append(f'/D"{define}"') return formated_defines diff --git a/conan/tools/microsoft/nmaketoolchain.py b/conan/tools/microsoft/nmaketoolchain.py index c016291547e..104e5a76192 100644 --- a/conan/tools/microsoft/nmaketoolchain.py +++ b/conan/tools/microsoft/nmaketoolchain.py @@ -58,7 +58,7 @@ def _cl(self): defines.extend(self.extra_defines) return (["/nologo"] + self._format_options(bt_flags + rt_flags + cflags + cxxflags) + - format_defines(defines)) + format_defines(defines, toolchain=True)) @property def _link(self): diff --git a/test/integration/toolchains/microsoft/test_nmakedeps.py b/test/integration/toolchains/microsoft/test_nmakedeps.py index abed9daf82e..338b40bfd85 100644 --- a/test/integration/toolchains/microsoft/test_nmakedeps.py +++ b/test/integration/toolchains/microsoft/test_nmakedeps.py @@ -43,9 +43,9 @@ def package_info(self): # Checking that defines are added to CL for flag in ( r'/D"TEST_DEFINITION1"', '/D"TEST_DEFINITION2#0"', - r'/D"TEST_DEFINITION3#"', r'/D"TEST_DEFINITION4#\"foo\""', - r'/D"TEST_DEFINITION5#\"__declspec(dllexport)\""', - r'/D"TEST_DEFINITION6#\"foo bar\""', + r'/D"TEST_DEFINITION3#"', '/D"TEST_DEFINITION4#"foo""', + r'/D"TEST_DEFINITION5#"__declspec(dllexport)""', + r'/D"TEST_DEFINITION6#"foo bar""', r'/D"TEST_DEFINITION7#7"', r'/D"TEST_WINVER#0x0601"' ): From 912a4f2711847fa2f3b816dc8094a093f49eda50 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 14 Apr 2026 11:59:07 +0200 Subject: [PATCH 090/110] remove pytest version limit (#19864) * remove pytest version limit * fix --- conans/requirements_dev.txt | 2 +- .../cmake/cmakedeps/test_cmakedeps_components_names.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/conans/requirements_dev.txt b/conans/requirements_dev.txt index 7d8ca9b1989..0a3919cf382 100644 --- a/conans/requirements_dev.txt +++ b/conans/requirements_dev.txt @@ -1,4 +1,4 @@ -pytest>=7, <8.0.0 +pytest pytest-xdist # To launch in N cores with pytest -n WebTest>=3.0.0, <4 bottle diff --git a/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_components_names.py b/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_components_names.py index b9ce1acf3b1..8ac66581aac 100644 --- a/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_components_names.py +++ b/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_components_names.py @@ -10,7 +10,6 @@ from conan.test.utils.tools import TestClient -@pytest.mark.tool("cmake") @pytest.fixture(scope="module") def setup_client_with_greetings(): """ From e54f73eaba63ae9fbd55161df25003d5db3a2d25 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 14 Apr 2026 16:27:07 +0200 Subject: [PATCH 091/110] refactor for loader api-helper (#19866) --- conan/api/conan_api.py | 7 ++++++- conan/api/subapi/cache.py | 2 +- conan/api/subapi/config.py | 2 +- conan/api/subapi/export.py | 2 +- conan/api/subapi/graph.py | 8 ++++---- conan/api/subapi/local.py | 6 +++--- conan/api/subapi/report.py | 2 +- conan/api/subapi/upload.py | 4 ++-- conan/api/subapi/workspace.py | 6 +++--- conan/internal/runner/docker.py | 2 +- 10 files changed, 23 insertions(+), 18 deletions(-) diff --git a/conan/api/conan_api.py b/conan/api/conan_api.py index c0f2f63458d..0d450defb3e 100644 --- a/conan/api/conan_api.py +++ b/conan/api/conan_api.py @@ -211,6 +211,11 @@ def editable_packages(self): # These are just the global editables, not including workspace ones return self._editable_packages + @property + def loader(self): + _, _, load, _ = self.get_loader() + return load + def get_loader(self): ws_editables = self._conan_api.workspace.packages() editable_packages = self._editable_packages.update_copy(ws_editables) @@ -230,4 +235,4 @@ def get_loader(self): pyreq_loader = PyRequireLoader(proxy, range_resolver, self.global_conf) # This is caching too! loader = ConanFileLoader(pyreq_loader, conanfile_helpers) - return proxy, range_resolver, loader + return proxy, range_resolver, loader, None diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index 7e43faaff17..8bc021adde3 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -193,7 +193,7 @@ def sign(self, package_list): "information on how to configure the plugin, please read the documentation at " "https://docs.conan.io/2/reference/extensions/package_signing.html.") - _, _, loader = self._api_helpers.get_loader() + loader = self._api_helpers.loader preparator = PackagePreparator(loader, self._api_helpers.cache, self._api_helpers.remote_manager, self._api_helpers.global_conf) diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index d8af964b5a3..d3a4e564894 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -211,7 +211,7 @@ def fetch_packages(self, requires, lockfile=None, remotes=None, profile=None): remotes = conan_api.remotes.list() if remotes is None else remotes profile_host = profile_build = profile or conan_api.profiles.get_profile([]) - proxy, range_resolver, loader = self._helpers.get_loader() + proxy, range_resolver, loader, _ = self._helpers.get_loader() cache = self._helpers.cache ConanOutput().title("Fetching requested configuration packages") diff --git a/conan/api/subapi/export.py b/conan/api/subapi/export.py index ca8a74cbf93..86473e4220c 100644 --- a/conan/api/subapi/export.py +++ b/conan/api/subapi/export.py @@ -43,7 +43,7 @@ def export(self, path, name: str = None, version: str = None, user: str = None, :return: A tuple of the exported RecipeReference and a ConanFile object """ ConanOutput().title("Exporting recipe to the cache") - _, _, loader = self._helpers.get_loader() + loader = self._helpers.loader hook_manager = self._helpers.hook_manager return cmd_export(loader,self._helpers.cache, hook_manager, self._helpers.global_conf, path, name, version, user, channel, graph_lock=lockfile, remotes=remotes) diff --git a/conan/api/subapi/graph.py b/conan/api/subapi/graph.py index cfa182b0641..1caa230bbe0 100644 --- a/conan/api/subapi/graph.py +++ b/conan/api/subapi/graph.py @@ -20,7 +20,7 @@ def _load_root_consumer_conanfile(self, path, profile_host, profile_build, name=None, version=None, user=None, channel=None, update=None, remotes=None, lockfile=None, is_build_require=False): - _, _, loader = self._helpers.get_loader() + loader = self._helpers.loader if path.endswith(".py"): conanfile = loader.load_consumer(path, @@ -66,7 +66,7 @@ def load_root_test_conanfile(self, path, tested_reference, profile_host, profile # necessary for correct resolution and update of remote python_requires - _, _, loader = self._helpers.get_loader() + loader = self._helpers.loader profile_host.options.scope(tested_reference) # do not try apply lock_python_requires for test_package/conanfile.py consumer @@ -90,7 +90,7 @@ def _load_root_virtual_conanfile(self, profile_host, profile_build, requires, to python_requires=None): if not python_requires and not requires and not tool_requires: raise ConanException("Provide requires or tool_requires") - _, _, loader = self._helpers.get_loader() + loader = self._helpers.loader conanfile = loader.load_virtual(requires=requires, tool_requires=tool_requires, python_requires=python_requires, @@ -184,7 +184,7 @@ def load_graph(self, root_node, profile_host, profile_build, lockfile=None, remo assert profile_host is not None assert profile_build is not None - proxy, range_resolver, loader = self._helpers.get_loader() + proxy, range_resolver, loader, _ = self._helpers.get_loader() remotes = remotes or [] cache = self._helpers.cache diff --git a/conan/api/subapi/local.py b/conan/api/subapi/local.py index 01c572954ee..dbcc4ead3b0 100644 --- a/conan/api/subapi/local.py +++ b/conan/api/subapi/local.py @@ -70,7 +70,7 @@ def editable_add(self, path, name=None, version=None, user=None, channel=None, c :return: RecipeReference of the added package """ path = self.get_conanfile_path(path, cwd, py=True) - _, _, loader = self._helpers.get_loader() + loader = self._helpers.loader conanfile = loader.load_named(path, name, version, user, channel, remotes=remotes) if conanfile.name is None or conanfile.version is None: raise ConanException("Editable package recipe should declare its name and version") @@ -115,7 +115,7 @@ def source(self, path, name=None, version=None, user=None, channel=None, :param channel: The channel of the package. If not defined, it is taken from conanfile :param remotes: The remotes to resolve possible ``python-requires`` for this recipe if needed. """ - _, _, loader = self._helpers.get_loader() + loader = self._helpers.loader conanfile = loader.load_consumer(path, name=name, version=version, user=user, channel=channel, graph_lock=None, remotes=remotes) @@ -181,7 +181,7 @@ def test(conanfile) -> None: def inspect(self, conanfile_path, remotes, lockfile, name=None, version=None, user=None, channel=None): - _, _, loader = self._helpers.get_loader() + loader = self._helpers.loader conanfile = loader.load_named(conanfile_path, name=name, version=version, user=user, channel=channel, remotes=remotes, graph_lock=lockfile) return conanfile diff --git a/conan/api/subapi/report.py b/conan/api/subapi/report.py index 21dcd960fae..94882a13829 100644 --- a/conan/api/subapi/report.py +++ b/conan/api/subapi/report.py @@ -85,7 +85,7 @@ def _source(path_to_conanfile, reference): def _configure_source(conan_api, hook_manager, conanfile_path, ref, remotes): - _, _, loader = conan_api._api_helpers.get_loader() # noqa + loader = conan_api._api_helpers.loader # noqa conanfile = loader.load_consumer(conanfile_path, name=ref.name, version=str(ref.version), user=ref.user, channel=ref.channel, graph_lock=None, remotes=remotes) diff --git a/conan/api/subapi/upload.py b/conan/api/subapi/upload.py index c8243790b45..c18d532449d 100644 --- a/conan/api/subapi/upload.py +++ b/conan/api/subapi/upload.py @@ -35,7 +35,7 @@ def check_upstream(self, package_list: PackagesList, remote: Remote, :parameter force: If ``True``, it will skip the check and mark that all items need to be uploaded. A ``force_upload`` key will be added to the entries that will be uploaded. """ - _, _, loader = self._api_helpers.get_loader() + loader = self._api_helpers.loader for ref, _ in package_list.items(): layout = self._api_helpers.cache.recipe_layout(ref) conanfile_path = layout.conanfile() @@ -63,7 +63,7 @@ def prepare(self, package_list: PackagesList, enabled_remotes: List[Remote], if metadata and metadata != [''] and '' in metadata: raise ConanException("Empty string and patterns can not be mixed for metadata.") - _, _, loader = self._api_helpers.get_loader() + loader = self._api_helpers.loader preparator = PackagePreparator(loader, self._api_helpers.cache, self._api_helpers.remote_manager, self._api_helpers.global_conf) diff --git a/conan/api/subapi/workspace.py b/conan/api/subapi/workspace.py index 00d7adeb912..8cc88d37a9b 100644 --- a/conan/api/subapi/workspace.py +++ b/conan/api/subapi/workspace.py @@ -141,7 +141,7 @@ def packages(self): def open(self, ref, remotes, cwd=None): cwd = cwd or os.getcwd() - proxy, _, loader = self._conan_api._api_helpers.get_loader() # noqa + proxy, _, loader, _ = self._conan_api._api_helpers.get_loader() # noqa ref = RecipeReference.loads(ref) if isinstance(ref, str) else ref recipe = proxy.get_recipe(ref, remotes, update=False, check_update=False) @@ -193,7 +193,7 @@ def add(self, path, name=None, version=None, user=None, channel=None, cwd=None, """ self._check_ws() full_path = self._conan_api.local.get_conanfile_path(path, cwd, py=True) - _, _, loader = self._conan_api._api_helpers.get_loader() # noqa + loader = self._conan_api._api_helpers.loader # noqa conanfile = loader.load_named(full_path, name, version, user, channel, remotes=remotes) if conanfile.name is None or conanfile.version is None: raise ConanException("Editable package recipe should declare its name and version") @@ -329,7 +329,7 @@ def find_folder(ref): if root_class is not None: conanfile = root_class(f"{WORKSPACE_PY} base project Conanfile") # To inject things like cmd_wrapper to the consumer conanfile, so self.run() works - _, _, loader = self._conan_api._api_helpers.get_loader() # noqa + loader = self._conan_api._api_helpers.loader # noqa helpers = loader._conanfile_helpers # noqa conanfile._conan_helpers = helpers conanfile._conan_is_consumer = True diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index 01fab2cc5b9..d21ed8a021e 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -244,7 +244,7 @@ def _run_command(self, command: str, workdir: Optional[str] = None, verbose: boo return stdout_log, stderr_log def _get_volumes_and_docker_path(self) -> tuple[dict, str]: - _, _, loader = self.conan_api._api_helpers.get_loader() # noqa + loader = self.conan_api._api_helpers.loader # noqa remotes = self.conan_api.remotes.list(self.args.remote) if not self.args.no_remote else [] conanfile = loader.load_consumer(self.abs_host_path / "conanfile.py", remotes=remotes) abs_docker_base_path = Path('/') / self.docker_user_name / 'conanrunner' From 44bac65a15188209c57ef2ef6465d3bd28ea940d Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 15 Apr 2026 10:19:47 +0200 Subject: [PATCH 092/110] fix error option output (#19867) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix error option output * Apply suggestion from @AbrilRBS --------- Co-authored-by: Abril Rincón Blanco <5364255+AbrilRBS@users.noreply.github.com> --- conan/cli/commands/list.py | 2 +- test/integration/command/list/list_test.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/conan/cli/commands/list.py b/conan/cli/commands/list.py index 37c642e8380..1ab1afa7291 100644 --- a/conan/cli/commands/list.py +++ b/conan/cli/commands/list.py @@ -34,7 +34,7 @@ def print_serial(item, indent=None, color_index=None): if isinstance(item, dict): for k, v in item.items(): if isinstance(v, (str, int)): - if "error" in k.lower(): + if k.lower() in ("error", "pkgsign_error"): color = Color.BRIGHT_RED k = "ERROR" elif k.lower() == "warning": diff --git a/test/integration/command/list/list_test.py b/test/integration/command/list/list_test.py index 579485f4287..fd137526d59 100644 --- a/test/integration/command/list/list_test.py +++ b/test/integration/command/list/list_test.py @@ -995,3 +995,10 @@ def test_list_local_recipe_index(): assert "ERROR: Recipe 'pkg/0.1@a' not found" in c.out c.run("list 'pkg%0.1#a@b/c' -r=local") assert "ERROR: Recipe 'pkg%0.1' not found" in c.out + +def test_list_error_option(): + c = TestClient(default_server_user=False) + c.save({"conanfile.py": GenConanfile("pkg", "1.0").with_option("my_error_option", [1, 2])}) + c.run("create . -o my_error_option=1") + c.run("list pkg/1.0#*:*") + assert "my_error_option: 1" in c.out From c3bc19c6dc9e4fd533484b9988fc3c129fcdb099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= <5364255+AbrilRBS@users.noreply.github.com> Date: Wed, 15 Apr 2026 10:59:29 +0200 Subject: [PATCH 093/110] Test for absolute core.sources:download_cache (#19869) --- conan/internal/rest/caching_file_downloader.py | 2 +- test/integration/cache/backup_sources_test.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/conan/internal/rest/caching_file_downloader.py b/conan/internal/rest/caching_file_downloader.py index 297a0698d21..fff4c8489af 100644 --- a/conan/internal/rest/caching_file_downloader.py +++ b/conan/internal/rest/caching_file_downloader.py @@ -54,7 +54,7 @@ def _caching_download(self, urls, file_path, # regular local shared download cache, not using Conan backup sources servers backups_urls = backups_urls or ["origin"] if download_cache_folder and not os.path.isabs(download_cache_folder): - raise ConanException("core.download:download_cache must be an absolute path") + raise ConanException("core.sources:download_cache must be an absolute path") download_cache = DownloadCache(download_cache_folder) cached_path = download_cache.source_path(sha256) diff --git a/test/integration/cache/backup_sources_test.py b/test/integration/cache/backup_sources_test.py index f959b4c4796..0b165939d8d 100644 --- a/test/integration/cache/backup_sources_test.py +++ b/test/integration/cache/backup_sources_test.py @@ -899,3 +899,19 @@ def source(self): self.client.run("upload * -c -r=default") assert "No backup sources files to upload" in self.client.out assert sha256 + ".dirty" not in os.listdir(os.path.join(self.download_cache_folder, "s")) + + def test_absolute_core_sources_conf(self): + client = TestClient(light=True) + client.save_home( + {"global.conf": f"core.sources:download_cache=relative\n" + "core.sources:download_urls=['origin']"}) + conanfile = textwrap.dedent(f""" + from conan import ConanFile + from conan.tools.files import download + class Pkg(ConanFile): + def source(self): + download(self, "badbad", "myfile.txt", sha256="sha256") + """) + client.save({"conanfile.py": conanfile}) + client.run("source .", assert_error=True) + assert "core.sources:download_cache must be an absolute path" in client.out From c2ec220462883d7a751507f543db1e2472cca09b Mon Sep 17 00:00:00 2001 From: James Date: Wed, 15 Apr 2026 12:26:13 +0200 Subject: [PATCH 094/110] api docstrings (#19871) * api docstrings * more --- conan/api/conan_api.py | 6 ++++-- conan/api/subapi/command.py | 14 ++++++++++++++ conan/api/subapi/profiles.py | 20 +++++++++++++++++++- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/conan/api/conan_api.py b/conan/api/conan_api.py index 0d450defb3e..d51d33434a7 100644 --- a/conan/api/conan_api.py +++ b/conan/api/conan_api.py @@ -71,10 +71,12 @@ def __init__(self, cache_folder=None): self.config: ConfigAPI = ConfigAPI(self, self._api_helpers) #: Used to interact with remotes self.remotes: RemotesAPI = RemotesAPI(self, self._api_helpers) - self.command = CommandAPI(self) + #: Used to call other commands + self.command: CommandAPI = CommandAPI(self) #: Used to get latest refs and list refs of recipes and packages self.list: ListAPI = ListAPI(self, self._api_helpers) - self.profiles = ProfilesAPI(self, self._api_helpers) + #: Used to process and load Conan profiles + self.profiles: ProfilesAPI = ProfilesAPI(self, self._api_helpers) #: Used to install binaries, sources, deploy packages and more self.install: InstallAPI = InstallAPI(self, self._api_helpers) self.graph = GraphAPI(self, self._api_helpers) diff --git a/conan/api/subapi/command.py b/conan/api/subapi/command.py index f0ec75a80b7..2c59d5a0c23 100644 --- a/conan/api/subapi/command.py +++ b/conan/api/subapi/command.py @@ -6,12 +6,26 @@ class CommandAPI: + """ This CommandAPI is useful to be able to launch full commands from the ConanAPI + + Sometimes some commands are built using several calls to the ConanAPI. If we want + to reuse the same functionality, then we would have to copy all that code into our + own commands. + Instead of doing that, it is possible to call Conan commands using this API, via + the ``run()`` method. + """ def __init__(self, conan_api): self._conan_api = conan_api self.cli = None def run(self, cmd): + """ Runs another Conan command via API + + :param cmd: Conan command to run. It can be either a string, or a list of strings. + :return: It will return what that command returns. Note that different commands can + return different things, so the caller needs to process it accordingly. + """ if isinstance(cmd, str): cmd = shlex.split(cmd) if isinstance(cmd, list): diff --git a/conan/api/subapi/profiles.py b/conan/api/subapi/profiles.py index 10b4d4f666a..dd679c3f53e 100644 --- a/conan/api/subapi/profiles.py +++ b/conan/api/subapi/profiles.py @@ -13,6 +13,8 @@ class ProfilesAPI: + """ This ProfilesAPI is used to list, manage and load Conan profiles + """ def __init__(self, conan_api, api_helpers): self._conan_api = conan_api @@ -77,6 +79,16 @@ def get_profile(self, profiles, settings=None, options=None, conf=None, cwd=None """ Computes a Profile as the result of aggregating all the user arguments, first it loads the "profiles", composing them in order (last profile has priority), and finally adding the individual settings, options (priority over the profiles) + + :param profiles: the list of profiles to load + :param settings: list of "key=value" settings to define the profile. Patterns allowed as + "pkg-pattern:key=value" + :param options: list of "key=value" options. Patterns allowed as "pkg-pattern:key=value" + :param conf: list of "key=value" configurations. Following "conf" definitions, patterns + are allowed as "pkg-pattern:key=value", values that are lists or dictionaries might be + allowed, and configuration operations like ``+=`` for appending are allowed. + :param cwd: the current working directory. If None, os.getcwd() will be used. + :param context: the context, "build" or "host" to which this profile belongs """ assert isinstance(profiles, list), "Please provide a list of profiles" global_conf = self._api_helpers.global_conf @@ -127,7 +139,8 @@ def get_path(self, profile, cwd=None, exists=True): def list(self): """ - List all the profiles file sin the cache + List all the profiles files in the cache + :return: an alphabetically ordered list of profile files in the default cache location """ # List is to be extended (directories should not have a trailing slash) @@ -151,6 +164,11 @@ def list(self): @staticmethod def detect(): """ + Detects a possible default profile. + + The output of this detection is not guaranteed to be complete or stable, it might + change in future releases, following the same rules as the "conan profile detect" command. + :return: an automatically detected Profile, with a "best guess" of the system settings """ profile = Profile() From de9f9c9340082e67b988d1ab7a2207f7ebc0c668 Mon Sep 17 00:00:00 2001 From: Carlos Zoido Date: Wed, 15 Apr 2026 14:10:25 +0200 Subject: [PATCH 095/110] Warn when credentials env vars are ignored due to anonymous server access (#19872) add warning --- conan/api/subapi/remotes.py | 10 +++++++ .../command/remote/test_remote_users.py | 29 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/conan/api/subapi/remotes.py b/conan/api/subapi/remotes.py index 51dd9f5be94..a15b82b0802 100644 --- a/conan/api/subapi/remotes.py +++ b/conan/api/subapi/remotes.py @@ -278,6 +278,16 @@ def user_auth(self, remote: Remote, with_user=False, force=False): return self._api_helpers.remote_manager.check_credentials(remote, force) user, token, _ = localdb.get_login(remote.url) + if not force and user is None: + remote_upper = remote.name.replace("-", "_").upper() + candidate_vars = [f"CONAN_LOGIN_USERNAME_{remote_upper}", "CONAN_LOGIN_USERNAME", + f"CONAN_PASSWORD_{remote_upper}", "CONAN_PASSWORD"] + found_vars = [v for v in candidate_vars if os.getenv(v)] + if found_vars: + ConanOutput().warning( + f"Remote '{remote.name}' accepted anonymous access. {', '.join(found_vars)} " + f"environment variables were not used. Use '--force' to force authentication." + ) return user diff --git a/test/integration/command/remote/test_remote_users.py b/test/integration/command/remote/test_remote_users.py index dfd42522875..bd913369b46 100644 --- a/test/integration/command/remote/test_remote_users.py +++ b/test/integration/command/remote/test_remote_users.py @@ -6,6 +6,7 @@ from unittest.mock import patch from conan.internal.api.remotes.localdb import LocalDB +from conan.internal.rest.rest_client_v2 import RestV2Methods from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient, TestServer from conan.test.utils.env import environment_update @@ -492,3 +493,31 @@ def test_auth_after_logout(self): c.run("remote auth *") assert "Remote 'default' needs authentication, obtaining credentials" in c.out assert "user: myuser" in c.out + + def test_remote_auth_env_vars_anonymous_warning(self): + """https://github.com/conan-io/conan/issues/19807 + Warn when env vars are set but server accepts anonymous access""" + server = TestServer(users={"admin": "password"}) + c = TestClient(light=True, servers={"default": server}) + + # env vars set + anonymous server → warning with var names + with patch.object(RestV2Methods, "check_credentials", return_value=None): + with environment_update({"CONAN_LOGIN_USERNAME": "admin", "CONAN_PASSWORD": "password"}): + c.run("remote auth default") + + assert "CONAN_LOGIN_USERNAME" in c.out + assert "environment variables were not used. Use '--force'" in c.out + + # env vars set + --force → authenticated, no warning + with environment_update({"CONAN_LOGIN_USERNAME": "admin", "CONAN_PASSWORD": "password"}): + c.run("remote auth default --force") + + assert "user: admin" in c.out + assert "environment variables were not used" not in c.out + + # no env vars + anonymous server → no warning + c.run("remote logout default") + with patch.object(RestV2Methods, "check_credentials", return_value=None): + c.run("remote auth default") + + assert "environment variables were not used" not in c.out From 19fe11265d5260a097bdd59e6a262f53bccc7d18 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Thu, 16 Apr 2026 10:47:11 +0200 Subject: [PATCH 096/110] tools.build:install_strip now accepts a list of possible build systems (#19874) * tools.build:install_strip now accepts a list of possible build systems --- conan/internal/model/conf.py | 3 +- conan/tools/cmake/cmake.py | 5 ++- conan/tools/gnu/autotools.py | 6 ++- conan/tools/meson/meson.py | 7 +++- .../tools/cmake/test_cmake_install.py | 37 +++++++++++++++++++ test/unittests/tools/gnu/autotools_test.py | 17 ++++++--- test/unittests/tools/meson/test_meson.py | 20 +++++----- 7 files changed, 77 insertions(+), 18 deletions(-) diff --git a/conan/internal/model/conf.py b/conan/internal/model/conf.py index 620bd4b0f67..65bffc073d2 100644 --- a/conan/internal/model/conf.py +++ b/conan/internal/model/conf.py @@ -156,7 +156,8 @@ "tools.build:rcflags": "List of extra RC (resource compiler) flags used by different toolchains like CMakeToolchain, MSBuildToolchain and MesonToolchain", "tools.build:linker_scripts": "List of linker script files to pass to the linker used by different toolchains like CMakeToolchain, AutotoolsToolchain, and MesonToolchain", # Toolchain installation - "tools.build:install_strip": "(boolean) Strip the binaries when installing them with CMake, Meson and Autotools", + "tools.build:install_strip": "(boolean or list) True/False to strip on install for every CMake, Meson and Autotools " + "integration, or a list of 'cmake', 'meson', 'autotools' to strip only for those.", # Package ID composition "tools.info.package_id:confs": "List of existing configuration to be part of the package ID", } diff --git a/conan/tools/cmake/cmake.py b/conan/tools/cmake/cmake.py index 46e8fc8bc94..e4576fee3ba 100644 --- a/conan/tools/cmake/cmake.py +++ b/conan/tools/cmake/cmake.py @@ -249,7 +249,10 @@ def install(self, build_type=None, component=None, cli_args=None, stdout=None, s "deprecated, use 'tools.build:install_strip' instead.", warn_tag="deprecated") - do_strip = self._conanfile.conf.get("tools.build:install_strip", check_type=bool) + try: + do_strip = self._conanfile.conf.get("tools.build:install_strip", check_type=bool) + except ConanException: + do_strip = "cmake" in self._conanfile.conf.get("tools.build:install_strip", check_type=list) if do_strip or deprecated_install_strip: arg_list.append("--strip") diff --git a/conan/tools/gnu/autotools.py b/conan/tools/gnu/autotools.py index a8aa7757951..0a04a6b93cb 100644 --- a/conan/tools/gnu/autotools.py +++ b/conan/tools/gnu/autotools.py @@ -1,6 +1,7 @@ import os import re +from conan.errors import ConanException from conan.tools.build import build_jobs, cmd_args_to_string, load_toolchain_args from conan.internal.subsystems import subsystem_path, deduce_subsystem from conan.tools.files import chdir @@ -99,7 +100,10 @@ def install(self, args=None, target=None, makefile=None): """ if target is None: target = "install" - do_strip = self._conanfile.conf.get("tools.build:install_strip", check_type=bool) + try: + do_strip = self._conanfile.conf.get("tools.build:install_strip", check_type=bool) + except ConanException: + do_strip = "autotools" in self._conanfile.conf.get("tools.build:install_strip", check_type=list) if do_strip: target += "-strip" args = args if args else [] diff --git a/conan/tools/meson/meson.py b/conan/tools/meson/meson.py index bca877e6624..3cb3b8d2a9f 100644 --- a/conan/tools/meson/meson.py +++ b/conan/tools/meson/meson.py @@ -1,5 +1,6 @@ import os +from conan.errors import ConanException from conan.tools.build import build_jobs from conan.tools.meson.toolchain import MesonToolchain @@ -85,7 +86,11 @@ def install(self, cli_args=None): verbosity = self._install_verbosity if verbosity: cmd += " " + verbosity - if self._conanfile.conf.get("tools.build:install_strip", check_type=bool): + try: + do_strip = self._conanfile.conf.get("tools.build:install_strip", check_type=bool) + except ConanException: + do_strip = "meson" in self._conanfile.conf.get("tools.build:install_strip", check_type=list) + if do_strip: cmd += " --strip" if cli_args: cmd += " " + " ".join(cli_args) diff --git a/test/unittests/tools/cmake/test_cmake_install.py b/test/unittests/tools/cmake/test_cmake_install.py index 2203070dbe9..35870191bdb 100644 --- a/test/unittests/tools/cmake/test_cmake_install.py +++ b/test/unittests/tools/cmake/test_cmake_install.py @@ -1,5 +1,6 @@ import pytest +from conan.errors import ConanException from conan.internal.default_settings import default_settings_yml from conan.tools.cmake import CMake from conan.tools.cmake.presets import write_cmake_presets @@ -80,6 +81,42 @@ def test_run_install_strip(config, deprecated): assert "--strip" in conanfile.command +@pytest.mark.parametrize("value, do_raise, do_strip", + [("cmake", True, False), + ("{'cmake': True', 'meson': False}", True, False), + (None, False, False), + ("['cmake']", True, True), + ("['autotools', 'meson']", True, False)]) +def test_run_install_strip_check_conf_values(value, do_raise, do_strip): + """ + Testing that ``tools.build:install_strip`` only accepts bool or list, + and that the install/strip rule is called when the value is correct. + """ + settings = Settings.loads(default_settings_yml) + settings.os = "Linux" + settings.arch = "x86_64" + settings.build_type = "Release" + settings.compiler = "gcc" + settings.compiler.version = "11" + + conanfile = ConanFileMock() + conanfile.conf = Conf() + conanfile.conf.define("tools.build:install_strip", value) + conanfile.folders.generators = "." + conanfile.folders.set_base_generators(temp_folder()) + conanfile.settings = settings + conanfile.folders.set_base_package(temp_folder()) + write_cmake_presets(conanfile, "toolchain", "Unix Makefiles", {}) + cmake = CMake(conanfile) + if do_raise: + with pytest.raises(ConanException) as exc_info: + cmake.install() + assert "tools.build:install_strip must be a list-like object" in str(exc_info.value) + else: + cmake.install() + assert "--strip" not in conanfile.command if not do_strip else "--strip" in conanfile.command + + def test_run_install_cli_args(): """ Testing that the passing cli_args to install works diff --git a/test/unittests/tools/gnu/autotools_test.py b/test/unittests/tools/gnu/autotools_test.py index 2708e4418e3..47194817bca 100644 --- a/test/unittests/tools/gnu/autotools_test.py +++ b/test/unittests/tools/gnu/autotools_test.py @@ -37,11 +37,18 @@ def test_source_folder_works(chdir_mock): assert conanfile.command == 'autoreconf -bar foo' -@pytest.mark.parametrize("install_strip", [False, True]) -def test_install_strip(install_strip): +@pytest.mark.parametrize("install_strip, expect_strip", [ + (False, False), + (True, True), + (["autotools"], True), + (["cmake"], False), + (["meson"], False), + (["cmake", "meson", "autotools"], True), +]) +def test_install_strip(install_strip, expect_strip): """ - When the configuration `tools.build:install_strip` is set to True, - the Autotools install command should invoke the `install-strip` target. + When `tools.build:install_strip` is True or lists ``autotools``, the install command + should use the ``install-strip`` target; list values only strip for named backends. """ folder = temp_folder() os.chdir(folder) @@ -59,7 +66,7 @@ def test_install_strip(install_strip): autotools = Autotools(conanfile) autotools.install() - assert ('install-strip' in str(conanfile.command)) == install_strip + assert ('install-strip' in str(conanfile.command)) == expect_strip def test_configure_arguments(): diff --git a/test/unittests/tools/meson/test_meson.py b/test/unittests/tools/meson/test_meson.py index b0cf0e82895..cf1ee1501ce 100644 --- a/test/unittests/tools/meson/test_meson.py +++ b/test/unittests/tools/meson/test_meson.py @@ -67,13 +67,15 @@ def test_meson_to_cppstd_flag(compiler, compiler_version, cppstd, expected): assert to_cppstd_flag(ConanFileMock(), compiler, compiler_version, cppstd) == expected -def test_meson_install_strip(): - """When the configuration `tools.build:install_strip` is set to True, - the Meson install command should include the `--strip` option. - """ - c = ConfDefinition() - c.loads("tools.build:install_strip=True") - +@pytest.mark.parametrize("conf_line, expect_strip", [ + (True, True), + (False, False), + (['meson'], True), + (['meson', 'cmake'], True), + (['autotools', 'cmake'], False), +]) +def test_meson_install_strip(conf_line, expect_strip): + """``tools.build:install_strip`` as True or a list containing ``meson`` adds ``--strip``.""" settings = MockSettings({"build_type": "Release", "compiler": "gcc", "compiler.version": "7", @@ -81,7 +83,7 @@ def test_meson_install_strip(): "arch": "x86_64"}) conanfile = ConanFileMock() conanfile.settings = settings - conanfile.conf = c.get_conanfile_conf(None) + conanfile.conf.define("tools.build:install_strip", conf_line) conanfile.folders.generators = "." conanfile.folders.set_base_generators(temp_folder()) conanfile.folders.set_base_package(temp_folder()) @@ -89,7 +91,7 @@ def test_meson_install_strip(): meson = Meson(conanfile) meson.install() - assert '--strip' in str(conanfile.command) + assert ('--strip' in str(conanfile.command)) == expect_strip def test_meson_install_cli_args(): """When the `cli_args` are provided, the Meson install command should include them. From 028c0c79cfc444853982aeeca92f53370ecb0a4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= <5364255+AbrilRBS@users.noreply.github.com> Date: Thu, 16 Apr 2026 13:35:44 +0200 Subject: [PATCH 097/110] Modernize build-order tests to use order-by (#19878) * Modernize build-order tests to use order-by * Modernize build-order tests to use order-by --- .../test_install_test_build_require.py | 4 +- .../command/info/test_info_build_order.py | 40 +++++++++++-------- .../graph/ux/loop_detection_test.py | 2 +- test/integration/lockfile/test_ci.py | 18 +++++---- .../integration/lockfile/test_ci_overrides.py | 19 +++++---- .../integration/lockfile/test_ci_revisions.py | 13 +++--- test/integration/lockfile/test_options.py | 4 +- .../integration/package_id/compatible_test.py | 4 +- 8 files changed, 60 insertions(+), 44 deletions(-) diff --git a/test/integration/build_requires/test_install_test_build_require.py b/test/integration/build_requires/test_install_test_build_require.py index 830b77b9747..b79616e3134 100644 --- a/test/integration/build_requires/test_install_test_build_require.py +++ b/test/integration/build_requires/test_install_test_build_require.py @@ -217,8 +217,8 @@ def build_requirements(self): c.assert_listed_binary({"tool/1.0": (win_pkg_id, "Build")}, build=True) c.run("graph build-order --requires=tool/1.0 -s:b os=Windows -s:h os=Linux --build=* " - "--format=json", redirect_stdout="o.json") - order = json.loads(c.load("o.json")) + "--order-by=recipe --format=json", redirect_stdout="o.json") + order = json.loads(c.load("o.json"))['order'] package1 = order[0][0]["packages"][0][0] package2 = order[0][0]["packages"][1][0] assert package1["package_id"] == win_pkg_id diff --git a/test/integration/command/info/test_info_build_order.py b/test/integration/command/info/test_info_build_order.py index f21fe24d3b8..9bfeddef414 100644 --- a/test/integration/command/info/test_info_build_order.py +++ b/test/integration/command/info/test_info_build_order.py @@ -15,6 +15,7 @@ def test_info_build_order(): "consumer/conanfile.txt": "[requires]\npkg/0.1"}) c.run("export dep --name=dep --version=0.1") c.run("export pkg --name=pkg --version=0.1") + # Old legacy syntax c.run("graph build-order consumer --build=missing --format=json") bo_json = json.loads(c.stdout) @@ -76,6 +77,7 @@ def test_info_build_order(): assert bo_json["order"] == result # test html format + # old legacy syntax c.run("graph build-order consumer --build=missing --format=html") assert "" in c.stdout c.run("graph build-order consumer --order-by=recipe --build=missing --format=html") @@ -162,7 +164,7 @@ def test_info_build_order_build_require(): "consumer/conanfile.txt": "[requires]\npkg/0.1"}) c.run("export dep --name=dep --version=0.1") c.run("export pkg --name=pkg --version=0.1") - c.run("graph build-order consumer --build=missing --format=json") + c.run("graph build-order consumer --build=missing --format=json --order-by=recipe") bo_json = json.loads(c.stdout) result = [ [ @@ -209,7 +211,7 @@ def test_info_build_order_build_require(): ] ] - assert bo_json == result + assert bo_json['order'] == result def test_info_build_order_options(): @@ -226,7 +228,7 @@ def test_info_build_order_options(): c.run("export dep1 --name=dep1 --version=0.1") c.run("export dep2 --name=dep2 --version=0.1") - c.run("graph build-order consumer --build=missing --format=json") + c.run("graph build-order consumer --build=missing --format=json --order-by=recipe") bo_json = json.loads(c.stdout) result = [ [ @@ -265,7 +267,7 @@ def test_info_build_order_options(): ]]} ] ] - assert bo_json == result + assert bo_json['order'] == result def test_info_build_order_merge_multi_product(): @@ -277,8 +279,10 @@ def test_info_build_order_merge_multi_product(): c.run("export dep --name=dep --version=0.1") c.run("export pkg --name=pkg --version=0.1") c.run("export pkg --name=pkg --version=0.2") - c.run("graph build-order consumer1 --build=missing --format=json", redirect_stdout="bo1.json") - c.run("graph build-order consumer2 --build=missing --format=json", redirect_stdout="bo2.json") + c.run("graph build-order consumer1 --build=missing --format=json --order-by=recipe", + redirect_stdout="bo1.json") + c.run("graph build-order consumer2 --build=missing --format=json --order-by=recipe", + redirect_stdout="bo2.json") c.run("graph build-order-merge --file=bo1.json --file=bo2.json --format=json", redirect_stdout="bo3.json") @@ -349,7 +353,7 @@ def test_info_build_order_merge_multi_product(): ] ] - assert bo_json == result + assert bo_json['order'] == result # test that html format for build-order-merge generates something c.run("graph build-order-merge --file=bo1.json --file=bo2.json --format=html") @@ -455,9 +459,9 @@ def requirements(self): c.run("export dep --name=depwin --version=0.1") c.run("export dep --name=depnix --version=0.1") c.run("export pkg --name=pkg --version=0.1") - c.run("graph build-order consumer --format=json --build=missing -s os=Windows", + c.run("graph build-order consumer --format=json --build=missing -s os=Windows --order-by=recipe", redirect_stdout="bo_win.json") - c.run("graph build-order consumer --format=json --build=missing -s os=Linux", + c.run("graph build-order consumer --format=json --build=missing -s os=Linux --order-by=recipe", redirect_stdout="bo_nix.json") c.run("graph build-order-merge --file=bo_win.json --file=bo_nix.json --format=json", redirect_stdout="bo3.json") @@ -542,7 +546,7 @@ def requirements(self): ] ] - assert bo_json == result + assert bo_json['order'] == result def test_info_build_order_lockfile_location(): @@ -555,7 +559,7 @@ def test_info_build_order_lockfile_location(): c.run("create dep") c.run("lock create pkg --lockfile-out=myconan.lock") assert os.path.exists(os.path.join(c.current_folder, "myconan.lock")) - c.run("graph build-order pkg --lockfile=myconan.lock --lockfile-out=myconan2.lock") + c.run("graph build-order pkg --lockfile=myconan.lock --lockfile-out=myconan2.lock --order-by=recipe") assert os.path.exists(os.path.join(c.current_folder, "myconan2.lock")) @@ -627,7 +631,8 @@ def export(self): """) c.save({"conanfile.py": dep}) c.run("export .") - c.run("graph build-order --requires=dep/0.1 --format=json", assert_error=True) + c.run("graph build-order --requires=dep/0.1 --format=json --order-by=recipe", + assert_error=True) assert "ImportError" in c.out assert "It is possible that this recipe is not Conan 2.0 ready" in c.out @@ -713,7 +718,7 @@ def test_build_order_merge_reduce(self, order): def test_error_reduced(self): c = TestClient() c.save({"conanfile.py": GenConanfile("liba", "0.1")}) - c.run("graph build-order . --format=json", redirect_stdout="bo1.json") + c.run("graph build-order . --order-by=recipe --format=json", redirect_stdout="bo1.json") c.run("graph build-order . --order-by=recipe --reduce --format=json", redirect_stdout="bo2.json") c.run(f"graph build-order-merge --file=bo1.json --file=bo2.json", assert_error=True) @@ -725,6 +730,7 @@ def test_error_reduced(self): def test_error_different_orders(self): c = TestClient() c.save({"conanfile.py": GenConanfile("liba", "0.1")}) + # old syntax c.run("graph build-order . --format=json", redirect_stdout="bo1.json") c.run("graph build-order . --order-by=recipe --format=json", redirect_stdout="bo2.json") c.run("graph build-order . --order-by=configuration --format=json", @@ -873,7 +879,7 @@ def validate(self): for approach in ("--build=missing -s foo/*:compiler.cppstd=17", '--build="compatible:foo/*" --build="missing:bar/*"', '--build="missing:foo/*" --build="compatible:foo/*" --build="missing:bar/*"'): - c.run(f'graph build-order --require=foo/1.0 --require=bar/1.0 -pr:a profile {approach}') + c.run(f'graph build-order --require=foo/1.0 --require=bar/1.0 -pr:a profile {approach} --order-by=recipe') c.assert_listed_binary({"foo/1.0": ["4e2ae338231ae18d0d43b9e119404d2b2c416758", "Build"], "bar/1.0": ["5e4ffcc1ff33697a4ee96f66f0d2228ec458f25c", "Build"]}) c.assert_listed_binary({"foo/1.0": ["4e2ae338231ae18d0d43b9e119404d2b2c416758", "Build"]}, @@ -906,11 +912,11 @@ def test_build_order_path_reqs_mixed_args(): # This used not to crash in previous versions # Also make sure we can properly used them separately tc = TestClient(light=True) - tc.run("graph build-order . --requires=foo/1.0", assert_error=True) + tc.run("graph build-order . --requires=foo/1.0 --order-by=recipe", assert_error=True) assert "ERROR: --requires and --tool-requires arguments are incompatible with [path] '.' argument" in tc.out - tc.run("graph build-order --requires=foo/1.0", assert_error=True) + tc.run("graph build-order --requires=foo/1.0 --order-by=recipe", assert_error=True) assert "Package 'foo/1.0' not resolved: No remote defined" in tc.out - tc.run("graph build-order .", assert_error=True) + tc.run("graph build-order . --order-by=recipe", assert_error=True) assert "Conanfile not found" in tc.out diff --git a/test/integration/graph/ux/loop_detection_test.py b/test/integration/graph/ux/loop_detection_test.py index d83a4546dcd..8a15dc65620 100644 --- a/test/integration/graph/ux/loop_detection_test.py +++ b/test/integration/graph/ux/loop_detection_test.py @@ -42,7 +42,7 @@ def test_install_order_infinite_loop(): assert "tool/1.0 (Build) -> ['fmt/1.0']" in c.out # Graph build-order fails in the same way - c.run("graph build-order tool -pr:h=tool_profile -b=missing", + c.run("graph build-order tool -pr:h=tool_profile -b=missing --order-by=recipe", assert_error=True) assert "ERROR: There is a loop in the graph" in c.out assert "fmt/1.0 (Build) -> ['tool/1.0']" in c.out diff --git a/test/integration/lockfile/test_ci.py b/test/integration/lockfile/test_ci.py index 25e1dd1a8c8..c6393f7dbf2 100644 --- a/test/integration/lockfile/test_ci.py +++ b/test/integration/lockfile/test_ci.py @@ -324,10 +324,11 @@ def test_single_config_decentralized(client_setup): # Now lets build the application, to see everything ok c.run("graph build-order --requires=app1/0.1@ --lockfile=app1_b_changed.lock " - "--build=missing --format=json -s os=Windows", redirect_stdout="build_order.json") + "--build=missing --format=json -s os=Windows --order-by=recipe", + redirect_stdout="build_order.json") json_file = c.load("build_order.json") - to_build = json.loads(json_file) + to_build = json.loads(json_file)['order'] level0 = to_build[0] assert len(level0) == 1 pkgawin = level0[0] @@ -391,14 +392,16 @@ def test_multi_config_decentralized(client_setup): # Now lets build the application, to see everything ok, for all the configs c.run("graph build-order --requires=app1/0.1@ --lockfile=app1_win.lock " - "--build=missing --format=json -s os=Windows", redirect_stdout="app1_win.json") + "--build=missing --format=json -s os=Windows --order-by=recipe", + redirect_stdout="app1_win.json") c.run("graph build-order --requires=app1/0.1@ --lockfile=app1_nix.lock " - "--build=missing --format=json -s os=Linux", redirect_stdout="app1_nix.json") + "--build=missing --format=json -s os=Linux --order-by=recipe", + redirect_stdout="app1_nix.json") c.run("graph build-order-merge --file=app1_win.json --file=app1_nix.json" " --format=json", redirect_stdout="build_order.json") json_file = c.load("build_order.json") - to_build = json.loads(json_file) + to_build = json.loads(json_file)['order'] level0 = to_build[0] assert len(level0) == 2 pkgawin = level0[0] @@ -499,8 +502,9 @@ def test_single_config_decentralized_overrides(): assert len(lock["overrides"]) == 1 assert set(lock["overrides"]["toolc/1.0"]) == {"toolc/3.0", "toolc/2.0", None} - c.run("graph build-order pkgc --lockfile=pkgc/conan.lock --format=json --build=missing") - to_build = json.loads(c.stdout) + c.run("graph build-order pkgc --lockfile=pkgc/conan.lock --format=json --build=missing" + " --order-by=recipe") + to_build = json.loads(c.stdout)['order'] for level in to_build: for elem in level: for package in elem["packages"][0]: # assumes no dependencies between packages diff --git a/test/integration/lockfile/test_ci_overrides.py b/test/integration/lockfile/test_ci_overrides.py index a40f9153973..4c057a28c68 100644 --- a/test/integration/lockfile/test_ci_overrides.py +++ b/test/integration/lockfile/test_ci_overrides.py @@ -144,8 +144,9 @@ def test_single_config_decentralized_overrides(): assert len(lock["overrides"]) == 1 assert set(lock["overrides"]["toolc/1.0"]) == {"toolc/3.0", "toolc/2.0", None} - c.run("graph build-order pkgc --lockfile=pkgc/conan.lock --format=json --build=missing") - to_build = json.loads(c.stdout) + c.run("graph build-order pkgc --lockfile=pkgc/conan.lock --format=json --build=missing" + " --order-by=recipe") + to_build = json.loads(c.stdout)['order'] for level in to_build: for elem in level: for package in elem["packages"][0]: # assumes no dependencies between packages @@ -195,8 +196,9 @@ def test_single_config_decentralized_overrides_nested(): assert lock["overrides"] == {"libf/1.0": ["libf/3.0"], "libf/2.0": ["libf/3.0"]} - c.run("graph build-order pkga --lockfile=pkga/conan.lock --format=json --build=missing") - to_build = json.loads(c.stdout) + c.run("graph build-order pkga --lockfile=pkga/conan.lock --format=json --build=missing" + " --order-by=recipe") + to_build = json.loads(c.stdout)['order'] for level in to_build: for elem in level: ref = elem["ref"] @@ -278,8 +280,9 @@ def test_single_config_decentralized_overrides_multi(forced): else: assert set(lock["overrides"]["libf/2.0"]) == {"libf/4.0", "libf/3.0"} - c.run("graph build-order pkgc --lockfile=pkgc/conan.lock --format=json --build=missing") - to_build = json.loads(c.stdout) + c.run("graph build-order pkgc --lockfile=pkgc/conan.lock --format=json --build=missing" + " --order-by=recipe") + to_build = json.loads(c.stdout)['order'] for level in to_build: for elem in level: ref = elem["ref"] @@ -359,9 +362,9 @@ def test_single_config_decentralized_overrides_multi_replace_requires(replace_pa # overrides will be different everytime, just checking that things can be built c.run("graph build-order pkgc --lockfile=pkgc/conan.lock --format=json -pr:b=profile " - "--build=missing") + "--build=missing --order-by=recipe") - to_build = json.loads(c.stdout) + to_build = json.loads(c.stdout)['order'] for level in to_build: for elem in level: ref = elem["ref"] diff --git a/test/integration/lockfile/test_ci_revisions.py b/test/integration/lockfile/test_ci_revisions.py index 7cb8a9be7d6..27432b7634f 100644 --- a/test/integration/lockfile/test_ci_revisions.py +++ b/test/integration/lockfile/test_ci_revisions.py @@ -319,9 +319,10 @@ def test_single_config_decentralized(client_setup): # Now lets build the application, to see everything ok c.run("graph build-order --requires=app1/0.1@ --lockfile=app1_b_changed.lock " - "--build=missing --format=json -s os=Windows", redirect_stdout="build_order.json") + "--build=missing --format=json -s os=Windows --order-by=recipe", + redirect_stdout="build_order.json") json_file = c.load("build_order.json") - to_build = json.loads(json_file) + to_build = json.loads(json_file)['order'] level0 = to_build[0] assert len(level0) == 1 pkgawin = level0[0] @@ -384,14 +385,16 @@ def test_multi_config_decentralized(client_setup): # Now lets build the application, to see everything ok, for all the configs c.run("graph build-order --requires=app1/0.1@ --lockfile=app1_win.lock " - "--build=missing --format=json -s os=Windows", redirect_stdout="app1_win.json") + "--build=missing --format=json -s os=Windows --order-by=recipe", + redirect_stdout="app1_win.json") c.run("graph build-order --requires=app1/0.1@ --lockfile=app1_nix.lock " - "--build=missing --format=json -s os=Linux", redirect_stdout="app1_nix.json") + "--build=missing --format=json -s os=Linux --order-by=recipe", + redirect_stdout="app1_nix.json") c.run("graph build-order-merge --file=app1_win.json --file=app1_nix.json" " --format=json", redirect_stdout="build_order.json") json_file = c.load("build_order.json") - to_build = json.loads(json_file) + to_build = json.loads(json_file)['order'] level0 = to_build[0] assert len(level0) == 2 pkgawin = level0[0] diff --git a/test/integration/lockfile/test_options.py b/test/integration/lockfile/test_options.py index 23ecbe30623..e03b9df290b 100644 --- a/test/integration/lockfile/test_options.py +++ b/test/integration/lockfile/test_options.py @@ -34,10 +34,10 @@ class Meta(ConanFile): client.run("lock create --requires=nano/1.0@ --build=*") client.run("graph build-order --requires=nano/1.0@ " "--lockfile-out=conan.lock --build=missing " - "--format=json", redirect_stdout="build_order.json") + "--format=json --order-by=recipe", redirect_stdout="build_order.json") json_file = client.load("build_order.json") - to_build = json.loads(json_file) + to_build = json.loads(json_file)['order'] ffmpeg = to_build[0][0] ref = ffmpeg["ref"] options = " ".join(f"-o {option}" for option in ffmpeg["packages"][0][0]["options"]) diff --git a/test/integration/package_id/compatible_test.py b/test/integration/package_id/compatible_test.py index 0ae4acdd9e3..6768220937d 100644 --- a/test/integration/package_id/compatible_test.py +++ b/test/integration/package_id/compatible_test.py @@ -614,8 +614,8 @@ def validate_build(self): c.run("export libb") settings = "-s os=Windows -s compiler=gcc -s compiler.version=11 " \ "-s compiler.libcxx=libstdc++11 -s compiler.cppstd=11" - c.run(f"graph build-order --requires=libb/0.1 {settings} --format=json", assert_error=True, - redirect_stdout="build_order.json") + c.run(f"graph build-order --requires=libb/0.1 {settings} --format=json --order-by=recipe", + assert_error=True) c.assert_listed_binary({"liba/0.1": ("bb33db23c961978d08dc0cdd6bc786b45b3e5943", "Missing"), "libb/0.1": ("144910d65b27bcbf7d544201f5578555bbd0376e", "Missing")}) From 1ee0b08b61880747e6ebdf43bcc0b7f460d511d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= <5364255+AbrilRBS@users.noreply.github.com> Date: Thu, 16 Apr 2026 13:38:53 +0200 Subject: [PATCH 098/110] Allow patterns in `--update` flag (#19856) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Pattern for ref name * Simplify diff * Update help text· --- conan/cli/args.py | 6 ++++-- conan/internal/graph/proxy.py | 5 ++++- test/integration/cache/cache2_update_test.py | 7 +++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/conan/cli/args.py b/conan/cli/args.py index 4e4b148a506..6ea92ee82d5 100644 --- a/conan/cli/args.py +++ b/conan/cli/args.py @@ -54,10 +54,12 @@ def add_common_install_arguments(parser): help='Do not use remote, resolve exclusively in the cache') update_help = ("Will install newer versions and/or revisions in the local cache " - "for the given reference name, or all references in the graph if no argument is supplied. " + "for the given references whose name matches the given pattern, " + "or all references in the graph if no argument is supplied. " "When using version ranges, it will install the latest version that " "satisfies the range. It will update to the " - "latest revision for the resolved version range.") + "latest revision for the resolved version range. " + "The consumer pattern (&) has no effect, and users should not specify versions.") group.add_argument("-u", "--update", action="append", nargs="?", help=update_help, const="*") add_profiles_args(parser) diff --git a/conan/internal/graph/proxy.py b/conan/internal/graph/proxy.py index 910970932b3..7b5a05f6358 100644 --- a/conan/internal/graph/proxy.py +++ b/conan/internal/graph/proxy.py @@ -1,3 +1,5 @@ +from fnmatch import fnmatch + from conan.api.output import ConanOutput from conan.internal.cache.conan_reference_layout import BasicLayout from conan.internal.graph.graph import (RECIPE_DOWNLOADED, RECIPE_INCACHE, RECIPE_NEWER, @@ -180,4 +182,5 @@ def should_update_reference(reference, update): if isinstance(update, bool): return update # Legacy syntax had --update without pattern, it manifests as a "*" pattern - return any(name == "*" or reference.name == name for name in update) + return any(fnmatch(reference.name, pattern) + for pattern in update) diff --git a/test/integration/cache/cache2_update_test.py b/test/integration/cache/cache2_update_test.py index 9acb947dd84..07209372afc 100644 --- a/test/integration/cache/cache2_update_test.py +++ b/test/integration/cache/cache2_update_test.py @@ -443,7 +443,6 @@ def test_version_ranges(self): @pytest.mark.parametrize("update,result", [ - # Not a real pattern, works to support legacy syntax ["*", {"liba/1.1": "Downloaded (default)", "libb/1.1": "Downloaded (default)"}], ["libc", {"liba/1.0": "Cache", @@ -458,9 +457,9 @@ def test_version_ranges(self): "libb/1.0": "Cache"}], ["", {"liba/1.0": "Cache", "libb/1.0": "Cache"}], - # Patterns not supported, only full name match - ["lib*", {"liba/1.0": "Cache", - "libb/1.0": "Cache"}], + # Patterns supported, but only on name + ["lib*", {"liba/1.1": "Downloaded (default)", + "libb/1.1": "Downloaded (default)"}], ["liba/*", {"liba/1.0": "Cache", "libb/1.0": "Cache"}], # None only passes legacy --update without args, From ffcb0405945ebe23087dd29d2e4976079df24680 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= <5364255+AbrilRBS@users.noreply.github.com> Date: Fri, 17 Apr 2026 12:16:22 +0200 Subject: [PATCH 099/110] Remove some deprecated features (#19877) * Remove deprecated Node::dependencies method * Remove deprecated cmake_set_interface_link_directories property * Remove deprecated methods from PackagesList * Remove deprecated deploy folder in conan home * Remove deprecated message for order-by in conan graph info build-order * Remove deprecated detect_compiler method in detect api * Remove deprecated system_tools profile section * Remove deprecated boolean powershell conf values * Revert "Remove deprecated message for order-by in conan graph info build-order" This reverts commit 1c09c83f7ac63077bc8e7f4d8e26d095e158f3c9. * Last test changes * Remove deoply usage in test * Remove last true for test --- conan/api/model/list.py | 51 ------------------- conan/internal/api/detect/detect_api.py | 7 --- conan/internal/api/profile/profile_loader.py | 6 +-- conan/internal/cache/home_paths.py | 5 -- conan/internal/graph/graph.py | 7 --- conan/internal/model/conf.py | 2 +- .../cmakeconfigdeps/target_configuration.py | 7 --- .../templates/target_configuration.py | 11 +--- conan/tools/env/environment.py | 17 +------ conan/tools/microsoft/visual.py | 14 +---- .../functional/command/test_install_deploy.py | 6 +-- .../env/test_virtualenv_powershell.py | 29 ++--------- .../cache/test_home_special_char.py | 2 +- test/integration/graph/test_system_tools.py | 7 ++- .../cmakeconfigdeps/test_cmakeconfigdeps.py | 20 -------- 15 files changed, 17 insertions(+), 174 deletions(-) diff --git a/conan/api/model/list.py b/conan/api/model/list.py index 63572209e0f..8b7bd1cc9e0 100644 --- a/conan/api/model/list.py +++ b/conan/api/model/list.py @@ -246,13 +246,6 @@ def only_recipes(self) -> None: for rrev_dict in ref_dict.get("revisions", {}).values(): rrev_dict.pop("packages", None) - def add_refs(self, refs): - ConanOutput().warning("PackagesLists.add_refs() non-public, non-documented method will be " - "removed, use .add_ref() instead", warn_tag="deprecated") - # RREVS alreday come in ASCENDING order, so upload does older revisions first - for ref in refs: - self.add_ref(ref) - def add_ref(self, ref: RecipeReference) -> None: """ Adds a new RecipeReference to a package list @@ -264,13 +257,6 @@ def add_ref(self, ref: RecipeReference) -> None: if ref.timestamp: rev_dict["timestamp"] = ref.timestamp - def add_prefs(self, rrev, prefs): - ConanOutput().warning("PackageLists.add_prefs() non-public, non-documented method will be " - "removed, use .add_pref() instead", warn_tag="deprecated") - # Prevs already come in ASCENDING order, so upload does older revisions first - for p in prefs: - self.add_pref(p) - def add_pref(self, pref: PkgReference, pkg_info: dict = None) -> None: """ Add a PkgReference to an already existing RecipeReference inside a package list @@ -287,30 +273,6 @@ def add_pref(self, pref: PkgReference, pkg_info: dict = None) -> None: if pkg_info is not None: package_dict["info"] = pkg_info - def add_configurations(self, confs): - ConanOutput().warning("PackageLists.add_configurations() non-public, non-documented method " - "will be removed, use .add_pref() instead", - warn_tag="deprecated") - for pref, conf in confs.items(): - rev_dict = self.recipe_dict(pref.ref) - try: - rev_dict["packages"][pref.package_id]["info"] = conf - except KeyError: # If package_id does not exist, do nothing, only add to existing prefs - pass - - def refs(self): - ConanOutput().warning("PackageLists.refs() non-public, non-documented method will be " - "removed, use .items() instead", warn_tag="deprecated") - result = {} - for ref, ref_dict in self._data.items(): - for rrev, rrev_dict in ref_dict.get("revisions", {}).items(): - t = rrev_dict.get("timestamp") - recipe = RecipeReference.loads(f"{ref}#{rrev}") # TODO: optimize this - if t is not None: - recipe.timestamp = t - result[recipe] = rrev_dict - return result - def items(self) -> Iterable[Tuple[RecipeReference, Dict[PkgReference, Dict]]]: """Iterate over the contents of the package list. @@ -371,19 +333,6 @@ def package_dict(self, pref: PkgReference): ref_dict = self.recipe_dict(pref.ref) return ref_dict["packages"][pref.package_id]["revisions"][pref.revision] - @staticmethod - def prefs(ref, recipe_bundle): - ConanOutput().warning("PackageLists.prefs() non-public, non-documented method will be " - "removed, use .items() instead", warn_tag="deprecated") - result = {} - for package_id, pkg_bundle in recipe_bundle.get("packages", {}).items(): - prevs = pkg_bundle.get("revisions", {}) - for prev, prev_bundle in prevs.items(): - t = prev_bundle.get("timestamp") - pref = PkgReference(ref, package_id, prev, t) - result[pref] = prev_bundle - return result - def serialize(self): """ Serialize the instance to a dictionary.""" return copy.deepcopy(self._data) diff --git a/conan/internal/api/detect/detect_api.py b/conan/internal/api/detect/detect_api.py index e39b81132ac..a59b92f5229 100644 --- a/conan/internal/api/detect/detect_api.py +++ b/conan/internal/api/detect/detect_api.py @@ -522,13 +522,6 @@ def detect_gcc_compiler(compiler_exe="gcc"): return None, None, None -def detect_compiler(): - ConanOutput(scope="detect_api").warning("detect_compiler() is deprecated, " - "use detect_default_compiler()", warn_tag="deprecated") - compiler, version, _ = detect_default_compiler() - return compiler, version - - def detect_intel_compiler(compiler_exe="icx"): try: ret, out = detect_runner(f'"{compiler_exe}" --version') diff --git a/conan/internal/api/profile/profile_loader.py b/conan/internal/api/profile/profile_loader.py index addf1f71b06..1428a079b51 100644 --- a/conan/internal/api/profile/profile_loader.py +++ b/conan/internal/api/profile/profile_loader.py @@ -229,7 +229,6 @@ class _ProfileValueParser: def get_profile(profile_text, base_profile=None): # Trying to strip comments might be problematic if things contain # doc = TextINIParse(profile_text, allowed_fields=["tool_requires", - "system_tools", # DEPRECATED: platform_tool_requires "platform_requires", "platform_tool_requires", "settings", "options", "conf", "buildenv", "runenv", @@ -241,10 +240,7 @@ def get_profile(profile_text, base_profile=None): tool_requires = _ProfileValueParser._parse_tool_requires(doc) doc_platform_requires = doc.platform_requires or "" - doc_platform_tool_requires = doc.platform_tool_requires or doc.system_tools or "" - if doc.system_tools: - ConanOutput().warning("Profile [system_tools] is deprecated," - " please use [platform_tool_requires]", warn_tag="deprecated") + doc_platform_tool_requires = doc.platform_tool_requires or "" def parse_replaces(replaces): result = [RecipeReference.loads(r) for r in replaces.splitlines()] diff --git a/conan/internal/cache/home_paths.py b/conan/internal/cache/home_paths.py index d7b81820d61..e85802b5768 100644 --- a/conan/internal/cache/home_paths.py +++ b/conan/internal/cache/home_paths.py @@ -22,11 +22,6 @@ def global_conf_path(self): @property def deployers_path(self): - deploy = os.path.join(self._home, _EXTENSIONS_FOLDER, "deploy") - if os.path.exists(deploy): - ConanOutput().warning("Use 'deployers' cache folder for deployers instead of 'deploy'", - warn_tag="deprecated") - return deploy return os.path.join(self._home, _EXTENSIONS_FOLDER, "deployers") @property diff --git a/conan/internal/graph/graph.py b/conan/internal/graph/graph.py index 9af88d7b07a..56c53ce2fc6 100644 --- a/conan/internal/graph/graph.py +++ b/conan/internal/graph/graph.py @@ -73,13 +73,6 @@ def __init__(self, ref, conanfile, context, recipe=None, path=None, test=False): self.skipped_build_requires = False self.editable_output_folder = None # In case this node is editable - @property - def dependencies(self): - ConanOutput().warning("Node.dependencies is private and shouldn't be used. It is now " - "node.edges. Please fix your code, Node.dependencies will be removed " - "in future versions", warn_tag="deprecated") - return self.edges - def subgraph(self): nodes = [self] opened = [self] diff --git a/conan/internal/model/conf.py b/conan/internal/model/conf.py index 65bffc073d2..87ae9a360ea 100644 --- a/conan/internal/model/conf.py +++ b/conan/internal/model/conf.py @@ -143,7 +143,7 @@ "tools.apple:enable_bitcode": "(boolean) Enable/Disable Bitcode Apple Clang flags", "tools.apple:enable_arc": "(boolean) Enable/Disable ARC Apple Clang flags", "tools.apple:enable_visibility": "(boolean) Enable/Disable Visibility Apple Clang flags", - "tools.env.virtualenv:powershell": "If specified, it generates PowerShell launchers (.ps1). Use this configuration setting the PowerShell executable you want to use (e.g., 'powershell.exe' or 'pwsh'). Setting it to True or False is deprecated as of Conan 2.11.0.", + "tools.env.virtualenv:powershell": "If specified, it generates PowerShell launchers (.ps1). Use this configuration setting the PowerShell executable you want to use (e.g., 'powershell.exe' or 'pwsh')", "tools.env:dotenv": "(Experimental) Generate dotenv environment files", "tools.env:deactivation_mode": "(Experimental) If 'function', generate a deactivate function instead of a script to unset the environment variables", # Compilers/Flags configurations diff --git a/conan/tools/cmake/cmakeconfigdeps/target_configuration.py b/conan/tools/cmake/cmakeconfigdeps/target_configuration.py index 29f19d95da1..d7b16c21cdb 100644 --- a/conan/tools/cmake/cmakeconfigdeps/target_configuration.py +++ b/conan/tools/cmake/cmakeconfigdeps/target_configuration.py @@ -22,13 +22,6 @@ def __init__(self, cmakedeps, conanfile, require, full_cpp_info): self._full_cpp_info = full_cpp_info def content(self): - auto_link = self._cmakedeps.get_property("cmake_set_interface_link_directories", - self._conanfile, check_type=bool) - if auto_link: - out = self._cmakedeps._conanfile.output # noqa - out.warning("CMakeConfigDeps: cmake_set_interface_link_directories deprecated and " - "invalid. The package 'package_info()' must correctly define the (CPS) " - "information", warn_tag="deprecated") t = Template(self._template, trim_blocks=True, lstrip_blocks=True, undefined=jinja2.StrictUndefined) return t.render(self._context) diff --git a/conan/tools/cmake/cmakedeps/templates/target_configuration.py b/conan/tools/cmake/cmakedeps/templates/target_configuration.py index d31cb1dc118..2ed26cd04f3 100644 --- a/conan/tools/cmake/cmakedeps/templates/target_configuration.py +++ b/conan/tools/cmake/cmakedeps/templates/target_configuration.py @@ -26,22 +26,13 @@ def context(self): components_names = [(components_target_name.replace("::", "_"), components_target_name) for components_target_name in components_targets_names] - is_win = self.conanfile.settings.get_safe("os") == "Windows" - auto_link = self.cmakedeps.get_property("cmake_set_interface_link_directories", - self.conanfile, check_type=bool) - if auto_link: - out = self.cmakedeps._conanfile.output # noqa - out.warning("CMakeDeps: cmake_set_interface_link_directories is legacy, not necessary", - warn_tag="deprecated") - return {"pkg_name": self.pkg_name, "root_target_name": self.root_target_name, "config_suffix": self.config_suffix, "config": self.configuration.upper(), "deps_targets_names": ";".join(deps_targets_names), "components_names": components_names, - "configuration": self.cmakedeps.configuration, - "set_interface_link_directories": auto_link and is_win} + "configuration": self.cmakedeps.configuration} @property def template(self): diff --git a/conan/tools/env/environment.py b/conan/tools/env/environment.py index 41ed9c8f7ac..87228d727a3 100644 --- a/conan/tools/env/environment.py +++ b/conan/tools/env/environment.py @@ -52,8 +52,7 @@ def environment_wrap_command(conanfile, env_filenames, env_folder, cmd, subsyste raise ConanException("Cannot wrap command with different envs," "{} - {}".format(bats+ps1s, shs)) - powershell = conanfile.conf.get("tools.env.virtualenv:powershell") or "powershell.exe" - powershell = "powershell.exe" if powershell is True else powershell + powershell = conanfile.conf.get("tools.env.virtualenv:powershell", default="powershell.exe") if bats: launchers = " && ".join('"{}"'.format(b) for b in bats) @@ -586,19 +585,7 @@ def save_script(self, filename): is_ps1 = ext == ".ps1" else: # Need to deduce it automatically is_bat = self._subsystem == WINDOWS - try: - is_ps1 = self._conanfile.conf.get("tools.env.virtualenv:powershell", check_type=bool) - if is_ps1 is not None: - ConanOutput().warning( - "Boolean values for 'tools.env.virtualenv:powershell' are deprecated. " - "Please specify 'powershell.exe' or 'pwsh' instead, appending arguments if needed " - "(for example: 'powershell.exe -argument'). " - "To unset this configuration, use `tools.env.virtualenv:powershell=!`, which matches " - "the previous 'False' behavior.", - warn_tag="deprecated" - ) - except ConanException: - is_ps1 = self._conanfile.conf.get("tools.env.virtualenv:powershell", check_type=str) + is_ps1 = self._conanfile.conf.get("tools.env.virtualenv:powershell", check_type=str) if is_ps1: filename = filename + ".ps1" is_bat = False diff --git a/conan/tools/microsoft/visual.py b/conan/tools/microsoft/visual.py index 1c56de02a07..4343b24445c 100644 --- a/conan/tools/microsoft/visual.py +++ b/conan/tools/microsoft/visual.py @@ -166,19 +166,7 @@ def generate(self, scope="build"): create_env_script(conanfile, content, conan_vcvars_bat, scope) _create_deactivate_vcvars_file(conanfile, conan_vcvars_bat) - try: - is_ps1 = self._conanfile.conf.get("tools.env.virtualenv:powershell", check_type=bool) - if is_ps1 is not None: - ConanOutput().warning( - "Boolean values for 'tools.env.virtualenv:powershell' are deprecated. " - "Please specify 'powershell.exe' or 'pwsh' instead, appending arguments " - "if needed (for example: 'powershell.exe -argument'). " - "To unset this configuration, use `tools.env.virtualenv:powershell=!`, " - "which matches the previous 'False' behavior.", - warn_tag="deprecated" - ) - except ConanException: - is_ps1 = self._conanfile.conf.get("tools.env.virtualenv:powershell", check_type=str) + is_ps1 = self._conanfile.conf.get("tools.env.virtualenv:powershell", check_type=str) if is_ps1: content_ps1 = textwrap.dedent(rf""" if (-not $env:VSCMD_ARG_VCVARS_VER){{ diff --git a/test/functional/command/test_install_deploy.py b/test/functional/command/test_install_deploy.py index 2cb6841ab7d..b8a8ed325b0 100644 --- a/test/functional/command/test_install_deploy.py +++ b/test/functional/command/test_install_deploy.py @@ -62,7 +62,7 @@ def deploy(graph, output_folder, **kwargs): "CMakeLists.txt": cmake, "main.cpp": gen_function_cpp(name="main", includes=["matrix"], calls=["matrix"])}, clean_first=True) - pwsh = "-c tools.env.virtualenv:powershell=True" if powershell else "" + pwsh = "-c tools.env.virtualenv:powershell=powershell.exe" if powershell else "" c.run("install . -o *:shared=True " f"--deployer=deploy.py -of=mydeploy -g CMakeToolchain -g CMakeDeps {pwsh}") c.run("remove * -c") # Make sure the cache is clean, no deps there @@ -184,9 +184,9 @@ def deploy(graph, output_folder, **kwargs): conanfile = graph.root.conanfile conanfile.output.info("deploy cache!!") """) - save(os.path.join(c.cache_folder, "extensions", "deploy", "deploy_cache.py"), deploy_cache) + save(os.path.join(c.cache_folder, "extensions", "deployers", "deploy_cache.py"), deploy_cache) # This should never be called in this test, always the local is found first - save(os.path.join(c.cache_folder, "extensions", "deploy", "mydeploy.py"), "CRASH!!!!") + save(os.path.join(c.cache_folder, "extensions", "deployers", "mydeploy.py"), "CRASH!!!!") c.save({"conanfile.txt": "", "mydeploy.py": deploy1, "sub/mydeploy2.py": deploy2}) diff --git a/test/functional/toolchains/env/test_virtualenv_powershell.py b/test/functional/toolchains/env/test_virtualenv_powershell.py index 3041c3a6fa9..d5288a644fa 100644 --- a/test/functional/toolchains/env/test_virtualenv_powershell.py +++ b/test/functional/toolchains/env/test_virtualenv_powershell.py @@ -73,7 +73,7 @@ def build(self): @pytest.mark.skipif(platform.system() != "Windows", reason="Requires Windows powershell") -@pytest.mark.parametrize("powershell", [True, "powershell.exe", "pwsh"]) +@pytest.mark.parametrize("powershell", ["powershell.exe", "pwsh"]) def test_virtualenv_test_package(powershell): """ The test_package could crash if not cleaning correctly the test_package output folder. This will still crassh if the layout is not creating different build folders @@ -119,7 +119,7 @@ def test(self): assert "MYVC_CUSTOMVAR2=PATATA2" in client.out @pytest.mark.skipif(platform.system() != "Windows", reason="Requires Windows powershell") -@pytest.mark.parametrize("powershell", [True, "powershell.exe", "pwsh"]) +@pytest.mark.parametrize("powershell", ["powershell.exe", "pwsh"]) def test_vcvars(powershell): client = TestClient() conanfile = textwrap.dedent(r""" @@ -164,7 +164,7 @@ def build(self): @pytest.mark.skipif(platform.system() != "Windows", reason="Test for powershell") -@pytest.mark.parametrize("powershell", [True, "powershell.exe", "pwsh", "powershell.exe -NoProfile", "pwsh -NoProfile"]) +@pytest.mark.parametrize("powershell", ["powershell.exe", "pwsh", "powershell.exe -NoProfile", "pwsh -NoProfile"]) def test_concatenate_build_and_run_env(powershell): # this tests that if we have both build and run env, they are concatenated correctly when using # powershell @@ -211,29 +211,8 @@ class Pkg(ConanFile): assert "MYTOOL 1!!" in client.out -@pytest.mark.parametrize("powershell", [None, True, False]) -def test_powershell_deprecated_message(powershell): - client = TestClient(light=True) - conanfile = textwrap.dedent("""\ - from conan import ConanFile - class Pkg(ConanFile): - settings = "os" - name = "pkg" - version = "0.1" - """) - - client.save({"conanfile.py": conanfile}) - powershell_arg = f'-c tools.env.virtualenv:powershell={powershell}' if powershell is not None else "" - client.run(f'install . {powershell_arg}') - # only show message if the value is set to False or True if not set do not show message - if powershell is not None: - assert "Boolean values for 'tools.env.virtualenv:powershell' are deprecated" in client.out - else: - assert "Boolean values for 'tools.env.virtualenv:powershell' are deprecated" not in client.out - - @pytest.mark.skipif(platform.system() != "Windows", reason="Test for powershell") -@pytest.mark.parametrize("powershell", [True, "pwsh", "powershell.exe"]) +@pytest.mark.parametrize("powershell", ["pwsh", "powershell.exe"]) def test_powershell_quoting(powershell): client = TestClient(path_with_spaces=False) conanfile = textwrap.dedent("""\ diff --git a/test/integration/cache/test_home_special_char.py b/test/integration/cache/test_home_special_char.py index b6234c304cd..843d31e6664 100644 --- a/test/integration/cache/test_home_special_char.py +++ b/test/integration/cache/test_home_special_char.py @@ -66,6 +66,6 @@ def test_reuse_buildenv(client_with_special_chars): @pytest.mark.skipif(platform.system() != "Windows", reason="powershell only win") def test_reuse_buildenv_powershell(client_with_special_chars): c = client_with_special_chars - c.run("create . -c tools.env.virtualenv:powershell=True") + c.run("create . -c tools.env.virtualenv:powershell=powershell.exe") assert _path_chars in c.out assert "MYTOOL WORKS!!" in c.out diff --git a/test/integration/graph/test_system_tools.py b/test/integration/graph/test_system_tools.py index 5526bb9c267..623449d66f1 100644 --- a/test/integration/graph/test_system_tools.py +++ b/test/integration/graph/test_system_tools.py @@ -18,16 +18,15 @@ def test_system_tool_require(self, revision): assert f"tool/1.0{revision or '#platform'} - Platform" in client.out def test_system_tool_require_non_matching(self): - """ if what is specified in [system_tool_require] doesn't match what the recipe requires, then - the system_tool_require will not be used, and the recipe will use its declared version + """ if what is specified in [platform_tool_requires] doesn't match what the recipe requires, then + the platform_tool_requires will not be used, and the recipe will use its declared version """ client = TestClient(light=True) client.save({"tool/conanfile.py": GenConanfile("tool", "1.0"), "conanfile.py": GenConanfile("pkg", "1.0").with_tool_requires("tool/1.0"), - "profile": "[system_tools]\ntool/1.1"}) + "profile": "[platform_tool_requires]\ntool/1.1"}) client.run("create tool") client.run("create . -pr=profile") - assert "WARN: deprecated: Profile [system_tools] is deprecated" in client.out assert "tool/1.0#60ed6e65eae112df86da7f6d790887fd - Cache" in client.out @pytest.mark.parametrize("revision", ["", "#myrev"]) diff --git a/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps.py b/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps.py index b12fffac0b0..262050ac99a 100644 --- a/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps.py +++ b/test/integration/toolchains/cmake/cmakeconfigdeps/test_cmakeconfigdeps.py @@ -219,26 +219,6 @@ def package_info(self): ' $<$:my_system_cool_lib>)' in cmake -def test_autolink_pragma(): - """https://github.com/conan-io/conan/issues/10837""" - c = TestClient() - conanfile = textwrap.dedent(""" - from conan import ConanFile - class Pkg(ConanFile): - def package_info(self): - self.cpp_info.set_property("cmake_set_interface_link_directories", True) - """) - c.save({"conanfile.py": conanfile, - "test_package/conanfile.py": GenConanfile().with_test("pass") - .with_settings("build_type") - .with_generator("CMakeConfigDeps")}) - c.run("create . --name=pkg --version=0.1") - assert "CMakeConfigDeps: cmake_set_interface_link_directories deprecated" in c.out - c.run(f"create . --name=pkg --version=0.1") - assert "CMakeConfigDeps: cmake_set_interface_link_directories deprecated and invalid. " \ - "The package 'package_info()' must correctly define the (CPS) information" in c.out - - def test_consuming_cpp_info_with_components_dependency_from_same_package(): c = TestClient() conanfile = textwrap.dedent(""" From 49f72a43d2acdb38b63ccc8b3f4f157d671d5d72 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 17 Apr 2026 16:43:56 +0200 Subject: [PATCH 100/110] Refactor/sources caching downloaded (#19789) * refactor, tests not passing yet * wip * wip * wip * refactor --- .../internal/rest/caching_file_downloader.py | 162 ++++++++---------- test/integration/cache/backup_sources_test.py | 27 +-- 2 files changed, 85 insertions(+), 104 deletions(-) diff --git a/conan/internal/rest/caching_file_downloader.py b/conan/internal/rest/caching_file_downloader.py index fff4c8489af..f64d863d8a8 100644 --- a/conan/internal/rest/caching_file_downloader.py +++ b/conan/internal/rest/caching_file_downloader.py @@ -29,101 +29,78 @@ def __init__(self, conanfile): def download(self, urls, file_path, retry, retry_wait, verify_ssl, auth, headers, md5, sha1, sha256): download_cache_folder = self._global_conf.get("core.sources:download_cache") - backups_urls = self._global_conf.get("core.sources:download_urls", check_type=list) - if not (backups_urls or download_cache_folder) or not sha256: - # regular, non backup/caching download - if backups_urls or download_cache_folder: - self._output.warning("Cannot cache download() without sha256 checksum") - self._download_from_urls(urls, file_path, retry, retry_wait, verify_ssl, auth, headers, - md5, sha1, sha256) - else: - self._caching_download(urls, file_path, - retry, retry_wait, verify_ssl, auth, headers, md5, sha1, sha256, - download_cache_folder, backups_urls) - - def _caching_download(self, urls, file_path, - retry, retry_wait, verify_ssl, auth, headers, md5, sha1, sha256, - download_cache_folder, backups_urls): - """ - this download will first check in the local cache, if not there, it will go to the list - of backup_urls defined by user conf (by default ["origin"]), and iterate it until - something is found. - """ - # We are going to use the download_urls definition for backups - download_cache_folder = download_cache_folder or HomePaths(self._home_folder).default_sources_backup_folder - # regular local shared download cache, not using Conan backup sources servers - backups_urls = backups_urls or ["origin"] + source_origins = self._global_conf.get("core.sources:download_urls", check_type=list) + if source_origins and not download_cache_folder: + # If backups are defined, but the download cache is not defined, use a default one + download_cache_folder = HomePaths(self._home_folder).default_sources_backup_folder if download_cache_folder and not os.path.isabs(download_cache_folder): raise ConanException("core.sources:download_cache must be an absolute path") - - download_cache = DownloadCache(download_cache_folder) - cached_path = download_cache.source_path(sha256) - with download_cache.lock(sha256): - remove_if_dirty(cached_path) - - if os.path.exists(cached_path): - self._output.info(f"Source {urls} retrieved from local download cache") - else: - with set_dirty_context_manager(cached_path): - if None in backups_urls: - raise ConanException("Trying to download sources from None backup remote." - f" Remotes were: {backups_urls}") - for backup_url in backups_urls: - is_last = backup_url is backups_urls[-1] - if backup_url == "origin": # recipe defined URLs - if self._origin_download(urls, cached_path, retry, retry_wait, - verify_ssl, auth, headers, md5, sha1, sha256, - is_last): - break - else: - if self._backup_download(backup_url, backups_urls, sha256, cached_path, - urls, is_last): - break - - download_cache.update_backup_sources_json(cached_path, self._conanfile, urls) - # Everything good, file in the cache, just copy it to final destination - mkdir(os.path.dirname(file_path)) - shutil.copy2(cached_path, file_path) - - def _origin_download(self, urls, cached_path, retry, retry_wait, - verify_ssl, auth, headers, md5, sha1, sha256, is_last): - """ download from the internet, the urls provided by the recipe (mirrors). - """ - try: - self._download_from_urls(urls, cached_path, retry, retry_wait, - verify_ssl, auth, headers, md5, sha1, sha256) - except ConanException as e: - if is_last: - raise - else: - # TODO: Improve printing of AuthenticationException - self._output.warning(f"Sources for {urls} failed in 'origin': {e}") - self._output.warning("Checking backups") + if download_cache_folder and not sha256: + self._output.warning("Cannot cache download() without sha256 checksum") + download_cache_folder = None # Cannot cache + source_origins = source_origins or ["origin"] + if None in source_origins: + raise ConanException(f"Incorrect 'core.sources:download_urls' contains invalid 'None'" + f"url: {source_origins}") + + # First, see if it is already in the download cache + if download_cache_folder: + download_cache = DownloadCache(download_cache_folder) + download_path = download_cache.source_path(sha256) + with download_cache.lock(sha256): + remove_if_dirty(download_path) + + if os.path.exists(download_path): + self._output.info(f"Source {urls} retrieved from local download cache") + else: + # not in cache, we need to actually download from internet or backup servers + with set_dirty_context_manager(download_path): + self._do_download(source_origins, urls, download_path, retry, retry_wait, + verify_ssl, auth, headers, md5, sha1, sha256) + + # copy it to the package "source" folder + os.makedirs(os.path.dirname(file_path), exist_ok=True) + shutil.copy2(download_path, file_path) + download_cache.update_backup_sources_json(download_path, self._conanfile, urls) else: - if not is_last: - self._output.info(f"Sources for {urls} found in origin") - return True - - def _backup_download(self, backup_url, backups_urls, sha256, cached_path, urls, is_last): - """ download from a Conan backup sources file server, like an Artifactory generic repo - All failures are bad, except NotFound. The server must be live, working and auth, we - don't want silently skipping a backup because it is down. - """ - try: - backup_url = backup_url if backup_url.endswith("/") else backup_url + "/" - self._file_downloader.download(backup_url + sha256, cached_path, sha256=sha256) - self._file_downloader.download(backup_url + sha256 + ".json", cached_path + ".json") - self._output.info(f"Sources for {urls} found in remote backup {backup_url}") - return True - except NotFoundException: - if is_last: - raise NotFoundException(f"File {urls} not found in {backups_urls}") - else: - self._output.warning(f"File {urls} not found in {backup_url}") - except (AuthenticationException, ForbiddenException) as e: - raise ConanException(f"Authentication to source backup server '{backup_url}' " - f"failed: {e}. " - f"Please check your 'source_credentials.json'") + # This doesn't need to be dirty-protected, as the full "source" folder is protected + self._do_download(source_origins, urls, file_path, retry, retry_wait, verify_ssl, auth, + headers, md5, sha1, sha256) + + def _do_download(self, source_origins, urls, download_path, retry, retry_wait, verify_ssl, + auth, headers, md5, sha1, sha256): + # iterates the origins until one works + for backup_url in source_origins: + if backup_url == "origin": # download from the internet + try: + self._download_from_urls(urls, download_path, retry, retry_wait, verify_ssl, + auth, headers, md5, sha1, sha256) + return + except Exception as e: + if backup_url is source_origins[-1]: + raise + self._output.warning(f"Sources for {urls} failed in 'origin': {e}") + else: # Download from a backup server + try: + self._output.info(f"Checking backup: {backup_url}") + backup_url = backup_url if backup_url.endswith("/") else backup_url + "/" + # The download happens to the user download folder, not to the download cache + self._file_downloader.download(backup_url + sha256, download_path, + sha256=sha256, overwrite=True) + self._file_downloader.download(backup_url + sha256 + ".json", + download_path + ".json", overwrite=True) + self._output.info(f"Sources for {urls} found in remote backup {backup_url}") + return + except NotFoundException: + msg = f"Sources for {urls} not found in remote backup {backup_url}" + if backup_url is source_origins[-1]: + raise NotFoundException(msg) + else: + self._output.warning(msg) + except (AuthenticationException, ForbiddenException) as e: + raise ConanException(f"Authentication to source backup server '{backup_url}' " + f"failed: {e}. " + f"Please check your 'source_credentials.json'") def _download_from_urls(self, urls, file_path, retry, retry_wait, verify_ssl, auth, headers, md5, sha1, sha256): @@ -142,6 +119,7 @@ def _download_from_urls(self, urls, file_path, retry, retry_wait, verify_ssl, au else: self._file_downloader.download(url, file_path, retry, retry_wait, verify_ssl, auth, True, headers, md5, sha1, sha256) + self._output.info(f"Sources correctly downloaded from {url}") return # Success! Return to caller except Exception as error: if url != urls[-1]: # If it is not the last one, do not raise, warn and move to next diff --git a/test/integration/cache/backup_sources_test.py b/test/integration/cache/backup_sources_test.py index 0b165939d8d..4862907a75d 100644 --- a/test/integration/cache/backup_sources_test.py +++ b/test/integration/cache/backup_sources_test.py @@ -145,7 +145,7 @@ def source(self): self.client.save({"conanfile.py": conanfile}) self.client.run("create .", assert_error=True) - assert "Trying to download sources from None backup remote" in self.client.out + assert "Incorrect 'core.sources:download_urls' contains invalid 'None'" in self.client.out self.client.save_home( {"global.conf": f"core.sources:download_cache={self.download_cache_folder}\n" @@ -190,10 +190,11 @@ def source(self): f"core.sources:upload_url={self.file_server.fake_url}/backups/\n" f"core.sources:exclude_urls=['{self.file_server.fake_url}/mycompanystorage/', '{self.file_server.fake_url}/mycompanystorage2/']"}) self.client.run("source .") + assert (f"Sources for {self.file_server.fake_url}/internet/myfile.txt found in " f"remote backup") in self.client.out - assert (f"File {self.file_server.fake_url}/mycompanystorage/mycompanyfile.txt not " - f"found in {self.file_server.fake_url}/backups/") in self.client.out + assert (f"Sources for {self.file_server.fake_url}/mycompanystorage/mycompanyfile.txt not " + f"found in remote backup {self.file_server.fake_url}/backups/") in self.client.out # Ensure defaults backup folder works if it's not set in global.conf # (The rest is needed to exercise the rest of the code) @@ -203,8 +204,8 @@ def source(self): self.client.run("source .") assert (f"Sources for {self.file_server.fake_url}/internet/myfile.txt found in " f"remote backup") in self.client.out - assert (f"File {self.file_server.fake_url}/mycompanystorage/mycompanyfile.txt not " - f"found in {self.file_server.fake_url}/backups/") in self.client.out + assert (f"Sources for {self.file_server.fake_url}/mycompanystorage/mycompanyfile.txt not " + f"found in remote backup {self.file_server.fake_url}/backups/") in self.client.out def test_unknown_handling(self): http_server_base_folder_internet = os.path.join(self.file_server.store, "internet") @@ -273,8 +274,8 @@ def source(self): rmdir(self.download_cache_folder) self.client.run("source .") - assert (f"Sources for {self.file_server.fake_url}/internet/myfile.txt found " - f"in origin") in self.client.out + assert (f"Sources correctly downloaded from " + f"{self.file_server.fake_url}/internet/myfile.txt") in self.client.out self.client.run("source .") assert (f"Source {self.file_server.fake_url}/internet/myfile.txt retrieved from " f"local download cache") in self.client.out @@ -303,8 +304,10 @@ def source(self): self.client.save({"conanfile.py": conanfile}) self.client.run("create . -vv") - assert (f"WARN: File {self.file_server.fake_url}/internet/myfile.txt not found " - f"in {self.file_server.fake_url}/backup/") in self.client.out + assert (f"WARN: Sources for {self.file_server.fake_url}/internet/myfile.txt not found " + f"in remote backup {self.file_server.fake_url}/backup/") in self.client.out + assert (f"Downloaded {self.file_server.fake_url}/internet/myfile.txt " + f"from {self.file_server.fake_url}/internet/myfile.txt") self.client.run("upload * -c -r=default") rmdir(self.download_cache_folder) @@ -546,9 +549,9 @@ def source(self): client.save({"conanfile.py": conanfile}) client.run("source .", assert_error=True) assert "WARN: Sources for http://fake/myfile.txt failed in 'origin'" in client.out - assert "WARN: Checking backups" in client.out - assert "NotFoundException: File http://fake/myfile.txt " \ - "not found in ['origin', 'http://extrafake/']" in client.out + assert "Checking backup" in client.out + assert "NotFoundException: Sources for http://fake/myfile.txt " \ + "not found in remote backup http://extrafake/" in client.out def test_ok_when_origin_breaks_midway_list(self): http_server_base_folder_backup2 = os.path.join(self.file_server.store, "backup2") From 703eae0e25a781b858bbc1d28fdd963fc2cbf3f9 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 21 Apr 2026 07:04:29 +0200 Subject: [PATCH 101/110] checking config_requires from lockfiles at conan-install time (#19875) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * checking config_requires from lockfiles at conan-install time * api docstrings (#19871) * api docstrings * more * Warn when credentials env vars are ignored due to anonymous server access (#19872) add warning * tools.build:install_strip now accepts a list of possible build systems (#19874) * tools.build:install_strip now accepts a list of possible build systems * Modernize build-order tests to use order-by (#19878) * Modernize build-order tests to use order-by * Modernize build-order tests to use order-by * Allow patterns in `--update` flag (#19856) * Pattern for ref name * Simplify diff * Update help text· * Remove some deprecated features (#19877) * Remove deprecated Node::dependencies method * Remove deprecated cmake_set_interface_link_directories property * Remove deprecated methods from PackagesList * Remove deprecated deploy folder in conan home * Remove deprecated message for order-by in conan graph info build-order * Remove deprecated detect_compiler method in detect api * Remove deprecated system_tools profile section * Remove deprecated boolean powershell conf values * Revert "Remove deprecated message for order-by in conan graph info build-order" This reverts commit 1c09c83f7ac63077bc8e7f4d8e26d095e158f3c9. * Last test changes * Remove deoply usage in test * Remove last true for test * Refactor/sources caching downloaded (#19789) * refactor, tests not passing yet * wip * wip * wip * refactor * wip * review * added to other commands * review * wip --------- Co-authored-by: Carlos Zoido Co-authored-by: PerseoGI Co-authored-by: Abril Rincón Blanco <5364255+AbrilRBS@users.noreply.github.com> --- conan/api/subapi/lockfile.py | 12 ++ conan/cli/commands/audit.py | 1 + conan/cli/commands/build.py | 1 + conan/cli/commands/create.py | 1 + conan/cli/commands/export_pkg.py | 1 + conan/cli/commands/graph.py | 4 + conan/cli/commands/install.py | 1 + conan/cli/commands/lock.py | 2 + conan/cli/commands/test.py | 1 + conan/cli/commands/workspace.py | 4 + conan/internal/model/lockfile.py | 28 +++++ .../command/test_config_install_pkg.py | 13 +++ test/integration/command/config/__init__.py | 0 .../command/{ => config}/config_test.py | 0 .../command/config/test_config_install_pkg.py | 103 ++++++++++++++++++ 15 files changed, 172 insertions(+) create mode 100644 test/integration/command/config/__init__.py rename test/integration/command/{ => config}/config_test.py (100%) create mode 100644 test/integration/command/config/test_config_install_pkg.py diff --git a/conan/api/subapi/lockfile.py b/conan/api/subapi/lockfile.py index dc056a3bb46..63ee687f3dd 100644 --- a/conan/api/subapi/lockfile.py +++ b/conan/api/subapi/lockfile.py @@ -2,8 +2,10 @@ from conan.api.output import ConanOutput from conan.cli import make_abs_path +from conan.internal.cache.home_paths import HomePaths from conan.internal.graph.graph import Overrides from conan.errors import ConanException +from conan.internal.model.conanconfig import loadconanconfig from conan.internal.model.lockfile import Lockfile, LOCKFILE @@ -60,6 +62,16 @@ def get_lockfile(lockfile=None, conanfile_path=None, cwd=None, partial=False, ConanOutput().info("Using lockfile: '{}'".format(lockfile_path)) return graph_lock + def check_lockfile_config(self, lockfile): + """Verify that installed configurations are aligned with lockfile config_requires. + """ + if lockfile is None: + return + + config_version_path = HomePaths(self._conan_api.home_folder).config_version_path + refs = loadconanconfig(config_version_path) if os.path.exists(config_version_path) else [] + lockfile.check_config_requires(refs) + def update_lockfile_export(self, lockfile, conanfile, ref, is_build_require=False): # The package_type is not fully processed at export is_python_require = conanfile.package_type == "python-require" diff --git a/conan/cli/commands/audit.py b/conan/cli/commands/audit.py index 32654e1dd66..32d27d4b731 100644 --- a/conan/cli/commands/audit.py +++ b/conan/cli/commands/audit.py @@ -78,6 +78,7 @@ def audit_scan(conan_api: ConanAPI, parser, subparser, *args) -> dict: overrides = eval(args.lockfile_overrides) if args.lockfile_overrides else None lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=path, cwd=cwd, partial=args.lockfile_partial, overrides=overrides) + conan_api.lockfile.check_lockfile_config(lockfile) profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) print_profiles(profile_host, profile_build) diff --git a/conan/cli/commands/build.py b/conan/cli/commands/build.py index d1a79cf9aaa..f6458885898 100644 --- a/conan/cli/commands/build.py +++ b/conan/cli/commands/build.py @@ -48,6 +48,7 @@ def build(conan_api, parser, *args): overrides = eval(args.lockfile_overrides) if args.lockfile_overrides else None lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=path, cwd=cwd, partial=args.lockfile_partial, overrides=overrides) + conan_api.lockfile.check_lockfile_config(lockfile) profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) print_profiles(profile_host, profile_build) diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index 25d5c3930af..96d0140f983 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -45,6 +45,7 @@ def create(conan_api, parser, *args): cwd=cwd, partial=args.lockfile_partial, overrides=overrides) + conan_api.lockfile.check_lockfile_config(lockfile) remotes = conan_api.remotes.list(args.remote) if not args.no_remote else [] profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) diff --git a/conan/cli/commands/export_pkg.py b/conan/cli/commands/export_pkg.py index f17b984cfae..21f684db254 100644 --- a/conan/cli/commands/export_pkg.py +++ b/conan/cli/commands/export_pkg.py @@ -43,6 +43,7 @@ def export_pkg(conan_api, parser, *args): lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=path, cwd=cwd, partial=args.lockfile_partial, overrides=overrides) + conan_api.lockfile.check_lockfile_config(lockfile) profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) remotes = conan_api.remotes.list(args.remote) if not args.no_remote else [] output_folder = make_abs_path(args.output_folder, cwd) if args.output_folder else None diff --git a/conan/cli/commands/graph.py b/conan/cli/commands/graph.py index 0e94e186c33..84a5b8503f7 100644 --- a/conan/cli/commands/graph.py +++ b/conan/cli/commands/graph.py @@ -87,6 +87,7 @@ def graph_build_order(conan_api, parser, subparser, *args): cwd=cwd, partial=args.lockfile_partial, overrides=overrides) + conan_api.lockfile.check_lockfile_config(lockfile) profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) if path: @@ -185,6 +186,7 @@ def graph_info(conan_api, parser, subparser, *args): cwd=cwd, partial=args.lockfile_partial, overrides=overrides) + conan_api.lockfile.check_lockfile_config(lockfile) profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) print_profiles(profile_host, profile_build) @@ -266,6 +268,7 @@ def graph_explain(conan_api, parser, subparser, *args): cwd=cwd, partial=args.lockfile_partial, overrides=overrides) + conan_api.lockfile.check_lockfile_config(lockfile) profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) if path: @@ -347,6 +350,7 @@ def graph_outdated(conan_api, parser, subparser, *args): cwd=cwd, partial=args.lockfile_partial, overrides=overrides) + conan_api.lockfile.check_lockfile_config(lockfile) profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) if path: diff --git a/conan/cli/commands/install.py b/conan/cli/commands/install.py index 61643aeeab7..38cf27d2075 100644 --- a/conan/cli/commands/install.py +++ b/conan/cli/commands/install.py @@ -75,6 +75,7 @@ def _run_install_command(conan_api, args, cwd, return_install_error=True): overrides = eval(args.lockfile_overrides) if args.lockfile_overrides else None lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=path, cwd=cwd, partial=args.lockfile_partial, overrides=overrides) + conan_api.lockfile.check_lockfile_config(lockfile) profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) print_profiles(profile_host, profile_build) diff --git a/conan/cli/commands/lock.py b/conan/cli/commands/lock.py index db27e806810..5161e43bb40 100644 --- a/conan/cli/commands/lock.py +++ b/conan/cli/commands/lock.py @@ -35,6 +35,7 @@ def lock_create(conan_api, parser, subparser, *args): overrides = eval(args.lockfile_overrides) if args.lockfile_overrides else None lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=path, cwd=cwd, partial=True, overrides=overrides) + conan_api.lockfile.check_lockfile_config(lockfile) profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) if path: @@ -204,6 +205,7 @@ def lock_upgrade(conan_api, parser, subparser, *args): cwd=cwd, partial=True, overrides=overrides) if lockfile is None: raise ConanException("No lockfile specified and default conan.lock not found") + conan_api.lockfile.check_lockfile_config(lockfile) profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) # Remove the lockfile entries that will be updated diff --git a/conan/cli/commands/test.py b/conan/cli/commands/test.py index ca8d461832b..d942c9da7bf 100644 --- a/conan/cli/commands/test.py +++ b/conan/cli/commands/test.py @@ -34,6 +34,7 @@ def test(conan_api, parser, *args): cwd=cwd, partial=args.lockfile_partial, overrides=overrides) + conan_api.lockfile.check_lockfile_config(lockfile) remotes = conan_api.remotes.list(args.remote) if not args.no_remote else [] profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) diff --git a/conan/cli/commands/workspace.py b/conan/cli/commands/workspace.py index 18ab3fb9245..e9985631a6b 100644 --- a/conan/cli/commands/workspace.py +++ b/conan/cli/commands/workspace.py @@ -94,6 +94,7 @@ def workspace_complete(conan_api: ConanAPI, parser, subparser, *args): ws_folder = conan_api.workspace.folder() lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=ws_folder, cwd=None, partial=args.lockfile_partial) + conan_api.lockfile.check_lockfile_config(lockfile) profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) print_profiles(profile_host, profile_build) @@ -171,6 +172,7 @@ def _install_build(conan_api: ConanAPI, parser, subparser, build, *args): ws_folder = conan_api.workspace.folder() lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=ws_folder, cwd=None, partial=args.lockfile_partial) + conan_api.lockfile.check_lockfile_config(lockfile) profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) print_profiles(profile_host, profile_build) @@ -257,6 +259,7 @@ def workspace_super_install(conan_api: ConanAPI, parser, subparser, *args): lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=ws_folder, cwd=None, partial=args.lockfile_partial, overrides=overrides) + conan_api.lockfile.check_lockfile_config(lockfile) profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) print_profiles(profile_host, profile_build) @@ -337,6 +340,7 @@ def workspace_create(conan_api: ConanAPI, parser, subparser, *args): ws_folder = conan_api.workspace.folder() lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=ws_folder, cwd=None, partial=args.lockfile_partial) + conan_api.lockfile.check_lockfile_config(lockfile) profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) print_profiles(profile_host, profile_build) diff --git a/conan/internal/model/lockfile.py b/conan/internal/model/lockfile.py index d93a5f7436e..f5a54684266 100644 --- a/conan/internal/model/lockfile.py +++ b/conan/internal/model/lockfile.py @@ -317,6 +317,34 @@ def resolve_prev(self, node): if prevs: return prevs.get(node.package_id) + def check_config_requires(self, installed_refs): + # Validates that the given installed configuration packages satisfy the current lockfile + # For that to happen, the installed conf packages must match the lockfile ones + # Lockfile ones can be partial, like not containing recipe-revision + # And also all 'config_requires' in the lockfile must have a configuration package for them + # In case of a lockfile containing several constraints, one per package name must exist + lockfile_refs = set(self._conf_requires.refs()) + if not lockfile_refs: + return # If lockfile is not locking config_requires, do nothing, would break + + # Existing installed config packages must exist in the lockfile + not_locked = [r for r in installed_refs if r not in lockfile_refs] + if not_locked: + raise ConanException(f"Installed config packages {not_locked} not in the lockfile") + + # Config-requires in the lockfile must be installed, at least 1 per package name + # so first, group by package name + lockfile_refs_by_name = {} + for r in self._conf_requires.refs(): + lockfile_refs_by_name.setdefault(r.name, []).append(r) + not_installed = [] + for refs in lockfile_refs_by_name.values(): + if not any(r in installed_refs for r in refs): + not_installed.extend(refs) + if not_installed: + raise ConanException("There are config packages in lockfile " + f"'config_requires' not installed: {not_installed}") + def _resolve(self, require, locked_refs, resolve_prereleases, kind): version_range = require.version_range ref = require.ref diff --git a/test/functional/command/test_config_install_pkg.py b/test/functional/command/test_config_install_pkg.py index f3f4408a15f..65e79c58f9d 100644 --- a/test/functional/command/test_config_install_pkg.py +++ b/test/functional/command/test_config_install_pkg.py @@ -185,6 +185,19 @@ def test_lockfile(self, servers): result = json.loads(c.load("config.lock")) assert "myconf_a/0.2" in result["config_requires"][0] + def test_lockfile_multiple(self, servers): + c = TestClient(servers=servers, light=True) + c.run("config install-pkg myconf_a/0.1 --lockfile-out=config.lock") + c.run("config install-pkg myconf_a/0.2 --lockfile=config.lock --lockfile-out=config.lock " + "--lockfile-partial") + _check_conf(c, "myconf_a/0.2") + _check_conf_file(c, ["myconf_a/0.2"]) + + # The lockfile can contain both, not ideal, but possible + result = json.loads(c.load("config.lock")) + assert "myconf_a/0.2" in result["config_requires"][0] + assert "myconf_a/0.1" in result["config_requires"][1] + def test_package_id_effect(self): # full integration, as "test_config_package_id.py" tests from hardcoded cache json files c = TestClient(light=True) diff --git a/test/integration/command/config/__init__.py b/test/integration/command/config/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/integration/command/config_test.py b/test/integration/command/config/config_test.py similarity index 100% rename from test/integration/command/config_test.py rename to test/integration/command/config/config_test.py diff --git a/test/integration/command/config/test_config_install_pkg.py b/test/integration/command/config/test_config_install_pkg.py new file mode 100644 index 00000000000..d7bcc71946a --- /dev/null +++ b/test/integration/command/config/test_config_install_pkg.py @@ -0,0 +1,103 @@ +import json + +from conan.test.assets.genconanfile import GenConanfile +from conan.test.utils.tools import TestClient + + +class TestInstallLockfileConfigRequires: + """Tests for the alignment check between lockfile config_requires and installed configs.""" + + def test_install_no_lockfile(self): + """conan install without a lockfile skips the check entirely.""" + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile("pkg", "1.0")}) + c.save_home({"config_version.json": json.dumps( + {"config_version": ["myconf/1.0#aabbcc"]})}) + c.run("install") + assert "ERROR" not in c.out + + def test_install_empty_lockfile(self): + """conan install with an empty lockfile skips the check entirely.""" + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile("pkg", "1.0"), + "conan.lock": "{}"}) + c.save_home({"config_version.json": json.dumps( + {"config_version": ["myconf/1.0#aabbcc"]})}) + c.run("install") + assert "ERROR" not in c.out + + def test_install_missing_installed_config_error(self): + """Lockfile with config_requires not installed => Error""" + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile("pkg", "1.0"), + "conan.lock": json.dumps({ + "version": "0.5", + "config_requires": ["myconf/1.0#aabbcc", "other/2.1#rev2"] + })}) + c.run("install . --lockfile=conan.lock", assert_error=True) + assert ("ERROR: There are config packages in lockfile 'config_requires' " + "not installed: [myconf/1.0#aabbcc, other/2.1#rev2]") in c.out + + def test_install_missing_installed_config_error_same(self): + """Same as above but with multiple versions of same package""" + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile("pkg", "1.0"), + "conan.lock": json.dumps({ + "version": "0.5", + "config_requires": ["myconf/2.0#aabbcc", "myconf/1.0#aabbcc"] + })}) + c.run("install . --lockfile=conan.lock", assert_error=True) + assert ("ERROR: There are config packages in lockfile 'config_requires' " + "not installed: [myconf/2.0#aabbcc, myconf/1.0#aabbcc]") in c.out + + def test_install_lockfile_config_match(self): + """Installed config matches lockfile entry -> passes.""" + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile("pkg", "1.0"), + "conan.lock": json.dumps({ + "version": "0.5", + "config_requires": ["myconf/1.0#rev1"] + })}) + c.save_home({"config_version.json": json.dumps( + {"config_version": ["myconf/1.0#rev1"]})}) + c.run("install . --lockfile=conan.lock") + assert "ERROR" not in c.out + + def test_install_lockfile_config_partial_match(self): + """Lockfile dont lock down to the revision, only version""" + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile("pkg", "1.0"), + "conan.lock": json.dumps({ + "version": "0.5", + "config_requires": ["myconf/1.0"] + })}) + c.save_home({"config_version.json": json.dumps( + {"config_version": ["myconf/1.0#rev1"]})}) + c.run("install . --lockfile=conan.lock") + assert "ERROR" not in c.out + + def test_install_lockfile_config_revision_mismatch_error(self): + """Installed config has different revision than lockfile -> error.""" + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile("pkg", "1.0"), + "conan.lock": json.dumps({ + "version": "0.5", + "config_requires": ["myconf/1.0#rev_in_lockfile"] + })}) + c.save_home({"config_version.json": json.dumps( + {"config_version": ["myconf/1.0#rev"]})}) + c.run("install . --lockfile=conan.lock", assert_error=True) + assert "ERROR: Installed config packages [myconf/1.0#rev] not in the lockfile" in c.out + + def test_install_lockfile_config_version_mismatch(self): + """Installed config has different version than lockfile entry -> error.""" + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile("pkg", "1.0"), + "conan.lock": json.dumps({ + "version": "0.5", + "config_requires": ["myconf/2.0#aabbcc"] + })}) + c.save_home({"config_version.json": json.dumps( + {"config_version": ["myconf/1.0#aabbcc"]})}) + c.run("install . --lockfile=conan.lock", assert_error=True) + assert "ERROR: Installed config packages [myconf/1.0#aabbcc] not in the lockfile" in c.out From 490772b05d0f81398f006712ac08c19d38298db0 Mon Sep 17 00:00:00 2001 From: Alexander Krabler Date: Tue, 21 Apr 2026 08:00:15 +0200 Subject: [PATCH 102/110] detect_libcxx: Remove temporary directory usage, using safe parameters instead (#19838) * detect_libcxx: Remove temporary directory after use `detect_libxx` tries to detect the C++ standard library implementation. When running `conan profile detect`, a temporary folder is created, where a minimal C++ translation unit is stored. This unit gets compiled in order to test which standard library is in use. Due to the use of `tempfile.mkdtemp()`, cleanup must be done manually, but was omitted. Migrate to using a context manager instead (`tempfile.TemporaryDirectory`), cleaning automatically when the scope ends. Ignoring cleanup errors as it is not a huge problem when there are some remaining files still available. Remove unnecessary using-directive for `std` namespace, too. Changelog: omit Docs: omit * detect_libcxx: Remove temporary directory completely The temporary directory was used for two purposes: * Storing the input file (`main.cpp`). * Storing the output file (`a.out`). Both of these can be quite easily replaced: * The input can be passed via stdin to the compiler. `-xc++` is necessary to tell the compiler which language the input is, as autodetection from filename doesn't work anymore. `-` is used to tell the compiler to use stdin. * The output can be omitted by using `-fsyntax-only`, which tells the compiler to stop early and don't write output at all. The output is not relevant anyway, only the returncode and error message was used. Together, the whole chdir and temporary file logic is obsolete and can be removed. --- conan/internal/api/detect/detect_api.py | 37 ++++++++++++------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/conan/internal/api/detect/detect_api.py b/conan/internal/api/detect/detect_api.py index a59b92f5229..6e380a2e492 100644 --- a/conan/internal/api/detect/detect_api.py +++ b/conan/internal/api/detect/detect_api.py @@ -1,6 +1,7 @@ import os import platform import re +import subprocess import tempfile import textwrap @@ -223,29 +224,25 @@ def _detect_gcc_libcxx(version_, executable): main = textwrap.dedent(""" #include - using namespace std; static_assert(sizeof(std::string) != sizeof(void*), "using libstdc++"); int main(){} """) - t = tempfile.mkdtemp() - filename = os.path.join(t, "main.cpp") - save(filename, main) - old_path = os.getcwd() - os.chdir(t) - try: - error, out_str = detect_runner(f'"{executable}" main.cpp -std=c++11') - if error: - if "using libstdc++" in out_str: - output.info("gcc C++ standard library: libstdc++") - return "libstdc++" - # Other error, but can't know, lets keep libstdc++11 - output.warning("compiler.libcxx check error: %s" % out_str) - output.warning("Couldn't deduce compiler.libcxx for gcc>=5.1, assuming libstdc++11") - else: - output.info("gcc C++ standard library: libstdc++11") - return "libstdc++11" - finally: - os.chdir(old_path) + # -fsyntax-only to omit the output and stop early (but enough for our static_assert). + # -xc++ and - to tell the compiler to compile code from stdin and treat it as C++. + completed_process = subprocess.run([executable, "-std=c++11", "-fsyntax-only", "-xc++", "-"], + input=main, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True) + error, out_str = completed_process.returncode, completed_process.stdout + if error: + if "using libstdc++" in out_str: + output.info("gcc C++ standard library: libstdc++") + return "libstdc++" + # Other error, but can't know, lets keep libstdc++11 + output.warning("compiler.libcxx check error: %s" % out_str) + output.warning("Couldn't deduce compiler.libcxx for gcc>=5.1, assuming libstdc++11") + else: + output.info("gcc C++ standard library: libstdc++11") + return "libstdc++11" # This is not really a detection in most cases # Get compiler C++ stdlib From a5cdf4c126908e543d60d07b481da21a24492c05 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 21 Apr 2026 10:32:39 +0200 Subject: [PATCH 103/110] poc of changing transitive statics package_id (#19705) * poc of changing transitive statics package_id * review * conf * wip * wip * new approach with required_conan_version>=2.28 * fix * wip * remove help line --- conan/internal/graph/compute_pid.py | 22 ++++++- conan/internal/graph/graph_binaries.py | 11 ++-- conan/internal/loader.py | 1 + conan/internal/model/conan_file.py | 1 + conan/internal/model/conf.py | 7 ++- conan/internal/model/requires.py | 19 ++++++- .../package_id_requires_modes_test.py | 57 +++++++++++++++++++ 7 files changed, 107 insertions(+), 11 deletions(-) diff --git a/conan/internal/graph/compute_pid.py b/conan/internal/graph/compute_pid.py index 94494f7f1cb..14c33f94a3c 100644 --- a/conan/internal/graph/compute_pid.py +++ b/conan/internal/graph/compute_pid.py @@ -6,9 +6,21 @@ from conan.internal.model.info import (ConanInfo, RequirementsInfo, RequirementInfo, PythonRequiresInfo) from conan.internal.model.pkg_type import PackageType +from conan.internal.model.version_range import VersionRange +from conan.internal.model.version import Version -def compute_package_id(node, modes, config_version, hook_manager): +def _compute_fix_transitive(conanfile, global_required_conan): + # fix for transitive static libraries + recipe_require_conan_version = global_required_conan or conanfile._conan_required_version # noqa + if recipe_require_conan_version: + version_range = VersionRange(recipe_require_conan_version) + for conditions in version_range.condition_sets: + conditions.prerelease = True + return not version_range.contains(Version("2.27.9"), resolve_prerelease=None) + + +def compute_package_id(node, modes, config_version, hook_manager, global_required_conan): """ Compute the binary package ID of this node """ @@ -20,10 +32,14 @@ def compute_package_id(node, modes, config_version, hook_manager): data = OrderedDict() build_data = OrderedDict() + + fix_transitive_static = _compute_fix_transitive(conanfile, global_required_conan) + for require, transitive in node.transitive_deps.items(): dep_node = transitive.node - require.deduce_package_id_mode(conanfile.package_type, dep_node, - non_embed_mode, embed_mode, build_mode, unknown_mode) + require.deduce_package_id_mode(conanfile, dep_node, + non_embed_mode, embed_mode, build_mode, unknown_mode, + fix_transitive_static) if require.package_id_mode is not None: req_info = RequirementInfo(dep_node.pref.ref, dep_node.pref.package_id, require.package_id_mode) diff --git a/conan/internal/graph/graph_binaries.py b/conan/internal/graph/graph_binaries.py index 09889f638f6..0249cfd1053 100644 --- a/conan/internal/graph/graph_binaries.py +++ b/conan/internal/graph/graph_binaries.py @@ -453,8 +453,9 @@ def _config_version(self): f" file in cache: {self._home_folder}: {e}") return RequirementsInfo(result) - def _evaluate_package_id(self, node, config_version): - compute_package_id(node, self._modes, config_version, self._hook_manager) + def _evaluate_package_id(self, node, config_version, global_required_conan): + compute_package_id(node, self._modes, config_version, self._hook_manager, + global_required_conan) # TODO: layout() execution don't need to be evaluated at GraphBuilder time. # it could even be delayed until installation time, but if we got enough info here for @@ -490,9 +491,10 @@ def _evaluate_single(n): root_pkg_type = deps_graph.root.edges[0].dst.conanfile.package_type \ if deps_graph.root.edges else None config_version = self._config_version() if root_pkg_type is not PackageType.CONF else None + global_required_conan = self._global_conf.get("core:required_conan_version") for level in levels[:-1]: # all levels but the last one, which is the single consumer for node in level: - self._evaluate_package_id(node, config_version) + self._evaluate_package_id(node, config_version, global_required_conan) # group by pref to paralelize, so evaluation is done only 1 per pref nodes = {} for node in level: @@ -521,7 +523,8 @@ def _evaluate_single(n): if node.path is not None: if node.path.endswith(".py"): # For .py we keep evaluating the package_id, validate(), etc - compute_package_id(node, self._modes, config_version, self._hook_manager) + compute_package_id(node, self._modes, config_version, self._hook_manager, + global_required_conan) # To support the ``[layout]`` in conanfile.txt if hasattr(node.conanfile, "layout"): with conanfile_exception_formatter(node.conanfile, "layout"): diff --git a/conan/internal/loader.py b/conan/internal/loader.py index 0dccf125a07..2e0dfeff703 100644 --- a/conan/internal/loader.py +++ b/conan/internal/loader.py @@ -57,6 +57,7 @@ def load_basic_module(self, conanfile_path, graph_lock=None, display="", remotes try: module, conanfile = _parse_conanfile(conanfile_path) + conanfile._conan_required_version = getattr(module, "required_conan_version", None) if isinstance(tested_python_requires, RecipeReference): if getattr(conanfile, "python_requires", None) == "tested_reference_str": conanfile.python_requires = tested_python_requires.repr_notime() diff --git a/conan/internal/model/conan_file.py b/conan/internal/model/conan_file.py index 313060cf40f..6c23642280b 100644 --- a/conan/internal/model/conan_file.py +++ b/conan/internal/model/conan_file.py @@ -59,6 +59,7 @@ class ConanFile: win_bash_run = None # For run scope _conan_is_consumer = False + _conan_required_version = None # #### Requirements requires = None diff --git a/conan/internal/model/conf.py b/conan/internal/model/conf.py index 87ae9a360ea..acf2b454ebd 100644 --- a/conan/internal/model/conf.py +++ b/conan/internal/model/conf.py @@ -17,8 +17,13 @@ from conan.internal.model.settings import SettingsItem from conan.internal.util.files import load, save +required_conan_version_msg = """\ +Raise if current version does not match the defined range. + - If required_conan_version>=2.28, bugfix https://github.com/conan-io/conan/pull/19705 for transitive static libraries package_id is applied + These behaviors also apply for 'required_conan_version' in recipes, but the global one has precedence.""" + BUILT_IN_CONFS = { - "core:required_conan_version": "Raise if current version does not match the defined range.", + "core:required_conan_version": required_conan_version_msg, "core:non_interactive": "Disable interactive user input, raises error if input necessary", "core:warnings_as_errors": "Treat warnings matching any of the patterns in this list as errors and then raise an exception. " "Current warning tags are 'network', 'deprecated'", diff --git a/conan/internal/model/requires.py b/conan/internal/model/requires.py index 62cbed2c1b3..d4c44dd8fb6 100644 --- a/conan/internal/model/requires.py +++ b/conan/internal/model/requires.py @@ -357,8 +357,8 @@ def transform_downstream(self, pkg_type, require, dep_pkg_type): downstream_require.direct = False return downstream_require - def deduce_package_id_mode(self, pkg_type, dep_node, non_embed_mode, embed_mode, build_mode, - unknown_mode): + def deduce_package_id_mode(self, conanfile, dep_node, non_embed_mode, embed_mode, build_mode, + unknown_mode, fix_transitive_static): # If defined by the ``require(package_id_mode=xxx)`` trait, that is higher priority # The "conf" values are defaults, no hard overrides if self.package_id_mode: @@ -374,6 +374,7 @@ def deduce_package_id_mode(self, pkg_type, dep_node, non_embed_mode, embed_mode, self.package_id_mode = build_mode return + pkg_type = conanfile.package_type if pkg_type is PackageType.HEADER: self.package_id_mode = "unrelated_mode" return @@ -391,8 +392,20 @@ def deduce_package_id_mode(self, pkg_type, dep_node, non_embed_mode, embed_mode, elif pkg_type is PackageType.STATIC: if dep_pkg_type is PackageType.HEADER: self.package_id_mode = embed_mode - else: + elif self.headers or not fix_transitive_static: self.package_id_mode = non_embed_mode + if not self.headers and not fix_transitive_static: + # Just to avoid multiple repeated warnings + warned = getattr(conanfile, "_conan_fix_transitive_static", False) + if not warned: + msg = ("Transitive dependencies with 'headers=False' effect in " + "'package_id' is not necessary and suboptimal. Use " + "required_conan_version='>=2.28' to activate it") + conanfile.output.warning(msg, warn_tag="risk") + conanfile._conan_fix_transitive_static = True + else: + self.package_id_mode = None + return if self.package_id_mode is None: self.package_id_mode = unknown_mode diff --git a/test/integration/package_id/package_id_requires_modes_test.py b/test/integration/package_id/package_id_requires_modes_test.py index dbffa8ad608..ea43050269e 100644 --- a/test/integration/package_id/package_id_requires_modes_test.py +++ b/test/integration/package_id/package_id_requires_modes_test.py @@ -216,3 +216,60 @@ def test_half_diamond_conflict(self, mode, pattern): c.run("create pkg") c.run("list pkg:*") assert f"liba/{pattern}" in c.out + + +class TestTransitiveStatic: + @pytest.mark.parametrize("apply_fix", [True, False, "conf"]) + def test_transitive_statics(self, apply_fix): + # https://github.com/conan-io/conan/issues/19664 + c = TestClient(light=True) + required_conan_version = 'required_conan_version = ">=2.28"' if apply_fix is True else "" + if apply_fix == "conf": + c.save_home({"global.conf": "core:required_conan_version=>=2.28"}) + libc = textwrap.dedent(f"""\ + from conan import ConanFile + + {required_conan_version} + class LibcConan(ConanFile): + name = "libc" + version = "1.0" + package_type = "static-library" + requires = "libb/1.0" + """) + c.save({"liba/conanfile.py": GenConanfile("liba", "1.0").with_package_type("static-library"), + "libb/conanfile.py": GenConanfile("libb", "1.0").with_package_type("static-library") + .with_requires("liba/1.0"), + "libc/conanfile.py": libc + }) + c.run("create liba") + c.run("create libb") + c.run(f"create libc") + if not apply_fix: + assert ("libc/1.0: WARN: risk: Transitive dependencies with " + "'headers=False' effect in 'package_id'") in c.out + c.run("list libc:*") + assert "libb/1.0.Z" in c.out + if apply_fix: + assert "liba/" not in c.out + else: + assert "liba/" in c.out + + def test_transitive_shared(self): + # https://github.com/conan-io/conan/issues/19664 + # This doesn't happen by default because the transitive shared do not propagate .libs + # linkage requirement trait + c = TestClient(light=True) + c.save({"liba/conanfile.py": GenConanfile("liba", "1.0").with_package_type("shared-library"), + "libb/conanfile.py": GenConanfile("libb", "1.0").with_package_type("shared-library") + .with_requires("liba/1.0"), + "libc/conanfile.py": GenConanfile("libc", "1.0").with_package_type("shared-library") + .with_requires("libb/1.0"), + }) + c.run("create liba") + c.run("create libb") + c.run("create libc") + + assert "libc/1.0: WARN" not in c.out + c.run("list libc:*") + assert "libb/1.0.Z" in c.out + assert "liba/" not in c.out From e726aad25e799d98c7287eb3c91ce7b1591bb324 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Tue, 21 Apr 2026 12:28:10 +0200 Subject: [PATCH 104/110] Scoped output: first approach (#19836) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * First approach for a contextual output * Support multi lines in messages * Simplify approach, limit to export command * Spaces only * Arrow with underline POC * Change approach to scope based output * Change scope color, remove underline and reduce changes * Disable contextual output behavior on tests and all commands but export * Fix old python issue * Simplify code * Remove ignore_indent paremeter * Add test case for scoped output * Close scoped output after export command * Update test/unittests/client/conan_output_test.py --------- Co-authored-by: Abril Rincón Blanco <5364255+AbrilRBS@users.noreply.github.com> --- conan/api/output.py | 40 +++++++++++++++++++++- conan/cli/commands/export.py | 7 +++- conan/test/utils/tools.py | 2 ++ test/unittests/client/conan_output_test.py | 34 ++++++++++++++++++ 4 files changed, 81 insertions(+), 2 deletions(-) diff --git a/conan/api/output.py b/conan/api/output.py index 520e5d560cb..aa9359587d8 100644 --- a/conan/api/output.py +++ b/conan/api/output.py @@ -101,6 +101,9 @@ class ConanOutput: _conan_output_level = LEVEL_STATUS _silent_warn_tags = [] _warnings_as_errors = [] + _last_scope_header = None + # Flag to enable/disable new ConanOutput contextual behavior + _scoped_recipe_output = None lock = Lock() def __init__(self, scope: str = ""): @@ -115,6 +118,8 @@ def __init__(self, scope: str = ""): # FIXME: This is needed because in testing we are redirecting the sys.stderr to a buffer # stream to capture it, so colorama is not there to strip the color bytes self._color = _color_enabled(self.stream) + if not scope: + ConanOutput._last_scope_header = None @classmethod def define_silence_warnings(cls, warnings): @@ -165,6 +170,14 @@ def define_log_level(cls, v): def level_allowed(cls, level): return cls._conan_output_level <= level + def _emit_scope_line_if_new(self): + """Print ``scope:`` once per scope change, when a message is about to be written.""" + scope = self._scope + if not scope or scope == ConanOutput._last_scope_header: + return + self.writeln(f"{scope}:", fg=Color.BRIGHT_BLUE) + ConanOutput._last_scope_header = scope + @property def color(self): return self._color @@ -176,6 +189,8 @@ def scope(self): @scope.setter def scope(self, out_scope): self._scope = out_scope + if not out_scope: + ConanOutput._last_scope_header = None @property def is_terminal(self): @@ -212,6 +227,18 @@ def login_msg(self, msg, newline=False): self._write_message(msg, newline=newline) return self + def _format_scoped_message(self, msg, fg=None, bg=None): + """Prefix every physical line with a short space gutter (block under the scope header). + Splitlines is used for handling multi-line messages, ensuring a gutter is added to each line + """ + parts = [] + for line in msg.splitlines(): + if self._color: + parts.append(f" {fg or ''}{bg or ''}{line}{Style.RESET_ALL}") + else: + parts.append(f" {line}") + return "\n".join(parts) + def _write_message(self, msg, fg=None, bg=None, newline=True): if isinstance(msg, dict): # For traces we can receive a dict already, we try to transform then into more natural @@ -220,7 +247,10 @@ def _write_message(self, msg, fg=None, bg=None, newline=True): msg = "=> {}".format(msg) # msg = json.dumps(msg, sort_keys=True, default=json_encoder) - if self._scope: + if ConanOutput._scoped_recipe_output and self._scope: + self._emit_scope_line_if_new() + ret = self._format_scoped_message(msg, fg, bg) + elif self._scope: if self._color: ret = f"{fg or ''}{bg or ''}{self._scope}: {msg}{Style.RESET_ALL}" else: @@ -292,15 +322,23 @@ def status(self, msg: str, fg: str = None, bg: str = None, newline: bool = True) def title(self, msg: str): """ Draws a title around the message, useful for important messages""" if self._conan_output_level <= LEVEL_NOTICE: + ConanOutput._last_scope_header = None + prev_scope = self._scope + self._scope = "" self._write_message("\n======== {} ========".format(msg), fg=Color.BRIGHT_MAGENTA) + self._scope = prev_scope return self def subtitle(self, msg: str): """ Draws a subtitle around the message, useful for important messages""" if self._conan_output_level <= LEVEL_NOTICE: + ConanOutput._last_scope_header = None + prev_scope = self._scope + self._scope = "" self._write_message("\n-------- {} --------".format(msg), fg=Color.BRIGHT_MAGENTA) + self._scope = prev_scope return self def highlight(self, msg: str): diff --git a/conan/cli/commands/export.py b/conan/cli/commands/export.py index cc6c3e53d00..963ee5cb6ca 100644 --- a/conan/cli/commands/export.py +++ b/conan/cli/commands/export.py @@ -2,7 +2,7 @@ import os from conan.api.model import MultiPackagesList, PackagesList -from conan.api.output import cli_out_write +from conan.api.output import ConanOutput, cli_out_write from conan.cli.command import conan_command, OnceArgument from conan.cli.args import add_reference_args @@ -43,6 +43,10 @@ def export(conan_api, parser, *args): help='Whether the provided reference is a build-require') args = parser.parse_args(*args) + # Only enable scoped output if None. If it is False, it means that + # we have explicitly disabled (e.g. tests), so we should not enable it + if ConanOutput._scoped_recipe_output is None: + ConanOutput._scoped_recipe_output = True cwd = os.getcwd() path = conan_api.local.get_conanfile_path(args.path, cwd, py=True) remotes = conan_api.remotes.list(args.remote) if not args.no_remote else [] @@ -64,6 +68,7 @@ def export(conan_api, parser, *args): pkglist = MultiPackagesList() pkglist.add("Local Cache", exported_list) + ConanOutput._scoped_recipe_output = None return { "pkglist": pkglist.serialize(), diff --git a/conan/test/utils/tools.py b/conan/test/utils/tools.py index 8ef1ebbeaea..6ae57eee6e0 100644 --- a/conan/test/utils/tools.py +++ b/conan/test/utils/tools.py @@ -23,6 +23,7 @@ from requests.exceptions import HTTPError from webtest.app import TestApp +from conan.api.output import ConanOutput from conan.api.subapi.audit import CONAN_CENTER_AUDIT_PROVIDER_NAME, _save_providers from conan.api.subapi.remotes import _save from conan.cli.exit_codes import SUCCESS @@ -531,6 +532,7 @@ def mock_get_pass(*args, **kwargs): yield def _run_cli(self, command_line, assert_error=False): + ConanOutput._scoped_recipe_output = False args = shlex.split(command_line) error = SUCCESS trace = None diff --git a/test/unittests/client/conan_output_test.py b/test/unittests/client/conan_output_test.py index ea35f762813..bdccdaa48de 100644 --- a/test/unittests/client/conan_output_test.py +++ b/test/unittests/client/conan_output_test.py @@ -2,6 +2,7 @@ from unittest import mock import pytest +import textwrap from conan.api.output import ConanOutput, init_colorama from conan.test.utils.mocks import RedirectedTestOutput @@ -48,3 +49,36 @@ def test_output_chainable(): assert "My title" in stderr.getvalue() assert "Worked" in stderr.getvalue() assert "But there was more that needed to be said" in stderr.getvalue() + + +def test_output_scoped(): + """ + Test that when scoped output is enabled, the scope is only printed once per scope change or title/subtitle is emitted + """ + stderr = RedirectedTestOutput() + with redirect_output(stderr): + ConanOutput._scoped_recipe_output = True + output = ConanOutput(scope="My package") + output.info("Hello") + output.highlight("Conan") + output.title("Title") + output.info("Package manager!") + output.subtitle("Subtitle") + output.highlight("Frog") + output = ConanOutput(scope="Other package") + output.info("Hello") + assert textwrap.dedent("""\ + My package: + Hello + Conan + + ======== Title ======== + My package: + Package manager! + + -------- Subtitle -------- + My package: + Frog + Other package: + Hello + """) == stderr.getvalue() From cf01c962e5279ebb6ed5d8c6694787acc733db0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= <5364255+AbrilRBS@users.noreply.github.com> Date: Tue, 21 Apr 2026 12:47:00 +0200 Subject: [PATCH 105/110] Colorize config command outputs (#19889) --- conan/cli/commands/config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conan/cli/commands/config.py b/conan/cli/commands/config.py index 0bcd1719700..d930a07ca10 100644 --- a/conan/cli/commands/config.py +++ b/conan/cli/commands/config.py @@ -1,7 +1,7 @@ import os from conan.api.model import Remote -from conan.api.output import cli_out_write +from conan.api.output import cli_out_write, Color from conan.cli import make_abs_path from conan.cli.command import conan_command, conan_subcommand, OnceArgument from conan.cli.formatters import default_json_formatter @@ -115,7 +115,8 @@ def config_install_pkg(conan_api, parser, subparser, *args): def _list_text_formatter(confs): for k, v in confs.items(): - cli_out_write(f"{k}: {v}") + cli_out_write(f"{k}: ", fg=Color.CYAN, endline="") + cli_out_write(v) @conan_subcommand(formatters={"text": cli_out_write}) From c3764a80f4661e94011fcbe088d22ab780b1ac1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= <5364255+AbrilRBS@users.noreply.github.com> Date: Tue, 21 Apr 2026 13:39:27 +0200 Subject: [PATCH 106/110] HTML updates for `conan report diff` and `conan graph info` (#19884) * Update html graph info * Audit * Show subgraph * Show subgraph exclusive edges * Fix missing nodes * Style information * Extension filter * Remove regex, fix extensions * Remove vonsole.log * Siggestions of style * Suggested changes * Apply suggestion from @memsharded * Back to black --------- Co-authored-by: James --- conan/cli/formatters/graph/info_graph_html.py | 133 +++++++++-- conan/cli/formatters/report/diff_html.py | 211 ++++++++++++++---- .../command/info/test_graph_info_graphical.py | 5 +- 3 files changed, 282 insertions(+), 67 deletions(-) diff --git a/conan/cli/formatters/graph/info_graph_html.py b/conan/cli/formatters/graph/info_graph_html.py index e2466d225b9..6ccf6692532 100644 --- a/conan/cli/formatters/graph/info_graph_html.py +++ b/conan/cli/formatters/graph/info_graph_html.py @@ -21,12 +21,46 @@ display: inline-block; font-size: 18px; } + label { + user-select: none + } + + #container { + display: grid; + grid-template-columns: 75% 25%; + grid-template-rows: 50px auto; + height: 100vh; + } + #mylegend { + border-bottom: 2px solid #f2f2f2; + grid-column-end: span 1; + font-size: 14px; + height: 100%; + } + #empty-sidebar, #sidebar { + background-color: #f9f9fe; + border: none; + min-height:100%; + height:0; + overflow-y: auto; + padding: 5px 10px; + } + #details { + background-color: #f3f3f3; + overflow-y: auto; + border: 1px solid #e4e4e4; + border-radius: 5px; + padding: 3px 5px; + } -
    -
    +
    +
    +
    +

    Controls

    +
    -
    +