From 94b1ac6465abe13ba73267c0b4da9048af2ae4c2 Mon Sep 17 00:00:00 2001 From: bbhtt Date: Mon, 27 Apr 2026 10:24:55 +0530 Subject: [PATCH 1/6] builder-manifest: Port to libappstream-compose API This also simplifies the compose handling by moving all related path and context logic into builder_appstreamcli_compose() as this is only ever used for Flatpak and the paths stay constant for all input build refs. This also allows us to remove the duplicate call due to media_dir and mirror url being set to NULL in one branch. The rest is an exact 1:1 port of what was present before from our side. --- .github/dependencies.apt.txt | 2 +- meson.build | 4 - src/builder-main.c | 6 -- src/builder-manifest.c | 147 ++++++++++++++++++----------------- src/builder-utils.c | 52 ------------- src/meson.build | 1 + tests/libtest.sh | 41 ---------- tests/test-builder.sh | 22 +++--- 8 files changed, 86 insertions(+), 189 deletions(-) diff --git a/.github/dependencies.apt.txt b/.github/dependencies.apt.txt index 4359473f..ea376c44 100644 --- a/.github/dependencies.apt.txt +++ b/.github/dependencies.apt.txt @@ -1,4 +1,3 @@ -appstream-compose dbus debugedit desktop-file-utils @@ -9,6 +8,7 @@ fuse3 gettext git git-lfs +libappstream-compose-dev libarchive-tools libcurl4-openssl-dev libelf-dev diff --git a/meson.build b/meson.build index f5e5d8c3..69117fe8 100644 --- a/meson.build +++ b/meson.build @@ -51,10 +51,6 @@ endif # The debugedit program is a hard dependency debugedit = find_program('debugedit', version: '>= 5.0') -# Require appstream with compose plugin installed -appstreamcli = find_program('appstreamcli', version: '>= 0.15.0') -appstreamcli_compose = run_command(appstreamcli, ['compose', '--help'], check: true) - fusermount = get_option('system_fusermount') if fusermount == '' fusermount_program = find_program(['fusermount3', 'fusermount'], required: true) diff --git a/src/builder-main.c b/src/builder-main.c index 8510da66..99352022 100644 --- a/src/builder-main.c +++ b/src/builder-main.c @@ -632,12 +632,6 @@ main (int argc, return 1; } - if (policy == BUILDER_AS_URL_POLICY_FULL && !appstream_has_version (0, 16, 3)) - { - g_printerr ("AppStream version >= 0.16.3 required for 'full' compose URL policy\n"); - return 1; - } - builder_context_set_as_url_policy (build_context, policy); } diff --git a/src/builder-manifest.c b/src/builder-manifest.c index 472b96e4..2cea588d 100644 --- a/src/builder-manifest.c +++ b/src/builder-manifest.c @@ -30,6 +30,10 @@ #include #include +/* Drop once AppStream 1.1.3 is required */ +#define I_KNOW_THE_APPSTREAM_COMPOSE_API_IS_SUBJECT_TO_CHANGE +#include + #include "builder-manifest.h" #include "builder-utils.h" #include "builder-flatpak-utils.h" @@ -2434,30 +2438,82 @@ cmpstringp (const void *p1, const void *p2) } static gboolean -appstreamcli_compose (GError **error, - BuilderAsUrlPolicy as_url_policy, - ...) +builder_appstreamcli_compose (const char *app_id, + GFile *app_root, + BuilderContext *context, + GError **error) { - g_autoptr(GPtrArray) args = NULL; - const gchar *arg; - va_list ap; + g_autoptr(AscCompose) compose = NULL; + g_autoptr(AscDirectoryUnit) dirunit = NULL; + g_autofree char *desktop_component = NULL; - args = g_ptr_array_new_with_free_func (g_free); - g_ptr_array_add (args, g_strdup ("appstreamcli")); - g_ptr_array_add (args, g_strdup ("compose")); + g_autoptr(GFile) data_out = NULL; + g_autoptr(GFile) icon_out = NULL; + const char *app_root_path = NULL; + const char *data_dir = NULL; + const char *icon_dir = NULL; + + const char *opt_mirror_screenshots_url = NULL; + gboolean opt_export_only; + BuilderAsUrlPolicy as_url_policy; + + g_return_val_if_fail (app_id != NULL, FALSE); + g_return_val_if_fail (G_IS_FILE (app_root), FALSE); + + app_root_path = flatpak_file_get_path_cached (app_root); + + data_out = flatpak_build_file (app_root, "share/app-info/xmls", NULL); + icon_out = flatpak_build_file (app_root, "share/app-info/icons/flatpak", NULL); + + data_dir = flatpak_file_get_path_cached (data_out); + icon_dir = flatpak_file_get_path_cached (icon_out); + + opt_mirror_screenshots_url = builder_context_get_opt_mirror_screenshots_url (context); + opt_export_only = builder_context_get_opt_export_only (context); + as_url_policy = builder_context_get_as_url_policy (context); + + compose = asc_compose_new (); + + asc_compose_set_format (compose, AS_FORMAT_KIND_XML); + asc_compose_set_origin (compose, app_id); + asc_compose_set_prefix (compose, "/"); + + asc_compose_add_allowed_cid (compose, app_id); + desktop_component = g_strdup_printf ("%s.desktop", app_id); + asc_compose_add_allowed_cid (compose, desktop_component); + + dirunit = asc_directory_unit_new (app_root_path); + asc_compose_add_unit (compose, ASC_UNIT (dirunit)); + + asc_compose_set_data_result_dir (compose, data_dir); + asc_compose_set_icons_result_dir (compose, icon_dir); + + if (opt_mirror_screenshots_url && !opt_export_only) + { + g_autoptr(GFile) media_out = flatpak_build_file (app_root, "share/app-info/media", NULL); + const char *media_dir = flatpak_file_get_path_cached (media_out); + + g_print ("Saving screenshots in %s\n", media_dir); + asc_compose_set_media_baseurl (compose, opt_mirror_screenshots_url); + asc_compose_set_media_result_dir (compose, media_dir); + } if (as_url_policy == BUILDER_AS_URL_POLICY_FULL) - g_ptr_array_add (args, g_strdup ("--no-partial-urls")); + asc_compose_add_flags (compose, ASC_COMPOSE_FLAG_NO_PARTIAL_URLS); - va_start (ap, as_url_policy); - while ((arg = va_arg (ap, const gchar *))) - g_ptr_array_add (args, g_strdup (arg)); - g_ptr_array_add (args, NULL); - va_end (ap); + g_print ("Running appstreamcli compose\n"); - if (!flatpak_spawnv (NULL, NULL, 0, error, (const char * const *)args->pdata, NULL)) + g_autoptr(GPtrArray) results = asc_compose_run (compose, NULL, error); + if (results == NULL) { - g_prefix_error (error, "ERROR: appstreamcli compose failed: "); + g_prefix_error (error, "AppStream compose failed: "); + return FALSE; + } + + if (asc_compose_has_errors (compose)) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "AppStream compose completed with errors"); return FALSE; } @@ -3084,61 +3140,8 @@ builder_manifest_cleanup (BuilderManifest *self, if (self->appstream_compose && appdata_file != NULL) { - g_autofree char *origin = g_strdup_printf ("--origin=%s", - builder_manifest_get_id (self)); - g_autofree char *components_arg = g_strdup_printf ("--components=%s,%s.desktop", - self->id, self->id); - const char *app_root_path = flatpak_file_get_path_cached (app_root); - g_autofree char *result_root_arg = g_strdup_printf ("--result-root=%s", app_root_path); - g_autoptr(GFile) xml_dir = flatpak_build_file (app_root, "share/app-info/xmls", NULL); - g_autoptr(GFile) icon_out = flatpak_build_file (app_root, "share/app-info/icons/flatpak", NULL); - g_autoptr(GFile) media_dir = flatpak_build_file (app_root, "share/app-info/media", NULL); - g_autofree char *data_dir = g_strdup_printf ("--data-dir=%s", - flatpak_file_get_path_cached (xml_dir)); - g_autofree char *icon_dir = g_strdup_printf ("--icons-dir=%s", - flatpak_file_get_path_cached (icon_out)); - const char *opt_mirror_screenshots_url = builder_context_get_opt_mirror_screenshots_url (context); - gboolean opt_export_only = builder_context_get_opt_export_only (context); - BuilderAsUrlPolicy as_url_policy = builder_context_get_as_url_policy (context); - - if (opt_mirror_screenshots_url && !opt_export_only) - { - g_autofree char *url = g_build_filename (opt_mirror_screenshots_url, NULL); - g_autofree char *arg_base_url = g_strdup_printf ("--media-baseurl=%s", url); - g_autofree char *arg_media_dir = g_strdup_printf ("--media-dir=%s", - flatpak_file_get_path_cached (media_dir)); - - g_print ("Running appstreamcli compose\n"); - g_print ("Saving screenshots in %s\n", flatpak_file_get_path_cached (media_dir)); - if (!appstreamcli_compose (error, - as_url_policy, - "--prefix=/", - origin, - arg_base_url, - arg_media_dir, - result_root_arg, - data_dir, - icon_dir, - components_arg, - app_root_path, - NULL)) - return FALSE; - } - else - { - g_print ("Running appstreamcli compose\n"); - if (!appstreamcli_compose (error, - as_url_policy, - "--prefix=/", - origin, - result_root_arg, - data_dir, - icon_dir, - components_arg, - app_root_path, - NULL)) - return FALSE; - } + if (!builder_appstreamcli_compose (self->id, app_root, context, error)) + return FALSE; } if (!builder_context_disable_rofiles (context, error)) diff --git a/src/builder-utils.c b/src/builder-utils.c index 01d981b5..7e43943d 100644 --- a/src/builder-utils.c +++ b/src/builder-utils.c @@ -1921,55 +1921,3 @@ flatpak_version_check (int major, return FALSE; } - -gboolean -appstream_has_version (int major, - int minor, - int micro) -{ - static int as_major = 0; - static int as_minor = 0; - static int as_micro = 0; - - if (as_major == 0 && - as_minor == 0 && - as_micro == 0) - { - const char * argv[] = { "appstreamcli", "--version", NULL }; - g_autoptr(GSubprocessLauncher) launcher = NULL; - g_autoptr(GSubprocess) subp = NULL; - g_autofree char *out = NULL; - g_auto(GStrv) lines = NULL; - - launcher = g_subprocess_launcher_new (G_SUBPROCESS_FLAGS_STDOUT_PIPE); - g_subprocess_launcher_setenv (launcher, "LANGUAGE", "C", TRUE); - subp = g_subprocess_launcher_spawnv (launcher, argv, NULL); - g_subprocess_communicate_utf8 (subp, NULL, NULL, &out, NULL, NULL); - - lines = g_strsplit (out, "\n", -1); - - for (size_t i = 0; lines[i] != NULL; i++) - { - /* Only prefer library version over cli version in case of mismatch */ - if (g_str_has_prefix (lines[i], "AppStream library version:")) - { - if (sscanf (lines[i], "AppStream library version: %d.%d.%d", &as_major, &as_minor, &as_micro) == 3) - break; - } - else if (g_str_has_prefix (lines[i], "AppStream version:")) - { - if (sscanf (lines[i], "AppStream version: %d.%d.%d", &as_major, &as_minor, &as_micro) == 3) - break; - } - } - - if (as_major == 0 && as_minor == 0 && as_micro == 0) - g_warning ("Failed to find appstream version"); - else - g_debug ("Found AppStream version %d.%d.%d", as_major, as_minor, as_micro); - } - - return (as_major > major) || - (as_major == major && as_minor > minor) || - (as_major == major && as_minor == minor && as_micro >= micro); -} diff --git a/src/meson.build b/src/meson.build index a737b41d..a09f741f 100644 --- a/src/meson.build +++ b/src/meson.build @@ -58,6 +58,7 @@ flatpak_builder_deps = [ dependency('libglnx', default_options: ['tests=false']), dependency('libxml-2.0', version: '>= 2.4'), dependency('ostree-1', version: '>= 2017.14'), + dependency('appstream-compose', version: '>=1.1.2'), yaml_dep, ] diff --git a/tests/libtest.sh b/tests/libtest.sh index 3537728d..a04cf2e0 100644 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -330,47 +330,6 @@ skip_without_python2 () { fi } -appstream_has_version () { - req_major=$1 - req_minor=$2 - req_micro=$3 - - maj=0; min=0; mic=0 - - out=$(LANGUAGE=C appstreamcli --version 2>/dev/null) || return 1 - - while IFS= read -r line; do - case "$line" in - "AppStream library version:"* ) - ver=$(echo "$line" | awk '{print $4}') - ;; - "AppStream version:"* ) - ver=$(echo "$line" | awk '{print $3}') - ;; - * ) continue ;; - esac - - maj=$(echo "$ver" | cut -d. -f1) - min=$(echo "$ver" | cut -d. -f2) - mic=$(echo "$ver" | cut -d. -f3) - break - done <&2 || true if test -n "${TEST_SKIP_CLEANUP:-}"; then diff --git a/tests/test-builder.sh b/tests/test-builder.sh index 68377470..e354afcd 100755 --- a/tests/test-builder.sh +++ b/tests/test-builder.sh @@ -180,19 +180,15 @@ gzip -cdq builddir_sc/files/share/app-info/xmls/org.flatpak.appstream_media.xml. echo "ok compose partial url policy" # test compose full url policy -if appstream_has_version 0 16 3; then - ${FLATPAK_BUILDER} --force-clean builddir_sc \ - --mirror-screenshots-url=https://example.org/media \ - --state-dir .fp-compose-url-policy-full \ - --compose-url-policy=full \ - org.flatpak.appstream_media.json >&2 - - gzip -cdq builddir_sc/files/share/app-info/xmls/org.flatpak.appstream_media.xml.gz|grep -Eq '>https://example.org/media/org/flatpak/appstream_media/[^/]+/icons/128x128/org.flatpak.appstream_media.png' - - echo "ok compose full url policy" -else - echo "ok # Skip AppStream < 0.16.3" -fi +${FLATPAK_BUILDER} --force-clean builddir_sc \ + --mirror-screenshots-url=https://example.org/media \ + --state-dir .fp-compose-url-policy-full \ + --compose-url-policy=full \ + org.flatpak.appstream_media.json >&2 + +gzip -cdq builddir_sc/files/share/app-info/xmls/org.flatpak.appstream_media.xml.gz|grep -Eq '>https://example.org/media/org/flatpak/appstream_media/[^/]+/icons/128x128/org.flatpak.appstream_media.png' + +echo "ok compose full url policy" # test install ${FLATPAK_BUILDER} --user --install \ From d1d163a4265a217b3bef48509f04da01511af286 Mon Sep 17 00:00:00 2001 From: bbhtt Date: Tue, 19 May 2026 08:46:18 +0530 Subject: [PATCH 2/6] builder-manifest: Allow all custom appstream tags in catalogue This allows all `` tags from metainfo files to reach catalogue data. Custom tags are used by Flathub to store verification and build information and often downstream vendors who use Flathub's catalogue for their own app store want custom tags to identify or highlight their own apps. For example cosmic using Flathub catalogue data to highlight cosmic apps on cosmic store app etc. Allowing individual custom tags one by one is annoying and this is what libappstream-glib did before the port in 1.3.4. So this brings back the old as-glib behaviour that was lost in >=1.3.4 --- src/builder-manifest.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/builder-manifest.c b/src/builder-manifest.c index 2cea588d..df08aa67 100644 --- a/src/builder-manifest.c +++ b/src/builder-manifest.c @@ -2501,6 +2501,8 @@ builder_appstreamcli_compose (const char *app_id, if (as_url_policy == BUILDER_AS_URL_POLICY_FULL) asc_compose_add_flags (compose, ASC_COMPOSE_FLAG_NO_PARTIAL_URLS); + asc_compose_add_flags (compose, ASC_COMPOSE_FLAG_PROPAGATE_CUSTOM); + g_print ("Running appstreamcli compose\n"); g_autoptr(GPtrArray) results = asc_compose_run (compose, NULL, error); From 33afe02f00a158e0c81cd33dc5fa4a1dc82e9884 Mon Sep 17 00:00:00 2001 From: bbhtt Date: Tue, 19 May 2026 08:46:59 +0530 Subject: [PATCH 3/6] builder-manifest: Disable processing screencasts in appstream Screencasts are generally unwanted in catalogue to as they are more expensive to download, store in ref and serve. Also no appstore currently supports showing them to my knowledge. This can later be changed by adding an argument if needed. --- src/builder-manifest.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/builder-manifest.c b/src/builder-manifest.c index df08aa67..3d6d8bb9 100644 --- a/src/builder-manifest.c +++ b/src/builder-manifest.c @@ -2502,6 +2502,7 @@ builder_appstreamcli_compose (const char *app_id, asc_compose_add_flags (compose, ASC_COMPOSE_FLAG_NO_PARTIAL_URLS); asc_compose_add_flags (compose, ASC_COMPOSE_FLAG_PROPAGATE_CUSTOM); + asc_compose_remove_flags (compose, ASC_COMPOSE_FLAG_ALLOW_SCREENCASTS); g_print ("Running appstreamcli compose\n"); From bfffdc55743a0ddbb1e270de18c03b72af69887f Mon Sep 17 00:00:00 2001 From: bbhtt Date: Sat, 23 May 2026 10:38:02 +0530 Subject: [PATCH 4/6] builder-main: Flip the default BuilderAsUrlPolicy to full See: https://github.com/flatpak/flatpak-builder/issues/743 --- doc/flatpak-builder.xml | 4 +++- src/builder-main.c | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/flatpak-builder.xml b/doc/flatpak-builder.xml index 4e3564f2..f39b52f7 100644 --- a/doc/flatpak-builder.xml +++ b/doc/flatpak-builder.xml @@ -633,7 +633,9 @@ Set the AppStream compose URL policy. Accepted values are partial and full. full requires AppStream version >= 0.16.3. - Defaults to partial if unspecified. + Defaults to full since + 1.5.0 but partial for earlier + versions, if unspecified. This policy only takes effect when used in conjunction with ; otherwise the Appstream catalogue will preserve diff --git a/src/builder-main.c b/src/builder-main.c index 99352022..f1b77940 100644 --- a/src/builder-main.c +++ b/src/builder-main.c @@ -150,7 +150,7 @@ static GOptionEntry entries[] = { { "assumeyes", 'y', 0, G_OPTION_ARG_NONE, &opt_yes, N_("Automatically answer yes for all questions"), NULL }, { "no-shallow-clone", 0, 0, G_OPTION_ARG_NONE, &opt_no_shallow_clone, "Don't use shallow clones when mirroring git repos", NULL }, { "override-source-date-epoch", 0, 0, G_OPTION_ARG_INT64, &opt_source_date_epoch, "Use this timestamp to perform the build, instead of the last modification time of the manifest.", NULL }, - { "compose-url-policy", 0, 0, G_OPTION_ARG_STRING, &opt_as_url_policy, "Set the AppStream compose URL policy to either 'partial' (default) or 'full'", "POLICY" }, + { "compose-url-policy", 0, 0, G_OPTION_ARG_STRING, &opt_as_url_policy, "Set the AppStream compose URL policy to either 'full' (default) or 'partial'", "POLICY" }, { NULL } }; @@ -620,7 +620,7 @@ main (int argc, if (opt_mirror_screenshots_url) { - BuilderAsUrlPolicy policy = BUILDER_AS_URL_POLICY_PARTIAL; + BuilderAsUrlPolicy policy = BUILDER_AS_URL_POLICY_FULL; if (g_strcmp0 (opt_as_url_policy, "full") == 0) policy = BUILDER_AS_URL_POLICY_FULL; From 71d88d717f7cdf0ff326abf4d611ad1927941c2c Mon Sep 17 00:00:00 2001 From: bbhtt Date: Wed, 27 May 2026 20:44:10 +0530 Subject: [PATCH 5/6] tests: Add asan suppressions for appstream --- tests/lsan.supp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/lsan.supp b/tests/lsan.supp index e66dc2be..70cf001b 100644 --- a/tests/lsan.supp +++ b/tests/lsan.supp @@ -5,4 +5,9 @@ leak:g_thread_pool_spawn_thread leak:g_main_context_iteration leak:sysprof_collector_get +# Fix in https://github.com/ximion/appstream/pull/752 +leak:asc_compose_init +leak:asc_icon_policy_init +leak:asc_compose_process_task_cb + # flatpak-builder From 119cdce7ce81593fc5f42e9bee971a994efb8f4b Mon Sep 17 00:00:00 2001 From: bbhtt Date: Wed, 27 May 2026 20:52:34 +0530 Subject: [PATCH 6/6] test --- .github/dependencies.apt.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/dependencies.apt.txt b/.github/dependencies.apt.txt index ea376c44..8b642397 100644 --- a/.github/dependencies.apt.txt +++ b/.github/dependencies.apt.txt @@ -9,6 +9,7 @@ gettext git git-lfs libappstream-compose-dev +libappstream-compose0-dbgsym libarchive-tools libcurl4-openssl-dev libelf-dev