Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,10 @@ jobs:
with:
persist-credentials: false

- name: Test Linux prebuilt package selection
if: runner.os == 'Linux'
run: bash Launcher/Test-AuroraPrebuilt.sh

- name: Configure without graphics dependencies or game assets
run: >-
cmake -S runtime -B build-platform -G Ninja
Expand Down Expand Up @@ -96,10 +100,20 @@ jobs:
shell: bash
run: |
cmake --build build-macos --target \
mkw_ppc_cr_logical_tests \
mkw_platform_paths_tests \
mkw_runtime_config_tests \
mkw_memory_access_tests \
mkw_nand_save_tests \
mkw_nand_guest_path_tests \
mkw_host_file_tests \
mkw_system_bridge_tests \
mkw_u8_archive_tests \
mkw_runtime_config_io_tests \
mkw_pad_trigger_tests \
mkw_fiber_manager_tests \
mkw_audio_backend_ownership_tests \
mkw_network_receive_tests \
mkw_nand_settings_tests \
mkw_sc_serial_tests \
mkw_input_expr_tests \
Expand Down
27 changes: 7 additions & 20 deletions Launcher/Prepare-NativePrebuilt.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ set -euo pipefail

script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
workspace=$(cd "$script_dir/.." && pwd)
source "$script_dir/build-cache.sh"

arch=""
output_dir=""
Expand Down Expand Up @@ -123,25 +124,13 @@ assert_file "$cmake_bin" "Portable CMake (run prepare-portable-tools.sh --arch $
assert_file "$ninja_bin" "Portable Ninja"
assert_file "$cc" "Portable C compiler"
assert_file "$cxx" "Portable C++ compiler"
assert_dir "$aurora_source" "aurora-main source tree"
assert_file "$aurora_source/CMakeLists.txt" "aurora-main source tree"
clang_binary=$(normalize "${llvm_dir:-$toolchain_dir}/bin/clang-22")
assert_file "$clang_binary" "Portable clang driver binary"

(( parallel > 0 )) || parallel=$(nproc)

fingerprint_tree() {
# $1 = root dir, remaining args = top-level subdirectory names to exclude
local root=$1; shift
root=$(normalize "$root")
[[ -d "$root" ]] || return 0
local find_args=("$root" -name .git -prune -o)
local ex
for ex in "$@"; do find_args+=(-path "$root/$ex" -prune -o); done
find_args+=(-type f -print)
find "${find_args[@]}" | LC_ALL=C sort | while IFS= read -r abs; do
printf '%s %s\n' "${abs#$root/}" "$(sha256_of "$abs")"
done | sha256sum | awk '{print $1}'
}


# The fixed (path-independent) half of the configure command line - identical for every
# harvest, which is what lets one package be valid regardless of where it was built.
Expand Down Expand Up @@ -191,13 +180,11 @@ if [[ -n "$sysroot" ]]; then
fi
flag_fingerprint=$(printf '%s\n' "${fixed_configure_flags[@]}" | sha256sum | awk '{print $1}')

# extern/ is excluded because the payload ships that tree separately (aurora-main/extern is bundled
# whole by build-appimage.sh); build/ is a plain developer build directory.
aurora_fingerprint=$(fingerprint_tree "$aurora_source" extern build)
aurora_fingerprint=$(mkw_fingerprint_tree "$aurora_source" extern build)
Comment thread
patchzyy marked this conversation as resolved.
Resolved
[[ -n "$aurora_fingerprint" ]] || fail "The aurora source tree could not be fingerprinted: $aurora_source"

# The harvested Crypto++ archive is consumed against this tree's headers, so it is fingerprinted.
third_party_fingerprint=$(fingerprint_tree "$runtime_source/third_party")
third_party_fingerprint=$(mkw_fingerprint_tree "$runtime_source/third_party")
[[ -n "$third_party_fingerprint" ]] || fail "The vendored third-party tree could not be fingerprinted: $runtime_source/third_party"
mbedtls_fingerprint=$(sha256_of "$runtime_source/cmake/MbedTLSPin.cmake")

Expand Down Expand Up @@ -246,6 +233,7 @@ echo "Prepare-NativePrebuilt.sh: configuring the aurora/third-party staging buil
-DCMAKE_C_COMPILER="$cc" -DCMAKE_CXX_COMPILER="$cxx" \
-DCMAKE_MAKE_PROGRAM="$ninja_bin" \
-DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" \
-DMKW_AURORA_DIR="$aurora_source" \
-DMKW_NATIVE_PREBUILT_EXPORT_DIR="$export_dir"

meta_path="$export_dir/meta.txt"
Expand Down Expand Up @@ -636,8 +624,7 @@ generated_cmake="$output_dir/native_prebuilt.cmake"
# ---------------------------------------------------------------------------
# Provenance: the pinned toolchain is what makes a precompiled archive interchangeable with
# locally compiled objects (bit-for-bit the compiler and flag set the user's machine will use);
# LocalBuild.ps1's Windows equivalent re-checks both before consuming a package, and a Linux
# consumer-side check (once local-build.sh gains --native-prebuilt-dir) should do the same.
# local-build.sh checks the Aurora source fingerprint before accepting this package.
# compiler_sha256/flag_fingerprint/aurora_fingerprint/third_party_fingerprint were already computed
# above (needed before the --print-fingerprint-only early exit).
# ---------------------------------------------------------------------------
Expand Down
73 changes: 73 additions & 0 deletions Launcher/Test-AuroraPrebuilt.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
#!/usr/bin/env bash
set -euo pipefail

launcher=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
scratch=$(mktemp -d)
trap 'rm -rf "$scratch"' EXIT
cd "$scratch"
mkdir -p 'workspace/aurora-main/include' package
cd workspace/aurora-main
printf 'project(aurora)\n' > CMakeLists.txt
printf 'header\n' > 'include/a header.h'
# Manifest format used by existing Linux packages.
fingerprint=$(
for path in CMakeLists.txt 'include/a header.h'; do
printf '%s %s\n' "$path" "$(sha256sum < "$path" | cut -d ' ' -f 1)"
done | sha256sum | cut -d ' ' -f 1
)
mkdir -p .git extern build
touch .git/config extern/dependency build/object
cd ../..
if command -v shasum >/dev/null 2>&1; then
mkdir fallback-tools
for tool in shasum awk find sort xargs; do
ln -s "$(command -v "$tool")" "fallback-tools/$tool"
done
(
PATH="$PWD/fallback-tools"
source "$launcher/build-cache.sh"
[[ $(mkw_fingerprint_tree workspace/aurora-main extern build) == "$fingerprint" ]]
)
fi
touch package/native_prebuilt.cmake
args=(--workspace workspace --output-dir output --translator-bin unused
--cc true --cxx true --ninja true --x86-cpu-profile v2 --native-prebuilt-dir package)
expect_error() {
local expected=$1 output
if output=$(bash "$launcher/local-build.sh" "${args[@]}" 2>&1); then
echo "unexpected build success" >&2
exit 1
fi
if [[ "$output" != *"$expected"* ]]; then
printf 'expected %s, got %s\n' "$expected" "$output" >&2
exit 1
fi
}
expect_error 'Native prebuilt provenance is missing'
printf '{"AuroraSourceFingerprint":"%s"}\n' "$fingerprint" > package/provenance.json
expect_error 'Extracted main.dol'
mv workspace 'workspace[old]'
args[1]='workspace[old]'
expect_error 'Extracted main.dol'
printf 'edited\n' >> 'workspace[old]/aurora-main/include/a header.h'
expect_error 'does not match the Aurora sources'
printf 'header\n' > 'workspace[old]/aurora-main/include/a header.h'
for invalid in '{}' 'null' 'not json' '{"AuroraSourceFingerprint":false}' \
"{\"nested\":{\"AuroraSourceFingerprint\":\"$fingerprint\"}}"; do
printf '%s\n' "$invalid" > package/provenance.json
expect_error 'does not match the Aurora sources'
done

mkdir -p producer/{Launcher,aurora-main,runtime/{third_party,cmake},tools/bin}
cp "$launcher"/{Prepare-NativePrebuilt.sh,build-cache.sh} producer/Launcher/
touch producer/runtime/cmake/MbedTLSPin.cmake producer/tools/bin/{clang,clang++,clang-22}
producer_args=(--arch aarch64 --print-fingerprint-only --toolchain-dir "$PWD/producer/tools"
--cmake-bin /bin/true --ninja-bin /bin/true)
if output=$(bash producer/Launcher/Prepare-NativePrebuilt.sh "${producer_args[@]}" 2>&1); then
echo 'accepted an empty aurora checkout' >&2
exit 1
fi
[[ "$output" == *'aurora-main/CMakeLists.txt'* ]]
cp 'workspace[old]/aurora-main/CMakeLists.txt' producer/aurora-main/
output=$(bash producer/Launcher/Prepare-NativePrebuilt.sh "${producer_args[@]}")
[[ "$output" =~ aurora_fingerprint=[0-9a-f]{64} ]]
Comment thread
patchzyy marked this conversation as resolved.
Resolved
21 changes: 21 additions & 0 deletions Launcher/build-cache.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,27 @@ mkw_sha256() {
"${mkw_sha256_cmd[@]}" "$@" | awk '{print $1}'
}

# Linux package provenance uses sorted relative paths followed by their file digests.
mkw_fingerprint_tree() (
set -o pipefail
cd -- "$1" || exit 1
shift
local args=(. -name .git -prune -o) dir record digest
for dir in "$@"; do args+=(-path "./$dir" -prune -o); done
find "${args[@]}" -type f -print0 | LC_ALL=C sort -z |
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
if [[ ${mkw_sha256_cmd[0]} == sha256sum ]]; then
xargs -0 -r sha256sum -z -- | while IFS= read -r -d '' record; do
# sha256sum -z emits the digest, two spaces, then the ./ path.
printf '%s %s\n' "${record:68}" "${record:0:64}"
done
else
while IFS= read -r -d '' record; do
digest=$(mkw_sha256 < "$record") || exit 1
printf '%s %s\n' "${record#./}" "$digest"
done
fi | mkw_sha256
)

# Prints "<sha> <relative path>" for every file under each directory, relative to $1.
mkw_digest_tree() {
local root=$1; shift
Expand Down
10 changes: 9 additions & 1 deletion Launcher/local-build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,12 @@ require_command "$cxx_bin" cxx

if [[ -n "$native_prebuilt_dir" ]]; then
assert_file "$native_prebuilt_dir/native_prebuilt.cmake" "Native prebuilt package"
assert_file "$native_prebuilt_dir/provenance.json" "Native prebuilt provenance"
native_prebuilt_dir=$(cd "$native_prebuilt_dir" && pwd)
assert_file "$workspace/aurora-main/CMakeLists.txt" "Aurora source tree"
aurora_fingerprint=$(mkw_fingerprint_tree "$workspace/aurora-main" extern build)
"$cmake_bin" "-DAURORA_FINGERPRINT=$aurora_fingerprint" "-DPROVENANCE_FILE=$native_prebuilt_dir/provenance.json" \
-P "$script_dir/../runtime/cmake/AuroraFingerprint.cmake"
fi

host_supports_x86_64_v3() {
Expand Down Expand Up @@ -497,7 +503,9 @@ if [[ -n "$fuse_ld_override" ]]; then
configure_args+=(-DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=$fuse_ld_quoted")
fi
if [[ -n "$native_prebuilt_dir" ]]; then
configure_args+=(-DMKW_NATIVE_PREBUILT_DIR="$native_prebuilt_dir")
configure_args+=(-DMKW_NATIVE_PREBUILT_DIR="$native_prebuilt_dir" -DMKW_AURORA_DIR="$workspace/aurora-main")
else
configure_args+=(-UMKW_NATIVE_PREBUILT_DIR)
fi
if [[ -n "$sysroot" ]]; then
configure_args+=(-DCMAKE_SYSROOT="$sysroot")
Expand Down
3 changes: 2 additions & 1 deletion runtime/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ endif()

option(MKW_BUILD_PRODUCTS "Build translated WiiCompiled product targets" ON)
option(MKW_BUILD_PSQ_TESTS "Build focused PSQ ISA tests" OFF)
option(MKW_BUILD_AURORA_TESTS "Build integrated Aurora regression tests" OFF)
set(MKW_X86_CPU_PROFILE "v3" CACHE STRING
"Minimum x86-64 CPU profile for translated products (v2 or v3)")
set_property(CACHE MKW_X86_CPU_PROFILE PROPERTY STRINGS v2 v3)
Expand Down Expand Up @@ -277,7 +278,7 @@ else()

# Aurora components the runtime does not use. They would otherwise fetch
# extra dependencies at configure time, which offline builds cannot do.
set(AURORA_ENABLE_TESTS OFF CACHE BOOL "" FORCE)
set(AURORA_ENABLE_TESTS ${MKW_BUILD_AURORA_TESTS} CACHE BOOL "" FORCE)
set(AURORA_ENABLE_EXAMPLES OFF CACHE BOOL "" FORCE)
set(AURORA_ENABLE_DVD OFF CACHE BOOL "" FORCE)
set(AURORA_ENABLE_THP OFF CACHE BOOL "" FORCE)
Expand Down
7 changes: 7 additions & 0 deletions runtime/cmake/AuroraFingerprint.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
cmake_minimum_required(VERSION 3.20)

file(READ "${PROVENANCE_FILE}" _provenance)
string(JSON _recorded ERROR_VARIABLE _error GET "${_provenance}" AuroraSourceFingerprint)
if(_error OR NOT _recorded STREQUAL AURORA_FINGERPRINT)
message(FATAL_ERROR "Native prebuilt package does not match the Aurora sources. Regenerate it with Prepare-NativePrebuilt.sh or omit --native-prebuilt-dir.")
endif()
Loading
Loading