Update for compatibility with LLVM 18

This update makes the library compatible with the
new version of Alusus (0.15) which is based on
LLVM18. This change involves:
* Switching to a new version of binaryen (129)
* Switching from stdlib-min to wasi-libc
* Update build sccript to fetch pre-built binaries
  for binaryen and wasi-libc.

Author: sarmadka

Scripts/build.alusus

@@ -21,55 +21,64 @@ import "Srl/System";
 import "Srl/Console";
 import "Srl/Net";
 import "Srl/String";
+import "Srl/Regex";
 import "Srl/Fs";
 import "Core/Data";
 import "Spp";
 import "Zip";
 
 use Srl;
 
-function downloadPackage (url: ptr[array[Char]], targetDir: ptr[array[Char]]) {
-    def result: ptr[array[Char]];
-    def size: Int;
-    if !Net.getFile(url, "/tmp/package.zip") {
-      System.fail(1, String("Failed to download ") + url);
-    };
-
-    if Zip.extractFromFile("/tmp/package.zip", targetDir, 0, 0) != 0 {
-      System.fail(1, String("Failed to extract downloaded package ") + url);
-    }
+function getFilenameFromPath (path: ptr[array[Char]]): String {
+    return String((String.findLast(path, '/') + 1)~cast[CharsPtr]);
 }
 
 function getDirectoryFromPath (path: ptr[array[Char]]): Srl.String {
     return Srl.String(path, Srl.String.findLast(path, '/')~cast[ArchInt] - path~cast[ArchInt]);
 }
 
+function downloadPackage (url: ptr[array[Char]], targetDir: ptr[array[Char]]) {
+    def filename: String = getFilenameFromPath(url);
+    Console.print(String("Downloading ") + url + " to " + targetDir + "\n");
+    Console.print(String("Filename extracted from URL: ") + filename + "\n");
+
+    if System.exec(String.format("curl -L -o /tmp/%s %s", filename.buf, url)) != 0 {
+        System.fail(1, String("Failed to download ") + url);
+    }
+
+    // Is it a zip file?
+    if Regex.match("\\.zip$", filename, 1).getLength() != 0 {
+        if Zip.extractFromFile(String.format("/tmp/%s", filename.buf), targetDir, 0, 0) != 0 {
+            System.fail(1, String("Failed to extract downloaded package ") + url);
+        }
+    } else if Regex.match("\\.tar\\.gz$", filename, 1).getLength() != 0 {
+        if System.exec(String.format("tar -xzf /tmp/%s -C %s", filename.buf, targetDir)) != 0 {
+            System.fail(1, String("Failed to extract downloaded package ") + url);
+        }
+    } else {
+        System.fail(1, String("Unknown package format for ") + url);
+    }
+    Console.print("Download and extraction successful\n\n");
+}
+
 def rootPath: Srl.String = getDirectoryFromPath(getThisSourceFullPath[]) + "/..";
 def buildPath: Srl.String = rootPath + "/Builds";
 
 System.exec(String.format("mkdir -p %s", buildPath.buf));
 
-// Build wabt
-def wabtVersion: "1.0.20";
-if !Fs.exists(String.format("%s/wabt-%s/bin/wat2wasm", buildPath.buf, wabtVersion)) {
-    Console.print("<<<< Building wabt >>>>\n\n");
-    System.exec(String.format("rm -rf %s/wabt-%s", buildPath.buf, wabtVersion));
-    downloadPackage(String("https://github.com/WebAssembly/wabt/archive/") + wabtVersion + ".zip", buildPath + "/");
-    System.exec(String.format(
-        "mkdir -p %s/wabt-%s/build && cd %s/wabt-%s/build && cmake -DBUILD_TESTS=OFF -DBUILD_LIBWASM=OFF .. && cmake --build .",
-        buildPath.buf, wabtVersion, buildPath.buf, wabtVersion
-    ));
-} else {
-    Console.print("<<<< wabt already available, skipping... >>>>\n\n");
-}
-
-// Build binaryen
-def binaryenVersion: "version_98";
-if !Fs.exists(String.format("%s/binaryen-%s/bin/wasm-opt", buildPath.buf, binaryenVersion)) {
-    Console.print("<<<< Building binaryen >>>>\n\n");
-    System.exec(String.format("rm -rf %s/binaryen-%s", buildPath.buf, binaryenVersion));
-    downloadPackage(String("https://github.com/WebAssembly/binaryen/archive/") + binaryenVersion + ".zip", buildPath + "/");
-    System.exec(String.format("cd %s/binaryen-%s/ && cmake . && make", buildPath.buf, binaryenVersion));
+// Fetch binaryen
+def binaryenVersion: "129";
+if !Fs.exists(String.format("%s/binaryen-version_%s/bin/wasm-opt", buildPath.buf, binaryenVersion)) {
+    Console.print("<<<< Fetching binaryen >>>>\n\n");
+    System.exec(String.format("rm -rf %s/binaryen-version_%s", buildPath.buf, binaryenVersion));
+    downloadPackage(
+        String.format(
+            "https://github.com/WebAssembly/binaryen/releases/download/version_%s/binaryen-version_%s-x86_64-linux.tar.gz",
+            binaryenVersion,
+            binaryenVersion
+        ), 
+        buildPath + "/"
+    );
 } else {
     Console.print("<<<< binaryen already available, skipping... >>>>\n\n");
 }
@@ -78,98 +87,36 @@ function getFilenameFromPath (path: ptr[array[Char]]): String {
     return String((String.findLast(path, '/') + 1)~cast[CharsPtr]);
 }
 
-// Build stdlib
-def stdlibPath: Srl.String = buildPath + "/wasm-stdlib-hack-master";
-def clangCmd: "clang-11 --target=wasm32-unknown-unknown-wasm -Iinclude/libc -Oz";
-def disableWarn: "-Wno-bitwise-op-parentheses -Wno-shift-op-parentheses";
-def mallocOpts: "-DMORECORE_CANNOT_TRIM=1 -DHAVE_MMAP=0 -DHAVE_MREMAP=0 -DLACKS_TIME_H=1 -DNO_MALLOC_STATS=1 "
-    "-Dmalloc_getpagesize=65536 -Wno-expansion-to-defined";
-if !Fs.exists(String.format("%s/dist/stdlib-min.wasm", stdlibPath.buf)) {
-    Console.print("<<<< Building wasm-stdlib-hack >>>>\n\n");
-    System.exec(String.format("rm -rf %s", stdlibPath.buf));
-    downloadPackage("https://github.com/guybedford/wasm-stdlib-hack/archive/master.zip", buildPath + "/");
-
-    System.exec(String.format("mkdir -p %s/lib", stdlibPath.buf));
-
-    // Build stdlib-base
-    System.exec(String.format(
-        "cd %s && ../wabt-%s/bin/wat2wasm -r -o lib/stdlib-base.wasm src/stdlib-base.wast", stdlibPath.buf, wabtVersion
-    ));
-    // Build malloc.wasm
-    System.exec(String.format(
-        "cd %s && %s -c src/dlmalloc.c -o lib/malloc.wasm %s %s", stdlibPath.buf, clangCmd, mallocOpts, disableWarn
-    ));
-    // Build other functions.
-    def fnFilenames: Array[String]({
-        String("stdlib/abs"),
-        String("string/memcpy"),
-        String("string/memset"),
-        String("string/memrchr"),
-        String("string/memcmp"),
-        String("string/memmove"),
-        String("string/stpcpy"),
-        String("string/stpncpy"),
-        String("string/strchr"),
-        String("string/memchr"),
-        String("string/strrchr"),
-        String("string/strstr"),
-        String("string/strcmp"),
-        String("string/strncmp"),
-        String("string/strcpy"),
-        String("string/strncpy"),
-        String("string/strcat"),
-        String("string/strncat"),
-        String("string/strlen"),
-        String("string/strchrnul"),
-        String("math/sin"),
-        String("math/sinf"),
-        String("math/__sin"),
-        String("math/__sindf"),
-        String("math/cos"),
-        String("math/cosf"),
-        String("math/__cos"),
-        String("math/__cosdf"),
-        String("math/__rem_pio2"),
-        String("math/__rem_pio2f"),
-        String("math/__rem_pio2_large"),
-        String("math/scalbn"),
-        String("math/powf"),
-        // String("math/abs"),
-        String("math/fabsf")
-    });
-    def i: Int;
-    for i = 0, i < fnFilenames.getLength(), ++i {
-        System.exec(String.format(
-            "cd %s && %s -Isrc/internal/ -fvisibility=hidden -c src/%s.c -o lib/%s.wasm %s",
-            stdlibPath.buf, clangCmd, fnFilenames(i).buf, getFilenameFromPath(fnFilenames(i)).buf, disableWarn
-        ));
-    }
-    
-    // Link into stdlib-min.wasm
-    def cmd: String = String.format(
-        "cd %s && wasm-ld-11 --no-entry --export-dynamic -r lib/stdlib-base.wasm lib/malloc.wasm", stdlibPath.buf
+// Fetch wasi-libc
+def wasiLibcVersion: "24";
+def wasiLibcInnerPath: "share/wasi-sysroot/lib/wasm32-wasi";
+if !Fs.exists(String.format("%s/wasi-sdk-%s.0-x86_64-linux/%s/libc.a", buildPath.buf, wasiLibcVersion, wasiLibcInnerPath)) {
+    Console.print("<<<< Fetching wasi-libc >>>>\n\n");
+    System.exec(String.format("rm -rf %s/wasi-sdk-%s.0-x86_64-linux", buildPath.buf, wasiLibcVersion));
+    downloadPackage(
+        String.format(
+            "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-%s/wasi-sdk-%s.0-x86_64-linux.tar.gz",
+            wasiLibcVersion,
+            wasiLibcVersion
+        ),
+        buildPath + "/"
     );
-    for i = 0, i < fnFilenames.getLength(), ++i {
-        cmd += " lib/";
-        cmd += getFilenameFromPath(fnFilenames(i));
-        cmd += ".wasm";
-    }
-    cmd += " -o dist/stdlib-min.wasm";
-    System.exec(cmd);
 } else {
-    Console.print("<<<< wasm-stdlib-hack already available, skipping... >>>>\n\n");
+    Console.print("<<<< wasi-libc already available, skipping... >>>>\n\n");
 }
 
 // Copy needed files.
-Console.print("<<<< Installing generated files >>>>\n\n");
+Console.print("<<<< Installing fetched files >>>>\n\n");
 System.exec(String.format("mkdir -p %s/Tools/bin && mkdir -p %s/Tools/lib", rootPath.buf, rootPath.buf));
 System.exec(
-    String.format("cp %s/binaryen-%s/bin/wasm-opt %s/Tools/bin/", buildPath.buf, binaryenVersion, rootPath.buf)
+    String.format("cp %s/binaryen-version_%s/bin/wasm-opt %s/Tools/bin/", buildPath.buf, binaryenVersion, rootPath.buf)
+);
+System.exec(
+    String.format("cp %s/wasi-sdk-%s.0-x86_64-linux/%s/libc.a %s/Tools/lib/", buildPath.buf, wasiLibcVersion, wasiLibcInnerPath, rootPath.buf)
 );
 System.exec(
-    String.format("cp %s/binaryen-%s/lib/libbinaryen.so %s/Tools/lib/", buildPath.buf, binaryenVersion, rootPath.buf)
+    String.format("cp %s/wasi-sdk-%s.0-x86_64-linux/%s/crt1.o %s/Tools/lib/", buildPath.buf, wasiLibcVersion, wasiLibcInnerPath, rootPath.buf)
 );
-System.exec(String.format("cp %s/dist/stdlib-min.wasm %s", stdlibPath.buf, rootPath.buf));
 
 Console.print("<<<< Build successful >>>>\n\n");
 

Tools/bin/wasm-opt

@@ -124,7 +124,8 @@
         def binding: ref[Core.Basic.Binding](Core.Basic.getInterface[fn, Core.Basic.Binding]);
         binding.setMember("name", Core.Basic.TiStr("wasmStart"));
         def wasm: Build.Wasm(fn, uiEndpointsPath + "_intermediate.wasm");
-        wasm.addDependency(webPlatformPath + String("stdlib-min.wasm"));
+        wasm.addDependency(webPlatformPath + String("Tools/lib/crt1.o"));
+        wasm.addDependency(webPlatformPath + String("Tools/lib/libc.a"));
         wasm.addFlags({ String("--export=malloc"), String("--export=realloc") });
         wasm.generate();
         def fnName: String = Spp.astMgr.getDefinitionName(fn);

Tools/lib/crt1.o

@@ -1105,14 +1105,32 @@ function toLocalISOString(date) {
 
 // Main Functions
 
+const wasiShim = {
+  proc_exit: (code) => console.log(`Process exited with code ${code}`),
+  fd_write: () => 0,
+  fd_close: () => 0,
+  fd_seek: () => 0,
+  fd_fdstat_get: () => 0,
+  args_get: () => 0,
+  args_sizes_get: () => 0,
+  environ_sizes_get: (outCount, outSize) => 0,
+  environ_get: (outPtr, outBuf) => 0,
+};
+
 async function loadWasm(filename, importTable) {
     let response = await fetch(filename);
     let binary = await response.arrayBuffer();
-    return WebAssembly.instantiate( binary, { "env": importTable } );
+    return WebAssembly.instantiate( binary, { "env": importTable, wasi_snapshot_preview1: wasiShim } );
 }
 
 async function start(moduleName) {
     program = await loadWasm(moduleName, wasmApi);
+
+    // Initialize wasi-libc
+    if (program.instance.exports._initialize) {
+        program.instance.exports._initialize();
+    }
+
     wasmMemory = program.instance.exports.memory;
 
     asyncifyDataPtr = program.instance.exports.malloc(STACK_SIZE + 8);