Replace setuptools/bundled-capnproto build with conan-py-build + CMake [looking for feedback]

.github/workflows/wheels.yml

@@ -44,14 +44,10 @@ jobs:
           output-dir: wheelhouse
         env:
           CIBW_ARCHS: ${{ matrix.arch }}
-          # Drop EOL CPython 3.8 (also avoids the macOS x86_64-only-installer warning)
           CIBW_SKIP: "cp38-*"
           CIBW_TEST_REQUIRES: pytest pytest-asyncio
           CIBW_TEST_COMMAND: pytest {project}
-          # Force libcapnp to build for the target arch on macOS (the runner
-          # is arm64, so without this an x86_64 wheel ends up linking against
-          # an arm64 libkj/libcapnp and failing to load at test time).
-          CMAKE_OSX_ARCHITECTURES: "${{ runner.os == 'macOS' && matrix.arch || '' }}"
+          CIBW_BEFORE_ALL_LINUX: "pip install conan && conan profile detect --force && conan install --requires=m4/1.4.19 --build=m4*"
 
       - uses: actions/upload-artifact@v6
         with:
@@ -62,7 +58,7 @@ jobs:
   # on release tags rather than every push/PR.
   build_wheels_exotic:
     name: Build wheels (exotic) ${{ matrix.arch }}
-    if: startsWith(github.ref, 'refs/tags/v')
+    if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/conan-py-build'
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -81,6 +77,7 @@ jobs:
         env:
           CIBW_ARCHS: ${{ matrix.arch }}
           CIBW_SKIP: "cp38-*"
+          CIBW_BEFORE_ALL_LINUX: "pip install conan && conan profile detect --force && conan install --requires=m4/1.4.19 --build=m4*"
           # Tests under QEMU are extremely slow; skip them for these arches.
           CIBW_TEST_SKIP: "*"
 

.gitignore

@@ -40,7 +40,6 @@ nosetests.xml
 # Cpp files
 capnp/*.cpp
 
-capnp/version.py
 MANIFEST
 docs/_build
 

CMakeLists.txt

@@ -0,0 +1,67 @@
+cmake_minimum_required(VERSION 3.15)
+project(pycapnp LANGUAGES CXX)
+
+find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module)
+# FindPython3 skips SOABI detection when CMAKE_OSX_ARCHITECTURES differs from the runner
+# arch (e.g. x86_64 wheel on arm64 runner), leaving Python3_SOABI empty. On macOS,
+# Rosetta 2 can transparently execute the cross-compile target Python, so re-derive it.
+if(APPLE AND NOT Python3_SOABI)
+    execute_process(
+        COMMAND "${Python3_EXECUTABLE}" -c
+            "import sysconfig; print(sysconfig.get_config_var('SOABI') or '', end='')"
+        OUTPUT_VARIABLE Python3_SOABI
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+endif()
+find_package(CapnProto REQUIRED)
+
+set(CAPNP_PYX "${CMAKE_SOURCE_DIR}/capnp/lib/capnp.pyx")
+set(CAPNP_CPP "${CMAKE_CURRENT_BINARY_DIR}/capnp/lib/capnp.cpp")
+
+file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/capnp/lib")
+
+add_custom_command(
+    OUTPUT "${CAPNP_CPP}"
+    COMMAND "${Python3_EXECUTABLE}" -m cython
+            --cplus
+            -I "${CMAKE_SOURCE_DIR}"
+            "${CAPNP_PYX}"
+            -o "${CAPNP_CPP}"
+    DEPENDS "${CAPNP_PYX}"
+            "${CMAKE_SOURCE_DIR}/capnp/includes/capnp_cpp.pxd"
+            "${CMAKE_SOURCE_DIR}/capnp/includes/schema_cpp.pxd"
+            "${CMAKE_SOURCE_DIR}/capnp/helpers/helpers.pxd"
+            "${CMAKE_SOURCE_DIR}/capnp/lib/capnp.pxd"
+    COMMENT "Running Cython on capnp/lib/capnp.pyx"
+)
+
+Python3_add_library(capnp_ext MODULE WITH_SOABI
+    "${CAPNP_CPP}"
+    "${CMAKE_SOURCE_DIR}/capnp/helpers/capabilityHelper.cpp"
+    "${CMAKE_SOURCE_DIR}/capnp/includes/PyCustomMessageBuilder.cpp"
+)
+
+set_target_properties(capnp_ext PROPERTIES
+    OUTPUT_NAME "capnp"
+    CXX_STANDARD 14
+    CXX_STANDARD_REQUIRED ON
+)
+
+target_include_directories(capnp_ext PRIVATE
+    "${CMAKE_SOURCE_DIR}"
+    "${CMAKE_CURRENT_BINARY_DIR}"
+    "${CMAKE_SOURCE_DIR}/capnp/includes"
+)
+
+target_link_libraries(capnp_ext PRIVATE
+    CapnProto::capnpc
+    CapnProto::capnp-rpc
+    CapnProto::capnp
+    CapnProto::kj-async
+    CapnProto::kj
+)
+
+install(TARGETS capnp_ext DESTINATION capnp/lib)
+
+install(FILES "${CMAKE_CURRENT_BINARY_DIR}/capnp/lib/capnp_api.h"
+        DESTINATION capnp/lib)

capnp/version.py

@@ -0,0 +1,9 @@
+from importlib.metadata import version as _pkg_version
+
+from .lib.capnp import _CAPNP_VERSION_MAJOR as LIBCAPNP_VERSION_MAJOR  # noqa: F401
+from .lib.capnp import _CAPNP_VERSION_MINOR as LIBCAPNP_VERSION_MINOR  # noqa: F401
+from .lib.capnp import _CAPNP_VERSION_MICRO as LIBCAPNP_VERSION_MICRO  # noqa: F401
+from .lib.capnp import _CAPNP_VERSION as LIBCAPNP_VERSION  # noqa: F401
+
+version = _pkg_version("pycapnp")
+short_version = version

conan-host.profile

@@ -0,0 +1,14 @@
+{% set cibw_arch = os.environ.get("CIBW_ARCHS", "") %}
+{% set conan_arch = {"x86_64": "x86_64", "arm64": "armv8", "x86": "x86"}.get(cibw_arch, "") %}
+{% set dt = os.environ.get("MACOSX_DEPLOYMENT_TARGET", "").replace(".", "_") %}
+include(default)
+{% if conan_arch %}
+
+[settings]
+arch={{ conan_arch }}
+{% endif %}
+{% if platform.system() == 'Darwin' and cibw_arch in ("x86_64", "arm64") and dt %}
+
+[buildenv]
+WHEEL_ARCH=macosx_{{ dt }}_{{ cibw_arch }}
+{% endif %}

conanfile.py

@@ -0,0 +1,37 @@
+import os
+import sys
+import sysconfig
+
+from conan import ConanFile
+from conan.tools.cmake import CMake, CMakeDeps, CMakeToolchain, cmake_layout
+from conan.tools.files import copy
+
+
+class PycapnpConan(ConanFile):
+    name = "pycapnp"
+    settings = "os", "compiler", "build_type", "arch"
+
+    def requirements(self):
+        self.requires("capnproto/1.0.1")
+
+    def layout(self):
+        cmake_layout(self)
+
+    def generate(self):
+        tc = CMakeToolchain(self)
+        tc.cache_variables["Python3_EXECUTABLE"] = sys.executable
+        tc.cache_variables["Python3_INCLUDE_DIR"] = sysconfig.get_path("include")
+        tc.generate()
+        CMakeDeps(self).generate()
+
+    def build(self):
+        cmake = CMake(self)
+        cmake.configure()
+        cmake.build()
+
+    def package(self):
+        cmake = CMake(self)
+        cmake.install()
+        capnp_include = os.path.join(str(self.dependencies["capnproto"].package_folder), "include", "capnp")
+        # excludes="*/*" skips subdirectories, top-level schemas only
+        copy(self, "*.capnp", src=capnp_include, dst=os.path.join(self.package_folder, "capnp"), excludes="*/*")