feat: expose staged index segment transactions

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "8.0.0-beta.1"
+current_version = "9.0.0-beta.2"
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)(-(?P<prerelease>(beta|rc))\\.(?P<prerelease_num>\\d+))?"
 serialize = [
     "{major}.{minor}.{patch}-{prerelease}.{prerelease_num}",
@@ -55,6 +55,11 @@ filename = "Cargo.toml"
 search = 'lance-datagen = {{ version = "={current_version}"'
 replace = 'lance-datagen = {{ version = "={new_version}"'
 
+[[tool.bumpversion.files]]
+filename = "Cargo.toml"
+search = 'lance-derive = {{ version = "={current_version}"'
+replace = 'lance-derive = {{ version = "={new_version}"'
+
 [[tool.bumpversion.files]]
 filename = "Cargo.toml"
 search = 'lance-encoding = {{ version = "={current_version}"'

.cargo/config.toml

@@ -9,6 +9,13 @@ debug = true
 codegen-units = 16
 lto = "thin"
 
+[profile.release-no-lto]
+inherits = "release"
+debug = true
+lto = false
+# Prioritize compile time when LTO is not relevant to the measurement.
+codegen-units = 16
+
 [profile.bench]
 inherits = "release"
 lto = "thin"

.github/dependabot.yml

@@ -0,0 +1,30 @@
+version: 2
+updates:
+  - package-ecosystem: "cargo"
+    directory: "/"
+    versioning-strategy: lockfile-only
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
+
+  - package-ecosystem: "cargo"
+    directory: "/python"
+    versioning-strategy: lockfile-only
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
+
+  - package-ecosystem: "cargo"
+    directory: "/java/lance-jni"
+    versioning-strategy: lockfile-only
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
+
+  - package-ecosystem: "uv"
+    directory: "/python"
+    versioning-strategy: lockfile-only
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
+

.github/workflows/build_linux_wheel/action.yml

@@ -3,7 +3,7 @@ name: build-linux-wheel
 description: "Build a manylinux wheel for lance"
 inputs:
   python-minor-version:
-    description: "9, 10, 11, 12"
+    description: "10, 11, 12, 13"
     required: true
   args:
     description: "--release"
@@ -41,11 +41,9 @@ runs:
         args: ${{ inputs.args }}
         maturin-version: "1.10.2"
         before-script-linux: |
-          set -e
-          yum install -y openssl-devel \
-            && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
-            && unzip /tmp/protoc.zip -d /usr/local \
-            && rm /tmp/protoc.zip
+          set -euo pipefail
+          yum install -y openssl-devel
+          bash "${GITHUB_WORKSPACE}/.github/workflows/build_linux_wheel/install-protoc.sh"
     - name: Build x86_64 Manylinux {manylinux} wheel
       if: ${{ inputs.arm-build == 'false' && inputs.manylinux != '2_17' }}
       uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1
@@ -60,11 +58,9 @@ runs:
         args: ${{ inputs.args }}
         maturin-version: "1.10.2"
         before-script-linux: |
-          set -e
-          yum install -y openssl-devel clang \
-            && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
-            && unzip /tmp/protoc.zip -d /usr/local \
-            && rm /tmp/protoc.zip
+          set -euo pipefail
+          yum install -y openssl-devel clang
+          bash "${GITHUB_WORKSPACE}/.github/workflows/build_linux_wheel/install-protoc.sh"
     - name: Build Arm Manylinux Wheel
       if: ${{ inputs.arm-build == 'true' }}
       uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1
@@ -76,8 +72,6 @@ runs:
         args: ${{ inputs.args }}
         maturin-version: "1.10.2"
         before-script-linux: |
-          set -e
-          yum install -y openssl-devel clang \
-            && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip > /tmp/protoc.zip \
-            && unzip /tmp/protoc.zip -d /usr/local \
-            && rm /tmp/protoc.zip
+          set -euo pipefail
+          yum install -y openssl-devel clang
+          bash "${GITHUB_WORKSPACE}/.github/workflows/build_linux_wheel/install-protoc.sh"

.github/workflows/build_linux_wheel/install-protoc.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+version="${PROTOC_VERSION:-24.4}"
+install_dir="${PROTOC_INSTALL_DIR:-/usr/local}"
+machine="${1:-$(uname -m)}"
+
+case "${machine}" in
+  aarch64 | arm64)
+    asset_arch="aarch_64"
+    ;;
+  x86_64)
+    asset_arch="x86_64"
+    ;;
+  *)
+    echo "Unsupported protoc architecture: ${machine}" >&2
+    exit 1
+    ;;
+esac
+
+zip_path="/tmp/protoc-${version}-linux-${asset_arch}.zip"
+url="https://github.com/protocolbuffers/protobuf/releases/download/v${version}/protoc-${version}-linux-${asset_arch}.zip"
+
+for attempt in 1 2 3 4 5; do
+  rm -f "${zip_path}"
+
+  if curl -fsSL --connect-timeout 15 --max-time 120 -o "${zip_path}" "${url}" \
+    && unzip -tq "${zip_path}" >/dev/null; then
+    break
+  fi
+
+  if [[ "${attempt}" == "5" ]]; then
+    echo "Failed to download a valid protoc archive from ${url}" >&2
+    exit 1
+  fi
+
+  sleep "$((attempt * 2))"
+done
+
+unzip -q -o "${zip_path}" -d "${install_dir}"
+rm -f "${zip_path}"
+
+if [[ "$(uname -s)" == "Linux" ]]; then
+  "${install_dir}/bin/protoc" --version
+else
+  test -f "${install_dir}/bin/protoc"
+fi

.github/workflows/build_mac_wheel/action.yml

@@ -3,7 +3,7 @@ name: build_wheel
 description: "Build a lance wheel"
 inputs:
   python-minor-version:
-    description: "9, 10, 11, 12"
+    description: "10, 11, 12, 13"
     required: true
   args:
     description: "--release"

.github/workflows/build_windows_wheel/action.yml

@@ -3,7 +3,7 @@ name: build_wheel
 description: "Build a lance wheel"
 inputs:
   python-minor-version:
-    description: "9, 10, 11, 12"
+    description: "10, 11, 12, 13"
     required: true
   args:
     description: "--release"

.github/workflows/compat-pair.yml

@@ -0,0 +1,78 @@
+# On-demand cross-version index compatibility run between two arbitrary refs.
+#
+# The PR path lives in python.yml (compat-sequence job), which ages the two previous
+# majors into the PR head reusing the prebuilt wheel. This workflow is the manual escape
+# hatch for any other pairing: each ref (version, sha, branch, or tag) is provisioned by
+# the framework -- a published release installs a wheel, anything else is built from a
+# worktree via maturin -- so two arbitrary refs can be compared even when neither has a
+# published wheel. The suite ages an index under the writer ref and exercises it under the
+# reader, searching maintenance-op sequences for panics or correctness divergence.
+name: Compat (index ref pair)
+
+on:
+  workflow_dispatch:
+    inputs:
+      from_ref:
+        description: "Writer ref (version / sha / branch). Blank = 2nd most recent release."
+        required: false
+        default: ""
+      to_ref:
+        description: "Reader ref (version / sha / branch). Blank = most recent release."
+        required: false
+        default: ""
+      kinds:
+        description: "Comma-separated index kinds (INVERTED,BTREE,...) or 'all'."
+        required: false
+        default: "all"
+      max_length:
+        description: "Max maintenance-op sequence length to search (deeper = slower)."
+        required: false
+        default: "5"
+
+jobs:
+  compat-pair:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    defaults:
+      run:
+        working-directory: python
+    env:
+      COMPAT_FROM_REF: ${{ inputs.from_ref }}
+      COMPAT_TO_REF: ${{ inputs.to_ref }}
+      COMPAT_MAX_LENGTH: ${{ inputs.max_length || '5' }}
+    steps:
+      # Full history so arbitrary refs can be checked out for builds.
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.11"
+      # Toolchain for the build-from-source provisioning path (refs without a wheel).
+      - uses: actions-rust-lang/setup-rust-toolchain@a0b538fa0b742a6aa35d6e2c169b4bd06d225a98 # v1
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
+      - name: Install build deps
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Install host deps
+        run: pip install pytest pytest-xdist pyarrow packaging maturin
+      - name: Resolve kinds
+        id: kinds
+        env:
+          KINDS_IN: ${{ inputs.kinds }}
+        run: |
+          if [ -z "$KINDS_IN" ] || [ "$KINDS_IN" = "all" ]; then
+            echo "value=INVERTED,BTREE,BITMAP,LABEL_LIST,NGRAM,ZONEMAP,BLOOMFILTER" >> "$GITHUB_OUTPUT"
+          else
+            echo "value=$KINDS_IN" >> "$GITHUB_OUTPUT"
+          fi
+      # Oversubscribe (4x cores): each scenario writes a small dataset to disk and the
+      # reader spends most of its time in short subprocess round-trips, so a worker waiting
+      # on disk or the sub-venv pipe overlaps another's compute.
+      - name: Run compat suite (${{ inputs.from_ref }} -> ${{ inputs.to_ref }})
+        env:
+          COMPAT_KINDS: ${{ steps.kinds.outputs.value }}
+        run: |
+          python -m pytest python/tests/compat/test_index_sequence.py \
+            --run-compat -n "$(( $(nproc) * 4 ))" -v --no-header