v0.3.0

.buildkite/developer.pipeline.yml

@@ -1,4 +1,13 @@
 steps:
+  - label: ":lock: Enter developer CI"
+    command: "true"
+    concurrency: 1
+    concurrency_group: "cunumeric/developer"
+    agents:
+      queue: "cuda"
+
+  - wait: ~
+
   - group: ":hammer: Developer"
     key: "developer"
     steps:
@@ -7,7 +16,7 @@ steps:
           - JuliaCI/julia#v1:
               version: "{{matrix.julia}}"
               # Per version AND fusion: dev wrappers are ABI-specific, and same-depot
-              # concurrent jobs contend on Pkg locks. Separate from the JLL cache.
+              # concurrent jobs contend on Pkg locks. Separate from the JLL cache. 
               cache_dir: "${HOME}/.cache/julia-buildkite-plugin-developer-{{matrix.julia}}-{{matrix.fusion}}"
         command: ".buildkite/run_developer_ci.sh"
         artifact_paths:
@@ -32,6 +41,17 @@ steps:
               - "1.10"
               - "1.11"
               - "1.12"
+              - "1.13"
             fusion:
               - "on"
               - "off"
+
+  - wait: ~
+    continue_on_failure: true
+
+  - label: ":unlock: Leave developer CI"
+    command: "true"
+    concurrency: 1
+    concurrency_group: "cunumeric/developer"
+    agents:
+      queue: "cuda"

.buildkite/jll.pipeline.yml

@@ -11,9 +11,9 @@ steps:
         plugins:
           - JuliaCI/julia#v1:
               version: "{{matrix.julia}}"
-              # Developer builds install local wrapper overrides into the depot.
-              # Keep JLL tests in a separate cache so those overrides cannot leak here.
-              cache_dir: "${HOME}/.cache/julia-buildkite-plugin-jll"
+              # Isolate Julia versions and compile-time fusion preferences, and keep
+              # JLL tests separate from developer wrapper overrides.
+              cache_dir: "${HOME}/.cache/julia-buildkite-plugin-jll-{{matrix.julia}}-{{matrix.fusion}}"
           - jquick/pre-hook#v1.2.0:
               command: |
                 if [ "{{matrix.fusion}}" = "on" ]; then
@@ -23,7 +23,7 @@ steps:
                 fi
                 julia --project -e 'using Pkg; Pkg.resolve(); Pkg.instantiate()'
           - JuliaCI/julia-test#v1:
-              test_args: "--quickfail --jobs=8 --verbose"
+              test_args: "--jobs=8 --verbose"
           - JuliaCI/julia-coverage#v1:
               dirs:
                 - src
@@ -50,6 +50,7 @@ steps:
               - "1.10"
               - "1.11"
               - "1.12"
+              - "1.13"
             fusion:
               - "on"
               - "off"

.buildkite/pipeline.yml

@@ -10,6 +10,6 @@ steps:
     command: ".buildkite/upload_gpu_ci.sh"
     agents:
       queue: "cuda"
-    if: build.message !~ /\[skip tests\]/
+    if: build.message !~ /\[skip ci\]/
     if_changed: "{src/**,scripts/**,deps/build.jl,Project.toml,lib/CNPreferences/src/**,lib/cunumeric_jl_wrapper/**}"
     timeout_in_minutes: 5

.buildkite/run_developer_ci.sh

@@ -20,13 +20,20 @@ sh "$CMAKE_INSTALLER" --skip-license --prefix="$CMAKE_ROOT"
 export PATH="$CMAKE_ROOT/bin:$PATH"
 cmake --version
 
+# Exercise libcxxwrap cache validation separately from package tests, which run in
+# JLL jobs where no build toolchain is installed.
+julia --startup-file=no test/build_cxxwrap.jl
+
 # Clean slate so cached state doesn't leak across Julia versions.
 rm -f Manifest.toml test/Manifest.toml dev/Manifest.toml \
       LocalPreferences.toml test/LocalPreferences.toml
 
-# Dev mode rebuilds the wrapper .so each run, so drop stale overrides and the .ji that
-# bake @wrapmodule bindings (cuNumeric/Legate + wrapper JLLs) — a cached .ji would
-# mismatch the fresh .so and segfault. libcxxwrap override is kept.
+# Build directly in the plugin's persistent cache so artifacts, precompile, and libcxxwrap all
+# stay warm across runs. Developer builds are serialized across PRs, while each matrix job in a
+# build uses a separate (Julia, fusion) cache.
+# Dev mode rebuilds the wrapper .so each run, so drop stale overrides and the .ji that bake
+# @wrapmodule bindings (cuNumeric/Legate + wrapper JLLs) — a cached .ji would mismatch the fresh
+# .so and segfault. libcxxwrap's dev build is kept and reused.
 DEPOT="$(julia --startup-file=no -e 'print(DEPOT_PATH[1])')"
 rm -rf "$DEPOT"/packages/*/*/override \
        "$DEPOT"/compiled/v*/{cuNumeric,Legate,cunumeric_jl_wrapper_jll,legate_jl_wrapper_jll}
@@ -83,5 +90,6 @@ cp LocalPreferences.toml test/LocalPreferences.toml
 
 julia --color=yes --project=. -e '
     using Pkg
-    Pkg.test("cuNumeric"; test_args = ["--quickfail", "--jobs=8", "--verbose"])
+    Pkg.develop(PackageSpec(path = "lib/CNPreferences"))
+    Pkg.test("cuNumeric"; test_args = ["--jobs=8", "--verbose"])
 '

.buildkite/upload_gpu_ci.sh

@@ -4,41 +4,42 @@ set -euo pipefail
 
 readonly JLL_PIPELINE=".buildkite/jll.pipeline.yml"
 readonly DEVELOPER_PIPELINE=".buildkite/developer.pipeline.yml"
-readonly WRAPPER_PATH="lib/cunumeric_jl_wrapper"
-readonly WRAPPER_BASE_BRANCH="main"
 
 branch="${BUILDKITE_BRANCH:-}"
 base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-}"
-pull_request="${BUILDKITE_PULL_REQUEST:-false}"
 message="${BUILDKITE_MESSAGE:-}"
 
 run_jll=true
 run_developer=true
 
-# Keep both suites for main and PRs into main. For non-main PRs, select the
-# suite whose wrapper matches the code under test.
+if [[ "$message" =~ \[skip[[:space:]]ci\] ]]; then
+    echo "Skipping all GPU CI because the build message requests it."
+    exit 0
+fi
+
+if [[ "$message" =~ \[skip[[:space:]]jll\] ]]; then
+    echo "Skipping JLL GPU CI because the build message contains [skip jll]."
+    run_jll=false
+fi
+if [[ "$message" =~ \[skip[[:space:]]dev\] ]]; then
+    echo "Skipping developer GPU CI because the build message contains [skip dev]."
+    run_developer=false
+fi
+
+# Keep both suites for main and PRs into main. Otherwise use the JLL suite only
+# when the wrapper matches the released JLL source.
 if [[ "$branch" != "main" && "$base_branch" != "main" ]]; then
-    if [[ "$message" =~ \[skip[[:space:]]jll\] ]]; then
-        echo "Skipping JLL GPU CI because the build message contains [skip jll]."
-        run_jll=false
-    elif [[ "$pull_request" != "false" && -n "$base_branch" ]]; then
-        base_ref="refs/remotes/origin/$WRAPPER_BASE_BRANCH"
-        # The published wrapper JLL tracks main, so compare against main even
-        # when the pull request targets develop.
-        git fetch --no-tags origin "+refs/heads/${WRAPPER_BASE_BRANCH}:${base_ref}"
-
-        if git diff --quiet "${base_ref}...HEAD" -- "$WRAPPER_PATH"; then
-            echo "No wrapper changes detected against origin/$WRAPPER_BASE_BRANCH; using JLL GPU CI."
-            run_developer=false
+    if scripts/wrapper_changed.sh; then
+        echo "Wrapper matches the released JLL source; using JLL GPU CI."
+        run_developer=false
+    else
+        diff_status=$?
+        if ((diff_status == 1)); then
+            echo "Wrapper differs from the released JLL source; using developer GPU CI."
+            run_jll=false
         else
-            diff_status=$?
-            if ((diff_status == 1)); then
-                echo "Wrapper changes detected against origin/$WRAPPER_BASE_BRANCH; using developer GPU CI."
-                run_jll=false
-            else
-                echo "Could not determine whether the wrapper changed against origin/$WRAPPER_BASE_BRANCH." >&2
-                exit "$diff_status"
-            fi
+            echo "Could not determine whether the wrapper matches the released JLL source." >&2
+            exit "$diff_status"
         fi
     fi
 fi

.dockerignore

@@ -0,0 +1,47 @@
+# Local Julia environments and generated build configuration.
+**/*Manifest.toml
+**/*LocalPreferences.toml
+dev/Project.toml
+deps/deps.jl
+build_wrapper.sh
+compile_wrapper.sh
+
+# Local caches, logs, and editor configuration.
+**/__pycache__
+**/*.pyc
+**/*.log
+**/*.err
+**/*.prof
+**/node_modules
+.vscode
+.envrc
+.localenv
+docs/.vitepress
+docs/src/.vitepress/cache
+docs/src/.vitepress/dist
+
+benchmark/results
+benchmark/plots
+# Host build products must not overwrite artifacts built inside the image.
+**/build
+**/*.o
+**/*.obj
+**/*.so
+**/*.so.*
+**/*.dylib
+**/*.dll
+**/*.a
+**/*.lib
+**/*.exe
+deps/cupynumeric-*
+deps/lapacke_build
+libcupynumeric
+install-libcupynumeric
+install-cupynumeric
+libcxxwrap-julia
+lapacke
+
+# Keep root repository metadata intact, including files matching rules above.
+**/.git
+!.git
+!.git/**

.github/scripts/check_versions.py

@@ -2,9 +2,12 @@
 """Version consistency check for pull requests targeting main."""
 
 import argparse
+import json
 import re
 import subprocess
 import sys
+import urllib.parse
+import urllib.request
 from pathlib import Path
 
 REPO_ROOT = Path(subprocess.run(
@@ -18,6 +21,10 @@
 
 WRAPPER_VERSION_FILE   = "lib/cunumeric_jl_wrapper/VERSION"
 WRAPPER_JLL_COMPAT_KEY = "cunumeric_jl_wrapper_jll"
+WRAPPER_RELEASED_COMMIT_FILE = "lib/cunumeric_jl_wrapper/RELEASED_COMMIT"
+WRAPPER_JLL_REPO       = "JuliaBinaryWrappers/cunumeric_jl_wrapper_jll.jl"
+WRAPPER_JLL_TAG_PREFIX = "cunumeric_jl_wrapper-v"
+WRAPPER_SOURCE_REPO    = "https://github.com/JuliaLegate/cuNumeric.jl.git"
 WRAPPER_SRC_PREFIXES   = (
     "lib/cunumeric_jl_wrapper/src/",
     "lib/cunumeric_jl_wrapper/include/",
@@ -149,6 +156,67 @@ def check_wrapper_compat_sync(pr_toml: str, errors: list):
         print(f"\t\tOK")
 
 
+def http_get(url: str) -> str:
+    request = urllib.request.Request(url, headers={"User-Agent": "cuNumeric-version-check"})
+    with urllib.request.urlopen(request, timeout=30) as response:
+        return response.read().decode()
+
+
+def released_wrapper_commit(version: str) -> tuple:
+    """Return (tag, source revision) of the newest JLL build of `version`."""
+    refs = json.loads(http_get(
+        f"https://api.github.com/repos/{WRAPPER_JLL_REPO}/git/matching-refs/tags/"
+        f"{WRAPPER_JLL_TAG_PREFIX}{version}+"
+    ))
+    builds = {}
+    for ref in refs:
+        tag = ref["ref"].removeprefix("refs/tags/")
+        m = re.fullmatch(re.escape(WRAPPER_JLL_TAG_PREFIX + version) + r"\+(\d+)", tag)
+        if m:
+            builds[int(m.group(1))] = tag
+    if not builds:
+        return None, None
+    tag = builds[max(builds)]
+    readme = http_get(
+        f"https://raw.githubusercontent.com/{WRAPPER_JLL_REPO}/{urllib.parse.quote(tag)}/README.md"
+    )
+    m = re.search(re.escape(WRAPPER_SOURCE_REPO) + r" \(revision: `([0-9a-f]{40})`\)", readme)
+    return tag, m.group(1) if m else None
+
+
+def check_wrapper_released_commit(pr_toml: str, errors: list):
+    compat_ver = parse_compat_section(pr_toml).get(WRAPPER_JLL_COMPAT_KEY)
+    recorded = (REPO_ROOT / WRAPPER_RELEASED_COMMIT_FILE).read_text().strip()
+
+    print(f"\t[wrapper released commit]")
+    print(f"\t\t{WRAPPER_RELEASED_COMMIT_FILE} = {recorded}")
+    if compat_ver is None:
+        return  # reported by check_wrapper_compat_sync
+    try:
+        tag, released = released_wrapper_commit(compat_ver)
+    except Exception as e:
+        errors.append(
+            f"Could not look up the source revision of {WRAPPER_JLL_COMPAT_KEY} {compat_ver}: {e}"
+        )
+        return
+
+    if tag is None:
+        errors.append(
+            f"{WRAPPER_JLL_COMPAT_KEY} {compat_ver} has no release in {WRAPPER_JLL_REPO}.\n"
+            f"\tRelease the wrapper JLL before merging into main."
+        )
+    elif released is None:
+        errors.append(f"Could not find the source revision in the README of {WRAPPER_JLL_REPO} {tag}.")
+    elif released != recorded:
+        errors.append(
+            f"{WRAPPER_RELEASED_COMMIT_FILE} is stale.\n"
+            f"\t{tag} was built from {released}, but the file records {recorded}.\n"
+            f"\tSet {WRAPPER_RELEASED_COMMIT_FILE} to {released}."
+        )
+    else:
+        print(f"\t\tOK (matches {tag})")
+
+
 def check_subpkg_version(base_ref: str, pr_toml: str, changed: list, errors: list):
     src_changed = [f for f in changed if any(f.startswith(p) for p in SUBPKG_SRC_PREFIXES)]
     if not src_changed:
@@ -210,6 +278,7 @@ def main():
     check_package_version(base_ref, pr_toml, errors)
     check_wrapper_version(base_ref, changed, errors)
     check_wrapper_compat_sync(pr_toml, errors)
+    check_wrapper_released_commit(pr_toml, errors)
     check_subpkg_version(base_ref, pr_toml, changed, errors)
     print("─" * 60)