From 1c08f4f01d6a8f12bb36419f55844b669d48750c Mon Sep 17 00:00:00 2001 From: Greg Magolan Date: Tue, 15 Sep 2026 21:22:42 -0700 Subject: [PATCH] feat(bazelrc): config groups, deployment sections, and a flag catalog with opt-outs --- .aspect/bazelrc | 40 ++ .aspect/bootstrap.sh | 19 +- .aspect/config.axl | 29 +- .aspect/generated/.gitignore | 2 + .bazelrc | 5 +- .../src/builtins/aspect/MODULE.aspect | 3 +- .../builtins/aspect/bazel/build_events.axl | 14 +- .../aspect/bazel/build_events_test.axl | 20 +- .../builtins/aspect/bazel/build_metadata.axl | 13 + .../src/builtins/aspect/bazel/flags.axl | 26 +- .../src/builtins/aspect/bazel/invocation.axl | 2 +- .../src/builtins/aspect/bazel/trait.axl | 1 + .../src/builtins/aspect/bazelrc.axl | 485 ++++++++++++------ .../src/builtins/aspect/delivery.axl | 3 +- .../builtins/aspect/feature/bazel_flags.axl | 44 ++ .../src/builtins/aspect/feature/workflows.axl | 97 ++-- .../aspect/private/lib/aspect_flags.axl | 458 +++++++++++++++++ .../aspect/private/lib/aspect_flags_test.axl | 314 ++++++++++++ .../aspect/private/lib/bazelrc_file.axl | 149 ++++++ .../aspect/private/lib/bazelrc_file_test.axl | 149 ++++++ .../aspect/private/lib/bazelrc_test.axl | 210 -------- .../aspect/private/lib/deployment_flags.axl | 46 +- .../private/lib/deployment_flags_test.axl | 43 +- .../aspect/private/lib/deployment_rc.axl | 168 ++++++ .../aspect/private/lib/deployment_rc_test.axl | 183 +++++++ .../aspect/private/lib/environment.axl | 233 ++------- .../builtins/aspect/private/lib/rc_groups.axl | 178 +++++++ .../aspect/private/lib/rc_groups_test.axl | 184 +++++++ crates/axl-runtime/src/engine/aspect/auth.rs | 148 +++++- crates/axl-runtime/src/engine/bazel/build.rs | 8 +- crates/axl-runtime/src/engine/bazel/mod.rs | 8 +- crates/axl-runtime/src/engine/std/process.rs | 117 +++++ crates/bazelrc/src/lib.rs | 80 ++- 33 files changed, 2771 insertions(+), 708 deletions(-) create mode 100644 .aspect/bazelrc create mode 100644 .aspect/generated/.gitignore create mode 100644 crates/aspect-cli/src/builtins/aspect/feature/bazel_flags.axl create mode 100644 crates/aspect-cli/src/builtins/aspect/private/lib/aspect_flags.axl create mode 100644 crates/aspect-cli/src/builtins/aspect/private/lib/aspect_flags_test.axl create mode 100644 crates/aspect-cli/src/builtins/aspect/private/lib/bazelrc_file.axl create mode 100644 crates/aspect-cli/src/builtins/aspect/private/lib/bazelrc_file_test.axl delete mode 100644 crates/aspect-cli/src/builtins/aspect/private/lib/bazelrc_test.axl create mode 100644 crates/aspect-cli/src/builtins/aspect/private/lib/deployment_rc.axl create mode 100644 crates/aspect-cli/src/builtins/aspect/private/lib/deployment_rc_test.axl create mode 100644 crates/aspect-cli/src/builtins/aspect/private/lib/rc_groups.axl create mode 100644 crates/aspect-cli/src/builtins/aspect/private/lib/rc_groups_test.axl diff --git a/.aspect/bazelrc b/.aspect/bazelrc new file mode 100644 index 000000000..e9678cdb4 --- /dev/null +++ b/.aspect/bazelrc @@ -0,0 +1,40 @@ +# Generated by `aspect setup bazelrc` as a starting point: edit as your repository needs; rerun the command to restore it. +# Configures vanilla `bazel` calls to use Aspect services and recommended optimizations. +# Flags are grouped by what they presume; the groups that apply here are enabled at the +# top, the rest reached by chaining or by name. `bazel build --announce_rc` shows which group set what. + + +# Tuning that presumes no endpoint. +common:aspect-common --heap_dump_on_oom +common:aspect-common --experimental_repository_cache_hardlinks + +# The remote cache (ASPECT_WORKFLOWS_REMOTE_CACHE where set) and the tuning a cache justifies. +common:aspect-cache --remote_upload_local_results +common:aspect-cache --remote_accept_cached +common:aspect-cache --remote_timeout=3600 +common:aspect-cache --remote_retries=360 +common:aspect-cache --grpc_keepalive_timeout=30s + +# The BES backend (ASPECT_WORKFLOWS_BES_BACKEND where set), its enrichment, and the build-diagnostics upload the Aspect backend reads. +common:aspect-bes --grpc_keepalive_timeout=30s +common:aspect-bes --generate_json_trace_profile +common:aspect-bes --noslim_profile +common:aspect-bes --experimental_profile_include_target_label +common:aspect-bes --experimental_profile_include_primary_output +common:aspect-bes --experimental_profile_include_target_configuration +common:aspect-bes --legacy_important_outputs +common:aspect-bes --remote_build_event_upload=all + +# The compact execution log, uploaded with the build diagnostics for per-action data in the Web UI. Slow on a thin link: remove this group and the --config lines naming it to skip it. +common:aspect-exec-log --execution_log_compact_file=%workspace%/.aspect/generated/exec.log.zstd + +# Aspect Cloud: remote cache + BES; auth via ASPECT_API_TOKEN or `aspect auth login`. +common:aspect-cloud --config=aspect-common +common:aspect-cloud --config=aspect-cache +common:aspect-cloud --config=aspect-bes +common:aspect-cloud --config=aspect-exec-log +common:aspect-cloud --remote_cache=grpcs://cache.aspect.build +common:aspect-cloud --bes_backend=grpcs://bes.aspect.build +common:aspect-cloud --bes_results_url=https://app.aspect.build/i/ +common:aspect-cloud --credential_helper=cache.aspect.build=aspect +common:aspect-cloud --credential_helper=bes.aspect.build=aspect diff --git a/.aspect/bootstrap.sh b/.aspect/bootstrap.sh index 9334002c3..f2adde167 100755 --- a/.aspect/bootstrap.sh +++ b/.aspect/bootstrap.sh @@ -20,17 +20,6 @@ BAZEL_REMOTE_FLAGS="" [ -n "${ASPECT_WORKFLOWS_BES_RESULTS_URL:-}" ] && BAZEL_REMOTE_FLAGS="${BAZEL_REMOTE_FLAGS} --bes_results_url=${ASPECT_WORKFLOWS_BES_RESULTS_URL}" [ -n "${ASPECT_WORKFLOWS_REMOTE_CACHE:-}" ] && BAZEL_REMOTE_FLAGS="${BAZEL_REMOTE_FLAGS} --remote_cache=${ASPECT_WORKFLOWS_REMOTE_CACHE}" [ -n "${ASPECT_WORKFLOWS_REMOTE_BYTESTREAM_URI_PREFIX:-}" ] && BAZEL_REMOTE_FLAGS="${BAZEL_REMOTE_FLAGS} --remote_bytestream_uri_prefix=${ASPECT_WORKFLOWS_REMOTE_BYTESTREAM_URI_PREFIX}" -# x-identity authenticates the runner to the backend. Bazel scopes headers per -# channel: --remote_header covers the remote cache and executor gRPC channels -# but NOT the BES channel, which needs its own --bes_header. Set both so the -# cache/executor and the pre-build's BES stream all authenticate. Gate on the -# identity alone (mirrors get_bazelrc_flags in -# crates/aspect-cli/src/builtins/aspect/lib/environment.axl). -if [ -n "${ASPECT_WORKFLOWS_RUNNER_IDENTITY:-}" ]; then - BAZEL_REMOTE_FLAGS="${BAZEL_REMOTE_FLAGS} --remote_header=x-identity=${ASPECT_WORKFLOWS_RUNNER_IDENTITY}" - BAZEL_REMOTE_FLAGS="${BAZEL_REMOTE_FLAGS} --bes_header=x-identity=${ASPECT_WORKFLOWS_RUNNER_IDENTITY}" -fi - # --build_metadata flags for the pre-build invocation. Only set when we're # forwarding events to a BES backend (the Aspect Web UI or similar) — # otherwise the metadata has nowhere to surface. @@ -275,7 +264,7 @@ if [ -n "${ASPECT_WORKFLOWS_RUNNER:-}" ]; then fi # Derive workspace subdir from the checkout path. - # Mirrors the aspect_root_dir derivation in get_bazelrc_flags in environment.axl. + # Mirrors the aspect_root_dir derivation in runner_startup_flags in aspect_flags.axl. WORKSPACE_DIR="${BUILDKITE_BUILD_CHECKOUT_PATH:-${GITHUB_WORKSPACE:-${CIRCLE_WORKING_DIRECTORY:-${CI_PROJECT_DIR:-$(pwd)}}}}" SUBDIR=$(basename "${WORKSPACE_DIR}" | sed 's|[^a-zA-Z0-9._-]|_|g') @@ -309,11 +298,7 @@ export DISABLE_PLUGINS_FLAG export LOCK_VERSION_FLAG echo "Startup opts: ${BAZEL_STARTUP_OPTS}" -# Redact x-identity header values before echoing — the runner identity is an -# auth credential, and the CLI's own log redaction treats remote_header/ -# bes_header values as secrets (crates/axl-runtime/src/engine/bazel/stream/ -# redaction.rs). The exported BAZEL_BUILD_OPTS keeps the real value. -echo "Build opts: $(printf '%s' "${BAZEL_BUILD_OPTS}" | sed 's/x-identity=[^ ]*/x-identity=/g')" +echo "Build opts: ${BAZEL_BUILD_OPTS}" USER_BAZELRC="${HOME}/.bazelrc" if [ -f "${USER_BAZELRC}" ]; then diff --git a/.aspect/config.axl b/.aspect/config.axl index 57524c55d..f11546d7a 100644 --- a/.aspect/config.axl +++ b/.aspect/config.axl @@ -13,15 +13,17 @@ load("@aspect//private/feature/buildkite_annotations_test.axl", "bk_annotation_s load("@aspect//private/feature/github_status_comments_test.axl", "pr_comment_snapshot_tests") load("@aspect//private/helpers_test.axl", "helpers_facade_tests") load("@aspect//private/lib/aspect_endpoint_auth_test.axl", "aspect_endpoint_auth_tests") +load("@aspect//private/lib/aspect_flags_test.axl", "aspect_flags_tests") load("@aspect//private/lib/bazel_flags_test.axl", "bazel_flags_tests") load("@aspect//private/lib/bazel_results_test.axl", "bazel_results_unit_tests", "template_snapshot_tests") -load("@aspect//private/lib/bazelrc_test.axl", "bazelrc_tests") +load("@aspect//private/lib/bazelrc_file_test.axl", "bazelrc_file_tests") load("@aspect//private/lib/cache_selection_test.axl", "cache_selection_tests") load("@aspect//private/lib/ci_test.axl", "ci_tests") load("@aspect//private/lib/circleci_test.axl", "circleci_tests") load("@aspect//private/lib/delivery_results_test.axl", "delivery_results_unit_tests", "delivery_template_snapshot_tests") load("@aspect//private/lib/deliveryd_test.axl", "deliveryd_tests") load("@aspect//private/lib/deployment_flags_test.axl", "deployment_flags_tests") +load("@aspect//private/lib/deployment_rc_test.axl", "deployment_rc_tests") load("@aspect//private/lib/format_results_test.axl", "format_template_snapshot_tests") load("@aspect//private/lib/format_spawn_test.axl", "format_spawn_tests") load("@aspect//private/lib/gazelle_results_test.axl", "gazelle_template_snapshot_tests") @@ -37,6 +39,7 @@ load("@aspect//private/lib/lifecycle_test.axl", "lifecycle_tests") load("@aspect//private/lib/lint_comments_test.axl", "lint_comments_tests") load("@aspect//private/lib/lint_results_test.axl", "detect_lint_tool_tests", "lint_annotation_plan_tests", "lint_template_snapshot_tests", "linter_rows_tests") load("@aspect//private/lib/rate_limit_test.axl", "rate_limit_tests") +load("@aspect//private/lib/rc_groups_test.axl", "rc_groups_tests") load("@aspect//private/lib/remote_executor_test.axl", "remote_executor_tests") load("@aspect//private/lib/repro_commands_test.axl", "repro_commands_tests") load("@aspect//private/lib/runnable_test.axl", "runnable_tests") @@ -334,11 +337,25 @@ def config(ctx: ConfigContext): ctx.tasks.add(runnable_tests) ctx.tasks.add(cache_selection_tests) - # `aspect ci bazelrc` pure helpers — version-constraint gating - # (assumed-latest when the Bazel version is unknown), mixed plain/tuple - # flag resolution, and rc rendering (startup vs common sections). - # Run with: aspect dev test-bazelrc - ctx.tasks.add(bazelrc_tests) + # The Bazel flags Aspect sets: the catalog's endpoint keying, the runner and + # env compositions, and the opt-out with its protected set. + # Run with: aspect dev test-aspect-flags + ctx.tasks.add(aspect_flags_tests) + + # The rc `aspect setup bazelrc` writes as a document — rendering (startup vs + # common sections, `--config` groups) and the `try-import` placement. + # Run with: aspect dev test-bazelrc-file + ctx.tasks.add(bazelrc_file_tests) + + # The named `--config` groups the rc is built from: what each holds, which + # are enabled where, and the host group's naming. + # Run with: aspect dev test-rc-groups + ctx.tasks.add(rc_groups_tests) + + # The `--config=aspect-*` sections built from deployment rows: naming, + # credential helpers, per-endpoint flags, aliases, and collisions. + # Run with: aspect dev test-deployment-rc + ctx.tasks.add(deployment_rc_tests) # tools/bazel wrapper helpers — workspace-root detection, raw-URL # building, wrapper-version parsing, install classification, shell diff --git a/.aspect/generated/.gitignore b/.aspect/generated/.gitignore new file mode 100644 index 000000000..d6b7ef32c --- /dev/null +++ b/.aspect/generated/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/.bazelrc b/.bazelrc index e98f6b64e..d27a8e7d1 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,3 +1,6 @@ +# Aspect's rc for vanilla `bazel`, maintained by `aspect setup bazelrc`. +try-import %workspace%/.aspect/bazelrc + import %workspace%/bazel/defaults.bazelrc common --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 @@ -46,4 +49,4 @@ common --test_output=errors # repo, so it exercises `resolve_query_flags` (lib/bazel_flags.axl) dropping # build-only options from the query expansion. Mirrors the customer rc that # surfaced the bug. -build --show_result=20 +build --show_result=20 \ No newline at end of file diff --git a/crates/aspect-cli/src/builtins/aspect/MODULE.aspect b/crates/aspect-cli/src/builtins/aspect/MODULE.aspect index 57bb6f1bc..39b81ae4b 100644 --- a/crates/aspect-cli/src/builtins/aspect/MODULE.aspect +++ b/crates/aspect-cli/src/builtins/aspect/MODULE.aspect @@ -18,12 +18,13 @@ use_task("wrapper.axl", "install") use_task("wrapper.axl", "uninstall") use_task("init.axl", "init") use_task("mcp.axl", "mcp") -use_task("bazelrc.axl", "bazelrc") +use_task("bazelrc.axl", "bazelrc", "ci_bazelrc") use_task("warming.axl", "warming") use_task("runner_metadata.axl", "runner_metadata") use_task("runner_health_check.axl", "runner_health_check") use_feature("feature/artifacts.axl", "ArtifactUpload") +use_feature("feature/bazel_flags.axl", "BazelFlags") use_feature("feature/circleci_test_results.axl", "CircleCITestResults") use_feature("feature/deployment.axl", "Deployment") use_feature("feature/github_lint_comments.axl", "GithubLintComments") diff --git a/crates/aspect-cli/src/builtins/aspect/bazel/build_events.axl b/crates/aspect-cli/src/builtins/aspect/bazel/build_events.axl index 04627891a..8dd0f6de3 100644 --- a/crates/aspect-cli/src/builtins/aspect/bazel/build_events.axl +++ b/crates/aspect-cli/src/builtins/aspect/bazel/build_events.axl @@ -231,13 +231,17 @@ def advertised_results_url(sources: list[(str, str)], rc, bes_sinks: list, comma return results_url return "" -def bes_results_url_flag(sources: list[(str, str)], rc, bes_sinks: list, command: str = "build") -> list: - """`["--bes_results_url="]` when *Bazel* uploads the BES itself, else `[]`. +def bes_results_url_flag(sources: list[(str, str)], rc, command: str = "build") -> list: + """`["--bes_results_url="]` when *Bazel* uploads the BES itself to a backend + with an advertised viewer, else `[]`. Only Bazel consumes this flag, and only its BES module — the one `--bes_backend` activates — reads it, to print "Streaming build results to: …" and to key the link - on its own `invocation_id`. Passing it on the CLI-streamed path would inject a flag - nothing acts on, so that path announces the URL itself; see + on its own `invocation_id`. So the viewer is matched against Bazel's backend + alone, never the CLI's sinks: when Bazel uploads to a third party while the CLI + streams to Aspect, Aspect's viewer would name an invocation the backend indexes + under the sink's id, not Bazel's, and the third party would be handed a URL it + does not serve. The CLI-streamed path announces its own URL instead; see `announce_bes_results_url`. A `--bes_results_url` the user already set — command line, `.bazelrc`, or an @@ -248,7 +252,7 @@ def bes_results_url_flag(sources: list[(str, str)], rc, bes_sinks: list, command if not bazel_bes_backend(rc, command): return [] - results_url = advertised_results_url(sources, rc, bes_sinks, command) + results_url = advertised_results_url(sources, rc, [], command) return ["--bes_results_url=" + results_url] if results_url else [] def bes_upload_line(uri: str, sent: int, acked: int, failed: bool) -> str: diff --git a/crates/aspect-cli/src/builtins/aspect/bazel/build_events_test.axl b/crates/aspect-cli/src/builtins/aspect/bazel/build_events_test.axl index b7f95c095..4b6eb0d40 100644 --- a/crates/aspect-cli/src/builtins/aspect/bazel/build_events_test.axl +++ b/crates/aspect-cli/src/builtins/aspect/bazel/build_events_test.axl @@ -282,12 +282,12 @@ def _test_bes_flag_only_when_bazel_uploads(_): URL itself instead).""" _eq( "bazel-side upload → wired", - bes_results_url_flag([_GCP], _fake_rc({"--bes_backend": "grpcs://bes.gcp"}), []), + bes_results_url_flag([_GCP], _fake_rc({"--bes_backend": "grpcs://bes.gcp"})), ["--bes_results_url=https://app.gcp/i/"], ) _eq( "cli internal client → no flag for Bazel to read", - bes_results_url_flag([_GCP], _fake_rc({}), _sinks("grpcs://bes.gcp")), + bes_results_url_flag([_GCP], _fake_rc({})), [], ) @@ -296,10 +296,21 @@ def _test_bes_flag_needs_an_advertised_backend(_): it.""" _eq( "third-party bazel backend → not wired", - bes_results_url_flag([_GCP], _fake_rc({"--bes_backend": "grpcs://bes.elsewhere"}), []), + bes_results_url_flag([_GCP], _fake_rc({"--bes_backend": "grpcs://bes.elsewhere"})), + [], + ) + _eq("no forwarding at all → not wired", bes_results_url_flag([_GCP], _fake_rc({})), []) + + # Dual write: Bazel uploads to a third party while the CLI streams to the + # deployment. Only Bazel's own backend decides the flag — the deployment's + # viewer would name an invocation indexed under the sink's id, not Bazel's. + # (The CLI's sinks are not an input here; `advertised_results_url` takes them + # for the CLI's own announce line.) + _eq( + "dual write → Bazel's third party gets no Aspect viewer", + bes_results_url_flag([_GCP], _fake_rc({"--bes_backend": "grpcs://bes.elsewhere"})), [], ) - _eq("no forwarding at all → not wired", bes_results_url_flag([_GCP], _fake_rc({}), []), []) def _test_bes_flag_yields_to_user_flag(_): _eq( @@ -307,7 +318,6 @@ def _test_bes_flag_yields_to_user_flag(_): bes_results_url_flag( [_GCP], _fake_rc({"--bes_results_url": "https://mine.example", "--bes_backend": "grpcs://bes.gcp"}), - _sinks("grpcs://bes.gcp"), ), [], ) diff --git a/crates/aspect-cli/src/builtins/aspect/bazel/build_metadata.axl b/crates/aspect-cli/src/builtins/aspect/bazel/build_metadata.axl index e875edd01..42b4dbc5f 100644 --- a/crates/aspect-cli/src/builtins/aspect/bazel/build_metadata.axl +++ b/crates/aspect-cli/src/builtins/aspect/bazel/build_metadata.axl @@ -484,6 +484,19 @@ def version_gte(a, b): # legacy two-field form (NAME=friendly_kind, ID=UUID). _TASK_METADATA_NEW_SCHEMA_MIN_VERSION = "5.18.0" +def register_bes_metadata(ctx, bazel_trait) -> None: + """Give the invocation the `--build_metadata` a BES backend reads: the task + identity (`get_task_metadata_flags`, a `task_flags` hook since it needs the + TaskContext) and the commit and CI context (`get_build_metadata_flags`). + Idempotent, so every feature that wires a BES backend — Workflows for the + runner's, Deployment for `--remote` — can call it and the flags appear once. + Without a backend the metadata has nowhere to surface, so nothing else + registers it.""" + if get_task_metadata_flags in bazel_trait.task_flags: + return + bazel_trait.task_flags.append(get_task_metadata_flags) + bazel_trait.extra_flags.extend(get_build_metadata_flags(ctx.std)) + def get_task_metadata_flags(ctx): """Generate --build_metadata flags for the current task identity. diff --git a/crates/aspect-cli/src/builtins/aspect/bazel/flags.axl b/crates/aspect-cli/src/builtins/aspect/bazel/flags.axl index cc182c73a..e7420714e 100644 --- a/crates/aspect-cli/src/builtins/aspect/bazel/flags.axl +++ b/crates/aspect-cli/src/builtins/aspect/bazel/flags.axl @@ -206,23 +206,30 @@ def _disclaim_passthrough(ctx) -> None: for bucket in _PASSTHROUGH_ARGS: _claim_bucket(ctx, bucket) +def _filter_injected(ctx, flags: list) -> list: + """`flags` through every `trait.injected_flag_filters` hook: the user's + opt-outs, applied only to what features inject.""" + for flag_filter in ctx.traits[BazelTrait].injected_flag_filters: + flags = flag_filter(flags) + return flags + def _resolve_startup_flags(ctx) -> list: """`ctx.args.bazel_startup_flags + pre-command passthrough + - trait.extra_startup_flags`, then the `trait.startup_flags(flags)` transform - if set. Requires a `bazel_startup_flags` CLI arg.""" + trait.extra_startup_flags` (filtered), then the `trait.startup_flags(flags)` + transform if set. Requires a `bazel_startup_flags` CLI arg.""" bazel_trait = ctx.traits[BazelTrait] _require_arg(ctx, "bazel_startup_flags") flags = list(ctx.args.bazel_startup_flags) flags.extend(_claim_bucket(ctx, _PASSTHROUGH_STARTUP_FLAGS_ARG)) - flags.extend(bazel_trait.extra_startup_flags) + flags.extend(_filter_injected(ctx, list(bazel_trait.extra_startup_flags))) if bazel_trait.startup_flags: flags = bazel_trait.startup_flags(flags) return flags def _resolve_flags(ctx) -> list: """`ctx.args.bazel_flags + post-command passthrough + trait.extra_flags + - trait.task_flags(ctx) hooks`, then the `trait.flags(flags)` transform if set. - Requires a `bazel_flags` CLI arg. + trait.task_flags(ctx) hooks` (the injected part filtered), then the + `trait.flags(flags)` transform if set. Requires a `bazel_flags` CLI arg. Passthrough flags follow the explicit `--bazel-flag` ones: both are user-typed, and on a collision Bazel takes the last, so the direct spelling @@ -231,9 +238,10 @@ def _resolve_flags(ctx) -> list: _require_arg(ctx, "bazel_flags") flags = list(ctx.args.bazel_flags) flags.extend(_claim_bucket(ctx, _PASSTHROUGH_FLAGS_ARG)) - flags.extend(bazel_trait.extra_flags) + injected = list(bazel_trait.extra_flags) for hook in bazel_trait.task_flags: - flags.extend(hook(ctx)) + injected.extend(hook(ctx)) + flags.extend(_filter_injected(ctx, injected)) if bazel_trait.flags: flags = bazel_trait.flags(flags) return flags @@ -254,7 +262,7 @@ def _setup_command(ctx, command: str, base_flags: list = []): active run command every Bazel call uses. """ startup_flags = _resolve_startup_flags(ctx) - flags = list(ctx.traits[BazelTrait].base_flags) + list(base_flags) + _resolve_flags(ctx) + flags = _filter_injected(ctx, list(ctx.traits[BazelTrait].base_flags)) + list(base_flags) + _resolve_flags(ctx) rc = ctx.bazel.parse_rc( startup_flags = startup_flags, flags = flags, @@ -280,7 +288,7 @@ def _sibling_rc(ctx, startup_transform, command: str = "build", base_flags: list the disclosure — pass the returned rc as a per-call `rc=` on `ctx.bazel.build`. """ startup_flags = startup_transform(_resolve_startup_flags(ctx)) - flags = list(ctx.traits[BazelTrait].base_flags) + list(base_flags) + _resolve_flags(ctx) + flags = _filter_injected(ctx, list(ctx.traits[BazelTrait].base_flags)) + list(base_flags) + _resolve_flags(ctx) return ctx.bazel.parse_rc( startup_flags = startup_flags, flags = flags, diff --git a/crates/aspect-cli/src/builtins/aspect/bazel/invocation.axl b/crates/aspect-cli/src/builtins/aspect/bazel/invocation.axl index 7f5c73db9..65241273c 100644 --- a/crates/aspect-cli/src/builtins/aspect/bazel/invocation.axl +++ b/crates/aspect-cli/src/builtins/aspect/bazel/invocation.axl @@ -136,7 +136,7 @@ def _resolve(ctx, command: str) -> _Invocation: # id), so the flag rides only that path — the CLI-streamed path announces # the URL itself. A value the user already set is never overridden. results_url = advertised_results_url(trait.bes_results_sources, rc, sinks, command) - rc_flags.extend(bes_results_url_flag(trait.bes_results_sources, rc, sinks, command)) + rc_flags.extend(bes_results_url_flag(trait.bes_results_sources, rc, command)) requested = ctx.args.bazel_retry_attempts if hasattr(ctx.args, "bazel_retry_attempts") else DEFAULT_BAZEL_RETRY_ATTEMPTS diff --git a/crates/aspect-cli/src/builtins/aspect/bazel/trait.axl b/crates/aspect-cli/src/builtins/aspect/bazel/trait.axl index a17bb1149..68e8ddf8a 100644 --- a/crates/aspect-cli/src/builtins/aspect/bazel/trait.axl +++ b/crates/aspect-cli/src/builtins/aspect/bazel/trait.axl @@ -71,6 +71,7 @@ BazelTrait = trait( flags = attr(typing.Callable[[list[str]], list[str]] | None, default = None, description = "Transform function called with the full flag list; return value replaces the list"), startup_flags = attr(typing.Callable[[list[str]], list[str]] | None, default = None, description = "Transform function called with the full startup flag list; return value replaces the list"), base_flags = attr(list[str], default = [], description = "Default Bazel flags prepended before the user's --bazel-flag overrides (user overrides win, unlike extra_flags which append after)"), + injected_flag_filters = attr(list[typing.Callable[[list], list]], default = [], description = "The user's flag policy for a task's Bazel call: filters applied to every flag features inject (base_flags, extra_flags, extra_startup_flags, task_flags results) and never to the user's own flags. The BazelFlags feature registers --bazel-flags:omit here; Workflows and Deployment inject, the spawn path filters"), bes_backends = attr(list[str], default = [], description = "Extra CLI-streamed BES backend URIs to stream build events to"), bes_results_sources = attr(list[(str, str)], default = [], description = "(BES backend, results viewer URL) pairs the invocation's build-result link is drawn from; matched against everywhere the build reports BES, first hit wins"), diff --git a/crates/aspect-cli/src/builtins/aspect/bazelrc.axl b/crates/aspect-cli/src/builtins/aspect/bazelrc.axl index a0592be9a..7f3485fe5 100644 --- a/crates/aspect-cli/src/builtins/aspect/bazelrc.axl +++ b/crates/aspect-cli/src/builtins/aspect/bazelrc.axl @@ -1,191 +1,344 @@ -"""`aspect ci bazelrc` — generate a Bazel rc that configures vanilla `bazel` calls to -use an Aspect Workflows CI runner's services and optimizations. - -On a Workflows runner, `aspect ` injects the runner's remote cache, -repository cache, and per-runner output paths into Bazel itself (see -`feature/workflows.axl` and `get_bazelrc_flags` in `lib/environment.axl`). A bare -`bazel build` started from the same job gets none of that. This command writes -those same flags to a Bazel rc file (`~/.bazelrc` by default — the first user rc -Bazel loads, picked up with no `--bazelrc` plumbing) so vanilla `bazel` calls in the -job pick up the identical configuration. - -It is the in-CLI successor to `rosetta bazelrc`: the CI integrations (the -setup-aspect GitHub Action, Buildkite plugin, CircleCI orb, GitLab component) -call this instead of `rosetta` once `rosetta` is gone from the runner image. - -What it writes depends on where it runs: - - - On a Workflows runner: the full set — generic flags, the remote-cache - endpoint (from env), and the runner-specific flags (repository cache, - per-runner output paths, identity header, the Workflows-cache compression - gates). - - On other CI, or with an explicit `--output`: only the generic flags (plus - the remote-cache endpoint when its env vars are set) — nothing tied to a - runner's mounts or identity. Useful when an Aspect remote cache / BES is - configured via env vars off a Workflows runner. - - Off a runner and off CI with no `--output`: writes nothing, so it doesn't - clobber the developer's `~/.bazelrc`. - -On Bazel >= 6.3.0 the build flags are emitted under a single `common` section -(applies to every command); older Bazel has no `common` pseudo-command, so -they're spelled out under `build` (inherited by test/run/coverage/cquery/aquery/ -fetch/…) and `query` (the one action-relevant command that doesn't inherit -`build`). Version-gated flags are resolved against the detected Bazel version -(assumed-latest when undetectable), matching how the workflows feature gates the -same flags at invocation time. - -Only file output is supported — there is no stdout mode. +"""`aspect setup bazelrc` — write a Bazel rc that configures vanilla `bazel` calls to +use Aspect's remote services and recommended optimizations. `aspect ci bazelrc`, +the name the CI integrations in the wild call, is the same task under its old +group. + +`aspect ` injects its Bazel flags itself (`feature/workflows.axl`, +`feature/deployment.axl`, and `lib/aspect_flags.axl`, the table of every flag +Aspect sets and why). A bare `bazel build` started from the same job gets none +of that. This command writes the equivalent where vanilla `bazel` will find it. +It is the in-CLI successor to `rosetta bazelrc`: the CI integrations +(the setup-aspect GitHub Action, Buildkite plugin, CircleCI orb, GitLab +component) call it, as `aspect ci bazelrc`, once `rosetta` is gone from the +runner image. + +Where the rc goes: + + - On an Aspect Workflows runner, `~/.bazelrc` is written whole. The runner is + ours, nothing else configures Bazel on it, and rewriting the file every job + is what keeps it in step with the runner's environment. Tasks run with + `--nohome_rc` and stream BES from the CLI, so nothing in this file reaches + them. + - Anywhere else, the rc is the checkout's, meant to be committed: + `/.aspect/bazelrc`, imported by the workspace `.bazelrc` through + `try-import %workspace%/.aspect/bazelrc`, which the command adds once at the + top of the file under a comment, a blank line apart from its other content — + at the top so the repository's own lines come after and win where they + disagree, which is how it overrides a recommendation. Both go into the + repository, so everyone who builds it gets the same rc. The file is a + starting point the repository then owns: edit it freely; a rerun of the + command restores the generated version. Nothing in + that file may be this machine's: the credential helper is the tool's name, + the CI host's group is left out, and the compact execution log is placed + through `%workspace%`, the one variable Bazel expands in rc values, at + `%workspace%/.aspect/generated/exec.log.zstd`. `.aspect/generated/` is the + home of what tools write into a checkout at build time; the command creates + it with a `.gitignore` of `*` and `!.gitignore`, committed so the directory + exists in every checkout — Bazel does not create the log's parent. The + workspace root is `ctx.std.env.bazel_root_dir()`, found by the boundary + files Bazel finds `%workspace%` by, and the command refuses where none is. + +What it holds: every flag sits in a named `--config` group from +`lib/rc_groups.axl`, and the rc has two shapes. Where the environment names the +deployment — the runner marker, or any `ASPECT_WORKFLOWS_*` endpoint — the +groups that apply are enabled at the top with `common --config=` lines, +`--config=aspect-exec` opts into the runner's executor, and no deployment +sections are written: a `--config=aspect-cloud` in a runner's rc would route a +vanilla build off the runner's own services. Where the environment is silent, +nothing is enabled and the configured deployments (`aspect auth status`) get +the opt-in sections of `lib/deployment_rc.axl`, each chaining the common and +CI-host groups and the tuning its endpoints justify, so the file is inert until +a section is named and safe to import into a developer's rc. With no deployment +on record there would be nothing to point at, so the command stops and says to +log in (`aspect auth login` records Aspect Cloud's endpoints; `aspect auth +configure` adds a single-tenant deployment) before writing anything. +`--announce_rc` +names the group every flag came from either way. Startup options have no +`--config`, so the runner's output paths are plain `startup` lines. The +credential helper the sections name is `aspect`, which Bazel resolves on PATH +like a shell would; a checkout's rc is shared by everyone who builds it, so it +names the tool, not a path, and the command warns when `aspect` is not on PATH +where it runs. A user who authenticates with a long-lived token instead drops +the helper in their own `~/.bazelrc` and sets `--remote_header`. + +Bazel reads the home rc after the workspace rc, so on a runner a repository +whose own rc names a different `--bes_backend` for vanilla calls keeps it by +opting out: `--omit-bazel-flag=--bes_backend`. That knob drops any tuning flag +inside whichever group holds it (a group emptied that way disappears along with +the lines that named it); the endpoints, credentials, and runner paths in +`PROTECTED_FLAGS` are refused. Tasks have their own knob, `--bazel-flags:omit` +(`feature/bazel_flags.axl`), since vanilla `bazel` and tasks may want different +omissions. + +How the rc renders and how the import line is placed live in +`lib/bazelrc_file.axl`. Version-gated flags are resolved against the detected +Bazel version (assumed-latest when undetectable) before writing. Only file +output is supported — there is no stdout mode. """ -load("@aspect//bazel/build_metadata.axl", "version_gte") load( - "./private/lib/environment.axl", - "detect_ci", - "error", - "get_bazelrc_flags", - "get_generic_bazelrc_flags", - "get_workflows_environment", - "info", - "is_aspect_workflows_runner", + "./private/lib/aspect_flags.axl", + "omit_flags", + "protected_omissions", + "runner_exec_log_path", + "runner_startup_flags", + "unscoped", +) +load( + "./private/lib/bazelrc_file.axl", + "GENERATED_HEADER", + "ImportAction", + "RUNNER_HEADER", + "RcConfig", + "command_sections", + "ensure_import", + "import_line", + "is_config_chain", + "prune_configs", + "render_bazelrc", + "startup_flags_for_rc", ) +load("./private/lib/deployment_rc.axl", "config_collisions", "deployment_configs") +load("./private/lib/environment.axl", "bazel_misdetects_color", "env_names_deployment", "get_workflows_environment", "info", "is_aspect_workflows_runner", "warn") +load("./private/lib/rc_groups.axl", "bes_chains", "rc_groups", "section_chains") -# Bazel gained the `common` pseudo-command (flags applied to every command) in -# 6.3.0. On older Bazel a `common` line is an unrecognized-command error, so we -# spell the flags out per command instead. -_COMMON_PSEUDO_COMMAND_MIN_VERSION = "6.3.0" - -# The command sections used on Bazel < 6.3.0 (no `common`). `build` is inherited -# by the commands that matter here — test, run, coverage, cquery, aquery, fetch, -# clean, mobile-install, … — so listing it covers them all. `query` is the one -# action-relevant command that does NOT inherit `build`, so it's named -# explicitly. Every flag get_bazelrc_flags emits is accepted by `query` on Bazel -# < 6.3.0 (the only range this branch runs in), so emitting them all under both -# is safe — the flags that later Bazel rejects on `query` are gated to those -# newer versions, where we use `common` instead and never reach this path. -_PRE_COMMON_COMMANDS = ["build", "query"] - -def _flag_str(flag) -> str: - """A flag is either a plain string or, for a surviving version-gated option, - a `(flag, version_condition)` tuple from `rc.expand` — take the flag text.""" - return flag[0] if type(flag) == "tuple" else flag - -def startup_flags_for_rc(startup_flags: list) -> list: - """Drop the rc-suppression startup flags, which are illegal inside an rc file. - - `get_bazelrc_flags` includes `--nohome_rc` and `--nosystem_rc` because the - workflows feature passes these as *command-line* startup flags to - `aspect `. Here we write the flags into the home rc itself - (~/.bazelrc), and Bazel rejects any `--no*_rc` option that appears inside a - bazelrc with a fatal `Can't specify --nosystem_rc in the .bazelrc file` - (likewise `--nohome_rc`, which would also be self-defeating — Bazel would be - told to ignore the very file it's reading). These flags only suppress *other* - rc files (`/etc/bazel.bazelrc`, `~/.bazelrc`); that's a command-line concern - and there's nothing the generated rc can do about it, so we drop them here. - - Public (not `_`-prefixed) so `bazelrc_test.axl` can load it.""" - _RC_SUPPRESSION_FLAGS = ["--nohome_rc", "--nosystem_rc", "--noworkspace_rc"] - return [f for f in startup_flags if f not in _RC_SUPPRESSION_FLAGS] - -def command_sections(bazel_version: str | None) -> list: - """The rc command sections the build flags go under. `common` (one section, - applies to every command) on Bazel >= 6.3.0 or when the version is unknown - (assume latest); the explicit `build`/`query` sections on older Bazel that - has no `common` pseudo-command. - - Public (not `_`-prefixed) so `bazelrc_test.axl` can load it.""" - if bazel_version == None or version_gte(bazel_version, _COMMON_PSEUDO_COMMAND_MIN_VERSION): - return ["common"] - return _PRE_COMMON_COMMANDS - -def render_bazelrc(startup_flags: list, build_flags: list, command_sections: list) -> str: - """Render the rc text: a `startup` line per startup flag, then every build - flag under each section in `command_sections` (`["common"]` on modern Bazel, - `["build", "query"]` on Bazel < 6.3.0). Build flags may be `(flag, condition)` - tuples (the shape `rc.expand` yields for version-gated options); only the flag - text is written.""" - lines = [ - "# Generated by `aspect ci bazelrc` — do not edit.", - "# Configures vanilla `bazel` calls to use Aspect Workflows services and optimizations.", - "", - ] - for flag in startup_flags: - lines.append("startup " + _flag_str(flag)) - for flag in build_flags: - for section in command_sections: - lines.append(section + " " + _flag_str(flag)) - return "\n".join(lines) + "\n" - -def _default_output(ctx: TaskContext) -> str | None: - """Default rc destination: `~/.bazelrc`. None (with an error) when the home - directory can't be resolved and no `--output` was given.""" +# A runner's rc, written whole. +_HOME_RC = ".bazelrc" + +# Everywhere else: the checkout's committed rc, the workspace rc that imports +# it through Bazel's `%workspace%`, and the committed self-ignoring directory +# build-time outputs land in. +_GENERATED_RC = ".aspect/bazelrc" +_WORKSPACE_RC = ".bazelrc" +_GENERATED_DIR = ".aspect/generated" +_EXEC_LOG_NAME = "exec.log.zstd" +_WORKSPACE_EXEC_LOG = "%workspace%/" + _GENERATED_DIR + "/" + _EXEC_LOG_NAME +_BAZEL_BOUNDARY_FILES = ["MODULE.bazel", "REPO.bazel", "WORKSPACE.bazel", "WORKSPACE"] + +_HELPER_NAME = "aspect" + +_IMPORT_VERBS = { + ImportAction("created"): "created %s with it", + ImportAction("unchanged"): "%s already imports it", + ImportAction("prepended"): "added the import at the top of %s", +} + +def _home_rc(ctx: TaskContext) -> str: + """`~/.bazelrc`; ends the task when the home directory can't be resolved.""" home = ctx.std.env.home_dir() if home == None: - error(ctx.std, "Could not resolve your home directory to default the output path. Pass --output= explicitly.") - return None - return home.rstrip("/") + "/.bazelrc" + ctx.std.process.exit(1, "Could not resolve your home directory for ~/.bazelrc. Pass --output= explicitly.") + return home.rstrip("/") + "/" + _HOME_RC -def _bazelrc_impl(ctx: TaskContext) -> int: - environment = get_workflows_environment(ctx.std) - on_runner = is_aspect_workflows_runner(ctx.std.env) - explicit_output = bool(ctx.args.output) +def _workspace(ctx: TaskContext) -> str: + """Bazel's workspace root — the directory it substitutes for `%workspace%` + in an rc `import`, found from the working directory the way Bazel finds it, + by the nearest boundary file. Ends the task where there is none: with no + workspace there is no `.bazelrc` for vanilla `bazel` to read.""" + root = ctx.std.env.bazel_root_dir().rstrip("/") + if not any([ctx.std.fs.exists(root + "/" + marker) for marker in _BAZEL_BOUNDARY_FILES]): + ctx.std.process.exit(1, "No Bazel workspace above %s (no MODULE.bazel, REPO.bazel, or WORKSPACE), so there is no .bazelrc for vanilla bazel to read. Run this from inside the workspace, or pass --output= and --import-into=." % ctx.std.env.current_dir()) + return root + +def _dirname(path: str) -> str: + return path.rsplit("/", 1)[0] if "/" in path else "." + +def _generated_dir(ctx: TaskContext, workspace: str) -> str: + """Create `/.aspect/generated/` with a `.gitignore` that ignores + everything but itself. Committed, it makes the directory exist in every + checkout — Bazel will not create the execution log's parent — while nothing + written there is ever tracked. Returns the `.gitignore` path.""" + dir = workspace + "/" + _GENERATED_DIR + ctx.std.fs.create_dir_all(dir) + ctx.std.fs.write(dir + "/.gitignore", "*\n!.gitignore\n") + return dir + "/.gitignore" + +def _write(ctx: TaskContext, path: str, content: str) -> None: + """Write `content` to `path`, creating parent directories.""" + parent = path.rstrip("/").rsplit("/", 1) + if len(parent) == 2 and parent[0] and not ctx.std.fs.exists(parent[0]): + ctx.std.fs.create_dir_all(parent[0]) + ctx.std.fs.write(path, content) - # Off a runner and off CI, don't write ~/.bazelrc by default — it's the - # developer's own. Require an explicit --output there. On CI/runner, expected. - if not on_runner and detect_ci(ctx.std.env) == None and not explicit_output: - error(ctx.std, "Not on an Aspect Workflows runner or other CI. Not writing ~/.bazelrc by default off CI; pass --output= to write somewhere explicit.") - return 1 +def _resolve(ctx: TaskContext, flags: list, bazel_version: str | None) -> list: + """Resolve the version-gated `(flag, constraint)` tuples in `flags` against the + detected Bazel version via the bazelrc runtime: a blank RunCommand whose + `common` bucket holds every flag, expanded assumed-latest when the version is + unknown. Config chain entries are kept verbatim ahead of the rest — the blank + RunCommand has no section for them to expand into. Command scopes are + dropped: the rc writes every flag under `common`, which Bazel applies only + where the option is accepted.""" + chain = [f for f in flags if is_config_chain(f)] + rest = [unscoped(f) for f in flags if not is_config_chain(f)] + rc = ctx.bazel.new_rc(flags = rest, version = bazel_version) + return chain + rc.expand(command = "common") +def _groups(ctx: TaskContext, environment, on_runner: bool, env_deployment: bool, exec_log_path: str) -> (list, list): + """`rc_groups` for this host: every group defined, and the names enabled.""" + return rc_groups(environment, on_runner, bazel_misdetects_color(ctx.std.env), exec_log_path, env_deployment) + +def _credential_helper(ctx: TaskContext) -> str: + """What the sections hand Bazel as the credential helper: `aspect`, which + Bazel resolves on PATH the way a shell does. A checkout's rc is read by + everyone who builds it, so it names the tool, never this machine's path to + it; where `aspect` is missing from PATH here, say so, since a vanilla build + on this machine would fail at the helper.""" + if ctx.std.process.which(_HELPER_NAME) == None: + warn(ctx.std, "`aspect` is not on PATH here. The rc names it as Bazel's credential helper, so put it on PATH before a vanilla `bazel` reads the rc (the CI integrations install it).") + return _HELPER_NAME + +def _sections(ctx: TaskContext, groups: list) -> list: + """The `--config` sections for this machine's configured deployments, each + chaining `section_chains(groups)` first and, with a BES, `bes_chains(groups)`. + Reads `deployments()`, not `list()`: nothing here needs login status.""" + return deployment_configs(ctx.aspect.auth.deployments(), _credential_helper(ctx), section_chains(groups), bes_chains(groups)) + +def _bazelrc_impl(ctx: TaskContext) -> int | TaskConclusion: + omitted = ctx.args.omit_bazel_flags + blocked = protected_omissions(omitted) + if blocked: + return TaskConclusion(exit_code = 1, message = "--omit-bazel-flag cannot drop %s: endpoints, credentials, and the runner's paths are what make the build Aspect's." % ", ".join(blocked)) + + environment = get_workflows_environment(ctx.std) + on_runner = is_aspect_workflows_runner(ctx.std.env) + startup_flags = [] if on_runner: - # Full set: generic CI flags + the runner-specific cache/identity/output - # flags. Gate on the runner metadata being present. if environment.runner == None: - error(ctx.std, "Running on an Aspect Workflows runner but the runner metadata env vars are missing, so the cache flags cannot be generated. This usually means an older runner image; upgrade the runner.") - return 1 - (startup_flags, build_flags) = get_bazelrc_flags( - environment = environment, - aspect_root_dir = ctx.std.env.aspect_root_dir(), - ) - startup_flags = startup_flags_for_rc(startup_flags) + return TaskConclusion(exit_code = 1, message = "Running on an Aspect Workflows runner but the runner metadata env vars are missing, so the cache flags cannot be generated. This usually means an older runner image; upgrade the runner.") + startup_flags = startup_flags_for_rc(runner_startup_flags(environment, ctx.std.env.aspect_root_dir())) + workspace = "" + exec_log_path = runner_exec_log_path(environment) else: - # Off a runner (CI, or explicit --output): only the generic flags that - # don't depend on the runner's mounts, identity, or the Workflows cache. - # The remote-cache endpoint flags are still picked up from env vars when - # set. No runner-specific startup flags. - build_flags = get_generic_bazelrc_flags(environment) - startup_flags = [] + # The workspace is needed only for the default paths. The committed rc + # places the log through `%workspace%`; an explicit --output, which may + # live anywhere, keeps it beside itself. + workspace = "" if ctx.args.output and ctx.args.import_into else _workspace(ctx) + exec_log_path = _dirname(ctx.args.output) + "/" + _EXEC_LOG_NAME if ctx.args.output else _WORKSPACE_EXEC_LOG - bazel_version = ctx.bazel.version() + env_deployment = env_names_deployment(ctx.std.env) + (groups, enabled) = _groups(ctx, environment, on_runner, env_deployment, exec_log_path) + sections = [] if env_deployment else _sections(ctx, groups) + if not env_deployment and not sections: + return TaskConclusion(exit_code = 1, message = "No Aspect deployment is configured here, so there is nothing for the rc to point `bazel` at. Log in first — `aspect auth login` records Aspect Cloud's endpoints — and to target a single-tenant Aspect Workflows deployment as well, run `aspect auth configure ` beforehand. Then rerun `aspect setup bazelrc`.") + configs = groups + sections + collisions = config_collisions(configs) + if collisions: + return TaskConclusion(exit_code = 1, message = "\n".join(collisions)) - # Resolve the version-gated `(flag, constraint)` tuples (the runner set has - # them) via the bazelrc runtime rather than by hand: a blank RunCommand whose - # `common` bucket holds every flag, expanded against the detected version - # (assumed-latest when unknown). The rendered section is chosen separately. - rc = ctx.bazel.new_rc(flags = build_flags, version = bazel_version) - build_flags = rc.expand(command = "common") + bazel_version = ctx.bazel.version() + startup_flags = omit_flags(startup_flags, omitted) + configs = [ + RcConfig(name = c.name, source = c.source, comment = c.comment, flags = _resolve(ctx, omit_flags(c.flags, omitted), bazel_version)) + for c in configs + ] + (configs, enabled) = prune_configs(configs, enabled) - output = ctx.args.output if explicit_output else _default_output(ctx) - if output == None: - return 1 + # A runner's home rc is written whole; elsewhere the rc is a generated file + # of the checkout that its workspace rc imports. + written = [] + if ctx.args.output: + output = ctx.args.output + elif on_runner: + output = _home_rc(ctx) + else: + written.append(_generated_dir(ctx, workspace)) + output = workspace + "/" + _GENERATED_RC + _write(ctx, output, render_bazelrc(startup_flags, enabled, command_sections(bazel_version), configs, header = RUNNER_HEADER if on_runner else GENERATED_HEADER)) + written.insert(0, output) - content = render_bazelrc(startup_flags, build_flags, command_sections(bazel_version)) + importer = ctx.args.import_into or (output if on_runner else workspace + "/" + _WORKSPACE_RC) + import_result = None + if importer != output: + # The default location is imported through `%workspace%`, so the + # workspace rc stays checkout-independent; an explicit --output is + # imported as given. + line = import_line(ctx.args.output or "%workspace%/" + _GENERATED_RC) + existing = ctx.std.fs.read_to_string(importer) if ctx.std.fs.exists(importer) else None + import_result = ensure_import(existing, line) + if import_result.action != ImportAction("unchanged"): + _write(ctx, importer, import_result.content) - parent = output.rstrip("/").rsplit("/", 1) - if len(parent) == 2 and parent[0] and not ctx.std.fs.exists(parent[0]): - ctx.std.fs.create_dir_all(parent[0]) + shown = lambda path: _relative(path, workspace) + if env_deployment: + optional = [c.name for c in configs if c.name not in enabled] + summary = "Wrote %s: enabled --config groups %s; opt-in %s (the ASPECT_WORKFLOWS_* environment names this machine's deployment, so no deployment sections)" % (shown(output), ", ".join(enabled), ", ".join(optional) or "none") + if import_result != None: + summary += "; " + _IMPORT_VERBS[import_result.action] % shown(importer) + info(ctx.std, summary + ".") + return 0 - ctx.std.fs.write(output, content) - info(ctx.std, "Wrote Aspect Workflows bazelrc to %s (%d build flags, %d startup flags)." % (output, len(build_flags), len(startup_flags))) + group_names = [g.name for g in groups] + sections = [c for c in configs if c.name not in group_names] + info(ctx.std, "Generating %s: Bazel flags that point plain `bazel` calls at Aspect Cloud and, when configured, Aspect single-tenant deployments." % shown(output)) + for line in _report(written, import_result, importer, ctx.args.output or "%workspace%/" + _GENERATED_RC, shown, sections): + print(line) return 0 +def _relative(path: str, workspace: str) -> str: + """`path` relative to `workspace` when under it, for the summary; the + absolute path otherwise.""" + prefix = workspace + "/" + return path[len(prefix):] if workspace and path.startswith(prefix) else path + +def _report(written: list, import_result, importer: str, imported: str, shown, sections: list) -> list: + """What the command did and how to use the result, printed after the lead + line: the files written and whether the import was added or already + there, then the `--config` names the rc defines and what each wires, then + that Aspect's own tasks reach the default deployment with `--remote` and + need none of it, last so it is what a developer sees.""" + lines = [""] + for path in written: + lines.append(" wrote " + shown(path)) + if import_result != None: + if import_result.action == ImportAction("unchanged"): + lines.append(" kept %s (already imports %s)" % (shown(importer), imported)) + else: + lines.append(" added try-import %s at the top of %s" % (imported, shown(importer))) + lines.append("Commit these with your repository.") + + lines.append("") + lines.append("Choose a deployment with --config on any bazel command:") + lines.append("") + width = max([len(c.name) for c in sections]) + len("--config=") + for c in sections: + flag = "--config=" + c.name + lines.append(" %s%s %s" % (flag, " " * (width - len(flag)), c.comment)) + lines.append("") + lines.append("The file is a starting point: edit it as your repository needs; rerunning `aspect setup bazelrc` restores it.") + + lines.append("") + lines.append("Aspect Workflows tasks need no --config: pass --remote (`aspect build --remote`, `aspect test --remote`) and the default deployment is used automatically.") + return lines + +_BAZELRC_ARGS = { + "output": args.string( + default = "", + description = "Rc file to write, rewritten on every run. Defaults to `~/.bazelrc` on an Aspect Workflows runner and to `/.aspect/bazelrc` elsewhere, a file to commit. Parent directories are created if needed. Only file output is supported — there is no stdout mode.", + ), + "import_into": args.string( + long = "import-into", + default = "", + description = "Rc that receives the `try-import` line for the written rc, added at its top only when not already present, so the file's own lines override the import. Defaults to the workspace `.bazelrc`, importing `%workspace%/.aspect/bazelrc`; on an Aspect Workflows runner the output is `~/.bazelrc` itself and no import is needed.", + ), + "omit_bazel_flags": args.string_list( + long = "omit-bazel-flag", + description = "A Bazel flag this rc would otherwise set that you do not want, named as Aspect spells it (e.g. --omit-bazel-flag=--heap_dump_on_oom; a value after `=` is ignored). Repeat for several. Set it once per repo in config.axl via ctx.tasks[\"setup/bazelrc\"].args.omit_bazel_flags. Endpoints, credentials, and the runner's output paths cannot be omitted. Tasks have their own knob, --bazel-flags:omit.", + ), +} + bazelrc = task( + group = ["setup"], + summary = "Write a Bazel rc that points vanilla `bazel` calls at Aspect's remote services and recommended optimizations.", + description = "Write a Bazel rc so vanilla `bazel` calls pick up the same configuration that `aspect ` injects. On an Aspect Workflows runner `~/.bazelrc` is written whole; elsewhere the rc goes to `/.aspect/bazelrc` and the workspace `.bazelrc` gains `try-import %workspace%/.aspect/bazelrc` at its top under a comment, added once, so the repository's own lines override — commit both. Every flag sits in a named `--config` group — `aspect-common`, `aspect-cache`, `aspect-bes`, `aspect-`, `aspect-runner`, and the opt-in `aspect-exec` — and the groups that apply on this host are enabled at the top of the file (`bazel build --announce_rc` shows which group set what). On a Workflows runner, or wherever the `ASPECT_WORKFLOWS_*` environment names an endpoint, that is the recommended CI/profiling flags plus that remote cache, BES, repository cache, and per-runner output paths, with `--config=aspect-exec` opting into its remote executor, and no deployment sections. Elsewhere nothing is enabled and the deployment sections are opt-in, each bringing the common and CI-host tuning with it: `--config=aspect-cloud` for Aspect Cloud's remote cache + BES, `--config=aspect-` for a configured deployment (`aspect auth status`), and `--config=aspect--exec` to add its remote execution — authenticated through the `aspect get` credential helper. No section names the default deployment, which `aspect auth use` could change under a committed file. `aspect ci bazelrc` is an alias.", + implementation = _bazelrc_impl, + args = _BAZELRC_ARGS, +) + +ci_bazelrc = task( group = ["ci"], - summary = "Write a Bazel rc that configures vanilla `bazel` calls to use Aspect Workflows services and optimizations.", - description = "Write a Bazel rc so vanilla `bazel` calls pick up the same Aspect Workflows configuration that `aspect ` injects. On a Workflows runner this is the full set (remote cache, repository cache, per-runner output paths); on other CI it's the generic flags plus any remote cache configured via env vars. Defaults to `~/.bazelrc`; off a runner and off CI it writes nothing unless `--output` is given, so it won't clobber your local `~/.bazelrc`.", + kind = "bazelrc", + summary = "Alias of `aspect setup bazelrc`, kept for the CI integrations that call it.", + description = "The same command as `aspect setup bazelrc`, under the name the setup-aspect GitHub Action, Buildkite plugin, CircleCI orb, and GitLab component call; new callers should use `aspect setup bazelrc`. Config overrides for this name go to ctx.tasks[\"ci/bazelrc\"].", implementation = _bazelrc_impl, - args = { - "output": args.string( - default = "", - description = "Rc file to write. Defaults to `~/.bazelrc` (the first user rc Bazel loads). Parent directories are created if needed. Only file output is supported — there is no stdout mode.", - ), - }, + args = _BAZELRC_ARGS, ) diff --git a/crates/aspect-cli/src/builtins/aspect/delivery.axl b/crates/aspect-cli/src/builtins/aspect/delivery.axl index 1d05254d4..2bc6e6d94 100644 --- a/crates/aspect-cli/src/builtins/aspect/delivery.axl +++ b/crates/aspect-cli/src/builtins/aspect/delivery.axl @@ -156,11 +156,12 @@ load("@bazel//proto/remote_logging.axl", "remote_logging") load("@std//time.axl", "sleep_iter", "time") load("./private/lib/ansi.axl", "ansi") load("./private/lib/artifacts.axl", "artifacts") +load("./private/lib/aspect_flags.axl", "apply_output_base_suffix") load("./private/lib/bazel_results.axl", "BES_DRAIN_TICK_MS", "MIN_HEARTBEAT_INTERVAL_MS", "archive_bazel_attempt", "bes_get", "now_ms", "process_event", bazel_init_data = "init_data") load("./private/lib/ci.axl", "resolve_build_url") load("./private/lib/delivery_results.axl", delivery_add_result = "add_result", delivery_build_manifest = "build_manifest", delivery_conclusion = "conclusion", delivery_init_data = "init_data") load("./private/lib/deliveryd.axl", "deliveryd") -load("./private/lib/environment.axl", "apply_output_base_suffix", "color_enabled", "error", "info", "sanitize_filename", "warn") +load("./private/lib/environment.axl", "color_enabled", "error", "info", "sanitize_filename", "warn") load("./private/lib/github.axl", "detect_commit_sha") load("./private/lib/health_check.axl", "HealthCheckTrait") load("./private/lib/lifecycle.axl", "phases") diff --git a/crates/aspect-cli/src/builtins/aspect/feature/bazel_flags.axl b/crates/aspect-cli/src/builtins/aspect/feature/bazel_flags.axl new file mode 100644 index 000000000..8994fc69b --- /dev/null +++ b/crates/aspect-cli/src/builtins/aspect/feature/bazel_flags.axl @@ -0,0 +1,44 @@ +"""Aspect's Bazel flag policy for tasks: the user's opt-outs from the flags other +features inject into a task's Bazel call. + +The flags themselves are decided in `lib/aspect_flags.axl` and injected by the +features that own their situation — `Workflows` on a runner, `Deployment` for +`--remote`. This feature owns only what the user may take away, so the knob has +one home whatever injects: `--bazel-flags:omit=` on any task, or once per +repo in `config.axl`: + + ctx.features[BazelFlags].args.omit = ["--heap_dump_on_oom"] + +It registers the omission on `BazelTrait.injected_flag_filters`, which the spawn +path applies to every injected flag and never to the user's own. The rc +`aspect setup bazelrc` writes has its own `--omit-bazel-flag`, since vanilla `bazel` +and tasks are different contexts and may want different omissions. +""" + +load("@aspect//bazel.axl", "BazelTrait") +load("../private/lib/aspect_flags.axl", "omit_flags", "protected_omissions") + +def _bazel_flags_impl(ctx: FeatureContext): + omitted = ctx.args.omit + if not omitted: + return + blocked = protected_omissions(omitted) + if blocked: + ctx.std.process.exit(1, "--bazel-flags:omit cannot drop %s: endpoints, credentials, and the runner's paths are what make the build Aspect's." % ", ".join(blocked)) + ctx.traits[BazelTrait].injected_flag_filters.append(lambda flags: omit_flags(flags, omitted)) + +BazelFlags = feature( + display_name = "Aspect Bazel flags", + summary = "Opt out of Bazel flags Aspect would otherwise set on a task's Bazel call.", + implementation = _bazel_flags_impl, + args = { + "omit": args.string_list( + description = "A Bazel flag Aspect would otherwise set on this task's Bazel call that you do not want, " + + "named as Aspect spells it (e.g. --bazel-flags:omit=--heap_dump_on_oom; a value after `=` " + + "is ignored). Repeat for several. Applies to what Aspect injects, never to your own flags; " + + "set it once per repo in config.axl via ctx.features[BazelFlags].args.omit. Endpoints, " + + "credentials, and the runner's output paths cannot be omitted. The rc written by " + + "`aspect setup bazelrc` has its own --omit-bazel-flag.", + ), + }, +) diff --git a/crates/aspect-cli/src/builtins/aspect/feature/workflows.axl b/crates/aspect-cli/src/builtins/aspect/feature/workflows.axl index 3c7f975be..42df9f96b 100644 --- a/crates/aspect-cli/src/builtins/aspect/feature/workflows.axl +++ b/crates/aspect-cli/src/builtins/aspect/feature/workflows.axl @@ -1,12 +1,20 @@ -load("@aspect//bazel/build_metadata.axl", "get_build_metadata_flags", "get_task_metadata_flags") +load("@aspect//bazel/build_metadata.axl", "register_bes_metadata") load("@aspect//bazel.axl", "BazelTrait") load( - "../private/lib/environment.axl", + "../private/lib/aspect_flags.axl", "apply_output_base_suffix", - "color_enabled", - "get_bazelrc_flags", - "get_build_diagnostics_flags", + "color_flags", + "env_task_flags", + "runner_diagnostics_flags", + "runner_executor_flags", + "runner_flags", + "temp_diagnostics_upload_flags", +) +load( + "../private/lib/environment.axl", + "bazel_misdetects_color", "get_workflows_environment", + "info", "print_environment_info", "sanitize_filename", ) @@ -19,28 +27,14 @@ def _workflows_impl(ctx: FeatureContext): hc_trait = ctx.traits[HealthCheckTrait] bazel_trait = ctx.traits[BazelTrait] - # Register task-identity --build_metadata as a task_flags hook. Runs - # with TaskContext at invocation time, so it has access to ctx.task.* - # (name, key, id) which FeatureContext doesn't expose. Every task that - # uses BazelTrait (build, test, lint, delivery) picks this up by - # calling `bazel_trait.task_flags` hooks from its impl — one place to - # register, zero per-task wiring. - bazel_trait.task_flags.append(get_task_metadata_flags) - - # `--color=yes` is wanted in every environment where aspect-cli will - # render ANSI elsewhere — not just on Workflows runners. Bazel's - # default `--color=auto` keys off stderr's isatty (since `ctx.bazel. - # build` inherits stderr by default that's normally a TTY locally), - # but several conditions still trip it to "no color" — `TERM=dumb`, - # being launched from a parent that's not on a tty (e.g. `cargo run - # -- format` from certain shells / editors), or CI log viewers that - # capture stderr to a pipe even though they render ANSI fine. Force - # it whenever `color_enabled` agrees output should be colored. On a - # plain TTY this matches what `auto` would have done. Deliberately - # not forcing `--curses=yes`: line-oriented log viewers garble - # cursor-control sequences. - if color_enabled(ctx.std): - bazel_trait.extra_flags.append("--color=yes") + bazel_trait.extra_flags.extend(color_flags(bazel_misdetects_color(ctx.std.env))) + + if ctx.args.remote_exec: + executor_flags = runner_executor_flags(environment) + if executor_flags: + bazel_trait.extra_flags.extend(executor_flags) + else: + info(ctx.std, "Remote execution was requested (--workflows:remote-exec) but ASPECT_WORKFLOWS_REMOTE_EXECUTOR is unset, so none is wired.") if environment.runner != None: # On a Workflows runner, register the runner-environment table and the @@ -63,11 +57,7 @@ def _workflows_impl(ctx: FeatureContext): bazel_trait.build_retry = recovery.build_retry bazel_trait.build_end.append(recovery.build_end) - # Flags to optimize build & test - (startup_flags, build_flags) = get_bazelrc_flags( - environment = environment, - aspect_root_dir = ctx.std.env.aspect_root_dir(), - ) + (startup_flags, build_flags) = runner_flags(environment, ctx.std.env.aspect_root_dir()) # Optionally move this task to its own Bazel server by suffixing the # runner-derived output base, so a job that flips Bazel options (e.g. @@ -79,12 +69,7 @@ def _workflows_impl(ctx: FeatureContext): bazel_trait.extra_startup_flags.extend(startup_flags) bazel_trait.extra_flags.extend(build_flags) - # BES-backend upload of the JSON profile + compact exec log. Driven by - # the deployment (ASPECT_WORKFLOWS_RUNNER_UPLOAD_BUILD_DIAGNOSTICS) or a - # workspace-level override arg. get_build_diagnostics_flags re-checks - # build_events/remote_cache and returns [] if either is missing. - diagnostics_on = ctx.args.upload_build_diagnostics or environment.runner.upload_build_diagnostics - bazel_trait.extra_flags.extend(get_build_diagnostics_flags(environment, diagnostics_on)) + bazel_trait.extra_flags.extend(runner_diagnostics_flags(environment)) # On older runners, `bazel` on the PATH is the legacy CLI, which we # disable the plugin feature on. `ASPECT_WORKFLOWS_RUNNER_NO_LEGACY_CLI` @@ -103,14 +88,15 @@ def _workflows_impl(ctx: FeatureContext): if is_legacy_cli: bazel_trait.extra_startup_flags.extend(["--aspect:disable_plugins", "--aspect:lock_version"]) - if environment.build_events != None: - metadata_flags = get_build_metadata_flags(ctx.std) + if environment.runner == None: + # A hand-set ASPECT_WORKFLOWS_* endpoint names the deployment the way a + # runner's does; only the runner's mounts are missing. + bazel_trait.extra_flags.extend(env_task_flags(environment)) + if environment.build_events != None: + bazel_trait.task_flags.append(temp_diagnostics_upload_flags) - # The runtime owns the BES connection, so send `x-identity` as sink - # metadata (the remote cache/RE path gets it via `--remote_header`). - bes_metadata = {} - if environment.runner and environment.runner.identity: - bes_metadata["x-identity"] = environment.runner.identity + if environment.build_events != None: + register_bes_metadata(ctx, bazel_trait) bessie_sinks = [] if environment.build_events and environment.build_events.backend: @@ -120,14 +106,12 @@ def _workflows_impl(ctx: FeatureContext): buffer_bytes = ctx.args.bes_retry_max_buffer_bytes bessie_sinks.append(bazel.build_events.grpc( uri = environment.build_events.backend, - metadata = bes_metadata, max_retries = ctx.args.bes_max_retries, retry_min_delay = ctx.args.bes_retry_min_delay, retry_max_buffer_bytes = buffer_bytes if buffer_bytes > 0 else None, timeout = ctx.args.bes_timeout, )) - bazel_trait.extra_flags.extend(metadata_flags) bazel_trait.build_event_sinks.extend(bessie_sinks) if environment.ci and environment.ci.host == "github": @@ -171,6 +155,16 @@ Workflows = feature( description = "Overall BES upload deadline. Duration string; '0s' " + "disables the deadline. Mirrors Bazel's --bes_timeout.", ), + "remote_exec": args.boolean( + default = False, + description = "Run actions on the remote executor the runner environment names " + + "(ASPECT_WORKFLOWS_REMOTE_EXECUTOR), by setting --remote_executor on this " + + "task's Bazel call. Off by default because remote execution relocates " + + "every action; the runner's cache and BES are wired regardless. Set it " + + "once per repo in config.axl via ctx.features[Workflows].args.remote_exec, " + + "or per task with --workflows:remote-exec. Vanilla `bazel` on the runner " + + "opts in with --config=aspect-exec from the rc `aspect setup bazelrc` writes.", + ), "runner_output_base_suffix": args.string( default = "", description = "Suffix appended to the runner-derived Bazel " + @@ -185,14 +179,5 @@ Workflows = feature( "ctx.features[Workflows].args.runner_output_base_suffix, or " + "per task with --workflows:runner-output-base-suffix.", ), - "upload_build_diagnostics": args.boolean( - default = False, - description = "Upload Bazel's JSON trace profile and compact " + - "execution log to the remote cache via BES for " + - "backend analysis. Normally driven by the deployment " + - "(ASPECT_WORKFLOWS_RUNNER_UPLOAD_BUILD_DIAGNOSTICS); " + - "this arg is a workspace-level override. No effect " + - "unless a BES backend and remote cache are configured.", - ), }, ) diff --git a/crates/aspect-cli/src/builtins/aspect/private/lib/aspect_flags.axl b/crates/aspect-cli/src/builtins/aspect/private/lib/aspect_flags.axl new file mode 100644 index 000000000..50d3582af --- /dev/null +++ b/crates/aspect-cli/src/builtins/aspect/private/lib/aspect_flags.axl @@ -0,0 +1,458 @@ +"""The Bazel flags Aspect sets on the user's behalf, in one place. + +Three surfaces set them: the Workflows feature injects them into every +`aspect ` Bazel call on an Aspect Workflows runner, the Deployment feature +into a task told `--remote`, and `aspect setup bazelrc` writes them to an rc for +vanilla `bazel` calls. All compose from the same selections here, so a flag is +set the same way — and for the same reason — wherever it appears. + +Every flag is tied to what it presumes (`Needs`). A flag that tunes the remote +cache is set only where a remote cache is wired, a flag that enriches the BES +stream only where a BES backend reads it. A runner or deployment with no cache, +no BES, or no executor gets none of the flags that presume the missing one, so +every line Aspect writes is accounted for by an endpoint that is present. In the +rc the same keying becomes named `--config` groups (`rc_groups.axl`), one per +thing presumed. + +`CATALOG`, `WORKFLOWS_CACHE_FLAGS`, and `NO_COMPRESSION_FLAGS` are the maintained +tables: each entry says what the flag does, why we set it, and what it presumes. +Read them as the documentation; the functions only select from them. Flags whose values come +from the environment or a deployment — the endpoints themselves, the runner's +paths, the diagnostics file locations — are built by the functions below and +documented there. + +Users may opt out of individual flags, per surface: `--bazel-flags:omit` for +tasks (`feature/bazel_flags.axl`) and `--omit-bazel-flag` on `aspect setup bazelrc`. +Both use `omit_flags`, which applies only to what Aspect injects, and +`PROTECTED_FLAGS`, the flags that cannot be dropped: endpoints, credentials, and +the runner's paths, without which the build stops talking to Aspect or breaks +the runner. + +A flag is a plain string, a `(flag, constraint)` tuple carrying a Bazel semver +constraint, or a `(flag, constraint, command)` tuple that also names the rc +section the flag belongs to — `"build"` for an option only build-like commands +accept, which the bazelrc runtime then withholds from `query` and its kin the +way an rc file's `build` line is. All three are the shape `ctx.bazel.parse_rc` +/ `new_rc` take and the trait's flag lists accept. +""" + +load("./environment.axl", "WorkflowsEnvironment", "sanitize_filename") + +# What a flag presumes before it is worth setting. `cache` is satisfied by a +# remote executor as well: Bazel routes CAS/AC through `--remote_executor` when +# no `--remote_cache` is set. `channel` is any gRPC endpoint at all. +Needs = enum("nothing", "cache", "bes", "channel") + +InjectedFlag = record( + flag = field(str), + why = field(str), + needs = field(Needs, default = Needs("nothing")), + # Semver constraint on the Bazel version, "" for every version. + bazel = field(str, default = ""), +) + +def _entry(flag: str, why: str, needs: str = "nothing", bazel: str = "") -> InjectedFlag: + return InjectedFlag(flag = flag, why = why, needs = Needs(needs), bazel = bazel) + +# The tuning and reporting flags Aspect sets wherever their endpoint is wired. +CATALOG = [ + _entry( + "--heap_dump_on_oom", + "A heap dump when the server runs out of memory. CI OOMs cannot be reproduced interactively; the dump is the only post-mortem.", + ), + _entry( + "--experimental_repository_cache_hardlinks", + "Hardlink archives out of the repository cache instead of copying them; saves disk and time on every external fetch.", + ), + _entry( + "--remote_upload_local_results", + "CI populates the cache: outputs built locally are written back so later jobs and developers hit them.", + needs = "cache", + ), + _entry( + "--remote_accept_cached", + "Reuse cached action results; stated explicitly so a workspace rc cannot leave the cache write-only.", + needs = "cache", + ), + _entry( + "--remote_timeout=3600", + "Bazel's 60s default fails large artifacts on slow links mid-transfer.", + needs = "cache", + ), + _entry( + "--remote_retries=360", + "Ride out cache restarts and network blips instead of failing the build; Bazel defaults to 5.", + needs = "cache", + ), + _entry( + "--grpc_keepalive_timeout=30s", + "Drop and reopen a channel a load balancer or NAT silently killed, rather than hang on it.", + needs = "channel", + ), + _entry( + "--generate_json_trace_profile", + "The JSON trace profile the Aspect backend analyzes.", + needs = "bes", + ), + _entry( + "--noslim_profile", + "Keep every profile event; the slim profile merges short ones and loses the per-action detail the backend needs.", + needs = "bes", + ), + _entry( + "--experimental_profile_include_target_label", + "Attribute profile time to targets.", + needs = "bes", + ), + _entry( + "--experimental_profile_include_primary_output", + "Tell apart the several actions of one target in the profile.", + needs = "bes", + ), + _entry( + "--experimental_profile_include_target_configuration", + "Separate the exec- and target-configuration builds of one label in the profile.", + needs = "bes", + bazel = ">=8.0.0", + ), + _entry( + "--legacy_important_outputs", + "Keep `important_output` on TargetComplete events, which the Aspect Web UI reads and Bazel intends to stop emitting by default.", + needs = "bes", + ), +] + +# Set on a runner whose cache does not accept compressed blobs +# (ASPECT_WORKFLOWS_REMOTE_CACHE_COMPRESSION unset): Bazel errors when a +# repository rc asks such a cache for compression, so the runner forces it off. +# A cache that declares support leaves the choice to the repository. Drop once no +# Workflows deployment without compression is left in the wild. +NO_COMPRESSION_FLAGS = [ + _entry( + "--noexperimental_remote_cache_compression", + "This Workflows remote cache does not support compressed blobs.", + bazel = "<8.0.0", + ), + _entry( + "--noremote_cache_compression", + "This Workflows remote cache does not support compressed blobs; Bazel 8 renamed the flag.", + bazel = ">=8.0.0", + ), +] + +# Flags specific to the Aspect Workflows runner's own remote cache, set only on a +# runner with that cache wired. They presume the Workflows cache implementation, +# not just any cache, so they are not in `CATALOG`. +WORKFLOWS_CACHE_FLAGS = [ + _entry( + "--incompatible_remote_results_ignore_disk", + "Give Bazel 6 the combined-cache semantics Bazel 7 made the default.", + bazel = "<7.0.0", + ), + _entry( + "--disk_cache=", + "With the runner's remote cache set, a disk cache from a workspace rc would only fill the runner's disk with a second copy of what the remote cache holds. Elsewhere a disk cache is left as the user set it.", + ), +] + +# Flag names (no value) that the opt-outs may not drop. Each is an endpoint, a +# credential, an rc chain, or a path the runner's isolation depends on; without +# it the build stops talking to Aspect or lands in the wrong place. The BES pair +# (`--bes_backend`, `--bes_results_url`) is deliberately absent: it is the one +# endpoint a repository may legitimately route elsewhere for vanilla `bazel`, +# and the opt-out is how the runner's rc yields to that. +PROTECTED_FLAGS = [ + "--remote_cache", + "--remote_executor", + "--remote_bytestream_uri_prefix", + "--credential_helper", + "--experimental_credential_helper", + "--remote_header", + "--bes_header", + "--config", + "--output_base", + "--output_user_root", + "--repository_cache", + "--nohome_rc", + "--nosystem_rc", + "--aspect:disable_plugins", + "--aspect:lock_version", +] + +# The value-bearing flags the functions below build, by name. +_REMOTE_CACHE = "--remote_cache" +_REMOTE_EXECUTOR = "--remote_executor" +_BYTESTREAM_URI_PREFIX = "--remote_bytestream_uri_prefix" +_REPOSITORY_CACHE = "--repository_cache" +_OUTPUT_USER_ROOT = "--output_user_root" +_OUTPUT_BASE = "--output_base" +_NOHOME_RC = "--nohome_rc" +_NOSYSTEM_RC = "--nosystem_rc" +_COLOR = "--color" +_BES_BACKEND = "--bes_backend" +_BES_RESULTS_URL = "--bes_results_url" +_PROFILE = "--profile" +_REMOTE_BUILD_EVENT_UPLOAD = "--remote_build_event_upload" +_EXECUTION_LOG_COMPACT_FILE = "--execution_log_compact_file" + +def flag_text(flag) -> str: + """The flag text of a plain or `(flag, constraint)` entry.""" + return flag[0] if type(flag) == "tuple" else flag + +def flag_name(flag) -> str: + """The name of a flag without its value: `--remote_timeout=3600` → `--remote_timeout`.""" + return flag_text(flag).split("=")[0] + +def unscoped(flag): + """`flag` without a command scope: a `(flag, constraint, command)` entry + becomes `(flag, constraint)`, or the bare flag when the constraint is empty. + For the rc `aspect setup bazelrc` writes, whose `common` sections Bazel already + applies only where the option is accepted.""" + if type(flag) != "tuple" or len(flag) < 3: + return flag + return (flag[0], flag[1]) if flag[1] else flag[0] + +def _as_flag(entry: InjectedFlag): + return (entry.flag, entry.bazel) if entry.bazel else entry.flag + +def independent_flags() -> list: + """The `CATALOG` flags that presume no endpoint. Kept apart from + `endpoint_flags` so each surface places them once: the head of a task's + injected flags, the `aspect-common` group of the rc.""" + return [_as_flag(e) for e in CATALOG if e.needs == Needs("nothing")] + +def endpoint_flags(cache: bool, bes: bool, exec: bool = False) -> list: + """The `CATALOG` flags the wired endpoints justify: cache tuning with a cache + or an executor, BES enrichment with a BES backend, channel tuning with any of + the three. Nothing when none is wired; `independent_flags` are separate so + each surface places them once.""" + satisfied = { + Needs("nothing"): False, + Needs("cache"): cache or exec, + Needs("bes"): bes, + Needs("channel"): cache or bes or exec, + } + return [_as_flag(e) for e in CATALOG if satisfied[e.needs]] + +def omit_flags(flags: list, omitted: list) -> list: + """`flags` minus those whose name is in `omitted`. Names compare without a + value, so `--remote_timeout` drops `--remote_timeout=3600`, and an omitted + entry may itself carry a value.""" + names = [flag_name(o) for o in omitted] + return [f for f in flags if flag_name(f) not in names] + +def protected_omissions(omitted: list) -> list: + """The entries of `omitted` that name a `PROTECTED_FLAGS` flag, as the user + spelled them; empty when every omission is allowed.""" + return [o for o in omitted if flag_name(o) in PROTECTED_FLAGS] + +def color_flags(misdetected: bool) -> list: + """`--color=yes` where Bazel's `--color=auto` would wrongly say no: a CI host + whose log viewer renders ANSI but runs the job with no TTY for Bazel to see + (`environment.bazel_misdetects_color`). Presumes no endpoint — the host, not + a deployment, decides — so in the rc it lives in the group named for that + host (`aspect-github-actions`, …).""" + return [_COLOR + "=yes"] if misdetected else [] + +def env_endpoint_flags(environment: WorkflowsEnvironment) -> list: + """The remote-cache endpoint the runner environment names: `--remote_cache` + and, when surfaced, `--remote_bytestream_uri_prefix`, which makes the + `bytestream://` URIs in BEP addressable across caches.""" + remote_cache = environment.remote_cache + if remote_cache == None: + return [] + flags = [] + if remote_cache.endpoint: + flags.append(_REMOTE_CACHE + "=" + remote_cache.endpoint) + if remote_cache.bytestream_uri_prefix: + flags.append(_BYTESTREAM_URI_PREFIX + "=" + remote_cache.bytestream_uri_prefix) + return flags + +def env_flags(environment: WorkflowsEnvironment) -> list: + """The task flags the `ASPECT_WORKFLOWS_*` env justifies: the cache endpoint + it names and the `CATALOG` selection for its cache and BES. Empty when the + env names neither. Nothing tied to a runner's mounts or its cache + implementation; `runner_cache_flags` adds those.""" + return endpoint_flags( + cache = environment.remote_cache != None, + bes = environment.build_events != None, + ) + env_endpoint_flags(environment) + +def env_task_flags(environment: WorkflowsEnvironment) -> list: + """The task flags a hand-set `ASPECT_WORKFLOWS_*` environment justifies off a + runner: the endpoint-independent flags and `env_flags`, the same set a + runner's task gets minus what its mounts add. Empty when the env names no + cache and no BES, so plain CI and developer machines get nothing. On a + runner `runner_flags` covers this.""" + if environment.remote_cache == None and environment.build_events == None: + return [] + return independent_flags() + env_flags(environment) + +def runner_bes_flags(environment: WorkflowsEnvironment) -> list: + """The BES backend the runner environment names, for vanilla `bazel` on the + runner: `--bes_backend` and, when surfaced, `--bes_results_url`. Tasks never + take these — they run with `--nohome_rc` and the CLI streams their build + events from its own sink — so only `aspect setup bazelrc` writes them. A + repository routing vanilla calls to another backend opts out with + `--omit-bazel-flag=--bes_backend`, which is why the pair is not in + `PROTECTED_FLAGS`. Empty when the env names no BES backend.""" + build_events = environment.build_events + if build_events == None or not build_events.backend: + return [] + flags = [_BES_BACKEND + "=" + build_events.backend] + if build_events.results_url: + flags.append(_BES_RESULTS_URL + "=" + build_events.results_url) + return flags + +def runner_executor_flags(environment: WorkflowsEnvironment) -> list: + """The remote executor the runner environment names, for a build that opted in: + `--remote_executor` plus, when the env names no separate cache, the tuning an + executor justifies (it doubles as the cache). Never part of `runner_flags` + or `env_flags` — remote execution relocates every action, so it is reached + only by naming it: `--workflows:remote-exec` on a task, `--config=aspect-exec` + in the rc. Empty when the env names no executor.""" + endpoint = runner_executor_endpoint(environment) + if not endpoint: + return [] + return endpoint + ([] if environment.remote_cache != None else endpoint_flags(cache = False, bes = False, exec = True)) + +def runner_executor_endpoint(environment: WorkflowsEnvironment) -> list: + """Just `--remote_executor=` from the environment, or empty. The rc's + `aspect-exec` group takes this and chains the cache tuning by name.""" + executor = environment.remote_executor + if executor == None or not executor.endpoint: + return [] + return [_REMOTE_EXECUTOR + "=" + executor.endpoint] + +def workflows_cache_flags(environment: WorkflowsEnvironment) -> list: + """The Workflows cache's specifics: `NO_COMPRESSION_FLAGS` unless the cache + declares compression support, then `WORKFLOWS_CACHE_FLAGS`. Presumes the + runner's own cache, so callers gate on the runner marker; fails without + `environment.remote_cache`.""" + remote_cache = environment.remote_cache + if remote_cache == None: + fail("workflows_cache_flags called without a remote cache — gate on environment.remote_cache != None") + compression = [] if remote_cache.supports_compression else NO_COMPRESSION_FLAGS + return [_as_flag(e) for e in compression + WORKFLOWS_CACHE_FLAGS] + +def runner_repository_cache_flags(environment: WorkflowsEnvironment) -> list: + """`--repository_cache` at the runner's warmed repository cache. Presumes + the runner marker.""" + runner = environment.runner + if runner == None: + fail("runner_repository_cache_flags called without a Workflows runner — gate on environment.runner != None") + return [_REPOSITORY_CACHE + "=" + runner.repository_cache_dir] + +def runner_cache_flags(environment: WorkflowsEnvironment) -> list: + """The runner's own cache specifics for a task: `workflows_cache_flags` when + the env names the runner's cache, plus `runner_repository_cache_flags`. + Presumes the runner marker.""" + if environment.runner == None: + fail("runner_cache_flags called without a Workflows runner — gate on environment.runner != None") + specifics = workflows_cache_flags(environment) if environment.remote_cache != None else [] + return specifics + runner_repository_cache_flags(environment) + +def runner_startup_flags(environment: WorkflowsEnvironment, aspect_root_dir: str) -> list: + """The startup flags that give each repository its own Bazel server on the + runner's storage. `--output_user_root` holds the install base and external + repos, shared across that repo's jobs; `--output_base` holds the server and + its analysis cache, so a persistent runner keeps it warm between jobs + (`apply_output_base_suffix` moves a task to a sibling server). + `--nohome_rc`/`--nosystem_rc` make the workspace rc plus the injected flags + the whole configuration; they are command-line only, and `aspect setup bazelrc` + drops them. Presumes the runner marker.""" + runner = environment.runner + if runner == None: + fail("runner_startup_flags called without a Workflows runner — gate on environment.runner != None") + repo_name = environment.ci.scm_repo_name if environment.ci else "" + subdir = sanitize_filename(aspect_root_dir.rstrip("/").split("/")[-1]) if aspect_root_dir else "__main__" + scope = sanitize_filename(repo_name) + "/" + subdir if repo_name else subdir + return [ + _NOHOME_RC, + _NOSYSTEM_RC, + _OUTPUT_USER_ROOT + "=" + runner.bazel_root_dir + "/" + scope, + _OUTPUT_BASE + "=" + runner.output_root_dir + "/" + scope, + ] + +def runner_flags(environment: WorkflowsEnvironment, aspect_root_dir: str) -> (list, list): + """The full flag set the Workflows feature injects into a task on a runner: + the endpoint-independent flags, `env_flags`, and `runner_cache_flags` as + build flags, and `runner_startup_flags` as startup flags. Caller must have + verified `environment.runner != None`. Returns `(startup_flags, build_flags)`.""" + return ( + runner_startup_flags(environment, aspect_root_dir), + independent_flags() + env_flags(environment) + runner_cache_flags(environment), + ) + +_OUTPUT_BASE_FLAG_PREFIX = _OUTPUT_BASE + "=" + +def apply_output_base_suffix(startup_flags: list, suffix: str) -> list: + """A copy of `startup_flags` with `suffix` appended to the `--output_base=` + value; every other flag passes through. `--output_base` names the Bazel + server, so this moves the invocation to a sibling server (`…/aspect-cli` → + `…/aspect-cli-delivery`), isolating a task whose options (`--stamp`) would + otherwise discard the shared warm analysis cache. `--output_user_root` stays + shared so the install base and external repos are not duplicated. + + A no-op copy when `suffix` is empty or no `--output_base` is present. The + suffix becomes a path segment; callers sanitize it (`sanitize_filename`).""" + if not suffix: + return list(startup_flags) + return [flag + suffix if flag.startswith(_OUTPUT_BASE_FLAG_PREFIX) else flag for flag in startup_flags] + +def diagnostics_upload_flags() -> list: + """The switch that carries build diagnostics to the BES backend: every + BEP-referenced local file — the JSON profile, and the execution log when + `exec_log_flags` writes one — uploaded to the remote cache so the backend + can read it. Set wherever a BES backend is wired; the upload needs a remote + cache to land in, so with a BES alone it is inert. Version-gated: Bazel 7 + renamed `--remote_build_event_upload` and flipped its default to `minimal` + (Bazel 6 only knows the `experimental_` spelling, whose default is already + `all`).""" + return [(_REMOTE_BUILD_EVENT_UPLOAD + "=all", ">=7.0.0")] + +def exec_log_flags(exec_log_path: str) -> list: + """Write the compact execution log at `exec_log_path`, so it rides the + diagnostics upload. Per-action cache and timing data in the Web UI comes + from this log, which is also why the per-action BEP events behind + `--build_event_publish_all_actions` are not requested. Bazel 7.1 added the + compact log; an execution option, so scoped to `build` and never reaching a + task's `query`. The rc keeps it in its own `aspect-exec-log` group, the + upload being slow on a thin link, so a repository can drop just this.""" + return [(_EXECUTION_LOG_COMPACT_FILE + "=" + exec_log_path, ">=7.1.0", "build")] + +def temp_exec_log_path(std, run_id: str) -> str: + """A place for a task's compact execution log off a Workflows runner: under + the OS temp dir so nothing lands in the checkout, namespaced by `run_id` + since concurrent tasks on one machine share that dir. Creates the directory, + at the task's own run — never for an rc, which outlives the directory.""" + dir = std.env.temp_dir().rstrip("/") + "/aspect-bazel-" + (run_id or "exec") + std.fs.create_dir_all(dir) + return dir + "/exec.log.zstd" + +def temp_diagnostics_upload_flags(ctx) -> list: + """The diagnostics upload with the execution log at `temp_exec_log_path` for + this run — a `BazelTrait.task_flags` hook, since only the task knows its run + id. For a BES wired off a runner (`--remote`, or a hand-set env).""" + return diagnostics_upload_flags() + exec_log_flags(temp_exec_log_path(ctx.std, ctx.task.id)) + +def runner_profile_flags(environment: WorkflowsEnvironment) -> list: + """`--profile` placed under the runner's job tmpdir, keeping the + `command.profile.gz` name the backend's ingest matches on. Empty off a runner + or without a BES backend to upload it to.""" + runner = environment.runner + if runner == None or environment.build_events == None: + return [] + return [_PROFILE + "=" + runner.job_tmpdir + "/command.profile.gz"] + +def runner_exec_log_path(environment: WorkflowsEnvironment) -> str: + """Where a runner's compact execution log is written: the job tmpdir.""" + return environment.runner.job_tmpdir + "/exec.log.zstd" + +def runner_diagnostics_flags(environment: WorkflowsEnvironment) -> list: + """`runner_profile_flags`, the diagnostics upload, and the execution log at + `runner_exec_log_path`, for a task on a runner whose env names a BES backend. + Empty off a runner or without one.""" + if environment.runner == None or environment.build_events == None: + return [] + return runner_profile_flags(environment) + diagnostics_upload_flags() + exec_log_flags(runner_exec_log_path(environment)) diff --git a/crates/aspect-cli/src/builtins/aspect/private/lib/aspect_flags_test.axl b/crates/aspect-cli/src/builtins/aspect/private/lib/aspect_flags_test.axl new file mode 100644 index 000000000..3a043c84f --- /dev/null +++ b/crates/aspect-cli/src/builtins/aspect/private/lib/aspect_flags_test.axl @@ -0,0 +1,314 @@ +"""Unit tests for `aspect_flags.axl`: which catalog flags each endpoint justifies +(`independent_flags` / `endpoint_flags`), the `env_flags` / `runner_flags` / +`runner_diagnostics_flags` compositions and the runner pieces the rc groups are +assembled from, `apply_output_base_suffix`, and the opt-out (`omit_flags`, +`protected_omissions`). + +Version-gating of `(flag, constraint)` tuples is the bazelrc runtime's job; these +assert the tuples carry the right constraints. Run with: aspect dev test-aspect-flags +""" + +load( + "./aspect_flags.axl", + "CATALOG", + "PROTECTED_FLAGS", + "apply_output_base_suffix", + "color_flags", + "diagnostics_upload_flags", + "endpoint_flags", + "env_flags", + "env_task_flags", + "exec_log_flags", + "flag_name", + "independent_flags", + "omit_flags", + "protected_omissions", + "runner_bes_flags", + "runner_cache_flags", + "runner_diagnostics_flags", + "runner_exec_log_path", + "runner_executor_endpoint", + "runner_executor_flags", + "runner_flags", + "runner_profile_flags", + "runner_repository_cache_flags", + "runner_startup_flags", + "unscoped", + "workflows_cache_flags", +) +load("./environment.axl", "BuildEvents", "CI", "RemoteCache", "RemoteExecutor", "Runner", "WorkflowsEnvironment", "bazel_misdetects_color") + +def _eq(label, got, want): + if got != want: + fail("%s: got %r, want %r" % (label, got, want)) + +def _texts(flags) -> list: + return [f[0] if type(f) == "tuple" else f for f in flags] + +def _has(label, flags, want): + _eq(label, want in _texts(flags), True) + +def _lacks(label, flags, fragment): + _eq(label, any([fragment in f for f in _texts(flags)]), False) + +# Flags only a Workflows runner's mounts or its own cache justify. Never in +# `env_flags`, whatever endpoints the environment names. +_RUNNER_ONLY_FRAGMENTS = ["--repository_cache=", "--output_base=", "--output_user_root=", "--nosystem_rc", "_remote_cache_compression", "--disk_cache="] + +_CACHE_TUNING = ["--remote_upload_local_results", "--remote_accept_cached", "--remote_timeout=3600", "--remote_retries=360"] +_BES_ENRICHMENT = ["--generate_json_trace_profile", "--noslim_profile", "--experimental_profile_include_target_label", "--experimental_profile_include_primary_output", "--legacy_important_outputs"] +_INDEPENDENT = ["--heap_dump_on_oom", "--experimental_repository_cache_hardlinks"] +_CHANNEL = "--grpc_keepalive_timeout=30s" + +_WITH_BES = WorkflowsEnvironment(build_events = BuildEvents(backend = "grpcs://b")) + +def _test_endpoint_flags_follow_endpoints(_ctx): + # No endpoint: nothing. The endpoint-independent flags are a separate group + # so each surface places them once. + _eq("nothing wired", endpoint_flags(cache = False, bes = False), []) + _eq("independent", sorted(_texts(independent_flags())), sorted(_INDEPENDENT)) + + # A cache brings its tuning and the channel keepalive, nothing BES-shaped. + cache = endpoint_flags(cache = True, bes = False) + for want in _CACHE_TUNING + [_CHANNEL]: + _has("cache has " + want, cache, want) + for absent in _BES_ENRICHMENT: + _lacks("cache lacks " + absent, cache, absent) + + # BES brings the profile/BEP enrichment and the keepalive, no cache tuning. + bes = endpoint_flags(cache = False, bes = True) + for want in _BES_ENRICHMENT + [_CHANNEL]: + _has("bes has " + want, bes, want) + for absent in _CACHE_TUNING: + _lacks("bes lacks " + absent, bes, absent) + _eq("target_configuration gated", ("--experimental_profile_include_target_configuration", ">=8.0.0") in bes, True) + + # An executor doubles as the cache, so it earns the cache tuning. + exec_only = endpoint_flags(cache = False, bes = False, exec = True) + for want in _CACHE_TUNING + [_CHANNEL]: + _has("exec has " + want, exec_only, want) + + # Every catalog entry is reachable between the two groups. + _eq("catalog covered", len(independent_flags()) + len(endpoint_flags(True, True, True)), len(CATALOG)) + +def _test_env_flags_follow_the_environment(_ctx): + # Nothing in the env: nothing unconditional, so an imported rc is inert. + _eq("bare", env_flags(WorkflowsEnvironment()), []) + + # The env's cache brings its endpoint flags and tuning, still nothing runner-only. + env = WorkflowsEnvironment(remote_cache = RemoteCache(endpoint = "unix:///x/grpc", bytestream_uri_prefix = "aw-remote.example")) + flags = env_flags(env) + _has("remote_cache", flags, "--remote_cache=unix:///x/grpc") + _has("bytestream prefix", flags, "--remote_bytestream_uri_prefix=aw-remote.example") + _has("cache tuning", flags, "--remote_timeout=3600") + _lacks("independent flags are not the env's", flags, "--heap_dump_on_oom") + for frag in _RUNNER_ONLY_FRAGMENTS: + _lacks("no runner-only " + frag, flags, frag) + + # The env's BES brings the enrichment. + _has("bes enrichment", env_flags(_WITH_BES), "--noslim_profile") + _lacks("no cache tuning without a cache", env_flags(_WITH_BES), "--remote_timeout") + +def _test_env_task_flags(_ctx): + # Off a runner a hand-set env gets what a runner's task gets minus its mounts; + # a silent env gets nothing at all. + env = WorkflowsEnvironment(remote_cache = RemoteCache(endpoint = "grpc://c"), build_events = BuildEvents(backend = "grpc://b")) + _eq("independent + env", env_task_flags(env), independent_flags() + env_flags(env)) + for fragment in _RUNNER_ONLY_FRAGMENTS: + _lacks("no runner-only " + fragment, env_task_flags(env), fragment) + _eq("silent env", env_task_flags(WorkflowsEnvironment(ci = CI(host = "github"))), []) + +def _test_runner_flags(_ctx): + env = WorkflowsEnvironment( + runner = Runner(repository_cache_dir = "/mnt/caches/repository"), + remote_cache = RemoteCache(endpoint = "grpcs://c"), + build_events = BuildEvents(backend = "grpcs://b"), + ) + startup, build = runner_flags(env, "/work/repo") + + # The independent flags, the env's tuning, the Workflows-cache specifics, and + # the repository cache. + _has("independent", build, "--heap_dump_on_oom") + _has("tuning", build, "--remote_timeout=3600") + _has("enrichment", build, "--noslim_profile") + _has("repository cache", build, "--repository_cache=/mnt/caches/repository") + _has("disk cache reset", build, "--disk_cache=") + _eq("compression <8", ("--noexperimental_remote_cache_compression", "<8.0.0") in build, True) + _eq("compression >=8", ("--noremote_cache_compression", ">=8.0.0") in build, True) + _eq("ignore_disk <7", ("--incompatible_remote_results_ignore_disk", "<7.0.0") in build, True) + _eq("startup", startup[:2], ["--nohome_rc", "--nosystem_rc"]) + + # Without the runner's cache in the env, its cache specifics stay out too. + _startup, no_cache = runner_flags(WorkflowsEnvironment(runner = Runner(), build_events = BuildEvents(backend = "b")), "/work/repo") + _lacks("no disk cache reset", no_cache, "--disk_cache=") + _lacks("no compression gate", no_cache, "_remote_cache_compression") + _lacks("no cache tuning", no_cache, "--remote_timeout") + +def _test_runner_flags_output_base(_ctx): + # The repo name and the checkout's last path segment scope both roots; the + # repo name is sanitized. subdir "aspect-cli" is clean. + env = WorkflowsEnvironment(runner = Runner(bazel_root_dir = "/b", output_root_dir = "/o"), ci = CI(scm_repo_name = "my repo")) + startup, _build = runner_flags(env, "/work/aspect-cli") + _eq("output_base", "--output_base=/o/my_repo/aspect-cli" in startup, True) + _eq("output_user_root", "--output_user_root=/b/my_repo/aspect-cli" in startup, True) + + # Only --output_base gains a suffix; --output_user_root stays shared. + suffixed = apply_output_base_suffix(startup, "-delivery") + _eq("output_base moved", "--output_base=/o/my_repo/aspect-cli-delivery" in suffixed, True) + _eq("output_user_root shared", "--output_user_root=/b/my_repo/aspect-cli" in suffixed, True) + _eq("empty suffix noop", apply_output_base_suffix(startup, ""), startup) + _eq("no output_base noop", apply_output_base_suffix(["--nohome_rc"], "-x"), ["--nohome_rc"]) + +def _test_runner_diagnostics_flags(_ctx): + env = WorkflowsEnvironment(runner = Runner(job_tmpdir = "/workflows"), build_events = BuildEvents(backend = "grpcs://bes")) + flags = runner_diagnostics_flags(env) + _eq("profile placed", "--profile=/workflows/command.profile.gz" in flags, True) + _eq("exec log gated", ("--execution_log_compact_file=/workflows/exec.log.zstd", ">=7.1.0", "build") in flags, True) + + # Bazel 6 only knows `--experimental_remote_build_event_upload` (default + # `all`), so the unprefixed spelling must be gated to 7+. + _eq("upload=all gated", ("--remote_build_event_upload=all", ">=7.0.0") in flags, True) + + # On by default wherever a BES backend is; nothing without one or off a runner. + _eq("no BES → empty", runner_diagnostics_flags(WorkflowsEnvironment(runner = Runner(), remote_cache = RemoteCache(endpoint = "x"))), []) + _eq("no runner → empty", runner_diagnostics_flags(_WITH_BES), []) + +def _test_runner_pieces(_ctx): + # The rc assembles `runner_flags` from its parts, so each part is exactly + # its share: startup paths; cache specifics with the repository cache; the + # profile placement; the exec-log path. + env = WorkflowsEnvironment(runner = Runner(bazel_root_dir = "/mnt/b", output_root_dir = "/mnt/o", repository_cache_dir = "/mnt/r", job_tmpdir = "/wf"), remote_cache = RemoteCache(endpoint = "grpc://c"), build_events = BuildEvents(backend = "grpc://b"), ci = CI(scm_repo_name = "repo")) + startup, build = runner_flags(env, "/work/repo") + _eq("startup is its part", startup, runner_startup_flags(env, "/work/repo")) + cache = runner_cache_flags(env) + _has("repository cache", cache, "--repository_cache=/mnt/r") + _has("disk cache reset", cache, "--disk_cache=") + _eq("build = independent + env + cache", _texts(build), _texts(independent_flags() + env_flags(env) + cache)) + _eq("no env cache → only the repository cache", runner_cache_flags(WorkflowsEnvironment(runner = Runner(repository_cache_dir = "/mnt/r"))), ["--repository_cache=/mnt/r"]) + _eq("cache = specifics + repository cache", cache, workflows_cache_flags(env) + runner_repository_cache_flags(env)) + _has("specifics carry the disk-cache reset", workflows_cache_flags(env), "--disk_cache=") + + # A cache that declares compression support (ASPECT_WORKFLOWS_REMOTE_CACHE_COMPRESSION) + # is no longer forced off; the repository decides. The other specifics stay. + compressing = WorkflowsEnvironment(runner = Runner(repository_cache_dir = "/mnt/r"), remote_cache = RemoteCache(endpoint = "grpc://c", supports_compression = True)) + _lacks("compression left to the repo", workflows_cache_flags(compressing), "_remote_cache_compression") + _has("disk-cache reset stays", workflows_cache_flags(compressing), "--disk_cache=") + _lacks("task flags follow", runner_flags(compressing, "/x")[1], "_remote_cache_compression") + _eq("profile", runner_profile_flags(env), ["--profile=/wf/command.profile.gz"]) + _eq("profile needs BES", runner_profile_flags(WorkflowsEnvironment(runner = Runner(job_tmpdir = "/wf"))), []) + _eq("exec log", runner_exec_log_path(env), "/wf/exec.log.zstd") + _eq("diagnostics = profile + upload + log", runner_diagnostics_flags(env), runner_profile_flags(env) + diagnostics_upload_flags() + exec_log_flags("/wf/exec.log.zstd")) + +def _test_runner_bes_flags(_ctx): + # The env's BES backend and viewer, for vanilla `bazel` on the runner. + env = WorkflowsEnvironment(runner = Runner(), build_events = BuildEvents(backend = "grpc://bes:1985", results_url = "https://ui/i/")) + _eq("both", runner_bes_flags(env), ["--bes_backend=grpc://bes:1985", "--bes_results_url=https://ui/i/"]) + _eq("no viewer", runner_bes_flags(WorkflowsEnvironment(build_events = BuildEvents(backend = "grpc://bes:1985"))), ["--bes_backend=grpc://bes:1985"]) + _eq("no BES", runner_bes_flags(WorkflowsEnvironment(remote_cache = RemoteCache(endpoint = "x"))), []) + + # Not part of `runner_flags`: tasks stream BES from the CLI, so only the rc + # takes these. + _startup, build = runner_flags(env, "/x") + _lacks("runner_flags omits bes", build, "--bes_backend") + +def _test_runner_executor_flags(_ctx): + # Opt-in only: never in runner_flags or env_flags. + with_cache = WorkflowsEnvironment(runner = Runner(), remote_cache = RemoteCache(endpoint = "grpc://c"), remote_executor = RemoteExecutor(endpoint = "grpc://e")) + _eq("executor only, cache tuning already unconditional", runner_executor_flags(with_cache), ["--remote_executor=grpc://e"]) + _startup, build = runner_flags(with_cache, "/x") + _lacks("runner_flags omits executor", build, "--remote_executor") + _lacks("env_flags omits executor", env_flags(with_cache), "--remote_executor") + + # Without a separate cache the executor doubles as one and earns its tuning. + exec_only = WorkflowsEnvironment(remote_executor = RemoteExecutor(endpoint = "grpc://e")) + flags = runner_executor_flags(exec_only) + _has("executor", flags, "--remote_executor=grpc://e") + _has("cache tuning via executor", flags, "--remote_accept_cached") + _has("keepalive", flags, "--grpc_keepalive_timeout=30s") + _eq("no executor → nothing", runner_executor_flags(WorkflowsEnvironment(runner = Runner())), []) + _eq("endpoint alone", runner_executor_endpoint(exec_only), ["--remote_executor=grpc://e"]) + _eq("no endpoint", runner_executor_endpoint(WorkflowsEnvironment()), []) + +def _test_color_flags(_ctx): + _eq("misdetected", color_flags(True), ["--color=yes"]) + _eq("detected fine", color_flags(False), []) + + # Forced only where a viewer renders ANSI but Bazel sees no TTY. Buildkite + # gives the job a PTY, so Bazel colors unaided and it is left out. + def env(**vars): + return struct(var = lambda k: vars.get(k, "")) + + _eq("github", bazel_misdetects_color(env(GITHUB_ACTIONS = "true")), True) + _eq("circleci", bazel_misdetects_color(env(CIRCLECI = "true")), True) + _eq("gitlab", bazel_misdetects_color(env(GITLAB_CI = "true")), True) + _eq("buildkite", bazel_misdetects_color(env(BUILDKITE = "true")), False) + _eq("plain CI", bazel_misdetects_color(env(CI = "1")), False) + +def _test_diagnostics_upload_flags(_ctx): + # Shared by the runner diagnostics, the rc's BES sections, and `--remote`. + _eq("upload switch", diagnostics_upload_flags(), [("--remote_build_event_upload=all", ">=7.0.0")]) + _eq("exec log, build-scoped", exec_log_flags("/x/e.zstd"), [("--execution_log_compact_file=/x/e.zstd", ">=7.1.0", "build")]) + _eq("scope stripped for the rc", unscoped(exec_log_flags("/x/e.zstd")[0]), ("--execution_log_compact_file=/x/e.zstd", ">=7.1.0")) + _eq("unscoped passthrough", [unscoped(f) for f in ["--a", ("--b", ">=1.0.0"), ("--c", "", "build")]], ["--a", ("--b", ">=1.0.0"), "--c"]) + + # Through the bazelrc runtime the scope holds: query never sees the + # execution log, test does (the cache-diff task queries with the task's rc). + rc = _ctx.bazel.new_rc(flags = diagnostics_upload_flags() + exec_log_flags("/x/e.zstd"), version = "8.0.0") + _eq("query", _texts(rc.expand(command = "query")), ["--remote_build_event_upload=all"]) + _eq("test", sorted(_texts(rc.expand(command = "test"))), ["--execution_log_compact_file=/x/e.zstd", "--remote_build_event_upload=all"]) + +def _test_omit_flags(_ctx): + flags = ["--heap_dump_on_oom", "--remote_timeout=3600", ("--noremote_cache_compression", ">=8.0.0"), "--remote_cache=grpcs://c"] + + # Matched by name, with or without a value on either side; tuples included. + got = omit_flags(flags, ["--heap_dump_on_oom", "--remote_timeout=999", "--noremote_cache_compression"]) + _eq("omitted", got, ["--remote_cache=grpcs://c"]) + _eq("untouched when nothing omitted", omit_flags(flags, []), flags) + _eq("unknown name is a no-op", omit_flags(flags, ["--not_ours"]), flags) + + # Names compare exactly as Aspect spells them: the `no` form is its own flag. + _eq("no-form is distinct", len(omit_flags(["--noslim_profile"], ["--slim_profile"])), 1) + +def _test_protected_omissions(_ctx): + _eq("allowed", protected_omissions(["--heap_dump_on_oom", "--remote_timeout"]), []) + _eq("blocked, as spelled", protected_omissions(["--heap_dump_on_oom", "--remote_cache=x", "--output_base"]), ["--remote_cache=x", "--output_base"]) + + # Every endpoint, credential, chain, and runner-path flag the generators can + # write is protected — otherwise an opt-out could point a build nowhere. + for name in ["--remote_cache", "--remote_executor", "--credential_helper", "--experimental_credential_helper", "--config", "--output_base", "--output_user_root", "--repository_cache"]: + _eq("protected " + name, name in PROTECTED_FLAGS, True) + + # The runner rc's BES pair stays omittable: a repository that streams its + # vanilla builds elsewhere drops it with --omit-bazel-flag=--bes_backend. + _eq("bes not protected", protected_omissions(["--bes_backend", "--bes_results_url"]), []) + _eq("flag_name strips value", flag_name("--remote_cache=grpcs://c"), "--remote_cache") + +_UNIT_TESTS = [ + _test_endpoint_flags_follow_endpoints, + _test_env_flags_follow_the_environment, + _test_env_task_flags, + _test_runner_flags, + _test_runner_flags_output_base, + _test_runner_pieces, + _test_runner_diagnostics_flags, + _test_runner_bes_flags, + _test_runner_executor_flags, + _test_color_flags, + _test_diagnostics_upload_flags, + _test_omit_flags, + _test_protected_omissions, +] + +def _test_impl(ctx): + for t in _UNIT_TESTS: + t(ctx) + print("aspect_flags.axl: OK (%d tests)" % len(_UNIT_TESTS)) + return 0 + +aspect_flags_tests = task( + summary = "Run the aspect_flags AXL unit tests.", + kind = "test-aspect-flags", + group = ["dev"], + implementation = _test_impl, + args = {}, +) diff --git a/crates/aspect-cli/src/builtins/aspect/private/lib/bazelrc_file.axl b/crates/aspect-cli/src/builtins/aspect/private/lib/bazelrc_file.axl new file mode 100644 index 000000000..c5f27745c --- /dev/null +++ b/crates/aspect-cli/src/builtins/aspect/private/lib/bazelrc_file.axl @@ -0,0 +1,149 @@ +"""The rc file `aspect setup bazelrc` writes, as a document: how flags render into +sections, how the file announces itself, and how it is placed in another rc — +the workspace `.bazelrc` — through a commented `try-import` line. + +Nothing here decides *which* flags are written — `aspect_flags.axl`, +`rc_groups.axl`, and `deployment_rc.axl` do — only how startup flags, enabled +groups, and `RcConfig` sections become text, and how that text lands without +disturbing what a user or another setup step already put in their rc. + +On Bazel >= 6.3.0 flags are emitted under `common` (applies to every command); +older Bazel has no `common` pseudo-command, so they're spelled out under `build` +(inherited by test/run/coverage/cquery/aquery/fetch/…) and `query` (the one +action-relevant command that doesn't inherit `build`). Every flag this file is +handed ungated is accepted by `query` on Bazel < 6.3.0, the only range that +branch runs in; flags later Bazel rejects on `query` are version-gated past it. +""" + +load("@aspect//bazel/build_metadata.axl", "version_gte") +load("./aspect_flags.axl", "flag_text") + +# Bazel gained the `common` pseudo-command in 6.3.0. +_COMMON_PSEUDO_COMMAND_MIN_VERSION = "6.3.0" +_PRE_COMMON_COMMANDS = ["build", "query"] + +# First line of the generated rc. A checkout's rc is a starting point its +# repository owns and edits; a runner's is rewritten every job. +GENERATED_HEADER = "# Generated by `aspect setup bazelrc` as a starting point: edit as your repository needs; rerun the command to restore it." +RUNNER_HEADER = "# Generated by `aspect setup bazelrc` for this runner — rewritten every job; opt out of a flag with --omit-bazel-flag." + +# One `--config` section of the rc. `flags` are `str | (str, constraint)` like +# everywhere else; a plain `--config=` entry chains to another section and +# is written as-is rather than version-resolved. `source` names what claims the +# section, for the collision error in `deployment_rc.axl`. +RcConfig = record( + name = field(str), + source = field(str), + comment = field(str, default = ""), + flags = field(list), +) + +# The comment written above the `try-import` line in the importing rc, so the +# import reads as ours next to whatever else the file holds. +IMPORT_COMMENT = "# Aspect's rc for vanilla `bazel`, maintained by `aspect setup bazelrc`." + +# What `ensure_import` did to the user rc. +ImportAction = enum("created", "unchanged", "prepended") + +ImportResult = record( + content = field(str), + action = field(ImportAction), +) + +def is_config_chain(flag) -> bool: + """Whether `flag` chains to another section (`--config=`), which is + written verbatim rather than version-resolved.""" + return type(flag) == "string" and flag.startswith("--config=") + +def startup_flags_for_rc(startup_flags: list) -> list: + """Drop the rc-suppression startup flags, which are illegal inside an rc file. + + `runner_startup_flags` includes `--nohome_rc` and `--nosystem_rc` because the + Workflows feature passes them as *command-line* startup flags to + `aspect `. Bazel rejects any `--no*_rc` option inside a bazelrc with a + fatal `Can't specify --nosystem_rc in the .bazelrc file` (`--nohome_rc` would + also be self-defeating — Bazel would be told to ignore the file importing + this one). They only suppress *other* rc files, a command-line concern the + generated rc cannot express, so they are dropped here.""" + _RC_SUPPRESSION_FLAGS = ["--nohome_rc", "--nosystem_rc", "--noworkspace_rc"] + return [f for f in startup_flags if f not in _RC_SUPPRESSION_FLAGS] + +def command_sections(bazel_version: str | None) -> list: + """The rc command sections build flags go under: `common` on Bazel >= 6.3.0 or + when the version is unknown (assume latest); `build` and `query` on older + Bazel that has no `common` pseudo-command.""" + if bazel_version == None or version_gte(bazel_version, _COMMON_PSEUDO_COMMAND_MIN_VERSION): + return ["common"] + return _PRE_COMMON_COMMANDS + +def render_bazelrc(startup_flags: list, enabled: list, command_sections: list, configs: list, header: str = GENERATED_HEADER) -> str: + """Render the rc: `header`, a `startup` line per startup flag, a + `
--config=` line per group in `enabled`, then each + `RcConfig` in `configs` as a commented `
:` group. Every + section is written once per entry in `command_sections` (`["common"]` on + modern Bazel, `["build", "query"]` on Bazel < 6.3.0). Flags may be + `(flag, condition)` tuples (the shape `rc.expand` yields for surviving + version-gated options); only the flag text is written.""" + lines = [ + header, + "# Configures vanilla `bazel` calls to use Aspect services and recommended optimizations.", + "# Flags are grouped by what they presume; the groups that apply here are enabled at the", + "# top, the rest reached by chaining or by name. `bazel build --announce_rc` shows which group set what.", + "", + ] + for flag in startup_flags: + lines.append("startup " + flag_text(flag)) + for group in enabled: + for section in command_sections: + lines.append(section + " --config=" + group) + for config in configs: + lines.append("") + if config.comment: + lines.append("# " + config.comment) + for flag in config.flags: + for section in command_sections: + lines.append(section + ":" + config.name + " " + flag_text(flag)) + return "\n".join(lines) + "\n" + +def prune_configs(configs: list, enabled: list) -> (list, list): + """`configs` and `enabled` without the sections left empty by the user's + omissions, and without the `--config=` chains and enable lines that + named them: Bazel fails on a `--config` no rc defines. Dropping a section can + empty one that only chained to it, so this runs to a fixed point.""" + for _ in range(len(configs) + 1): + dropped = [c.name for c in configs if not c.flags] + if not dropped: + break + configs = [ + RcConfig(name = c.name, source = c.source, comment = c.comment, flags = [f for f in c.flags if not (is_config_chain(f) and f[len("--config="):] in dropped)]) + for c in configs + if c.flags + ] + enabled = [name for name in enabled if name not in dropped] + return (configs, enabled) + +def import_line(generated_rc: str) -> str: + """The line that pulls `generated_rc` into another rc. `try-import` rather + than `import` so a generated file that later disappears degrades to no + Aspect configuration instead of failing every `bazel` call. Bazel resolves + the path as written except for a leading `%workspace%`, its one rc variable, + so the caller passes `%workspace%/…` for a file in the checkout and an + absolute path otherwise; a bare relative path would resolve against Bazel's + working directory.""" + return "try-import " + generated_rc + +def ensure_import(existing: str | None, line: str) -> ImportResult: + """The rc content with `line` present exactly once: unchanged when it is + already there, otherwise placed at the top under `IMPORT_COMMENT`, a blank + line apart from whatever the file already holds. At the top so the file's + own lines come after the imported ones and win where they disagree — Bazel + takes the last value of a flag — which is how a repository overrides a + recommendation, say `--noremote_upload_local_results` for local work. + `None`/empty `existing` is a new file.""" + block = IMPORT_COMMENT + "\n" + line + "\n" + if not existing: + return ImportResult(content = block, action = ImportAction("created")) + if line in existing.split("\n"): + return ImportResult(content = existing, action = ImportAction("unchanged")) + content = block + ("\n" + existing.lstrip("\n") if existing.strip() else "") + return ImportResult(content = content, action = ImportAction("prepended")) diff --git a/crates/aspect-cli/src/builtins/aspect/private/lib/bazelrc_file_test.axl b/crates/aspect-cli/src/builtins/aspect/private/lib/bazelrc_file_test.axl new file mode 100644 index 000000000..626a517c5 --- /dev/null +++ b/crates/aspect-cli/src/builtins/aspect/private/lib/bazelrc_file_test.axl @@ -0,0 +1,149 @@ +"""Unit tests for `bazelrc_file.axl`: `command_sections`, `render_bazelrc`, +`prune_configs`, `startup_flags_for_rc`, `import_line`, and `ensure_import` (the +one-time `try-import` block). +Run with: aspect dev test-bazelrc-file +""" + +load( + "./bazelrc_file.axl", + "GENERATED_HEADER", + "IMPORT_COMMENT", + "ImportAction", + "RcConfig", + "command_sections", + "ensure_import", + "import_line", + "prune_configs", + "render_bazelrc", + "startup_flags_for_rc", +) + +def _eq(label, got, want): + if got != want: + fail("%s: got %r, want %r" % (label, got, want)) + +_SECTION = RcConfig(name = "aspect-x", source = "x", comment = "x: remote cache.", flags = ["--remote_cache=grpcs://x", ("--credential_helper=x=h", ">=6.4.0")]) +_CHAIN = RcConfig(name = "aspect-x-exec", source = "x exec", flags = ["--config=aspect-x", "--remote_executor=grpcs://x"]) + +def _test_startup_flags_for_rc(_ctx): + # The `--no*_rc` suppression flags must all be dropped: Bazel fatals on any of + # them inside a bazelrc. Output paths stay. + got = startup_flags_for_rc(["--nohome_rc", "--nosystem_rc", "--output_user_root=/b/x", "--output_base=/o/x"]) + _eq("suppression dropped, paths kept", got, ["--output_user_root=/b/x", "--output_base=/o/x"]) + +def _test_command_sections(_ctx): + # >= 6.3.0 and the unknown-version (assume latest) case use `common`. + for v in [None, "6.3.0", "7.4.1", "8.0.0"]: + _eq("%s -> common" % v, command_sections(v), ["common"]) + + # < 6.3.0 has no `common`; spell out build (inherited) + query (not inherited). + for v in ["6.2.1", "6.0.0", "5.4.0"]: + _eq("%s -> build/query" % v, command_sections(v), ["build", "query"]) + +_COMMON = RcConfig(name = "aspect-common", source = "common", comment = "Common tuning.", flags = ["--heap_dump_on_oom"]) + +def _test_render_common_section(_ctx): + out = render_bazelrc(["--output_base=/mnt/output"], ["aspect-common"], ["common"], [_COMMON]) + lines = out.split("\n") + _eq("header first", lines[0], GENERATED_HEADER) + _eq("startup line", "startup --output_base=/mnt/output" in lines, True) + _eq("enable line", "common --config=aspect-common" in lines, True) + _eq("group flag", "common:aspect-common --heap_dump_on_oom" in lines, True) + _eq("enable before definition", lines.index("common --config=aspect-common") < lines.index("common:aspect-common --heap_dump_on_oom"), True) + _eq("no unscoped flag", "common --heap_dump_on_oom" in lines, False) + _eq("no common startup_flag", "common --output_base=/mnt/output" in lines, False) + _eq("trailing newline", out.endswith("\n"), True) + +def _test_render_pre_common_sections(_ctx): + # < 6.3.0: each enable and group line is emitted once per section (build + + # query); startup flags are emitted once regardless of the command sections. + lines = render_bazelrc(["--output_base=/o"], ["aspect-common"], ["build", "query"], [_COMMON]).split("\n") + _eq("build enable", "build --config=aspect-common" in lines, True) + _eq("query enable", "query --config=aspect-common" in lines, True) + _eq("build group", "build:aspect-common --heap_dump_on_oom" in lines, True) + _eq("query group", "query:aspect-common --heap_dump_on_oom" in lines, True) + _eq("no common", any([ln.startswith("common") for ln in lines]), False) + _eq("startup once", len([ln for ln in lines if ln == "startup --output_base=/o"]), 1) + +def _test_render_configs(_ctx): + # Config flags go under `
:`, tuples unwrapped, chain lines + # verbatim, each group behind its comment; an unenabled group is defined only. + lines = render_bazelrc([], [], ["common"], [_SECTION, _CHAIN]).split("\n") + _eq("section flag", "common:aspect-x --remote_cache=grpcs://x" in lines, True) + _eq("tuple unwrapped", "common:aspect-x --credential_helper=x=h" in lines, True) + _eq("chain verbatim", "common:aspect-x-exec --config=aspect-x" in lines, True) + _eq("comment", "# x: remote cache." in lines, True) + _eq("nothing enabled", any([ln.startswith("common --config=") for ln in lines]), False) + _eq("no condition leaked", any([">=" in ln for ln in lines]), False) + +def _test_prune_configs(_ctx): + cache = RcConfig(name = "aspect-cache", source = "g", flags = []) + exec = RcConfig(name = "aspect-exec", source = "e", flags = ["--config=aspect-cache", "--remote_executor=grpcs://e"]) + alias = RcConfig(name = "aspect", source = "a", flags = ["--config=aspect-cache"]) + (configs, enabled) = prune_configs([_COMMON, cache, exec, alias], ["aspect-common", "aspect-cache"]) + + # The emptied group goes, with its enable line and every chain to it; a + # section that only chained to it empties in turn and goes too. + _eq("survivors", [c.name for c in configs], ["aspect-common", "aspect-exec"]) + _eq("chain dropped", configs[1].flags, ["--remote_executor=grpcs://e"]) + _eq("enable dropped", enabled, ["aspect-common"]) + _eq("nothing to prune", prune_configs([_COMMON], ["aspect-common"]), ([_COMMON], ["aspect-common"])) + +_LINE = import_line("%workspace%/.aspect/bazelrc") +_BLOCK = IMPORT_COMMENT + "\n" + _LINE + "\n" + +def _ensure(existing): + return ensure_import(existing, _LINE) + +def _test_import_line(_ctx): + # `try-import`, so a generated file that later disappears is not fatal. + _eq("try-import", _LINE, "try-import %workspace%/.aspect/bazelrc") + _eq("explicit path as given", import_line("/srv/ci/aspect.bazelrc"), "try-import /srv/ci/aspect.bazelrc") + +def _test_ensure_import_new_file(_ctx): + got = _ensure(None) + _eq("created content", got.content, _BLOCK) + _eq("created", got.action, ImportAction("created")) + _eq("empty is new", _ensure("").action, ImportAction("created")) + +def _test_ensure_import_appends_once(_ctx): + # A user's rc keeps every line; ours is added after them, and a second run + # finds it and changes nothing — so re-running never accumulates lines. + first = _ensure("build --disk_cache=/tmp/dc\n") + + # At the top, so the file's own lines come after and override the import. + _eq("prepended a blank line apart, under the comment", first.content, _BLOCK + "\nbuild --disk_cache=/tmp/dc\n") + _eq("prepended", first.action, ImportAction("prepended")) + _eq("no trailing newline", _ensure("build --x").content, _BLOCK + "\nbuild --x") + _eq("leading blank lines folded", _ensure("\n\nbuild --x\n").content, _BLOCK + "\nbuild --x\n") + + second = _ensure(first.content) + _eq("unchanged content", second.content, first.content) + _eq("unchanged", second.action, ImportAction("unchanged")) + _eq("mid-file", _ensure(_LINE + "\nbuild --later\n").action, ImportAction("unchanged")) + +_UNIT_TESTS = [ + _test_startup_flags_for_rc, + _test_command_sections, + _test_render_common_section, + _test_render_pre_common_sections, + _test_render_configs, + _test_prune_configs, + _test_import_line, + _test_ensure_import_new_file, + _test_ensure_import_appends_once, +] + +def _test_impl(ctx): + for t in _UNIT_TESTS: + t(ctx) + print("bazelrc_file.axl: OK (%d tests)" % len(_UNIT_TESTS)) + return 0 + +bazelrc_file_tests = task( + summary = "Run the bazelrc_file AXL unit tests.", + kind = "test-bazelrc-file", + group = ["dev"], + implementation = _test_impl, + args = {}, +) diff --git a/crates/aspect-cli/src/builtins/aspect/private/lib/bazelrc_test.axl b/crates/aspect-cli/src/builtins/aspect/private/lib/bazelrc_test.axl deleted file mode 100644 index 20239b0a5..000000000 --- a/crates/aspect-cli/src/builtins/aspect/private/lib/bazelrc_test.axl +++ /dev/null @@ -1,210 +0,0 @@ -"""Unit tests for the pure bazelrc/output-base flag helpers: - - - `bazelrc.axl` rendering: `command_sections` (which rc command sections to - use for a Bazel version), `render_bazelrc` (rc text), `startup_flags_for_rc`. - - `environment.axl` flags: `get_generic_bazelrc_flags` / `get_bazelrc_flags` - output, and `apply_output_base_suffix`. - -Version-gating of the `(flag, constraint)` tuples from `get_bazelrc_flags` is -delegated to the bazelrc runtime (`ctx.bazel.new_rc(...).expand(...)`), so it's -covered by the runtime's own tests rather than here. Run with: -aspect dev test-bazelrc -""" - -load("@aspect//bazelrc.axl", "command_sections", "render_bazelrc", "startup_flags_for_rc") -load( - "./environment.axl", - "CI", - "RemoteCache", - "Runner", - "WorkflowsEnvironment", - "apply_output_base_suffix", - "get_bazelrc_flags", - "get_generic_bazelrc_flags", -) - -def _eq(label, got, want): - if got != want: - fail("%s: got %r, want %r" % (label, got, want)) - -# Runner-specific flag fragments that must NEVER appear in the generic set — -# they depend on a runner's mounts, identity, or the Workflows cache. -_RUNNER_ONLY_FRAGMENTS = [ - "--repository_cache=", - "--output_base=", - "--output_user_root=", - "--remote_header=", - "--nosystem_rc", - "_remote_cache_compression", -] - -def _test_generic_flags_are_runner_independent(_ctx): - flags = get_generic_bazelrc_flags(WorkflowsEnvironment()) - - # The generic profiling/remote tuning flags are present... - for want in ["--remote_upload_local_results", "--disk_cache=", "--remote_timeout=3600"]: - _eq("generic has " + want, want in flags, True) - - # ...and nothing runner-specific leaks in. - for frag in _RUNNER_ONLY_FRAGMENTS: - _eq("no runner-only " + frag, any([frag in f for f in flags]), False) - - # No remote-cache endpoint without the env (no RemoteCache on the environment). - _eq("no remote_cache", any(["--remote_cache=" in f for f in flags]), False) - -def _test_generic_flags_include_remote_cache_env(_ctx): - # When the remote-cache env vars are set (surfaced as environment.remote_cache), - # the endpoint flags are emitted — even off a runner. - env = WorkflowsEnvironment(remote_cache = RemoteCache( - endpoint = "unix:///x/grpc", - bytestream_uri_prefix = "aw-remote.example", - )) - flags = get_generic_bazelrc_flags(env) - _eq("remote_cache", "--remote_cache=unix:///x/grpc" in flags, True) - _eq("bytestream prefix", "--remote_bytestream_uri_prefix=aw-remote.example" in flags, True) - -def _test_startup_flags_for_rc(_ctx): - # The `--no*_rc` suppression flags must all be dropped: Bazel fatals on any of - # them appearing inside a bazelrc ("Can't specify --nosystem_rc in the - # .bazelrc file"), and the generated file IS the home rc. Output paths stay. - got = startup_flags_for_rc([ - "--nohome_rc", - "--nosystem_rc", - "--output_user_root=/mnt/ephemeral/bazel/x", - "--output_base=/mnt/ephemeral/output/x", - ]) - _eq("nohome_rc dropped", "--nohome_rc" in got, False) - _eq("nosystem_rc dropped", "--nosystem_rc" in got, False) - _eq("output_user_root kept", "--output_user_root=/mnt/ephemeral/bazel/x" in got, True) - _eq("output_base kept", "--output_base=/mnt/ephemeral/output/x" in got, True) - -def _test_command_sections(_ctx): - # >= 6.3.0 and the unknown-version (assume latest) case use `common`. - _eq("unknown -> common", command_sections(None), ["common"]) - _eq("6.3.0 -> common", command_sections("6.3.0"), ["common"]) - _eq("7.4.1 -> common", command_sections("7.4.1"), ["common"]) - _eq("8.0.0 -> common", command_sections("8.0.0"), ["common"]) - - # < 6.3.0 has no `common`; spell out build (inherited) + query (not inherited). - _eq("6.2.1 -> build/query", command_sections("6.2.1"), ["build", "query"]) - _eq("6.0.0 -> build/query", command_sections("6.0.0"), ["build", "query"]) - _eq("5.4.0 -> build/query", command_sections("5.4.0"), ["build", "query"]) - -def _test_render_common_section(_ctx): - out = render_bazelrc( - ["--nohome_rc", "--output_base=/mnt/output"], - ["--remote_cache=grpcs://c:443", "--disk_cache="], - ["common"], - ) - lines = out.split("\n") - - # Startup flags get a `startup ` prefix; build flags get a `common ` prefix. - _eq("startup line", "startup --nohome_rc" in lines, True) - _eq("startup output_base", "startup --output_base=/mnt/output" in lines, True) - _eq("common remote_cache", "common --remote_cache=grpcs://c:443" in lines, True) - _eq("common disk_cache", "common --disk_cache=" in lines, True) - - # No build flag leaks into a startup line and vice versa. - _eq("no common startup_flag", "common --nohome_rc" in lines, False) - _eq("no startup build_flag", "startup --remote_cache=grpcs://c:443" in lines, False) - - # Trailing newline so the file ends cleanly. - _eq("trailing newline", out.endswith("\n"), True) - -def _test_render_pre_common_sections(_ctx): - # < 6.3.0: each build flag is emitted once per section (build + query); - # startup flags are emitted once regardless of the command sections. - out = render_bazelrc( - ["--nohome_rc"], - ["--remote_cache=grpcs://c:443"], - ["build", "query"], - ) - lines = out.split("\n") - - _eq("build section", "build --remote_cache=grpcs://c:443" in lines, True) - _eq("query section", "query --remote_cache=grpcs://c:443" in lines, True) - _eq("no common", "common --remote_cache=grpcs://c:443" in lines, False) - - # startup flag appears exactly once (not multiplied by the command sections). - _eq("startup once", len([ln for ln in lines if ln == "startup --nohome_rc"]), 1) - -def _test_render_unwraps_version_gated_tuple(_ctx): - # `rc.expand` yields surviving version-gated options as (flag, condition) - # tuples; render writes only the flag text. - out = render_bazelrc([], [("--noremote_cache_compression", ">=8.0.0"), "--disk_cache="], ["common"]) - lines = out.split("\n") - _eq("tuple flag unwrapped", "common --noremote_cache_compression" in lines, True) - _eq("no condition leaked", any([">=8.0.0" in ln for ln in lines]), False) - -def _test_render_empty(_ctx): - # With no flags, the header is still written. - out = render_bazelrc([], [], ["common"]) - _eq("header present", out.startswith("# Generated by `aspect ci bazelrc`"), True) - -def _test_apply_output_base_suffix_only_output_base(_ctx): - # Only --output_base gains the suffix; --output_user_root (shared install - # base + external-repo cache) and other flags pass through unchanged. - got = apply_output_base_suffix( - ["--output_base=/o/repo/sub", "--output_user_root=/b/repo/sub", "--nohome_rc"], - "-delivery", - ) - _eq("output_base suffixed", "--output_base=/o/repo/sub-delivery" in got, True) - _eq("output_user_root untouched", "--output_user_root=/b/repo/sub" in got, True) - _eq("nohome_rc untouched", "--nohome_rc" in got, True) - _eq("no unsuffixed output_base", "--output_base=/o/repo/sub" in got, False) - -def _test_apply_output_base_suffix_empty_noop(_ctx): - # Empty suffix returns the list unchanged. - flags = ["--output_base=/o/x", "--output_user_root=/b/x"] - _eq("empty suffix noop", apply_output_base_suffix(flags, ""), flags) - -def _test_apply_output_base_suffix_no_output_base_noop(_ctx): - # Off-runner shape: no --output_base, so nothing to suffix. - flags = ["--nohome_rc", "--nosystem_rc"] - _eq("no-output-base noop", apply_output_base_suffix(flags, "-delivery"), flags) - -def _test_get_bazelrc_flags_output_base_and_suffix(_ctx): - # Direct assertion on the runner-derived output base, then confirm the - # suffix helper moves --output_base only. subdir = last path segment of root. - env = WorkflowsEnvironment( - runner = Runner(bazel_root_dir = "/b", output_root_dir = "/o"), - ci = CI(scm_repo_name = "my repo"), - ) - startup, _build = get_bazelrc_flags(env, "/work/aspect-cli") - - # repo name "my repo" sanitizes to "my_repo"; subdir "aspect-cli" is clean. - _eq("output_base", "--output_base=/o/my_repo/aspect-cli" in startup, True) - _eq("output_user_root", "--output_user_root=/b/my_repo/aspect-cli" in startup, True) - - suffixed = apply_output_base_suffix(startup, "-delivery") - _eq("output_base moved", "--output_base=/o/my_repo/aspect-cli-delivery" in suffixed, True) - _eq("output_user_root shared", "--output_user_root=/b/my_repo/aspect-cli" in suffixed, True) - -_UNIT_TESTS = [ - _test_generic_flags_are_runner_independent, - _test_generic_flags_include_remote_cache_env, - _test_startup_flags_for_rc, - _test_command_sections, - _test_render_common_section, - _test_render_pre_common_sections, - _test_render_unwraps_version_gated_tuple, - _test_render_empty, - _test_apply_output_base_suffix_only_output_base, - _test_apply_output_base_suffix_empty_noop, - _test_apply_output_base_suffix_no_output_base_noop, - _test_get_bazelrc_flags_output_base_and_suffix, -] - -def _test_impl(ctx): - for t in _UNIT_TESTS: - t(ctx) - print("bazelrc.axl: OK (%d tests)" % len(_UNIT_TESTS)) - return 0 - -bazelrc_tests = task( - summary = "Run the bazelrc AXL unit tests.", - kind = "test-bazelrc", - group = ["dev"], - implementation = _test_impl, - args = {}, -) diff --git a/crates/aspect-cli/src/builtins/aspect/private/lib/deployment_flags.axl b/crates/aspect-cli/src/builtins/aspect/private/lib/deployment_flags.axl index 239a41831..1d5f28488 100644 --- a/crates/aspect-cli/src/builtins/aspect/private/lib/deployment_flags.axl +++ b/crates/aspect-cli/src/builtins/aspect/private/lib/deployment_flags.axl @@ -55,6 +55,10 @@ Failures: - An unrecognized capability token, a value naming no capabilities at all, or `none` combined with anything else. - `--deployment` with no `--remote` (nothing to wire). + - `--remote` where the `ASPECT_WORKFLOWS_*` environment already names this + machine's deployment (a Workflows runner, or the endpoint variables set by + hand): the Workflows feature wires that deployment, and a second one would + route the build off it. `REMOTE_REFUSED_BY_ENV` is the message. - `--remote` with no deployment to draw from — no `--deployment` and no default Aspect Workflows deployment configured (the target resolves to Aspect Cloud before discovery has recorded its endpoints). @@ -66,11 +70,26 @@ Failures: Auth attaches automatically: the injected hosts are the deployment's own, so they match the endpoint-auth host gate and receive its login JWT (see `aspect_endpoint_auth.axl`). + +The wired endpoints also bring the `aspect_flags.axl` tuning they justify (cache +tuning, BES enrichment, the keepalive) and, with BES, the build-diagnostics +upload and the `--build_metadata` the Web UI reads — the same flags the +`--config=aspect-*` sections of `aspect setup bazelrc` carry, so `aspect build +--remote` and `bazel build --config=aspect-` run alike. The user's +`--bazel-flags:omit` applies to them like any injected flag. """ +load("@aspect//bazel/build_metadata.axl", "register_bes_metadata") load("@aspect//bazel.axl", "BazelTrait") load("./aspect_endpoint_auth.axl", "credential_helper_covers_host", "endpoint_host", "headers_have_auth", "needs_aspect_token", "resolve_aspect_bearer", "same_bes_endpoint") -load("./environment.axl", "info") +load("./aspect_flags.axl", "endpoint_flags", "independent_flags", "temp_diagnostics_upload_flags") +load("./environment.axl", "env_names_deployment", "info") + +REMOTE_REFUSED_BY_ENV = ( + "--remote would wire a configured deployment, but the ASPECT_WORKFLOWS_* environment already names this machine's deployment, " + + "which every task here builds against without --remote. Unset ASPECT_WORKFLOWS_RUNNER and the ASPECT_WORKFLOWS_REMOTE_CACHE / " + + "BES_BACKEND / REMOTE_EXECUTOR variables to build against a configured deployment instead." +) # Bazel-facing Aspect endpoints are TLS gRPC. _SCHEME = "grpcs://" @@ -166,6 +185,8 @@ DeploymentFlags = record( capabilities = field(list, default = []), results_url = field(str, default = ""), bes_uri = field(str, default = ""), + # The capability names actually wired, for the tuning flags they justify. + wired = field(list, default = []), ) # Which capabilities `--remote` resolved to: `on` maps every capability in @@ -279,6 +300,9 @@ def deployment_endpoint_flags(ctx) -> DeploymentFlags: if not any(spec.on.values()): return DeploymentFlags(base_flags = [], bes_backends = []) + if env_names_deployment(ctx.std.env): + ctx.std.process.exit(1, REMOTE_REFUSED_BY_ENV) + endpoints = ctx.aspect.auth.deployment_endpoints(deployment = ctx.args.deployment) # --remote needs a deployment to draw endpoints from. With no --deployment the @@ -295,6 +319,7 @@ def deployment_endpoint_flags(ctx) -> DeploymentFlags: base_flags = [] bes_backends = [] capabilities = [] + wired = [] for (name, capability) in _CAPABILITY_SPECS.items(): if not spec.on[name]: continue @@ -314,6 +339,7 @@ def deployment_endpoint_flags(ctx) -> DeploymentFlags: bes_backends.append(uri) if capability.announced: capabilities.append((capability.label, uri)) + wired.append(name) return DeploymentFlags( base_flags = base_flags, @@ -322,6 +348,20 @@ def deployment_endpoint_flags(ctx) -> DeploymentFlags: capabilities = capabilities, results_url = endpoints.results_url, bes_uri = _endpoint_uri(endpoints, _CAPABILITY_SPECS["bes"]), + wired = wired, + ) + +def deployment_tuning_flags(flags: DeploymentFlags) -> list: + """The `aspect_flags.axl` flags the wired capabilities justify: the + endpoint-independent ones plus cache tuning, BES enrichment, and the + keepalive as applicable. Empty when nothing was wired. Public so + `deployment_flags_test.axl` can assert the pairing.""" + if not flags.wired: + return [] + return independent_flags() + endpoint_flags( + cache = "cache" in flags.wired, + bes = "bes" in flags.wired, + exec = "exec" in flags.wired, ) def _advertised_bes_sources(ctx, flags: DeploymentFlags) -> list[(str, str)]: @@ -428,6 +468,10 @@ def _deployment_impl(ctx): t.bes_backends.extend(dep.bes_backends) t.bes_results_sources.extend(_advertised_bes_sources(ctx, dep)) t.rc_flags.append(deployment_auth_flags) + t.extra_flags.extend(deployment_tuning_flags(dep)) + if "bes" in dep.wired: + register_bes_metadata(ctx, t) + t.task_flags.append(temp_diagnostics_upload_flags) if dep.capabilities: t.build_start.append(lambda tctx: announce_deployment_flags(tctx, dep)) diff --git a/crates/aspect-cli/src/builtins/aspect/private/lib/deployment_flags_test.axl b/crates/aspect-cli/src/builtins/aspect/private/lib/deployment_flags_test.axl index 01449b653..717be2c0f 100644 --- a/crates/aspect-cli/src/builtins/aspect/private/lib/deployment_flags_test.axl +++ b/crates/aspect-cli/src/builtins/aspect/private/lib/deployment_flags_test.axl @@ -15,7 +15,7 @@ when Bazel is the uploader (the CLI-streamed path announces the URL itself; see `bes_sinks.axl::announce_bes_results_url`). """ -load("./deployment_flags.axl", "BARE", "REMOTE_DEFAULT_CAPS", "capability_labels", "deployment_announcement", "deployment_endpoint_flags", "join_and", "parse_remote") +load("./deployment_flags.axl", "BARE", "REMOTE_DEFAULT_CAPS", "REMOTE_REFUSED_BY_ENV", "capability_labels", "deployment_announcement", "deployment_endpoint_flags", "deployment_tuning_flags", "join_and", "parse_remote") def _eq(label, got, want): if got != want: @@ -24,16 +24,20 @@ def _eq(label, got, want): def _endpoints(name, cache = "", bes = "", exec = "", results_url = ""): return struct(name = name, cache = cache, bes = bes, exec = exec, results_url = results_url) -def _ctx(endpoints, deployment = "", remote = "", env = {}): +def _ctx(endpoints, deployment = "", remote = "", env = {}, exits = []): """A fake ctx: fixed deployment_endpoints, args carrying --deployment/--remote, - and an env map for the ASPECT_WORKFLOWS_BES_RESULTS_URL lookup. Pass + an env map for the ASPECT_WORKFLOWS_* lookups, and a `process.exit` that + records `(code, message)` into `exits` instead of unwinding. Pass `remote = BARE` for a valueless `--remote`, matching what clap substitutes.""" return struct( args = struct( deployment = deployment, remote = remote, ), - std = struct(env = struct(var = lambda name: env.get(name, ""))), + std = struct( + env = struct(var = lambda name: env.get(name, "")), + process = struct(exit = lambda code, message: exits.append((code, message))), + ), aspect = struct(auth = struct( deployment_endpoints = lambda deployment: endpoints, )), @@ -214,7 +218,38 @@ def _test_join_and(ctx): _eq("two (oxford)", join_and(["remote cache", "bes backend"], oxford = True), "remote cache, and bes backend") _eq("three", join_and(["remote cache", "remote execution", "bes backend"]), "remote cache, remote execution, and bes backend") +def _test_tuning_follows_wired_capabilities(ctx): + # `--remote` brings the same catalog tuning the rc's `--config` sections get, + # for exactly the capabilities it wired; nothing when nothing was wired. + _eq("nothing wired", deployment_tuning_flags(deployment_endpoint_flags(_ctx(_ALL))), []) + + both = deployment_tuning_flags(deployment_endpoint_flags(_ctx(_ALL, remote = BARE))) + texts = [f[0] if type(f) == "tuple" else f for f in both] + _eq("independent", "--heap_dump_on_oom" in texts, True) + _eq("cache tuning", "--remote_timeout=3600" in texts, True) + _eq("bes enrichment", "--noslim_profile" in texts, True) + + cache_only = [f[0] if type(f) == "tuple" else f for f in deployment_tuning_flags(deployment_endpoint_flags(_ctx(_ALL, remote = "no-bes")))] + _eq("no bes enrichment without bes", "--noslim_profile" in cache_only, False) + _eq("cache tuning stays", "--remote_accept_cached" in cache_only, True) + +def _test_remote_refused_where_env_names_deployment(_ctx_unused): + # A runner, or hand-set endpoint variables, already answer which deployment + # this machine is on; --remote would route the build off it and is refused. + for var in ["ASPECT_WORKFLOWS_RUNNER", "ASPECT_WORKFLOWS_REMOTE_CACHE", "ASPECT_WORKFLOWS_BES_BACKEND", "ASPECT_WORKFLOWS_REMOTE_EXECUTOR"]: + exits = [] + deployment_endpoint_flags(_ctx(_ALL, remote = BARE, env = {var: "x"}, exits = exits)) + _eq("refused with " + var, exits, [(1, REMOTE_REFUSED_BY_ENV)]) + + # Wiring nothing is fine anywhere, and a silent environment never refuses. + exits = [] + deployment_endpoint_flags(_ctx(_ALL, remote = "none", env = {"ASPECT_WORKFLOWS_RUNNER": "1"}, exits = exits)) + deployment_endpoint_flags(_ctx(_ALL, remote = BARE, env = {"GITHUB_ACTIONS": "true"}, exits = exits)) + _eq("no refusal", exits, []) + _UNIT_TESTS = [ + _test_remote_refused_where_env_names_deployment, + _test_tuning_follows_wired_capabilities, _test_parse_defaults, _test_parse_omitted_is_off, _test_parse_exec_is_additive, diff --git a/crates/aspect-cli/src/builtins/aspect/private/lib/deployment_rc.axl b/crates/aspect-cli/src/builtins/aspect/private/lib/deployment_rc.axl new file mode 100644 index 000000000..75226542b --- /dev/null +++ b/crates/aspect-cli/src/builtins/aspect/private/lib/deployment_rc.axl @@ -0,0 +1,168 @@ +"""The `--config=aspect-*` sections `aspect setup bazelrc` writes, one per Aspect +deployment this machine knows (`ctx.aspect.auth.deployments()`) that advertises +an endpoint: + + --config=aspect-cloud Aspect Cloud: remote cache + BES + --config=aspect- a configured Aspect Workflows deployment + --config=aspect--exec the same, plus its remote execution + +No section names "the default deployment": the rc is committed and shared, and +the default (`aspect auth use`) is one machine's setting that could change +under it. A deployment is chosen by its own name. + +Aspect Cloud's section takes its own name; a configured deployment `` +gets `aspect-`. A cache or BES gives a base section, an executor an +`-exec` section chained to it (standalone for an executor-only deployment), and +a deployment with neither gets nothing. + +Each section wires the endpoints the deployment advertises (`--remote_cache`, +`--bes_backend`, `--bes_results_url`, `--remote_executor`), authenticates +through the `aspect get` credential helper scoped to each distinct host, and +chains `rc_groups.axl` groups rather than repeating flags: first the groups the +rc would otherwise enable for every command (`aspect-common`, the CI host's) — +the rc that carries sections enables nothing, so it is inert until a section is +named — then the tuning groups its endpoints justify (`aspect-cache`, +`aspect-bes`). Remote execution is never part of the base section: it relocates +every action, so it is only ever reached by naming `-exec`, mirroring +`--remote=exec` on `aspect `. + +Two deployments can land on one section name (one named `cloud`, or `silo` +beside `silo-exec`); Bazel would merge such sections and route the build to +whichever host came last, so `config_collisions` reports them for the task to +refuse. +""" + +load("./aspect_endpoint_auth.axl", "endpoint_host") +load("./bazelrc_file.axl", "RcConfig") +load("./rc_groups.axl", "GROUP_BES", "GROUP_CACHE") + +# Bazel-facing Aspect endpoints are TLS gRPC. +_SCHEME = "grpcs://" + +# Bazel 6.0.0 introduced the credential-helper protocol as +# `--experimental_credential_helper`; 6.4.0 dropped the prefix. +_CREDENTIAL_HELPER_MIN_VERSION = "6.0.0" +_CREDENTIAL_HELPER_UNPREFIXED_MIN_VERSION = "6.4.0" + +_CONFIG_PREFIX = "aspect-" +_EXEC_SUFFIX = "-exec" + +def config_name(deployment) -> str: + """The `--config` name for a deployment row: Aspect Cloud's own name for the + built-in entry, `aspect-` for a configured deployment.""" + return deployment.name if deployment.builtin else _CONFIG_PREFIX + deployment.name + +def credential_helper_flags(hosts: list, helper: str) -> list: + """`--credential_helper==` for each host, scoped so the helper is + only ever invoked for a deployment's own endpoints. Version-gated tuples: the + unprefixed spelling from 6.4.0 (the old one still works there, with a + deprecation warning), `--experimental_credential_helper` from 6.0.0 to 6.3.""" + flags = [] + for host in hosts: + scoped = host + "=" + helper + flags.append(("--credential_helper=" + scoped, ">=" + _CREDENTIAL_HELPER_UNPREFIXED_MIN_VERSION)) + flags.append(("--experimental_credential_helper=" + scoped, ">=" + _CREDENTIAL_HELPER_MIN_VERSION + ", <" + _CREDENTIAL_HELPER_UNPREFIXED_MIN_VERSION)) + return flags + +def _unique_hosts(uris: list) -> list: + hosts = [] + for uri in uris: + host = endpoint_host(uri) if uri else "" + if host and host not in hosts: + hosts.append(host) + return hosts + +def _label(deployment) -> str: + return "Aspect Cloud" if deployment.builtin else "Aspect Workflows deployment '" + deployment.name + "'" + +def _login_hint(deployment) -> str: + """How CI authenticates this deployment's section: its API-token variable, + or the login command for a machine with a browser.""" + login = "aspect auth login" if deployment.builtin else "aspect auth login --deployment " + deployment.name + if deployment.api_token_env: + return "auth via " + deployment.api_token_env + " or `" + login + "`" + return "auth via `" + login + "`" + +def _base_config(d, name: str, credential_helper: str, host_groups: list, bes_groups: list) -> RcConfig: + """`--config=`: the deployment's remote cache and/or BES, the credential + helper for their hosts, and the groups it chains — `host_groups` first, then + the tuning groups those endpoints justify (`bes_groups` for a BES).""" + flags = ["--config=" + g for g in host_groups] + wired = [] + if d.cache: + flags.append("--remote_cache=" + _SCHEME + d.cache) + wired.append("remote cache") + if d.bes: + flags.append("--bes_backend=" + _SCHEME + d.bes) + if d.results_url: + flags.append("--bes_results_url=" + d.results_url) + wired.append("BES") + flags.extend(credential_helper_flags(_unique_hosts([d.cache, d.bes]), credential_helper)) + if d.cache: + flags.append("--config=" + GROUP_CACHE) + if d.bes: + flags.extend(["--config=" + g for g in bes_groups]) + return RcConfig( + name = name, + source = _label(d), + comment = _label(d) + ": " + " + ".join(wired) + "; " + _login_hint(d) + ".", + flags = flags, + ) + +def _exec_config(d, name: str, credential_helper: str, has_base: bool, host_groups: list) -> RcConfig: + """`--config=-exec`: the base section plus the deployment's executor. An + executor-only deployment has no base to chain to, so its section stands alone + and chains `host_groups` and the tuning an executor justifies (it doubles as + the cache). A credential helper is added for the exec host only when the base + section's don't cover it.""" + if has_base: + flags = ["--config=" + name] + covered = _unique_hosts([d.cache, d.bes]) + else: + flags = ["--config=" + g for g in host_groups] + ["--config=" + GROUP_CACHE] + covered = [] + flags.append("--remote_executor=" + _SCHEME + d.exec) + exec_host = endpoint_host(d.exec) + if exec_host not in covered: + flags.extend(credential_helper_flags([exec_host], credential_helper)) + return RcConfig( + name = name + _EXEC_SUFFIX, + source = _label(d) + " (remote execution)", + comment = ("--config=" + name + " plus its" if has_base else _label(d) + ":") + " remote execution" + ("" if has_base else "; " + _login_hint(d)) + ".", + flags = flags, + ) + +def deployment_configs(deployments: list, credential_helper: str, host_groups: list = [], bes_groups: list = [GROUP_BES]) -> list: + """The sections for every deployment in `deployments` (rows of + `aspect.DeploymentEndpoints`, or anything with the same `name` / `builtin` / + `default` / `cache` / `bes` / `exec` / `results_url` / `api_token_env` + fields), in order. `credential_helper` is the name or path Bazel invokes as + ` get`. Every section starts by chaining + `host_groups` — the rc's `aspect-common` and CI-host groups, which the + deployment shape of the rc leaves unenabled so the file is inert until a + section is named — then the tuning groups its endpoints justify, a BES + chaining `bes_groups` (`aspect-bes`, plus `aspect-exec-log` where the rc + defines it); all are `rc_groups.axl` groups the rc defines alongside.""" + configs = [] + for d in deployments: + has_base = bool(d.cache or d.bes) + if not (has_base or d.exec): + continue + name = config_name(d) + if has_base: + configs.append(_base_config(d, name, credential_helper, host_groups, bes_groups)) + if d.exec: + configs.append(_exec_config(d, name, credential_helper, has_base, host_groups)) + return configs + +def config_collisions(configs: list) -> list: + """One message per section name that more than one `RcConfig` claims, naming + the claimants and the fix; empty when every name is unique.""" + claimants = {} + for c in configs: + claimants[c.name] = claimants.get(c.name, []) + [c.source] + return [ + "--config=%s would be written for both %s. Rename the deployment: `aspect auth remove `, then `aspect auth configure --name `." % (name, " and ".join(sources)) + for (name, sources) in claimants.items() + if len(sources) > 1 + ] diff --git a/crates/aspect-cli/src/builtins/aspect/private/lib/deployment_rc_test.axl b/crates/aspect-cli/src/builtins/aspect/private/lib/deployment_rc_test.axl new file mode 100644 index 000000000..e82dc37ea --- /dev/null +++ b/crates/aspect-cli/src/builtins/aspect/private/lib/deployment_rc_test.axl @@ -0,0 +1,183 @@ +"""Unit tests for `deployment_rc.axl`: the `--config=aspect-*` sections built from +deployment rows — naming, credential helpers per distinct host, the flags each +endpoint combination justifies, and name collisions. +Run with: aspect dev test-deployment-rc +""" + +load( + "./deployment_rc.axl", + "config_collisions", + "config_name", + "credential_helper_flags", + "deployment_configs", +) + +def _eq(label, got, want): + if got != want: + fail("%s: got %r, want %r" % (label, got, want)) + +def _texts(flags) -> list: + return [f[0] if type(f) == "tuple" else f for f in flags] + +def _has(label, flags, want): + _eq(label, want in _texts(flags), True) + +def _lacks(label, flags, fragment): + _eq(label, any([fragment in f for f in _texts(flags)]), False) + +def _summary(name, builtin = False, default = False, cache = "", bes = "", exec = "", results_url = "", api_token_env = ""): + """A stand-in for an `aspect.DeploymentEndpoints` row from `ctx.aspect.auth.deployments()`.""" + return struct(name = name, builtin = builtin, default = default, cache = cache, bes = bes, exec = exec, results_url = results_url, api_token_env = api_token_env) + +_CLOUD = _summary("aspect-cloud", builtin = True, default = True, cache = "cache.aspect.build", bes = "bes.aspect.build", results_url = "https://app.aspect.build/i/", api_token_env = "ASPECT_API_TOKEN") +_SILO = _summary("silo-gcp", cache = "remote.silo-gcp.aspect.build", bes = "remote.silo-gcp.aspect.build", exec = "remote.silo-gcp.aspect.build", results_url = "https://app.silo-gcp.aspect.build/i/", api_token_env = "ASPECT_API_TOKEN_SILO_GCP") +_HELPER = "/opt/aspect/aspect-cli" + +def _by_name(configs) -> dict: + return {c.name: c for c in configs} + +def _test_config_name(_ctx): + # Aspect Cloud keeps its own (already `aspect-` prefixed) name; a configured + # deployment is prefixed so the names share one namespace. A self-hosted + # deployment keeps its dotted domain as its name; Bazel accepts dots. + _eq("builtin", config_name(_CLOUD), "aspect-cloud") + _eq("configured", config_name(_SILO), "aspect-silo-gcp") + _eq("dotted", config_name(_summary("remote.acme.com")), "aspect-remote.acme.com") + +def _test_cloud_section(_ctx): + configs = _by_name(deployment_configs([_CLOUD], _HELPER)) + _eq("names", sorted(configs.keys()), ["aspect-cloud"]) + flags = configs["aspect-cloud"].flags + + _has("cache", flags, "--remote_cache=grpcs://cache.aspect.build") + _has("bes", flags, "--bes_backend=grpcs://bes.aspect.build") + _has("results url", flags, "--bes_results_url=https://app.aspect.build/i/") + _lacks("no remote_executor", flags, "--remote_executor") + + # One scoped credential helper per distinct host, both spellings gated. + _eq("cache helper", ("--credential_helper=cache.aspect.build=" + _HELPER, ">=6.4.0") in flags, True) + _eq("bes helper", ("--credential_helper=bes.aspect.build=" + _HELPER, ">=6.4.0") in flags, True) + _eq("cache helper (6.0–6.3)", ("--experimental_credential_helper=cache.aspect.build=" + _HELPER, ">=6.0.0, <6.4.0") in flags, True) + _lacks("no unscoped helper", flags, "--credential_helper=" + _HELPER) + + # Both endpoints are wired, so the section chains to both tuning groups the + # rc defines and repeats none of their flags. Without `host_groups` nothing + # else is chained. + _eq("chains", [f for f in flags if _texts([f])[0].startswith("--config=")], ["--config=aspect-cache", "--config=aspect-bes"]) + _lacks("no inlined tuning", flags, "--remote_upload_local_results") + _lacks("no inlined enrichment", flags, "--legacy_important_outputs") + _lacks("no host tuning", flags, "--heap_dump_on_oom") + + # Each section says what it wires and how it authenticates. + _eq("comment", configs["aspect-cloud"].comment.startswith("Aspect Cloud: remote cache + BES; auth via ASPECT_API_TOKEN"), True) + +def _test_host_groups_chained_first(_ctx): + # The rc that carries sections enables nothing, so every section opens by + # chaining what the env shape would have enabled; aliases chain a section. + configs = _by_name(deployment_configs([_SILO, _summary("e", exec = "exec.e")], _HELPER, host_groups = ["aspect-common", "aspect-github-actions"])) + _eq("base opens with host groups", configs["aspect-silo-gcp"].flags[:2], ["--config=aspect-common", "--config=aspect-github-actions"]) + _eq("exec chains its base only", configs["aspect-silo-gcp-exec"].flags[0], "--config=aspect-silo-gcp") + _lacks("exec does not repeat them", configs["aspect-silo-gcp-exec"].flags, "--config=aspect-common") + _eq("standalone exec opens with them", configs["aspect-e-exec"].flags[:3], ["--config=aspect-common", "--config=aspect-github-actions", "--config=aspect-cache"]) + + # A BES-wired section chains the groups the rc defines for a BES, so the + # execution log rides along where the rc has one and is absent where not. + with_log = _by_name(deployment_configs([_CLOUD], _HELPER, bes_groups = ["aspect-bes", "aspect-exec-log"]))["aspect-cloud"].flags + _eq("bes chains", [f for f in with_log if _texts([f])[0] in ["--config=aspect-bes", "--config=aspect-exec-log"]], ["--config=aspect-bes", "--config=aspect-exec-log"]) + _lacks("default chains bes alone", deployment_configs([_CLOUD], _HELPER)[0].flags, "exec-log") + _eq("none by default", [f for f in deployment_configs([_SILO], _HELPER)[0].flags if f == "--config=aspect-common"], []) + +def _test_exec_section(_ctx): + configs = _by_name(deployment_configs([_SILO], _HELPER)) + _eq("names", sorted(configs.keys()), ["aspect-silo-gcp", "aspect-silo-gcp-exec"]) + + base = configs["aspect-silo-gcp"].flags + _lacks("base has no executor", base, "--remote_executor") + _eq("one host, one helper", len([f for f in base if _texts([f])[0].startswith("--credential_helper=")]), 1) + + exec = configs["aspect-silo-gcp-exec"].flags + _eq("chains to base first", exec[0], "--config=aspect-silo-gcp") + _has("executor", exec, "--remote_executor=grpcs://remote.silo-gcp.aspect.build") + _lacks("tuning comes from the base", exec, "--remote_timeout") + _lacks("exec host already covered", exec, "--credential_helper") + + # An exec host the base doesn't cover gets its own helper. + split = _by_name(deployment_configs([_summary("split", cache = "cache.split", bes = "bes.split", exec = "exec.split")], _HELPER))["aspect-split-exec"].flags + _eq("exec host helper", ("--credential_helper=exec.split=" + _HELPER, ">=6.4.0") in split, True) + +def _test_partial_deployments(_ctx): + # Each deployment gets exactly the flags its endpoints justify. + bes_only = _by_name(deployment_configs([_summary("b", bes = "bes.b")], _HELPER))["aspect-b"].flags + _lacks("no cache", bes_only, "--remote_cache") + _has("bes", bes_only, "--bes_backend=grpcs://bes.b") + _has("bes group", bes_only, "--config=aspect-bes") + _lacks("no cache group", bes_only, "--config=aspect-cache") + + cache_only = _by_name(deployment_configs([_summary("c", cache = "cache.c")], _HELPER))["aspect-c"].flags + _lacks("no bes", cache_only, "--bes_backend") + _has("cache group", cache_only, "--config=aspect-cache") + _lacks("no bes group", cache_only, "--config=aspect-bes") + + # An executor-only deployment gets a standalone `-exec` section: no base to + # chain to, the executor standing in as the cache. + exec_only = _by_name(deployment_configs([_summary("e", default = True, exec = "exec.e")], _HELPER)) + _eq("exec-only sections", sorted(exec_only.keys()), ["aspect-e-exec"]) + flags = exec_only["aspect-e-exec"].flags + _lacks("no base to chain", flags, "--config=aspect-e") + _has("executor", flags, "--remote_executor=grpcs://exec.e") + _has("exec earns the cache group", flags, "--config=aspect-cache") + _eq("helper", ("--credential_helper=exec.e=" + _HELPER, ">=6.4.0") in flags, True) + + # A fresh install's Aspect Cloud seed advertises nothing: no section, and no + # dangling `aspect` alias even though it is the default. + _eq("nothing advertised → nothing", deployment_configs([_summary("aspect-cloud", builtin = True, default = True)], _HELPER), []) + +def _test_no_default_alias(_ctx): + # The rc is committed and shared, and the default deployment is one machine's + # setting that can change under it, so no section names "the default". + names = [c.name for c in deployment_configs([_summary("aspect-cloud", builtin = True, cache = "c", bes = "b"), _summary("silo", default = True, cache = "c", bes = "b", exec = "e")], _HELPER)] + _eq("one section per deployment, none for the default", names, ["aspect-cloud", "aspect-silo", "aspect-silo-exec"]) + +def _test_credential_helper_scope(_ctx): + # A host advertised with a port still scopes the helper by bare host, which + # is what Bazel matches the request URI against. + flags = deployment_configs([_summary("p", cache = "cache.p:8980")], _HELPER)[0].flags + _eq("bare host scope", ("--credential_helper=cache.p=" + _HELPER, ">=6.4.0") in flags, True) + _eq("helper shape", credential_helper_flags(["h"], "x")[0], ("--credential_helper=h=x", ">=6.4.0")) + +def _test_config_collisions(_ctx): + _eq("no collision", config_collisions(deployment_configs([_CLOUD, _SILO], _HELPER)), []) + + # A deployment named `cloud` lands on Aspect Cloud's own section name. + msgs = config_collisions(deployment_configs([_CLOUD, _summary("cloud", cache = "c")], _HELPER)) + _eq("one collision", len(msgs), 1) + _eq("names both", msgs[0].startswith("--config=aspect-cloud would be written for both Aspect Cloud and Aspect Workflows deployment 'cloud'."), True) + + # `silo-gcp` with remote execution and a deployment named `silo-gcp-exec`. + clash = deployment_configs([_SILO, _summary("silo-gcp-exec", cache = "c")], _HELPER) + _eq("exec suffix collision", [m.split(" ")[0] for m in config_collisions(clash)], ["--config=aspect-silo-gcp-exec"]) + +_UNIT_TESTS = [ + _test_config_name, + _test_cloud_section, + _test_host_groups_chained_first, + _test_exec_section, + _test_partial_deployments, + _test_no_default_alias, + _test_credential_helper_scope, + _test_config_collisions, +] + +def _test_impl(ctx): + for t in _UNIT_TESTS: + t(ctx) + print("deployment_rc.axl: OK (%d tests)" % len(_UNIT_TESTS)) + return 0 + +deployment_rc_tests = task( + summary = "Run the deployment_rc AXL unit tests.", + kind = "test-deployment-rc", + group = ["dev"], + implementation = _test_impl, + args = {}, +) diff --git a/crates/aspect-cli/src/builtins/aspect/private/lib/environment.axl b/crates/aspect-cli/src/builtins/aspect/private/lib/environment.axl index ff6e2ec6c..1c9acb980 100644 --- a/crates/aspect-cli/src/builtins/aspect/private/lib/environment.axl +++ b/crates/aspect-cli/src/builtins/aspect/private/lib/environment.axl @@ -2,7 +2,8 @@ Workflows Environment Library Reads runner environment from environment variables and exposes it as typed -records. Also provides bazelrc flag generation and host/CI detection. +records, plus host/CI detection and the shared console-output helpers. The +Bazel flags derived from this environment live in `aspect_flags.axl`. """ load("./ansi.axl", "ansi") @@ -16,6 +17,11 @@ RemoteCache = record( endpoint = field(str, default = ""), # → Bazel `--remote_bytestream_uri_prefix`; cross-cache de-dup. "" when runner didn't surface one. bytestream_uri_prefix = field(str, default = ""), + # Whether the cache accepts compressed blobs (ASPECT_WORKFLOWS_REMOTE_CACHE_COMPRESSION + # set). False forces `--noremote_cache_compression`, since Bazel errors when a + # repository asks a cache that lacks it for compression; true leaves the + # choice to the repository. Goes away once no deployment lacks compression. + supports_compression = field(bool, default = False), ) BuildEvents = record( @@ -23,11 +29,13 @@ BuildEvents = record( results_url = field(str, default = ""), ) +# → Bazel `--remote_executor=`, only where a build opts in. +RemoteExecutor = record( + endpoint = field(str, default = ""), +) + Runner = record( storage_path = field(str, default = DEFAULT_STORAGE_PATH), - # Opaque value sent verbatim as the `x-identity` header on the build's - # remote connections. "" when unset (no header sent). - identity = field(str, default = ""), product_version = field(str, default = ""), instance_id = field(str, default = ""), instance_name = field(str, default = ""), @@ -40,8 +48,6 @@ Runner = record( az = field(str, default = ""), preemptible = field(bool, default = False), warming_enabled = field(bool, default = False), - # → enables BES-backend upload of the JSON profile + compact exec log. - upload_build_diagnostics = field(bool, default = False), cloud_provider = field(str, default = ""), data_dir = field(str, default = ""), bin_dir = field(str, default = DEFAULT_BIN_DIR), @@ -70,6 +76,7 @@ WorkflowsEnvironment = record( # Each field is independently None when its `_read_*` found no signals; # callers gate per field (e.g. plain GHA: runner == None but ci != None). remote_cache = field(RemoteCache | None, default = None), + remote_executor = field(RemoteExecutor | None, default = None), build_events = field(BuildEvents | None, default = None), runner = field(Runner | None, default = None), ci = field(CI | None, default = None), @@ -77,18 +84,11 @@ WorkflowsEnvironment = record( # Terminal / ANSI detection -def color_enabled(std): - """Should ANSI color be emitted to stdout? The "emit color?" predicate. - - True on a real TTY, or on a recognized CI host (GHA, Buildkite, CircleCI, - GitLab) — those render ANSI in log viewers despite a non-TTY pipe. - - Broader than `CI.supports_curses`: curses cursor control breaks - line-oriented log viewers, but plain ANSI color renders fine. - """ - if std.io.stdout.is_tty: - return True - env = std.env +def ci_renders_ansi(env) -> bool: + """Whether this is a CI host whose log viewer renders ANSI color despite a + non-TTY pipe: GitHub Actions, Buildkite, CircleCI, GitLab. Broader than + `CI.supports_curses`: curses cursor control breaks line-oriented log viewers, + but plain ANSI color renders fine.""" return bool( env.var("GITHUB_ACTIONS") or env.var("BUILDKITE") or @@ -96,6 +96,18 @@ def color_enabled(std): env.var("GITLAB_CI"), ) +def color_enabled(std): + """Should ANSI color be emitted to stdout? True on a real TTY, or on a CI + host that renders ANSI (`ci_renders_ansi`).""" + return std.io.stdout.is_tty or ci_renders_ansi(std.env) + +def bazel_misdetects_color(env) -> bool: + """Whether Bazel's `--color=auto` would wrongly decide against color here: a + CI host whose log viewer renders ANSI but which runs the job with no TTY for + Bazel to see — GitHub Actions, CircleCI, GitLab. Buildkite renders ANSI too + but gives the job a PTY, so Bazel gets it right unaided and is left out.""" + return bool(env.var("GITHUB_ACTIONS") or env.var("CIRCLECI") or env.var("GITLAB_CI")) + # Severity-prefixed output. Channels (increasing alarm): # trace.log(msg) — debug only, ASPECT_DEBUG=1-gated; never on a clean run. # print(msg) — narrative beats every run produces (banners, tables, outcomes). @@ -215,8 +227,19 @@ def _read_remote_cache(std) -> RemoteCache | None: return RemoteCache( endpoint = endpoint, bytestream_uri_prefix = std.env.var("ASPECT_WORKFLOWS_REMOTE_BYTESTREAM_URI_PREFIX") or "", + supports_compression = bool(std.env.var("ASPECT_WORKFLOWS_REMOTE_CACHE_COMPRESSION")), ) +def _read_remote_executor(std) -> RemoteExecutor | None: + """Read the remote-execution endpoint; None when + ASPECT_WORKFLOWS_REMOTE_EXECUTOR is unset. Never wired on its own: remote + execution relocates every action, so a build names it (`--workflows:remote-exec` + on a task, `--config=aspect-exec` for vanilla `bazel`).""" + endpoint = std.env.var("ASPECT_WORKFLOWS_REMOTE_EXECUTOR") or "" + if not endpoint: + return None + return RemoteExecutor(endpoint = endpoint) + def get_workflows_environment(std) -> WorkflowsEnvironment: """Read all Aspect-Workflows-managed environment state in one call. @@ -225,6 +248,7 @@ def get_workflows_environment(std) -> WorkflowsEnvironment: """ return WorkflowsEnvironment( remote_cache = _read_remote_cache(std), + remote_executor = _read_remote_executor(std), build_events = _read_build_events(std), runner = _read_runner(std), ci = _read_ci(std), @@ -264,7 +288,6 @@ def _read_runner(std) -> Runner | None: return Runner( account = std.env.var("ASPECT_WORKFLOWS_RUNNER_CLOUD_ACCOUNT") or "", aspect_launcher_version = std.env.var("ASPECT_WORKFLOWS_RUNNER_ASPECT_LAUNCHER_VERSION") or "", - identity = std.env.var("ASPECT_WORKFLOWS_RUNNER_IDENTITY") or "", az = std.env.var("ASPECT_WORKFLOWS_RUNNER_AZ") or "", bazel_root_dir = std.env.var("ASPECT_WORKFLOWS_RUNNER_BAZEL_ROOT_DIR") or (storage_path + "/bazel"), bin_dir = std.env.var("ASPECT_WORKFLOWS_RUNNER_BIN_DIR") or DEFAULT_BIN_DIR, @@ -288,7 +311,6 @@ def _read_runner(std) -> Runner | None: warming_complete = warming_complete, warming_current_cache = warming_current_cache, warming_enabled = bool(std.env.var("ASPECT_WORKFLOWS_RUNNER_WARMING_ENABLED")), - upload_build_diagnostics = bool(std.env.var("ASPECT_WORKFLOWS_RUNNER_UPLOAD_BUILD_DIAGNOSTICS")), runner_group_name = std.env.var("ASPECT_WORKFLOWS_RUNNER_GROUP_NAME") or "", runner_group_queue = std.env.var("ASPECT_WORKFLOWS_RUNNER_GROUP_QUEUE") or "", runner_resource_type = std.env.var("ASPECT_WORKFLOWS_RUNNER_RESOURCE_TYPE") or "", @@ -331,164 +353,6 @@ def sanitize_filename(name: str) -> str: result += "_" return result -def get_generic_bazelrc_flags(environment: WorkflowsEnvironment) -> list: - """Build flags that are safe on ANY CI, not just an Aspect Workflows runner. - - These tune Bazel's remote/cache/profiling behavior in ways that don't depend - on the runner's mounts, identity, or the Workflows cache implementation, so - they can be written into a CI bazelrc even off a Workflows runner. Includes - the remote-cache endpoint flags when the `ASPECT_WORKFLOWS_REMOTE_CACHE` / - `ASPECT_WORKFLOWS_REMOTE_BYTESTREAM_URI_PREFIX` env vars are set (they can be - set off a runner too), surfaced via `environment.remote_cache`. - - Excludes runner-specific flags — the per-runner mount paths - (`--repository_cache`, output base/root), the runner identity header, the - `--nosystem_rc` startup flag, and the cache-compression gates (Aspect - Workflows' own remote cache does not support compression, so those only - apply when talking to it on a runner). `get_bazelrc_flags` adds those. - - Returns a flat list of build flags (no version-gated tuples — none of the - generic flags are version-conditional). - """ - build_flags = [ - "--remote_upload_local_results", - "--heap_dump_on_oom", - "--generate_json_trace_profile", - "--experimental_repository_cache_hardlinks", - "--remote_accept_cached", - "--disk_cache=", - "--remote_timeout=3600", - "--remote_retries=360", - "--grpc_keepalive_timeout=30s", - ] - - remote_cache = environment.remote_cache - if remote_cache: - if remote_cache.endpoint: - build_flags.append("--remote_cache=" + remote_cache.endpoint) - if remote_cache.bytestream_uri_prefix: - build_flags.append("--remote_bytestream_uri_prefix=" + remote_cache.bytestream_uri_prefix) - - return build_flags - -def get_bazelrc_flags(environment: WorkflowsEnvironment, aspect_root_dir: str) -> (list, list): - """Generate the full bazelrc flag set for an Aspect Workflows runner. - - The generic CI flags (`get_generic_bazelrc_flags`) plus the runner-specific - ones: the cache-compression gates (the Workflows remote cache doesn't support - compression), the runner identity header, the per-runner repository cache and - output paths, and `--nohome_rc`/`--nosystem_rc`. - - The `--output_base` startup flag in the result is what `apply_output_base_suffix` - rewrites to move a task to its own Bazel server (`--output_user_root` stays - shared so the install base and external-repo cache are not duplicated). - - Args: - environment: WorkflowsEnvironment. Caller must have verified - `runner != None` (output_base/output_user_root come from it). - aspect_root_dir: absolute path to the Aspect project root. - - Returns: - (startup_flags, build_flags): two lists; build_flags may contain - version-gated `(flag, constraint)` tuples. - """ - runner = environment.runner - if runner == None: - fail("get_bazelrc_flags called without a Workflows runner — gate on environment.runner != None") - - repo_name = environment.ci.scm_repo_name if environment.ci else "" - subdir = sanitize_filename(aspect_root_dir.rstrip("/").split("/")[-1]) if aspect_root_dir else "__main__" - - build_flags = get_generic_bazelrc_flags(environment) - - # Aspect Workflows' remote cache does not support compression, so disable it - # (the flag name changed in Bazel 8). The disk-results gate is a <7 quirk. - build_flags.append(("--noexperimental_remote_cache_compression", "<8.0.0")) - build_flags.append(("--noremote_cache_compression", ">=8.0.0")) - build_flags.append(("--incompatible_remote_results_ignore_disk", "<7.0.0")) - - # `--remote_header` is the shared header Bazel applies to both the remote - # cache and the executor channel, so gate on the identity alone: this also - # covers executor-only setups, and Bazel ignores the flag with no backend. - if runner.identity: - build_flags.append("--remote_header=x-identity=" + runner.identity) - - build_flags.append("--repository_cache=" + runner.repository_cache_dir) - - # In the runner, home/system rc files are redundant — workspace .bazelrc - # plus these synthesized flags cover everything. - startup_flags = ["--nohome_rc", "--nosystem_rc"] - - bazel_root = runner.bazel_root_dir - output_root = runner.output_root_dir - if repo_name: - sanitized = sanitize_filename(repo_name) - startup_flags.append("--output_user_root=" + bazel_root + "/" + sanitized + "/" + subdir) - startup_flags.append("--output_base=" + output_root + "/" + sanitized + "/" + subdir) - else: - startup_flags.append("--output_user_root=" + bazel_root + "/" + subdir) - startup_flags.append("--output_base=" + output_root + "/" + subdir) - - return (startup_flags, build_flags) - -_OUTPUT_BASE_FLAG_PREFIX = "--output_base=" - -def _is_output_base_flag(flag: str) -> bool: - """True for the `--output_base=` startup flag that names this invocation's - Bazel server (and its analysis cache). - - Deliberately NOT `--output_user_root`: that root holds state we want shared - across servers — the install base, the extracted external-repo trees, and - the action cache — so only `--output_base` is unique per isolated server.""" - return flag.startswith(_OUTPUT_BASE_FLAG_PREFIX) - -def apply_output_base_suffix(startup_flags: list, suffix: str) -> list: - """Return a copy of `startup_flags` with `suffix` appended to the path value - of the `--output_base=` entry; every other flag passes through unchanged. - - `--output_base` names the Bazel server (and its analysis cache), so suffixing - it moves the invocation to a sibling server (e.g. `.../myrepo/aspect-cli` → - `.../myrepo/aspect-cli-delivery`) — isolating a task whose build flips Bazel - options (`--stamp`) from the shared warm analysis cache. `--output_user_root` - is left untouched so the install base and external-repo cache stay shared. - - A no-op returning an unchanged copy when `suffix` is empty or no - `--output_base` is present. `suffix` becomes part of an on-disk path segment, - so the caller must sanitize it (e.g. via `sanitize_filename`). - """ - if not suffix: - return list(startup_flags) - return [flag + suffix if _is_output_base_flag(flag) else flag for flag in startup_flags] - -def get_build_diagnostics_flags(environment: WorkflowsEnvironment, enabled: bool) -> list: - """BES-backend upload flags for the JSON profile + compact exec log. - - `enabled` is the opt-in decision the caller owns — the deployment env var - (runner.upload_build_diagnostics) OR the workspace-level override arg — so - both paths reach the flags. Even when enabled, returns empty unless both a - BES backend and a remote cache are wired up: the files ride the BES stream - to the remote cache as bytestream:// refs, so both are required for the - upload to land. `--generate_json_trace_profile` is already set - unconditionally in get_bazelrc_flags; this only enriches/uploads it. - """ - runner = environment.runner - if not enabled or runner == None: - return [] - if environment.build_events == None or environment.remote_cache == None: - return [] - - tmpdir = runner.job_tmpdir - return [ - "--experimental_profile_include_target_label", - "--profile=" + tmpdir + "/command.profile.gz", - # Version-gated tuple like the cache-compression flags above; - # --execution_log_compact_file needs Bazel >= 7.1. - ("--execution_log_compact_file=" + tmpdir + "/exec.log.zstd", ">=7.1.0"), - # Upload ALL BEP-referenced local outputs (default is `minimal`) so - # the profile + exec log actually reach the cache for ivy-bep-etl. - "--remote_build_event_upload=all", - ] - # Display helpers def _url_encode(s: str) -> str: @@ -524,6 +388,19 @@ def is_aspect_workflows_runner(env): """True when the current process is running on an Aspect Workflows runner.""" return bool(env.var("ASPECT_WORKFLOWS_RUNNER")) +# The variables by which the environment names this machine's deployment. +_DEPLOYMENT_VARS = ["ASPECT_WORKFLOWS_RUNNER", "ASPECT_WORKFLOWS_REMOTE_CACHE", "ASPECT_WORKFLOWS_BES_BACKEND", "ASPECT_WORKFLOWS_REMOTE_EXECUTOR"] + +def env_names_deployment(env) -> bool: + """Whether the environment says which deployment this machine is on: the + runner marker, or any `ASPECT_WORKFLOWS_*` endpoint. Then that is the + deployment for every Bazel call here — `aspect setup bazelrc` takes its + endpoints from the env and writes no `--config=aspect-*` sections, and a + task refuses `--remote`, which would point at a configured deployment + instead. The two are answers to the same question; a build routed off the + runner's own services by a second answer is never what anyone meant.""" + return any([bool(env.var(name)) for name in _DEPLOYMENT_VARS]) + def detect_ci(env): """Detect the CI platform from env vars (env = ctx.std.env). Returns: - "github"/"buildkite"/"gitlab"/"circleci" — gated on a host-exclusive marker. diff --git a/crates/aspect-cli/src/builtins/aspect/private/lib/rc_groups.axl b/crates/aspect-cli/src/builtins/aspect/private/lib/rc_groups.axl new file mode 100644 index 000000000..ba56cc157 --- /dev/null +++ b/crates/aspect-cli/src/builtins/aspect/private/lib/rc_groups.axl @@ -0,0 +1,178 @@ +"""The named `--config` groups every rc `aspect setup bazelrc` writes is built from. + +Bazel collects every rc section before expanding any `--config`, so a group can +be enabled by a `common --config=` line and defined further down the same +file. The rc uses that to say what it does in Bazel's own vocabulary: each flag +sits in the group named for what it presumes, the groups that apply on this host +are enabled at the top, and `--announce_rc` names the group every flag came from. +The startup options are the exception, since Bazel gives them no `--config`. + + aspect-common tuning that presumes no endpoint + aspect-cache the cache — its endpoint where the env names one — and the tuning a cache justifies + aspect-bes the BES — its endpoint where the env names one — and the tuning a BES justifies + aspect-exec-log the compact execution log for the diagnostics upload; apart so a repo can drop it + aspect- what that CI host needs (`--color=yes`); only in a machine's own rc + aspect-runner the runner's storage (`--repository_cache`); only with the runner marker + aspect-exec the runner's remote executor; only where the env names one + +The rc has two shapes, decided by `environment.env_names_deployment`. Where the environment +names the deployment — the runner marker, or any `ASPECT_WORKFLOWS_*` endpoint — +every group but `aspect-exec` is enabled (`aspect-cache` / `aspect-bes` only +where the env names their endpoint) and no deployment sections are written. +Where it is silent, the rc is the checkout's, committed and shared by every +host that builds it: the host group is left out, nothing is enabled, and the +deployment sections of `deployment_rc.axl` are written, each chaining +`section_chains` (`aspect-common`) and then the tuning groups its endpoints +justify, so the file is inert until a section is named. Either way `aspect-exec` is reached only +by name, since remote execution relocates every action. + +The host group is named for the detected CI host — `aspect-github-actions`, +`aspect-circleci`, `aspect-gitlab`, `aspect-buildkite` — and exists only when +that host needs something; today that is `--color=yes` where Bazel's +`--color=auto` misdetects (`environment.bazel_misdetects_color`), which leaves +Buildkite with none. `aspect-exec` is the runner's executor, the one endpoint +a build opts into by name. +""" + +load( + "./aspect_flags.axl", + "color_flags", + "diagnostics_upload_flags", + "endpoint_flags", + "env_endpoint_flags", + "exec_log_flags", + "independent_flags", + "runner_bes_flags", + "runner_executor_endpoint", + "runner_profile_flags", + "runner_repository_cache_flags", + "workflows_cache_flags", +) +load("./bazelrc_file.axl", "RcConfig") +load("./environment.axl", "WorkflowsEnvironment") + +GROUP_COMMON = "aspect-common" +GROUP_CACHE = "aspect-cache" +GROUP_BES = "aspect-bes" +GROUP_RUNNER = "aspect-runner" +GROUP_EXEC_LOG = "aspect-exec-log" +GROUP_EXEC = "aspect-exec" + +# `CI.host` → the host group's name. +HOST_GROUPS = { + "github": "aspect-github-actions", + "buildkite": "aspect-buildkite", + "circleci": "aspect-circleci", + "gitlab": "aspect-gitlab", +} + +_GROUP_SOURCE = "the %s group" + +def _group(name: str, comment: str, flags: list, source: str = "") -> RcConfig: + return RcConfig(name = name, source = source or _GROUP_SOURCE % name, comment = comment, flags = flags) + +def _host_label(host: str) -> str: + return {"github": "GitHub Actions", "buildkite": "Buildkite", "circleci": "CircleCI", "gitlab": "GitLab CI"}[host] + +def cache_group(environment: WorkflowsEnvironment, on_runner: bool) -> RcConfig: + """`aspect-cache`: the cache endpoint the env names (`--remote_cache`, the + bytestream prefix) and, with the runner marker, the Workflows cache's + specifics, followed by the tuning any remote cache justifies. Tuning alone + when the env names no cache.""" + endpoint = env_endpoint_flags(environment) + if endpoint and on_runner: + endpoint.extend(workflows_cache_flags(environment)) + return _group(GROUP_CACHE, "The remote cache (ASPECT_WORKFLOWS_REMOTE_CACHE where set) and the tuning a cache justifies.", endpoint + endpoint_flags(cache = True, bes = False)) + +def bes_group(environment: WorkflowsEnvironment, on_runner: bool) -> RcConfig: + """`aspect-bes`: the BES backend and viewer the env names and, with the + runner marker, `--profile` under the job tmpdir, followed by the enrichment + any BES justifies and the diagnostics upload switch. Tuning alone when the + env names no BES. The execution log is `exec_log_group`, apart.""" + endpoint = runner_bes_flags(environment) + if endpoint and on_runner: + endpoint.extend(runner_profile_flags(environment)) + return _group(GROUP_BES, "The BES backend (ASPECT_WORKFLOWS_BES_BACKEND where set), its enrichment, and the build-diagnostics upload the Aspect backend reads.", endpoint + endpoint_flags(cache = False, bes = True) + diagnostics_upload_flags()) + +def exec_log_group(exec_log_path: str) -> list: + """`aspect-exec-log`: the compact execution log at `exec_log_path` — a + runner's job tmpdir, or `%workspace%/.aspect/generated/…` in a checkout's + rc — for the diagnostics upload to carry. Its own group, enabled or chained + beside `aspect-bes`, so a repository on a thin link can drop the upload of + this one large file by removing the lines that name the group. Empty (no + group) without a path.""" + if not exec_log_path: + return [] + return [_group(GROUP_EXEC_LOG, "The compact execution log, uploaded with the build diagnostics for per-action data in the Web UI. Slow on a thin link: remove this group and the --config lines naming it to skip it.", exec_log_flags(exec_log_path))] + +def host_group(environment: WorkflowsEnvironment, misdetects_color: bool) -> list: + """`aspect-`: what the detected CI host needs — `--color=yes` where + `misdetects_color`. Empty (no group) off a recognized host or when the host + needs nothing.""" + name = HOST_GROUPS.get(environment.ci.host) if environment.ci else None + flags = color_flags(misdetects_color) + if not name or not flags: + return [] + return [_group(name, "What %s needs: its log viewer renders ANSI but the job has no TTY, so Bazel's --color=auto misdetects." % _host_label(environment.ci.host), flags)] + +def runner_group(environment: WorkflowsEnvironment, on_runner: bool) -> list: + """`aspect-runner`: `--repository_cache` on the runner's warmed storage, the + build-option half of the runner's paths (the server paths are startup + lines). Empty without the runner marker.""" + if not on_runner: + return [] + return [_group(GROUP_RUNNER, "The runner's storage: the repository cache its warming restores.", runner_repository_cache_flags(environment))] + +def exec_group(environment: WorkflowsEnvironment) -> list: + """`aspect-exec`: the runner's remote executor, chaining `aspect-cache` when + the env names no cache (the executor doubles as one; with a cache the group + is already enabled). Defined but never enabled — remote execution relocates + every action, so a call names `--config=aspect-exec`. Empty when the env + names no executor.""" + endpoint = runner_executor_endpoint(environment) + if not endpoint: + return [] + chain = [] if environment.remote_cache != None else ["--config=" + GROUP_CACHE] + return [_group(GROUP_EXEC, "The runner's remote execution (ASPECT_WORKFLOWS_REMOTE_EXECUTOR); opt in per call.", chain + endpoint, source = "the runner's remote executor")] + +def rc_groups(environment: WorkflowsEnvironment, on_runner: bool, misdetects_color: bool, exec_log_path: str, names_deployment: bool) -> (list, list): + """Every group the rc defines, in the order written, and the names among + them it enables with `common --config=` lines. Where the env names + the deployment (`names_deployment`, from `environment.env_names_deployment`) + the rc is this machine's: every group but `aspect-exec` is enabled, with + `aspect-cache` / `aspect-bes` (and `aspect-exec-log` with the latter) only + where the env names their endpoint. + Elsewhere the rc is the checkout's, shared by every host that builds it, so + the CI host's group is not defined at all and nothing is enabled; the + deployment sections chain `section_chains` instead. Returns + `(groups, enabled)`.""" + groups = [_group(GROUP_COMMON, "Tuning that presumes no endpoint.", independent_flags())] + if names_deployment: + groups.extend(host_group(environment, misdetects_color)) + groups.extend(runner_group(environment, on_runner)) + groups.append(cache_group(environment, on_runner)) + groups.append(bes_group(environment, on_runner)) + groups.extend(exec_log_group(exec_log_path)) + groups.extend(exec_group(environment)) + + if not names_deployment: + return (groups, []) + enabled = section_chains(groups) + enabled.extend([g.name for g in groups if g.name == GROUP_RUNNER]) + if environment.remote_cache != None: + enabled.append(GROUP_CACHE) + if environment.build_events != None: + enabled.append(GROUP_BES) + enabled.extend([g.name for g in groups if g.name == GROUP_EXEC_LOG]) + return (groups, enabled) + +def bes_chains(groups: list) -> list: + """The groups a BES-wired deployment section chains: `aspect-bes`, and + `aspect-exec-log` when `groups` defines it.""" + return [g.name for g in groups if g.name in [GROUP_BES, GROUP_EXEC_LOG]] + +def section_chains(groups: list) -> list: + """The groups every deployment section chains first, so a section brings + them along where nothing is enabled: `aspect-common`, and a host group if + `groups` defines one.""" + return [g.name for g in groups if g.name == GROUP_COMMON or g.name in HOST_GROUPS.values()] diff --git a/crates/aspect-cli/src/builtins/aspect/private/lib/rc_groups_test.axl b/crates/aspect-cli/src/builtins/aspect/private/lib/rc_groups_test.axl new file mode 100644 index 000000000..c2b068357 --- /dev/null +++ b/crates/aspect-cli/src/builtins/aspect/private/lib/rc_groups_test.axl @@ -0,0 +1,184 @@ +"""Unit tests for `rc_groups.axl`: what each named `--config` group holds on and +off a runner, the CI host group's naming, the two rc shapes (which groups are +enabled, which a deployment section chains), and `environment.env_names_deployment`. +Run with: aspect dev test-rc-groups +""" + +load("./aspect_flags.axl", "diagnostics_upload_flags", "endpoint_flags", "exec_log_flags", "independent_flags") +load("./environment.axl", "BuildEvents", "CI", "RemoteCache", "RemoteExecutor", "Runner", "WorkflowsEnvironment", "env_names_deployment") +load( + "./rc_groups.axl", + "GROUP_BES", + "GROUP_CACHE", + "GROUP_COMMON", + "GROUP_EXEC", + "GROUP_EXEC_LOG", + "GROUP_RUNNER", + "HOST_GROUPS", + "bes_chains", + "host_group", + "rc_groups", + "section_chains", +) + +def _eq(label, got, want): + if got != want: + fail("%s: got %r, want %r" % (label, got, want)) + +def _texts(flags) -> list: + return [f[0] if type(f) == "tuple" else f for f in flags] + +def _by_name(configs) -> dict: + return {c.name: c for c in configs} + +_RUNNER = Runner(bazel_root_dir = "/mnt/b", output_root_dir = "/mnt/o", repository_cache_dir = "/mnt/r", job_tmpdir = "/wf") +_CACHE = RemoteCache(endpoint = "grpc://cache:8980", bytestream_uri_prefix = "cache:8980/main") +_BES = BuildEvents(backend = "grpc://bes:1985", results_url = "https://ui/i/") +_EXEC = RemoteExecutor(endpoint = "grpc://exec:8980") + +def _test_bare_host(_ctx): + # Nothing in the env, no recognized host: the tuning groups alone, pure and + # none enabled — this is the deployment shape, inert until a section is named. + (groups, enabled) = rc_groups(WorkflowsEnvironment(), on_runner = False, misdetects_color = False, exec_log_path = "", names_deployment = False) + by = _by_name(groups) + _eq("names", [g.name for g in groups], [GROUP_COMMON, GROUP_CACHE, GROUP_BES]) + _eq("nothing enabled", enabled, []) + _eq("sections chain common", section_chains(groups), [GROUP_COMMON]) + _eq("common", by[GROUP_COMMON].flags, independent_flags()) + _eq("cache is tuning alone", by[GROUP_CACHE].flags, endpoint_flags(cache = True, bes = False)) + _eq("bes sections chain bes alone", bes_chains(groups), [GROUP_BES]) + _eq("bes is tuning + upload switch", by[GROUP_BES].flags, endpoint_flags(cache = False, bes = True) + diagnostics_upload_flags()) + + # With a log path the execution log is its own group, chained beside bes. + (groups, enabled) = rc_groups(WorkflowsEnvironment(), on_runner = False, misdetects_color = False, exec_log_path = "%workspace%/.aspect/generated/exec.log.zstd", names_deployment = False) + by = _by_name(groups) + _eq("exec log group", by[GROUP_EXEC_LOG].flags, exec_log_flags("%workspace%/.aspect/generated/exec.log.zstd")) + _eq("apart from bes", any(["execution_log" in f for f in _texts(by[GROUP_BES].flags)]), False) + _eq("bes sections chain both", bes_chains(groups), [GROUP_BES, GROUP_EXEC_LOG]) + _eq("still nothing enabled", enabled, []) + for g in groups: + _eq(g.name + " commented", bool(g.comment), True) + +def _test_env_off_runner(_ctx): + # Endpoint variables set by hand, no runner marker: each endpoint joins its + # tuning group, ahead of the tuning, and enables it; nothing tied to a + # runner's mounts. + env = WorkflowsEnvironment(remote_cache = _CACHE, build_events = _BES) + (groups, enabled) = rc_groups(env, on_runner = False, misdetects_color = False, exec_log_path = "/x", names_deployment = True) + by = _by_name(groups) + _eq("enabled, the log with bes", enabled, [GROUP_COMMON, GROUP_CACHE, GROUP_BES, GROUP_EXEC_LOG]) + _eq("no runner group", GROUP_RUNNER in by, False) + cache = by[GROUP_CACHE].flags + _eq("cache endpoint first", cache[:2], ["--remote_cache=grpc://cache:8980", "--remote_bytestream_uri_prefix=cache:8980/main"]) + _eq("then cache tuning", cache[2:], endpoint_flags(cache = True, bes = False)) + _eq("no workflows specifics off runner", "--disk_cache=" in _texts(cache), False) + bes = by[GROUP_BES].flags + _eq("bes endpoint first", bes[:2], ["--bes_backend=grpc://bes:1985", "--bes_results_url=https://ui/i/"]) + _eq("no profile off runner", any([f.startswith("--profile=") for f in _texts(bes)]), False) + +def _test_runner(_ctx): + env = WorkflowsEnvironment(runner = _RUNNER, remote_cache = _CACHE, build_events = _BES, remote_executor = _EXEC, ci = CI(host = "buildkite")) + (groups, enabled) = rc_groups(env, on_runner = True, misdetects_color = False, exec_log_path = "/wf/exec.log.zstd", names_deployment = True) + by = _by_name(groups) + _eq("order", [g.name for g in groups], [GROUP_COMMON, GROUP_RUNNER, GROUP_CACHE, GROUP_BES, GROUP_EXEC_LOG, GROUP_EXEC]) + _eq("enabled: all but exec", enabled, [GROUP_COMMON, GROUP_RUNNER, GROUP_CACHE, GROUP_BES, GROUP_EXEC_LOG]) + + # Storage is the repository cache alone; the server paths are startup lines. + _eq("runner storage", by[GROUP_RUNNER].flags, ["--repository_cache=/mnt/r"]) + + # With the marker the cache group carries the Workflows cache's specifics … + cache = _texts(by[GROUP_CACHE].flags) + for want in ["--remote_cache=grpc://cache:8980", "--disk_cache=", "--remote_timeout=3600"]: + _eq("cache has " + want, want in cache, True) + _eq("compression gated", ("--noremote_cache_compression", ">=8.0.0") in by[GROUP_CACHE].flags, True) + _eq("no bes in cache", any(["bes" in f or "profile" in f for f in cache]), False) + + # … and the BES group the profile under the job tmpdir next to its enrichment. + bes = _texts(by[GROUP_BES].flags) + for want in ["--bes_backend=grpc://bes:1985", "--profile=/wf/command.profile.gz", "--generate_json_trace_profile"]: + _eq("bes has " + want, want in bes, True) + _eq("exec log in its group", _texts(by[GROUP_EXEC_LOG].flags), ["--execution_log_compact_file=/wf/exec.log.zstd"]) + _eq("no cache in bes", any(["remote_cache" in f for f in bes]), False) + + # The executor stands alone when the cache group is already enabled. + _eq("exec", by[GROUP_EXEC].flags, ["--remote_executor=grpc://exec:8980"]) + _eq("exec source", by[GROUP_EXEC].source, "the runner's remote executor") + +def _test_exec_without_cache(_ctx): + # Without a cache the executor doubles as one: chain the (tuning-only) + # cache group, which is not otherwise enabled. + (groups, enabled) = rc_groups(WorkflowsEnvironment(remote_executor = _EXEC), on_runner = False, misdetects_color = False, exec_log_path = "/x", names_deployment = True) + by = _by_name(groups) + _eq("chains cache tuning", by[GROUP_EXEC].flags, ["--config=" + GROUP_CACHE, "--remote_executor=grpc://exec:8980"]) + _eq("cache not enabled", GROUP_CACHE in enabled, False) + _eq("exec never enabled", GROUP_EXEC in enabled, False) + +def _test_host_group(_ctx): + # Named for the detected host; exists only where the host needs something. + gha = host_group(WorkflowsEnvironment(ci = CI(host = "github")), misdetects_color = True) + _eq("github actions", [(g.name, g.flags) for g in gha], [("aspect-github-actions", ["--color=yes"])]) + _eq("gitlab", [g.name for g in host_group(WorkflowsEnvironment(ci = CI(host = "gitlab")), True)], ["aspect-gitlab"]) + _eq("buildkite needs nothing", host_group(WorkflowsEnvironment(ci = CI(host = "buildkite")), misdetects_color = False), []) + _eq("no host", host_group(WorkflowsEnvironment(), misdetects_color = True), []) + _eq("unknown host", host_group(WorkflowsEnvironment(ci = CI(host = "unknown")), misdetects_color = True), []) + _eq("every detected host has a name", sorted(HOST_GROUPS.keys()), ["buildkite", "circleci", "github", "gitlab"]) + + # The checkout's rc is shared across hosts, so with nothing in the env the + # host group is not defined; the deployment sections chain common alone. + (groups, enabled) = rc_groups(WorkflowsEnvironment(ci = CI(host = "github")), on_runner = False, misdetects_color = True, exec_log_path = "/x", names_deployment = False) + _eq("no host group in a shared rc", [g.name for g in groups], [GROUP_COMMON, GROUP_CACHE, GROUP_BES, GROUP_EXEC_LOG]) + _eq("nothing enabled", enabled, []) + _eq("sections chain common", section_chains(groups), [GROUP_COMMON]) + + # Where the env names the deployment the rc is this machine's, and the host + # group sits between common and the endpoint groups, enabled. + (groups, enabled) = rc_groups(WorkflowsEnvironment(ci = CI(host = "github"), remote_cache = _CACHE), on_runner = False, misdetects_color = True, exec_log_path = "/x", names_deployment = True) + _eq("placed", [g.name for g in groups], [GROUP_COMMON, "aspect-github-actions", GROUP_CACHE, GROUP_BES, GROUP_EXEC_LOG]) + _eq("enabled with env", enabled, [GROUP_COMMON, "aspect-github-actions", GROUP_CACHE]) + +def _test_env_names_deployment(_ctx): + # The runner marker or any one endpoint variable is the environment's answer. + def env(**vars): + return struct(var = lambda k: vars.get(k, "")) + + _eq("silent", env_names_deployment(env()), False) + _eq("marker", env_names_deployment(env(ASPECT_WORKFLOWS_RUNNER = "1")), True) + _eq("cache", env_names_deployment(env(ASPECT_WORKFLOWS_REMOTE_CACHE = "grpc://c")), True) + _eq("bes", env_names_deployment(env(ASPECT_WORKFLOWS_BES_BACKEND = "grpc://b")), True) + _eq("executor", env_names_deployment(env(ASPECT_WORKFLOWS_REMOTE_EXECUTOR = "grpc://e")), True) + _eq("ci host alone is not a deployment", env_names_deployment(env(GITHUB_ACTIONS = "true")), False) + +def _test_enabled_are_defined(_ctx): + for env, on_runner in [ + (WorkflowsEnvironment(runner = _RUNNER, remote_cache = _CACHE, build_events = _BES, remote_executor = _EXEC), True), + (WorkflowsEnvironment(remote_cache = _CACHE), False), + (WorkflowsEnvironment(runner = _RUNNER), True), + ]: + (groups, enabled) = rc_groups(env, on_runner, False, "/x", True) + defined = [g.name for g in groups] + for name in enabled: + _eq(name + " defined", name in defined, True) + +_UNIT_TESTS = [ + _test_bare_host, + _test_env_off_runner, + _test_runner, + _test_exec_without_cache, + _test_host_group, + _test_env_names_deployment, + _test_enabled_are_defined, +] + +def _test_impl(ctx): + for t in _UNIT_TESTS: + t(ctx) + print("rc_groups.axl: OK (%d tests)" % len(_UNIT_TESTS)) + return 0 + +rc_groups_tests = task( + summary = "Run the rc_groups AXL unit tests.", + kind = "test-rc-groups", + group = ["dev"], + implementation = _test_impl, + args = {}, +) diff --git a/crates/axl-runtime/src/engine/aspect/auth.rs b/crates/axl-runtime/src/engine/aspect/auth.rs index 394f7892f..f38021d51 100644 --- a/crates/axl-runtime/src/engine/aspect/auth.rs +++ b/crates/axl-runtime/src/engine/aspect/auth.rs @@ -3618,12 +3618,12 @@ fn deployment_summary_methods(registry: &mut MethodsBuilder) { fn summary_inputs() -> anyhow::Result<(Vec, ProfileCredentials, bool)> { let deployments = load_deployments()?; let creds = load_profile_credentials(&resolve_profile(None))?; - let any_configured_default = deployments.iter().any(|d| d.default && !d.builtin); - Ok((deployments, creds, any_configured_default)) + let any_default = any_configured_default(&deployments); + Ok((deployments, creds, any_default)) } fn list_deployment_summaries() -> anyhow::Result> { - let (deployments, creds, any_configured_default) = summary_inputs()?; + let (deployments, creds, any_default) = summary_inputs()?; // Reported here rather than at load: `auth status` is where a user is looking // at their deployments, so it is where a clash between two of them is // actionable. @@ -3636,7 +3636,7 @@ fn list_deployment_summaries() -> anyhow::Result> { } Ok(deployments .iter() - .map(|d| summarize_deployment(d, &creds, any_configured_default)) + .map(|d| summarize_deployment(d, &creds, any_default)) .collect()) } @@ -3650,11 +3650,7 @@ fn summarize_deployment( ) -> DeploymentSummary { let builtin = d.builtin; let entry = creds.get(&login_profile_for(d)); - let default = if builtin { - !any_configured_default - } else { - d.default - }; + let default = effective_default(d, any_configured_default); // Identity + expiry come from the stored credential (when present). let (email, display_name, status) = match entry { Some(e) => { @@ -3688,29 +3684,71 @@ fn summarize_deployment( /// such deployment is configured. Used by `configure` to render the recorded /// deployment with the same detail as `list`. fn one_deployment_summary(name: &str) -> anyhow::Result> { - let (deployments, creds, any_configured_default) = summary_inputs()?; + let (deployments, creds, any_default) = summary_inputs()?; Ok(deployments .iter() .find(|d| d.name == name) - .map(|d| summarize_deployment(d, &creds, any_configured_default))) + .map(|d| summarize_deployment(d, &creds, any_default))) } -/// The resolved Bazel-facing endpoints for `ctx.aspect.auth.deployment_endpoints(name)`: -/// `cache` (→ --remote_cache), `bes` (→ the CLI BES sink / --bes_backend), and -/// `exec` (→ --remote_executor). Each is a bare host, or "" when the deployment -/// doesn't advertise/serve that capability. `name` is the resolved deployment -/// name. Consumed by bazel-spawning tasks' `--remote` to auto-wire the flags. +/// A deployment as Bazel sees it, for `ctx.aspect.auth.deployment_endpoints(name)` +/// and `ctx.aspect.auth.deployments()`: `cache` (→ --remote_cache), `bes` (→ the +/// CLI BES sink / --bes_backend), and `exec` (→ --remote_executor). Each is a bare +/// host, or "" when the deployment doesn't advertise/serve that capability. +/// `name` is the resolved deployment name. Consumed by bazel-spawning tasks' +/// `--remote` to auto-wire the flags, and by `aspect ci bazelrc` to write them. /// /// `results_url` is a full URL rather than a host — the build-result viewer /// (→ --bes_results_url), "" when the deployment advertises no web UI. +/// +/// Built from the deployment config alone, never from the credential store, so +/// it is available where no keyring or login exists (a CI runner writing an rc). +/// `default` follows [`DeploymentSummary`]: the built-in entry is default only +/// when no configured deployment claims it. `api_token_env` names the variable +/// whose token authenticates the deployment unattended. #[derive(Debug, Display, ProvidesStaticType, NoSerialize, Allocative, Clone)] #[display("")] pub struct DeploymentEndpoints { pub name: String, + pub builtin: bool, + pub default: bool, pub cache: String, pub bes: String, pub exec: String, pub results_url: String, + pub api_token_env: String, +} + +impl DeploymentEndpoints { + fn of(d: &Deployment, any_configured_default: bool) -> Self { + Self { + name: d.name.clone(), + builtin: d.builtin, + default: effective_default(d, any_configured_default), + cache: d.endpoints.cache.clone(), + bes: d.endpoints.bes.clone(), + exec: d.endpoints.exec.clone(), + results_url: d.endpoints.results_url.clone(), + api_token_env: api_token_env_var(&d.name), + } + } +} + +/// Whether any configured (non-built-in) deployment is marked default, which +/// demotes the built-in Aspect Cloud entry from default. +fn any_configured_default(deployments: &[Deployment]) -> bool { + deployments.iter().any(|d| d.default && !d.builtin) +} + +/// Whether `d` is the deployment `--remote` targets by default: a configured +/// deployment by its own flag, the built-in entry only when no configured one +/// claims it. +fn effective_default(d: &Deployment, any_configured_default: bool) -> bool { + if d.builtin { + !any_configured_default + } else { + d.default + } } starlark_simple_value!(DeploymentEndpoints); @@ -3730,6 +3768,21 @@ fn deployment_endpoints_methods(registry: &mut MethodsBuilder) { attr_str!(this, DeploymentEndpoints, name) } + #[starlark(attribute)] + fn builtin<'v>(this: values::Value<'v>) -> anyhow::Result { + attr_bool!(this, DeploymentEndpoints, builtin) + } + + #[starlark(attribute)] + fn default<'v>(this: values::Value<'v>) -> anyhow::Result { + attr_bool!(this, DeploymentEndpoints, default) + } + + #[starlark(attribute)] + fn api_token_env<'v>(this: values::Value<'v>) -> anyhow::Result { + attr_str!(this, DeploymentEndpoints, api_token_env) + } + #[starlark(attribute)] fn cache<'v>(this: values::Value<'v>) -> anyhow::Result { attr_str!(this, DeploymentEndpoints, cache) @@ -4144,14 +4197,26 @@ fn auth_methods(registry: &mut MethodsBuilder) { heap: values::Heap<'v>, ) -> anyhow::Result> { let deployments = load_deployments()?; + let any_default = any_configured_default(&deployments); let selected = select_deployment(&deployments, deployment.into_option().as_deref())?; - Ok(heap.alloc(DeploymentEndpoints { - name: selected.name, - cache: selected.endpoints.cache, - bes: selected.endpoints.bes, - exec: selected.endpoints.exec, - results_url: selected.endpoints.results_url, - })) + Ok(heap.alloc(DeploymentEndpoints::of(&selected, any_default))) + } + + /// Every effective deployment (built-in seed + configured) as + /// [`DeploymentEndpoints`] rows, in config order. Reads only the deployment + /// config — never the credential store — so it works where `list` cannot + /// (no keyring, no login); use `list` when login status matters. + fn deployments<'v>( + #[allow(unused)] this: values::Value<'v>, + heap: values::Heap<'v>, + ) -> anyhow::Result> { + let deployments = load_deployments()?; + let any_default = any_configured_default(&deployments); + let rows: Vec = deployments + .iter() + .map(|d| DeploymentEndpoints::of(d, any_default)) + .collect(); + Ok(heap.alloc(rows)) } /// The name of the configured deployment that owns `host` (its advertised @@ -6392,6 +6457,43 @@ mod tests { assert!(!summarize_deployment(&aspect_cloud_deployment(), &creds, true).default); } + #[test] + fn deployment_endpoints_come_from_config_alone() { + // No credential store is consulted: the row carries the endpoints, the + // default marking, and the API-token variable, nothing login-derived. + let mut acme = dep("acme", true); + acme.endpoints = Endpoints { + cache: "remote.acme".to_string(), + bes: "bes.acme".to_string(), + exec: "exec.acme".to_string(), + api: String::new(), + results_url: "https://app.acme/i/".to_string(), + }; + let e = DeploymentEndpoints::of(&acme, true); + assert!(e.default && !e.builtin); + assert_eq!( + (e.cache.as_str(), e.bes.as_str(), e.exec.as_str()), + ("remote.acme", "bes.acme", "exec.acme") + ); + assert_eq!(e.results_url, "https://app.acme/i/"); + assert_eq!(e.api_token_env, api_token_env_var("acme")); + + // The built-in seed is default only when nothing configured claims it, + // the same rule `summarize_deployment` applies. + let seed = DeploymentEndpoints::of(&aspect_cloud_deployment(), false); + assert!(seed.builtin && seed.default); + assert_eq!(seed.api_token_env, API_TOKEN_ENV); + assert!(!DeploymentEndpoints::of(&aspect_cloud_deployment(), true).default); + + let deployments = vec![ + aspect_cloud_deployment(), + dep("acme", true), + dep("other", false), + ]; + assert!(any_configured_default(&deployments)); + assert!(!any_configured_default(&deployments[..1])); + } + #[test] fn is_https_guards_oidc_endpoints() { assert!(is_https("https://host/authorize")); diff --git a/crates/axl-runtime/src/engine/bazel/build.rs b/crates/axl-runtime/src/engine/bazel/build.rs index c3b440bee..f17d2bf88 100644 --- a/crates/axl-runtime/src/engine/bazel/build.rs +++ b/crates/axl-runtime/src/engine/bazel/build.rs @@ -859,10 +859,14 @@ impl Build { // find the path when it opens the BEP file. The reader-side thread // is started later — once we have the spawned child's pid in hand // for the per-invocation liveness check. + // + // Bazel's default action publishing (failed actions only) is all any + // consumer of the stream reads; per-action data comes from the + // execution log, so `--build_event_publish_all_actions` is not asked + // for. let bes_path = if build_events { let p = BuildEventStream::reserve_path()?; - cmd.arg("--build_event_publish_all_actions") - .arg("--build_event_binary_file_upload_mode=fully_async") + cmd.arg("--build_event_binary_file_upload_mode=fully_async") .arg("--build_event_binary_file") .arg(&p); Some(p) diff --git a/crates/axl-runtime/src/engine/bazel/mod.rs b/crates/axl-runtime/src/engine/bazel/mod.rs index d4e031309..b2c01b49c 100644 --- a/crates/axl-runtime/src/engine/bazel/mod.rs +++ b/crates/axl-runtime/src/engine/bazel/mod.rs @@ -1055,8 +1055,10 @@ pub(crate) fn bazel_methods(registry: &mut MethodsBuilder) { /// root here in a sub-workspace layout would read the outer /// `.bazelrc` and leak the parent project's flags. /// * `startup_flags` - Startup flags (e.g. `["--bazelrc=/path/to/extra.bazelrc"]`). - /// * `flags` - Command flags to inject as synthetic `always` options; each - /// element is a `str` or a `(flag, version_constraint)` tuple. + /// * `flags` - Command flags to inject as synthetic rc entries: a `str`, a + /// `(flag, version_condition)` tuple, or `(flag, version_condition, command)` + /// naming the section (`"build"` for an option only build-like commands + /// accept; default `always`, which every command reads). /// * `skip_config_if_missing` - `--config` names to drop if undefined. /// * `version` - Bazel version for evaluating version-gated options. When /// unset, the running Bazel is probed (only if a gated option exists). @@ -1100,7 +1102,7 @@ pub(crate) fn bazel_methods(registry: &mut MethodsBuilder) { /// /// # Arguments /// * `startup_flags` - Startup flags carried by the run command. - /// * `flags` - Command flags, same `str | (str, str)` shape as `parse_rc`. + /// * `flags` - Command flags, same `str | (str, str) | (str, str, str)` shape as `parse_rc`. /// * `version` - Bazel version for evaluating version-gated options. fn new_rc<'v>( #[allow(unused)] this: values::Value<'v>, diff --git a/crates/axl-runtime/src/engine/std/process.rs b/crates/axl-runtime/src/engine/std/process.rs index 723d4b87e..5554773dd 100644 --- a/crates/axl-runtime/src/engine/std/process.rs +++ b/crates/axl-runtime/src/engine/std/process.rs @@ -95,6 +95,36 @@ pub(crate) fn process_methods(registry: &mut MethodsBuilder) { u8::try_from(code).map_err(|_| anyhow!("exit code must be 0..=255, got {code}"))?; Err(TaskExit::new(code, message.into_option().map(str::to_owned)).into()) } + + /// The absolute path `command(name)` would run, or `None` when no `PATH` + /// entry holds an executable of that name — `which` / `command -v`. + /// + /// A bare name is looked up along `PATH`; a name containing a path + /// separator is checked as given. On Unix the file must carry an execute + /// bit; on Windows the `PATHEXT` extensions are tried. + /// + /// **Examples** + /// + /// ```python + /// helper = "aspect" if ctx.std.process.which("aspect") else ctx.std.env.current_exe() + /// ``` + fn which<'v>( + #[allow(unused)] this: values::Value<'v>, + #[starlark(require = pos)] name: &str, + heap: Heap<'v>, + ) -> anyhow::Result>> { + Ok( + match find_executable(name, std::env::var_os("PATH").as_deref()) { + Some(path) => NoneOr::Other( + heap.alloc_str( + path.to_str() + .ok_or_else(|| anyhow::anyhow!("path of `{name}` is non utf-8"))?, + ), + ), + None => NoneOr::None, + }, + ) + } } #[derive(Debug, Display, Trace, ProvidesStaticType, NoSerialize, Allocative)] @@ -611,3 +641,90 @@ mod tests { assert_eq!(description, r#"program "--flag1" "--flag2" "with spaces""#); } } + +/// Resolve `name` the way a shell does against `path` (the `PATH` value): a +/// bare name against each entry in order, a name with a separator as given. +/// The first existing executable wins. Behind `which` above. +fn find_executable(name: &str, path: Option<&std::ffi::OsStr>) -> Option { + let candidate = std::path::Path::new(name); + if candidate.components().count() > 1 { + return executable_variants(candidate) + .into_iter() + .find(|p| is_executable(p)); + } + std::env::split_paths(path?) + .filter(|dir| !dir.as_os_str().is_empty()) + .flat_map(|dir| executable_variants(&dir.join(name))) + .find(|p| is_executable(p)) +} + +#[cfg(unix)] +fn executable_variants(base: &std::path::Path) -> Vec { + vec![base.to_path_buf()] +} + +#[cfg(windows)] +fn executable_variants(base: &std::path::Path) -> Vec { + let mut variants = vec![base.to_path_buf()]; + if let Some(exts) = std::env::var_os("PATHEXT") { + for ext in std::env::split_paths(&exts) { + let mut with_ext = base.as_os_str().to_owned(); + with_ext.push(ext.as_os_str()); + variants.push(std::path::PathBuf::from(with_ext)); + } + } + variants +} + +#[cfg(unix)] +fn is_executable(path: &std::path::Path) -> bool { + use std::os::unix::fs::PermissionsExt; + path.metadata() + .map(|m| m.is_file() && m.permissions().mode() & 0o111 != 0) + .unwrap_or(false) +} + +#[cfg(windows)] +fn is_executable(path: &std::path::Path) -> bool { + path.is_file() +} + +#[cfg(all(test, unix))] +mod which_tests { + use super::find_executable; + use std::os::unix::fs::PermissionsExt; + + fn file(dir: &std::path::Path, name: &str, mode: u32) -> std::path::PathBuf { + let p = dir.join(name); + std::fs::write(&p, "#!/bin/sh\n").unwrap(); + std::fs::set_permissions(&p, std::fs::Permissions::from_mode(mode)).unwrap(); + p + } + + #[test] + fn which_walks_path_in_order_and_requires_an_execute_bit() { + let first = tempfile::tempdir().unwrap(); + let second = tempfile::tempdir().unwrap(); + file(first.path(), "aspect", 0o644); + let runnable = file(second.path(), "aspect", 0o755); + let path = std::env::join_paths([first.path(), second.path()]).unwrap(); + + // The non-executable file in the first entry is skipped; the second wins. + assert_eq!( + find_executable("aspect", Some(&path)), + Some(runnable.clone()) + ); + assert_eq!(find_executable("bazel", Some(&path)), None); + assert_eq!(find_executable("aspect", None), None); + + // A name with a separator is checked as given, not along PATH. + assert_eq!( + find_executable(runnable.to_str().unwrap(), Some(&path)), + Some(runnable) + ); + assert_eq!( + find_executable(first.path().join("aspect").to_str().unwrap(), Some(&path)), + None + ); + } +} diff --git a/crates/bazelrc/src/lib.rs b/crates/bazelrc/src/lib.rs index 40e39a76c..12755430b 100644 --- a/crates/bazelrc/src/lib.rs +++ b/crates/bazelrc/src/lib.rs @@ -51,6 +51,29 @@ pub struct RcOption { pub version_condition: Option, } +/// Caller-supplied flags become synthetic rc entries under the section each +/// names, or `always` when it names none, so a task can scope a flag to the +/// commands that accept it (`build` reaches test/run/coverage/cquery/aquery +/// through `command_ancestors`, never `query`). +fn push_caller_flags( + options: &mut HashMap>, + flags: &[RcOption], + source_index: usize, +) { + for flag in flags { + let command = if flag.command.is_empty() { + "always".to_owned() + } else { + flag.command.clone() + }; + options.entry(command.clone()).or_default().push(RcOption { + source_index, + command, + ..flag.clone() + }); + } +} + impl fmt::Display for RcOption { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "[{}] {}", self.command, self.value)?; @@ -176,19 +199,13 @@ impl BazelRC { )?; } - // Append caller-supplied flags as synthetic `always` options so they participate in - // options_for() and expand_configs() like any rc-file entry. + // Append caller-supplied flags as synthetic rc entries so they participate in + // options_for() and expand_configs() like any rc-file entry: under `always` + // unless the flag names its own command section. if !flags.is_empty() { let cli_source_index = sources.len(); sources.push(PathBuf::from("")); - let always_opts = options.entry("always".to_owned()).or_default(); - for flag in flags { - always_opts.push(RcOption { - source_index: cli_source_index, - command: "always".to_owned(), - ..flag.clone() - }); - } + push_caller_flags(&mut options, flags, cli_source_index); } Ok(BazelRC { @@ -244,14 +261,7 @@ impl BazelRC { if !flags.is_empty() { let cli_source_index = 0; sources.push(PathBuf::from("")); - let always_opts = options.entry("always".to_owned()).or_default(); - for flag in flags { - always_opts.push(RcOption { - source_index: cli_source_index, - command: "always".to_owned(), - ..flag.clone() - }); - } + push_caller_flags(&mut options, flags, cli_source_index); } BazelRC { options, @@ -881,9 +891,14 @@ impl<'v> UnpackValue<'v> for RcOption { { let mut items = tup.items.into_iter(); if let Some(flag) = items.next() { + // `(flag, version_condition)` or `(flag, version_condition, command)`; + // an empty condition means none, an empty command means `always`. + let version_condition = items.next().filter(|cond| !cond.is_empty()); + let command = items.next().unwrap_or_default(); return Ok(Some(RcOption { value: flag, - version_condition: items.next(), + version_condition, + command, ..RcOption::default() })); } @@ -1892,6 +1907,33 @@ build --build-flag assert_eq!(opts, vec!["--always-flag", "--common-flag", "--build-flag"]); } + #[test] + fn caller_flag_scoped_to_build_skips_query() { + // A caller flag naming its section applies where an rc-file entry in that + // section would: `build` reaches test but not query. Unscoped flags stay + // `always` and reach both. + let scoped = RcOption { + value: "--execution_log_compact_file=/tmp/exec.log".to_owned(), + command: "build".to_owned(), + ..RcOption::default() + }; + let unscoped = RcOption { + value: "--remote_timeout=3600".to_owned(), + ..RcOption::default() + }; + let rc = BazelRC::blank(&[scoped, unscoped]); + let values = |command: &str| -> Vec { rc.resolve_for_command(command).unwrap().1 }; + assert_eq!( + values("test"), + vec![ + "--execution_log_compact_file=/tmp/exec.log", + "--remote_timeout=3600" + ] + ); + assert_eq!(values("query"), vec!["--remote_timeout=3600"]); + assert_eq!(values("build").len(), 2); + } + #[test] fn test_inherits_build_flags() { let dir = make_workspace();