diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b1eb78e..8fa5efc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -64,6 +64,32 @@ jobs: - name: Build sdist and wheel run: python -m build + # Only one release/tag is ever kept per branch — delete whatever this + # branch's previous release+tag was (main: a plain "v*" tag; staging: + # a "v*-staging" tag) before creating the new one below. Best-effort: + # a delete failure (e.g. nothing to delete yet) must not fail the + # release itself. + - name: Delete this branch's previous release and tag + env: + GH_TOKEN: ${{ github.token }} + run: | + set +e + if [ "${{ github.ref_name }}" = "staging" ]; then + SUFFIX="-staging" + else + SUFFIX="" + fi + for TAG in $(gh release list --limit 100 --json tagName -q '.[].tagName'); do + case "$TAG" in + *-staging) TAG_SUFFIX="-staging" ;; + *) TAG_SUFFIX="" ;; + esac + if [ "$TAG_SUFFIX" = "$SUFFIX" ]; then + echo "Deleting previous release/tag for ${{ github.ref_name }}: $TAG" + gh release delete "$TAG" --cleanup-tag -y + fi + done + - name: Create GitHub release uses: softprops/action-gh-release@v3 with: diff --git a/README.md b/README.md index 99edc24..b8aa20c 100644 --- a/README.md +++ b/README.md @@ -19,10 +19,17 @@ Two main class domains for the EEA data lakehouse: ## Install -Install directly from GitHub, pinned to a released tag: +[![Latest release](https://img.shields.io/github/v/release/eeadata/EEALakeHouse.python?label=latest%20release)](https://github.com/eeadata/EEALakeHouse.python/releases/latest) + +The badge above always shows the current latest release tag — substitute it for `v0.1.5` below +if it's moved on since this was written (or check the [Releases page](https://github.com/eeadata/EEALakeHouse.python/releases/latest) directly). ```bash -pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@v0.1.0" +# latest release (currently v0.1.5) — recommended: stable, pinned to a tag +pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@v0.1.5" + +# latest main — bleeding edge, whatever's currently merged, not pinned to a release +pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@main" ``` ## Usage @@ -169,12 +176,23 @@ pytest Pushing to `main` without bumping the version re-publishes the release for the current version with the latest build artifacts. +**Only one release/tag exists per branch at a time** — `main` always has exactly one `vX.Y.Z` +release, `staging` exactly one `vX.Y.Z-staging` prerelease. Each new release deletes its branch's +previous release *and* tag first, so **tags aren't permanent** — pin to whatever the +[Install](#install) badge shows *now*, not to an old tag number, since it won't exist once a +newer release replaces it. + ## Install in JupyterLab -Run this in a notebook cell, pinned to the release tag you want: +Run this in a notebook cell — see the badge under [Install](#install) for the current latest +release tag (`v0.1.5` as of this writing): ```python -%pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@v0.1.0" +# latest release (currently v0.1.5) — recommended +%pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@v0.1.5" + +# latest main — bleeding edge, not pinned to a release +%pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@main" ``` Use the `%pip` magic rather than `!pip` — it installs into the kernel the @@ -186,9 +204,9 @@ Kernel...**) so the import below picks up the newly installed package: from eea_datalakehouse.dds_ingestion import FolderIngest ``` -Alternatively, download the wheel attached to the GitHub Release page for -that tag and install the local file instead of pulling from git: +Alternatively, download the wheel attached to the [GitHub Release page](https://github.com/eeadata/EEALakeHouse.python/releases/latest) +for that tag and install the local file instead of pulling from git: ```python -%pip install /path/to/EEADataLakehouse-0.1.0-py3-none-any.whl +%pip install /path/to/EEADataLakehouse-0.1.5-py3-none-any.whl ``` diff --git a/debugger/debug_run.py b/debugger/debug_run.py index b9cfc67..5f08b1f 100644 --- a/debugger/debug_run.py +++ b/debugger/debug_run.py @@ -27,6 +27,16 @@ import os import json import httpx + + + +import argparse +import atexit +import subprocess +import sys + +import requests + from pathlib import Path from xmlrpc.client import Boolean from eea_datalakehouse.catalog import Catalog @@ -38,7 +48,12 @@ BASE_URL = "https://dds.debug.local" # This script always lives directly in debugger/, so its own directory *is* # the folder to look in — independent of whatever cwd the debugger/terminal -# happened to launch with. +# happened to launch with. +CACHE_PATH = os.path.expanduser("~/.dremio_msal_cache.json") +TOKEN_EXCHANGE = "urn:ietf:params:oauth:grant-type:token-exchange" +JWT_TYPE = "urn:ietf:params:oauth:token-type:jwt" +PAT_TYPE = "urn:ietf:params:oauth:token-type:dremio:personal-access-token" + ENV_PATH = Path(__file__).resolve().parent / ".env" @@ -346,6 +361,126 @@ def run_catalog_bulk_close() -> None: print(f"catalog_bulk_close after a.closed={a._closed} b.closed={b._closed}") + +# -------------------------------------------------------------------------- +# Step 1 — get an Entra ID JWT +# -------------------------------------------------------------------------- +def entra_token_azcli(scope: str) -> str: + """Reuse the Azure CLI's cached login. Silent, no prompts.""" + out = subprocess.run( + ["az", "account", "get-access-token", "--scope", scope, "-o", "json"], + capture_output=True, + text=True, + check=True, + ) + return json.loads(out.stdout)["accessToken"] + + +def _msal_app(cls, **kwargs): + import msal + + cache = msal.SerializableTokenCache() + if os.path.exists(CACHE_PATH): + cache.deserialize(open(CACHE_PATH).read()) + + def _flush(): + if cache.has_state_changed: + fd = os.open(CACHE_PATH, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as fh: + fh.write(cache.serialize()) + + atexit.register(_flush) + return cls(token_cache=cache, **kwargs) + + +def entra_token_device(tenant: str, client_id: str, scope: str) -> str: + """Device-code flow with a persistent cache: interactive once, silent after.""" + import msal + + app = _msal_app( + msal.PublicClientApplication, + client_id=client_id, + authority=f"https://login.microsoftonline.com/{tenant}", + ) + + accounts = app.get_accounts() + result = app.acquire_token_silent([scope], account=accounts[0]) if accounts else None + + if not result: + flow = app.initiate_device_flow(scopes=[scope]) + if "user_code" not in flow: + raise RuntimeError(f"device flow failed: {flow}") + print(flow["message"], file=sys.stderr) + result = app.acquire_token_by_device_flow(flow) + + if "access_token" not in result: + raise RuntimeError(f"Entra ID error: {result.get('error_description', result)}") + return result["access_token"] + + +def entra_token_sp(tenant: str, client_id: str, client_secret: str, scope: str) -> str: + """Client credentials — fully unattended, app identity.""" + import msal + + app = msal.ConfidentialClientApplication( + client_id=client_id, + client_credential=client_secret, + authority=f"https://login.microsoftonline.com/{tenant}", + ) + result = app.acquire_token_for_client(scopes=[scope]) + if "access_token" not in result: + raise RuntimeError(f"Entra ID error: {result.get('error_description', result)}") + return result["access_token"] + + +# -------------------------------------------------------------------------- +# Step 2 — exchange the Entra JWT for a Dremio access token +# -------------------------------------------------------------------------- +def dremio_exchange(host: str, subject_token: str, token_type: str = JWT_TYPE, + verify: bool | str = True) -> dict: + r = requests.post( + f"https://{host}/oauth/token", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + data={ + "grant_type": TOKEN_EXCHANGE, + "subject_token": subject_token, + "subject_token_type": token_type, + "scope": "dremio.all", + }, + timeout=30, + verify=verify, + ) + if not r.ok: + raise RuntimeError(f"Dremio token exchange failed [{r.status_code}]: {r.text}") + return r.json() + + +# -------------------------------------------------------------------------- +# Step 3 — optional: mint a long-lived PAT with that access token +# -------------------------------------------------------------------------- +def create_pat(host: str, access_token: str, username: str, label: str, + days: int = 90, verify: bool | str = True) -> dict: + h = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} + + u = requests.get(f"https://{host}/api/v3/user/by-name/{username}", + headers=h, timeout=30, verify=verify) + if not u.ok: + raise RuntimeError(f"user lookup failed [{u.status_code}]: {u.text}") + user_id = u.json()["id"] + + p = requests.post( + f"https://{host}/api/v3/user/{user_id}/token", + headers=h, + json={"label": label, "millisecondsToExpire": days * 86_400_000}, + timeout=30, + verify=verify, + ) + if not p.ok: + raise RuntimeError(f"PAT creation failed [{p.status_code}]: {p.text}") + return p.json() + + + if __name__ == "__main__": if not ENV_PATH.exists(): raise RuntimeError(f"could not find {ENV_PATH}") @@ -458,8 +593,59 @@ def run_catalog_bulk_close() -> None: #run_deleteview() #run_createfolder() - run_deletefolder() + #run_deletefolder() + + + + + + + + + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--mode", choices=["azcli", "device", "sp"], default="device") + ap.add_argument("--host", default=os.getenv("DREMIO_HOST")) + ap.add_argument("--tenant", default=os.getenv("ENTRA_TENANT_ID")) + ap.add_argument("--client-id", default=os.getenv("ENTRA_CLIENT_ID")) + ap.add_argument("--client-secret", default=os.getenv("ENTRA_CLIENT_SECRET")) + ap.add_argument("--scope", default=os.getenv("ENTRA_SCOPE")) + ap.add_argument("--create-pat", metavar="USERNAME", + help="also mint a PAT for this Dremio username") + ap.add_argument("--pat-label", default="automated") + ap.add_argument("--pat-days", type=int, default=90, help="1-180, default 90") + ap.add_argument("--ca-bundle", help="path to CA bundle for self-signed Dremio certs") + ap.add_argument("--insecure", action="store_true", help="skip TLS verification") + args = ap.parse_args() + + missing = [n for n, v in [("--host", args.host), ("--scope", args.scope)] if not v] + if args.mode != "azcli": + missing += [n for n, v in [("--tenant", args.tenant), + ("--client-id", args.client_id)] if not v] + if args.mode == "sp" and not args.client_secret: + missing.append("--client-secret") + if missing: + ap.error("missing required: " + ", ".join(missing)) + + verify: bool | str = args.ca_bundle or (not args.insecure) + + if args.mode == "azcli": + jwt = entra_token_azcli(args.scope) + elif args.mode == "device": + jwt = entra_token_device(args.tenant, args.client_id, args.scope) + else: + jwt = entra_token_sp(args.tenant, args.client_id, args.client_secret, args.scope) + tok = dremio_exchange(args.host, jwt, verify=verify) + access_token = tok["access_token"] + print(f"# Dremio access token (expires in {tok.get('expires_in')}s)", file=sys.stderr) + print(access_token) + if args.create_pat: + pat = create_pat(args.host, access_token, args.create_pat, + args.pat_label, args.pat_days, verify=verify) + print("# PAT (store it now, it is not retrievable again)", file=sys.stderr) + print(pat.get("token") or json.dumps(pat)) + print ("OK")