diff --git a/README.md b/README.md index 0af89e5..7744727 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,16 @@ When you are done, `cd ..` out of the folder and run `labagent rag destroy --dir On Windows PowerShell use `$HOME\rag-lab` and `cd $HOME\rag-lab`. If you installed `rag-local` instead, add `pip install jupyterlab`, or open the notebooks in VS Code and pick the Python environment you installed into as the kernel. See [docs/rag-local.md](docs/rag-local.md). +## Azure Labs: move Azure VMs to another resource group + +Lift and shift VMs, disks and network from one Azure resource group to another with snapshots, or snapshot a dead VM or disk into a group of its own. It never changes the source. +An alternative for people leaving Azure Lab Services (retiring June 28, 2027), not Azure Lab Services itself. [Guide](docs/azure-labs.md) + +```bash +labagent launch azure-labs --name move1 --param source_rg=azure-labs-legacy --param target_rg=azure-labs-new +labagent destroy azure-labs --name move1 # deletes only what that run created +``` + ## AWS sandboxes ```bash diff --git a/docs/azure-labs.md b/docs/azure-labs.md new file mode 100644 index 0000000..dba2197 --- /dev/null +++ b/docs/azure-labs.md @@ -0,0 +1,99 @@ +# Azure Labs: move Azure VMs from one resource group to another + +A simple lift and shift for Azure VMs. It copies the VMs, their disks and their network into another resource group, and it never changes the source. +It can also just snapshot disks, for an old VM or disk you want to keep somewhere else. + +This is an alternative for people moving off Azure Lab Services (retiring June 28, 2027). It is not Azure Lab Services and not from Microsoft. +Lab VMs are ordinary Azure VMs, so this works for any VM in a resource group, not only lab VMs. + +## What you need + +- The Azure CLI, signed in: `az login`. +- Your subscription ID in the allow list. labagent refuses to run on any other subscription. + +```bash +export LABAGENT_ALLOWED_AZURE_SUBSCRIPTIONS=$(az account show --query id -o tsv) +``` + +PowerShell: `$env:LABAGENT_ALLOWED_AZURE_SUBSCRIPTIONS = az account show --query id -o tsv` + +## Copy VMs to another resource group + +```bash +labagent launch azure-labs --name move1 --param source_rg=azure-labs-legacy --param target_rg=azure-labs-new +``` + +| Parameter | Meaning | +|---|---| +| `source_rg` | Group to copy from. It is only read. | +| `target_rg` | Group to copy into. Created if it does not exist. | +| `vms` | Comma list of VM names. Default: every VM in the source group. | +| `snapshot_only` | `true` copies only disk snapshots (no VMs, no network). With no `vms`, it also takes disks that no VM is using. | +| `subscription` | Subscription ID or name. Default: the one `az login` selected. | + +What it does, in order: reads the source, checks vCPU quota, creates the target group if needed, snapshots every disk into the target, +copies the virtual network, subnets, network security groups (with their rules), public IPs and NICs, creates disks from the snapshots, +creates the VMs on those disks, and checks that each VM is provisioned. Every copied resource keeps its name and gets these tags: +`labagent=1`, `pack=azure-labs`, `name`, `migrated_from`, `created_at`. + +Run it again after a failure: it skips what already exists and carries on. Add `--dry-run` to see the steps without running them. + +The new VMs boot the same operating system with the same accounts and data (a specialized copy), so no password is needed or created. +They start running. The source VMs keep running too, and both count against your vCPU quota. Stop the sources, or copy fewer VMs at a time with `vms=`. + +## Snapshot an old VM or disk + +```bash +labagent launch azure-labs --name old1 --param source_rg=azure-labs-legacy --param target_rg=azure-labs-archive --param snapshot_only=true +``` + +Snapshots land in the target group as `snap-`. Create a disk or VM from one whenever you need it. + +## Delete + +```bash +labagent destroy azure-labs --name move1 +``` + +Deletes only what that migration created in the target group (found by its tags), dependants first. It deletes the target group too if labagent created it +and nothing else is in it. Anything in the group without the tags is reported and left alone. The source group is never touched. + +## What it does not copy + +It stops before creating anything, and lists the reasons, if a VM is in an availability set or scale set, uses an unmanaged VHD, +uses encrypted disks with your own keys, has a NIC with more than one IP configuration, or uses application security groups. + +- Public IP addresses change: you get a new address with the same name. +- Data disk order is kept, but LUN numbers and caching settings are not. +- VM extensions, managed identities, boot diagnostics settings and backups are not copied. +- Same subscription and same region only. Copying across subscriptions is not built. +- Managed images are not copied. Create a VM from the image, then snapshot that. + +## Try it on a test group + +Create a small legacy group first. Three B1s VMs fit a 10-vCPU quota twice over. + +```bash +az group create -n azure-labs-legacy -l canadacentral +az network nsg create -g azure-labs-legacy -n legacy-nsg +az network nsg rule create -g azure-labs-legacy --nsg-name legacy-nsg -n allow-ssh --priority 100 --destination-port-ranges 22 --access Allow --protocol Tcp +az network vnet create -g azure-labs-legacy -n legacy-vnet --address-prefixes 10.30.0.0/16 --subnet-name app --subnet-prefixes 10.30.1.0/24 --network-security-group legacy-nsg +for i in 1 2 3; do + az vm create -g azure-labs-legacy -n legacy-vm$i --image Ubuntu2204 --size Standard_B1s --vnet-name legacy-vnet --subnet app \ + --public-ip-address "" --nsg "" --admin-username azureuser --generate-ssh-keys +done +az vm disk attach -g azure-labs-legacy --vm-name legacy-vm1 --name legacy-vm1-data --new --size-gb 8 +az disk create -g azure-labs-legacy -n old-dead-disk --size-gb 4 +``` + +Then run the copy, check the result, and delete it: + +```bash +labagent launch azure-labs --name test1 --param source_rg=azure-labs-legacy --param target_rg=azure-labs-new +az vm list -g azure-labs-new -d -o table # 3 VMs, running +az resource list -g azure-labs-legacy -o table # unchanged +labagent destroy azure-labs --name test1 +az group exists -n azure-labs-new # false +``` + +Clean up the test source with `az group delete -n azure-labs-legacy --yes`. diff --git a/labagent/packs/azure_labs/__init__.py b/labagent/packs/azure_labs/__init__.py new file mode 100644 index 0000000..b73b788 --- /dev/null +++ b/labagent/packs/azure_labs/__init__.py @@ -0,0 +1 @@ +"""azure-labs: lift and shift Azure VMs from one resource group to another with snapshots. Not Azure Lab Services.""" diff --git a/labagent/packs/azure_labs/azcli.py b/labagent/packs/azure_labs/azcli.py new file mode 100644 index 0000000..4cdb79d --- /dev/null +++ b/labagent/packs/azure_labs/azcli.py @@ -0,0 +1,148 @@ +"""Thin wrapper over the `az` CLI. The runner is injectable so tests never touch Azure. Error text is redacted.""" +from __future__ import annotations + +import json +import re +import shutil +import subprocess +from typing import Any, Dict, List, Optional + +from ..base import PackError + +SECRET = re.compile(r"(eyJ[\w-]{20,}\.[\w-]{10,}\.[\w-]{10,}|(?i:password|secret|key|token)\S*\s*[=:]\s*\S+)") + + +def redact(text: str, tail: int = 6) -> str: + return SECRET.sub("", "\n".join(str(text).strip().splitlines()[-tail:]))[:600] + + +def tag_args(tags: Dict[str, str]) -> List[str]: + return ["--tags", *[f"{k}={v}" for k, v in tags.items()]] + + +class AzCli: + def __init__(self, subscription: Optional[str] = None, run=subprocess.run, binary: Optional[str] = None): + self.subscription = subscription + self._run = run + self._bin = binary + + def binary(self) -> str: + b = self._bin or shutil.which("az") + if not b: + raise PackError("Azure CLI not found. Install it (https://learn.microsoft.com/cli/azure/install-azure-cli) and run: az login") + return b + + def json(self, *args: str) -> Any: + cmd = [self.binary(), *args, "--only-show-errors", "-o", "json"] + if self.subscription: + cmd += ["--subscription", self.subscription] + p = self._run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace") + if p.returncode != 0: + raise PackError(f"az {' '.join(args[:2])} failed: {redact(p.stderr or p.stdout)}") + return json.loads(p.stdout) if (p.stdout or "").strip() else {} + + # ---- account and capacity ------------------------------------------------------------------------------------------ + def account(self) -> Dict[str, Any]: + try: + return self.json("account", "show") + except PackError as e: + raise PackError(f"could not read the Azure login. Run: az login ({e})") from None + + def usage(self, region: str) -> Dict[str, Dict[str, int]]: + rows = self.json("vm", "list-usage", "-l", region) + return {r["name"]["value"]: {"used": int(r["currentValue"]), "limit": int(r["limit"])} for r in rows} + + def skus(self, region: str) -> Dict[str, Dict[str, Any]]: + """Every VM size in the region: family, vCPUs, restricted. One call per run (az filters on the client, so it takes about a minute).""" + cache = self.__dict__.setdefault("_sku_cache", {}) + if region not in cache: + rows = self.json("vm", "list-skus", "-l", region, "--resource-type", "virtualMachines", "--query", + "[].{name:name,family:family,restricted:length(restrictions),vcpus:capabilities[?name=='vCPUs'].value|[0]}") + cache[region] = {r["name"]: r for r in rows} + return cache[region] + + # ---- groups and resources ----------------------------------------------------------------------------------------- + def group_exists(self, name: str) -> bool: + return bool(self.json("group", "exists", "-n", name)) + + def group_show(self, name: str) -> Optional[Dict[str, Any]]: + return self.json("group", "show", "-n", name) if self.group_exists(name) else None + + def group_create(self, name: str, region: str, tags: Dict[str, str]) -> None: + self.json("group", "create", "-n", name, "-l", region, *tag_args(tags)) + + def group_delete(self, name: str) -> None: + self.json("group", "delete", "-n", name, "--yes") + + def resources(self, group: str) -> List[Dict[str, Any]]: + return self.json("resource", "list", "-g", group) or [] + + def resource_delete(self, rid: str) -> None: + self.json("resource", "delete", "--ids", rid) + + def show(self, rid: str) -> Dict[str, Any]: + return self.json("resource", "show", "--ids", rid) + + # ---- reading the source (read-only) --------------------------------------------------------------------------------- + def list_vms(self, group: str) -> List[Dict[str, Any]]: + return self.json("vm", "list", "-g", group) or [] + + def list_disks(self, group: str) -> List[Dict[str, Any]]: + return self.json("disk", "list", "-g", group) or [] + + def vm_state(self, group: str, name: str) -> Dict[str, Any]: + return self.json("vm", "show", "-d", "-g", group, "-n", name) + + # ---- creating in the target --------------------------------------------------------------------------------------- + def snapshot_create(self, group: str, name: str, source_id: str, location: str, tags: Dict[str, str], hyper_v: Optional[str] = None) -> None: + extra = ["--hyper-v-generation", hyper_v] if hyper_v else [] + self.json("snapshot", "create", "-g", group, "-n", name, "--source", source_id, "-l", location, "--incremental", "false", *extra, *tag_args(tags)) + + def disk_create(self, group: str, name: str, snapshot_id: str, sku: str, location: str, tags: Dict[str, str], + zone: Optional[str] = None, hyper_v: Optional[str] = None, os_type: Optional[str] = None) -> None: + extra = (["--zone", zone] if zone else []) + (["--hyper-v-generation", hyper_v] if hyper_v else []) + (["--os-type", os_type] if os_type else []) + self.json("disk", "create", "-g", group, "-n", name, "--source", snapshot_id, "--sku", sku, "-l", location, *extra, *tag_args(tags)) + + def nsg_create(self, group: str, name: str, location: str, tags: Dict[str, str]) -> None: + self.json("network", "nsg", "create", "-g", group, "-n", name, "-l", location, *tag_args(tags)) + + def nsg_rule_create(self, group: str, nsg: str, rule: Dict[str, Any]) -> None: + args = ["network", "nsg", "rule", "create", "-g", group, "--nsg-name", nsg, "-n", rule["name"], "--priority", str(rule["priority"]), + "--direction", rule["direction"], "--access", rule["access"], "--protocol", rule["protocol"]] + for flag, key in (("--source-address-prefixes", "source_addresses"), ("--source-port-ranges", "source_ports"), + ("--destination-address-prefixes", "destination_addresses"), ("--destination-port-ranges", "destination_ports")): + args += [flag, *rule[key]] + self.json(*args) + + def vnet_create(self, group: str, name: str, prefixes: List[str], subnet: str, subnet_prefix: str, location: str, tags: Dict[str, str]) -> None: + self.json("network", "vnet", "create", "-g", group, "-n", name, "--address-prefixes", *prefixes, "--subnet-name", subnet, + "--subnet-prefixes", subnet_prefix, "-l", location, *tag_args(tags)) + + def subnet_create(self, group: str, vnet: str, name: str, prefix: str) -> None: + self.json("network", "vnet", "subnet", "create", "-g", group, "--vnet-name", vnet, "-n", name, "--address-prefixes", prefix) + + def subnet_set_nsg(self, group: str, vnet: str, subnet: str, nsg: str) -> None: + self.json("network", "vnet", "subnet", "update", "-g", group, "--vnet-name", vnet, "-n", subnet, "--network-security-group", nsg) + + def public_ip_create(self, group: str, name: str, location: str, tags: Dict[str, str]) -> None: + self.json("network", "public-ip", "create", "-g", group, "-n", name, "-l", location, "--sku", "Standard", "--allocation-method", "Static", *tag_args(tags)) + + def nic_create(self, group: str, name: str, vnet: str, subnet: str, location: str, tags: Dict[str, str], + nsg: Optional[str] = None, public_ip: Optional[str] = None, accelerated: bool = False) -> None: + args = ["network", "nic", "create", "-g", group, "-n", name, "--vnet-name", vnet, "--subnet", subnet, "-l", location] + if nsg: + args += ["--network-security-group", nsg] + if public_ip: + args += ["--public-ip-address", public_ip] + if accelerated: + args += ["--accelerated-networking", "true"] + self.json(*args, *tag_args(tags)) + + def vm_create(self, group: str, name: str, os_disk: str, os_type: str, size: str, nics: List[str], data_disks: List[str], + location: str, tags: Dict[str, str], zone: Optional[str] = None) -> None: + args = ["vm", "create", "-g", group, "-n", name, "--attach-os-disk", os_disk, "--os-type", os_type, "--size", size, "--nics", *nics, "-l", location] + if data_disks: + args += ["--attach-data-disks", *data_disks] + if zone: + args += ["--zone", zone] + self.json(*args, *tag_args(tags)) diff --git a/labagent/packs/azure_labs/lifecycle.py b/labagent/packs/azure_labs/lifecycle.py new file mode 100644 index 0000000..1b66a2c --- /dev/null +++ b/labagent/packs/azure_labs/lifecycle.py @@ -0,0 +1,68 @@ +"""vCPU quota arithmetic and tag-only teardown.""" +from __future__ import annotations + +from typing import Any, Callable, Dict + +from ..base import PackError +from . import safety + +# Delete in this order so dependants go first. +ORDER = ["microsoft.compute/virtualmachines", "microsoft.network/networkinterfaces", "microsoft.compute/disks", "microsoft.compute/snapshots", + "microsoft.network/publicipaddresses", "microsoft.network/virtualnetworks", "microsoft.network/networksecuritygroups"] +ROUNDS = 4 + + +def check_quota(usage: Dict[str, Dict[str, int]], needs: Dict[str, int], region: str) -> None: + """`needs` is vCPUs per VM family. Also checks the regional total.""" + def free(key: str) -> int: + u = usage.get(key) + return (u["limit"] - u["used"]) if u else 0 + total = sum(needs.values()) + short = [f"total regional vCPUs: need {total}, free {free('cores')}"] if free("cores") < total else [] + short += [f"{fam}: need {n}, free {free(fam)}" for fam, n in sorted(needs.items()) if free(fam) < n] + if short: + raise PackError(f"not enough vCPU quota in {region} ({'; '.join(short)}). The source VMs still count while they run: stop them, " + "migrate fewer VMs at once (vms=a,b), or ask for more in the Azure portal under Subscriptions > Usage + quotas.") + + +def _rank(rtype: str) -> int: + t = rtype.lower() + return ORDER.index(t) if t in ORDER else len(ORDER) + + +def destroy_owned(az: Any, group: str, name: str, log: Callable[[str], None] = lambda m: None) -> Dict[str, Any]: + """Delete only the resources in `group` that carry this migration's tags, then the group if labagent created it and it is empty. + + Raises PackError, deleting nothing more, when something without these tags is left. The source group is never passed here.""" + info = az.group_show(group) + if info is None: + return {"group_deleted": False, "resources_deleted": 0} + deleted = 0 + for _ in range(ROUNDS): + mine = [r for r in az.resources(group) if safety.owned(r.get("tags"), name) and r["type"].count("/") == 1] + if not mine: + break + failed = 0 + for r in sorted(mine, key=lambda x: _rank(x["type"])): + try: + az.resource_delete(r["id"]) + deleted += 1 + log(f" deleted {r['type'].split('/')[-1]} {r['name']}") + except PackError: + failed += 1 # usually a dependency that is not gone yet: the next round retries + if not failed: + break + left = [r for r in az.resources(group) if r["type"].count("/") == 1] + mine_left = [r["name"] for r in left if safety.owned(r.get("tags"), name)] + foreign = [r["name"] for r in left if not safety.owned(r.get("tags"), name)] + if mine_left: + raise PackError(f"could not delete {', '.join(sorted(mine_left))} in {group}. Run destroy again.") + created = (info.get("tags") or {}).get("created_by_labagent") == "1" + if foreign: + if created: + raise PackError(f"{len(foreign)} resource(s) in {group} do not carry this migration's tags and were not deleted: " + f"{', '.join(sorted(foreign))}. Remove them yourself, then run destroy again.") + return {"group_deleted": False, "resources_deleted": deleted} + if created: + az.group_delete(group) + return {"group_deleted": created, "resources_deleted": deleted} diff --git a/labagent/packs/azure_labs/migrate.py b/labagent/packs/azure_labs/migrate.py new file mode 100644 index 0000000..65b2870 --- /dev/null +++ b/labagent/packs/azure_labs/migrate.py @@ -0,0 +1,216 @@ +"""Lift and shift: read a source resource group (read-only), then copy its VMs, disks and network into a target group. + +The source is never changed. Disks are copied through snapshots, so the new VMs boot the same operating system, accounts and data. +Every resource created in the target keeps its source name and carries this migration's tags, so destroy can find exactly what was made. +""" +from __future__ import annotations + +from typing import Any, Callable, Dict, List, Set, Tuple + +from ..base import PackError + +SNAPSHOT = "microsoft.compute/snapshots" +DISK = "microsoft.compute/disks" + + +def last(rid: str) -> str: + return rid.rstrip("/").split("/")[-1] + + +def after(rid: str, segment: str) -> str: + parts = rid.rstrip("/").split("/") + return parts[[p.lower() for p in parts].index(segment.lower()) + 1] + + +def _rule(r: Dict[str, Any]) -> Dict[str, Any]: + p = r["properties"] + + def many(single: str, plural: str) -> List[str]: + return [p[single]] if p.get(single) else list(p.get(plural) or ["*"]) + return {"name": r["name"], "priority": p["priority"], "direction": p["direction"], "access": p["access"], "protocol": p["protocol"], + "source_addresses": many("sourceAddressPrefix", "sourceAddressPrefixes"), "source_ports": many("sourcePortRange", "sourcePortRanges"), + "destination_addresses": many("destinationAddressPrefix", "destinationAddressPrefixes"), + "destination_ports": many("destinationPortRange", "destinationPortRanges")} + + +def inspect(az: Any, source: str, wanted: List[str], snapshot_only: bool) -> Dict[str, Any]: + """What would be copied. Raises one PackError listing everything that cannot be copied, so nothing is half done.""" + vms = az.list_vms(source) + if wanted: + missing = sorted(set(wanted) - {v["name"] for v in vms}) + if missing: + raise PackError(f"VM(s) not found in {source}: {', '.join(missing)}") + vms = [v for v in vms if v["name"] in wanted] + disks = {d["id"].lower(): d for d in az.list_disks(source)} + problems: List[str] = [] + inv: Dict[str, Any] = {"vms": [], "disks": [], "nics": {}, "nsgs": {}, "vnets": {}, "pips": {}} + nsg_ids: Dict[str, str] = {} + used: Set[str] = set() + for v in vms: + name = v["name"] + if v.get("availabilitySet") or v.get("virtualMachineScaleSet"): + problems.append(f"{name}: is in an availability set or scale set") + osd = v["storageProfile"]["osDisk"] + vm_disks = [osd] + sorted(v["storageProfile"].get("dataDisks") or [], key=lambda d: d.get("lun", 0)) + entries = [] + for d in vm_disks: + md = d.get("managedDisk") + if not md: + problems.append(f"{name}: disk {d.get('name')} is an unmanaged VHD (convert it to a managed disk first)") + continue + info = disks.get(md["id"].lower()) + if info is None: + problems.append(f"{name}: disk {d.get('name')} is not in {source}") + continue + if (info.get("encryption") or {}).get("diskEncryptionSetId") or info.get("encryptionSettingsCollection"): + problems.append(f"{name}: disk {info['name']} uses customer-managed or Azure Disk Encryption keys") + used.add(info["id"].lower()) + entries.append({"name": info["name"], "id": info["id"], "sku": (info.get("sku") or {}).get("name", "Premium_LRS"), + "hyper_v": info.get("hyperVGeneration"), "os_type": info.get("osType"), "location": info["location"]}) + if not entries: + continue + nic_names = [] + for ref in v["networkProfile"]["networkInterfaces"]: + nic = az.show(ref["id"]) + cfgs = nic["properties"]["ipConfigurations"] + if len(cfgs) != 1: + problems.append(f"{name}: NIC {nic['name']} has more than one IP configuration") + continue + ip = cfgs[0]["properties"] + sid = ip["subnet"]["id"] + nsg_id = (nic["properties"].get("networkSecurityGroup") or {}).get("id") + pip_id = (ip.get("publicIPAddress") or {}).get("id") + inv["nics"][nic["name"]] = {"vnet": after(sid, "virtualNetworks"), "subnet": last(sid), "vnet_id": sid.split("/subnets/")[0], + "nsg": last(nsg_id) if nsg_id else None, "pip": last(pip_id) if pip_id else None, + "accelerated": bool(nic["properties"].get("enableAcceleratedNetworking"))} + nic_names.append(nic["name"]) + if nsg_id: + nsg_ids[last(nsg_id)] = nsg_id + if pip_id: + inv["pips"][last(pip_id)] = {"location": nic["location"]} + inv["vms"].append({"name": name, "size": v["hardwareProfile"]["vmSize"], "location": v["location"], + "zone": (v.get("zones") or [None])[0], "os_type": (entries[0]["os_type"] or "Linux").lower(), + "disks": entries, "nics": nic_names}) + if snapshot_only and not wanted: # also the disks nothing is using: the "old dead" ones + for did, d in disks.items(): + if did not in used and not d.get("managedBy"): + inv["disks"].append({"name": d["name"], "id": d["id"], "location": d["location"], "hyper_v": d.get("hyperVGeneration")}) + for nic in inv["nics"].values(): + if nic["vnet"] not in inv["vnets"]: + vn = az.show(nic["vnet_id"]) + subs = [] + for s in vn["properties"]["subnets"]: + sp = s["properties"] + prefix = sp.get("addressPrefix") or (sp.get("addressPrefixes") or [None])[0] + nsg = (sp.get("networkSecurityGroup") or {}).get("id") + if nsg: + nsg_ids[last(nsg)] = nsg + subs.append({"name": s["name"], "prefix": prefix, "nsg": last(nsg) if nsg else None}) + inv["vnets"][nic["vnet"]] = {"prefixes": vn["properties"]["addressSpace"]["addressPrefixes"], "location": vn["location"], "subnets": subs} + for nsg_name, nsg_id in nsg_ids.items(): + rules = az.show(nsg_id)["properties"].get("securityRules") or [] + if any(r["properties"].get("sourceApplicationSecurityGroups") or r["properties"].get("destinationApplicationSecurityGroups") for r in rules): + problems.append(f"NSG {nsg_name}: rules use application security groups") + inv["nsgs"][nsg_name] = [_rule(r) for r in rules] + if not inv["vms"] and not inv["disks"]: + problems.append(f"nothing to copy in {source}") + if problems: + raise PackError("cannot migrate: " + "; ".join(problems)) + return inv + + +def existing(az: Any, target: str) -> Set[Tuple[str, str]]: + return {(r["type"].lower(), r["name"].lower()) for r in az.resources(target)} if az.group_exists(target) else set() + + +def has(seen: Set[Tuple[str, str]], rtype: str, name: str) -> bool: + return (rtype.lower(), name.lower()) in seen + + +def snapshot_name(disk: str) -> str: + return f"snap-{disk}"[:80] + + +def snapshots(az: Any, inv: Dict[str, Any], target: str, tags: Dict[str, str], log: Callable[[str], None]) -> int: + seen = existing(az, target) + made = 0 + for d in [x for v in inv["vms"] for x in v["disks"]] + inv["disks"]: + sn = snapshot_name(d["name"]) + if has(seen, SNAPSHOT, sn): + continue + az.snapshot_create(target, sn, d["id"], d["location"], tags, d.get("hyper_v")) + log(f" snapshot {sn}") + made += 1 + return made + + +def network(az: Any, inv: Dict[str, Any], target: str, tags: Dict[str, str], log: Callable[[str], None]) -> int: + seen = existing(az, target) + made = 0 + for nsg, rules in inv["nsgs"].items(): + if has(seen, "microsoft.network/networksecuritygroups", nsg): + continue + loc = next(iter(inv["vnets"].values()))["location"] + az.nsg_create(target, nsg, loc, tags) + for r in rules: + az.nsg_rule_create(target, nsg, r) + log(f" network security group {nsg} ({len(rules)} rules)") + made += 1 + for vname, vn in inv["vnets"].items(): + if has(seen, "microsoft.network/virtualnetworks", vname): + continue + first, rest = vn["subnets"][0], vn["subnets"][1:] + az.vnet_create(target, vname, vn["prefixes"], first["name"], first["prefix"], vn["location"], tags) + for s in rest: + az.subnet_create(target, vname, s["name"], s["prefix"]) + for s in vn["subnets"]: + if s["nsg"]: + az.subnet_set_nsg(target, vname, s["name"], s["nsg"]) + log(f" virtual network {vname} ({len(vn['subnets'])} subnets)") + made += 1 + for pip, info in inv["pips"].items(): + if not has(seen, "microsoft.network/publicipaddresses", pip): + az.public_ip_create(target, pip, info["location"], tags) + log(f" public IP {pip} (a new address: the old one is not moved)") + made += 1 + for nname, n in inv["nics"].items(): + if has(seen, "microsoft.network/networkinterfaces", nname): + continue + loc = next(v["location"] for v in inv["vms"] if nname in v["nics"]) + az.nic_create(target, nname, n["vnet"], n["subnet"], loc, tags, nsg=n["nsg"], public_ip=n["pip"], accelerated=n["accelerated"]) + log(f" network interface {nname}") + made += 1 + return made + + +def disks(az: Any, inv: Dict[str, Any], target: str, tags: Dict[str, str], log: Callable[[str], None]) -> int: + seen = existing(az, target) + snaps = {r["name"]: r["id"] for r in az.resources(target) if r["type"].lower() == SNAPSHOT} + made = 0 + for v in inv["vms"]: + for i, d in enumerate(v["disks"]): + if has(seen, DISK, d["name"]): + continue + snap = snaps.get(snapshot_name(d["name"])) + if snap is None: + raise PackError(f"snapshot of {d['name']} is not in {target} yet: run launch again") + az.disk_create(target, d["name"], snap, d["sku"], d["location"], tags, zone=v["zone"], hyper_v=d.get("hyper_v"), + os_type=d["os_type"] if i == 0 else None) + log(f" disk {d['name']}") + made += 1 + return made + + +def vms(az: Any, inv: Dict[str, Any], target: str, tags: Dict[str, str], log: Callable[[str], None]) -> int: + seen = existing(az, target) + ids = {(r["type"].lower(), r["name"]): r["id"] for r in az.resources(target)} + made = 0 + for v in inv["vms"]: + if has(seen, "microsoft.compute/virtualmachines", v["name"]): + continue + os_disk, data = v["disks"][0], v["disks"][1:] + az.vm_create(target, v["name"], ids[(DISK, os_disk["name"])], v["os_type"], v["size"], [ids[("microsoft.network/networkinterfaces", n)] for n in v["nics"]], + [ids[(DISK, d["name"])] for d in data], v["location"], tags, zone=v["zone"]) + log(f" VM {v['name']} ({v['size']}, {1 + len(data)} disk(s))") + made += 1 + return made diff --git a/labagent/packs/azure_labs/pack.py b/labagent/packs/azure_labs/pack.py new file mode 100644 index 0000000..7594e03 --- /dev/null +++ b/labagent/packs/azure_labs/pack.py @@ -0,0 +1,172 @@ +"""azure-labs: lift and shift Azure VMs from one resource group to another, or just snapshot them. + +An alternative for Azure Lab Services customers who need their lab VMs somewhere else. Not Microsoft's service. +`launch` copies a source group's VMs, disks and network into a target group and never changes the source. +`destroy` deletes only what that launch created. +""" +from __future__ import annotations + +from typing import Any, Dict, List + +from ..base import Pack, PackContext, PackError, Param, Step +from . import lifecycle, migrate, safety +from .azcli import AzCli + + +class _LazyAz: + """Creates the real `az` wrapper on first use, so `--dry-run` works without Azure installed.""" + + def __init__(self) -> None: + self._az = None + self.subscription = None + + def __getattr__(self, name: str): + if self._az is None: + self._az = AzCli(self.subscription) + return getattr(self._az, name) + + +def _names(text: Any) -> List[str]: + return [t.strip() for t in str(text or "").split(",") if t.strip()] + + +class AzureLabs(Pack): + id = safety.PACK_ID + title = "Azure Labs: move Azure VMs between resource groups" + summary = "Lift and shift Azure VMs, disks and network to another resource group" + aliases = ("azure-lab", "als-migration") + platforms = ("azure",) + cost_tier = "metered" + default_name = "migration" + params = [ + Param("source_rg", None, "Resource group to copy from. It is only read, never changed"), + Param("target_rg", None, "Resource group to copy into. Created if it does not exist"), + Param("vms", "", "Comma list of VM names to copy (default: every VM in the source group)"), + Param("snapshot_only", False, "Only snapshot the disks into the target group: no VMs, no network. Also takes unattached disks when vms is empty"), + Param("subscription", None, "Subscription ID or name (default: the one `az login` selected)"), + ] + output_keys = ("source_rg", "target_rg", "vms", "snapshots") + + def services(self, profile): + return {"az": _LazyAz()} + + def _az(self, ctx: PackContext): + az = ctx.services["az"] + if getattr(az, "subscription", None) is None and ctx.params.get("subscription"): + az.subscription = ctx.params["subscription"] + return az + + def _login(self, ctx: PackContext) -> str: + sub = self._az(ctx).account()["id"] + safety.check_subscription(sub) + return sub + + def _tags(self, ctx: PackContext, created_group: bool = False) -> Dict[str, str]: + return safety.tags(ctx.name, ctx.state["source_rg"], created_group) + + # ---- launch ------------------------------------------------------------------------------------------------------------- + def _preflight(self, ctx: PackContext): + p = ctx.params + source, target = p["source_rg"], p["target_rg"] + if not source or not target: + raise PackError("give --param source_rg= and --param target_rg=") + if source.lower() == target.lower(): + raise PackError("source_rg and target_rg must be different") + safety.lab_name(ctx.name) + self._login(ctx) + az = self._az(ctx) + if not az.group_exists(source): + raise PackError(f"source resource group {source} does not exist") + inv = migrate.inspect(az, source, _names(p["vms"]), bool(p["snapshot_only"])) + ctx.mem["inv"] = inv + n_disks = sum(len(v["disks"]) for v in inv["vms"]) + len(inv["disks"]) + if not p["snapshot_only"] and inv["vms"]: + regions = {v["location"] for v in inv["vms"]} + if len(regions) > 1: + raise PackError(f"the VMs are in more than one region ({', '.join(sorted(regions))}): copy them in separate runs with vms=") + region = regions.pop() + ctx.log(" checking VM sizes and quota (the size list can take a minute) ...") + skus = az.skus(region) + needs: Dict[str, int] = {} + for v in inv["vms"]: + s = skus.get(v["size"]) + if s is None or s.get("restricted"): + raise PackError(f"{v['size']} ({v['name']}) is not available to this subscription in {region}") + needs[s["family"]] = needs.get(s["family"], 0) + int(s["vcpus"]) + lifecycle.check_quota(az.usage(region), needs, region) + ctx.log(f" {source} -> {target}: {len(inv['vms'])} VM(s), {n_disks} disk(s)" + (", snapshots only" if p["snapshot_only"] else "")) + return {"source_rg": source, "target_rg": target, "vms": [v["name"] for v in inv["vms"]], "snapshot_only": bool(p["snapshot_only"])} + + def _inv(self, ctx: PackContext) -> Dict[str, Any]: + return ctx.mem["inv"] + + def _target(self, ctx: PackContext): + az = self._az(ctx) + target = ctx.state["target_rg"] + if not az.group_exists(target): + inv = self._inv(ctx) + region = (inv["vms"] or inv["disks"])[0]["location"] + az.group_create(target, region, self._tags(ctx, created_group=True)) + ctx.log(f" created resource group {target}") + return {} + + def _snapshot(self, ctx: PackContext): + n = migrate.snapshots(self._az(ctx), self._inv(ctx), ctx.state["target_rg"], self._tags(ctx), ctx.log) + return {"snapshots": sum(len(v["disks"]) for v in self._inv(ctx)["vms"]) + len(self._inv(ctx)["disks"]), "snapshots_made": n} + + def _network(self, ctx: PackContext): + migrate.network(self._az(ctx), self._inv(ctx), ctx.state["target_rg"], self._tags(ctx), ctx.log) + return {} + + def _disks(self, ctx: PackContext): + migrate.disks(self._az(ctx), self._inv(ctx), ctx.state["target_rg"], self._tags(ctx), ctx.log) + return {} + + def _vms(self, ctx: PackContext): + migrate.vms(self._az(ctx), self._inv(ctx), ctx.state["target_rg"], self._tags(ctx), ctx.log) + return {} + + def _verify(self, ctx: PackContext): + az, target = self._az(ctx), ctx.state["target_rg"] + found = {(r["type"].lower(), r["name"]) for r in az.resources(target)} + missing = [f"snapshot {migrate.snapshot_name(d['name'])}" for v in self._inv(ctx)["vms"] for d in v["disks"] + if (migrate.SNAPSHOT, migrate.snapshot_name(d["name"])) not in found] + missing += [f"snapshot {migrate.snapshot_name(d['name'])}" for d in self._inv(ctx)["disks"] if (migrate.SNAPSHOT, migrate.snapshot_name(d["name"])) not in found] + report = {} + if not ctx.state.get("snapshot_only"): + for v in self._inv(ctx)["vms"]: + info = az.vm_state(target, v["name"]) + ok = info.get("provisioningState") == "Succeeded" + report[v["name"]] = {"state": info.get("powerState"), "private_ip": info.get("privateIps"), "public_ip": info.get("publicIps") or None} + ctx.log(f" {v['name']}: {info.get('provisioningState')}, {info.get('powerState')}, {info.get('privateIps')}") + if not ok: + missing.append(f"VM {v['name']} ({info.get('provisioningState')})") + if missing: + raise PackError(f"verify failed: {', '.join(missing)}") + return {"report": report} + + def deploy_steps(self, ctx: PackContext) -> List[Step]: + only = bool(ctx.params.get("snapshot_only")) + steps = [Step("preflight", "Read the source group, check the login, allow list and quota", self._preflight), + Step("target", "Create the target resource group if it is missing", self._target), + Step("snapshot", "Snapshot every disk into the target group", self._snapshot)] + if not only: + steps += [Step("network", "Copy the virtual network, security groups, public IPs and NICs", self._network), + Step("disks", "Create disks from the snapshots", self._disks), + Step("vms", "Create the VMs from the copied disks", self._vms)] + return steps + [Step("verify", "Check that everything exists and the VMs are provisioned", self._verify)] + + # ---- destroy ------------------------------------------------------------------------------------------------------------ + def _destroy(self, ctx: PackContext): + target = ctx.params.get("target_rg") or ctx.state.get("target_rg") + if not target: + raise PackError("give --param target_rg=") + if ctx.state.get("source_rg", "").lower() == target.lower(): + raise PackError("refusing to delete from the source group") + self._login(ctx) + out = lifecycle.destroy_owned(self._az(ctx), target, ctx.name, ctx.log) + ctx.log(f" resources deleted: {out['resources_deleted']}; resource group {'deleted' if out['group_deleted'] else 'kept'}") + return {"group_deleted": out["group_deleted"]} + + def destroy_steps(self, ctx: PackContext) -> List[Step]: + return [Step("destroy", "Delete what this migration created in the target group (the source is never touched)", self._destroy)] diff --git a/labagent/packs/azure_labs/safety.py b/labagent/packs/azure_labs/safety.py new file mode 100644 index 0000000..be32e6a --- /dev/null +++ b/labagent/packs/azure_labs/safety.py @@ -0,0 +1,51 @@ +"""Guards for azure-labs: the subscription allow-list, names and tags. Destroy only touches what carries this migration's tags.""" +from __future__ import annotations + +import os +import re +from datetime import datetime, timezone +from typing import Dict, List, Optional + +from ..base import PackError + +PACK_ID = "azure-labs" +ALLOWED_ENV = "LABAGENT_ALLOWED_AZURE_SUBSCRIPTIONS" + + +def slug(text: str, n: int = 40) -> str: + return re.sub(r"[^a-z0-9-]+", "-", str(text).lower()).strip("-")[:n].strip("-") + + +def lab_name(name: str) -> str: + s = slug(name) + if len(s) < 3: + raise PackError("the name must have at least 3 letters or digits (a-z, 0-9 and -)") + return s + + +def allowed_subscriptions(environ: Optional[Dict[str, str]] = None) -> List[str]: + raw = (environ if environ is not None else os.environ).get(ALLOWED_ENV, "") + return [s.strip().lower() for s in raw.split(",") if s.strip()] + + +def check_subscription(subscription_id: str, environ: Optional[Dict[str, str]] = None) -> None: + """Refuse to run unless the subscription is explicitly allowed. An empty list refuses everything.""" + allowed = allowed_subscriptions(environ) + if not allowed: + raise PackError(f"{ALLOWED_ENV} is empty: refusing to run. Set it to the subscription ID you may use.") + if str(subscription_id).lower() not in allowed: + raise PackError(f"the current Azure subscription is not in {ALLOWED_ENV}: refusing to run.") + + +def tags(name: str, source_rg: str, created_group: bool = False, now: Optional[datetime] = None) -> Dict[str, str]: + now = now or datetime.now(timezone.utc) + t = {"labagent": "1", "pack": PACK_ID, "name": lab_name(name), "migrated_from": source_rg, "created_at": now.strftime("%Y-%m-%dT%H:%M:%SZ")} + if created_group: + t["created_by_labagent"] = "1" + return t + + +def owned(resource_tags: Optional[Dict[str, str]], name: str) -> bool: + """True only for a resource that carries this migration's tags. Anything else is never ours to delete.""" + t = resource_tags or {} + return t.get("labagent") == "1" and t.get("pack") == PACK_ID and t.get("name") == lab_name(name) diff --git a/labagent/packs/registry.py b/labagent/packs/registry.py index eb39689..f96bbac 100644 --- a/labagent/packs/registry.py +++ b/labagent/packs/registry.py @@ -13,7 +13,8 @@ def builtin() -> Dict[str, Pack]: from .aml.pack import AmlLab from .dbx_workspace.pack import DatabricksWorkspace from .dbx_lakehouse.pack import DatabricksLakehouse - return {p.id: p for p in (RagSecTutorial(), DbAgent(), RagIde(), AmlLab(), DatabricksWorkspace(), DatabricksLakehouse())} + from .azure_labs.pack import AzureLabs + return {p.id: p for p in (RagSecTutorial(), DbAgent(), RagIde(), AmlLab(), DatabricksWorkspace(), DatabricksLakehouse(), AzureLabs())} def all_packs() -> Dict[str, Pack]: diff --git a/tests/test_azure_labs.py b/tests/test_azure_labs.py new file mode 100644 index 0000000..7df8845 --- /dev/null +++ b/tests/test_azure_labs.py @@ -0,0 +1,347 @@ +"""azure-labs: copy VMs, disks and network from one resource group to another, snapshot only, and delete what was made.""" +import json + +import pytest + +from labagent.cli import main +from labagent.packs.azure_labs import lifecycle, migrate, safety +from labagent.packs.azure_labs.azcli import AzCli, redact +from labagent.packs.azure_labs.pack import AzureLabs +from labagent.packs.base import PackError, run_pack, state_path + +SUB = "00000000-0000-0000-0000-000000000001" +SRC, DST = "azure-labs-legacy", "azure-labs-new" +BASE = f"/subscriptions/{SUB}/resourceGroups/{SRC}/providers" +LOC = "canadacentral" + + +def rid(group, rtype, name): + return f"/subscriptions/{SUB}/resourceGroups/{group}/providers/{rtype}/{name}" + + +def vm_doc(name, size, os_disk, data=(), nic=None, zone=None, **extra): + return {"name": name, "location": LOC, "zones": [zone] if zone else None, "hardwareProfile": {"vmSize": size}, + "storageProfile": {"osDisk": {"name": os_disk, "managedDisk": {"id": rid(SRC, "Microsoft.Compute/disks", os_disk)}}, + "dataDisks": [{"name": d, "lun": i, "managedDisk": {"id": rid(SRC, "Microsoft.Compute/disks", d)}} for i, d in enumerate(data)]}, + "networkProfile": {"networkInterfaces": [{"id": rid(SRC, "Microsoft.Network/networkInterfaces", nic or f"{name}-nic")}]}, **extra} + + +def disk_doc(name, os_type=None, managed_by=None, sku="StandardSSD_LRS", **extra): + return {"name": name, "id": rid(SRC, "Microsoft.Compute/disks", name), "location": LOC, "sku": {"name": sku}, "osType": os_type, + "hyperVGeneration": "V2" if os_type else None, "managedBy": managed_by, **extra} + + +def nic_doc(name, pip=None, nsg=None, ip_configs=1): + cfg = {"properties": {"subnet": {"id": rid(SRC, "Microsoft.Network/virtualNetworks", "legacy-vnet") + "/subnets/app"}, + **({"publicIPAddress": {"id": rid(SRC, "Microsoft.Network/publicIPAddresses", pip)}} if pip else {})}} + return {"name": name, "location": LOC, "properties": {"ipConfigurations": [cfg] * ip_configs, "enableAcceleratedNetworking": False, + **({"networkSecurityGroup": {"id": rid(SRC, "Microsoft.Network/networkSecurityGroups", nsg)}} if nsg else {})}} + + +RULE = {"name": "allow-ssh", "properties": {"priority": 100, "direction": "Inbound", "access": "Allow", "protocol": "Tcp", "sourceAddressPrefix": "*", + "sourcePortRange": "*", "destinationAddressPrefix": "*", "destinationPortRange": "22"}} + + +class FakeAz: + """In-memory az with a prefilled source group. Records every write so tests can prove what was and was not touched.""" + + def __init__(self, usage=None): + self.subscription = None + self.writes = [] # (what, group, name) + self.groups = {SRC: {"location": LOC, "tags": {}, "resources": []}} + self.fail_on = {} # (what, name) -> remaining failures + self._usage = usage or {"cores": {"used": 0, "limit": 10}, "standardBSFamily": {"used": 0, "limit": 10}, "standardDSv3Family": {"used": 0, "limit": 10}} + self.vms = [vm_doc("vm1", "Standard_B1s", "vm1_osdisk", data=("vm1-data",), nic="vm1-nic"), + vm_doc("vm2", "Standard_D2s_v3", "vm2_osdisk", nic="vm2-nic", zone="1")] + self.disks = [disk_doc("vm1_osdisk", "Linux", managed_by="vm1"), disk_doc("vm1-data", managed_by="vm1"), + disk_doc("vm2_osdisk", "Windows", managed_by="vm2"), disk_doc("old-dead-disk")] + self.docs = { + rid(SRC, "Microsoft.Network/networkInterfaces", "vm1-nic"): nic_doc("vm1-nic", pip="vm1-pip"), + rid(SRC, "Microsoft.Network/networkInterfaces", "vm2-nic"): nic_doc("vm2-nic"), + rid(SRC, "Microsoft.Network/virtualNetworks", "legacy-vnet"): { + "name": "legacy-vnet", "location": LOC, "properties": {"addressSpace": {"addressPrefixes": ["10.30.0.0/16"]}, "subnets": [ + {"name": "app", "properties": {"addressPrefix": "10.30.1.0/24", "networkSecurityGroup": {"id": rid(SRC, "Microsoft.Network/networkSecurityGroups", "legacy-nsg")}}}, + {"name": "db", "properties": {"addressPrefix": "10.30.2.0/24"}}]}}, + rid(SRC, "Microsoft.Network/networkSecurityGroups", "legacy-nsg"): {"name": "legacy-nsg", "properties": {"securityRules": [RULE]}}, + } + self.states = {} + + # reads + def account(self): return {"id": SUB, "name": "t"} + def usage(self, region): return self._usage + def skus(self, region): + return {"Standard_B1s": {"family": "standardBSFamily", "vcpus": "1", "restricted": 0}, + "Standard_D2s_v3": {"family": "standardDSv3Family", "vcpus": "2", "restricted": 0}} + def group_exists(self, n): return n in self.groups + def group_show(self, n): + g = self.groups.get(n) + return {"name": n, "location": g["location"], "tags": dict(g["tags"])} if g else None + def resources(self, g): return [dict(r) for r in self.groups[g]["resources"]] + def show(self, r): return self.docs[r] + def list_vms(self, g): + assert g == SRC + return self.vms + def list_disks(self, g): + assert g == SRC + return self.disks + def vm_state(self, g, n): return {"provisioningState": "Succeeded", "powerState": "VM running", "privateIps": "10.30.1.4", "publicIps": ""} + + # writes + def _add(self, what, rtype, group, name, tags): + if self.fail_on.get((what, name), 0): + self.fail_on[(what, name)] -= 1 + raise PackError(f"az {what} failed: boom") + self.writes.append((what, group, name)) + self.groups[group]["resources"].append({"id": rid(group, rtype, name), "name": name, "type": rtype, "tags": dict(tags)}) + + def group_create(self, n, region, tags): + self.writes.append(("group", n, n)) + self.groups[n] = {"location": region, "tags": dict(tags), "resources": []} + def group_delete(self, n): + self.writes.append(("group_delete", n, n)) + del self.groups[n] + def resource_delete(self, r): + self.writes.append(("delete", r.split("/resourceGroups/")[1].split("/")[0], r.split("/")[-1])) + for g in self.groups.values(): + g["resources"] = [x for x in g["resources"] if x["id"] != r] + def snapshot_create(self, g, n, src, loc, tags, hyper_v=None): self._add("snapshot", "Microsoft.Compute/snapshots", g, n, tags) + def disk_create(self, g, n, snap, sku, loc, tags, zone=None, hyper_v=None, os_type=None): + self.disk_args = getattr(self, "disk_args", {}) | {n: (snap, sku, zone, os_type)} + self._add("disk", "Microsoft.Compute/disks", g, n, tags) + def nsg_create(self, g, n, loc, tags): self._add("nsg", "Microsoft.Network/networkSecurityGroups", g, n, tags) + def nsg_rule_create(self, g, nsg, rule): self.writes.append(("rule", g, f"{nsg}/{rule['name']}")) + def vnet_create(self, g, n, prefixes, sub, pre, loc, tags): self._add("vnet", "Microsoft.Network/virtualNetworks", g, n, tags); self.writes.append(("subnet", g, f"{n}/{sub}")) + def subnet_create(self, g, vnet, n, prefix): self.writes.append(("subnet", g, f"{vnet}/{n}")) + def subnet_set_nsg(self, g, vnet, sub, nsg): self.writes.append(("subnet_nsg", g, f"{vnet}/{sub}={nsg}")) + def public_ip_create(self, g, n, loc, tags): self._add("pip", "Microsoft.Network/publicIPAddresses", g, n, tags) + def nic_create(self, g, n, vnet, sub, loc, tags, nsg=None, public_ip=None, accelerated=False): + self.nic_args = getattr(self, "nic_args", {}) | {n: (vnet, sub, nsg, public_ip)} + self._add("nic", "Microsoft.Network/networkInterfaces", g, n, tags) + def vm_create(self, g, n, os_disk, os_type, size, nics, data_disks, loc, tags, zone=None): + self.vm_args = getattr(self, "vm_args", {}) | {n: (os_disk, os_type, size, nics, data_disks, zone)} + self._add("vm", "Microsoft.Compute/virtualMachines", g, n, tags) + + +@pytest.fixture(autouse=True) +def env(tmp_path, monkeypatch): + monkeypatch.setenv("LABAGENT_HOME", str(tmp_path)) + monkeypatch.setenv(safety.ALLOWED_ENV, SUB) + + +def launch(az, **params): + return run_pack(AzureLabs(), "launch", "mig", {"source_rg": SRC, "target_rg": DST, **params}, services={"az": az}, log=lambda m: None) + + +def destroy(az, **params): + return run_pack(AzureLabs(), "destroy", "mig", params, services={"az": az}, log=lambda m: None) + + +def names(az, group, rtype): + return sorted(r["name"] for r in az.groups[group]["resources"] if r["type"].lower() == rtype.lower()) + + +# ---- copying ------------------------------------------------------------------------------------------------------------ +def test_launch_copies_vms_disks_and_network_with_the_same_names_and_tags(): + az = FakeAz() + env = launch(az) + assert env.status == "ok", env.error + assert names(az, DST, "Microsoft.Compute/virtualMachines") == ["vm1", "vm2"] + assert names(az, DST, "Microsoft.Compute/disks") == ["vm1-data", "vm1_osdisk", "vm2_osdisk"] + assert names(az, DST, "Microsoft.Compute/snapshots") == ["snap-vm1-data", "snap-vm1_osdisk", "snap-vm2_osdisk"] + assert names(az, DST, "Microsoft.Network/networkInterfaces") == ["vm1-nic", "vm2-nic"] + assert names(az, DST, "Microsoft.Network/virtualNetworks") == ["legacy-vnet"] + assert names(az, DST, "Microsoft.Network/networkSecurityGroups") == ["legacy-nsg"] + assert names(az, DST, "Microsoft.Network/publicIPAddresses") == ["vm1-pip"] + assert ("rule", DST, "legacy-nsg/allow-ssh") in az.writes + assert {w[2] for w in az.writes if w[0] == "subnet"} == {"legacy-vnet/app", "legacy-vnet/db"} + assert ("subnet_nsg", DST, "legacy-vnet/app=legacy-nsg") in az.writes + assert all(safety.owned(r["tags"], "mig") and r["tags"]["migrated_from"] == SRC for r in az.groups[DST]["resources"]) + assert az.groups[DST]["tags"]["created_by_labagent"] == "1" + assert "old-dead-disk" not in names(az, DST, "Microsoft.Compute/disks") # not attached to a VM: only snapshot_only takes it + + +def test_the_vms_boot_from_the_copied_disks_and_keep_size_zone_and_data_disks(): + az = FakeAz() + launch(az) + os_disk, os_type, size, nics, data, zone = az.vm_args["vm1"] + assert os_disk == rid(DST, "Microsoft.Compute/disks", "vm1_osdisk") and os_type == "linux" and size == "Standard_B1s" + assert data == [rid(DST, "Microsoft.Compute/disks", "vm1-data")] and nics == [rid(DST, "Microsoft.Network/networkInterfaces", "vm1-nic")] and zone is None + assert az.vm_args["vm2"][1] == "windows" and az.vm_args["vm2"][5] == "1" + snap, sku, zone, os_type = az.disk_args["vm2_osdisk"] + assert snap == rid(DST, "Microsoft.Compute/snapshots", "snap-vm2_osdisk") and sku == "StandardSSD_LRS" and zone == "1" and os_type == "Windows" + assert az.disk_args["vm1-data"][3] is None # only the OS disk carries an OS type + assert az.nic_args["vm1-nic"] == ("legacy-vnet", "app", None, "vm1-pip") + + +def test_the_source_group_is_never_written(): + az = FakeAz() + launch(az) + destroy(az) + assert all(w[1] != SRC for w in az.writes) + assert len(az.groups[SRC]["resources"]) == 0 and len(az.disks) == 4 # nothing removed from the source either + + +def test_a_second_run_creates_nothing_and_a_failed_run_resumes_where_it_stopped(): + az = FakeAz() + az.fail_on[("vm", "vm2")] = 1 + first = launch(az) + assert first.status == "failed" and "boom" in first.error and names(az, DST, "Microsoft.Compute/virtualMachines") == ["vm1"] + second = launch(az) + assert second.status == "ok" and names(az, DST, "Microsoft.Compute/virtualMachines") == ["vm1", "vm2"] + before = list(az.writes) + assert launch(az).status == "ok" and az.writes == before + + +def test_vms_filter_copies_only_the_named_vms_and_unknown_names_are_an_error(): + az = FakeAz() + assert launch(az, vms="vm1").status == "ok" + assert names(az, DST, "Microsoft.Compute/virtualMachines") == ["vm1"] and names(az, DST, "Microsoft.Network/networkInterfaces") == ["vm1-nic"] + bad = launch(FakeAz(), vms="vm1,nope") + assert bad.status == "failed" and "nope" in bad.error + + +def test_snapshot_only_takes_every_disk_including_unattached_ones_and_nothing_else(): + az = FakeAz() + env = launch(az, snapshot_only=True) + assert env.status == "ok", env.error + assert names(az, DST, "Microsoft.Compute/snapshots") == ["snap-old-dead-disk", "snap-vm1-data", "snap-vm1_osdisk", "snap-vm2_osdisk"] + assert {r["type"] for r in az.groups[DST]["resources"]} == {"Microsoft.Compute/snapshots"} + only = FakeAz() + launch(only, snapshot_only=True, vms="vm1") + assert names(only, DST, "Microsoft.Compute/snapshots") == ["snap-vm1-data", "snap-vm1_osdisk"] + + +def test_what_cannot_be_copied_is_reported_together_before_anything_is_created(): + az = FakeAz() + az.vms[0]["availabilitySet"] = {"id": "x"} + az.vms[1]["storageProfile"]["osDisk"]["managedDisk"] = None + az.docs[rid(SRC, "Microsoft.Network/networkInterfaces", "vm1-nic")] = nic_doc("vm1-nic", pip="vm1-pip", ip_configs=2) + env = launch(az) + assert env.status == "failed" and "availability set" in env.error and "unmanaged" in env.error and "more than one IP" in env.error + assert DST not in az.groups and not az.writes + asg = FakeAz() + asg.docs[rid(SRC, "Microsoft.Network/networkSecurityGroups", "legacy-nsg")]["properties"]["securityRules"] = [ + {**RULE, "properties": {**RULE["properties"], "sourceApplicationSecurityGroups": [{"id": "x"}]}}] + assert "application security groups" in launch(asg).error + enc = FakeAz() + enc.disks[0]["encryption"] = {"diskEncryptionSetId": "x"} + assert "customer-managed" in launch(enc).error + + +# ---- refusals ----------------------------------------------------------------------------------------------------------- +def test_bad_input_creates_nothing(): + for params, msg in (({"source_rg": None}, "source_rg"), ({"target_rg": SRC}, "must be different"), ({"source_rg": "missing-rg"}, "does not exist")): + az = FakeAz() + env = run_pack(AzureLabs(), "launch", "mig", {"source_rg": SRC, "target_rg": DST, **params}, services={"az": az}, log=lambda m: None) + assert env.status == "failed" and msg in env.error and not az.writes + + +def test_the_subscription_allow_list_is_enforced_for_launch_and_destroy(monkeypatch): + for value in ("", "11111111-1111-1111-1111-111111111111"): + monkeypatch.setenv(safety.ALLOWED_ENV, value) + az = FakeAz() + assert launch(az).status == "failed" and "refusing to run" in launch(az).error and not az.writes + assert destroy(az, target_rg=DST).status == "failed" + + +def test_not_enough_quota_stops_before_any_copy_and_says_why(): + az = FakeAz(usage={"cores": {"used": 8, "limit": 10}, "standardBSFamily": {"used": 1, "limit": 10}, "standardDSv3Family": {"used": 0, "limit": 10}}) + env = launch(az) + assert env.status == "failed" and "total regional vCPUs: need 3, free 2" in env.error and "source VMs still count" in env.error and not az.writes + assert launch(FakeAz(usage={"cores": {"used": 0, "limit": 10}}), snapshot_only=True).status == "ok" # snapshots need no vCPUs + + +def test_dry_run_reads_nothing(): + az = FakeAz() + env = run_pack(AzureLabs(), "launch", "mig", {}, services={"az": az}, dry_run=True) + assert env.status == "ok" and not az.writes and len(env.items) == 7 + + +# ---- destroy ----------------------------------------------------------------------------------------------------------- +def test_destroy_deletes_what_the_migration_made_dependants_first_and_the_group(): + az = FakeAz() + launch(az) + env = destroy(az) + assert env.status == "ok" and DST not in az.groups and not state_path("azure-labs", "mig").exists() + deleted = [w[2] for w in az.writes if w[0] == "delete"] + assert deleted.index("vm1") < deleted.index("vm1-nic") < deleted.index("vm1_osdisk") < deleted.index("snap-vm1_osdisk") < deleted.index("legacy-vnet") + + +def test_destroy_never_deletes_an_untagged_resource_and_keeps_a_group_it_did_not_create(): + az = FakeAz() + launch(az) + az.groups[DST]["resources"].append({"id": rid(DST, "Microsoft.Compute/disks", "precious"), "name": "precious", "type": "Microsoft.Compute/disks", "tags": {}}) + env = destroy(az) + assert env.status == "failed" and "precious" in env.error and DST in az.groups + assert ("delete", DST, "precious") not in az.writes + pre = FakeAz() + pre.groups[DST] = {"location": LOC, "tags": {"team": "x"}, "resources": [{"id": "i", "name": "keep-me", "type": "Microsoft.Compute/disks", "tags": {}}]} + launch(pre) + assert destroy(pre).status == "ok" and DST in pre.groups and [r["name"] for r in pre.groups[DST]["resources"]] == ["keep-me"] + + +def test_destroy_never_works_on_the_source_group_and_needs_a_target(): + az = FakeAz() + launch(az) + assert "refusing to delete from the source" in destroy(az, target_rg=SRC).error + assert "target_rg" in run_pack(AzureLabs(), "destroy", "other", {}, services={"az": FakeAz()}, log=lambda m: None).error + + +def test_destroy_reports_a_resource_it_cannot_delete(): + az = FakeAz() + launch(az) + real = az.resource_delete + az.resource_delete = lambda r: (_ for _ in ()).throw(PackError("in use")) if r.endswith("/legacy-vnet") else real(r) + env = destroy(az) + assert env.status == "failed" and "legacy-vnet" in env.error and DST in az.groups + + +# ---- the az wrapper ----------------------------------------------------------------------------------------------------- +class Proc: + def __init__(self, out="", err="", rc=0): + self.stdout, self.stderr, self.returncode = out, err, rc + + +def recorder(out="{}"): + seen = [] + return seen, AzCli("sub-1", run=lambda cmd, **kw: seen.append((cmd, kw)) or Proc(out), binary="az") + + +def test_azcli_builds_the_commands_the_copy_depends_on(): + seen, az = recorder() + az.vm_create("g", "vm1", "DISK", "linux", "Standard_B1s", ["NIC"], ["D1", "D2"], "canadacentral", {"a": "1"}, zone="1") + cmd = seen[-1][0] + assert cmd[:6] == ["az", "vm", "create", "-g", "g", "-n"] and "--attach-os-disk" in cmd and cmd[cmd.index("--attach-data-disks") + 1:cmd.index("--attach-data-disks") + 3] == ["D1", "D2"] + assert cmd[cmd.index("--zone") + 1] == "1" and "--tags" in cmd and cmd[-2:] == ["--subscription", "sub-1"] + az.snapshot_create("g", "s", "SRC", "canadacentral", {"a": "1"}, "V2") + cmd = seen[-1][0] + assert cmd[cmd.index("--incremental") + 1] == "false" and cmd[cmd.index("--hyper-v-generation") + 1] == "V2" and cmd[cmd.index("--source") + 1] == "SRC" + az.nsg_rule_create("g", "nsg", migrate._rule(RULE)) + cmd = seen[-1][0] + assert cmd[cmd.index("--destination-port-ranges") + 1] == "22" and cmd[cmd.index("--priority") + 1] == "100" + assert all(kw["encoding"] == "utf-8" and kw["text"] is True for _, kw in seen) + + +def test_azcli_errors_are_redacted_and_a_missing_binary_is_explained(monkeypatch): + az = AzCli(run=lambda cmd, **kw: Proc(err="ERROR: bad adminPassword=Sup3rSecret! token: abcdef123456", rc=1), binary="az") + with pytest.raises(PackError) as e: + az.json("vm", "create") + assert "Sup3rSecret" not in str(e.value) and "abcdef123456" not in str(e.value) and "Sup3rSecret" not in redact("password: Sup3rSecret!") + monkeypatch.setattr("shutil.which", lambda n: None) + with pytest.raises(PackError, match="Azure CLI not found"): + AzCli().json("account", "show") + + +def test_azcli_group_exists_usage_and_the_size_list_is_fetched_once(): + assert AzCli(run=lambda c, **k: Proc("true"), binary="az").group_exists("g") is True + usage = [{"name": {"value": "cores"}, "currentValue": "2", "limit": "10"}] + assert AzCli(run=lambda c, **k: Proc(json.dumps(usage)), binary="az").usage("r") == {"cores": {"used": 2, "limit": 10}} + seen, az = recorder(json.dumps([{"name": "Standard_B1s", "family": "standardBSFamily", "restricted": 0, "vcpus": "1"}])) + az.skus("r"), az.skus("r") + assert len(seen) == 1 + + +def test_the_pack_is_listed_with_no_credential(capsys): + assert main(["packs", "--json"]) == 0 + out = capsys.readouterr().out + assert {p["id"]: p for p in json.loads(out)["packs"]}["azure-labs"]["platforms"] == ["azure"] and SUB not in out diff --git a/tests/test_nl.py b/tests/test_nl.py index d15937a..a63f334 100644 --- a/tests/test_nl.py +++ b/tests/test_nl.py @@ -94,7 +94,7 @@ def test_prompt_is_small_lists_real_packs_and_asks_only_for_action_and_pack(): sys_prompt = msgs[0]["content"] assert "rag-sec-tutorial" in sys_prompt and "rag-tutorial" in sys_prompt and "db-agent" in sys_prompt and "rag-ide" in sys_prompt assert "idle_minutes" not in sys_prompt and "profile" not in sys_prompt # parameters are not the model's job - assert len(sys_prompt) < 2500 # short enough for a 0.5b model to follow + assert len(sys_prompt) < 3000 # short enough for a 0.5b model to follow; each pack adds about 300 assert msgs[1]["content"] == "Request: deploy rag-tutorial" diff --git a/tests/test_packs.py b/tests/test_packs.py index 84ec472..bf877cf 100644 --- a/tests/test_packs.py +++ b/tests/test_packs.py @@ -75,7 +75,7 @@ def go(c): def test_registry_lists_the_current_packs_and_the_cli_refuses_unknown_ones(capsys, home): - assert sorted(registry.all_packs()) == ["aml", "db-agent", "dbx-lakehouse", "dbx-workspace", "rag-ide", "rag-sec-tutorial"] + assert sorted(registry.all_packs()) == ["aml", "azure-labs", "db-agent", "dbx-lakehouse", "dbx-workspace", "rag-ide", "rag-sec-tutorial"] assert main(["packs"]) == 0 and "rag-sec-tutorial" in capsys.readouterr().out assert main(["launch", "nope", "--name", "x"]) == 2 @@ -149,7 +149,7 @@ def test_an_explicit_profile_is_never_overridden(home): def test_packs_json_describes_every_pack_with_platforms_and_params(capsys, home): assert main(["packs", "--json"]) == 0 cat = {p["id"]: p for p in json.loads(capsys.readouterr().out)["packs"]} - assert sorted(cat) == ["aml", "db-agent", "dbx-lakehouse", "dbx-workspace", "rag-ide", "rag-sec-tutorial"] + assert sorted(cat) == ["aml", "azure-labs", "db-agent", "dbx-lakehouse", "dbx-workspace", "rag-ide", "rag-sec-tutorial"] assert cat["rag-ide"]["platforms"] == ["local", "ec2", "modal"] and cat["rag-ide"]["cost_tier"] == "metered" assert {"name": "hours", "default": 4, "help": "Hard cap per session, 1 to 24", "required": False} in cat["rag-ide"]["params"] assert all(p["title"] and p["summary"] and p["platforms"] and p["cost_tier"] for p in cat.values())