From bf3d8a337de48dfb8b9ec7fe8d43afd9f67ecaaa Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Thu, 30 Jul 2026 15:01:27 +0200 Subject: [PATCH] Direct Routed (L3) guest networks: route public IPv4/IPv6 directly to Instances Adds a new guest network type in which the hypervisor performs L3 routing for the Instance: no Virtual Router, no NAT and no DHCP. Each Instance receives a public IPv4 address as a /32 and/or an IPv6 address as a /128, with a shared, host-independent gateway (169.254.0.1 and fe80::1) that every hypervisor carries on the network's bridge. All addressing reaches the Instance exclusively via ConfigDrive/cloud-init; a routing daemon on the host (FRR, BIRD, ...) advertises the addresses to the fabric and is deliberately out of scope for CloudStack. Management server: - GuestType.L3; the guest_type column is char(32), so no schema change. - Offering validation: UserData via ConfigDrive is mandatory, Dns optional but ConfigDrive-only, SecurityGroup permitted (now allowed for L3 alongside Shared), Dhcp rejected as not supported and not needed. Network mode, specifyVlan and VPC use are rejected. - DirectRoutedNetworkGuru subclasses DirectNetworkGuru, inheriting the Shared-network address lifecycle. canHandle() selects on the offering's guest type alone; design() produces a Native broadcast domain with no isolation id. After allocation the NicProfile is forced into host-route form, which is also the signature by which the agent and ConfigDrive recognise these NICs. - createNetwork treats L3 like Shared for the subnet: explicit IP range mandatory, vlan/IP-range row created at network creation, IPv6 accepted without the /64 restriction, aclType Account. - Zone-wide IPv4 overlap validation for L3 ranges: all L3 subnets share one host routing table and one fabric, so an overlap is an address conflict. The IPv6 vlan check was already zone-wide. ConfigDrive: - Network data is always generated for a direct routed NIC; the historical gate (Dhcp or Dns supported) held while ConfigDrive supplemented a VR but would leave these NICs with no addressing at all. Route generation itself is unchanged: cloud-init detects an IPv4 gateway inside 169.254.0.0/16 and sets on-link on the rendered route by itself. KVM agent: - One uplink-less bridge per network, brdr-, created and removed by the new modifybrdr.sh (flock'd, idempotent, refuses to remove a bridge still in use). The bridge carries the gateway addresses, forwarding and strict rp_filter; separate bridges make isolation between networks topological rather than a filtering concern. - BridgeVifDriver plugs direct routed NICs into their brdr bridge and runs the existing modifymacip.sh hook per NIC to install the static neighbour entry and host route, regardless of the host-wide EVPN property, whose meaning is unchanged. The design document, including the decision log and the verification notes behind each choice, is added under docs/design/. --- .../main/java/com/cloud/network/Network.java | 4 +- .../user/network/CreateNetworkCmd.java | 4 +- .../command/user/vm/RemoveIpFromVmNicCmd.java | 12 +- .../api/NetworkRulesVmSecondaryIpCommand.java | 16 + docs/design/direct-routed-networks.md | 1258 +++++++++++++++++ .../configdrive/ConfigDriveBuilder.java | 28 +- .../configdrive/ConfigDriveBuilderTest.java | 95 +- .../kvm/resource/BridgeVifDriver.java | 112 +- .../resource/LibvirtComputingResource.java | 63 +- ...tworkRulesVmSecondaryIpCommandWrapper.java | 2 +- ...bvirtSecurityGroupRulesCommandWrapper.java | 11 +- scripts/vm/network/security_group.py | 344 +++-- .../network/tests/golden_add_fw_framework.txt | 44 + .../tests/golden_default_network_rules.txt | 92 ++ .../vm/network/tests/test_security_group.py | 245 ++++ scripts/vm/network/vnet/modifybrdr.sh | 198 +++ scripts/vm/network/vnet/modifymacip.sh | 24 +- .../ConfigurationManagerImpl.java | 65 +- .../com/cloud/network/NetworkServiceImpl.java | 43 +- .../network/guru/DirectRoutedNetworkGuru.java | 178 +++ .../security/SecurityGroupManagerImpl.java | 24 +- .../spring-server-network-context.xml | 3 + .../ConfigurationManagerImplTest.java | 75 + .../guru/DirectRoutedNetworkGuruTest.java | 161 +++ test/integration/smoke/test_l3_networks.py | 186 +++ tools/marvin/marvin/config/test_data.py | 22 + ui/public/locales/en.json | 4 + ui/src/views/network/CreateL3NetworkForm.vue | 402 ++++++ ui/src/views/network/CreateNetwork.vue | 9 + ui/src/views/offering/AddNetworkOffering.vue | 52 +- .../java/com/cloud/utils/net/NetUtils.java | 20 +- 31 files changed, 3604 insertions(+), 192 deletions(-) create mode 100644 docs/design/direct-routed-networks.md create mode 100644 scripts/vm/network/tests/golden_add_fw_framework.txt create mode 100644 scripts/vm/network/tests/golden_default_network_rules.txt create mode 100755 scripts/vm/network/tests/test_security_group.py create mode 100755 scripts/vm/network/vnet/modifybrdr.sh create mode 100644 server/src/main/java/com/cloud/network/guru/DirectRoutedNetworkGuru.java create mode 100644 server/src/test/java/com/cloud/network/guru/DirectRoutedNetworkGuruTest.java create mode 100644 test/integration/smoke/test_l3_networks.py create mode 100644 ui/src/views/network/CreateL3NetworkForm.vue diff --git a/api/src/main/java/com/cloud/network/Network.java b/api/src/main/java/com/cloud/network/Network.java index 2f0bcdd5ef9a..d7b467bab5a7 100644 --- a/api/src/main/java/com/cloud/network/Network.java +++ b/api/src/main/java/com/cloud/network/Network.java @@ -43,7 +43,7 @@ public interface Network extends ControlledEntity, StateObject, InternalIdentity, Identity, Serializable, Displayable { enum GuestType { - Shared, Isolated, L2; + Shared, Isolated, L2, L3; public static GuestType fromValue(String type) { if (StringUtils.isBlank(type)) { @@ -54,6 +54,8 @@ public static GuestType fromValue(String type) { return Isolated; } else if (type.equalsIgnoreCase("L2")) { return L2; + } else if (type.equalsIgnoreCase("L3")) { + return L3; } else { throw new InvalidParameterValueException("Unexpected Guest type : " + type); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java index 79fb5f6d01cf..e32ab4e77719 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java @@ -340,10 +340,10 @@ public Long getPhysicalNetworkId() { } } if (physicalNetworkId != null) { - if ((offering.getGuestType() == GuestType.Shared) || (offering.getGuestType() == GuestType.L2)) { + if ((offering.getGuestType() == GuestType.Shared) || (offering.getGuestType() == GuestType.L2) || (offering.getGuestType() == GuestType.L3)) { return physicalNetworkId; } else { - throw new InvalidParameterValueException("Physical network ID can be specified for networks of guest IP type " + GuestType.Shared + " or " + GuestType.L2 + " only."); + throw new InvalidParameterValueException(String.format("Physical network ID can be specified for networks of guest IP type %s, %s or %s only.", GuestType.Shared, GuestType.L2, GuestType.L3)); } } else { if (zoneId == null) { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RemoveIpFromVmNicCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RemoveIpFromVmNicCmd.java index f4c4d82b30d4..f8bdfe718d1b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RemoveIpFromVmNicCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RemoveIpFromVmNicCmd.java @@ -124,6 +124,16 @@ public NetworkType getNetworkType() { } + /** + * A Direct Routed (L3) network needs the agent told when a secondary IP goes away, so the + * host route and neighbour entry are removed - otherwise the host keeps routing an address + * the Instance no longer owns, and the routing daemon keeps advertising it. + */ + private boolean isDirectRoutedNetwork() { + Network ntwk = _entityMgr.findById(Network.class, getNetworkId()); + return ntwk != null && Network.GuestType.L3.equals(ntwk.getGuestType()); + } + private boolean isZoneSGEnabled() { Network ntwk = _entityMgr.findById(Network.class, getNetworkId()); DataCenter dc = _entityMgr.findById(DataCenter.class, ntwk.getDataCenterId()); @@ -144,7 +154,7 @@ public void execute() throws InvalidParameterValueException { secIp = nicSecIp.getIp6Address(); } - if (isZoneSGEnabled()) { + if (isZoneSGEnabled() || isDirectRoutedNetwork()) { //remove the security group rules for this secondary ip boolean success = false; success = _securityGroupService.securityGroupRulesForVmSecIp(nicSecIp.getNicId(), secIp, false); diff --git a/core/src/main/java/com/cloud/agent/api/NetworkRulesVmSecondaryIpCommand.java b/core/src/main/java/com/cloud/agent/api/NetworkRulesVmSecondaryIpCommand.java index c0752a1e97e5..ed6949a48095 100644 --- a/core/src/main/java/com/cloud/agent/api/NetworkRulesVmSecondaryIpCommand.java +++ b/core/src/main/java/com/cloud/agent/api/NetworkRulesVmSecondaryIpCommand.java @@ -28,12 +28,28 @@ public class NetworkRulesVmSecondaryIpCommand extends Command { private String vmSecIp; private String vmMac; private String action; + private boolean directRouted; + private boolean applySecurityGroupRules = true; public NetworkRulesVmSecondaryIpCommand(String vmName, VirtualMachine.Type type) { this.vmName = vmName; this.type = type; } + public NetworkRulesVmSecondaryIpCommand(String vmName, String vmMac, String secondaryIp, boolean action, boolean directRouted, boolean applySecurityGroupRules) { + this(vmName, vmMac, secondaryIp, action); + this.directRouted = directRouted; + this.applySecurityGroupRules = applySecurityGroupRules; + } + + public boolean isDirectRouted() { + return directRouted; + } + + public boolean isApplySecurityGroupRules() { + return applySecurityGroupRules; + } + public NetworkRulesVmSecondaryIpCommand(String vmName, String vmMac, String secondaryIp, boolean action) { this.vmName = vmName; this.vmMac = vmMac; diff --git a/docs/design/direct-routed-networks.md b/docs/design/direct-routed-networks.md new file mode 100644 index 000000000000..555a164667a0 --- /dev/null +++ b/docs/design/direct-routed-networks.md @@ -0,0 +1,1258 @@ +# Direct Routed Networks + +**Status:** design agreed — open items are implementation and verification only +**Branch:** `direct-routed-network` +**Author:** Wido den Hollander +**Last updated:** 2026-07-30 + +--- + +## 1. Summary + +A new guest network type in which the **hypervisor performs L3 routing for the guest**. There is +no Virtual Router, no NAT, and **no DHCP** — the defining characteristic of this network type. + +The operator creates a network and adds a subnet to it, as for a Shared network. CloudStack then +allocates **individual addresses** out of that subnet to guest NICs, and hands each NIC: + +* its IPv4 address as a **/32** and its IPv6 address as a **/128** +* a **shared, host-independent gateway**: `169.254.0.1` (IPv4) and `fe80::1` (IPv6), configured on + the network's bridge on every hypervisor +* the whole configuration via **ConfigDrive / cloud-init** — mandatory, since without DHCP or RA + there is no other way for the guest to learn its address + +The hypervisor's CloudStack agent installs, per guest address, a host route and a static neighbour +entry on the guest bridge, binding the address to that guest's MAC. This reuses the existing +`modifymacip.sh` hook unchanged (§9.1). A routing daemon on the host advertises those addresses to +the fabric. **The routing daemon is out of scope for this feature** — see §10. + +## 2. Motivation + +Compared with what CloudStack offers today: + +| | Shared | Isolated (NATTED) | Isolated (ROUTED, 4.20+) | **Direct Routed** | +|---|---|---|---|---| +| VR in data path | for DHCP/DNS | yes | yes | **no** | +| DHCP | yes | yes | yes | **no** | +| VM gets routable IP | yes | no (NAT) | yes | yes | +| Guest netmask | subnet mask | subnet mask | subnet mask | **/32, /128** | +| Shared broadcast domain | yes | yes | yes | **no** | +| Guest-to-guest | switched | switched | switched | **routed by host** | +| Isolation ID | VLAN/VXLAN | VLAN/VXLAN | VLAN/VXLAN | **none** | + +What this closes: + +* **No VR in the data path.** `ROUTED` networks (4.20) removed NAT but kept a VR in the path and an + L2 segment per network. Throughput, failover, and per-network VR footprint remain concerns. +* **No shared broadcast domain between networks.** Each network gets its own uplink-less bridge + (§9.2), so there is no L2 path of any kind between networks — no ARP spoofing, no rogue DHCP + server, no rogue RA reaching another tenant. Isolation is topological, so it does not depend on a + rule set being correct, and an administrator can turn security groups off for a network without + weakening it. Within a single network guests still share a bridge; that residual exposure is + intra-tenant and is documented in §12.3. +* **No isolation id at all.** No VLAN, no VXLAN, no encapsulation, nothing to allocate or size — the + bridge is named from the network's own database id (§9.2.1). Guest network count is bounded by no + ID space, and each network still gets a real L2 boundary. +* **Per-network routing policy on the hypervisor.** Each network is a distinct, named L3 interface + (`brdr-42`), so the operator can apply different route-maps, redistribution filters, policy + routing or QoS per network in the host's own configuration. CloudStack does not need to know or + care; it is entirely a local network design decision (§9.2). +* **IP mobility.** Because the gateway is identical on every hypervisor, the guest's network + configuration is entirely host-independent. A VM can start, stop, and migrate anywhere in the + routing domain with no reconfiguration — its address follows it as an advertised route. +* **Fits L3-to-the-host fabrics.** Operators already running BGP or OSPF on the hypervisor get an + addressing model that matches their fabric instead of fighting it with stretched VLANs. +* **Denser subnet use.** No broadcast domain means no network or broadcast address to reserve — + every address in the subnet is usable (§6.3.1). + +## 3. Non-goals + +* Not a replacement for `NetworkMode.ROUTED` + BGP-per-network (4.20). That stays. +* No NAT, source NAT, static NAT, port forwarding, LB, or VPN in this network type. +* No VR at all — not even for DHCP/DNS/UserData. +* No DHCP, DHCPv6, or SLAAC/RA for guest addressing. +* No support for guests that cannot consume ConfigDrive (§6.5). +* **No routing-daemon management.** CloudStack does not install, configure, or monitor FRR/BIRD + (§10). +* No per-tenant VRFs, and therefore no overlapping subnets between networks (§6.3.2). +* **Never part of a VPC** — VPC already covers BGP-routed subnets with a VR; this is the different + case of a public address on the Instance itself (§6.6). + +## 4. Terminology + +| Term | Meaning | +|---|---| +| Direct routed network | The new guest network type described here | +| `brdr-` | Bridge-DirectRouted: the uplink-less per-network bridge, named from the network's database id; created by `modifybrdr.sh` | +| Shared gateway | `169.254.0.1` / `fe80::1`, present on **every** `brdr-*` bridge on **every** host | +| Host route | `ip route replace /32 dev `, installed by `modifymacip.sh` | +| Static neighbour | `ip neigh replace lladdr dev nud permanent` | +| Routing daemon | FRR/BIRD/other, run and configured by the operator — not by CloudStack | + +## 5. Network model + +### 5.1 Addressing + +* Guest NIC: `203.0.113.55/32`, `2001:db8:1::55/128` +* Guest default route: `default via 169.254.0.1 dev eth0 onlink` and `default via fe80::1 dev eth0` +* Host guest bridge: `169.254.0.1/32` and `fe80::1/64` — identical on every host +* Host, per guest address: a /32 (or /128) route and a permanent neighbour entry, both on the + **bridge** (§9.1.2) + +### 5.2 Packet walk — guest to elsewhere + +1. The guest has no on-link neighbours (it is a /32) → everything goes to the default route. +2. The guest ARPs for `169.254.0.1` — permitted because the route is `onlink` — or ND-solicits + `fe80::1`. The host bridge answers. +3. The host routes the packet per its own routing table, out to the fabric. + +### 5.3 Packet walk — fabric to guest + +1. The fabric has learned `203.0.113.55/32` from this host via BGP/OSPF. +2. The packet arrives; the host matches `203.0.113.55/32 dev `. +3. The host does **not** need to ARP — `modifymacip.sh` installed a permanent neighbour entry + mapping the address to the guest's MAC. The frame is handed to the bridge, which forwards it to + the port where that MAC was learned. +4. Delivered. + +Static neighbour entries rather than ARP are deliberate: they remove a resolution round-trip from +VM start, make host→guest delivery independent of whether the guest answers ARP, and close off +ARP-based address takeover between guests on the same host. + +Note the address is pinned to a **MAC**, not to a port — the MAC-to-port mapping comes from ordinary +bridge FDB learning. See §12.1 for why that is still sound, and what it depends on. + +### 5.4 Packet walk — guest to guest, same host + +Between guests of **different** networks, the two addresses are on different bridges (§9.2), so the +host routes between them and there is no L2 path at all. + +Between guests of the **same** network, both host routes are local to one bridge and the host +hairpins via that bridge. Either way the traffic passes through the host's forwarding table, and +therefore through security group filtering where those are in use (§12.2). + +Remaining asymmetry: same-host traffic never reaches the fabric, so fabric-level policy does not see +it. Since security groups are optional here, same-network guest-to-guest traffic on one host may be +unfiltered — accepted for v1, see §12.3. + +### 5.5 What the network object looks like + +Like a **Shared** network, not like L2: the network has a subnet. The operator supplies the CIDR, +and optionally an IPv6 prefix, via the usual IP-range mechanism (`vlan` rows — `vlan_gateway`, +`vlan_netmask`, `ip4_range`, `ip6_gateway`, `ip6_cidr`, `ip6_range` in +`engine/schema/src/main/java/com/cloud/dc/VlanVO.java`). + +The difference is what CloudStack does with it: the subnet is an **allocation pool that is routed to +the hypervisors**, not a broadcast domain. Guests never see the subnet mask or its gateway. + +**The subnet gateway is required, and ignored. DECIDED for v1.** + +`vlan_gateway` / `ip6_gateway` are meaningless for this network type — nothing reads them, because +the guest's gateway is always the shared link-local address and the whole subnet is routed to the +hosts. `createVlanIpRange` requires a gateway today, and it stays required: relaxing it would mean +touching validation shared with every other network type for no functional gain. + +The cost is a small wart — the operator has to nominate an address in the subnet that will never be +configured anywhere or answer anything. Two practical notes: + +* It should be **documented as unused**, so nobody wastes time debugging why traffic is not reaching + it, and so nobody assumes reserving it is necessary. +* Whichever address is given still gets consumed from the allocation pool unless explicitly kept + out of the IP range. Since every address in the subnet is otherwise usable (§6.3.1), an operator + can simply give the range as the full subnet and point the gateway at an address inside it — but + confirm existing validation does not reject a gateway that falls within the range. + +Making it optional remains available later if it proves annoying; it is a validation change, not a +model change, so nothing here forecloses it. + +## 6. Design decisions + +### 6.1 How is the network type modelled? **DECIDED — `GuestType.L3`** + +`Network.GuestType` becomes `Shared, Isolated, L2, L3` +(`api/src/main/java/com/cloud/network/Network.java:45`). + +Chosen because it is self-describing and symmetrical with the existing `L2`, and because every +existing `GuestType.L2` / `Shared` branch then becomes an obvious place to decide what `L3` should +do. The cost is accepted: a new enum value touches API responses, UI and upgrade, and roughly a +dozen files special-case `L2` with a similar number special-casing `Shared`. + +Rejected alternatives, for the record: + +* **`GuestType.Shared` plus a flag or `NetworkMode` on the offering** — no new enum, and it would + inherit Shared's subnet and allocation handling for free, but it overloads `NetworkMode.ROUTED` + which already means "VR routes, no NAT". Every Shared code path would have to ask "but is it the + routed kind?", which is worse than a new value. +* **`GuestType.DirectRouted`** — explicit, but mixes a topology concept (`L2`) with a routing one in + the same enum. + +Files to review — those branching on `GuestType.L2`: + +* `server/src/main/java/com/cloud/network/NetworkServiceImpl.java` +* `server/src/main/java/com/cloud/network/NetworkModelImpl.java` +* `server/src/main/java/com/cloud/network/guru/GuestNetworkGuru.java` +* `engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java` +* `engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java` +* `server/src/main/java/com/cloud/vm/UserVmManagerImpl.java` +* `api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java` +* `engine/schema/src/main/java/com/cloud/offerings/dao/NetworkOfferingDaoImpl.java` +* `plugins/network-elements/vxlan/.../VxlanGuestNetworkGuru.java` +* `server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java` + +…plus the `GuestType.Shared` branches, which is where subnet and IP-range handling lives. + +### 6.2 Which gateway address? **DECIDED — static `169.254.0.1` / `fe80::1`** + +Fixed, not configurable. Requirements met: identical on every host, never globally routable, cannot +collide with guest address space. + +`fe80::1` is unambiguously correct — link-local next-hops are the norm in IPv6 and need no special +handling. + +`169.254.0.1` requires `onlink` on the guest route because it falls outside the guest's /32 (§8.1), +and that is fine: **the address is a /32, so the guest has to treat its gateway as on-link whatever +that gateway is.** There is no configuration of the gateway address that would avoid needing +`onlink`, so making it configurable would buy nothing — any guest that can work at all here can use +an on-link next-hop. + +An earlier draft left open whether to add a global setting for operators whose images might refuse a +link-local next-hop. Closed: no setting. Keeping it fixed preserves the "guest configuration is +completely host-independent" property by construction, and removes a knob that could only ever be set +wrong. + +Consequences: + +* §8.1's rule — emit `on-link: true` when the gateway is in `169.254.0.0/16` — is now permanently + sufficient. There is no need to generalise it to "gateway outside the address's own prefix", + because the gateway will never be anything else. +* `modifybrdr.sh` keeps its `-4` / `-6` options with these values as defaults, **deliberately**. + Nothing calls them with anything other than the defaults today, and they are not a CloudStack + setting — but they cost nothing, make the script testable in isolation, and mean a future change of + heart is a caller-side change rather than a script rewrite. They should not be removed as dead + options. + +### 6.3 Where do the addresses come from? **DECIDED** + +Users add a subnet to the network; CloudStack picks **individual** IPv4 and IPv6 addresses from the +existing address tables: + +* IPv4 → `user_ip_address` (`engine/schema/src/main/java/com/cloud/network/dao/IPAddressVO.java`) +* IPv6 → `user_ipv6_address` (`engine/schema/src/main/java/com/cloud/network/UserIpv6AddressVO.java`) +* Subnet definition → `vlan` rows, as for Shared networks + +This is the mechanism Shared networks already use: `DirectNetworkGuru.allocateDirectIp()` +(`server/src/main/java/com/cloud/network/guru/DirectNetworkGuru.java:316`) → +`IpAddressManagerImpl.allocateDirectIp()` +(`server/src/main/java/com/cloud/network/IpAddressManagerImpl.java:2434`). + +Consequences, all good: + +* No new tables, no new allocation logic, no new capacity accounting. +* Existing IP reservation, `listPublicIpAddresses`, and quota/usage machinery apply. +* Requested-IP (`ipaddress=` on `deployVirtualMachine`) works for free. + +The one thing the new guru **must** override: `allocateDirectIp()` sets the NIC's gateway and +netmask from the vlan row (lines 2459–2461). For this network type they must instead be forced to +`255.255.255.255` / `169.254.0.1` and `/128` / `fe80::1`. + +#### 6.3.1 Network and broadcast addresses are usable **DECIDED — must work** + +There is no broadcast domain, so the first and last address of a subnet are ordinary, routable, +assignable addresses. `203.0.113.0` and `203.0.113.255` in a /24 are just addresses; nothing +broadcasts to them, and every guest is a /32 behind the host's routing table. + +**They must be assignable.** The gain is proportionally largest exactly where address space is +tightest, which is the common case for this feature: + +| Subnet | Addresses | Usable today (minus network, broadcast, gateway) | Usable here | Gain | +|---|---|---|---|---| +| /29 | 8 | 5 | 7 | +40% | +| /28 | 16 | 13 | 15 | +15% | +| /27 | 32 | 29 | 31 | +7% | +| /24 | 256 | 253 | 255 | +0.8% | + +(The gateway is still deducted because §5.5 keeps it required, even though nothing uses it.) + +**Verified during implementation: no code change was needed.** The earlier draft assumed the +exclusion lived in the `NetUtils` CIDR helpers (`getIpRangeFromCidr()` and friends, whose +`start = (ip & netmask) + 1` / `end - 2` arithmetic does skip the two addresses) and would need +inclusive variants. Tracing the actual path an operator-supplied range takes showed those helpers +are only used where CloudStack *derives* a range from a CIDR — Isolated networks. The explicit +start/end path this network type uses (Shared-style `createVlanIpRange`) validates with +`sameSubnet()` — a plain bitmask comparison that `.0` and `.255` pass — plus start≤end and +gateway-not-in-range, and `savePublicIPRange()` then iterates the range without exclusions into +`user_ip_address`. Allocation is row-based from that table and never re-derives from the CIDR. + +**`.0` and `.255` are therefore already assignable end-to-end on this path.** The smoke test should +still assert it (an Instance actually receiving `.0` or `.255` and working), since this rests on +tracing rather than an existing guarantee anyone maintains. + +#### 6.3.2 Subnets must be unique across the routing domain **DECIDED — constraint** + +Because every direct routed network on a host shares one routing table, and all subnets are +advertised into one fabric, **subnets cannot overlap between networks**. Two tenants cannot both +use `10.0.0.0/24`. + +This follows from §9.2: networks separate tenants in the API and UI, not in address space. There is +no VRF, no per-tenant routing table, and no NAT to hide behind. Addresses must be unique and +routable within the routing domain. + +Implications: + +* Tenants cannot bring their own overlapping RFC1918 space. Operators assign from a pool they + control — public space, or private space that is unique zone-wide. +* **Validation must reject a new subnet that overlaps an existing direct routed subnet anywhere in + the zone. Required — an overlap is an address conflict, not a policy preference.** Two networks + sharing a subnet would produce duplicate /32s in one host routing table and duplicate + advertisements into the fabric, with traffic delivered to whichever Instance the host resolved + last. +* **Implemented and verified:** the IPv6 vlan overlap check was already zone-wide + (`_vlanDao.listByZone()`). IPv4 was not — the `user_ip_address` unique key is + `(public_ip_address, source_network_id)`, i.e. per network — so `createVlanAndPublicIpRange` now + calls the existing zone-wide `checkOverlapPublicIpRange()` for L3 networks. Shared networks keep + their historical behaviour, where the same IPv4 range in two VLANs is legitimate. +* This is the main user-visible limitation of "networks separate tenants administratively only" and + must be explicit in the documentation. +* Per-tenant VRFs would lift the restriction but mean per-VRF routing tables on the host and + per-VRF sessions in the routing daemon — well beyond v1 (§15). + +#### 6.3.3 Route scale **DECIDED — out of scope** + +Each guest address is advertised as an individual /32 or /128, so a zone with 50k Instances puts 50k +routes into the fabric. This is inherent to the design: any aggregation scheme pins addresses to +hosts and breaks migration (§11), so per-address advertisement is deliberate. + +**How that scales is the operator's concern, not CloudStack's.** Fabric capacity, aggregation and +route policy are local network design — the same boundary already drawn around the routing daemon in +§10. CloudStack states no supported ceiling, because it has no way to know one: the answer depends +entirely on the hardware and topology in front of it. + +For context rather than as a commitment: route counts in the order of 100k are not usually a problem +for modern equipment. Operators running L3-to-the-host fabrics are typically already carrying host +routes at that scale. + +### 6.4 Service/provider matrix for the offering **DECIDED** + +`ConfigDriveNetworkElement` advertises `UserData`, `Dhcp`, `Dns` +(`server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java:203`). This type uses +two of the three. + +| Service | Provider | Note | +|---|---|---| +| `UserData` | `ConfigDrive` | **mandatory** | +| `Dns` | `ConfigDrive` | optional but **strongly recommended** — the only way a guest learns its resolvers unless its template already has them (§6.5) | +| `SecurityGroup` | `SecurityGroupProvider` | optional (§12.2) | +| `Dhcp` | — | **not supported, and not needed** | +| `SourceNat`, `StaticNat`, `PortForwarding`, `Lb`, `Firewall`, `Vpn`, `NetworkACL`, `Gateway` | — | not supported | + +* `specifyIpRanges` → `true` (the operator supplies the subnet) +* `specifyVlan` → `false`; there is no isolation id of any kind to specify (§9.2.1) +* `NetworkMode` → not applicable; reject `NATTED`/`ROUTED` for this guest type + +**Why DNS is recommended rather than mandatory.** There is no VR and no resolver on the host, so +`network_data.json` `services` is the only channel by which CloudStack can tell a guest its DNS +servers (§8.3). An offering without it leaves Instances with addresses and routing but no name +resolution — unless the template already carries resolvers, which is unusual but legitimate and the +operator's call (§6.5). Note this depends on the §8.2 gate being fixed first. + +**Why DHCP is not merely unsupported but unnecessary.** ConfigDrive delivers the address, netmask, +gateway and routes directly (§8.1). DHCP would have nothing left to hand out, and offering it would +reintroduce exactly the L2 broadcast dependency this network type removes. + +Validation to add: + +* reject the offering unless `UserData` is provided by `ConfigDrive`; `Dns` is permitted but not + required (§6.5) +* reject `Dhcp` outright for this guest type +* `NetworkServiceImpl.java:657` currently rejects DNS on L2 networks. This type *requires* DNS, so + that branch must distinguish L2 from L3 rather than treating them alike. + +### 6.5 ConfigDrive is mandatory, DNS is not **DECIDED** + +**`UserData` via ConfigDrive is mandatory. `Dns` is optional but strongly recommended.** + +ConfigDrive itself cannot be optional: with no DHCP and no RA, it is the only channel that carries +the address, netmask, gateway and routes. A guest that ignores it comes up with no addresses at all, +and nothing in CloudStack reports an error. + +DNS is a different matter. A template may already have resolvers baked in, or be configured by +whatever provisions it afterwards. That is unusual, but it is the operator's call, not something the +network type needs to enforce — so an offering may omit `Dns`. The documentation should recommend +`UserData` + `Dns` together, since omitting DNS leaves an Instance with connectivity but no name +resolution unless its template handles it. + +**This has a hard prerequisite — see §8.2.** `ConfigDriveBuilder.needForGeneratingNetworkData()` +currently writes network data only when the network supports `Dhcp` **or** `Dns`. Since this type +never has `Dhcp`, an offering without `Dns` would today produce an **empty `network_data.json` and an +Instance with no addressing at all** — a far worse outcome than missing resolvers. Making `Dns` +optional therefore requires changing that gate first; it is not merely a validation relaxation. + +**No warning is raised when a template ignores ConfigDrive. DECIDED.** Whether cloud-init is present +and configured inside the guest is the operator's responsibility, not something CloudStack should +police. There is no reliable way to detect it from outside the Instance in any case, so any check +would be a guess presented as a fact. Documented as a requirement of the network type; not enforced, +not warned about. + +### 6.6 VPC support **DECIDED — never** + +**Not out of scope for v1; out of scope permanently.** Standalone networks only. + +The purpose of this network type is to route a public IPv4 and IPv6 address directly to an Instance. +A VPC is the opposite proposition: a private, self-contained address space with a VR, tiers holding +their own CIDRs, and ACLs between them. Every one of those is something this type deliberately +removes, so "direct routed inside a VPC" would not be a reduced VPC — it would be a contradiction. + +The case that *sounds* like it overlaps is already covered: **VPC supports BGP routing with subnets +today** (`NetworkMode.ROUTED`, 4.20). An operator who wants tenant subnets advertised into the fabric +with a VR in the path should use that. This feature addresses the different case where the Instance +itself holds the public address and there is no VR at all. + +Keeping the two apart is deliberate. They are not two settings of one feature, and treating them as +such would make both harder to reason about. + +### 6.7 What makes a network direct routed **DECIDED** + +**The network offering, and nothing else.** There is no broadcast domain type, no isolation method, +no id to allocate, and nothing for the user to choose — the same shape as an L2 network, where the +offering's guest type is the whole story. + +* **Guru selection is on guest type alone.** `canHandle()` tests the zone type, `isMyTrafficType()` + and `offering.getGuestType() == GuestType.L3`. It deliberately does **not** call + `isMyIsolationMethod()`, unlike its siblings + (`DirectNetworkGuru.java:147`, `VxlanGuestNetworkGuru.java:56`), because there is no isolation to + select — no VLAN, no VXLAN, no encapsulation of any kind. `GuestType.L3` is new, so no other guru + claims it and there is no ambiguity to resolve. +* **No isolation method to register.** An earlier draft proposed `new IsolationMethod("ROUTED")`. + Dropped: it would force the operator to add an isolation method to the physical network for no + benefit, since nothing would ever be selected by it. +* **No broadcast domain type.** An earlier draft proposed `routed://` as a new + `BroadcastDomainType`. Also dropped — see §9.2.1 for how the bridge name is derived instead, which + needs no new plumbing at all. +* **No id for the user to pick.** The bridge is named from the network's own database id, which is + unique by construction and requires no allocation, no range to configure, and no `specifyVlan` + handling. + +Offering creation rejects `Dhcp` for this guest type (§6.4), so a direct routed offering cannot be +built with DHCP in the first place. + +#### 6.7.1 Changing a network's offering afterwards **ACCEPTED** + +`NetworkServiceImpl.canUpgrade()` (`server/src/main/java/com/cloud/network/NetworkServiceImpl.java:4193`) +gates `updateNetwork`'s offering change on SecurityGroup parity, tag equality, `specifyVlan` +equality, `NetworkMode` equality, and `canMoveToPhysicalNetwork()`. It does **not** compare guest +type, so an administrator can in principle move a network onto an offering of a different type — for +example one without ConfigDrive, leaving Instances with no way to get their addressing. + +**Accepted, not guarded.** This is an administrative action with an obvious cause and effect; adding +a special case to `canUpgrade()` for this type is not worth the complexity. Worth a line in the +documentation, nothing more. + +One incidental note from reading that method: the **SecurityGroup parity check** means security +groups cannot be toggled on a live network by swapping offerings — enabling them is a choice made +when the network is created. + +### 6.8 Hypervisor support + +**KVM only for v1.** The host must program routes and neighbour entries and run a routing daemon; +that is only realistic where the host OS is ours to configure. Other hypervisors: reject at network +creation with a clear error. + +## 7. API and data model changes + +### 7.1 API + +* `createNetworkOffering` — accept the new `guestiptype`; validate the service matrix (§6.4). +* `createNetwork` — accept the subnet as for Shared networks; `createVlanIpRange` gateway handling + per §5.5; zone-wide overlap validation per §6.3.2. +* `listNetworks` / `NetworkResponse` — expose the type. The network's CIDR is meaningful (it is the + pool) and should be shown; its gateway is not. +* `listNics` / `NicResponse` — report the /32 and /128 plus the shared gateway. `netmask` is + already a dotted quad, so `255.255.255.255` needs no schema change. +* `listPublicIpAddresses` — works as for Shared networks. +* New APIs: **none.** + +### 7.2 Database + +* `network_offerings.guest_type` — new enum value (`L3`). +* `nics` — all needed columns exist: `ip4_address`, `netmask`, `gateway`, `ip6_address`, + `ip6_cidr`, `ip6_gateway` (`engine/schema/src/main/java/com/cloud/vm/NicVO.java:55-108`). +* `user_ip_address`, `user_ipv6_address`, `vlan` — reused unchanged. +* Upgrade: new enum value only. No data migration. + +### 7.3 New components + +* **Network guru — a subclass of `DirectNetworkGuru`. DECIDED.** `DirectNetworkGuru` already + implements "operator defines a subnet, CloudStack assigns individual v4 and v6 addresses", which + is exactly the allocation behaviour wanted, so the allocate/release lifecycle is inherited rather + than duplicated. The subclass overrides: + * `canHandle()` — accept `GuestType.L3`, and drop the `isMyIsolationMethod()` test (§6.7) + * the `NicProfile` after allocation — force `255.255.255.255` / `169.254.0.1` and `/128` / + `fe80::1` over the vlan row's values (`IpAddressManagerImpl.allocateDirectIp()` lines 2459–2461, + §6.3) + * `design()` — no broadcast domain or isolation to assign (§9.2.1) + + The known cost of subclassing is inheriting `DirectNetworkGuru`'s Shared-network assumptions. + Accepted: the alternative is duplicating the address lifecycle, which is the part most likely to + drift and the part where bugs are least visible. **TODO:** during implementation, note any + inherited behaviour that only makes sense for `GuestType.Shared` and override it explicitly rather + than letting it apply by accident. +* **Network element** — none new. `ConfigDriveNetworkElement` covers UserData/DNS; the security + group element covers filtering; host routes ride on NIC plug/unplug (§9), not on element + `implement()`. + +## 8. Guest configuration via ConfigDrive + +### 8.1 `on-link` for the link-local gateway **DECIDED** + +`ConfigDriveBuilder.getNetworksJsonArrayForNic()` +(`engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java:365`) +today emits OpenStack **network_data.json v1**: + +```json +{"id":"eth0","ip_address":"...","netmask":"...","link":"eth0","type":"ipv4", + "routes":[{"gateway":"...","netmask":"0.0.0.0","network":"0.0.0.0"}]} +``` + +A /32 address with a gateway outside its own subnet cannot be expressed here — v1 has no way to +mark a next-hop as on-link, and the guest kernel will reject the resulting config with +`Nexthop has invalid gateway`. + +**Resolved — verified against cloud-init: no on-link plumbing is needed at all.** + +Two findings settled this, one from code and one from testing: + +1. The ISO is labelled `config-2` (`VirtualMachineManager.VmConfigDriveLabel`), so cloud-init + selects its **OpenStack/ConfigDrive datasource**, which takes network configuration from + `openstack/latest/network_data.json`. A `network-config` file (Network Config v2) is only read + by the NoCloud datasource (label `cidata`) and would be ignored on the default boot path. +2. **Verified by Wido:** when consuming `network_data.json`, cloud-init itself detects an IPv4 + gateway inside `169.254.0.0/16` and sets the on-link flag on the route it renders. The existing + v1 emission — a plain default route via the gateway — therefore works unchanged. + +So ConfigDrive's route generation is **not modified at all**. What did change (§8.2) is only the +gate: network data is now always generated for a direct routed NIC, since ConfigDrive is its only +addressing channel. + +Emitting a Network Config v2 `network-config` file for NoCloud-configured images was implemented +and then removed — deferred to a **later PR** (§15). A v1 host-route-to-the-gateway workaround was +likewise removed as redundant: cloud-init handles the case natively, and extra routes with a +`0.0.0.0` gateway are the kind of thing that can confuse a renderer. + +The guest-side requirement is accordingly not "Network Config v2 support" but **a cloud-init recent +enough to apply on-link for link-local gateways from OpenStack network data** — which the smoke +test must pin down to a minimum version for the documentation. + +Target guest config: + +```yaml +version: 2 +ethernets: + eth0: + match: {macaddress: "02:00:...:5a"} + addresses: [203.0.113.55/32, "2001:db8:1::55/128"] + routes: + - to: default + via: 169.254.0.1 + on-link: true + - to: default + via: "fe80::1" + nameservers: + addresses: [...] +``` + +Notes and follow-ups: + +* The on-link trigger — the gateway matching `169.254.0.0/16` — lives in **cloud-init**, not in + CloudStack. It is a property of the address, so it stays correct however the guest receives it. +* IPv6 needs no `on-link` — `fe80::1` is link-local by definition and always on-link. +* **The guest requirement is a cloud-init recent enough to set on-link for link-local IPv4 + gateways when consuming OpenStack network data.** Guests whose cloud-init predates that are not + supported on this network type, in the same way that guests without ConfigDrive are not (§6.5). +* **TODO:** verify against the images used in the smoke tests, and state the minimum cloud-init + version in the documentation so the requirement is visible to operators rather than discovered at + boot. + +### 8.2 network_data generation is gated on DHCP or DNS **REQUIRED CHANGE** + +```java +static boolean needForGeneratingNetworkData(Map> supportedServices) { + return supportedServices.values().stream() + .anyMatch(services -> services.contains(Network.Service.Dhcp) + || services.contains(Network.Service.Dns)); +} +``` +(`engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java:266`, +called from `writeNetworkData()` at `:252`.) + +If neither service is supported, `writeNetworkData()` writes an empty `{}` and **the guest receives +no network configuration whatsoever** — no address, no netmask, no gateway, no routes. + +That gate is wrong for this network type. It equates "does this network have DHCP or DNS?" with +"does this NIC need its addressing written into ConfigDrive?", which held while ConfigDrive was a +supplement to a VR but does not hold when ConfigDrive is the *only* channel. This type never has +`Dhcp`, and §6.5 makes `Dns` optional, so the two conditions can both be false while the NIC still +very much needs its /32 written. + +**Required:** extend the condition so network data is generated whenever the NIC belongs to a direct +routed network — or, more generally, whenever the NIC has an address to convey and no other means of +conveying it. Until that lands, a `Dns`-less offering is not merely degraded, it is non-functional. + +Worth a code comment either way, because the coupling is invisible from the offering side. + +### 8.3 DNS **DECIDED — per network, falling back to the zone** + +DNS servers reach the guest through `network_data.json` `services` +(`getServicesJsonArrayForNic`). No resolver on the host, no VR. + +Resolvers come from the network when set, and from the zone otherwise — so an operator configures +DNS once per zone and overrides it only on the networks that need something different. + +**This already works and needs no new code.** `NetworkModelImpl.getNetworkIp4Dns()` and +`getNetworkIp6Dns()` (`server/src/main/java/com/cloud/network/NetworkModelImpl.java:3023` and +`:3037`) implement exactly that precedence: network `dns1`/`dns2` if set, then the VPC, then the +zone. The VPC branch is simply never reached here (§6.6). The `networks` table already carries +`dns1`, `dns2`, `ip6_dns1`, `ip6_dns2`, and `createNetwork`/`updateNetwork` already expose them. + +Note the interaction with §6.5: DNS is optional on the offering. If the offering omits the `Dns` +service, these values are never written to ConfigDrive regardless of being configured on the network +or zone. + +### 8.4 Metadata service **DECIDED — none in v1** + +With no VR there is no `data-server` at the gateway and no link-local metadata endpoint. **The +ConfigDrive ISO is the only source of metadata and user data**, and there is no +`169.254.169.254`-style HTTP endpoint. + +**Note this explicitly for operators**, because it is a real behavioural difference from other +network types rather than an omission. Tooling that expects to `curl 169.254.169.254` — cloud-init +in some configurations, Kubernetes cloud providers, various agents — will not work unmodified. A +template that reads its metadata from ConfigDrive is fine; one that assumes the HTTP endpoint is not. + +A host-side responder on the `brdr-*` bridge is entirely feasible later: the gateway address is +already there, the host already routes for the guest, and the data is already assembled for the ISO. +It is deferred rather than ruled out — **a candidate for v2** (§15). + +## 9. Hypervisor (KVM) implementation + +### 9.1 The agent's entire job + +Per guest address, on NIC plug: install a static neighbour entry and a host route. On unplug / VM +stop / migrate-away: remove them. **That is the whole contract.** Nothing else on the host is +CloudStack's concern. + +#### 9.1.1 This already exists — reuse `modifymacip.sh` **DECIDED** + +The mechanism was added to `main` by `4816e059383` ("KVM: add configurable MAC/IP script hook for +static ARP/NDP and routes", PR #13495, 2026-07-10) for the VXLAN/EVPN static MAC-IP work. It does +almost exactly what this design needs: + +* `scripts/vm/network/vnet/modifymacip.sh` — `-o add -b -m [-4 ]... [-6 ]...` + runs `ip neigh replace lladdr dev nud permanent` and + `ip route replace /32 dev `, and the `-6` equivalents with `/128`. `-o delete -b + -m ` discovers the addresses to remove by querying the neighbour table for that MAC, so no + state file is needed. +* `BridgeVifDriver.executeMacIpScript()` + (`plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java:437` + for add, `:418` for delete) is already wired into NIC plug (`:291`) and unplug (`:298`). +* It also passes the MAC-derived IPv6 link-local automatically + (`NetUtils.ipv6LinkLocal(mac)`), handles secondary IPs, and sets + `net.ipv6.conf..disable_ipv6=0` before installing NDP entries. +* Gated by the agent property `vm.network.macip.static` + (`AgentProperties.VM_NETWORK_MACIP_STATIC`, default `false`). + +So §9.1 is largely an integration exercise rather than new code. Four gaps to close. + +#### 9.1.2 Bridge, not tap **DECIDED** + +Routes and neighbour entries go on the **bridge**, exactly as `modifymacip.sh` and the existing +`BridgeVifDriver` hook already do (`intf.getBrName()` is passed at `:291`). **No change to the +script or the hook for this.** The same code serves both the EVPN case and this one; there is no +reason to write it twice. + +Consequence to be aware of: the host route pins each address to the bridge, and the static neighbour +entry pins it to a **MAC** — but the MAC-to-port mapping comes from ordinary bridge FDB learning, so +the routing table alone is not an anti-spoofing boundary. §12.1 sets out why the design is still +sound (short version: the per-network bridge is the isolation boundary, so MAC-to-port pinning only +matters within one account's own network), and §12.3 for what remains exposed. + +Per-tap routes were considered and rejected for v1. Recorded for completeness in case the anti-spoof +story needs tightening later: `-b` is passed verbatim as the `dev` argument to every `ip` command, +and the delete path's `ip neigh show dev ` works on a tap too, so targeting a tap would need no +script change at all — only a rename of `-b` to something less misleading. + +#### 9.1.3 Gating is per NIC, inferred **DECIDED** + +`vm.network.macip.static` is resolved once in `configure()` (`BridgeVifDriver.java:87`) and is +all-or-nothing for the host. A host runs direct routed guests *and* ordinary bridged guests side by +side, so the behaviour is decided **per NIC**, inferred from what the `NicTO` already carries — not +from a host property, and without adding a flag. + +**What the agent can actually see.** `NicTO`/`NetworkTO` expose `broadcastType`, `type` +(TrafficType), `networkId`, `gateway`, `netmask`, `ip6Cidr` and `securityGroupEnabled` +(`api/src/main/java/com/cloud/agent/api/to/NicTO.java`, +`api/src/main/java/com/cloud/agent/api/to/NetworkTO.java`). Two things it does **not** carry: + +* **Guest type is not on the TO at all.** There is no `GuestType` field, so `GuestType.L3` cannot be + tested directly on the agent. +* **Broadcast type is ambiguous.** Since §9.2.1 dropped the broadcast domain, these NICs are + `BroadcastDomainType.Native`, which is shared with other untagged cases. + +**The signature to infer from** is therefore the addressing itself, which no other CloudStack network +type produces: an IPv4 netmask of `255.255.255.255` (and/or an IPv6 `/128`) together with a gateway +inside `169.254.0.0/16`. Shared and Isolated networks always hand out a real subnet mask; L2 hands +out none. This is the same key §8.1 uses to decide the ConfigDrive `on-link` rule, so the two stay +consistent by construction. + +**One inference, used twice.** The same test decides both which bridge to use (`brdr-`, +§9.2.1) and whether to run the MAC/IP hook. They are not separate decisions — if the driver has +chosen a `brdr-` bridge, the hook applies. + +The agent property `vm.network.macip.static` stays as an independent host-wide opt-in for the EVPN +use case and is unaffected. + +**Caveat, worth stating:** this is an implicit contract. Nothing stops a future change from handing a +guest a /32 for some other reason and silently activating this path. If that becomes a real risk, the +explicit alternative is a boolean on `NicTO` set by the guru — a small change, deliberately not taken +now because it adds plumbing for a case that does not yet exist. + +#### 9.1.4 Silent failures are kept **DECIDED — accepted** + +Both `executeMacIpScript()` overloads catch everything and only log, deliberately — "managing host +neighbour/route entries is best-effort and must never break VM lifecycle operations" +(`BridgeVifDriver.java:432`). + +**That behaviour is kept unchanged.** No new failure handling for this network type in v1, which +also means the existing EVPN path cannot be regressed by this feature. + +The consequence, stated so it is not a surprise later: if the route or neighbour install fails, the +Instance starts normally and appears healthy, but has **no connectivity at all**, and nothing in +CloudStack reports why. Diagnosis means looking at the agent log for the warning from +`executeMacIpScript()`, or checking `ip route` / `ip neigh` on the host. + +Making it fatal to the NIC plug, or raising an alert, remains available later (§15) and would be a +small change — the call sites already distinguish success from failure, they simply do not act on it. + +#### 9.1.5 Secondary IPs **IMPLEMENTED** + +CloudStack lets an Instance hold secondary IPv4 and IPv6 addresses on a NIC, and they need the +same treatment as the primary: a host route and a static neighbour entry, or the address is not +reachable. + +**At Instance start this already worked.** `modifymacip.sh` accepts repeated `-4`/`-6`, and +`BridgeVifDriver` passes `nic.getNicSecIps()` alongside the primary addresses, so every address the +NIC holds is installed on plug. Unplug removes them all by MAC. + +**Adding or removing one on a *running* Instance did not, and was fixed.** That path +(`NetworkRulesVmSecondaryIpCommand`) only ever updated ipsets and ebtables, and the management +server only sent it when security groups were in play — three separate gates +(`SecurityGroupManagerImpl`: Instance in no security group, network without the SG service; +`NetworkServiceImpl.configureNicSecondaryIp` / `RemoveIpFromVmNicCmd`: zone without SG). On a +Direct Routed network with security groups disabled, none of them fired, so a newly added secondary +IP stayed dark until the Instance was restarted — and a removed one kept being routed and +advertised. + +The command now carries `directRouted` and `applySecurityGroupRules`; the agent installs or removes +the host route and neighbour entry for the address whenever the former is set, independently of +`canBridgeFirewall`, and skips the security group script when the latter is not. `modifymacip.sh` +gained per-address delete (`-o delete` with `-4`/`-6` removes just those; without them it keeps its +delete-everything-for-this-MAC behaviour, which is what unplug uses). + +**The ipset-based dispatch pays off here.** Because to-Instance traffic on L3 matches +`--match-set dst` (§12.2), and secondary IPs are added to that same ipset, security group +dispatch covers a new secondary IP with no rule changes at all. + +#### 9.1.6 Not a gap: the shared gateway addresses + +`modifymacip.sh` does not configure `169.254.0.1` / `fe80::1` — that is `modifybrdr.sh`'s job when it +creates the network's bridge (§9.2). The two scripts have a clean split: `modifybrdr.sh` owns the +bridge and its gateway addresses, `modifymacip.sh` owns per-guest routes and neighbour entries on it. + +Minor robustness note: because the delete path derives addresses from the neighbour table, a route +leaks if its neighbour entry has already been flushed. Reconciliation (§9.6) covers this. + +### 9.2 One bridge per network **DECIDED** + +**Each direct routed network gets its own bridge on every hypervisor that runs one of its +Instances**, named `brdr-` (Bridge-DirectRouted) — for example `brdr-42`. + +The `` is simply `networks.id`, the network's numeric database id — unique by construction, with +nothing to allocate and nothing for the user to choose (§9.2.1). + +The bridge is created and removed by a new script, +`scripts/vm/network/vnet/modifybrdr.sh`, modelled on `modifyvxlan.sh`: + +``` +modifybrdr.sh -o add -n [-4 ] [-6 ] +modifybrdr.sh -o delete -n +``` + +On `add` it creates the bridge if absent (STP off, `forward_delay 0` — there is no uplink, so no +loop to detect and no reason to hold ports down at Instance start), enables IPv4/IPv6 forwarding on +it, disables RA acceptance, and configures the gateway addresses. On `delete` it removes the bridge, +but only after confirming nothing is still attached — an Instance may have started on the network +while the last one was stopping. The whole script runs under `flock`, like `modifyvxlan.sh`, because +concurrent Instance starts on one network will race to create the bridge. + +Consequences: + +* **Networks are isolated from each other at layer 2 by the topology**, not by filtering. This is + the first reason for the design: a guest on `brdr-42` has no L2 path of any kind to a guest on + `brdr-43`, and no rule set has to be correct for that to hold (§12). +* **Each network becomes a named L3 interface on the hypervisor.** This is the second reason, and it + is an operational one: `brdr-42` is something the operator can attach local policy to. Different + route-maps or redistribution filters per network, per-network policy routing, QoS, or later a VRF + per bridge — all expressible in the host's own network configuration, matched on interface name, + with no involvement from CloudStack. The routing daemon is already the operator's (§10); giving + each network its own interface is what makes per-network routing decisions possible at all. +* **Bridge name is identical on every host** — it derives only from the network id — which is what + keeps migration a no-op for the guest. +* **The vif driver now needs work.** The earlier shared-bridge design could ride on + `BridgeVifDriver`'s existing `brname = trafficLabel` fallback + (`plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java:255`); + that no longer applies. The driver must derive `brdr-` from `NicTO.getNetworkId()` and invoke + `modifybrdr.sh` on plug, and on unplug when the last interface leaves — the same shape as its + existing `createVnetBr()` handling for VXLAN. + +#### 9.2.1 How the agent learns the bridge name **DECIDED** + +**`NicTO` already carries the network id.** `NicTO.networkId` (`api/src/main/java/com/cloud/agent/api/to/NicTO.java:35`) +is populated for every NIC by `HypervisorGuruBase.toNicTO()` +(`server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java:208`, +`to.setNetworkId(profile.getNetworkId())`). + +`BridgeVifDriver` passes the network id to `modifybrdr.sh`, which creates the bridge and **prints +the name it chose** — the agent uses whatever comes back. On unplug the agent asks the same script +(`-o delete -b `), which answers `notmine`, `kept` or `deleted`; `notmine` sends the agent +down its regular unplug path. **How the bridges are named is known only to the script**; no +`brdr-` prefix appears anywhere in Java. + +That is the entire mechanism. No new `BroadcastDomainType`, no broadcast URI, no isolation method, no +id allocation, no new field on any TO — and nothing the user has to choose. The id is unique by +construction because it is the network's primary key. + +Earlier drafts proposed a `routed://` broadcast domain with an operator-choosable id allocated +from the physical network's VNET range. Dropped: it added an allocation mechanism, a range to +configure, and a choice to make, all to convey a number the agent already has. + +Consequences: + +* The network's `broadcast_uri` stays empty and its broadcast domain type is `Native`. Nothing is + allocated, so nothing has to be released on network deletion. +* `specifyVlan` is `false` and stays that way (§6.4). +* An operator who wants `brdr-42` to be a *specific* number cannot have it. Bridge names are + discoverable from the network's id in the UI and API, so per-network host routing policy (§9.2) is + still perfectly writable — it just has to be written after the network exists rather than chosen + in advance. + +Per bridge, `modifybrdr.sh` sets: + +* `169.254.0.1/32` and `fe80::1/64` +* `net.ipv4.conf..forwarding=1`, `net.ipv6.conf..forwarding=1` +* `net.ipv6.conf..disable_ipv6=0` and `accept_ra=0` +* `arp_ignore=1` / `arp_announce=2` — see §9.2.1 +* **no physical uplink** — see §9.3, this is mandatory +* proxy ARP: **not needed** — the gateway addresses are local to the bridge, so the bridge answers + guest ARP/ND directly, and host→guest neighbour entries are static rather than resolved + +#### 9.2.2 The same gateway address on many bridges **DECIDED** + +Every `brdr-*` bridge on a host carries the *same* `169.254.0.1` and `fe80::1`. This is intended, +and it is what makes a guest's configuration identical no matter which network or host it lands on. + +For IPv6 it is unremarkable: link-local addresses are per-link and scoped by interface, so `fe80::1` +on twenty bridges is normal and correct. + +For IPv4 the sysctls are what make it correct, and `modifybrdr.sh` sets both: + +* `arp_ignore=1` — answer ARP only for addresses configured on the interface the request arrived on, + so a request reaching `brdr-42` is never answered on behalf of `brdr-43` +* `arp_announce=2` — always source ARP from the address of the interface the request goes out of + +Each bridge is its own L2 domain, so the ARP exchange stays within the right one regardless; the +sysctls remove the cases where the host might otherwise answer or source from the wrong interface. +The host's local route table gains one `local 169.254.0.1` entry per bridge, which is harmless — the +host never originates traffic from that address, it only replies on-link. + +### 9.3 The bridges have no physical uplink **DECIDED — correctness requirement** + +Unlike `cloudbr0`, a `brdr-*` bridge has **no physical port**. It is purely host-local: the only L3 +presence on it is `169.254.0.1` / `fe80::1`, and all guest traffic leaves the host via the host's own +routed uplink rather than being bridged. `modifybrdr.sh` never enslaves an interface, so this holds +by construction as long as nothing else adds one. + +If a bridge did have an uplink onto a shared L2 segment, two things break: + +1. **Duplicate gateway addresses.** Every hypervisor configures `169.254.0.1` and `fe80::1` on every + `brdr-*` bridge. Put those on a common segment and every host answers ARP/ND for the same + addresses; guests would resolve the gateway to an arbitrary host's MAC. +2. **Isolation collapses.** Guests of that network across all hosts would share one broadcast domain, + and the per-network bridge would stop being a boundary. + +### 9.4 Layer 2 isolation comes from the bridges **DECIDED** + +Separate bridges per network are the isolation mechanism. A guest on `brdr-42` cannot send a frame of +any kind — ARP, raw L2, rogue RA, anything — to a guest on `brdr-43`. There is no shared broadcast +domain, no shared FDB, and no path that filtering would have to police. + +This replaces the earlier shared-bridge design, in which separate networks were only administrative +and L2 separation had to be recovered with filtering. **No libvirt nwfilter is used**: the +`no-mac-spoofing` / `clean-traffic` approach was for the shared-bridge model and is dropped entirely. + +Two consequences worth stating plainly: + +* **Isolation between networks needs no filtering at all.** The boundary is the bridge rather than + a rule set. That was the motivation for this change. +* **Guests within one network still share a bridge**, so spoofing between them remains possible + (§12.3). Since a network belongs to one account, that is intra-tenant exposure rather than + cross-tenant. + +Bridge port isolation (`bridge link set dev vnetX isolated on`) remains available as a further step +if intra-network isolation is ever wanted; it is not applied in v1 (§12.3, §15). + +### 9.5 Sysctls + +Routing happens on the bridge, so the bridge is the L3 input interface for guest traffic and these +apply per `brdr-*` bridge rather than per tap. `modifybrdr.sh` sets them at creation: + +* `net.ipv4.conf..forwarding=1`, `net.ipv6.conf..forwarding=1` +* `net.ipv6.conf..disable_ipv6=0` — also set by `modifymacip.sh` before installing NDP entries +* `net.ipv6.conf..accept_ra=0` — a guest must never be able to send an RA the host acts on +* `net.ipv4.conf..arp_ignore=1` / `arp_announce=2` — required because every `brdr-*` bridge on + the host carries the same gateway address (§9.2.2) + +**`rp_filter=1` (strict) is set on every `brdr-*` bridge. DECIDED.** An Instance may only send from +an address that routes back out of the bridge it arrived on — which, given per-guest /32 routes, is +its own address. Source spoofing is therefore blocked at the host. + +Set **on the bridge only, never on `all`**, so no other interface on the host changes behaviour. The +kernel takes `max(conf.all.rp_filter, conf..rp_filter)`, so a per-bridge value of 1 is effective +regardless of what `all` is, without touching the uplink — which matters, because strict mode on a +host uplink can break legitimately asymmetric fabric routing. + +Strict mode is safe for the paths this design creates: + +* an Instance's own traffic — source `S`, route `S/32 dev brdr-N`, arrives on `brdr-N` → passes +* same-network hairpin (§5.4) — arrives on `brdr-N` from an address routed via `brdr-N` → passes +* cross-network — arrives on `brdr-N`, forwarded out `brdr-M`; the check is on ingress only → passes + +**IPv4 only.** The kernel has no IPv6 `rp_filter`, so IPv6 source spoofing is not bounded by this and +must be handled by security groups (§12.2) where it matters. + +### 9.6 No reconciliation on agent restart **DECIDED** + +**The agent does not scan or rebuild routes and neighbour entries at startup.** The existing hook +fires only on NIC plug/unplug (`BridgeVifDriver.java:291,298`), and that is sufficient. + +The reasoning, which is worth writing down because "kernel state is not persistent" invites the +opposite conclusion: + +* **Host reboot** clears the routes — but it also destroys every Instance on that host. The + management server starts them again, each start goes through the plug path, and the routes are + reinstalled as a side effect. There is nothing to reconcile, because there is nothing running whose + state could be missing. +* **Agent restart without a host reboot** does not clear anything. Routes and neighbour entries live + in the kernel, not in the agent, so running Instances keep working across an agent restart with no + action at all. +* **While the agent is down**, CloudStack cannot stop or migrate Instances on that host, so host + state cannot drift from what the management server believes. + +`modifymacip.sh` uses `ip route replace` / `ip neigh replace`, so any repeated plug is idempotent +regardless. + +**Residual, accepted:** if an Instance disappears without an unplug — libvirt kills it, or it crashes +in a way that skips the normal path — its route and neighbour entry linger. The practical risk is not +the stale kernel state itself but that the routing daemon keeps advertising that /32, so if the +Instance is started on another host the fabric may see the address from two places. Rare, and not +worth a startup sweep to prevent; noted so it is recognisable if it ever shows up. + +## 10. Routing daemon — explicitly out of scope + +**CloudStack does not install, configure, or monitor the routing daemon.** + +The contract is one-directional: *CloudStack guarantees the correct routes and neighbour entries +exist in the host kernel.* The operator configures their daemon to redistribute them, e.g. with +FRR: + +``` +router bgp 65001 + address-family ipv4 unicast + redistribute kernel + address-family ipv6 unicast + redistribute kernel +``` + +Why this is the right boundary: + +* Identical behaviour for FRR, BIRD, or anything else that redistributes kernel routes. +* Identical behaviour for BGP, OSPF, or IS-IS. +* No daemon version coupling, no config-file ownership conflict with the operator's automation, no + `vtysh`/reload dependency in the agent. +* Nothing new to fail: if the route is in the kernel the guest works locally, and advertisement is + the fabric's business. + +Consequences to accept and document: + +* We ship **documentation and reference configuration**, not code. Redistribution filtering via + route-maps is the operator's job. +* **No feedback loop.** CloudStack cannot tell whether a guest's address is reachable from the + fabric. Accepted as a known gap for v1; a health signal from the agent is listed under future work + (§15) rather than treated as an open question. +* This is the **opposite** choice from the 4.20 BGP work, where CloudStack manages peers + (`BgpPeerVO`, `SetBgpPeersCommand`, `systemvm/debian/opt/cloud/bin/cs/CsBgpPeers.py`). The doc + should say why explicitly — reviewers will ask. +* Design should not preclude an optional managed mode later, but v1 does not have one. + +## 11. Live migration + +The guest's configuration is host-independent, so the guest needs no reconfiguration. What moves is +host state: + +1. Destination host installs the route and neighbour entry when the NIC is plugged. +2. Source host removes both when the NIC goes away. +3. Each host's routing daemon advertises/withdraws; the fabric reconverges. + +To work through: + +* **Ordering.** Install-then-remove is safer than remove-then-install; a transient duplicate + advertisement is less harmful than a black hole. Confirm the plug/unplug sequence during + migration actually gives that ordering. +* **Convergence gap.** Traffic may be black-holed until the fabric reconverges. **TODO:** quantify + on a normal iBGP/OSPF setup — is sub-second realistic? +* **No GARP needed.** Normally a migrating VM sends a gratuitous ARP to update switch tables. Here + there is no L2 path to update, and the destination host's neighbour entry is installed statically + by the agent rather than learned — so this problem disappears. Verify. +* **Per-address advertisement is mandatory.** Any aggregation scheme that pins prefixes to hosts + breaks migration. This is the price of §6.3.3. +* **Routing domain boundaries.** Migration works as long as source and destination are in the same + routing domain. **Left to the operator, not validated by CloudStack** — consistent with §10 and + §6.3.3, where fabric topology is local network design. CloudStack has no model of routing domains + and inventing one to police migration would be a larger change than the problem warrants. + +## 12. Security + +### 12.1 The isolation model + +The boundary between networks is **topological**: one bridge per network (§9.2, §9.4), with no +uplink and therefore no path between bridges except through the host's routing table. Nothing has to +be configured correctly for that to hold, and nothing degrades if filtering is disabled. + +Within one network the guests still share a bridge, so the properties there are weaker: + +* **No L3 adjacency.** Each guest is a /32 behind the host's routing table; there is no + subnet-mates relationship to exploit even between guests on the same bridge. +* **Static neighbour entries** prevent ARP-based address takeover on the host→guest path: the + IP-to-MAC mapping is asserted by `modifymacip.sh`, never learned (§5.3). The host is therefore not + susceptible to a guest claiming another guest's address. +* **`rp_filter=1` on the bridge** (§9.5) confines an Instance to sending from its own /32, since + that is the only address routed back out of that bridge. IPv4 only. + +What remains open inside a network is set out in §12.3. Because a network belongs to one account, +that is intra-tenant exposure. + +### 12.2 Security groups — supported, via the unified script rules **DECIDED** + +Security groups are supported on L3 networks, programmed by `security_group.py` as for Shared +networks. There is **no separate L3 rule function**: the classic and Direct Routed paths share one +implementation, parameterized only by how each direction identifies the Instance, plus small +conditionals for what does not exist on L3 (DHCP, DHCPv6, router advertisements towards guests). + +| Direction | Classic bridge | Direct Routed bridge | +|---|---|---| +| from the Instance | `-m physdev --physdev-in ` | same rule, identical | +| towards the Instance | `-m physdev --physdev-is-bridged --physdev-out ` | `-m set --match-set dst` | + +**The `--physdev-is-bridged` question, settled in kernel source** (`net/netfilter/xt_physdev.c`): + +* On `--physdev-in` rules the flag was **removable**. Match-time semantics: a bridged-then-routed + packet carries bridge info with the ingress port set and no bridged egress port, so a plain + `--physdev-in` matches it while `--physdev-is-bridged` excludes it. On classic bridges removing + the flag changes nothing observable — the BF- framework hook (which keeps the flag) already + restricts what reaches the per-VM chains to bridged traffic. Removing it is what lets both paths + share the from-Instance rules verbatim. The golden test was regenerated once, deliberately, for + exactly this diff. +* On `--physdev-out` rules the flag **stays**: for a routed packet no bridge info exists at all at + FORWARD time (`nf_bridge_info_exists()` is false and every physdev variant returns false), so the + kernel is structurally incapable of identifying the bridged egress port there. Removing the flag + would change nothing and merely deviate from upstream convention. This is why the to-Instance + direction matches the destination against the Instance's ipsets instead — exact, since L3 + addresses are /32s and /128s. + +The IPv6 source-spoof drop in the shared rules doubles as the missing IPv6 `rp_filter` (§9.5). +Teardown uses one awk pattern on the chain names, which also covers rules created by older versions +that still carry the flag on `physdev-in` lines. + +**Framework setup is shared too.** `enable_bridge_netfilter()`, `create_bridge_fw_chains()` and +`add_notrack_ipset_rules()` were extracted from `add_fw_framework()` and are used by both it and +`add_l3_fw_framework()`; the two differ only in their FORWARD hooks (classic gates on +`physdev-is-bridged` and consults chain reference counts; L3 jumps unconditionally, with the same +default-deny backstop). A second golden test pins `add_fw_framework`'s command stream, proving the +extraction left it byte-identical — 44 commands, unchanged. + +**How the script knows: the Agent tells it.** `security_group.py` performs no classification of its +own — no bridge-name check, no gateway inspection. The Agent already identifies these NICs for the +bridge and MAC/IP hook (§9.1.3), so it passes `--directrouted` on `default_network_rules` and +`add_network_rules`; the script's own re-entry points thread the flag through. This keeps the +decision in exactly one host-side place and leaves the script with no inference to get wrong. + +One path cannot receive the flag: `network_rules_for_rebooted_vm` runs without the Agent. It +restores dispatch rules in whichever form the Instance's per-VM chain already uses (destination +ipset for Direct Routed, `physdev-out` for bridged), so a rebooted Instance is reprogrammed +correctly either way. + +**Investigated and rejected: libvirt nwfilters.** An nwfilter-based implementation was built and +then discarded. Its ebtables layer would have served anti-spoofing well (it sees routed delivery), +but stateful ingress cannot work: libvirt's own to-Instance iptables hook is hard-coded as +`-m physdev --physdev-is-bridged --physdev-out` (`src/nwfilter/nwfilter_ebiptables_driver.c`), the +same structural blindness to routed delivery — inside libvirt, where it cannot be patched or +augmented with destination matching. The script approach filters correctly in both directions and, +after the unification above, without duplicate code. + +### 12.3 Residual risk within one network **ACCEPTED for v1** + +Guests of the same network share a bridge. With security groups enabled, RA and gateway +impersonation between them are handled by the shared rules (ebtables ARP pinning, NDP source +checks, RA drop) and source spoofing is bounded by the ipset checks plus `rp_filter` (§9.5). An +operator who disables security groups accepts intra-tenant spoofing between guests of that one +network — a deliberate operator choice; isolation between *tenants* is topological and unaffected +(§12.1). The host itself is immune either way: its neighbour entries are static (§5.3). + +### 12.4 Sharp edge + +**Guests are directly reachable from the fabric.** No NAT, no VR firewall. Whatever the fabric +permits reaches the guest, subject only to the guest's security groups if they are in use. This is a +meaningful change in default posture versus an Isolated network and must be prominent in the +documentation. + +## 13. Orchestration touchpoints + +Checklist to work through: + +- [ ] `NetworkOrchestrator.allocate()` / `prepare()` / `release()` — address lifecycle +- [ ] `NetworkOrchestrator` and `VirtualMachineManagerImpl` — `L2` and `Shared` branches +- [ ] `UserVmManagerImpl` — `addNicToVm`, `updateDefaultNic`, IP change +- [ ] `NetworkModelImpl` — capability lookups, `getNetworkTag`, `isSecurityGroupSupportedInNetwork` +- [ ] `IpAddressManagerImpl.allocateDirectIp()` — gateway/netmask override for /32 and /128 +- [ ] `BridgeVifDriver` — infer direct-routed from the /32 + link-local gateway; gate the MAC/IP + hook and bridge selection on it. Script and bridge targeting reused unchanged (§9.1.3) +- [ ] `scripts/vm/network/vnet/modifybrdr.sh` — per-network bridge lifecycle (§9.2) +- [ ] `BridgeVifDriver` — derive `brdr-`, call `modifybrdr.sh` on plug and last unplug +- [ ] New guru with `canHandle()` on guest type alone, no isolation method (§6.7) +- [ ] Offering validation — require ConfigDrive `UserData`, reject `Dhcp` (§6.4) +- [ ] `ConfigDriveBuilder.needForGeneratingNetworkData()` — must not gate on Dhcp/Dns (§8.2) +- [ ] `NetworkServiceImpl.java:657` — stop rejecting DNS for this type as it does for L2 (§6.4) +- [ ] `security_group.py` unified rules — verify on a real host that both directions match live + traffic on classic and Direct Routed bridges, and that ARP for `169.254.0.1` / ND for + `fe80::1` pass (§12.2) +- [ ] `createVlanIpRange` — zone-wide subnet overlap validation (§6.3.2) +- [ ] `NetUtils` — inclusive range variants so `.0` and `.255` are assignable (§6.3.1) +- [ ] VM snapshot / restore, VM import (`UnmanagedVMsManagerImpl`), template creation +- [ ] Network restart — no-op without a VR? +- [ ] IP capacity reporting and usage records — is a directly routed address a billable public IP? +- [ ] UI: network creation wizard, network detail page, NIC display, offering creation + +## 14. Upgrade and compatibility + +* Additive: new enum value, new offering type. No change to existing networks. +* **Agent version gating is out of scope. DECIDED.** No capability flag, no version check, and no + management-server logic to keep Instances of this type away from agents that predate it. Operators + are expected to upgrade their agents as part of upgrading CloudStack, as they already are. The + failure mode if they do not is that `modifybrdr.sh` is missing on the old host and the Instance + fails to get connectivity — visible in the agent log, consistent with §9.1.4. +* Downgrade unsupported once networks of this type exist, as usual. + +## 15. Future work / explicitly deferred + +* Non-KVM hypervisors +* Optional CloudStack-managed routing daemon configuration +* **Host-side metadata service** (`169.254.169.254`) — a v2 candidate; the gateway address and the + data are both already present, so it is a natural addition (§8.4) +* Multiple addresses per NIC — additional /32s fit the model naturally, but v1 is one v4 + one v6 +* Reachability/health feedback from the routing daemon +* Network Config v2 `network-config` emission for NoCloud-configured images — a later PR (§8.1) +* Making a failed route/neighbour install fatal or alert-raising, rather than silent (§9.1.4) +* Closing the §12.3 intra-network spoofing gaps — bridge port isolation (§9.4) first, or libvirt + nwfilter if a narrower fix is preferred +* **Per-tenant VRFs**, which would lift the non-overlapping-subnet constraint of §6.3.2 + +## 16. Decision log and open questions + +### Settled + +* Network type is "no DHCP", not "like L2" (§1) +* Addresses come from `user_ip_address` / `user_ipv6_address` with an operator-supplied subnet + (§6.3); subnets must not overlap zone-wide (§6.3.2) +* New `GuestType.L3`, chosen over overloading `Shared`/`NetworkMode` (§6.1) +* Gateway is a static, non-configurable `169.254.0.1` / `fe80::1`; a /32 means the guest must treat + its gateway as on-link regardless, so configurability would buy nothing (§6.2) +* ConfigDrive emits `on-link: true` for gateways in `169.254.0.0/16` (§8.1) +* The agent writes only routes and neighbour entries; FRR is out of scope (§9.1, §10) +* That work is done by reusing `modifymacip.sh` + the `BridgeVifDriver` hook already in main from + `4816e059383` / PR #13495, rather than new code (§9.1.1) +* Routes and neighbour entries go on the **bridge**, not the tap — the script and hook are reused + unchanged (§9.1.2) +* **One bridge per network**, named `brdr-`, created and removed by the new + `scripts/vm/network/vnet/modifybrdr.sh` (§9.2) +* No broadcast domain, no isolation method, no id allocation and nothing for the user to choose — + the agent already has the network id on `NicTO` (§9.2.1) +* Per-network bridges also give the operator a named interface per network to hang local routing + policy off — a deliberate benefit, not just a side effect (§9.2) +* Those bridges have no physical uplink (§9.3) +* L2 isolation between networks is topological — separate bridges — not filtering. **No libvirt + nwfilter is used**; the earlier `no-mac-spoofing` / `clean-traffic` plan is dropped (§9.4) +* Security groups are supported on L3 via **one unified rule implementation** shared with classic + bridges: `--physdev-is-bridged` dropped from `physdev-in` rules (verified harmless in kernel + source), destination-ipset matching towards the Instance. nwfilter was investigated and rejected + — libvirt's own to-Instance hook has the same physdev-is-bridged blindness (§12.2) +* Within one network, RA and gateway-impersonation protection depends on security groups being + enabled; leaving them off is a deliberate operator choice (§12.3) +* The `--physdev-is-bridged` rework is **required** — L3 filtering must work — and must be strictly + additive so existing Basic-zone and Shared-network rules are unchanged (§12.2) +* The **offering alone** determines that a network is direct routed, on guest type — as for an L2 + network. Changing a network's offering afterwards is not guarded (§6.7, §6.7.1) +* The offering requires ConfigDrive `UserData`; `Dns` is optional but strongly recommended, and + `SecurityGroup` is optional. `Dhcp` is rejected — not just unsupported but unnecessary (§6.4, §6.5) +* `rp_filter=1` is set on each `brdr-*` bridge, bounding IPv4 source spoofing to the Instance's own + address; IPv6 has no kernel equivalent (§9.5) +* **No reconciliation at agent startup** — a host reboot destroys the Instances too, and an agent + restart does not clear kernel state (§9.6) +* The MAC/IP hook is gated **per NIC, inferred** from the /32 + link-local-gateway signature rather + than a host property or a new `NicTO` flag; the same test picks the bridge (§9.1.3) +* The same `169.254.0.1` on every `brdr-*` bridge is **correct as designed**; `arp_ignore=1` and + `arp_announce=2` handle it (§9.2.2) +* The guru is a **subclass of `DirectNetworkGuru`**, inheriting the address lifecycle rather than + duplicating it (§7.3) +* **No on-link plumbing in ConfigDrive** — cloud-init itself sets on-link for an IPv4 gateway in + `169.254.0.0/16` when consuming `network_data.json` (verified); the v2 `network-config` file is + deferred to a later PR (§8.1, §15) +* DNS is **per network, falling back to the zone**; already implemented by + `NetworkModelImpl.getNetworkIp4Dns()` (§8.3) +* **No metadata service** in v1 — ConfigDrive only, no `169.254.169.254`; a v2 candidate (§8.4) +* Route/neighbour install failures **stay silent** for v1, as they are for EVPN (§9.1.4) +* Route scale is **out of scope** — fabric capacity and aggregation are local network design; + ~100k routes is not usually a problem on modern equipment, but CloudStack states no ceiling (§6.3.3) +* Network and broadcast addresses **must be assignable** — verified already true on the explicit + start/end range path; the `NetUtils` exclusion only affects CIDR-derived Isolated ranges (§6.3.1) +* The subnet's gateway stays **required and ignored**; relaxing validation shared with other + network types is not worth it for v1 (§5.5) +* The `--physdev-is-bridged` rework **lands in v1**; security groups are not shipped half-working + (§12.2) +* Zone-wide subnet overlap validation is **required** — an overlap is an address conflict (§6.3.2) +* **No warning** when a template ignores ConfigDrive; cloud-init inside the guest is the operator's + responsibility (§6.5) +* **Agent version gating is out of scope** — operators upgrade agents with CloudStack (§14) +* KVM only for v1 (§6.8); **VPC never** — it is a separate use case already served by VPC's own + BGP-routed subnets (§6.6) + +### Still open + +**None.** Every design question raised in this document has been decided. + +What remains is implementation work, tracked in §13 and as `TODO` markers in the sections above. +Three of those are verification rather than coding, and are the ones most likely to change a +decision if they come out badly: + +* §12.2 — confirm on a real host that the unified rules match live traffic in both directions on + both bridge types, and that ARP for the gateway and neighbour discovery pass +* §6.3.1 — confirm nothing downstream of `createVlanIpRange` re-derives the usable range and + re-excludes `.0` and `.255` +* §6.3.2 — confirm the existing overlap checks are zone-wide, and widen them if not diff --git a/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java b/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java index 15febbe972c4..9b27cd726591 100644 --- a/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java +++ b/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java @@ -48,6 +48,7 @@ import com.cloud.network.NetworkModel; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.NetUtils; import com.cloud.utils.script.Script; import com.google.gson.JsonArray; import com.google.gson.JsonElement; @@ -249,8 +250,15 @@ static void writeVmMetadata(List vmData, String tempDirName, File open */ static void writeNetworkData(List nics, Map> supportedServices, File openStackFolder) { JsonObject finalNetworkData = new JsonObject(); - if (needForGeneratingNetworkData(supportedServices)) { + // A direct routed NIC always needs its network data written: ConfigDrive is the only + // channel that carries its addressing, whatever services the offering does or does not + // have. For every other NIC the historical gate (Dhcp or Dns supported) is unchanged. + boolean generateForAllNics = needForGeneratingNetworkData(supportedServices); + if (generateForAllNics || nics.stream().anyMatch(ConfigDriveBuilder::isDirectRoutedNic)) { for (NicProfile nic : nics) { + if (!generateForAllNics && !isDirectRoutedNic(nic)) { + continue; + } List supportedService = supportedServices.get(nic.getId()); JsonObject networkData = getNetworkDataJsonObjectForNic(nic, supportedService); @@ -267,6 +275,24 @@ static boolean needForGeneratingNetworkData(Map> sup return supportedServices.values().stream().anyMatch(services -> services.contains(Network.Service.Dhcp) || services.contains(Network.Service.Dns)); } + /** + * A NIC on a Direct Routed (L3) network is recognised by the form of its addressing, not by a + * flag: an IPv4 host netmask with a link-local gateway, or an IPv6 /128 with the fixed + * link-local gateway. No other network type produces this combination. ConfigDrive is the + * only channel that carries such a NIC's network configuration (there is no DHCP and no RA), + * so network data must always be generated for it, whatever services the offering carries. + */ + static boolean isDirectRoutedNic(NicProfile nic) { + if (nic == null) { + return false; + } + boolean directRoutedIpv4 = StringUtils.isNotBlank(nic.getIPv4Address()) && NetUtils.IPV4_HOST_NETMASK.equals(nic.getIPv4Netmask()) + && StringUtils.isNotBlank(nic.getIPv4Gateway()) && NetUtils.isIpWithInCidrRange(nic.getIPv4Gateway(), NetUtils.getLinkLocalCIDR()); + boolean directRoutedIpv6 = StringUtils.isNotBlank(nic.getIPv6Address()) && StringUtils.isNotBlank(nic.getIPv6Cidr()) + && nic.getIPv6Cidr().endsWith("/" + NetUtils.IPV6_HOST_PREFIX_LENGTH) && NetUtils.getIpv6LinkLocalGateway().equals(nic.getIPv6Gateway()); + return directRoutedIpv4 || directRoutedIpv6; + } + /** * Writes an empty JSON file named vendor_data.json in openStackFolder * diff --git a/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java b/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java index 03ceac843997..9b785ae95900 100644 --- a/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java +++ b/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java @@ -52,6 +52,7 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; +import com.google.gson.JsonArray; import com.google.gson.JsonObject; @RunWith(MockitoJUnitRunner.class) @@ -659,4 +660,96 @@ public void testWriteNetworkDataEmptyJson() throws Exception { Assert.assertEquals(expectedJsonObject, actualJson); folder.delete(); } -} + + private NicProfile directRoutedNicProfile() { + NicProfile nic = new NicProfile(); + nic.setId(1L); + nic.setDeviceId(0); + nic.setMacAddress("02:00:4c:5f:00:01"); + nic.setIPv4Address("203.0.113.55"); + nic.setIPv4Netmask("255.255.255.255"); + nic.setIPv4Gateway("169.254.0.1"); + nic.setIPv6Address("2001:db8:1::55"); + nic.setIPv6Cidr("2001:db8:1::55/128"); + nic.setIPv6Gateway("fe80::1"); + nic.setIPv4Dns1("8.8.8.8"); + return nic; + } + + private NicProfile sharedNicProfile() { + NicProfile nic = new NicProfile(); + nic.setId(1L); + nic.setDeviceId(0); + nic.setMacAddress("02:00:4c:5f:00:02"); + nic.setIPv4Address("10.1.1.55"); + nic.setIPv4Netmask("255.255.255.0"); + nic.setIPv4Gateway("10.1.1.1"); + return nic; + } + + @Test + public void isDirectRoutedNicRecognisesHostRouteForm() { + Assert.assertTrue(ConfigDriveBuilder.isDirectRoutedNic(directRoutedNicProfile())); + } + + @Test + public void isDirectRoutedNicRecognisesIpv6OnlyForm() { + NicProfile nic = directRoutedNicProfile(); + nic.setIPv4Address(null); + nic.setIPv4Netmask(null); + nic.setIPv4Gateway(null); + Assert.assertTrue(ConfigDriveBuilder.isDirectRoutedNic(nic)); + } + + @Test + public void isDirectRoutedNicRejectsOrdinaryNics() { + Assert.assertFalse(ConfigDriveBuilder.isDirectRoutedNic(sharedNicProfile())); + Assert.assertFalse(ConfigDriveBuilder.isDirectRoutedNic(null)); + // a /32 with an ordinary gateway is not direct routed + NicProfile hostMaskOnly = sharedNicProfile(); + hostMaskOnly.setIPv4Netmask("255.255.255.255"); + Assert.assertFalse(ConfigDriveBuilder.isDirectRoutedNic(hostMaskOnly)); + } + + @Test + public void ordinaryNicRouteGenerationIsUnchanged() { + JsonArray networks = ConfigDriveBuilder.getNetworksJsonArrayForNic(sharedNicProfile()); + JsonObject ipv4Network = networks.get(0).getAsJsonObject(); + JsonArray routes = ipv4Network.getAsJsonArray("routes"); + Assert.assertEquals(1, routes.size()); + JsonObject defaultRoute = routes.get(0).getAsJsonObject(); + Assert.assertEquals("0.0.0.0", defaultRoute.get("network").getAsString()); + Assert.assertEquals("0.0.0.0", defaultRoute.get("netmask").getAsString()); + Assert.assertEquals("10.1.1.1", defaultRoute.get("gateway").getAsString()); + } + + @Test + public void networkDataIsGeneratedForDirectRoutedNicWithoutDhcpOrDns() throws Exception { + TemporaryFolder folder = new TemporaryFolder(); + folder.create(); + try { + Map> userDataOnly = Map.of(1L, List.of(Network.Service.UserData)); + ConfigDriveBuilder.writeNetworkData(List.of(directRoutedNicProfile()), userDataOnly, folder.getRoot()); + String json = FileUtils.readFileToString(new File(folder.getRoot(), "network_data.json"), com.cloud.utils.StringUtils.getPreferredCharset()); + Assert.assertTrue("direct routed nic must appear in network_data.json", json.contains("203.0.113.55")); + Assert.assertTrue(json.contains("169.254.0.1")); + } finally { + folder.delete(); + } + } + + @Test + public void networkDataStaysEmptyForOrdinaryNicWithoutDhcpOrDns() throws Exception { + TemporaryFolder folder = new TemporaryFolder(); + folder.create(); + try { + Map> userDataOnly = Map.of(1L, List.of(Network.Service.UserData)); + ConfigDriveBuilder.writeNetworkData(List.of(sharedNicProfile()), userDataOnly, folder.getRoot()); + String json = FileUtils.readFileToString(new File(folder.getRoot(), "network_data.json"), com.cloud.utils.StringUtils.getPreferredCharset()); + Assert.assertEquals("historical gate must be preserved for ordinary nics", "{}", json); + } finally { + folder.delete(); + } + } + +} \ No newline at end of file diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java index 327ec46e0ecf..1af8b893c51a 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java @@ -49,9 +49,28 @@ public class BridgeVifDriver extends VifDriverBase { private String _modifyVlanPath; private String _modifyVxlanPath; private String _macIpScriptPath; + private boolean _macIpStaticEnabled; + private String _modifyBrdrPath; private String _controlCidr = NetUtils.getLinkLocalCIDR(); private Long libvirtVersion; + /** + * A NIC on a Direct Routed (L3) network is recognised by the form of its addressing, never by + * a flag: an IPv4 host netmask with a link-local gateway, or an IPv6 /128 with the fixed + * link-local gateway. No other network type hands an Instance this combination. The same + * signature drives ConfigDrive's on-link emission, so the two stay consistent by construction. + */ + public static boolean isDirectRoutedNic(NicTO nic) { + if (nic == null) { + return false; + } + boolean directRoutedIpv4 = nic.getIp() != null && NetUtils.IPV4_HOST_NETMASK.equals(nic.getNetmask()) + && nic.getGateway() != null && NetUtils.isIpWithInCidrRange(nic.getGateway(), NetUtils.getLinkLocalCIDR()); + boolean directRoutedIpv6 = nic.getIp6Address() != null && nic.getIp6Cidr() != null + && nic.getIp6Cidr().endsWith("/" + NetUtils.IPV6_HOST_PREFIX_LENGTH) && NetUtils.getIpv6LinkLocalGateway().equals(nic.getIp6Gateway()); + return directRoutedIpv4 || directRoutedIpv6; + } + private static boolean isVxlanOrNetris(String protocol) { return protocol.equals(Networks.BroadcastDomainType.Vxlan.scheme()) || protocol.equals(Networks.BroadcastDomainType.Netris.scheme()); } @@ -84,14 +103,19 @@ public void configure(Map params) throws ConfigurationException throw new ConfigurationException("Unable to find " + vxlanScript); } - if (Boolean.TRUE.equals(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VM_NETWORK_MACIP_STATIC))) { - _macIpScriptPath = Script.findScript(networkScriptsDir, "modifymacip.sh"); + // Resolved unconditionally: Direct Routed (L3) NICs need the MAC/IP script regardless of + // the host-wide property, which remains the opt-in for running it on every NIC (EVPN). + _macIpScriptPath = Script.findScript(networkScriptsDir, "modifymacip.sh"); + _macIpStaticEnabled = Boolean.TRUE.equals(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VM_NETWORK_MACIP_STATIC)); + if (_macIpStaticEnabled) { if (_macIpScriptPath == null) { throw new ConfigurationException("Unable to find modifymacip.sh"); } logger.info("VM network MAC/IP static script configured: {}", _macIpScriptPath); } + _modifyBrdrPath = Script.findScript(networkScriptsDir, "modifybrdr.sh"); + libvirtVersion = (Long) params.get("libvirtVersion"); if (libvirtVersion == null) { libvirtVersion = 0L; @@ -241,7 +265,10 @@ public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicA } if (nic.getType() == Networks.TrafficType.Guest) { - if (isBroadcastTypeVlanOrVxlan(nic) && isValidProtocolAndVnetId(vNetId, protocol)) { + if (isDirectRoutedNic(nic)) { + String brName = createDirectRoutedBridge(nic); + intf.defBridgeNet(brName, null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps); + } else if (isBroadcastTypeVlanOrVxlan(nic) && isValidProtocolAndVnetId(vNetId, protocol)) { if (trafficLabel != null && !trafficLabel.isEmpty()) { logger.debug("creating a vNet dev and bridge for guest traffic per traffic label " + trafficLabel); String brName = createVnetBr(vNetId, trafficLabel, protocol); @@ -288,17 +315,92 @@ public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicA } intf.setLinkStateUp(nic.isEnabled()); - executeMacIpScript(intf.getBrName(), nic.getMac(), nic.getIp(), nic.getIp6Address(), nic.getNicSecIps()); + // Host-wide property (EVPN use case) runs the MAC/IP script for every NIC; a Direct + // Routed NIC needs it regardless, since the host route and static neighbour entry are + // what deliver its traffic. + if (_macIpStaticEnabled || isDirectRoutedNic(nic)) { + executeMacIpScript(intf.getBrName(), nic.getMac(), nic.getIp(), nic.getIp6Address(), nic.getNicSecIps()); + } return intf; } @Override public void unplug(LibvirtVMDef.InterfaceDef iface, boolean deleteBr) { - executeMacIpScript(iface.getBrName(), iface.getMacAddress()); + // How the bridges of Direct Routed (L3) networks are named is known only to + // modifybrdr.sh; the agent classifies an interface at unplug by asking it. "notmine" + // means this is not such a bridge and the regular unplug handling applies. + boolean directRouted = deleteDirectRoutedBridge(iface.getBrName()); + if (_macIpStaticEnabled || directRouted) { + executeMacIpScript(iface.getBrName(), iface.getMacAddress()); + } + if (directRouted) { + return; + } deleteVnetBr(iface.getBrName(), deleteBr); } + /** + * Ensures the per-network bridge for a Direct Routed NIC exists, with the shared gateway + * addresses and sysctls applied. Idempotent and flock'd in the script itself. A failure here + * is fatal to the NIC plug: without the bridge the domain XML would reference a nonexistent + * device and the Instance would fail to start with a far less useful error. + */ + private String createDirectRoutedBridge(NicTO nic) throws InternalErrorException { + if (_modifyBrdrPath == null) { + throw new InternalErrorException("Unable to find modifybrdr.sh: this host cannot run Instances on Direct Routed (L3) networks"); + } + Script command = new Script(_modifyBrdrPath, _timeout, logger); + command.add("-o", "add"); + command.add("-n", String.valueOf(nic.getNetworkId())); + // The gateway addresses come from the NIC, whose values the management server stamped at + // allocation (NetUtils.getLinkLocalGateway() / getIpv6LinkLocalGateway()) — the script has + // no defaults of its own, so the addresses are defined in exactly one place. + if (StringUtils.isNotBlank(nic.getGateway())) { + command.add("-4", nic.getGateway()); + } + if (StringUtils.isNotBlank(nic.getIp6Gateway())) { + command.add("-6", nic.getIp6Gateway()); + } + // The script prints the bridge name it created: how these bridges are named is known + // only to modifybrdr.sh, never to the agent. + OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); + String result = command.execute(parser); + if (result != null || StringUtils.isBlank(parser.getLine())) { + throw new InternalErrorException("Failed to create bridge for network " + nic.getNetworkId() + ": " + result); + } + return parser.getLine().trim(); + } + + /** + * Asks modifybrdr.sh to remove the bridge if it is one of its own and nothing is attached to + * it any more. Returns whether the bridge belongs to a Direct Routed network at all — + * "notmine" means it does not, and the caller falls back to the regular unplug handling. + * Best-effort beyond that: the script keeps the bridge while other Instances of the network + * still use it, and a leftover empty bridge is harmless and re-used on the next plug. + */ + private boolean deleteDirectRoutedBridge(String brName) { + if (_modifyBrdrPath == null || brName == null) { + return false; + } + try { + Script command = new Script(_modifyBrdrPath, _timeout, logger); + command.add("-o", "delete"); + command.add("-b", brName); + OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); + String result = command.execute(parser); + String verdict = parser.getLine() != null ? parser.getLine().trim() : ""; + if (result != null) { + logger.warn("Failed to delete bridge {}: {}", brName, result); + return false; + } + return !"notmine".equals(verdict); + } catch (Exception e) { + logger.warn("Failed to delete bridge {}", brName, e); + return false; + } + } + @Override public void attach(LibvirtVMDef.InterfaceDef iface) { Script.runSimpleBashScript("ip link set " + iface.getDevName() + " master " + iface.getBrName()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 4281036d9456..3bbffb655386 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -427,6 +427,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private boolean imageServerTlsEnabled = false; private String imageServerListenAddress; private String securityGroupPath; + private String macIpPath; private String ovsPvlanDhcpHostPath; private String ovsPvlanVmPath; private String routerProxyPath; @@ -1195,6 +1196,8 @@ public boolean configure(final String name, final Map params) th } securityGroupPath = Script.findScript(networkScriptsDir, "security_group.py"); + // Not fatal when absent: only Direct Routed networks need it, and a host may run none + macIpPath = Script.findScript(networkScriptsDir, "modifymacip.sh"); if (securityGroupPath == null) { throw new ConfigurationException("Unable to find the security_group.py"); } @@ -5724,6 +5727,10 @@ public boolean defaultNetworkRules(final Connect conn, final String vmName, fina if (checkBeforeApply) { cmd.add("--check"); } + // The Agent is the only component that knows the network type; the script never infers it + if (BridgeVifDriver.isDirectRoutedNic(nic)) { + cmd.add("--directrouted"); + } final String result = cmd.execute(); if (result != null) { return false; @@ -5852,7 +5859,7 @@ private Answer listLvmVolumes(String localPath, int startIndex, int pageSize) { } public boolean addNetworkRules(final String vmName, final String vmId, final String guestIP, final String guestIP6, final String sig, final String seq, final String mac, final String rules, final String vif, final String brname, - final String secIps) { + final String secIps, final boolean directRouted) { if (!canBridgeFirewall) { return false; } @@ -5875,6 +5882,9 @@ public boolean addNetworkRules(final String vmName, final String vmId, final Str if (newRules != null && !newRules.isEmpty()) { cmd.add("--rules", newRules); } + if (directRouted) { + cmd.add("--directrouted"); + } final String result = cmd.execute(); if (result != null) { return false; @@ -5883,11 +5893,27 @@ public boolean addNetworkRules(final String vmName, final String vmId, final Str } public boolean configureNetworkRulesVMSecondaryIP(final Connect conn, final String vmName, final String vmMac, final String secIp, final String action) { + return configureNetworkRulesVMSecondaryIP(conn, vmName, vmMac, secIp, action, false, true); + } - if (!canBridgeFirewall) { + public boolean configureNetworkRulesVMSecondaryIP(final Connect conn, final String vmName, final String vmMac, final String secIp, final String action, + final boolean directRouted, final boolean applySecurityGroupRules) { + + // On a Direct Routed network the address only reaches the Instance once the host has a + // route and a neighbour entry for it. Security groups may be disabled there, so this is + // done regardless of canBridgeFirewall, and before any firewall rules. + if (directRouted && !configureDirectRoutedSecondaryIp(conn, vmName, vmMac, secIp, action)) { return false; } + if (!applySecurityGroupRules) { + return true; + } + + if (!canBridgeFirewall) { + return directRouted; + } + final Script cmd = new Script(securityGroupPath, timeout, LOGGER); cmd.add("network_rules_vmSecondaryIp"); cmd.add("--vmname", vmName); @@ -5902,6 +5928,39 @@ public boolean configureNetworkRulesVMSecondaryIP(final Connect conn, final Stri return true; } + /** + * Adds or removes the host route and static neighbour entry for a secondary IP of an + * Instance on a Direct Routed network, so the address is reachable without restarting it. + */ + private boolean configureDirectRoutedSecondaryIp(final Connect conn, final String vmName, final String vmMac, final String secIp, final String action) { + if (macIpPath == null) { + LOGGER.warn("Unable to find modifymacip.sh, cannot configure secondary IP {} for {}", secIp, vmName); + return false; + } + String brName = null; + for (final InterfaceDef intf : getInterfaces(conn, vmName)) { + if (vmMac.equalsIgnoreCase(intf.getMacAddress())) { + brName = intf.getBrName(); + break; + } + } + if (brName == null) { + LOGGER.warn("Unable to find the interface of {} with MAC {}", vmName, vmMac); + return false; + } + final Script cmd = new Script(macIpPath, timeout, LOGGER); + cmd.add("-o", "-A".equals(action) ? "add" : "delete"); + cmd.add("-b", brName); + cmd.add("-m", vmMac); + cmd.add(NetUtils.isValidIp6(secIp) ? "-6" : "-4", secIp); + final String result = cmd.execute(); + if (result != null) { + LOGGER.warn("Failed to configure secondary IP {} for {}: {}", secIp, vmName, result); + return false; + } + return true; + } + public boolean setupTungstenVRouter(final String oper, final String inf, final String subnet, final String route, final String vrf) { final Script cmd = new Script(setupTungstenVrouterPath, timeout, LOGGER); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtNetworkRulesVmSecondaryIpCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtNetworkRulesVmSecondaryIpCommandWrapper.java index 890558ca3651..018d3b929284 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtNetworkRulesVmSecondaryIpCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtNetworkRulesVmSecondaryIpCommandWrapper.java @@ -39,7 +39,7 @@ public Answer execute(final NetworkRulesVmSecondaryIpCommand command, final Libv final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); final Connect conn = libvirtUtilitiesHelper.getConnectionByVmName(command.getVmName()); - result = libvirtComputingResource.configureNetworkRulesVMSecondaryIP(conn, command.getVmName(), command.getVmMac(), command.getVmSecIp(), command.getAction()); + result = libvirtComputingResource.configureNetworkRulesVMSecondaryIP(conn, command.getVmName(), command.getVmMac(), command.getVmSecIp(), command.getAction(), command.isDirectRouted(), command.isApplySecurityGroupRules()); } catch (final LibvirtException e) { logger.debug("Could not configure VM secondary IP! => " + e.getLocalizedMessage()); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSecurityGroupRulesCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSecurityGroupRulesCommandWrapper.java index 33164264ae80..fc65e1ac8ca3 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSecurityGroupRulesCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSecurityGroupRulesCommandWrapper.java @@ -28,6 +28,7 @@ import com.cloud.agent.api.SecurityGroupRuleAnswer; import com.cloud.agent.api.SecurityGroupRulesCmd; import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.hypervisor.kvm.resource.BridgeVifDriver; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef; import com.cloud.resource.CommandWrapper; @@ -59,8 +60,16 @@ public Answer execute(final SecurityGroupRulesCmd command, final LibvirtComputin return new SecurityGroupRuleAnswer(command, false, e.toString()); } + // The rules are programmed for the Instance's first NIC (vif/brname above); its network + // type decides which dispatch form the script must use. + boolean directRouted = false; + final VirtualMachineTO vmTO = command.getVmTO(); + if (vmTO != null && vmTO.getNics() != null && vmTO.getNics().length > 0) { + directRouted = BridgeVifDriver.isDirectRoutedNic(vmTO.getNics()[0]); + } + final boolean result = libvirtComputingResource.addNetworkRules(command.getVmName(), Long.toString(command.getVmId()), command.getGuestIp(), command.getGuestIp6(), command.getSignature(), - Long.toString(command.getSeqNum()), command.getGuestMac(), command.stringifyRules(), vif, brname, command.getSecIpsString()); + Long.toString(command.getSeqNum()), command.getGuestMac(), command.stringifyRules(), vif, brname, command.getSecIpsString(), directRouted); if (!result) { logger.warn("Failed to program network rules for vm " + command.getVmName()); diff --git a/scripts/vm/network/security_group.py b/scripts/vm/network/security_group.py index 235e6342feea..1286fa95fd11 100755 --- a/scripts/vm/network/security_group.py +++ b/scripts/vm/network/security_group.py @@ -527,7 +527,7 @@ def ebtables_rules_vmip(vmname, vmmac, ips, action): except: logging.debug("Failed to program ebtables rules for secondary ip %s for vm %s with action %s" % (ip, vmname, action)) -def check_default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, sec_ips, is_first_nic=False): +def check_default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, sec_ips, is_first_nic=False, direct_routed=False): brfw = get_br_fw(brname) vmchain_default = '-'.join(vm_name.split('-')[:-1]) + "-def" try: @@ -536,7 +536,7 @@ def check_default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brna rules = None if not rules: logging.debug("iptables rules do not exist, programming default rules for %s %s" % (vm_name,vif)) - default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, sec_ips, is_first_nic) + default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, sec_ips, is_first_nic, direct_routed) else: vmchain_in = vm_name + "-in" try: @@ -551,9 +551,84 @@ def check_default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brna ebtables_rules_vmip(vm_name, vm_mac, ips, "-I") return True -def default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, sec_ips, is_first_nic=False): - if not add_fw_framework(brname): - return False +def enable_bridge_netfilter(): + """ Make bridged traffic traverse ip(6)tables and arptables, which every rule below relies + on. Shared by all bridge types. """ + try: + execute("modprobe br_netfilter") + except: + logging.debug("failed to load kernel module br_netfilter") + + try: + execute("sysctl -w net.bridge.bridge-nf-call-arptables=1") + execute("sysctl -w net.bridge.bridge-nf-call-iptables=1") + execute("sysctl -w net.bridge.bridge-nf-call-ip6tables=1") + except: + logging.warning("failed to turn on bridge netfilter") + + +def create_bridge_fw_chains(brfw): + """ Create this bridge's BF chains in both ip and ip6 tables if they are not there yet. """ + for ipt in ['iptables', 'ip6tables']: + for chain in [brfw, brfw + "-OUT", brfw + "-IN"]: + try: + execute("%s -L %s" % (ipt, chain)) + except: + execute("%s -N %s" % (ipt, chain)) + + +def add_notrack_ipset_rules(): + """ Addresses in the notrack ipsets are excluded from connection tracking. Host wide and + idempotent, so every bridge type sets this up the same way. """ + execute(f"ipset -! create {NOTRACK_IPV4_IPSET} hash:ip family inet") + execute(f"ipset -! create {NOTRACK_IPV6_IPSET} hash:ip family inet6") + + for ipt, ipset_name in [('iptables', NOTRACK_IPV4_IPSET), ('ip6tables', NOTRACK_IPV6_IPSET)]: + for direction in ['dst', 'src']: + try: + execute(f"{ipt} -t raw -n -L PREROUTING | grep -q 'match-set {ipset_name} {direction} NOTRACK'") + except: + execute(f"{ipt} -t raw -A PREROUTING -m set --match-set {ipset_name} {direction} -j NOTRACK") + + +def add_l3_fw_framework(brname): + """ FORWARD hooks for a Direct Routed (L3) bridge. Guest traffic on these bridges is routed + by the host, not bridged, so the classic hooks - which reach the BF chains only for + --physdev-is-bridged traffic and drop the rest - would drop everything. These hooks jump + unconditionally for traffic on the bridge, with the same default-deny backstop. """ + enable_bridge_netfilter() + + brfw = get_br_fw(brname) + create_bridge_fw_chains(brfw) + + for ipt in ['iptables', 'ip6tables']: + # Idempotent hook installation; -C fails when the rule is absent + for rule in ["-i %s -j DROP" % brname, + "-o %s -j DROP" % brname, + "-i %s -j %s" % (brname, brfw + "-IN"), + "-o %s -j %s" % (brname, brfw + "-OUT")]: + try: + execute("%s -C FORWARD %s" % (ipt, rule)) + except: + execute("%s -I FORWARD %s" % (ipt, rule)) + + add_notrack_ipset_rules() + + return True + + +def default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, sec_ips, is_first_nic=False, direct_routed=False): + """ direct_routed marks an Instance on a Direct Routed (L3) network, whose traffic the host + routes rather than bridges. The Agent decides this - it is the only component that knows the + network type - and passes --directrouted; the script never infers it. """ + l3 = direct_routed + + if l3: + if not add_l3_fw_framework(brname): + return False + else: + if not add_fw_framework(brname): + return False vmName = vm_name brfw = get_br_fw(brname) @@ -568,6 +643,22 @@ def default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, se vmipsetName = ipset_chain_name(vm_name) vmipsetName6 = vmipsetName + '-6' + # Direction matchers. Traffic from the Instance is identified by the bridge port it entered + # through; --physdev-is-bridged is deliberately absent there: bridged-then-routed packets + # (Direct Routed networks) carry no bridged egress port yet must still match, and on classic + # bridges the BF- framework hook already restricts what reaches these chains to bridged + # traffic, so dropping the flag changes nothing for them. Traffic towards the Instance cannot + # be identified by bridge port on a routed path at all - the kernel only knows the egress + # port for bridged packets - so Direct Routed networks match the destination against the + # Instance's ipsets instead. + from_vm = "-m physdev --physdev-in " + vif + if l3: + to_vm4 = "-m set --match-set " + vmipsetName + " dst" + to_vm6 = "-m set --match-set " + vmipsetName6 + " dst" + else: + to_vm4 = "-m physdev --physdev-is-bridged --physdev-out " + vif + to_vm6 = to_vm4 + if is_first_nic: delete_rules_for_vm_in_bridge_firewall_chain(vmName) destroy_ebtables_rules(vmName, vif) @@ -611,26 +702,28 @@ def default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, se add_to_ipset(vmipsetName6, ip6s, action) try: - execute("iptables -A " + brfw + "-OUT" + " -m physdev --physdev-is-bridged --physdev-out " + vif + " -j " + vmchain_default) - execute("iptables -A " + brfw + "-IN" + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -j " + vmchain_default) - #allow dhcp - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -p udp --dport 67 --sport 68 -j ACCEPT") - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-out " + vif + " -p udp --dport 68 --sport 67 -j ACCEPT") - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -p udp --sport 67 -j DROP") + execute("iptables -A " + brfw + "-OUT " + to_vm4 + " -j " + vmchain_default) + execute("iptables -A " + brfw + "-IN " + from_vm + " -j " + vmchain_default) + if not l3: + #allow dhcp + execute("iptables -A " + vmchain_default + " " + from_vm + " -p udp --dport 67 --sport 68 -j ACCEPT") + execute("iptables -A " + vmchain_default + " " + to_vm4 + " -p udp --dport 68 --sport 67 -j ACCEPT") + execute("iptables -A " + vmchain_default + " " + from_vm + " -p udp --sport 67 -j DROP") #don't let vm spoof its ip address if vm_ip: - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -m set ! --match-set " + vmipsetName + " src -j DROP") - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-out " + vif + " -m set ! --match-set " + vmipsetName + " dst -j DROP") - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -m set --match-set " + vmipsetName + " src -p udp --dport 53 -j ACCEPT") - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -m set --match-set " + vmipsetName + " src -p tcp --dport 53 -j ACCEPT") - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -m set --match-set " + vmipsetName + " src -j " + vmchain_egress) - - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-out " + vif + " -j " + vmchain) - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -m state --state ESTABLISHED,RELATED -j ACCEPT") - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-out " + vif + " -m state --state ESTABLISHED,RELATED -j ACCEPT") - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-in " + vif + " -j DROP") - execute("iptables -A " + vmchain_default + " -m physdev --physdev-is-bridged --physdev-out " + vif + " -j DROP") + execute("iptables -A " + vmchain_default + " " + from_vm + " -m set ! --match-set " + vmipsetName + " src -j DROP") + if not l3: + execute("iptables -A " + vmchain_default + " " + to_vm4 + " -m set ! --match-set " + vmipsetName + " dst -j DROP") + execute("iptables -A " + vmchain_default + " " + from_vm + " -m set --match-set " + vmipsetName + " src -p udp --dport 53 -j ACCEPT") + execute("iptables -A " + vmchain_default + " " + from_vm + " -m set --match-set " + vmipsetName + " src -p tcp --dport 53 -j ACCEPT") + execute("iptables -A " + vmchain_default + " " + from_vm + " -m set --match-set " + vmipsetName + " src -j " + vmchain_egress) + + execute("iptables -A " + vmchain_default + " " + to_vm4 + " -j " + vmchain) + execute("iptables -A " + vmchain_default + " " + from_vm + " -m state --state ESTABLISHED,RELATED -j ACCEPT") + execute("iptables -A " + vmchain_default + " " + to_vm4 + " -m state --state ESTABLISHED,RELATED -j ACCEPT") + execute("iptables -A " + vmchain_default + " " + from_vm + " -j DROP") + execute("iptables -A " + vmchain_default + " " + to_vm4 + " -j DROP") execute("iptables -A " + vmchain + " -j RETURN") except: logging.debug("Failed to program default rules for vm " + vm_name) @@ -645,57 +738,63 @@ def default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, se logging.debug("Failed to log default network rules, ignoring") try: - execute('ip6tables -A ' + brfw + '-OUT' + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -j ' + vmchain_default) - execute('ip6tables -A ' + brfw + '-IN' + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -j ' + vmchain_default) + execute('ip6tables -A ' + brfw + '-OUT ' + to_vm6 + ' -j ' + vmchain_default) + execute('ip6tables -A ' + brfw + '-IN ' + from_vm + ' -j ' + vmchain_default) - # Allow Instances to receive Router Advertisements, send out solicitations, but block any outgoing Advertisement from a Instance - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' --src fe80::/64 --dst ff02::1 -p icmpv6 --icmpv6-type router-advertisement -m hl --hl-eq 255 -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' --dst ff02::2 -p icmpv6 --icmpv6-type router-solicitation -m hl --hl-eq 255 -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p icmpv6 --icmpv6-type router-advertisement -j DROP') + # Allow Instances to receive Router Advertisements, send out solicitations, but block any outgoing Advertisement from a Instance. + # On a Direct Routed network no router advertises anything, so only the block applies. + if not l3: + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' --src fe80::/64 --dst ff02::1 -p icmpv6 --icmpv6-type router-advertisement -m hl --hl-eq 255 -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' --dst ff02::2 -p icmpv6 --icmpv6-type router-solicitation -m hl --hl-eq 255 -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p icmpv6 --icmpv6-type router-advertisement -j DROP') # Allow neighbor solicitations and advertisements - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p icmpv6 --icmpv6-type neighbor-solicitation -m hl --hl-eq 255 -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -p icmpv6 --icmpv6-type neighbor-solicitation -m hl --hl-eq 255 -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p icmpv6 --icmpv6-type neighbor-advertisement -m set --match-set ' + vmipsetName6 + ' src -m hl --hl-eq 255 -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -p icmpv6 --icmpv6-type neighbor-advertisement -m hl --hl-eq 255 -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p icmpv6 --icmpv6-type neighbor-solicitation -m hl --hl-eq 255 -j ACCEPT') + if not l3: + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' -p icmpv6 --icmpv6-type neighbor-solicitation -m hl --hl-eq 255 -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p icmpv6 --icmpv6-type neighbor-advertisement -m set --match-set ' + vmipsetName6 + ' src -m hl --hl-eq 255 -j ACCEPT') + if not l3: + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' -p icmpv6 --icmpv6-type neighbor-advertisement -m hl --hl-eq 255 -j ACCEPT') # Packets to allow as per RFC4890 - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p icmpv6 --icmpv6-type packet-too-big -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -p icmpv6 --icmpv6-type packet-too-big -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p icmpv6 --icmpv6-type packet-too-big -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' -p icmpv6 --icmpv6-type packet-too-big -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p icmpv6 --icmpv6-type destination-unreachable -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -p icmpv6 --icmpv6-type destination-unreachable -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p icmpv6 --icmpv6-type destination-unreachable -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' -p icmpv6 --icmpv6-type destination-unreachable -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p icmpv6 --icmpv6-type time-exceeded -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -p icmpv6 --icmpv6-type time-exceeded -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p icmpv6 --icmpv6-type time-exceeded -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' -p icmpv6 --icmpv6-type time-exceeded -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p icmpv6 --icmpv6-type parameter-problem -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -p icmpv6 --icmpv6-type parameter-problem -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p icmpv6 --icmpv6-type parameter-problem -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' -p icmpv6 --icmpv6-type parameter-problem -j ACCEPT') # MLDv2 discovery packets - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p icmpv6 --dst ff02::16 -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p icmpv6 --dst ff02::16 -j ACCEPT') - # Allow Instances to send out DHCPv6 client messages, but block server messages - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p udp --sport 546 --dst ff02::1:2 --src ' + str(ipv6_link_local) + ' -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -p udp --src fe80::/64 --dport 546 --dst ' + str(ipv6_link_local) + ' -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p udp --sport 547 ! --dst fe80::/64 -j DROP') + # Allow Instances to send out DHCPv6 client messages, but block server messages. Not on + # Direct Routed networks: there is no DHCP there. + if not l3: + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p udp --sport 546 --dst ff02::1:2 --src ' + str(ipv6_link_local) + ' -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' -p udp --src fe80::/64 --dport 546 --dst ' + str(ipv6_link_local) + ' -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p udp --sport 547 ! --dst fe80::/64 -j DROP') # Always allow outbound DNS over UDP and TCP - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p udp --dport 53 -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -p tcp --dport 53 -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p udp --dport 53 -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -p tcp --dport 53 -m set --match-set ' + vmipsetName6 + ' src -j ACCEPT') # Prevent source address spoofing - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -m set ! --match-set ' + vmipsetName6 + ' src -j DROP') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -m set ! --match-set ' + vmipsetName6 + ' src -j DROP') # Send proper traffic to the egress chain of the Instance - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -m set --match-set ' + vmipsetName6 + ' src -j ' + vmchain_egress) + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -m set --match-set ' + vmipsetName6 + ' src -j ' + vmchain_egress) - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -j ' + vmchain) + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' -j ' + vmchain) - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -m state --state ESTABLISHED,RELATED -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -m state --state ESTABLISHED,RELATED -j ACCEPT') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-in ' + vif + ' -j DROP') - execute('ip6tables -A ' + vmchain_default + ' -m physdev --physdev-is-bridged --physdev-out ' + vif + ' -j DROP') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -m state --state ESTABLISHED,RELATED -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' -m state --state ESTABLISHED,RELATED -j ACCEPT') + execute('ip6tables -A ' + vmchain_default + ' ' + from_vm + ' -j DROP') + execute('ip6tables -A ' + vmchain_default + ' ' + to_vm6 + ' -j DROP') # Drop all other traffic into the Instance execute('ip6tables -A ' + vmchain + ' -j RETURN') @@ -747,21 +846,19 @@ def delete_rules_for_vm_in_bridge_firewall_chain(vmName): vmchain = iptables_chain_name(vm_name) - delcmd = """iptables-save | awk '/BF(.*)physdev-is-bridged(.*)%s/ { sub(/-A/, "-D", $1) ; print }'""" % vmchain - delcmds = [_f for _f in execute(delcmd).split('\n') if _f] - for cmd in delcmds: - try: - execute("iptables " + cmd) - except: - logging.exception("Ignoring failure to delete rules for vm " + vmName) - - delcmd = """ip6tables-save | awk '/BF(.*)physdev-is-bridged(.*)%s/ { sub(/-A/, "-D", $1) ; print }'""" % vmchain - delcmds = [_f for _f in execute(delcmd).split('\n') if _f] - for cmd in delcmds: - try: - execute('ip6tables ' + cmd) - except: - logging.exception("Ignoring failure to delete rules for vm " + vmName) + # Dispatch rules for this Instance in the BF- chains take several forms: physdev-in (from + # the Instance; historically with --physdev-is-bridged, which rules created by older + # versions still carry), physdev-out with --physdev-is-bridged (towards the Instance on a + # classic bridge) and ipset destination matches (towards the Instance on a Direct Routed + # bridge). One pattern on the chain names covers all of them. + for ipt in ['iptables', 'ip6tables']: + delcmd = """%s-save | awk '/BF(.*)%s/ { sub(/-A/, "-D", $1) ; print }'""" % (ipt, vmchain) + delcmds = [_f for _f in execute(delcmd).split('\n') if _f] + for cmd in delcmds: + try: + execute(ipt + ' ' + cmd) + except: + logging.exception("Ignoring failure to delete rules for vm " + vmName) def rewrite_rule_log_for_vm(vm_name, new_domid): @@ -844,13 +941,31 @@ def network_rules_for_rebooted_vm(vmName): vmchain = iptables_chain_name(vm_name) vmchain_default = '-'.join(vmchain.split('-')[:-1]) + "-def" + # This path runs without the Agent, so the --directrouted flag is not available here. The + # Instance's existing OUT dispatch rule tells us which form to restore: a Direct Routed + # Instance is dispatched by destination ipset, a bridged one by physdev-out. Captured before + # the rules are deleted above? No - delete_rules_for_vm_in_bridge_firewall_chain has already + # run, so read it from the per-VM chain, which is untouched by that function. + vmipsetName = ipset_chain_name(vm_name) + try: + l3 = bool(execute("""iptables-save | grep -w %s | grep -q -- '--match-set %s dst' && echo yes""" % (vmchain_default, vmipsetName)).strip()) + except: + l3 = False + vifs = get_vifs(vmName) logging.debug(vifs, brName) for v in vifs: - execute("iptables -A " + get_br_fw(brName) + "-IN " + " -m physdev --physdev-is-bridged --physdev-in " + v + " -j " + vmchain_default) - execute("iptables -A " + get_br_fw(brName) + "-OUT " + " -m physdev --physdev-is-bridged --physdev-out " + v + " -j " + vmchain_default) - execute("ip6tables -A " + get_br_fw(brName) + "-IN " + " -m physdev --physdev-is-bridged --physdev-in " + v + " -j " + vmchain_default) - execute("ip6tables -A " + get_br_fw(brName) + "-OUT " + " -m physdev --physdev-is-bridged --physdev-out " + v + " -j " + vmchain_default) + from_vm = "-m physdev --physdev-in " + v + if l3: + to_vm4 = "-m set --match-set " + vmipsetName + " dst" + to_vm6 = "-m set --match-set " + vmipsetName + "-6 dst" + else: + to_vm4 = "-m physdev --physdev-is-bridged --physdev-out " + v + to_vm6 = to_vm4 + execute("iptables -A " + get_br_fw(brName) + "-IN " + from_vm + " -j " + vmchain_default) + execute("iptables -A " + get_br_fw(brName) + "-OUT " + to_vm4 + " -j " + vmchain_default) + execute("ip6tables -A " + get_br_fw(brName) + "-IN " + from_vm + " -j " + vmchain_default) + execute("ip6tables -A " + get_br_fw(brName) + "-OUT " + to_vm6 + " -j " + vmchain_default) #change antispoof rule in vmchain try: @@ -1105,7 +1220,7 @@ def parse_network_rules(rules): return ret -def add_network_rules(vm_name, vm_id, vm_ip, vm_ip6, signature, seqno, vmMac, rules, vif, brname, sec_ips): +def add_network_rules(vm_name, vm_id, vm_ip, vm_ip6, signature, seqno, vmMac, rules, vif, brname, sec_ips, direct_routed=False): try: vmName = vm_name domId = get_vm_id(vmName) @@ -1128,7 +1243,7 @@ def add_network_rules(vm_name, vm_id, vm_ip, vm_ip6, signature, seqno, vmMac, ru execute('ip6tables -F ' + chain) except: logging.debug("Error flushing iptables rules for " + vm_name + ". Presuming firewall rules deleted, re-initializing." ) - default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vmMac, vif, brname, sec_ips, True) + default_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vmMac, vif, brname, sec_ips, True, direct_routed) egressrule_v4 = 0 egressrule_v6 = 0 @@ -1312,78 +1427,16 @@ def get_br_fw(brname): def add_fw_framework(brname): - try: - execute("modprobe br_netfilter") - except: - logging.debug("failed to load kernel module br_netfilter") - - try: - execute("sysctl -w net.bridge.bridge-nf-call-arptables=1") - execute("sysctl -w net.bridge.bridge-nf-call-iptables=1") - execute("sysctl -w net.bridge.bridge-nf-call-ip6tables=1") - except: - logging.warning("failed to turn on bridge netfilter") + enable_bridge_netfilter() brfw = get_br_fw(brname) - try: - execute("iptables -L " + brfw) - except: - execute("iptables -N " + brfw) - - brfwout = brfw + "-OUT" - try: - execute("iptables -L " + brfwout) - except: - execute("iptables -N " + brfwout) + create_bridge_fw_chains(brfw) brfwin = brfw + "-IN" - try: - execute("iptables -L " + brfwin) - except: - execute("iptables -N " + brfwin) - - try: - execute('ip6tables -L ' + brfw) - except: - execute('ip6tables -N ' + brfw) - brfwout = brfw + "-OUT" - try: - execute('ip6tables -L ' + brfwout) - except: - execute('ip6tables -N ' + brfwout) - - brfwin = brfw + "-IN" - try: - execute('ip6tables -L ' + brfwin) - except: - execute('ip6tables -N ' + brfwin) - physdev = get_bridge_physdev(brname) - # Add notrack ipset. The IP addresses from it will be excluded from connection tracking - execute(f"ipset -! create {NOTRACK_IPV4_IPSET} hash:ip family inet") - execute(f"ipset -! create {NOTRACK_IPV6_IPSET} hash:ip family inet6") - - # Create IPv4 rules that disable connection tracking - try: - execute(f"iptables -t raw -n -L PREROUTING | grep -q 'match-set {NOTRACK_IPV4_IPSET} dst NOTRACK'") - except: - execute(f"iptables -t raw -A PREROUTING -m set --match-set {NOTRACK_IPV4_IPSET} dst -j NOTRACK") - try: - execute(f"iptables -t raw -n -L PREROUTING | grep -q 'match-set {NOTRACK_IPV4_IPSET} src NOTRACK'") - except: - execute(f"iptables -t raw -A PREROUTING -m set --match-set {NOTRACK_IPV4_IPSET} src -j NOTRACK") - - # Create IPv6 rules that disable connection tracking - try: - execute(f"ip6tables -t raw -n -L PREROUTING | grep -q 'match-set {NOTRACK_IPV6_IPSET} dst NOTRACK'") - except: - execute(f"ip6tables -t raw -A PREROUTING -m set --match-set {NOTRACK_IPV6_IPSET} dst -j NOTRACK") - try: - execute(f"ip6tables -t raw -n -L PREROUTING | grep -q 'match-set {NOTRACK_IPV6_IPSET} src NOTRACK'") - except: - execute(f"ip6tables -t raw -A PREROUTING -m set --match-set {NOTRACK_IPV6_IPSET} src -j NOTRACK") + add_notrack_ipset_rules() try: refs = int(execute("""iptables -n -L %s | awk '/%s(.*)references/ {gsub(/\(/, "") ;print $3}'""" % (brfw,brfw)).strip()) @@ -1418,7 +1471,7 @@ def add_fw_framework(brname): return False return False -def verify_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, sec_ips): +def verify_network_rules(vm_name, vm_id, vm_ip, vm_ip6, vm_mac, vif, brname, sec_ips, direct_routed=False): if vm_name is None or vm_ip is None or vm_mac is None: print("vm_name, vm_ip and vm_mac must be specifed") sys.exit(1) @@ -1665,6 +1718,7 @@ def verify_expected_rules_in_order(expected_rules, rules): parser.add_argument("--privnic", dest="privnic") parser.add_argument("--isFirstNic", action="store_true", dest="isFirstNic") parser.add_argument("--check", action="store_true", dest="check") + parser.add_argument("--directrouted", action="store_true", dest="directRouted") args = parser.parse_args() cmd = args.command logging.debug("Executing command: %s", cmd) @@ -1679,9 +1733,9 @@ def verify_expected_rules_in_order(expected_rules, rules): if cmd == "can_bridge_firewall": can_bridge_firewall(args.privnic) elif cmd == "default_network_rules" and args.check: - check_default_network_rules(args.vmName, args.vmID, args.vmIP, args.vmIP6, args.vmMAC, args.vif, args.brname, args.nicSecIps, args.isFirstNic) + check_default_network_rules(args.vmName, args.vmID, args.vmIP, args.vmIP6, args.vmMAC, args.vif, args.brname, args.nicSecIps, args.isFirstNic, args.directRouted) elif cmd == "default_network_rules" and not args.check: - default_network_rules(args.vmName, args.vmID, args.vmIP, args.vmIP6, args.vmMAC, args.vif, args.brname, args.nicSecIps, args.isFirstNic) + default_network_rules(args.vmName, args.vmID, args.vmIP, args.vmIP6, args.vmMAC, args.vif, args.brname, args.nicSecIps, args.isFirstNic, args.directRouted) elif cmd == "destroy_network_rules_for_vm": if not args.vmIP: destroy_network_rules_for_vm(args.vmName, args.vif) @@ -1692,7 +1746,7 @@ def verify_expected_rules_in_order(expected_rules, rules): elif cmd == "get_rule_logs_for_vms": get_rule_logs_for_vms() elif cmd == "add_network_rules": - add_network_rules(args.vmName, args.vmID, args.vmIP, args.vmIP6, args.sig, args.seq, args.vmMAC, args.rules, args.vif, args.brname, args.nicSecIps) + add_network_rules(args.vmName, args.vmID, args.vmIP, args.vmIP6, args.sig, args.seq, args.vmMAC, args.rules, args.vif, args.brname, args.nicSecIps, args.directRouted) elif cmd == "network_rules_vmSecondaryIp": network_rules_vmSecondaryIp(args.vmName, args.vmMAC, args.nicSecIps, args.action) elif cmd == "cleanup_rules": @@ -1700,7 +1754,7 @@ def verify_expected_rules_in_order(expected_rules, rules): elif cmd == "post_default_network_rules": post_default_network_rules(args.vmName, args.vmID, args.vmIP, args.vmMAC, args.vif, args.brname, args.dhcpSvr, args.hostIp, args.hostMacAddr) elif cmd == "verify_network_rules": - verify_network_rules(args.vmName, args.vmID, args.vmIP, args.vmIP6, args.vmMAC, args.vif, args.brname, args.nicSecIps) + verify_network_rules(args.vmName, args.vmID, args.vmIP, args.vmIP6, args.vmMAC, args.vif, args.brname, args.nicSecIps, args.directRouted) else: logging.debug("Unknown command: " + cmd) sys.exit(1) diff --git a/scripts/vm/network/tests/golden_add_fw_framework.txt b/scripts/vm/network/tests/golden_add_fw_framework.txt new file mode 100644 index 000000000000..6ecfe18702ea --- /dev/null +++ b/scripts/vm/network/tests/golden_add_fw_framework.txt @@ -0,0 +1,44 @@ +modprobe br_netfilter +sysctl -w net.bridge.bridge-nf-call-arptables=1 +sysctl -w net.bridge.bridge-nf-call-iptables=1 +sysctl -w net.bridge.bridge-nf-call-ip6tables=1 +iptables -L BF-cloudbr0 +iptables -N BF-cloudbr0 +iptables -L BF-cloudbr0-OUT +iptables -N BF-cloudbr0-OUT +iptables -L BF-cloudbr0-IN +iptables -N BF-cloudbr0-IN +ip6tables -L BF-cloudbr0 +ip6tables -N BF-cloudbr0 +ip6tables -L BF-cloudbr0-OUT +ip6tables -N BF-cloudbr0-OUT +ip6tables -L BF-cloudbr0-IN +ip6tables -N BF-cloudbr0-IN +ipset -! create cs_notrack hash:ip family inet +ipset -! create cs_notrack6 hash:ip family inet6 +iptables -t raw -n -L PREROUTING | grep -q 'match-set cs_notrack dst NOTRACK' +iptables -t raw -A PREROUTING -m set --match-set cs_notrack dst -j NOTRACK +iptables -t raw -n -L PREROUTING | grep -q 'match-set cs_notrack src NOTRACK' +iptables -t raw -A PREROUTING -m set --match-set cs_notrack src -j NOTRACK +ip6tables -t raw -n -L PREROUTING | grep -q 'match-set cs_notrack6 dst NOTRACK' +ip6tables -t raw -A PREROUTING -m set --match-set cs_notrack6 dst -j NOTRACK +ip6tables -t raw -n -L PREROUTING | grep -q 'match-set cs_notrack6 src NOTRACK' +ip6tables -t raw -A PREROUTING -m set --match-set cs_notrack6 src -j NOTRACK +iptables -n -L BF-cloudbr0 | awk '/BF-cloudbr0(.*)references/ {gsub(/\(/, "") ;print $3}' +iptables -n -L BF-cloudbr0-IN | awk '/BF-cloudbr0-IN(.*)references/ {gsub(/\(/, "") ;print $3}' +iptables -n -L BF-cloudbr0-OUT | awk '/BF-cloudbr0-OUT(.*)references/ {gsub(/\(/, "") ;print $3}' +ip6tables -n -L BF-cloudbr0 | awk '/BF-cloudbr0(.*)references/ {gsub(/\(/, "") ;print $3}' +iptables -I FORWARD -i cloudbr0 -j DROP +iptables -I FORWARD -o cloudbr0 -j DROP +iptables -I FORWARD -i cloudbr0 -m physdev --physdev-is-bridged -j BF-cloudbr0 +iptables -I FORWARD -o cloudbr0 -m physdev --physdev-is-bridged -j BF-cloudbr0 +iptables -A BF-cloudbr0 -m physdev --physdev-is-bridged --physdev-is-in -j BF-cloudbr0-IN +iptables -A BF-cloudbr0 -m physdev --physdev-is-bridged --physdev-is-out -j BF-cloudbr0-OUT +iptables -A BF-cloudbr0 -m physdev --physdev-is-bridged --physdev-out eth0 -j ACCEPT +ip6tables -I FORWARD -i cloudbr0 -j DROP +ip6tables -I FORWARD -o cloudbr0 -j DROP +ip6tables -I FORWARD -i cloudbr0 -m physdev --physdev-is-bridged -j BF-cloudbr0 +ip6tables -I FORWARD -o cloudbr0 -m physdev --physdev-is-bridged -j BF-cloudbr0 +ip6tables -A BF-cloudbr0 -m physdev --physdev-is-bridged --physdev-is-in -j BF-cloudbr0-IN +ip6tables -A BF-cloudbr0 -m physdev --physdev-is-bridged --physdev-is-out -j BF-cloudbr0-OUT +ip6tables -A BF-cloudbr0 -m physdev --physdev-is-bridged --physdev-out eth0 -j ACCEPT diff --git a/scripts/vm/network/tests/golden_default_network_rules.txt b/scripts/vm/network/tests/golden_default_network_rules.txt new file mode 100644 index 000000000000..891b53aaaac5 --- /dev/null +++ b/scripts/vm/network/tests/golden_default_network_rules.txt @@ -0,0 +1,92 @@ +iptables-save |grep physdev-is-bridged |grep FORWARD |grep BF |grep '\-o' | grep -w cloudbr0|awk '{print $9}' | head -1 +iptables -N i-2-7-VM +ip6tables -N i-2-7-VM +iptables -N i-2-7-VM-eg +ip6tables -N i-2-7-VM-eg +iptables -N i-2-7-def +ip6tables -N i-2-7-def +ipset -F i-2-7-VM +ipset -X i-2-7-VM +ipset -N i-2-7-VM iphash family inet +ipset -F i-2-7-VM-6 +ipset -X i-2-7-VM-6 +ipset -N i-2-7-VM-6 hash:net family inet6 +ipset -! -A i-2-7-VM 10.1.1.55 +ipset -! -A i-2-7-VM 10.1.1.55 +ipset -! -A i-2-7-VM-6 fd00::55 +ipset -! -A i-2-7-VM-6 fe80::4cff:fe5f:1 +iptables -A BF-cloudbr0-OUT -m physdev --physdev-is-bridged --physdev-out vnet3 -j i-2-7-def +iptables -A BF-cloudbr0-IN -m physdev --physdev-in vnet3 -j i-2-7-def +iptables -A i-2-7-def -m physdev --physdev-in vnet3 -p udp --dport 67 --sport 68 -j ACCEPT +iptables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -p udp --dport 68 --sport 67 -j ACCEPT +iptables -A i-2-7-def -m physdev --physdev-in vnet3 -p udp --sport 67 -j DROP +iptables -A i-2-7-def -m physdev --physdev-in vnet3 -m set ! --match-set i-2-7-VM src -j DROP +iptables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -m set ! --match-set i-2-7-VM dst -j DROP +iptables -A i-2-7-def -m physdev --physdev-in vnet3 -m set --match-set i-2-7-VM src -p udp --dport 53 -j ACCEPT +iptables -A i-2-7-def -m physdev --physdev-in vnet3 -m set --match-set i-2-7-VM src -p tcp --dport 53 -j ACCEPT +iptables -A i-2-7-def -m physdev --physdev-in vnet3 -m set --match-set i-2-7-VM src -j i-2-7-VM-eg +iptables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -j i-2-7-VM +iptables -A i-2-7-def -m physdev --physdev-in vnet3 -m state --state ESTABLISHED,RELATED -j ACCEPT +iptables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -m state --state ESTABLISHED,RELATED -j ACCEPT +iptables -A i-2-7-def -m physdev --physdev-in vnet3 -j DROP +iptables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -j DROP +iptables -A i-2-7-VM -j RETURN +ebtables -t nat -N i-2-7-VM-in +ebtables -t nat -N i-2-7-VM-out +ebtables -t nat -N i-2-7-VM-in-ips +ebtables -t nat -N i-2-7-VM-out-ips +ebtables -t nat -N i-2-7-VM-in-src +ebtables -t nat -N i-2-7-VM-out-dst +ebtables -t nat -A PREROUTING -i vnet3 -j i-2-7-VM-in +ebtables -t nat -A POSTROUTING -o vnet3 -j i-2-7-VM-out +ebtables -t nat -A i-2-7-VM-in-ips -j DROP +ebtables -t nat -A i-2-7-VM-out-ips -j DROP +ebtables -t nat -A i-2-7-VM-in-src -j DROP +ebtables -t nat -A i-2-7-VM-out-dst -p ARP --arp-op Reply -j DROP +ebtables -t nat -A i-2-7-VM-in -j i-2-7-VM-in-src +ebtables -t nat -I i-2-7-VM-in-src -s 02:00:4c:5f:00:01 -j RETURN +ebtables -t nat -A i-2-7-VM-in -p ARP -j i-2-7-VM-in-ips +ebtables -t nat -I i-2-7-VM-in-ips -p ARP -s 02:00:4c:5f:00:01 --arp-mac-src 02:00:4c:5f:00:01 --arp-ip-src 10.1.1.55 -j RETURN +ebtables -t nat -A i-2-7-VM-in -p ARP --arp-op Request -j ACCEPT +ebtables -t nat -A i-2-7-VM-in -p ARP --arp-op Reply -j ACCEPT +ebtables -t nat -A i-2-7-VM-in -p ARP -j DROP +ebtables -t nat -A i-2-7-VM-out -p ARP --arp-op Reply -j i-2-7-VM-out-dst +ebtables -t nat -I i-2-7-VM-out-dst -p ARP --arp-op Reply --arp-mac-dst 02:00:4c:5f:00:01 -j RETURN +ebtables -t nat -A i-2-7-VM-out -p ARP -j i-2-7-VM-out-ips +ebtables -t nat -I i-2-7-VM-out-ips -p ARP --arp-ip-dst 10.1.1.55 -j RETURN +ebtables -t nat -A i-2-7-VM-out -p ARP --arp-op Request -j ACCEPT +ebtables -t nat -A i-2-7-VM-out -p ARP --arp-op Reply -j ACCEPT +ebtables -t nat -A i-2-7-VM-out -p ARP -j DROP +ebtables -t nat -I i-2-7-VM-in-ips -p ARP -s 02:00:4c:5f:00:01 --arp-mac-src 02:00:4c:5f:00:01 --arp-ip-src 10.1.1.55 -j RETURN +ebtables -t nat -I i-2-7-VM-out-ips -p ARP --arp-ip-dst 10.1.1.55 -j RETURN +ip6tables -A BF-cloudbr0-OUT -m physdev --physdev-is-bridged --physdev-out vnet3 -j i-2-7-def +ip6tables -A BF-cloudbr0-IN -m physdev --physdev-in vnet3 -j i-2-7-def +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 --src fe80::/64 --dst ff02::1 -p icmpv6 --icmpv6-type router-advertisement -m hl --hl-eq 255 -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 --dst ff02::2 -p icmpv6 --icmpv6-type router-solicitation -m hl --hl-eq 255 -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --icmpv6-type router-advertisement -j DROP +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --icmpv6-type neighbor-solicitation -m hl --hl-eq 255 -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -p icmpv6 --icmpv6-type neighbor-solicitation -m hl --hl-eq 255 -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --icmpv6-type neighbor-advertisement -m set --match-set i-2-7-VM-6 src -m hl --hl-eq 255 -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -p icmpv6 --icmpv6-type neighbor-advertisement -m hl --hl-eq 255 -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --icmpv6-type packet-too-big -m set --match-set i-2-7-VM-6 src -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -p icmpv6 --icmpv6-type packet-too-big -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --icmpv6-type destination-unreachable -m set --match-set i-2-7-VM-6 src -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -p icmpv6 --icmpv6-type destination-unreachable -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --icmpv6-type time-exceeded -m set --match-set i-2-7-VM-6 src -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -p icmpv6 --icmpv6-type time-exceeded -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --icmpv6-type parameter-problem -m set --match-set i-2-7-VM-6 src -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -p icmpv6 --icmpv6-type parameter-problem -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --dst ff02::16 -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p udp --sport 546 --dst ff02::1:2 --src fe80::4cff:fe5f:1 -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -p udp --src fe80::/64 --dport 546 --dst fe80::4cff:fe5f:1 -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p udp --sport 547 ! --dst fe80::/64 -j DROP +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p udp --dport 53 -m set --match-set i-2-7-VM-6 src -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p tcp --dport 53 -m set --match-set i-2-7-VM-6 src -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -m set ! --match-set i-2-7-VM-6 src -j DROP +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -m set --match-set i-2-7-VM-6 src -j i-2-7-VM-eg +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -j i-2-7-VM +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -m state --state ESTABLISHED,RELATED -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -m state --state ESTABLISHED,RELATED -j ACCEPT +ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -j DROP +ip6tables -A i-2-7-def -m physdev --physdev-is-bridged --physdev-out vnet3 -j DROP +ip6tables -A i-2-7-VM -j RETURN diff --git a/scripts/vm/network/tests/test_security_group.py b/scripts/vm/network/tests/test_security_group.py new file mode 100755 index 000000000000..84db7e58a9e7 --- /dev/null +++ b/scripts/vm/network/tests/test_security_group.py @@ -0,0 +1,245 @@ +#!/usr/bin/python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" Unit tests for security_group.py rule generation. + +The golden test pins the exact command stream the classic (bridged) path produces. It was +regenerated once, deliberately, when --physdev-is-bridged was removed from the --physdev-in +rules: the flag restricted them to bridged traffic, which is all that can reach these chains on +a classic bridge anyway (the BF- framework hook gates on it), while bridged-then-routed traffic +on Direct Routed networks must match them too. --physdev-out rules keep the flag: the kernel +cannot know the bridged egress port for routed packets, so removing it there changes nothing. +Regenerate only when a change to the classic rules is intended: + + python3 tests/test_security_group.py --regenerate +""" + +import importlib.util +import os +import sys +import unittest +from unittest import mock + +HERE = os.path.dirname(os.path.abspath(__file__)) +SCRIPT = os.path.join(HERE, os.pardir, 'security_group.py') +GOLDEN = os.path.join(HERE, 'golden_default_network_rules.txt') +GOLDEN_FRAMEWORK = os.path.join(HERE, 'golden_add_fw_framework.txt') + +VM_ARGS = dict(vm_name="i-2-7-VM", vm_id="7", vm_ip="10.1.1.55", vm_ip6="fd00::55", + vm_mac="02:00:4c:5f:00:01", vif="vnet3", sec_ips="0:") + + +def load_script(): + sys.modules.setdefault('libvirt', mock.MagicMock()) + spec = importlib.util.spec_from_file_location("security_group", SCRIPT) + sg = importlib.util.module_from_spec(spec) + spec.loader.exec_module(sg) + return sg + + +def capture_framework(sg, fn, brname): + """ Run a framework function with every probe failing (so create/insert branches are taken) + and chain references reported as zero (so hooks are installed). """ + captured = [] + + def fake_execute(cmd): + captured.append(cmd) + if ' -L ' in cmd and 'awk' not in cmd: + raise Exception("absent") + if 'grep -q' in cmd: + raise Exception("absent") + if ' -C ' in cmd: + raise Exception("absent") + if 'awk' in cmd and 'references' in cmd: + return "0" + return "" + + sg.execute = fake_execute + sg.get_br_fw = lambda b: "BF-" + b + sg.get_bridge_physdev = lambda b: "eth0" + fn(brname) + return captured + + +def capture_default_network_rules(sg, brname, direct_routed): + """ Run default_network_rules with all side effects stubbed, returning the commands the + script would have executed. """ + captured = [] + + def fake_execute(cmd): + captured.append(cmd) + return "" + + sg.execute = fake_execute + sg.add_fw_framework = lambda brname: True + sg.add_l3_fw_framework = lambda brname: True + sg.get_vm_id = lambda name: "7" + sg.write_rule_log_for_vm = lambda *a, **k: True + sg.write_secip_log_for_vm = lambda *a, **k: True + sg.delete_rules_for_vm_in_bridge_firewall_chain = lambda name: None + sg.destroy_ebtables_rules = lambda name, vif: None + + ok = sg.default_network_rules(VM_ARGS['vm_name'], VM_ARGS['vm_id'], VM_ARGS['vm_ip'], VM_ARGS['vm_ip6'], + VM_ARGS['vm_mac'], VM_ARGS['vif'], brname, VM_ARGS['sec_ips'], + is_first_nic=True, direct_routed=direct_routed) + return ok, captured + + +class TestClassicRulesUnchanged(unittest.TestCase): + """ The classic path must stay byte-identical: it serves Basic zones and every Shared + network in every existing deployment. """ + + def test_default_network_rules_matches_golden(self): + sg = load_script() + ok, captured = capture_default_network_rules(sg, "cloudbr0", direct_routed=False) + self.assertTrue(ok) + with open(GOLDEN) as f: + golden = f.read().splitlines() + self.assertEqual(golden, captured) + + def test_classic_physdev_direction_flags(self): + """ from-Instance rules identify the bridge port without --physdev-is-bridged (so the + same rules work for bridged-then-routed traffic); towards-Instance rules keep it, since + the kernel only knows the bridged egress port for bridged packets. """ + sg = load_script() + ok, captured = capture_default_network_rules(sg, "cloudbr0", direct_routed=False) + self.assertTrue(ok) + for cmd in captured: + if '--physdev-in' in cmd: + self.assertNotIn('--physdev-is-bridged', cmd, cmd) + if '--physdev-out' in cmd: + self.assertIn('--physdev-is-bridged', cmd, cmd) + + +class TestDirectRoutedRules(unittest.TestCase): + + def setUp(self): + self.sg = load_script() + ok, self.captured = capture_default_network_rules(self.sg, "brdr-42", direct_routed=True) + self.assertTrue(ok) + self.iptables = [c for c in self.captured if c.startswith('iptables ') or c.startswith('ip6tables ')] + + def test_no_physdev_is_bridged_anywhere(self): + # The defining property of the routed path: physdev-is-bridged matches only bridged + # traffic, which routed traffic is not. + for cmd in self.captured: + self.assertNotIn('--physdev-is-bridged', cmd, cmd) + + def test_no_dhcp_rules(self): + for cmd in self.iptables: + for marker in ['dport 67', 'sport 67', 'dport 68', 'sport 546', 'sport 547', 'dport 546']: + self.assertNotIn(marker, cmd, cmd) + + def test_from_instance_dispatch_uses_bridge_port(self): + self.assertIn("iptables -A BF-brdr-42-IN -m physdev --physdev-in vnet3 -j i-2-7-def", self.captured) + self.assertIn("ip6tables -A BF-brdr-42-IN -m physdev --physdev-in vnet3 -j i-2-7-def", self.captured) + + def test_towards_instance_dispatch_uses_ipset_destination(self): + self.assertIn("iptables -A BF-brdr-42-OUT -m set --match-set i-2-7-VM dst -j i-2-7-def", self.captured) + self.assertIn("ip6tables -A BF-brdr-42-OUT -m set --match-set i-2-7-VM-6 dst -j i-2-7-def", self.captured) + + def test_source_spoofing_dropped_both_families(self): + self.assertIn("iptables -A i-2-7-def -m physdev --physdev-in vnet3 -m set ! --match-set i-2-7-VM src -j DROP", self.captured) + self.assertIn("ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -m set ! --match-set i-2-7-VM-6 src -j DROP", self.captured) + + def test_router_advertisements_from_instance_dropped(self): + self.assertIn("ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --icmpv6-type router-advertisement -j DROP", self.captured) + + def test_neighbor_discovery_between_instances_allowed(self): + self.assertIn("ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --icmpv6-type neighbor-solicitation -m hl --hl-eq 255 -j ACCEPT", self.captured) + self.assertIn("ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -p icmpv6 --icmpv6-type neighbor-advertisement -m set --match-set i-2-7-VM-6 src -m hl --hl-eq 255 -j ACCEPT", self.captured) + + def test_default_deny_in_both_directions_and_families(self): + self.assertIn("iptables -A i-2-7-def -m physdev --physdev-in vnet3 -j DROP", self.captured) + self.assertIn("iptables -A i-2-7-def -m set --match-set i-2-7-VM dst -j DROP", self.captured) + self.assertIn("ip6tables -A i-2-7-def -m physdev --physdev-in vnet3 -j DROP", self.captured) + self.assertIn("ip6tables -A i-2-7-def -m set --match-set i-2-7-VM-6 dst -j DROP", self.captured) + + def test_user_rule_chains_are_wired(self): + self.assertIn("iptables -A i-2-7-def -m physdev --physdev-in vnet3 -m set --match-set i-2-7-VM src -j i-2-7-VM-eg", self.captured) + self.assertIn("iptables -A i-2-7-def -m set --match-set i-2-7-VM dst -j i-2-7-VM", self.captured) + + def test_gateway_arp_protection_unchanged(self): + # The ebtables rules are shared with the classic path: they pin the ARP *source* to the + # Instance's own address and never filter the ARP target, so resolving the gateway works. + ebtables = [c for c in self.captured if c.startswith('ebtables')] + self.assertTrue(any('--arp-ip-src 10.1.1.55' in c for c in ebtables)) + self.assertFalse(any('arp-ip-dst 169.254' in c for c in ebtables)) + + +class TestSharedFrameworkHelpers(unittest.TestCase): + """ enable_bridge_netfilter, create_bridge_fw_chains and add_notrack_ipset_rules are shared + by the classic and Direct Routed frameworks. The golden proves extracting them left the + classic framework byte-identical. """ + + def test_add_fw_framework_matches_golden(self): + sg = load_script() + captured = capture_framework(sg, sg.add_fw_framework, "cloudbr0") + with open(GOLDEN_FRAMEWORK) as f: + golden = f.read().splitlines() + self.assertEqual(golden, captured) + + def test_both_frameworks_share_the_helpers(self): + sg = load_script() + classic = capture_framework(sg, sg.add_fw_framework, "cloudbr0") + l3 = capture_framework(sg, sg.add_l3_fw_framework, "brdr-42") + for stream in (classic, l3): + self.assertIn("modprobe br_netfilter", stream) + self.assertTrue(any('ipset -! create' in c for c in stream)) + self.assertTrue(any('-j NOTRACK' in c for c in stream)) + # the L3 hooks jump unconditionally; the classic ones gate on physdev-is-bridged + self.assertTrue(any('-I FORWARD -i brdr-42 -j BF-brdr-42-IN' in c for c in l3)) + for cmd in l3: + if 'FORWARD' in cmd: + self.assertNotIn('physdev', cmd, cmd) + self.assertTrue(any('physdev-is-bridged' in c and 'FORWARD' in c for c in classic)) + + +class TestDirectRoutedFlagPlumbing(unittest.TestCase): + """ The Agent decides the network type and passes --directrouted; the script never infers + it from the bridge name, the gateway, or anything else. """ + + def test_flag_reaches_default_network_rules(self): + sg = load_script() + self.assertFalse(hasattr(sg, 'is_direct_routed_bridge'), + "the script must not classify bridges itself") + _, classic = capture_default_network_rules(sg, "brdr-42", direct_routed=False) + # Same bridge name, flag off: the classic rules must be produced + self.assertTrue(any('--physdev-is-bridged --physdev-out' in c for c in classic)) + + def test_cli_exposes_directrouted(self): + with open(SCRIPT) as f: + source = f.read() + self.assertIn('"--directrouted"', source) + self.assertIn('dest="directRouted"', source) + + +def regenerate_golden(): + sg = load_script() + ok, captured = capture_default_network_rules(sg, "cloudbr0", direct_routed=False) + assert ok + with open(GOLDEN, 'w') as f: + f.write("\n".join(captured) + "\n") + print("wrote %d commands to %s" % (len(captured), GOLDEN)) + + +if __name__ == '__main__': + if '--regenerate' in sys.argv: + regenerate_golden() + else: + unittest.main() diff --git a/scripts/vm/network/vnet/modifybrdr.sh b/scripts/vm/network/vnet/modifybrdr.sh new file mode 100755 index 000000000000..98284d8d5416 --- /dev/null +++ b/scripts/vm/network/vnet/modifybrdr.sh @@ -0,0 +1,198 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# modifybrdr.sh -- Manage per-network bridges for Direct Routed Networks +# +# One bridge is created per network, named brdr- +# (Bridge-DirectRouted), using the network's numeric database id. The bridge +# carries the shared, host-independent gateway addresses and is identical on +# every hypervisor, so an Instance needs no reconfiguration when it migrates. +# +# The bridge never has a physical uplink: all Instance traffic is routed by +# the hypervisor, not bridged. Separate bridges are what keep networks +# isolated from one another at layer 2, and they give the operator a named +# interface per network to hang local routing policy off. +# +# The gateway addresses are always passed in by the CloudStack agent, taken +# from the Instance's NIC; this script carries no defaults of its own, so the +# addresses are defined in exactly one place (NetUtils on the management +# server). +# +# This script is the only place that knows how these bridges are named. 'add' +# prints the bridge name on stdout for the agent to use, and 'delete' takes a +# bridge name and answers on stdout: +# notmine - not a bridge this script manages; the agent falls back to its +# regular unplug handling +# kept - ours, but other Instances still use it +# deleted - ours, removed +# +# Usage: +# add: modifybrdr.sh -o add -n -4 and/or -6 +# delete: modifybrdr.sh -o delete -b + +usage() { + echo "Usage: $0 -o add -n [-4 ] [-6 ] | -o delete -b " +} + +OP= +NETWORK_ID= +BRNAME= +IPV4_GATEWAY= +IPV6_GATEWAY= + +while getopts 'o:n:b:4:6:' OPTION; do + case $OPTION in + o) oflag=1 + OP="$OPTARG" + ;; + n) NETWORK_ID="$OPTARG" + ;; + b) BRNAME="$OPTARG" + ;; + 4) IPV4_GATEWAY="$OPTARG" + ;; + 6) IPV6_GATEWAY="$OPTARG" + ;; + ?) usage + exit 2 + ;; + esac +done + +if [[ "$oflag" != "1" ]]; then + usage + exit 2 +fi + +if [[ "$OP" == "add" ]]; then + if [[ ! "$NETWORK_ID" =~ ^[0-9]+$ ]]; then + echo "Network id must be numeric: ${NETWORK_ID}" + exit 2 + fi + + if [[ -z "$IPV4_GATEWAY" && -z "$IPV6_GATEWAY" ]]; then + echo "At least one of -4 or -6 must be given for add" + usage + exit 2 + fi + + BRNAME="brdr-${NETWORK_ID}" + + # Linux caps interface names at 15 characters + if [[ ${#BRNAME} -gt 15 ]]; then + echo "Bridge name ${BRNAME} exceeds the 15 character interface name limit" + exit 2 + fi +elif [[ "$OP" == "delete" ]]; then + if [[ -z "$BRNAME" ]]; then + usage + exit 2 + fi + + # Not one of ours: tell the agent so it can fall back to its regular + # unplug handling. This is what keeps the bridge naming knowledge in + # this script and nowhere else. + if [[ "$BRNAME" != brdr-* ]]; then + echo "notmine" + exit 0 + fi +else + usage + exit 2 +fi + +addBr() { + if [[ ! -d /sys/class/net/${BRNAME} ]]; then + # No STP and no forwarding delay: the bridge has no uplink, so there is + # no loop to detect and no reason to hold ports down when an Instance starts + ip link add name ${BRNAME} type bridge stp_state 0 forward_delay 0 + ip link set ${BRNAME} up + fi + + # The bridge routes on behalf of every Instance attached to it + sysctl -qw net.ipv4.conf.${BRNAME}.forwarding=1 + sysctl -qw net.ipv6.conf.${BRNAME}.disable_ipv6=0 + sysctl -qw net.ipv6.conf.${BRNAME}.forwarding=1 + + # Never act on a router advertisement sent by an Instance + sysctl -qw net.ipv6.conf.${BRNAME}.accept_ra=0 + + # The same gateway address is configured on every brdr bridge on this host. + # Only answer ARP for the address on the interface the request arrived on, + # and always source ARP from that interface's own address. + sysctl -qw net.ipv4.conf.${BRNAME}.arp_ignore=1 + sysctl -qw net.ipv4.conf.${BRNAME}.arp_announce=2 + + if [[ -n "${IPV4_GATEWAY}" ]]; then + ip address replace ${IPV4_GATEWAY}/32 dev ${BRNAME} + fi + if [[ -n "${IPV6_GATEWAY}" ]]; then + ip -6 address replace ${IPV6_GATEWAY}/64 dev ${BRNAME} + fi + + # Strict reverse path filtering: an Instance may only send from an address + # that is routed back out of this bridge, which is its own /32. Set on the + # bridge only, never on 'all', so no other interface changes behaviour. + # Note this is IPv4 only; the kernel has no IPv6 equivalent. + sysctl -qw net.ipv4.conf.${BRNAME}.rp_filter=1 + +} + +deleteBr() { + if [[ ! -d /sys/class/net/${BRNAME} ]]; then + echo "deleted" + return 0 + fi + + # An Instance may have been started on this network while the last one was + # being stopped; leave the bridge alone if anything is still attached + if [[ -n "$(ls -A /sys/class/net/${BRNAME}/brif 2>/dev/null)" ]]; then + echo "kept" + return 0 + fi + + ip link set ${BRNAME} down + ip link delete ${BRNAME} type bridge + echo "deleted" +} + +# +# Add a lockfile to prevent this script from running twice on the same host +# this can cause a race condition +# + +LOCKFILE=/var/run/cloud/brdr.lock + +# ensures that parent directories exists and prepares the lock file +mkdir -p "${LOCKFILE%/*}" + +( + flock -x -w 10 200 || exit 1 + if [[ "$OP" == "add" ]]; then + addBr + + if [[ $? -gt 0 ]]; then + exit 1 + fi + + # The agent uses this as the bridge name; naming lives here only + echo "${BRNAME}" + elif [[ "$OP" == "delete" ]]; then + deleteBr + fi +) 200>${LOCKFILE} diff --git a/scripts/vm/network/vnet/modifymacip.sh b/scripts/vm/network/vnet/modifymacip.sh index c8d0b0e290ca..d6cd82ae5eae 100755 --- a/scripts/vm/network/vnet/modifymacip.sh +++ b/scripts/vm/network/vnet/modifymacip.sh @@ -24,8 +24,11 @@ # # Both -4 and -6 may be specified multiple times to cover primary and secondary # addresses (e.g. link-local + global unicast for IPv6). -# On delete the bridge neighbour table is queried for all entries matching the -# MAC address; no separate state file is required. +# On delete without any -4/-6, the bridge neighbour table is queried for all +# entries matching the MAC address and every one of them is removed; no separate +# state file is required. This is what an unplug uses. Given -4/-6 addresses, +# delete removes only those, which is what removing a single secondary IP from a +# running Instance needs. usage() { echo "Usage: $0 -o -b -m [-4 ] ... [-6 ] ..." @@ -69,7 +72,24 @@ add_entries() { fi } +delete_listed_entries() { + for addr in "${IPV4_LIST[@]}"; do + ip neigh del "${addr}" dev "${BRIDGE}" 2>/dev/null || true + ip route del "${addr}/32" dev "${BRIDGE}" 2>/dev/null || true + done + + for addr in "${IPV6_LIST[@]}"; do + ip -6 neigh del "${addr}" dev "${BRIDGE}" 2>/dev/null || true + ip -6 route del "${addr}/128" dev "${BRIDGE}" 2>/dev/null || true + done +} + delete_entries() { + if [[ "${#IPV4_LIST[@]}" -gt 0 || "${#IPV6_LIST[@]}" -gt 0 ]]; then + delete_listed_entries + return 0 + fi + # Find all IPv4 neighbour entries on the bridge matching this MAC and remove them while read -r addr; do ip neigh del "${addr}" dev "${BRIDGE}" 2>/dev/null || true diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index 837254ed8b36..fb79295931b9 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -5560,6 +5560,16 @@ public Vlan createVlanAndPublicIpRange(final CreateVlanIpRangeCmd cmd) throws In checkOverlapPrivateIpRange(zoneId, startIP, endIP); } + // On an L3 (Direct Routed) network an overlap is an address conflict, not a policy + // preference: all subnets share one host routing table and are advertised into one + // fabric, so the check must be zone-wide. The IPv6 vlan check above already is; for + // IPv4 the user_ip_address unique key is per network, hence this explicit check. + // Shared networks keep their historical behaviour: the same IPv4 range in two VLANs + // is legitimate there. + if (ipv4 && network.getGuestType() == GuestType.L3) { + checkOverlapPublicIpRange(zoneId, startIP, endIP); + } + long reservedIpAddressesAmount = 0L; if (forVirtualNetwork && vlanOwner != null) { reservedIpAddressesAmount = NetUtils.ip2Long(endIP) - NetUtils.ip2Long(startIP) + 1; @@ -7214,7 +7224,7 @@ public NetworkOffering createNetworkOffering(final NetworkOfferingBaseCmd cmd) { } if (guestType == null) { - throw new InvalidParameterValueException("Invalid \"type\" parameter is given; can have Shared and Isolated values"); + throw new InvalidParameterValueException("Invalid \"type\" parameter is given; supported values are " + Arrays.toString(Network.GuestType.values())); } if (internetProtocol != null) { @@ -7271,9 +7281,9 @@ public NetworkOffering createNetworkOffering(final NetworkOfferingBaseCmd cmd) { } if (service == Service.SecurityGroup) { - // allow security group service for Shared networks only - if (guestType != GuestType.Shared) { - throw new InvalidParameterValueException("Security group service is supported for network offerings with guest ip type " + GuestType.Shared); + // allow security group service for Shared and L3 (Direct Routed) networks only + if (guestType != GuestType.Shared && guestType != GuestType.L3) { + throw new InvalidParameterValueException(String.format("Security group service is supported for network offerings with guest ip type %s or %s", GuestType.Shared, GuestType.L3)); } final Set sgProviders = new HashSet<>(); sgProviders.add(Provider.SecurityGroupProvider); @@ -7369,6 +7379,10 @@ public NetworkOffering createNetworkOffering(final NetworkOfferingBaseCmd cmd) { // validate providers combination here _networkModel.canProviderSupportServices(providerCombinationToVerify); + if (guestType == GuestType.L3) { + validateL3NetworkOffering(serviceProviderMap, networkMode, specifyVlan, specifyIpRanges, forVpc); + } + // validate the LB service capabilities specified in the network // offering final Map lbServiceCapabilityMap = cmd.getServiceCapabilities(Service.Lb); @@ -7471,6 +7485,49 @@ public NetworkOffering createNetworkOffering(final NetworkOfferingBaseCmd cmd) { return offering; } + /** + * Validates a network offering for the L3 (Direct Routed) guest type. There is no Virtual + * Router and no DHCP on these networks: the Instance learns its /32 (and /128) address, + * on-link gateway and routes exclusively from ConfigDrive. UserData via ConfigDrive is + * therefore mandatory. Dns is optional (a template may carry its own resolvers) but must be + * provided by ConfigDrive when present. SecurityGroup is the only other permitted service. + */ + protected void validateL3NetworkOffering(final Map> serviceProviderMap, final NetworkOffering.NetworkMode networkMode, + final boolean specifyVlan, final boolean specifyIpRanges, final Boolean forVpc) { + if (Boolean.TRUE.equals(forVpc)) { + throw new InvalidParameterValueException(String.format("VPC is not supported for network offerings with guest type %s", GuestType.L3)); + } + if (networkMode != null) { + throw new InvalidParameterValueException(String.format("Network mode can not be specified for network offerings with guest type %s", GuestType.L3)); + } + if (specifyVlan) { + throw new InvalidParameterValueException(String.format("VLAN can not be specified for network offerings with guest type %s; these networks use no isolation id", GuestType.L3)); + } + if (!specifyIpRanges) { + throw new InvalidParameterValueException(String.format("Network offerings with guest type %s must specify IP ranges", GuestType.L3)); + } + if (serviceProviderMap.containsKey(Service.Dhcp)) { + throw new InvalidParameterValueException(String.format("DHCP is not supported (and not needed) for network offerings with guest type %s; addressing is delivered via ConfigDrive", GuestType.L3)); + } + final Set allowedL3Services = new HashSet<>(Arrays.asList(Service.UserData, Service.Dns, Service.SecurityGroup)); + for (final Service service : serviceProviderMap.keySet()) { + if (!allowedL3Services.contains(service)) { + throw new InvalidParameterValueException(String.format("Service %s is not supported for network offerings with guest type %s; supported services are %s", + service.getName(), GuestType.L3, StringUtils.join(allowedL3Services.stream().map(Service::getName).toArray(), ", "))); + } + } + final Set configDriveOnly = Collections.singleton(Provider.ConfigDrive); + final Set userDataProviders = serviceProviderMap.get(Service.UserData); + if (!configDriveOnly.equals(userDataProviders)) { + throw new InvalidParameterValueException(String.format("UserData with provider %s is mandatory for network offerings with guest type %s; it is the only channel that carries the Instance's network configuration", + Provider.ConfigDrive.getName(), GuestType.L3)); + } + final Set dnsProviders = serviceProviderMap.get(Service.Dns); + if (dnsProviders != null && !configDriveOnly.equals(dnsProviders)) { + throw new InvalidParameterValueException(String.format("DNS on network offerings with guest type %s must use provider %s", GuestType.L3, Provider.ConfigDrive.getName())); + } + } + public static NetworkOffering.RoutingMode verifyRoutingMode(String routingModeString) { NetworkOffering.RoutingMode routingMode = null; if (routingModeString != null) { diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index 2853fa96330d..f61dddcc65ca 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -868,6 +868,19 @@ public boolean stop() { protected NetworkServiceImpl() { } + /** + * True when the NIC belongs to a Direct Routed (L3) network, where secondary IPs need a host + * route and neighbour entry on the hypervisor whether or not security groups are in use. + */ + protected boolean isDirectRoutedNic(long nicId) { + NicVO nic = _nicDao.findById(nicId); + if (nic == null) { + return false; + } + Network network = _networksDao.findById(nic.getNetworkId()); + return network != null && GuestType.L3.equals(network.getGuestType()); + } + @Override @ActionEvent(eventType = EventTypes.EVENT_NIC_SECONDARY_IP_CONFIGURE, eventDescription = "Configuring secondary IP " + "rules", async = true) public boolean configureNicSecondaryIp(NicSecondaryIp secIp, boolean isZoneSgEnabled) { @@ -877,9 +890,11 @@ public boolean configureNicSecondaryIp(NicSecondaryIp secIp, boolean isZoneSgEna secondaryIp = secIp.getIp6Address(); } - if (isZoneSgEnabled) { + // A Direct Routed network needs the agent told regardless of the zone's security group + // setting: the host route and neighbour entry are what make the address reachable. + if (isZoneSgEnabled || isDirectRoutedNic(secIp.getNicId())) { success = _securityGroupService.securityGroupRulesForVmSecIp(secIp.getNicId(), secondaryIp, true); - logger.info("Associated IP address to NIC : " + secIp.getIp4Address()); + logger.info("Associated IP address to NIC : " + secondaryIp); } else { success = true; } @@ -1605,8 +1620,8 @@ public Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapac } } - // Start and end IP address are mandatory for shared networks. - if (ntwkOff.getGuestType() == GuestType.Shared && vpcId == null) { + // Start and end IP address are mandatory for shared and L3 (Direct Routed) networks. + if ((ntwkOff.getGuestType() == GuestType.Shared || ntwkOff.getGuestType() == GuestType.L3) && vpcId == null) { if (!AllowEmptyStartEndIpAddress.valueIn(owner.getAccountId()) && (startIP == null && endIP == null) && (startIPv6 == null && endIPv6 == null)) { @@ -1655,12 +1670,12 @@ public Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapac endIPv6 = startIPv6; } _networkModel.checkIp6Parameters(startIPv6, endIPv6, ip6Gateway, ip6Cidr); - if (!GuestType.Shared.equals(ntwkOff.getGuestType())) { + if (!GuestType.Shared.equals(ntwkOff.getGuestType()) && !GuestType.L3.equals(ntwkOff.getGuestType())) { _networkModel.checkIp6CidrSizeEqualTo64(ip6Cidr); } - if (zone.getNetworkType() != NetworkType.Advanced || ntwkOff.getGuestType() != Network.GuestType.Shared) { - throw new InvalidParameterValueException("Can only support create IPv6 network with advance shared network!"); + if (zone.getNetworkType() != NetworkType.Advanced || (ntwkOff.getGuestType() != Network.GuestType.Shared && ntwkOff.getGuestType() != Network.GuestType.L3)) { + throw new InvalidParameterValueException(String.format("Can only support create IPv6 network with advanced %s or %s network!", GuestType.Shared, GuestType.L3)); } if(StringUtils.isAllBlank(ip6Dns1, ip6Dns2, zone.getIp6Dns1(), zone.getIp6Dns2())) { @@ -1722,7 +1737,7 @@ public Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapac } // Ignore vlanId if it is passed but specifyvlan=false in network offering - if (ntwkOff.getGuestType() == GuestType.Shared && ! ntwkOff.isSpecifyVlan() && vlanId != null) { + if ((ntwkOff.getGuestType() == GuestType.Shared || ntwkOff.getGuestType() == GuestType.L3) && ! ntwkOff.isSpecifyVlan() && vlanId != null) { throw new InvalidParameterValueException("Cannot specify vlanId when create a network from network offering with specifyvlan=false"); } @@ -1778,9 +1793,11 @@ public Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapac } } - // Vlan is created in 1 cases - works in Advance zone only: - // 1) GuestType is Shared + // Vlan is created in these cases - works in Advance zone only: + // 1) GuestType is Shared or L3 (Direct Routed) + // 2) GuestType is Isolated without SourceNat boolean createVlan = (startIP != null && endIP != null && zone.getNetworkType() == NetworkType.Advanced && ((ntwkOff.getGuestType() == Network.GuestType.Shared) + || (ntwkOff.getGuestType() == Network.GuestType.L3) || (ntwkOff.getGuestType() == GuestType.Isolated && !areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat)))); if (!createVlan) { @@ -1797,9 +1814,9 @@ public Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapac - if (GuestType.Shared == ntwkOff.getGuestType()) { + if (GuestType.Shared == ntwkOff.getGuestType() || GuestType.L3 == ntwkOff.getGuestType()) { if (!ntwkOff.isSpecifyIpRanges()) { - throw new CloudRuntimeException("The 'specifyipranges' parameter should be true for Shared Networks"); + throw new CloudRuntimeException(String.format("The 'specifyipranges' parameter should be true for %s Networks", ntwkOff.getGuestType())); } if (ipv4 && Objects.isNull(startIP)) { throw new CloudRuntimeException("IPv4 address range needs to be provided"); @@ -2018,7 +2035,7 @@ private static ACLType getAclType(String aclTypeStr, NetworkOffering ntwkOff) { } private ACLType getAclType(Account caller, NetworkOffering ntwkOff, ACLType aclType) { - if (ntwkOff.getGuestType() == GuestType.Isolated || ntwkOff.getGuestType() == GuestType.L2) { + if (ntwkOff.getGuestType() == GuestType.Isolated || ntwkOff.getGuestType() == GuestType.L2 || ntwkOff.getGuestType() == GuestType.L3) { aclType = ACLType.Account; } else if (ntwkOff.getGuestType() == GuestType.Shared) { if (_accountMgr.isRootAdmin(caller.getId())) { diff --git a/server/src/main/java/com/cloud/network/guru/DirectRoutedNetworkGuru.java b/server/src/main/java/com/cloud/network/guru/DirectRoutedNetworkGuru.java new file mode 100644 index 000000000000..c1675fe45bf2 --- /dev/null +++ b/server/src/main/java/com/cloud/network/guru/DirectRoutedNetworkGuru.java @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.guru; + +import org.apache.commons.lang3.StringUtils; + +import com.cloud.dc.DataCenter; +import com.cloud.dc.DataCenter.NetworkType; +import com.cloud.deploy.DeployDestination; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientAddressCapacityException; +import com.cloud.exception.InsufficientVirtualNetworkCapacityException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.network.Network; +import com.cloud.network.Network.GuestType; +import com.cloud.network.Network.State; +import com.cloud.network.Networks.BroadcastDomainType; +import com.cloud.network.Networks.Mode; +import com.cloud.network.PhysicalNetwork; +import com.cloud.network.PhysicalNetwork.IsolationMethod; +import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.PhysicalNetworkVO; +import com.cloud.offering.NetworkOffering; +import com.cloud.user.Account; +import com.cloud.utils.net.NetUtils; +import com.cloud.vm.NicProfile; +import com.cloud.vm.ReservationContext; +import com.cloud.vm.VirtualMachineProfile; + +/** + * Network guru for L3 (Direct Routed) guest networks: the hypervisor routes a public IPv4 + * and/or IPv6 address directly to the Instance. There is no Virtual Router, no NAT and no + * DHCP; the Instance learns its addressing exclusively from ConfigDrive. + * + * Address allocation is inherited from {@link DirectNetworkGuru}: the operator supplies a + * subnet, CloudStack assigns individual addresses out of it. What differs is the form those + * addresses take on the NIC — an IPv4 address is a /32 and an IPv6 address a /128, with the + * shared, host-independent on-link gateway (169.254.0.1 / fe80::1) that every hypervisor + * carries on the network's bridge. + * + * There is no isolation method and no broadcast domain: the network offering's guest type is + * the sole selector, and networks of this type consume no VLAN/VXLAN id. Layer 2 isolation + * between networks is provided by a dedicated, uplink-less bridge per network on each host. + */ +public class DirectRoutedNetworkGuru extends DirectNetworkGuru { + + public DirectRoutedNetworkGuru() { + super(); + // No isolation of any kind: no VLAN, no VXLAN, no encapsulation. canHandle() does not + // consult the physical network's isolation methods. + _isolationMethods = new IsolationMethod[] {}; + } + + @Override + protected boolean canHandle(NetworkOffering offering, DataCenter dc, PhysicalNetwork physnet) { + if (dc.getNetworkType() == NetworkType.Advanced && isMyTrafficType(offering.getTrafficType()) && offering.getGuestType() == GuestType.L3) { + return true; + } + logger.trace("We only take care of {} guest networks in zones of type {}", GuestType.L3, NetworkType.Advanced); + return false; + } + + @Override + public Network design(NetworkOffering offering, DeploymentPlan plan, Network userSpecified, String name, Long vpcId, Account owner) { + DataCenter dc = _dcDao.findById(plan.getDataCenterId()); + PhysicalNetworkVO physnet = _physicalNetworkDao.findById(plan.getPhysicalNetworkId()); + + if (!canHandle(offering, dc, physnet)) { + return null; + } + + if (vpcId != null) { + throw new InvalidParameterValueException(String.format("%s networks are never part of a VPC", GuestType.L3)); + } + + // Mode.Static: the Instance is statically configured via ConfigDrive, there is no DHCP. + // BroadcastDomainType.Native: no isolation id is consumed; the broadcast_uri stays empty. + NetworkVO config = new NetworkVO(offering.getTrafficType(), Mode.Static, BroadcastDomainType.Native, offering.getId(), State.Allocated, + plan.getDataCenterId(), plan.getPhysicalNetworkId(), false); + + if (userSpecified != null) { + if ((userSpecified.getCidr() == null && userSpecified.getGateway() != null) || (userSpecified.getCidr() != null && userSpecified.getGateway() == null)) { + throw new InvalidParameterValueException("cidr and gateway must be specified together."); + } + + if ((userSpecified.getIp6Cidr() == null && userSpecified.getIp6Gateway() != null) || + (userSpecified.getIp6Cidr() != null && userSpecified.getIp6Gateway() == null)) { + throw new InvalidParameterValueException("cidrv6 and gatewayv6 must be specified together."); + } + + // The subnet is an allocation pool routed to the hypervisors, not a broadcast domain. + // Its gateway is stored but never used: the Instance's gateway is always the shared + // link-local address. + if (userSpecified.getCidr() != null) { + config.setCidr(userSpecified.getCidr()); + config.setGateway(userSpecified.getGateway()); + } + + if (userSpecified.getIp6Cidr() != null) { + config.setIp6Cidr(userSpecified.getIp6Cidr()); + config.setIp6Gateway(userSpecified.getIp6Gateway()); + } + + if (userSpecified.getPublicMtu() != null) { + config.setPublicMtu(userSpecified.getPublicMtu()); + } + if (userSpecified.getPrivateMtu() != null) { + config.setPrivateMtu(userSpecified.getPrivateMtu()); + } + + if (StringUtils.isNotBlank(userSpecified.getDns1())) { + config.setDns1(userSpecified.getDns1()); + } + if (StringUtils.isNotBlank(userSpecified.getDns2())) { + config.setDns2(userSpecified.getDns2()); + } + if (StringUtils.isNotBlank(userSpecified.getIp6Dns1())) { + config.setIp6Dns1(userSpecified.getIp6Dns1()); + } + if (StringUtils.isNotBlank(userSpecified.getIp6Dns2())) { + config.setIp6Dns2(userSpecified.getIp6Dns2()); + } + } + + return config; + } + + @Override + public NicProfile allocate(Network network, NicProfile nic, VirtualMachineProfile vm) throws InsufficientVirtualNetworkCapacityException, + InsufficientAddressCapacityException, ConcurrentOperationException { + NicProfile profile = super.allocate(network, nic, vm); + applyDirectRoutedAddressing(profile); + return profile; + } + + @Override + public void reserve(NicProfile nic, Network network, VirtualMachineProfile vm, DeployDestination dest, ReservationContext context) + throws InsufficientVirtualNetworkCapacityException, InsufficientAddressCapacityException, ConcurrentOperationException { + super.reserve(nic, network, vm, dest, context); + applyDirectRoutedAddressing(nic); + } + + /** + * The inherited allocation ({@code IpAddressManagerImpl.allocateDirectIp()}) sets the NIC's + * gateway and netmask from the vlan row, as a Shared network needs. Here the address is a + * host route, not a subnet membership: force the /32 (or /128) form and the shared on-link + * gateway over whatever the vlan row provided. This is also the signature by which the KVM + * agent recognises a direct routed NIC. + */ + protected void applyDirectRoutedAddressing(NicProfile nic) { + if (nic == null) { + return; + } + if (nic.getIPv4Address() != null) { + nic.setIPv4Netmask(NetUtils.IPV4_HOST_NETMASK); + nic.setIPv4Gateway(NetUtils.getLinkLocalGateway()); + } + if (nic.getIPv6Address() != null) { + nic.setIPv6Cidr(nic.getIPv6Address() + "/" + NetUtils.IPV6_HOST_PREFIX_LENGTH); + nic.setIPv6Gateway(NetUtils.getIpv6LinkLocalGateway()); + } + } +} diff --git a/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java b/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java index 585b65aa4d94..6ca249e34fc3 100644 --- a/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java +++ b/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java @@ -1451,18 +1451,27 @@ public boolean securityGroupRulesForVmSecIp(long nicId, String secondaryIp, bool // Verify permissions _accountMgr.checkAccess(caller, null, false, vm); + Network network = _networkModel.getNetwork(nic.getNetworkId()); + + // On a Direct Routed network the host needs a route and a static neighbour entry for the + // secondary IP before it is reachable at all. That is independent of security groups, + // which are optional there and which the Instance may not be using, so the agent is told + // either way - otherwise the address would stay dark until the Instance was restarted. + boolean directRouted = Network.GuestType.L3.equals(network.getGuestType()); + // Validate parameters List vmSgGrps = getSecurityGroupsForVm(vmId); + boolean applySecurityGroupRules = true; if (vmSgGrps.isEmpty()) { logger.debug("Vm is not in any Security group "); - return true; - } - - //If network does not support SG service, no need add SG rules for secondary ip - Network network = _networkModel.getNetwork(nic.getNetworkId()); - if (!_networkModel.isSecurityGroupSupportedInNetwork(network)) { + applySecurityGroupRules = false; + } else if (!_networkModel.isSecurityGroupSupportedInNetwork(network)) { logger.debug("Network " + network + " is not enabled with security group service, "+ "so not applying SG rules for secondary ip"); + applySecurityGroupRules = false; + } + + if (!applySecurityGroupRules && !directRouted) { return true; } @@ -1473,7 +1482,8 @@ public boolean securityGroupRulesForVmSecIp(long nicId, String secondaryIp, bool } //create command for the to add ip in ipset and arptables rules - NetworkRulesVmSecondaryIpCommand cmd = new NetworkRulesVmSecondaryIpCommand(vmName, vmMac, secondaryIp, ruleAction); + NetworkRulesVmSecondaryIpCommand cmd = new NetworkRulesVmSecondaryIpCommand(vmName, vmMac, secondaryIp, ruleAction, + directRouted, applySecurityGroupRules); logger.debug("Asking agent to configure rules for vm secondary ip"); Commands cmds = null; diff --git a/server/src/main/resources/META-INF/cloudstack/server-network/spring-server-network-context.xml b/server/src/main/resources/META-INF/cloudstack/server-network/spring-server-network-context.xml index 97ed36e515c7..f3d518cd185c 100644 --- a/server/src/main/resources/META-INF/cloudstack/server-network/spring-server-network-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/server-network/spring-server-network-context.xml @@ -48,6 +48,9 @@ + + + diff --git a/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java b/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java index 9a0b150780e4..08c0e41872dd 100644 --- a/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java +++ b/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java @@ -1410,4 +1410,79 @@ public void testGetExternalNetworkProviderReturnsNullWhenNoExternalProviders() { mapWithEmptySet.put(Network.Service.Firewall, Collections.emptySet()); Assert.assertNull(ConfigurationManagerImpl.getExternalNetworkProvider(null, mapWithEmptySet)); } + + private Map> validL3ServiceProviderMap() { + Map> map = new HashMap<>(); + map.put(Network.Service.UserData, Collections.singleton(Network.Provider.ConfigDrive)); + map.put(Network.Service.Dns, Collections.singleton(Network.Provider.ConfigDrive)); + map.put(Network.Service.SecurityGroup, Collections.singleton(Network.Provider.SecurityGroupProvider)); + return map; + } + + @Test + public void validateL3NetworkOfferingAcceptsUserDataDnsAndSecurityGroup() { + configurationManagerImplSpy.validateL3NetworkOffering(validL3ServiceProviderMap(), null, false, true, false); + } + + @Test + public void validateL3NetworkOfferingAcceptsOfferingWithoutDns() { + Map> map = validL3ServiceProviderMap(); + map.remove(Network.Service.Dns); + configurationManagerImplSpy.validateL3NetworkOffering(map, null, false, true, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsDhcp() { + Map> map = validL3ServiceProviderMap(); + map.put(Network.Service.Dhcp, Collections.singleton(Network.Provider.ConfigDrive)); + configurationManagerImplSpy.validateL3NetworkOffering(map, null, false, true, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsMissingUserData() { + Map> map = validL3ServiceProviderMap(); + map.remove(Network.Service.UserData); + configurationManagerImplSpy.validateL3NetworkOffering(map, null, false, true, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsUserDataWithoutProvider() { + Map> map = validL3ServiceProviderMap(); + map.put(Network.Service.UserData, Collections.emptySet()); + configurationManagerImplSpy.validateL3NetworkOffering(map, null, false, true, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsNonConfigDriveDns() { + Map> map = validL3ServiceProviderMap(); + map.put(Network.Service.Dns, Collections.singleton(Network.Provider.VirtualRouter)); + configurationManagerImplSpy.validateL3NetworkOffering(map, null, false, true, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsUnsupportedService() { + Map> map = validL3ServiceProviderMap(); + map.put(Network.Service.SourceNat, Collections.singleton(Network.Provider.VirtualRouter)); + configurationManagerImplSpy.validateL3NetworkOffering(map, null, false, true, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsNetworkMode() { + configurationManagerImplSpy.validateL3NetworkOffering(validL3ServiceProviderMap(), NetworkOffering.NetworkMode.ROUTED, false, true, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsSpecifyVlan() { + configurationManagerImplSpy.validateL3NetworkOffering(validL3ServiceProviderMap(), null, true, true, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsWithoutSpecifyIpRanges() { + configurationManagerImplSpy.validateL3NetworkOffering(validL3ServiceProviderMap(), null, false, false, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsVpc() { + configurationManagerImplSpy.validateL3NetworkOffering(validL3ServiceProviderMap(), null, false, true, true); + } } diff --git a/server/src/test/java/com/cloud/network/guru/DirectRoutedNetworkGuruTest.java b/server/src/test/java/com/cloud/network/guru/DirectRoutedNetworkGuruTest.java new file mode 100644 index 000000000000..0198bfc7629e --- /dev/null +++ b/server/src/test/java/com/cloud/network/guru/DirectRoutedNetworkGuruTest.java @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.guru; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.dc.DataCenter.NetworkType; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.network.Network; +import com.cloud.network.Network.GuestType; +import com.cloud.network.Networks.BroadcastDomainType; +import com.cloud.network.Networks.Mode; +import com.cloud.network.Networks.TrafficType; +import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.PhysicalNetworkDao; +import com.cloud.network.dao.PhysicalNetworkVO; +import com.cloud.offering.NetworkOffering; +import com.cloud.user.Account; +import com.cloud.vm.NicProfile; + +@RunWith(MockitoJUnitRunner.Silent.class) +public class DirectRoutedNetworkGuruTest { + + @InjectMocks + protected DirectRoutedNetworkGuru guru = new DirectRoutedNetworkGuru(); + + @Mock + DataCenterDao dcDao; + @Mock + PhysicalNetworkDao physicalNetworkDao; + + @Mock + NetworkOffering offering; + @Mock + DataCenterVO dc; + @Mock + PhysicalNetworkVO physicalNetwork; + @Mock + DeploymentPlan plan; + @Mock + Account owner; + + @Before + public void setUp() { + lenient().when(dc.getNetworkType()).thenReturn(NetworkType.Advanced); + lenient().when(offering.getTrafficType()).thenReturn(TrafficType.Guest); + lenient().when(offering.getGuestType()).thenReturn(GuestType.L3); + lenient().when(plan.getDataCenterId()).thenReturn(1L); + lenient().when(plan.getPhysicalNetworkId()).thenReturn(1L); + lenient().when(dcDao.findById(1L)).thenReturn(dc); + lenient().when(physicalNetworkDao.findById(1L)).thenReturn(physicalNetwork); + } + + @Test + public void canHandleAcceptsL3InAdvancedZone() { + assertTrue(guru.canHandle(offering, dc, physicalNetwork)); + } + + @Test + public void canHandleRejectsBasicZone() { + when(dc.getNetworkType()).thenReturn(NetworkType.Basic); + assertFalse(guru.canHandle(offering, dc, physicalNetwork)); + } + + @Test + public void canHandleRejectsOtherGuestTypes() { + for (GuestType type : new GuestType[] {GuestType.Shared, GuestType.Isolated, GuestType.L2}) { + when(offering.getGuestType()).thenReturn(type); + assertFalse("guru must not claim guest type " + type, guru.canHandle(offering, dc, physicalNetwork)); + } + } + + @Test + public void designProducesNativeStaticNetwork() { + Network network = guru.design(offering, plan, null, "test", null, owner); + assertNotNull(network); + NetworkVO config = (NetworkVO)network; + assertEquals(BroadcastDomainType.Native, config.getBroadcastDomainType()); + assertEquals(Mode.Static, config.getMode()); + assertNull(config.getBroadcastUri()); + } + + @Test(expected = InvalidParameterValueException.class) + public void designRejectsVpc() { + guru.design(offering, plan, null, "test", 42L, owner); + } + + @Test + public void designReturnsNullForOtherGuestTypes() { + when(offering.getGuestType()).thenReturn(GuestType.Shared); + assertNull(guru.design(offering, plan, null, "test", null, owner)); + } + + @Test + public void applyDirectRoutedAddressingForcesHostRouteForm() { + NicProfile nic = new NicProfile(); + nic.setIPv4Address("203.0.113.55"); + nic.setIPv4Netmask("255.255.255.0"); + nic.setIPv4Gateway("203.0.113.1"); + nic.setIPv6Address("2001:db8:1::55"); + nic.setIPv6Cidr("2001:db8:1::/64"); + nic.setIPv6Gateway("2001:db8:1::1"); + + guru.applyDirectRoutedAddressing(nic); + + assertEquals("255.255.255.255", nic.getIPv4Netmask()); + assertEquals("169.254.0.1", nic.getIPv4Gateway()); + assertEquals("2001:db8:1::55/128", nic.getIPv6Cidr()); + assertEquals("fe80::1", nic.getIPv6Gateway()); + } + + @Test + public void applyDirectRoutedAddressingLeavesAbsentFamiliesAlone() { + NicProfile nic = new NicProfile(); + nic.setIPv4Address("203.0.113.55"); + nic.setIPv4Netmask("255.255.255.0"); + nic.setIPv4Gateway("203.0.113.1"); + + guru.applyDirectRoutedAddressing(nic); + + assertEquals("255.255.255.255", nic.getIPv4Netmask()); + assertEquals("169.254.0.1", nic.getIPv4Gateway()); + assertNull(nic.getIPv6Cidr()); + assertNull(nic.getIPv6Gateway()); + } + + @Test + public void applyDirectRoutedAddressingIsNullSafe() { + guru.applyDirectRoutedAddressing(null); + } +} diff --git a/test/integration/smoke/test_l3_networks.py b/test/integration/smoke/test_l3_networks.py new file mode 100644 index 000000000000..b2bb88a15e19 --- /dev/null +++ b/test/integration/smoke/test_l3_networks.py @@ -0,0 +1,186 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" Tests for L3 (Direct Routed) guest networks: the hypervisor routes a public IPv4/IPv6 + address directly to the Instance - no Virtual Router, no NAT, no DHCP. Instances receive + a /32 (or /128) with the shared link-local gateway, delivered via ConfigDrive. +""" + +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.cloudstackException import CloudstackAPIException +from marvin.lib.base import (Account, + Network, + NetworkOffering, + ServiceOffering, + VirtualMachine, + Zone) +from marvin.lib.common import (get_domain, + get_template, + get_zone) +from marvin.lib.utils import cleanup_resources +from nose.plugins.attrib import attr + + +class TestL3Networks(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + testClient = super(TestL3Networks, cls).getClsTestClient() + cls.apiclient = testClient.getApiClient() + cls.services = testClient.getParsedTestDataConfig() + + cls.domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests()) + cls.template = get_template(cls.apiclient, cls.zone.id, cls.services["ostype"]) + + cls._cleanup = [] + cls.hypervisor = testClient.getHypervisorInfo() + cls.skip = False + + zone = Zone(cls.zone.__dict__) + if zone.networktype != 'Advanced': + cls.skip = True + return + + cls.services["virtual_machine"]["zoneid"] = cls.zone.id + cls.services["virtual_machine"]["template"] = cls.template.id + + cls.service_offering = ServiceOffering.create( + cls.apiclient, + cls.services["service_offering"] + ) + cls._cleanup.append(cls.service_offering) + + cls.network_offering = NetworkOffering.create( + cls.apiclient, + cls.services["l3_network_offering"] + ) + cls._cleanup.append(cls.network_offering) + cls.network_offering.update(cls.apiclient, state='Enabled') + + cls.account = Account.create( + cls.apiclient, + cls.services["account"], + admin=True, + domainid=cls.domain.id + ) + cls._cleanup.append(cls.account) + + @classmethod + def tearDownClass(cls): + try: + cleanup_resources(cls.apiclient, reversed(cls._cleanup)) + except Exception as e: + raise Exception("Warning: Exception during class cleanup : %s" % e) + + def setUp(self): + if self.skip: + self.skipTest("L3 networks require an Advanced zone, skipping") + self.cleanup = [] + + def tearDown(self): + try: + cleanup_resources(self.apiclient, reversed(self.cleanup)) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + def create_l3_network(self, startip="203.0.113.10", endip="203.0.113.50"): + services = dict(self.services["l3_network"]) + services["startip"] = startip + services["endip"] = endip + return Network.create( + self.apiclient, + services, + zoneid=self.zone.id, + networkofferingid=self.network_offering.id, + accountid=self.account.name, + domainid=self.account.domainid + ) + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_01_create_l3_network(self): + """ An L3 network is created like a Shared network: with a subnet. The subnet is an + allocation pool routed to the hypervisors, not a broadcast domain, and consumes + no isolation id. """ + network = self.create_l3_network() + self.cleanup.append(network) + + self.assertEqual(network.type, "L3", "network type should be L3") + self.assertEqual(network.broadcastdomaintype, "Native", "an L3 network consumes no isolation id") + self.assertIn(network.state, ["Setup", "Allocated"], "unexpected network state") + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_02_deploy_vm_in_l3_network(self): + """ The Instance's NIC carries its address as a host route: a /32 with the shared, + host-independent link-local gateway. The subnet's own gateway is never used. """ + network = self.create_l3_network() + self.cleanup.append(network) + + virtual_machine = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering.id, + networkids=[network.id] + ) + self.cleanup.append(virtual_machine) + + self.assertEqual(virtual_machine.state, "Running") + nic = virtual_machine.nic[0] + self.assertEqual(nic.netmask, "255.255.255.255", "an L3 NIC address is a host route (/32)") + self.assertEqual(nic.gateway, "169.254.0.1", "an L3 NIC uses the shared link-local gateway") + self.assertTrue(nic.ipaddress.startswith("203.0.113."), "the address must come from the network's subnet") + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_03_l3_offering_rejects_dhcp(self): + """ DHCP is not supported and not needed on L3 networks: ConfigDrive carries the + address, netmask, gateway and routes, so DHCP would have nothing left to hand out. """ + services = dict(self.services["l3_network_offering"]) + services["name"] = "Test L3 offering with Dhcp - must fail" + services["supportedservices"] = "UserData,Dns,Dhcp" + services["serviceProviderList"] = { + "UserData": "ConfigDrive", + "Dns": "ConfigDrive", + "Dhcp": "ConfigDrive" + } + with self.assertRaises(Exception): + NetworkOffering.create(self.apiclient, services) + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_04_l3_offering_requires_userdata(self): + """ UserData via ConfigDrive is mandatory: it is the only channel that carries the + Instance's network configuration. """ + services = dict(self.services["l3_network_offering"]) + services["name"] = "Test L3 offering without UserData - must fail" + services["supportedservices"] = "Dns" + services["serviceProviderList"] = {"Dns": "ConfigDrive"} + with self.assertRaises(Exception): + NetworkOffering.create(self.apiclient, services) + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_05_l3_subnets_may_not_overlap_zone_wide(self): + """ All L3 subnets share one host routing table and one routing fabric, so an overlap + is an address conflict, not a policy preference. The check is zone wide. """ + network = self.create_l3_network(startip="203.0.113.10", endip="203.0.113.30") + self.cleanup.append(network) + + try: + overlapping = self.create_l3_network(startip="203.0.113.20", endip="203.0.113.40") + self.cleanup.append(overlapping) + self.fail("creating an L3 network overlapping another must fail") + except (CloudstackAPIException, Exception): + pass diff --git a/tools/marvin/marvin/config/test_data.py b/tools/marvin/marvin/config/test_data.py index e3d4022cf0f9..c9bd4736430b 100644 --- a/tools/marvin/marvin/config/test_data.py +++ b/tools/marvin/marvin/config/test_data.py @@ -221,6 +221,28 @@ "endip": "172.16.15.41", "acltype": "Account" }, + "l3_network_offering": { + "name": "Test L3 Direct Routed - Network offering", + "displaytext": "Test L3 Direct Routed - Network offering", + "guestiptype": "L3", + "supportedservices": "UserData,Dns", + "specifyIpRanges": "True", + "specifyVlan": "False", + "traffictype": "GUEST", + "availability": "Optional", + "serviceProviderList": { + "UserData": "ConfigDrive", + "Dns": "ConfigDrive" + } + }, + "l3_network": { + "name": "Test L3 Direct Routed Network", + "displaytext": "Test L3 Direct Routed Network", + "gateway": "203.0.113.1", + "netmask": "255.255.255.0", + "startip": "203.0.113.10", + "endip": "203.0.113.50" + }, "l2-network_offering": { "name": "Test L2 - Network offering", "displaytext": "Test L2 - Network offering", diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 775de26103a0..778ac37af7dc 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -1554,6 +1554,7 @@ "label.kvm": "KVM", "label.kvmnetworklabel": "KVM traffic label", "label.l2": "L2", +"label.l3": "L3 (Direct Routed)", "label.l2gatewayserviceuuid": "L2 Gateway Service UUID", "label.l3gatewayserviceuuid": "L3 Gateway Service UUID", "label.label": "Label", @@ -3908,6 +3909,9 @@ "message.note.about.keypair.permissions.title": "Note about API key pair rule permissions", "message.note.about.keypair.permissions.body": "During the creation of API key pairs, it is possible to define a corresponding set of rule permissions. If a rule set is defined, the API key pair will only have access to APIs for which access has been explicitly granted (i.e., APIs whose corresponding rules are marked as allowed). On the other hand, if no rule set is specified, the API key pair permissions will follow and adapt to the permission set of the user's account role.", "message.offering.internet.protocol.warning": "WARNING: IPv6 supported Networks use static routing and will require upstream routes to be configured manually.", +"message.create.l3.network": "The hypervisor routes a public IPv4/IPv6 address directly to each Instance on this network: no Virtual Router, no NAT and no DHCP. The subnet is an allocation pool routed to the hypervisors; its gateway is required but never used, since Instances always use the shared link-local gateway. Instances receive their configuration via ConfigDrive.", +"message.success.create.l3.network": "Successfully created L3 (Direct Routed) network", +"message.offering.l3": "The hypervisor routes a public IPv4/IPv6 address directly to each Instance: no Virtual Router, no NAT and no DHCP. UserData via ConfigDrive is always enabled, since it is the only channel that carries the Instance's network configuration. DNS is strongly recommended; Security Groups are optional.", "message.offering.ipv6.warning": "Please refer documentation for creating IPv6 enabled Network/VPC offering IPv6 support in CloudStack - Isolated Networks and VPC Network Tiers", "message.oobm.configured": "Successfully configured out-of-band management for host", "message.ovf.configurations": "OVF configurations available for the selected appliance. Please select the desired value. Incompatible compute offerings will get disabled.", diff --git a/ui/src/views/network/CreateL3NetworkForm.vue b/ui/src/views/network/CreateL3NetworkForm.vue new file mode 100644 index 000000000000..b6a04e802976 --- /dev/null +++ b/ui/src/views/network/CreateL3NetworkForm.vue @@ -0,0 +1,402 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + diff --git a/ui/src/views/network/CreateNetwork.vue b/ui/src/views/network/CreateNetwork.vue index 46ed9d2e039f..385bb1bba98e 100644 --- a/ui/src/views/network/CreateNetwork.vue +++ b/ui/src/views/network/CreateNetwork.vue @@ -34,6 +34,13 @@ @refresh-data="refreshParent" @refresh="handleRefresh"/> + + + {{ $t('label.l2') }} + + {{ $t('label.l3') }} + {{ $t('label.shared') }} @@ -93,7 +96,7 @@ - +