Terraform NSG rule (#1498)

* Terraform NSG rule * Split Subnet to own resource and add azurerm_subnet_network_security_group_association * Remove script from bootstrap * Modify pre-clusters output on vlan and subnets --------- Co-authored-by: Automatic Update <[email protected]>
equinor · Nov 25, 2024 · b0301fe · b0301fe
1 parent e69793e
commit b0301fe
Show file tree

Hide file tree

Showing 16 changed files with 134 additions and 170 deletions.
diff --git a/scripts/aks/util-pdb-patch.sh b/scripts/aks/util-pdb-patch.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+#######################################################################################
+### PURPOSE
+###
+
+# Patch all PDB's in cluster to allow teardown
+
+#######################################################################################
+
+for row in $(kubectl get pdb -A -o json | jq -c '.items[] | select(.spec.minAvailable == 1) | {namespace: .metadata.namespace, name: .metadata.name, minAvailable: .spec.minAvailable}'); do
+  namespace=$(echo "$row" | jq -r '.namespace')
+  name=$(echo "$row" | jq -r '.name')
+  minAvailable=$(echo "$row" | jq -r '.minAvailable')
+  kubectl patch pdb -n ${namespace} ${name} -p '{"spec":{"minAvailable":0}}'
+done
diff --git a/scripts/ingress-nginx/bootstrap.sh b/scripts/ingress-nginx/bootstrap.sh
@@ -302,35 +302,35 @@ create-a-record "*.${CLUSTER_NAME}" "$SELECTED_INGRESS_IP_RAW_ADDRESS" "$AZ_RESO
 
 # create nsg rule, update subnet.
 # Create network security group rule
-printf "Creating azure NSG rule %s-rule... " "${NSG_NAME}"
-az network nsg rule create \
-    --nsg-name "${NSG_NAME}" \
-    --name "${NSG_NAME}-rule" \
-    --resource-group "${AZ_RESOURCE_GROUP_CLUSTERS}" \
-    --subscription "${AZ_SUBSCRIPTION_ID}" \
-    --destination-address-prefixes "${SELECTED_INGRESS_IP_RAW_ADDRESS}" \
-    --destination-port-ranges 80 443 \
-    --access "Allow" \
-    --direction "Inbound" \
-    --priority 100 \
-    --protocol Tcp \
-    --source-address-prefixes "*" \
-    --source-port-ranges "*" \
-    --output none \
-    --only-show-errors
-
-printf "Done.\n"
-
-printf "    Updating subnet %s to associate NSG... " "${SUBNET_NAME}"
-az network vnet subnet update \
-    --vnet-name "${VNET_NAME}" \
-    --resource-group "${AZ_RESOURCE_GROUP_CLUSTERS}" \
-    --name "${SUBNET_NAME}" \
-    --subscription "${AZ_SUBSCRIPTION_ID}" \
-    --network-security-group "${NSG_NAME}" \
-    --output none \
-    --only-show-errors || { echo "ERROR: Could not update subnet." >&2; }
-printf "Done.\n"
+# printf "Creating azure NSG rule %s-rule... " "${NSG_NAME}"
+# az network nsg rule create \
+#     --nsg-name "${NSG_NAME}" \
+#     --name "${NSG_NAME}-rule" \
+#     --resource-group "${AZ_RESOURCE_GROUP_CLUSTERS}" \
+#     --subscription "${AZ_SUBSCRIPTION_ID}" \
+#     --destination-address-prefixes "${SELECTED_INGRESS_IP_RAW_ADDRESS}" \
+#     --destination-port-ranges 80 443 \
+#     --access "Allow" \
+#     --direction "Inbound" \
+#     --priority 100 \
+#     --protocol Tcp \
+#     --source-address-prefixes "*" \
+#     --source-port-ranges "*" \
+#     --output none \
+#     --only-show-errors
+
+# printf "Done.\n"
+
+# printf "    Updating subnet %s to associate NSG... " "${SUBNET_NAME}"
+# az network vnet subnet update \
+#     --vnet-name "${VNET_NAME}" \
+#     --resource-group "${AZ_RESOURCE_GROUP_CLUSTERS}" \
+#     --name "${SUBNET_NAME}" \
+#     --subscription "${AZ_SUBSCRIPTION_ID}" \
+#     --network-security-group "${NSG_NAME}" \
+#     --output none \
+#     --only-show-errors || { echo "ERROR: Could not update subnet." >&2; }
+# printf "Done.\n"
 
 kubectl create namespace ingress-nginx --dry-run=client -o yaml |
     kubectl apply -f -

diff --git a/scripts/move_custom_ingresses.sh b/scripts/move_custom_ingresses.sh
@@ -277,30 +277,6 @@ printf "%s► Execute %s%s\n" "${grn}" "$UPDATE_AUTH_PROXY_SECRET_FOR_CONSOLE_SC
 wait # wait for subshell to finish
 
 #######################################################################################
-### Tag $DEST_CLUSTER to have tag: autostartupschedule="true"
-### Used in GHA to determine which cluster shall be powered on daily
-###
-
-echo ""
-if [[ $CLUSTER_TYPE == "development" ]]; then
-    CLUSTERS=$(az aks list -ojson | jq '[{k8s:[.[] | select((.name | startswith("playground") or startswith('\"$DEST_CLUSTER\"') | not) and (.powerState.code!="Stopped") and (.tags.autostartupschedule == null) or (.name == '\"$SOURCE_CLUSTER\"')) | {name: .name, powerstate: .powerState.code, id: .id}]}]')
-
-    while read -r list; do
-        CLUSTER=$(jq -n "${list}" | jq -r .name)
-        ID=$(jq -n "${list}" | jq -r .id)
-        printf "Clear tag 'autostartupschedule' on cluster %s\n" "${CLUSTER}"
-        az resource tag \
-            --ids "${ID}" \
-            --tags autostartupschedule=false \
-            --is-incremental
-    done < <(printf "%s" "${CLUSTERS}" | jq -c '.[].k8s[]')
-
-    printf "Tag cluster %s to autostartupschedule\n" "${DEST_CLUSTER}"
-    az resource tag \
-        --ids "/subscriptions/${AZ_SUBSCRIPTION_ID}/resourcegroups/${AZ_RESOURCE_GROUP_CLUSTERS}/providers/Microsoft.ContainerService/managedClusters/${DEST_CLUSTER}" \
-        --tags autostartupschedule=true \
-        --is-incremental
-fi
 
 if [[ -z $CI ]]; then
     echo ""

diff --git a/terraform/subscriptions/modules/aks/network.tf b/terraform/subscriptions/modules/aks/network.tf
@@ -2,6 +2,19 @@ resource "azurerm_network_security_group" "this" {
   name                = "nsg-${var.cluster_name}"
   location            = var.location
   resource_group_name = var.resource_group
+  security_rule {
+
+    access                     = "Allow"
+    destination_address_prefix = var.ingressIP
+    destination_port_ranges    = ["80", "443"]
+    direction                  = "Inbound"
+    name                       = "nsg-${var.cluster_name}-rule"
+    priority                   = 100
+    protocol                   = "Tcp"
+    source_address_prefix      = "*"
+    source_port_range          = "*"
+
+  }
 
   tags = {
     IaC = "terraform"
@@ -39,13 +52,6 @@ resource "azurerm_virtual_network" "this" {
   resource_group_name = var.resource_group
   address_space       = ["${var.address_space}/16"]
 
-  subnet {
-    name                            = "subnet-${var.cluster_name}"
-    address_prefixes                = ["${var.address_space}/18"]
-    security_group                  = azurerm_network_security_group.this.id
-    default_outbound_access_enabled = false
-    service_endpoints               = var.service_endpoints
-  }
   dynamic "ddos_protection_plan" {
     for_each = var.enviroment == "platform" || var.enviroment == "c2" ? [1] : []
     content {
@@ -60,6 +66,20 @@ resource "azurerm_virtual_network" "this" {
   }
 }
 
+resource "azurerm_subnet" "this" {
+  name                            = "subnet-${var.cluster_name}"
+  resource_group_name             = var.resource_group
+  virtual_network_name            = azurerm_virtual_network.this.name
+  address_prefixes                = ["${var.address_space}/18"]
+  default_outbound_access_enabled = false
+}
+
+resource "azurerm_subnet_network_security_group_association" "this" {
+  subnet_id                 = azurerm_subnet.this.id
+  network_security_group_id = azurerm_network_security_group.this.id
+  depends_on                = [azurerm_virtual_network.this]
+}
+
 
 resource "azurerm_management_lock" "network" {
   for_each   = var.enviroment == "platform" || var.enviroment == "c2" ? { "${azurerm_virtual_network.this.name}" : true } : {}
@@ -72,3 +92,7 @@ resource "azurerm_management_lock" "network" {
 output "vnet" {
   value = azurerm_virtual_network.this
 }
+
+output "subnet" {
+  value = azurerm_subnet.this
+}
diff --git a/terraform/subscriptions/modules/aks/variables.tf b/terraform/subscriptions/modules/aks/variables.tf
@@ -109,15 +109,6 @@ variable "nodepools" {
   }))
 }
 
-# variable "autostartupschedule" {
-#   type    = bool
-#   default = false
-# }
-
-# variable "migrationStrategy" {
-#   type = string
-# }
-
 variable "storageaccount_id" {
   description = "The ID of the Storage Account"
   type        = string
@@ -140,4 +131,8 @@ variable "containers_workspace_id" {
 variable "service_endpoints" {
   type    = list(string)
   default = []
+}
+
+variable "ingressIP" {
+  type = string
 }
diff --git a/terraform/subscriptions/s940/c2/pre-clusters/aks.tf b/terraform/subscriptions/s940/c2/pre-clusters/aks.tf
@@ -46,6 +46,7 @@ module "aks" {
   workload_identity_enabled = each.value.workload_identity_enabled
   network_policy            = each.value.network_policy
   developers                = module.config.developers
+  ingressIP                 = each.value.ingressIP
 }
 
 locals {
@@ -54,8 +55,8 @@ locals {
       cluster     = key
       vnet_name   = value.vnet.name
       vnet_id     = value.vnet.id
-      subnet_id   = tolist(value.vnet.subnet)[0].id
-      subnet_name = tolist(value.vnet.subnet)[0].name
+      subnet_id   = value.subnet.id
+      subnet_name = value.subnet.name
     }
   }
 }

diff --git a/terraform/subscriptions/s940/c2/pre-clusters/main.tf b/terraform/subscriptions/s940/c2/pre-clusters/main.tf
@@ -12,30 +12,3 @@ data "azurerm_storage_account" "this" {
   name                = "radixlog${module.config.environment}"
   resource_group_name = module.config.common_resource_group
 }
-
-# module "clusternetwork" {
-#   source              = "../../../modules/cluster_network"
-#   for_each            = { for k, v in jsondecode(nonsensitive(data.azurerm_key_vault_secret.this.value)).clusters : v.name => v.ip }
-#   cluster_name        = each.key
-#   resource_group_name = module.config.cluster_resource_group
-#   location            = module.config.location
-#   storageaccount_id   = data.azurerm_storage_account.this.id
-#   address_space       = each.value
-#   enviroment          = module.config.environment
-# }
-
-# locals {
-#   flattened_vnets = {
-#     for key, value in module.clusternetwork : key => {
-#       cluster     = key
-#       vnet_name   = value.vnet.name
-#       vnet_id     = value.vnet.id
-#       subnet_id   = tolist(value.vnet.subnet)[0].id
-#       subnet_name = tolist(value.vnet.subnet)[0].name
-#     }
-#   }
-# }
-
-# output "vnets" {
-#   value = local.flattened_vnets
-# }
diff --git a/terraform/subscriptions/s940/extmon/pre-clusters/aks.tf b/terraform/subscriptions/s940/extmon/pre-clusters/aks.tf
@@ -44,11 +44,12 @@ module "aks" {
   identity_kublet_object      = data.azurerm_user_assigned_identity.akskubelet.principal_id
   identity_kublet_identity_id = data.azurerm_user_assigned_identity.akskubelet.id
   defender_workspace_id       = data.azurerm_log_analytics_workspace.defender.id
-  # containers_workspace_id     = data.azurerm_log_analytics_workspace.containers.id
-  cost_analysis             = each.value.cost_analysis
-  workload_identity_enabled = each.value.workload_identity_enabled
-  network_policy            = each.value.network_policy
-  developers                = module.config.developers
+  containers_workspace_id     = data.azurerm_log_analytics_workspace.containers.id
+  cost_analysis               = each.value.cost_analysis
+  workload_identity_enabled   = each.value.workload_identity_enabled
+  network_policy              = each.value.network_policy
+  developers                  = module.config.developers
+  ingressIP                   = each.value.ingressIP
 }
 
 locals {
@@ -57,8 +58,8 @@ locals {
       cluster     = key
       vnet_name   = value.vnet.name
       vnet_id     = value.vnet.id
-      subnet_id   = tolist(value.vnet.subnet)[0].id
-      subnet_name = tolist(value.vnet.subnet)[0].name
+      subnet_id   = value.subnet.id
+      subnet_name = value.subnet.name
     }
   }
 }

diff --git a/terraform/subscriptions/s940/extmon/pre-clusters/main.tf b/terraform/subscriptions/s940/extmon/pre-clusters/main.tf
@@ -12,30 +12,3 @@ data "azurerm_storage_account" "this" {
   name                = "radixlog${module.config.environment}"
   resource_group_name = module.config.common_resource_group
 }
-
-# module "clusternetwork" {
-#   source              = "../../../modules/cluster_network"
-#   for_each            = { for k, v in jsondecode(nonsensitive(data.azurerm_key_vault_secret.this.value)).clusters : v.name => v.ip }
-#   cluster_name        = each.key
-#   resource_group_name = module.config.cluster_resource_group
-#   location            = module.config.location
-#   storageaccount_id   = data.azurerm_storage_account.this.id
-#   address_space       = each.value
-#   enviroment          = module.config.environment
-# }
-
-# locals {
-#   flattened_vnets = {
-#     for key, value in module.clusternetwork : key => {
-#       cluster     = key
-#       vnet_name   = value.vnet.name
-#       vnet_id     = value.vnet.id
-#       subnet_id   = tolist(value.vnet.subnet)[0].id
-#       subnet_name = tolist(value.vnet.subnet)[0].name
-#     }
-#   }
-# }
-
-# output "vnets" {
-#   value = local.flattened_vnets
-# }
diff --git a/terraform/subscriptions/s940/prod/pre-clusters/aks.tf b/terraform/subscriptions/s940/prod/pre-clusters/aks.tf
@@ -47,6 +47,7 @@ module "aks" {
   workload_identity_enabled = each.value.workload_identity_enabled
   network_policy            = each.value.network_policy
   developers                = module.config.developers
+  ingressIP                 = each.value.ingressIP
 }
 
 locals {
@@ -55,8 +56,8 @@ locals {
       cluster     = key
       vnet_name   = value.vnet.name
       vnet_id     = value.vnet.id
-      subnet_id   = tolist(value.vnet.subnet)[0].id
-      subnet_name = tolist(value.vnet.subnet)[0].name
+      subnet_id   = value.subnet.id
+      subnet_name = value.subnet.name
     }
   }
 }

diff --git a/terraform/subscriptions/s940/prod/pre-clusters/main.tf b/terraform/subscriptions/s940/prod/pre-clusters/main.tf
@@ -12,30 +12,3 @@ data "azurerm_storage_account" "this" {
   name                = "radixlog${module.config.environment}"
   resource_group_name = module.config.common_resource_group
 }
-
-# module "clusternetwork" {
-#   source              = "../../../modules/cluster_network"
-#   for_each            = { for k, v in jsondecode(nonsensitive(data.azurerm_key_vault_secret.this.value)).clusters : v.name => v.ip }
-#   cluster_name        = each.key
-#   resource_group_name = "clusters" #TODO
-#   location            = module.config.location
-#   storageaccount_id   = data.azurerm_storage_account.this.id
-#   address_space       = each.value
-#   enviroment          = module.config.environment
-# }
-
-# locals {
-#   flattened_vnets = {
-#     for key, value in module.clusternetwork : key => {
-#       cluster     = key
-#       vnet_name   = value.vnet.name
-#       vnet_id     = value.vnet.id
-#       subnet_id   = tolist(value.vnet.subnet)[0].id
-#       subnet_name = tolist(value.vnet.subnet)[0].name
-#     }
-#   }
-# }
-
-# output "vnets" {
-#   value = local.flattened_vnets
-# }
diff --git a/terraform/subscriptions/s941/dev/pre-clusters/.terraform.lock.hcl b/terraform/subscriptions/s941/dev/pre-clusters/.terraform.lock.hcl