feat(ci): add sdn for e2e nested clusters

.github/workflows/e2e-matrix.yml

@@ -335,6 +335,7 @@ jobs:
     with:
       storage_type: replicated
       nested_storageclass_name: nested-thin-r1
+      nested_cluster_network_name: cn-4006-for-e2e-test
       branch: main
       virtualization_tag: main
       deckhouse_channel: alpha
@@ -359,6 +360,7 @@ jobs:
     with:
       storage_type: nfs
       nested_storageclass_name: nfs
+      nested_cluster_network_name: cn-4006-for-e2e-test
       branch: main
       virtualization_tag: main
       deckhouse_channel: alpha

.github/workflows/e2e-reusable-pipeline.yml

@@ -93,6 +93,11 @@ on:
         type: string
         default: ""
         description: "E2E tests focus tests like 'VirtualMachineConfiguration' and so on (by default all tests are run)"
+      nested_cluster_network_name:
+        required: false
+        type: string
+        default: "cn-4006-for-e2e-test"
+        description: "ClusterNetwork name for nested VM additional network interface"
     secrets:
       DEV_REGISTRY_DOCKER_CFG:
         required: true
@@ -235,6 +240,8 @@ jobs:
                   size: ${{ inputs.cluster_config_workers_memory }}
                 additionalDisks:
                 - size: 50Gi
+          networkConfig:
+            clusterNetworkName: ${{ inputs.nested_cluster_network_name }}
           EOF
 
           mkdir -p tmp
@@ -253,12 +260,15 @@ jobs:
       - name: Bootstrap cluster [dhctl-bootstrap]
         id: dhctl-bootstrap
         working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}
-        env:
-          # Proxy settings will be added to values.yaml if proxyEnabled is true via task render-cluster-config-proxy
-          HTTP_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }}
-          HTTPS_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }}
         run: |
+          if [[ $(yq eval '.deckhouse.proxyEnabled' values.yaml) == true ]]; then
+            export HTTP_PROXY="${{ secrets.BOOTSTRAP_DEV_PROXY }}"
+            export HTTPS_PROXY="${{ secrets.BOOTSTRAP_DEV_PROXY }}"
+            echo "Proxy settings - configured"
+          fi
+
           task dhctl-bootstrap
+          echo "[SUCCESS] Done"
         timeout-minutes: 30
       - name: Bootstrap cluster [show-connection-info]
         working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}
@@ -432,10 +442,171 @@ jobs:
           include-hidden-files: true
           retention-days: 3
 
+  configure-sdn:
+    name: Configure SDN
+    runs-on: ubuntu-latest
+    needs: bootstrap
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Task
+        uses: arduino/setup-task@v2
+        with:
+          version: 3.x
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup d8
+        uses: ./.github/actions/install-d8
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install kubectl CLI
+        uses: azure/setup-kubectl@v4
+
+      - name: Check nested kube-api via generated kubeconfig
+        run: |
+          mkdir -p ~/.kube
+          echo "[INFO] Configure kubeconfig for nested cluster"
+          echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config
+
+          echo "[INFO] Show paths and files content"
+          ls -la ~/.kube
+          echo "[INFO] Set permissions for kubeconfig"
+          chmod 600 ~/.kube/config
+
+          echo "[INFO] Show current kubeconfig context"
+          kubectl config get-contexts
+
+          echo "[INFO] Show nodes in cluster"
+          # `kubectl get nodes` may return error, so we need to retry.
+          count=30
+          success=false
+          for i in $(seq 1 $count); do
+            echo "[INFO] Attempt $i/$count..."
+            if kubectl get nodes; then
+              echo "[SUCCESS] Successfully retrieved nodes."
+              success=true
+              break
+            fi
+
+            if [ $i -lt $count ]; then
+              echo "[INFO] Retrying in 10 seconds..."
+              sleep 10
+            fi
+          done
+
+          if [ "$success" = false ]; then
+            echo "[ERROR] Failed to retrieve nodes after $count attempts."
+            exit 1
+          fi
+      - name: Enable SDN
+        run: |
+          echo "[INFO] Enable SDN"
+          d8 system module enable sdn
+          echo "[INFO] Wait for sdn modules to be ready, timeout: 300s"
+          kubectl wait --for=jsonpath='{.status.phase}'=Ready modules sdn --timeout=300s
+          echo "[INFO] Wait for sdn deployments to be ready, timeout: 300s"
+          kubectl -n d8-sdn wait --for=condition=Available deploy --all --timeout 300s
+          echo "[INFO] Wait for sdn daemonset agent to be ready, timeout: 300s"
+          kubectl -n d8-sdn rollout status daemonset agent --timeout=300s
+          echo "[SUCCESS] Done"
+
+      - name: Wait for nodenetworkinterfaces to be ready
+        run: |
+          count=60
+          success=false
+          wait_time_seconds=5
+
+          for i in $(seq 1 $count); do
+            nodes=$(kubectl get nodes -o name | wc -l)
+            actual=$(kubectl get nodenetworkinterfaces -o name 2>/dev/null | wc -l) || true
+            expected=$((nodes * 2))
+
+            echo "[INFO] Attempt $i/$count: expected=$expected, actual=$actual"
+
+            if [ "$actual" -ge "$expected" ]; then
+              echo "[SUCCESS] All nodenetworkinterfaces are present (expected=$expected, actual=$actual)"
+              kubectl get nodenetworkinterfaces
+              success=true
+              break
+            fi
+
+            if (( i % 5 == 0 )) ; then
+              echo ::group::📝 [DEBUG] show namespaces d8-sdn
+              kubectl -n d8-sdn get pods || true
+              echo ::endgroup::
+
+              echo ::group::📝 [DEBUG] show nodenetworkinterfaces d8-sdn
+              kubectl get nodenetworkinterfaces || true
+              echo ::endgroup::
+
+              echo "[INFO] Retrying in 10 seconds..."
+              sleep $wait_time_seconds
+            elif [ $i -lt $count ]; then
+              echo "[INFO] Retrying in 10 seconds..."
+              sleep $wait_time_seconds
+            fi
+          done
+
+          if [ "$success" = false ]; then
+            echo "[ERROR] Failed to get all nodenetworkinterfaces after $count attempts (expected=$expected)"
+            echo "[DEBUG] Show namespaces d8-sdn"
+            kubectl -n d8-sdn get pods || true
+            echo "[DEBUG] Show nodenetworkinterfaces d8-sdn"
+            kubectl get nodenetworkinterfaces || true
+            exit 1
+          fi
+
+      - name: Configure ClusterNetwork
+        run: |
+          extraNic=$(kubectl get nodenetworkinterfaces -l network.deckhouse.io/interface-type=NIC -o json | jq -r '.items[] | select(.status.operationalState == "Up") | select(.status.ifName != "eno1" and .status.ifName != "enp1s0") | .metadata.name')
+
+          for nic in $extraNic; do
+            echo "[INFO] Label nodenetworkinterface $nic nic-group=extra"
+            kubectl label nodenetworkinterfaces $nic nic-group=extra
+          done
+
+          kubectl get nodenetworkinterface -l nic-group=extra
+
+          cat <<'EOF' | kubectl apply -f -
+          ---
+          apiVersion: network.deckhouse.io/v1alpha1
+          kind: ClusterNetwork
+          metadata:
+            name: cn-4006-for-e2e-test
+          spec:
+            parentNodeNetworkInterfaces:
+              labelSelector:
+                matchLabels:
+                  nic-group: extra
+            type: Access
+          ---
+          apiVersion: network.deckhouse.io/v1alpha1
+          kind: ClusterNetwork
+          metadata:
+            name: cn-4007-for-e2e-test
+          spec:
+            parentNodeNetworkInterfaces:
+              labelSelector:
+                matchLabels:
+                  nic-group: extra
+            type: VLAN
+            vlan:
+              id: 4007
+          EOF
+
+          echo "[INFO] Wait for ClusterNetwork cn-4006-for-e2e-test to be ready"
+          kubectl wait clusternetworks.network.deckhouse.io --for=condition=Ready cn-4006-for-e2e-test --timeout=120s
+
+          echo "[INFO] Wait for ClusterNetwork cn-4007-for-e2e-test to be ready"
+          kubectl wait clusternetworks.network.deckhouse.io --for=condition=Ready cn-4007-for-e2e-test --timeout=120s
+
   configure-storage:
     name: Configure storage
     runs-on: ubuntu-latest
-    needs: bootstrap
+    needs:
+      - configure-sdn
+      - bootstrap
     steps:
       - uses: actions/checkout@v4
 
@@ -1338,10 +1509,11 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - bootstrap
+      - configure-sdn
       - configure-storage
       - configure-virtualization
       - e2e-test
-    if: cancelled() || success()
+    if: (cancelled() || success()) && (needs.configure-sdn.result == 'success')
     steps:
       - uses: actions/checkout@v4
 

test/dvp-static-cluster/Taskfile.yaml

@@ -116,7 +116,7 @@ tasks:
         sh: date +%s
     cmds:
       - kubectl apply -f {{ .TMP_DIR }}/infra.yaml
-      - kubectl -n {{ .NAMESPACE }} get all
+      - kubectl -n {{ .NAMESPACE }} get vm,vd,vi,svc,pod,deploy
       - kubectl -n {{ .NAMESPACE }} wait --for=condition=Ready pod -l app=jump-host --timeout=300s
       - kubectl -n {{ .NAMESPACE }} get vi -o name | xargs kubectl -n {{ .NAMESPACE }} wait --for='jsonpath={.status.phase}=Ready' --timeout=600s
       - kubectl -n {{ .NAMESPACE }} get vd -o name | xargs kubectl -n {{ .NAMESPACE }} wait --for='jsonpath={.status.phase}=Ready' --timeout=600s
@@ -140,7 +140,7 @@ tasks:
     desc: Add proxy if enabled
     cmds:
       - |
-        if yq eval '.deckhouse.proxyEnabled' values.yaml; then
+        if [[ $(yq eval '.deckhouse.proxyEnabled' values.yaml) == true ]]; then
           yq eval --inplace '.proxy.httpProxy = env(HTTP_PROXY) | .proxy.httpsProxy = env(HTTPS_PROXY)' values.yaml
         fi
 

test/dvp-static-cluster/charts/cluster-config/templates/master-nodes.yaml

@@ -4,6 +4,17 @@
   {{- $totalNodes = add $totalNodes .count -}}
 {{- end -}}
 
+{{/*
+  CAPS (Cluster API Static Provider) manages static nodes, but during bootstrap it may fail
+  to take control of the first master node, which can break the bootstrap process.
+
+  To avoid this, the first master (index 0) — the one that was bootstrapped initially —
+  is intentionally excluded from CAPS management. Only masters with index >= 1 are registered
+  as StaticInstance resources and counted in the NodeGroup.
+*/}}
+{{- $masterCount := $.Values.instances.masterNodes.count | int -}}
+{{- if gt $masterCount 1 -}}
+  {{- $staticCount := sub $masterCount 1 -}}
 ---
 apiVersion: deckhouse.io/v1
 kind: NodeGroup
@@ -24,27 +35,24 @@ spec:
       node-role.kubernetes.io/master: ""
   nodeType: Static
   staticInstances:
-    count: {{ .Values.instances.masterNodes.count }}
+    count: {{ $staticCount }}
     labelSelector:
       matchLabels:
         role: master
 
-{{- range $_, $i := untilStep 0 (.Values.instances.masterNodes.count | int) 1}}
+{{- range $_, $i := untilStep 1 $masterCount 1}}
   {{- $vmName := printf "%s-master-%d" $.Values.storageType $i }}
 ---
 apiVersion: deckhouse.io/v1alpha1
 kind: StaticInstance
 metadata:
-  {{- if eq $i 0 }}
-  annotations:
-    static.node.deckhouse.io/skip-bootstrap-phase: ""
-  {{- end }}
   name: {{ $vmName }}
   labels:
     role: master
 spec:
   address: {{ index $.Values.discovered.vmIPs $vmName }}
   credentialsRef:
     kind: SSHCredentials
-    name: mvp-static
+    name: dvp-stataic
+{{- end }}
 {{- end }}

test/dvp-static-cluster/charts/cluster-config/templates/nodes.yaml

@@ -38,7 +38,7 @@ spec:
   address: {{ index $.Values.discovered.vmIPs $vmName }}
   credentialsRef:
     kind: SSHCredentials
-    name: mvp-static
+    name: dvp-stataic
   {{- end }}
 
 {{- end }}

test/dvp-static-cluster/charts/cluster-config/templates/ssh-creds.yaml

@@ -2,7 +2,7 @@
 apiVersion: deckhouse.io/v1alpha2
 kind: SSHCredentials
 metadata:
-  name: mvp-static
+  name: dvp-stataic
 spec:
   user: cloud
   privateSSHKey: {{ .Values.discovered.privateSSHKeyBase64 }}

test/dvp-static-cluster/charts/infra/templates/_helpers.tpl

@@ -16,6 +16,7 @@ group: {{ $prefix }}
 {{- $ctx := index . 0 -}}
 {{- $name := index . 1 -}}
 {{- $cfg := index . 2 -}}
+{{- $clusterNetworkName := dig $ctx.Values "networkConfig" "clusterNetworkName" "" -}}
 ---
 apiVersion: virtualization.deckhouse.io/v1alpha2
 kind: VirtualMachine
@@ -33,6 +34,11 @@ spec:
   - kind: VirtualDisk
     name: {{ printf "%s-%d" $name $i }}
 {{- end }}
+  networks:
+    - type: Main
+{{- if $clusterNetworkName }}
+    - type: ClusterNetwork
+      name: {{ $clusterNetworkName }}
 {{- end }}
   bootloader: {{ $ctx.Values.image.bootloader }}
   liveMigrationPolicy: PreferForced
@@ -51,6 +57,18 @@ spec:
       #cloud-config
       ssh_pwauth: true
       package_update: true
+      write_files:
+        - path: /etc/netplan/99-eno2.yaml
+          content: |
+            network:
+              version: 2
+              ethernets:
+                eno2:
+                  dhcp4: false
+                  dhcp6: false
+                  addresses: []
+                  link-local: []
+                  optional: true
       packages:
         - qemu-guest-agent
         - jq
@@ -68,6 +86,8 @@ spec:
             - {{ $ctx.Values.discovered.publicSSHKey }}
 
       runcmd:
+        - netplan apply
+        - ip link set eno2 up
         - systemctl enable --now qemu-guest-agent.service
       final_message: "\U0001F525\U0001F525\U0001F525 The system is finally up, after $UPTIME seconds \U0001F525\U0001F525\U0001F525"
   runPolicy: AlwaysOn
@@ -108,4 +128,5 @@ spec:
     {{- end }}
   {{- end }}
 {{- end }}
+{{- end }}
 {{- end }}