feat(ci): add sdn for e2e nested cluster

universal-itengineer · universal-itengineer · commit e50f430aa0c0 · 2026-04-14T10:20:53.000+03:00
Signed-off-by: Nikita Korolev &lt;nikita.korolev@flant.com&gt;
diff --git a/.github/workflows/e2e-matrix.yml b/.github/workflows/e2e-matrix.yml
@@ -16,8 +16,13 @@ name: E2E Matrix Tests (nested clusters)
 
 on:
   workflow_dispatch:
-  schedule:
-    - cron: "40 4 * * *"
+  pull_request:
+    types: [opened, reopened, synchronize, labeled, unlabeled]
+    branches:
+      - main
+      - feat/ci/e2e-nested-add-sdn
+  # schedule:
+  #   - cron: "40 4 * * *"
 
 concurrency:
   group: "${{ github.workflow }}-${{ github.event.number || github.ref }}"
@@ -29,6 +34,7 @@ defaults:
 
 jobs:
   cleanup-nested-clusters:
+    if: github.event_name != 'pull_request'
     name: Cleanup nested clusters
     runs-on: ubuntu-latest
     steps:
@@ -100,6 +106,7 @@ jobs:
           cleanup_kind "vmclass"
 
   power-off-vms-for-nested:
+    if: github.event_name != 'pull_request'
     name: Power off VMs for nested clusters
     needs: cleanup-nested-clusters
     runs-on: ubuntu-latest
@@ -315,7 +322,7 @@ jobs:
           fi
   set-vars:
     name: Set vars
-    needs: power-off-vms-for-nested
+    # needs: power-off-vms-for-nested
     runs-on: ubuntu-latest
     outputs:
       date_start: ${{ steps.vars.outputs.date-start }}
@@ -345,13 +352,15 @@ jobs:
       randuuid4c: ${{ needs.set-vars.outputs.randuuid4c }}
       cluster_config_workers_memory: "9Gi"
       cluster_config_k8s_version: "1.34"
+      e2e_focus_tests: "VirtualMachineAdditionalNetworkInterfaces"
     secrets:
       DEV_REGISTRY_DOCKER_CFG: ${{ secrets.DEV_REGISTRY_DOCKER_CFG }}
       VIRT_E2E_NIGHTLY_SA_TOKEN: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }}
       PROD_IO_REGISTRY_DOCKER_CFG: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }}
       BOOTSTRAP_DEV_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }}
 
   e2e-nfs:
+    if: github.event_name != 'pull_request'
     name: E2E Pipeline (NFS)
     needs:
       - set-vars
@@ -380,7 +389,7 @@ jobs:
     name: End-to-End tests report
     needs:
       - e2e-replicated
-      - e2e-nfs
+      # - e2e-nfs
     if: ${{ always()}}
     env:
       STORAGE_TYPES: '["replicated", "nfs"]'
@@ -647,4 +656,5 @@ jobs:
             curl --request POST --header 'Content-Type: application/json' --data "{\"text\": \"${COMBINED_SUMMARY}\"}" "$LOOP_WEBHOOK_URL"
           fi
         env:
-          LOOP_WEBHOOK_URL: ${{ secrets.LOOP_WEBHOOK_URL }}
+          LOOP_WEBHOOK_URL: ${{ secrets.LOOP_TEST_CHANNEL }}
+          # LOOP_WEBHOOK_URL: ${{ secrets.LOOP_WEBHOOK_URL }}
diff --git a/.github/workflows/e2e-reusable-pipeline.yml b/.github/workflows/e2e-reusable-pipeline.yml
@@ -141,7 +141,7 @@ jobs:
         run: |
           GIT_SHORT_HASH=$(git rev-parse --short HEAD)
 
-          namespace="nightly-e2e-$STORAGE_TYPE-$GIT_SHORT_HASH-$RANDUUID4C"
+          namespace="test-sdn-e2e-$STORAGE_TYPE-$GIT_SHORT_HASH-$RANDUUID4C"
 
           echo "namespace=$namespace" >> $GITHUB_OUTPUT
           echo "sha_short=$GIT_SHORT_HASH" >> $GITHUB_OUTPUT
@@ -253,12 +253,15 @@ jobs:
       - name: Bootstrap cluster [dhctl-bootstrap]
         id: dhctl-bootstrap
         working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}
-        env:
-          # Proxy settings will be added to values.yaml if proxyEnabled is true via task render-cluster-config-proxy
-          HTTP_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }}
-          HTTPS_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }}
         run: |
+          if [[ $(yq eval '.deckhouse.proxyEnabled' values.yaml) == true ]]; then
+            export HTTP_PROXY="${{ secrets.BOOTSTRAP_DEV_PROXY }}"
+            export HTTPS_PROXY="${{ secrets.BOOTSTRAP_DEV_PROXY }}"
+            echo "Proxy settings - configured"
+          fi
+
           task dhctl-bootstrap
+          echo "[SUCCESS] Done"
         timeout-minutes: 30
       - name: Bootstrap cluster [show-connection-info]
         working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}
@@ -432,10 +435,171 @@ jobs:
           include-hidden-files: true
           retention-days: 3
 
+  configure-sdn:
+    name: Configure SDN
+    runs-on: ubuntu-latest
+    needs: bootstrap
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Task
+        uses: arduino/setup-task@v2
+        with:
+          version: 3.x
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup d8
+        uses: ./.github/actions/install-d8
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install kubectl CLI
+        uses: azure/setup-kubectl@v4
+
+      - name: Check nested kube-api via generated kubeconfig
+        run: |
+          mkdir -p ~/.kube
+          echo "[INFO] Configure kubeconfig for nested cluster"
+          echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config
+
+          echo "[INFO] Show paths and files content"
+          ls -la ~/.kube
+          echo "[INFO] Set permissions for kubeconfig"
+          chmod 600 ~/.kube/config
+
+          echo "[INFO] Show current kubeconfig context"
+          kubectl config get-contexts
+
+          echo "[INFO] Show nodes in cluster"
+          # `kubectl get nodes` may return error, so we need to retry.
+          count=30
+          success=false
+          for i in $(seq 1 $count); do
+            echo "[INFO] Attempt $i/$count..."
+            if kubectl get nodes; then
+              echo "[SUCCESS] Successfully retrieved nodes."
+              success=true
+              break
+            fi
+
+            if [ $i -lt $count ]; then
+              echo "[INFO] Retrying in 10 seconds..."
+              sleep 10
+            fi
+          done
+
+          if [ "$success" = false ]; then
+            echo "[ERROR] Failed to retrieve nodes after $count attempts."
+            exit 1
+          fi
+      - name: Enable SDN
+        run: |
+          echo "[INFO] Enable SDN"
+          d8 system module enable sdn
+          echo "[INFO] Wait for sdn modules to be ready, timeout: 300s"
+          kubectl wait --for=jsonpath='{.status.phase}'=Ready modules sdn --timeout=300s
+          echo "[INFO] Wait for sdn deployments to be ready, timeout: 300s"
+          kubectl -n d8-sdn wait --for=condition=Available deploy --all --timeout 300s
+          echo "[INFO] Wait for sdn daemonset agent to be ready, timeout: 300s"
+          kubectl -n d8-sdn rollout status daemonset agent --timeout=300s
+          echo "[SUCCESS] Done"
+
+      - name: Wait for nodenetworkinterfaces to be ready
+        run: |
+          count=60
+          success=false
+          wait_time_seconds=5
+
+          for i in $(seq 1 $count); do
+            nodes=$(kubectl get nodes -o name | wc -l)
+            actual=$(kubectl get nodenetworkinterfaces -o name 2>/dev/null | wc -l) || true
+            expected=$((nodes * 2))
+
+            echo "[INFO] Attempt $i/$count: expected=$expected, actual=$actual"
+
+            if [ "$actual" -ge "$expected" ]; then
+              echo "[SUCCESS] All nodenetworkinterfaces are present (expected=$expected, actual=$actual)"
+              kubectl get nodenetworkinterfaces
+              success=true
+              break
+            fi
+
+            if (( i % 5 == 0 )) ; then
+              echo ::group::📝 [DEBUG] show namespaces d8-sdn
+              kubectl -n d8-sdn get pods || true
+              echo ::endgroup::
+              
+              echo ::group::📝 [DEBUG] show nodenetworkinterfaces d8-sdn
+              kubectl get nodenetworkinterfaces || true
+              echo ::endgroup::
+              
+              echo "[INFO] Retrying in 10 seconds..."
+              sleep $wait_time_seconds
+            elif [ $i -lt $count ]; then
+              echo "[INFO] Retrying in 10 seconds..."
+              sleep $wait_time_seconds
+            fi
+          done
+
+          if [ "$success" = false ]; then
+            echo "[ERROR] Failed to get all nodenetworkinterfaces after $count attempts (expected=$expected)"
+            echo "[DEBUG] Show namespaces d8-sdn"
+            kubectl -n d8-sdn get pods || true
+            echo "[DEBUG] Show nodenetworkinterfaces d8-sdn"
+            kubectl get nodenetworkinterfaces || true
+            exit 1
+          fi
+
+      - name: Configure ClusterNetwork
+        run: |
+          extraNic=$(kubectl get nodenetworkinterfaces -l network.deckhouse.io/interface-type=NIC -o json | jq -r '.items[] | select(.status.operationalState == "Up") | select(.status.ifName != "eno1" and .status.ifName != "enp1s0") | .metadata.name')
+
+          for nic in $extraNic; do
+            echo "[INFO] Label nodenetworkinterface $nic nic-group=extra"
+            kubectl label nodenetworkinterfaces $nic nic-group=extra
+          done
+
+          kubectl get nodenetworkinterface -l nic-group=extra
+
+          cat <<'EOF' | kubectl apply -f -
+          ---
+          apiVersion: network.deckhouse.io/v1alpha1
+          kind: ClusterNetwork
+          metadata:
+            name: cn-4006-for-e2e-test
+          spec:
+            parentNodeNetworkInterfaces:
+              labelSelector:
+                matchLabels:
+                  nic-group: extra
+            type: Access
+          ---
+          apiVersion: network.deckhouse.io/v1alpha1
+          kind: ClusterNetwork
+          metadata:
+            name: cn-4007-for-e2e-test
+          spec:
+            parentNodeNetworkInterfaces:
+              labelSelector:
+                matchLabels:
+                  nic-group: extra
+            type: VLAN
+            vlan:
+              id: 4007
+          EOF
+
+          echo "[INFO] Wait for ClusterNetwork cn-4006-for-e2e-test to be ready"
+          kubectl wait clusternetworks.network.deckhouse.io --for=condition=Ready cn-4006-for-e2e-test --timeout=120s
+
+          echo "[INFO] Wait for ClusterNetwork cn-4007-for-e2e-test to be ready"
+          kubectl wait clusternetworks.network.deckhouse.io --for=condition=Ready cn-4007-for-e2e-test --timeout=120s
+
   configure-storage:
     name: Configure storage
     runs-on: ubuntu-latest
-    needs: bootstrap
+    needs:
+      - configure-sdn
+      - bootstrap
     steps:
       - uses: actions/checkout@v4
 
@@ -1144,6 +1308,7 @@ jobs:
 
           echo "[INFO] Exit code: $GINKGO_EXIT_CODE"
           exit $GINKGO_EXIT_CODE
+          # exit 1
       - name: Upload summary test results (junit/xml)
         uses: actions/upload-artifact@v4
         id: e2e-report-artifact
@@ -1338,10 +1503,11 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - bootstrap
+      - configure-sdn
       - configure-storage
       - configure-virtualization
       - e2e-test
-    if: cancelled() || success()
+    if: (cancelled() || success()) && (needs.configure-sdn.result == 'success')
     steps:
       - uses: actions/checkout@v4
 
diff --git a/test/dvp-static-cluster/Taskfile.yaml b/test/dvp-static-cluster/Taskfile.yaml
@@ -116,7 +116,7 @@ tasks:
         sh: date +%s
     cmds:
       - kubectl apply -f {{ .TMP_DIR }}/infra.yaml
-      - kubectl -n {{ .NAMESPACE }} get all
+      - kubectl -n {{ .NAMESPACE }} get vm,vd,vi,svc,pod,deploy
       - kubectl -n {{ .NAMESPACE }} wait --for=condition=Ready pod -l app=jump-host --timeout=300s
       - kubectl -n {{ .NAMESPACE }} get vi -o name | xargs kubectl -n {{ .NAMESPACE }} wait --for='jsonpath={.status.phase}=Ready' --timeout=600s
       - kubectl -n {{ .NAMESPACE }} get vd -o name | xargs kubectl -n {{ .NAMESPACE }} wait --for='jsonpath={.status.phase}=Ready' --timeout=600s
@@ -140,7 +140,7 @@ tasks:
     desc: Add proxy if enabled
     cmds:
       - |
-        if yq eval '.deckhouse.proxyEnabled' values.yaml; then
+        if [[ $(yq eval '.deckhouse.proxyEnabled' values.yaml) == true ]]; then
           yq eval --inplace '.proxy.httpProxy = env(HTTP_PROXY) | .proxy.httpsProxy = env(HTTPS_PROXY)' values.yaml
         fi
 
diff --git a/test/dvp-static-cluster/charts/cluster-config/templates/master-nodes.yaml b/test/dvp-static-cluster/charts/cluster-config/templates/master-nodes.yaml
@@ -4,6 +4,9 @@
   {{- $totalNodes = add $totalNodes .count -}}
 {{- end -}}
 
+{{- $masterCount := $.Values.instances.masterNodes.count | int -}}
+{{- if gt $masterCount 1 -}}
+  {{- $staticCount := sub $masterCount 1 -}}
 ---
 apiVersion: deckhouse.io/v1
 kind: NodeGroup
@@ -24,21 +27,17 @@ spec:
       node-role.kubernetes.io/master: ""
   nodeType: Static
   staticInstances:
-    count: {{ .Values.instances.masterNodes.count }}
+    count: {{ $staticCount }}
     labelSelector:
       matchLabels:
         role: master
 
-{{- range $_, $i := untilStep 0 (.Values.instances.masterNodes.count | int) 1}}
+{{- range $_, $i := untilStep 1 $masterCount 1}}
   {{- $vmName := printf "%s-master-%d" $.Values.storageType $i }}
 ---
 apiVersion: deckhouse.io/v1alpha1
 kind: StaticInstance
 metadata:
-  {{- if eq $i 0 }}
-  annotations:
-    static.node.deckhouse.io/skip-bootstrap-phase: ""
-  {{- end }}
   name: {{ $vmName }}
   labels:
     role: master
@@ -48,3 +47,4 @@ spec:
     kind: SSHCredentials
     name: mvp-static
 {{- end }}
+{{- end }}
diff --git a/test/dvp-static-cluster/charts/infra/templates/_helpers.tpl b/test/dvp-static-cluster/charts/infra/templates/_helpers.tpl
@@ -33,7 +33,10 @@ spec:
   - kind: VirtualDisk
     name: {{ printf "%s-%d" $name $i }}
 {{- end }}
-{{- end }}
+  networks:
+    - type: Main
+    - type: ClusterNetwork
+      name: cn-4006-for-e2e-test
   bootloader: {{ $ctx.Values.image.bootloader }}
   liveMigrationPolicy: PreferForced
   cpu:
@@ -51,6 +54,18 @@ spec:
       #cloud-config
       ssh_pwauth: true
       package_update: true
+      write_files:
+        - path: /etc/netplan/99-eno2.yaml
+          content: |
+            network:
+              version: 2
+              ethernets:
+                eno2:
+                  dhcp4: false
+                  dhcp6: false
+                  addresses: []
+                  link-local: []
+                  optional: true
       packages:
         - qemu-guest-agent
         - jq
@@ -68,6 +83,8 @@ spec:
             - {{ $ctx.Values.discovered.publicSSHKey }}
 
       runcmd:
+        - netplan apply
+        - ip link set eno2 up
         - systemctl enable --now qemu-guest-agent.service
       final_message: "\U0001F525\U0001F525\U0001F525 The system is finally up, after $UPTIME seconds \U0001F525\U0001F525\U0001F525"
   runPolicy: AlwaysOn
@@ -108,4 +125,5 @@ spec:
     {{- end }}
   {{- end }}
 {{- end }}
+{{- end }}
 {{- end }}
