feat(kubevirt): move virt-handler to hostNetwork, bump 3p-kubevirt fork version

build/components/versions.yml

@@ -3,7 +3,7 @@ firmware:
   libvirt: v10.9.0
   edk2: stable202411
 core:
-  3p-kubevirt: v1.6.2-v12n.24
+  3p-kubevirt: feat/virt-handler-to-hostnetwork # v1.6.2-v12n.24
   3p-containerized-data-importer: v1.60.3-v12n.18
   distribution: 2.8.3
 package:

images/virt-artifact/werf.inc.yaml

@@ -9,6 +9,7 @@
 image: {{ .ModuleNamePrefix }}{{ .ImageName }}-src-artifact
 final: false
 fromImage: builder/src
+fromCacheVersion: "020" # TODO: DELETE ME
 secrets:
 - id: SOURCE_REPO
   value: {{ $.SOURCE_REPO }}

templates/_hostnetwork_ports.tpl

@@ -0,0 +1,47 @@
+{{- /*
+Port constants for DaemonSets running with hostNetwork: true.
+
+All three DaemonSets — virt-handler, vm-route-forge, virtualization-dra —
+run with hostNetwork, so every bound port is exposed on the node's network
+interfaces. Ports below are chosen outside the KubeVirt live-migration range
+(4135-4199) and must not overlap with other well-known services on cluster nodes.
+
+Port map:
+
+  virt-handler (kube-api-rewriter runs as its sidecar):
+  4135-4199  virt-handler: live-migration tunnels (KubeVirt migration range).
+  4100       virt-handler: healthz and Prometheus metrics (--port flag), kube-rbac-proxy implemented natively.
+  4101       virt-handler: Console server port (--console-server-port flag).
+  4102       kube-api-rewriter sidecar: Prometheus metrics (MONITORING_BIND_ADDRESS), bound to pod IP.
+             liveness and readiness probes (/proxy/healthz, /proxy/readyz).
+  4103       kube-api-rewriter sidecar: pprof (PPROF_BIND_ADDRESS), bound to pod IP, debug mode only.
+  4104       kube-api-rewriter sidecar: Kubernetes API proxy (CLIENT_PROXY_PORT),
+             virt-handler connects here instead of the real API server.
+
+  vm-route-forge:
+  4105       vm-route-forge: liveness and readiness probes (HEALTH_PROBE_BIND_ADDRESS).
+  4106       vm-route-forge: pprof (PPROF_BIND_ADDRESS), debug mode only.
+
+  virtualization-dra:
+  4107       virtualization-dra: gRPC liveness and readiness probes.
+  4280       virtualization-dra: USB/IP daemon (--usbipd-port flag).
+*/ -}}
+
+{{- /* virt-handler */ -}}
+{{- define "virt_handler.migration_port_first" -}}4135{{- end -}}
+{{- define "virt_handler.migration_port_last" -}}4199{{- end -}}
+
+{{- define "virt_handler.port" -}}4100{{- end -}}
+{{- define "virt_handler.console_server_port" -}}4101{{- end -}}
+{{- define "virt_handler.rewriter_healthz_port" -}}4102{{- end -}}
+{{- define "virt_handler.rewriter_monitoring_port" -}}4102{{- end -}}
+{{- define "virt_handler.rewriter_pprof_port" -}}4103{{- end -}}
+{{- define "virt_handler.rewriter_proxy_port" -}}4104{{- end -}}
+
+{{- /* vm-route-forge */ -}}
+{{- define "vm_route_forge.health_port" -}}4105{{- end -}}
+{{- define "vm_route_forge.pprof_port" -}}4106{{- end -}}
+
+{{- /* virtualization-dra */ -}}
+{{- define "virtualization_dra.health_port" -}}4107{{- end -}}
+{{- define "virtualization_dra.usbipd_port" -}}4280{{- end -}}

templates/kube-api-rewriter/_customize_patch_helpers.tpl

@@ -30,7 +30,7 @@ spec:
       {{- include "kube_api_rewriter.sidecar_container" (tuple $ctx $settings) | nindent 6 }}
       - name: {{ $mainContainerName }}
         env:
-        {{- include "kube_api_rewriter.kubeconfig_env" . | nindent 8 }}
+        {{- include "kube_api_rewriter.kubeconfig_env" (tuple $ctx $settings) | nindent 8 }}
         volumeMounts:
         {{- include "kube_api_rewriter.kubeconfig_volume_mount" . | nindent 8 }}
 {{- end -}}

templates/kube-api-rewriter/_settings.tpl

@@ -7,13 +7,11 @@
 
 {{- define "kube_api_rewriter.pprof_port" -}}8129{{- end -}}
 
+{{- define "kube_api_rewriter.client_proxy_port" -}}23915{{- end -}}
+
 {{- define "kube_api_rewriter.env" -}}
 - name: LOG_LEVEL
   value: {{ include "moduleLogLevel" . }}
-{{- if eq (include "moduleLogLevel" .) "debug" }}
-- name: PPROF_BIND_ADDRESS
-  value: ":{{ include "kube_api_rewriter.pprof_port" . }}"
-{{- end }}
 {{- end -}}
 
 {{- define "kube_api_rewriter.resources" -}}

templates/kube-api-rewriter/_sidecar_helpers.tpl

@@ -1,98 +1,34 @@
-{{- /* Helpers to add kube-api-rewriter sidecar container to a pod.
-
-To connect to kube-api-rewriter main controller should has KUBECONFIG env,
-volumeMount with kubeconfig, and Pod should has volume with kubeconfig ConfigMap.
-
-These settings are provided by helpers:
-
-- kube_api_rewriter.kubeconfig_env defines KUBECONFIG env with file from the
-  mounted ConfigMap.
-- kube_api_rewriter.kubeconfig_volume_mount defines volumeMount for kubeconfig ConfigMap.
-- kube_api_rewriter.kubeconfig_volume defines volume with kubeconfig ConfigMap.
-
-Kube-api-rewriter sidecar should be the first container in the Pod, to
-main controller not fail on start.
-
-Kube-api-rewriter sidecar works in 2 modes: without webhook or with webhook rewriting.
-
-Sidecar without webhook is the simplest one:
-
-spec:
-  template:
-    spec:
-      containers:
-        {{ include "kube_api_rewriter.sidecar_container" . | nindent 8 }}
-        - name: main-controller
-          ...
-          env:
-            {{- include "kube_api_rewriter.kubeconfig_env" . | nindent 12 }}
-            ...
-          volumeMounts:
-            {{- include "kube_api_rewriter.kubeconfig_volume_mount" . | nindent 12 }}
-            ...
-      volumes:
-        {{- include "kube_api_rewriter.kubeconfig_volume" | nindent 8 }}
-        ...
-
-
-Webhook mode requires additional settings:
-
-- WEBHOOK_ADDRESS - address of the webhook in the main controller
-- WEBHOOK_CERT_FILE - path to the webhook certificate file.
-- WEBHOOK_KEY_FILE - path to the webhook key file.
-- webhookCertsVolumeName - name of the Pod volume with webhook certificates.
-- webhookCertsMountPath - path to mount the webhook certificates.
-
-The assumption here is that main controller has a webhook server and
-certificates are already mounted in the Pod, so kube-api-rewriter
-can use certificates from that volume to impersonate the webhook server.
-
-Example of adding kube-api-rewriter to the Deployment:
-
-spec:
-  template:
-    spec:
-      containers:
-      {{- $rewriterSettings := dict }}
-      {{- $_ := set $rewriterSettings "WEBHOOK_ADDRESS" "https://127.0.0.1:6443" }}
-      {{- $_ := set $rewriterSettings "WEBHOOK_CERT_FILE" "/etc/webhook-certificates/tls.crt" }}
-      {{- $_ := set $rewriterSettings "WEBHOOK_KEY_FILE" "/etc/webhook-certificates/tls.key" }}
-      {{- $_ := set $rewriterSettings "webhookCertsVolumeName" "webhook-certs" }}
-      {{- $_ := set $rewriterSettings "webhookCertsMountPath" "/etc/webhook-certificates" }}
-      {{- include "kube_api_rewriter.sidecar_container" (tuple . $rewriterSettings) | nindent 6 }}
-        - name: main-controller
-          ...
-          env:
-            {{- include "kube_api_rewriter.kubeconfig_env" . | nindent 12 }}
-            ...
-          ports:
-            - containerPort: 6443  # Goes to the WEBHOOK_ADDRESS
-              name: webhooks
-              protocol: TCP
-          volumeMounts:
-            {{- include "kube_api_rewriter.kubeconfig_volume_mount" . | nindent 12 }}
-            - name: webhook-certs
-              mountPath: /etc/webhook-certificates  # Goes to the webhookCertsMountPath
-              readOnly: true
-            ...
-      volumes:
-        {{- include "kube_api_rewriter.kubeconfig_volume" | nindent 8 }}
-        - name: webhook-certs  # Name of the existing volume goes to the webhookCertsVolumeName.
-          secret:
-            optional: true
-            secretName: webhook-certs
-        ...
-
- */ -}}
-
+{{- /*
+Helpers to add the kube-api-rewriter sidecar to a Pod.
+
+The main container must use kube-api-rewriter via a kubeconfig mounted from the
+ConfigMap exposed by these helpers:
+- kube_api_rewriter.kubeconfig_env
+- kube_api_rewriter.kubeconfig_volume_mount
+- kube_api_rewriter.kubeconfig_volume
+
+The sidecar supports two modes:
+- plain API proxying
+- webhook rewriting, when WEBHOOK_* settings and certificate mounts are passed
+
+The sidecar should be placed before the main container, so the main container
+does not start before the local API proxy is ready.
+*/ -}}
 {{- define "kube_api_rewriter.image" -}}
 {{- include "helm_lib_module_image" (list . "kubeApiRewriter") | toJson -}}
 {{- end -}}
 
-
+{{- /* KUBECONFIG for the main container pointing to the local kube-api-rewriter proxy. */ -}}
 {{- define "kube_api_rewriter.kubeconfig_env" -}}
+{{- $settings := dict -}}
+{{- if (kindIs "slice" .) -}}
+{{-   if ge (len .) 2 -}}
+{{-     $settings = index . 1 -}}
+{{-   end -}}
+{{- end -}}
+{{- $kubeconfigFilename := $settings.kubeconfigFilename | default "kube-api-rewriter.kubeconfig" -}}
 - name: KUBECONFIG
-  value: /kubeconfig.local/kube-api-rewriter.kubeconfig
+  value: /kubeconfig.local/{{ $kubeconfigFilename }}
 {{- end }}
 
 {{- define "kube_api_rewriter.kubeconfig_volume" -}}
@@ -107,7 +43,6 @@ spec:
   mountPath: /kubeconfig.local
 {{- end }}
 
-
 {{- define "kube_api_rewriter.webhook_volume_mount" -}}
 {{- $volumeName := index . 0 -}}
 {{- $mountPath := index . 1 -}}
@@ -122,16 +57,20 @@ spec:
   protocol: TCP
 {{- end }}
 
-{{- /* Container port for the pprof server */ -}}
+{{- /* Container port for the pprof server. */ -}}
 {{- define "kube_api_rewriter.pprof_container_port" -}}
 - containerPort: {{ include "kube_api_rewriter.pprof_port" . }}
   name: pprof
   protocol: TCP
 {{- end }}
 
-{{- /* Sidecar container spec with kube-api-rewriter */ -}}
-{{- /* Usage without the webhook proxy: {{ include kube_api_rewriter.sidecar_container . }} */ -}}
-{{- /* Usage with the webhook: {{ include kube_api_rewriter.sidecar_container (tuple . $webhookSettings) }} */ -}}
+{{- /*
+Sidecar container spec with kube-api-rewriter.
+
+Usage:
+- {{ include "kube_api_rewriter.sidecar_container" . }}
+- {{ include "kube_api_rewriter.sidecar_container" (tuple . $settings) }}
+*/ -}}
 {{- define "kube_api_rewriter.sidecar_container" -}}
   {{- $ctx := . -}}
   {{- $settings := dict -}}
@@ -142,6 +81,15 @@ spec:
   {{-   end -}}
   {{- end -}}
   {{- $isWebhook := hasKey $settings "WEBHOOK_ADDRESS" -}}
+  {{- $injectPodIP := $settings.injectPodIP | default false -}}
+  {{- $healthzPort := $settings.healthzPort | default 8082 -}}
+  {{- $healthzPath := $settings.healthzPath | default "/proxy/healthz" -}}
+  {{- $readyzPath := $settings.readyzPath | default "/proxy/readyz" -}}
+  {{- $clientProxyPort := $settings.clientProxyPort | default (include "kube_api_rewriter.client_proxy_port" $ctx | int) -}}
+  {{- $monitoringBindAddress := $settings.monitoringBindAddress | default "127.0.0.1:9090" -}}
+  {{- $pprofBindAddress := $settings.pprofBindAddress | default (printf ":%s" (include "kube_api_rewriter.pprof_port" $ctx)) -}}
+  {{- $pprofPort := last (splitList ":" $pprofBindAddress) | int -}}
+  {{- $probeScheme := $settings.probeScheme | default "HTTPS" -}}
 - name: {{ include "kube_api_rewriter.sidecar_name" $ctx }}
   image: {{ include "kube_api_rewriter.image" $ctx }}
   imagePullPolicy: IfNotPresent
@@ -154,8 +102,24 @@ spec:
     - name: WEBHOOK_KEY_FILE
       value: "{{ $settings.WEBHOOK_KEY_FILE }}"
     {{- end }}
+    {{- if $injectPodIP }}
+    - name: POD_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.podIP
+    {{- end }}
+    - name: CLIENT_PROXY_PORT
+      value: "{{ $clientProxyPort }}"
     - name: MONITORING_BIND_ADDRESS
-      value: "127.0.0.1:9090"
+      value: "{{ $monitoringBindAddress }}"
+    {{- if $settings.monitoringAuth }}
+    - name: MONITORING_AUTH
+      value: {{ $settings.monitoringAuth | toJson | quote }}
+    {{- end }}
+    {{- if eq (include "moduleLogLevel" $ctx) "debug" }}
+    - name: PPROF_BIND_ADDRESS
+      value: "{{ $pprofBindAddress }}"
+    {{- end }}
     {{- include "kube_api_rewriter.env" $ctx | nindent 4 }}
   resources:
     requests:
@@ -173,15 +137,15 @@ spec:
       type: RuntimeDefault
   livenessProbe:
     httpGet:
-      path: /proxy/healthz
-      port: 8082
-      scheme: HTTPS
+      path: {{ $healthzPath }}
+      port: {{ $healthzPort }}
+      scheme: {{ $probeScheme }}
     initialDelaySeconds: 10
   readinessProbe:
     httpGet:
-      path: /proxy/readyz
-      port: 8082
-      scheme: HTTPS
+      path: {{ $readyzPath }}
+      port: {{ $healthzPort }}
+      scheme: {{ $probeScheme }}
     initialDelaySeconds: 10
   terminationMessagePath: /dev/termination-log
   terminationMessagePolicy: File
@@ -191,9 +155,13 @@ spec:
   {{- end }}
   ports:
   {{- if eq (include "moduleLogLevel" $ctx) "debug" }}
-  {{-   include "kube_api_rewriter.pprof_container_port" . | nindent 4 }}
+  - containerPort: {{ $pprofPort }}
+    name: pprof
+    protocol: TCP
   {{- end }}
-  {{- if $isWebhook -}}
-  {{-   include "kube_api_rewriter.webhook_container_port" .| nindent 4 }}
+  {{- if $isWebhook }}
+  - containerPort: {{ include "kube_api_rewriter.webhook_port" $ctx }}
+    name: {{ include "kube_api_rewriter.webhook_port_name" $ctx }}
+    protocol: TCP
   {{- end -}}
 {{- end -}}

templates/kube-api-rewriter/cm-kubeconfig-local.yaml

@@ -18,3 +18,4 @@ data:
           cluster: kube-api-rewriter
         name: kube-api-rewriter
     current-context: kube-api-rewriter
+