Fix KVM driver (tests) timeouts (#20852)

* Fix KVM driver tests timeouts

Rewrite KVM driver waiting logic for domain start, getting ip address
and shutting domain down. Add more config/state outputs to aid future
debugging.

Bump go/libvirt to v1.11002.0 and set the minimum memory required for
running all tests to 3GB to avoid some really weird system behaviour.

* revert reduction of timelimit for TestCert tests run

* set memory and debug output in TestNoKubernetes tests

* extend kvm waitForStaticIP timeout

* add console log to debug output

* Updating ISO to v1.36.0-1748823857-20852

---------

Co-authored-by: minikube-bot <[email protected]>

Author: prezha

Makefile

@@ -24,7 +24,7 @@ KIC_VERSION ?= $(shell grep -E "Version =" pkg/drivers/kic/types.go | cut -d \"
 HUGO_VERSION ?= $(shell grep -E "HUGO_VERSION = \"" netlify.toml | cut -d \" -f2)
 
 # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions
-ISO_VERSION ?= v1.36.0
+ISO_VERSION ?= v1.36.0-1748823857-20852
 
 # Dashes are valid in semver, but not Linux packaging. Use ~ to delimit alpha/beta
 DEB_VERSION ?= $(subst -,~,$(RAW_VERSION))
@@ -103,7 +103,7 @@ $(shell mkdir -p $(BUILD_DIR))
 CURRENT_GIT_BRANCH ?= $(shell git branch | grep \* | cut -d ' ' -f2)
 
 # Use system python if it exists, otherwise use Docker.
-PYTHON := $(shell command -v python || echo "docker run --rm -it -v $(shell pwd):/minikube:Z -w /minikube python python")
+PYTHON := $(shell command -v python || echo "docker run --rm -it -v $(shell pwd):/minikube -w /minikube python python")
 BUILD_OS := $(shell uname -s)
 
 SHA512SUM=$(shell command -v sha512sum || echo "shasum -a 512")
@@ -189,7 +189,7 @@ endef
 
 # $(call DOCKER, image, command)
 define DOCKER
-	docker run --rm -e GOCACHE=/app/.cache -e IN_DOCKER=1 --user $(shell id -u):$(shell id -g) -w /app -v $(PWD):/app:Z -v $(GOPATH):/go --init $(1) /bin/bash -c '$(2)'
+	docker run --rm -e GOCACHE=/app/.cache -e IN_DOCKER=1 --user $(shell id -u):$(shell id -g) -w /app -v $(PWD):/app -v $(GOPATH):/go --init $(1) /bin/bash -c '$(2)'
 endef
 
 ifeq ($(BUILD_IN_DOCKER),y)
@@ -341,13 +341,13 @@ out/minikube-%.iso: $(shell find "deploy/iso/minikube-iso" -type f)
 ifeq ($(IN_DOCKER),1)
 	$(MAKE) minikube-iso-$*
 else
-	docker run --rm --workdir /mnt --volume $(CURDIR):/mnt:Z $(ISO_DOCKER_EXTRA_ARGS) \
+	docker run --rm --workdir /mnt --volume $(CURDIR):/mnt $(ISO_DOCKER_EXTRA_ARGS) \
 		--user $(shell id -u):$(shell id -g) --env HOME=/tmp --env IN_DOCKER=1 \
 		$(ISO_BUILD_IMAGE) /bin/bash -lc '/usr/bin/make minikube-iso-$*'
 endif
 
 iso_in_docker:
-	docker run -it --rm --workdir /mnt --volume $(CURDIR):/mnt:Z $(ISO_DOCKER_EXTRA_ARGS) \
+	docker run -it --rm --workdir /mnt --volume $(CURDIR):/mnt $(ISO_DOCKER_EXTRA_ARGS) \
 		--user $(shell id -u):$(shell id -g) --env HOME=/tmp --env IN_DOCKER=1 \
 		$(ISO_BUILD_IMAGE) /bin/bash
 
@@ -523,7 +523,7 @@ out/linters/golangci-lint-$(GOLINT_VERSION):
 .PHONY: lint
 ifeq ($(MINIKUBE_BUILD_IN_DOCKER),y)
 lint:
-	docker run --rm -v `pwd`:/app:Z -w /app golangci/golangci-lint:$(GOLINT_VERSION) \
+	docker run --rm -v `pwd`:/app -w /app golangci/golangci-lint:$(GOLINT_VERSION) \
 	golangci-lint run ${GOLINT_OPTIONS} ./..." 
 	# --skip-dirs "cmd/drivers/kvm|cmd/drivers/hyperkit|pkg/drivers/kvm|pkg/drivers/hyperkit" 
 	# The "--skip-dirs" parameter is no longer supported in the V2 version. If you need to skip the directory, 
@@ -657,7 +657,7 @@ out/docker-machine-driver-hyperkit:
 ifeq ($(MINIKUBE_BUILD_IN_DOCKER),y)
 	docker run --rm -e GOCACHE=/app/.cache -e IN_DOCKER=1 \
 		--user $(shell id -u):$(shell id -g) -w /app \
-		-v $(PWD):/app:Z -v $(GOPATH):/go:Z --init --entrypoint "" \
+		-v $(PWD):/app -v $(GOPATH):/go --init --entrypoint "" \
 		$(HYPERKIT_BUILD_IMAGE) /bin/bash -c 'CC=o64-clang CXX=o64-clang++ /usr/bin/make $@'
 else
 	$(if $(quiet),@echo "  GO       $@")

cmd/minikube/cmd/start.go

@@ -1098,8 +1098,8 @@ func suggestMemoryAllocation(sysLimit, containerLimit, nodes int) int {
 		return mem
 	}
 
-	const fallback = 2200
-	maximum := 6000
+	const fallback = 3072
+	maximum := 6144
 
 	if sysLimit > 0 && fallback > sysLimit {
 		return sysLimit

cmd/minikube/cmd/start_test.go

@@ -277,26 +277,26 @@ func TestSuggestMemoryAllocation(t *testing.T) {
 		nodes          int
 		want           int
 	}{
-		{"128GB sys", 128000, 0, 1, 6000},
-		{"64GB sys", 64000, 0, 1, 6000},
-		{"32GB sys", 32768, 0, 1, 6000},
+		{"128GB sys", 128000, 0, 1, 6144},
+		{"64GB sys", 64000, 0, 1, 6144},
+		{"32GB sys", 32768, 0, 1, 6144},
 		{"16GB sys", 16384, 0, 1, 4000},
 		{"odd sys", 14567, 0, 1, 3600},
-		{"4GB sys", 4096, 0, 1, 2200},
+		{"4GB sys", 4096, 0, 1, 3072},
 		{"2GB sys", 2048, 0, 1, 2048},
-		{"Unable to poll sys", 0, 0, 1, 2200},
+		{"Unable to poll sys", 0, 0, 1, 3072},
 		{"128GB sys, 16GB container", 128000, 16384, 1, 16336},
 		{"64GB sys, 16GB container", 64000, 16384, 1, 16000},
 		{"16GB sys, 4GB container", 16384, 4096, 1, 4000},
 		{"4GB sys, 3.5GB container", 16384, 3500, 1, 3452},
 		{"16GB sys, 2GB container", 16384, 2048, 1, 2048},
 		{"16GB sys, unable to poll container", 16384, 0, 1, 4000},
-		{"128GB sys 2 nodes", 128000, 0, 2, 6000},
-		{"8GB sys 3 nodes", 8192, 0, 3, 2200},
-		{"16GB sys 2 nodes", 16384, 0, 2, 2200},
+		{"128GB sys 2 nodes", 128000, 0, 2, 6144},
+		{"8GB sys 3 nodes", 8192, 0, 3, 3072},
+		{"16GB sys 2 nodes", 16384, 0, 2, 3072},
 		{"32GB sys 2 nodes", 32768, 0, 2, 4050},
-		{"odd sys 2 nodes", 14567, 0, 2, 2200},
-		{"4GB sys 2 nodes", 4096, 0, 2, 2200},
+		{"odd sys 2 nodes", 14567, 0, 2, 3072},
+		{"4GB sys 2 nodes", 4096, 0, 2, 3072},
 		{"2GB sys 3 nodes", 2048, 0, 3, 2048},
 	}
 	for _, test := range tests {

deploy/iso/minikube-iso/go.hash

@@ -35,4 +35,4 @@ sha256  36930162a93df417d90bd22c6e14daff4705baac2b02418edda671cdfa9cd07f  go1.23
 sha256  8d6a77332487557c6afa2421131b50f83db4ae3c579c3bc72e670ee1f6968599  go1.23.3.src.tar.gz
 sha256  ad345ac421e90814293a9699cca19dd5238251c3f687980bbcae28495b263531  go1.23.4.src.tar.gz
 sha256  d14120614acb29d12bcab72bd689f257eb4be9e0b6f88a8fb7e41ac65f8556e5  go1.24.0.src.tar.gz
-sha256 6924efde5de86fe277676e929dc9917d466efa02fb934197bc2eba35d5680971 go1.23.4.linux-amd64.tar.gz
+sha256  6924efde5de86fe277676e929dc9917d466efa02fb934197bc2eba35d5680971  go1.23.4.linux-amd64.tar.gz

go.mod

@@ -87,7 +87,7 @@ require (
 	k8s.io/klog/v2 v2.130.1
 	k8s.io/kubectl v0.32.2
 	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738
-	libvirt.org/go/libvirt v1.11001.0
+	libvirt.org/go/libvirt v1.11002.0
 	sigs.k8s.io/sig-storage-lib-external-provisioner/v6 v6.3.0
 )
 

go.sum

@@ -3219,8 +3219,8 @@ k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/
 k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro=
 k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
-libvirt.org/go/libvirt v1.11001.0 h1:QJgpslxY7qkpXZIDxdMHpkDl7FfhgQJwqRTGBbg/S8E=
-libvirt.org/go/libvirt v1.11001.0/go.mod h1:1WiFE8EjZfq+FCVog+rvr1yatKbKZ9FaFMZgEqxEJqQ=
+libvirt.org/go/libvirt v1.11002.0 h1:cb8KJG3D97pc/hxQ2n6P82hRX3rlgdzO7bih6W1AAQ8=
+libvirt.org/go/libvirt v1.11002.0/go.mod h1:1WiFE8EjZfq+FCVog+rvr1yatKbKZ9FaFMZgEqxEJqQ=
 lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
 lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
 modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=

hack/metrics/metrics.go

@@ -145,7 +145,7 @@ func getLabels(containerRuntime string) *stackdriver.Labels {
 func minikubeStartTime(ctx context.Context, projectID, minikubePath, containerRuntime string) (float64, error) {
 	defer deleteMinikube(ctx, minikubePath)
 
-	cmd := exec.CommandContext(ctx, minikubePath, "start", "--driver=docker", "-p", profile, "--memory=2048", "--trace=gcp", fmt.Sprintf("--container-runtime=%s", containerRuntime))
+	cmd := exec.CommandContext(ctx, minikubePath, "start", "--driver=docker", "-p", profile, "--memory=3072", "--trace=gcp", fmt.Sprintf("--container-runtime=%s", containerRuntime))
 	cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%s", pkgtrace.ProjectEnvVar, projectID))
 	cmd.Stdout = os.Stderr
 	cmd.Stderr = os.Stderr

pkg/drivers/kvm/domain.go

@@ -31,12 +31,12 @@ import (
 func (d *Driver) getDomain() (*libvirt.Domain, *libvirt.Connect, error) {
 	conn, err := getConnection(d.ConnectionURI)
 	if err != nil {
-		return nil, nil, errors.Wrap(err, "getting libvirt connection")
+		return nil, nil, fmt.Errorf("failed opening libvirt connection: %w", err)
 	}
 
 	dom, err := conn.LookupDomainByName(d.MachineName)
 	if err != nil {
-		return nil, nil, errors.Wrap(err, "looking up domain")
+		return nil, nil, fmt.Errorf("failed looking up domain: %w", lvErr(err))
 	}
 
 	return dom, conn, nil
@@ -45,13 +45,17 @@ func (d *Driver) getDomain() (*libvirt.Domain, *libvirt.Connect, error) {
 func getConnection(connectionURI string) (*libvirt.Connect, error) {
 	conn, err := libvirt.NewConnect(connectionURI)
 	if err != nil {
-		return nil, errors.Wrap(err, "connecting to libvirt socket")
+		return nil, fmt.Errorf("failed connecting to libvirt socket: %w", lvErr(err))
 	}
 
 	return conn, nil
 }
 
 func closeDomain(dom *libvirt.Domain, conn *libvirt.Connect) error {
+	if dom == nil {
+		return fmt.Errorf("nil domain, cannot close")
+	}
+
 	if err := dom.Free(); err != nil {
 		return err
 	}
@@ -62,25 +66,31 @@ func closeDomain(dom *libvirt.Domain, conn *libvirt.Connect) error {
 	return err
 }
 
-func (d *Driver) createDomain() (*libvirt.Domain, error) {
-	// create the XML for the domain using our domainTmpl template
+// defineDomain defines the XML for the domain using our domainTmpl template
+func (d *Driver) defineDomain() (*libvirt.Domain, error) {
 	tmpl := template.Must(template.New("domain").Parse(domainTmpl))
 	var domainXML bytes.Buffer
-	if err := tmpl.Execute(&domainXML, d); err != nil {
+	dlog := struct {
+		Driver
+		ConsoleLogPath string
+	}{
+		Driver:         *d,
+		ConsoleLogPath: consoleLogPath(*d),
+	}
+	if err := tmpl.Execute(&domainXML, dlog); err != nil {
 		return nil, errors.Wrap(err, "executing domain xml")
 	}
 	conn, err := getConnection(d.ConnectionURI)
 	if err != nil {
-		return nil, errors.Wrap(err, "getting libvirt connection")
+		return nil, fmt.Errorf("failed opening libvirt connection: %w", err)
 	}
 	defer func() {
 		if _, err := conn.Close(); err != nil {
-			log.Errorf("unable to close libvirt connection: %v", err)
+			log.Errorf("failed closing libvirt connection: %v", lvErr(err))
 		}
 	}()
 
-	log.Infof("define libvirt domain using xml: %v", domainXML.String())
-	// define the domain in libvirt using the generated XML
+	log.Infof("defining domain using XML: %v", domainXML.String())
 	dom, err := conn.DomainDefineXML(domainXML.String())
 	if err != nil {
 		return nil, errors.Wrapf(err, "error defining domain xml: %s", domainXML.String())

pkg/drivers/kvm/domain_definition_arm64.go

@@ -27,16 +27,16 @@ const domainTmpl = `
     <acpi/>
     <apic/>
     <pae/>
-    {{if .Hidden}}
+    {{- if .Hidden}}
     <kvm>
       <hidden state='on'/>
     </kvm>
-    {{end}}
+    {{- end}}
   </features>
   <cpu mode='host-passthrough'>
-  {{if gt .NUMANodeCount 1}}
+  {{- if gt .NUMANodeCount 1}}
   {{.NUMANodeXML}}
-  {{end}}
+  {{- end}}
   </cpu>
   <os>
     <type machine='virt-4.2' arch='aarch64'>hvm</type>
@@ -75,12 +75,12 @@ const domainTmpl = `
     <rng model='virtio'>
       <backend model='random'>/dev/random</backend>
     </rng>
-    {{if .GPU}}
+    {{- if .GPU}}
     {{.DevicesXML}}
-    {{end}}
-    {{if gt .ExtraDisks 0}}
+    {{- end}}
+    {{- if gt .ExtraDisks 0}}
     {{.ExtraDisksXML}}
-    {{end}}
+    {{- end}}
   </devices>
 </domain>
 `

pkg/drivers/kvm/domain_definition_x86.go

@@ -27,16 +27,16 @@ const domainTmpl = `
     <acpi/>
     <apic/>
     <pae/>
-    {{if .Hidden}}
+    {{- if .Hidden}}
     <kvm>
       <hidden state='on'/>
     </kvm>
-    {{end}}
+    {{- end}}
   </features>
   <cpu mode='host-passthrough'>
-  {{if gt .NUMANodeCount 1}}
+  {{- if gt .NUMANodeCount 1}}
   {{.NUMANodeXML}}
-  {{end}}
+  {{- end}}
   </cpu>
   <os>
     <type>hvm</type>
@@ -55,6 +55,7 @@ const domainTmpl = `
       <source file='{{.DiskPath}}'/>
       <target dev='hda' bus='virtio'/>
     </disk>
+    <controller type='virtio-serial'/>
     <interface type='network'>
       <source network='{{.PrivateNetwork}}'/>
       <model type='virtio'/>
@@ -65,19 +66,23 @@ const domainTmpl = `
     </interface>
     <serial type='pty'>
       <target port='0'/>
+      <log file='{{.ConsoleLogPath}}' append='on'/>
     </serial>
     <console type='pty'>
       <target type='serial' port='0'/>
     </console>
+    <console type='pty'>
+      <target type="virtio" port="1"/>
+    </console>
     <rng model='virtio'>
       <backend model='random'>/dev/random</backend>
     </rng>
-    {{if .GPU}}
+    {{- if .GPU}}
     {{.DevicesXML}}
-    {{end}}
-    {{if gt .ExtraDisks 0}}
+    {{- end}}
+    {{- if gt .ExtraDisks 0}}
     {{.ExtraDisksXML}}
-    {{end}}
+    {{- end}}
   </devices>
 </domain>
 `