diff --git a/misc/snapshotter/Dockerfile b/misc/snapshotter/Dockerfile index f0c698f20e..2c0208553a 100644 --- a/misc/snapshotter/Dockerfile +++ b/misc/snapshotter/Dockerfile @@ -15,9 +15,10 @@ RUN apk add -q --no-cache curl && \ FROM base AS kubectl-sourcer ARG TARGETARCH +ARG KUBE_VER=v1.30.2 RUN apk add -q --no-cache curl && \ - curl -fsSL -o /usr/bin/kubectl https://dl.k8s.io/release/"$(curl -L -s https://dl.k8s.io/release/stable.txt)"/bin/linux/"$TARGETARCH"/kubectl && \ + curl -fsSL -o /usr/bin/kubectl https://dl.k8s.io/release/${KUBE_VER}/bin/linux/${TARGETARCH}/kubectl && \ chmod +x /usr/bin/kubectl FROM base diff --git a/tests/e2e/k8s/containerd.config.toml b/tests/e2e/k8s/containerd.config.toml deleted file mode 100644 index 09a406cd93..0000000000 --- a/tests/e2e/k8s/containerd.config.toml +++ /dev/null @@ -1,33 +0,0 @@ -# explicitly use v2 config format -version = 2 - -[debug] -level = "debug" - -[plugins."io.containerd.grpc.v1.cri".containerd] -# save disk space when using a single snapshotter -discard_unpacked_layers = false -# explicitly use default snapshotter so we can sed it in entrypoint -snapshotter = "overlayfs" -# explicit default here, as we're configuring it below -default_runtime_name = "runc" -# nydus snapshotter needs this config -disable_snapshot_annotations = false - -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] -# set default runtime handler to v2, which has a per-pod shim -runtime_type = "io.containerd.runc.v2" - -# Setup a runtime with the magic name ("test-handler") used for Kubernetes -# runtime class tests ... -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler] -runtime_type = "io.containerd.runc.v2" - -[plugins."io.containerd.grpc.v1.cri"] -# use fixed sandbox image -sandbox_image = "registry.k8s.io/pause:3.6" -# allow hugepages controller to be missing -# see https://github.com/containerd/cri/pull/1501 -tolerate_missing_hugepages_controller = true -# restrict_oom_score_adj needs to be true when running inside UserNS (rootless) -restrict_oom_score_adj = false diff --git a/tests/e2e/k8s/kind.yaml b/tests/e2e/k8s/kind.yaml index c9ea078d05..54bb88552f 100644 --- a/tests/e2e/k8s/kind.yaml +++ b/tests/e2e/k8s/kind.yaml @@ -2,11 +2,13 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 networking: ipFamily: dual +containerdConfigPatches: + - |- + [plugins."io.containerd.grpc.v1.cri".containerd] + discard_unpacked_layers = false + disable_snapshot_annotations = false nodes: - role: control-plane - image: kindest/node:v1.30.2 extraMounts: - - hostPath: ./tests/e2e/k8s/containerd.config.toml - containerPath: /etc/containerd/config.toml - hostPath: /dev/fuse containerPath: /dev/fuse diff --git a/tests/helpers/kind.sh b/tests/helpers/kind.sh index 82c5738540..74ce6754cb 100755 --- a/tests/helpers/kind.sh +++ b/tests/helpers/kind.sh @@ -15,6 +15,7 @@ readonly root KIND_VERSION=v0.23.0 +KUBE_VERSION=v1.30.2 NYDUS_VERSION=v2.3.0 DOCKER_USER=testuser DOCKER_PASSWORD=testpassword @@ -47,7 +48,7 @@ cp -r misc/snapshotter/* ./ log::info "Building... docker image" pwd -exec::docker build --build-arg NYDUS_VER="$NYDUS_VERSION" -t local-dev:e2e . +exec::docker build --build-arg NYDUS_VER="$NYDUS_VERSION" --build-arg KUBE_VER="$KUBE_VERSION" -t local-dev:e2e . # Start local registry, and configure docker log::info "Starting registry" @@ -66,7 +67,7 @@ docker::login "$DOCKER_USER" "$DOCKER_PASSWORD" "$registry_url" # Install dependencies log::info "Installing host dependencies" install::kind "$KIND_VERSION" -install::kubectl +install::kubectl "$KUBE_VERSION" install::nydus "$NYDUS_VERSION" # Convert a nydus image and push it @@ -80,7 +81,7 @@ exec::nydusify convert \ # Create fresh cluster log::info "Creating new cluster" exec::kind delete cluster 2>/dev/null || true -exec::kind create cluster +exec::kind create cluster --config tests/e2e/k8s/kind.yaml --image "kindest/node:$KUBE_VERSION" exec::kind load docker-image local-dev:e2e # Deploy nydus @@ -106,6 +107,17 @@ exec::kubectl create --namespace "$NAMESPACE" secret generic regcred \ if [ "$AUTH_TYPE" == "cri" ]; then exec::docker exec kind-control-plane sh -c 'echo " --image-service-endpoint=unix:///run/containerd-nydus/containerd-nydus-grpc.sock" >> /etc/default/kubelet' exec::docker exec kind-control-plane sh -c 'systemctl daemon-reload && systemctl restart kubelet' + # The API server may become briefly unavailable after the kubelet + # restart. Wait for it to recover before issuing kubectl commands. + log::info "Waiting for API server to recover" + for _ in $(seq 1 10); do + kubectl get --raw /healthz &>/dev/null && break + sleep 2 + done + kubectl get --raw /healthz &>/dev/null || { + log::error "API server did not recover within 20s" + exit 1 + } fi exec::kubectl apply -f tests/e2e/k8s/test-pod.yaml