diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 78e278e0..08eaba97 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -142,6 +142,21 @@ jobs: run: | kubectl wait --for=condition=available --timeout=300s deployment/rbgs-controller-manager -n rbgs-system + - name: Pre-load e2e test images + run: | + # Pull and load e2e test images to avoid image pull timeout during tests + E2E_IMAGES=( + "registry.cn-hangzhou.aliyuncs.com/acs-sample/nginx:latest" + "registry-cn-hangzhou.ack.aliyuncs.com/dev/patio-runtime:v0.2.0" + ) + for image in "${E2E_IMAGES[@]}"; do + echo "Pulling image: ${image}" + docker pull "${image}" + echo "Loading image into Kind: ${image}" + kind load docker-image "${image}" --name rbgs-e2e + done + echo "All e2e test images loaded successfully" + - name: Run E2E tests run: | make test-e2e diff --git a/internal/controller/workloads/discovery_config_mode_test.go b/internal/controller/workloads/discovery_config_mode_test.go index 65dc6e1e..e14c93f4 100644 --- a/internal/controller/workloads/discovery_config_mode_test.go +++ b/internal/controller/workloads/discovery_config_mode_test.go @@ -30,7 +30,7 @@ func TestEnsureDiscoveryConfigMode(t *testing.T) { tests := []testCase{ { - name: "missing annotation with legacy role configmap should set legacy mode and requeue", + name: "missing annotation with legacy role configmap should set legacy mode without requeue", buildExtraObjects: func(rbg *workloadsv1alpha1.RoleBasedGroup) []runtime.Object { return []runtime.Object{ &corev1.ConfigMap{ @@ -41,13 +41,13 @@ func TestEnsureDiscoveryConfigMode(t *testing.T) { }, } }, - wantRequeue: true, + wantRequeue: false, wantMode: workloadsv1alpha1.LegacyDiscoveryConfigMode, wantModeAnnotation: true, }, { - name: "missing annotation without legacy signal should set refine mode and requeue", - wantRequeue: true, + name: "missing annotation without legacy signal should set refine mode without requeue", + wantRequeue: false, wantMode: workloadsv1alpha1.RefineDiscoveryConfigMode, wantModeAnnotation: true, }, diff --git a/internal/controller/workloads/pod_controller.go b/internal/controller/workloads/pod_controller.go index 845e8f77..965ac7e8 100644 --- a/internal/controller/workloads/pod_controller.go +++ b/internal/controller/workloads/pod_controller.go @@ -128,7 +128,9 @@ func (r *PodReconciler) setRestartCondition( setCondition(rbg, restartCondition) - rbgApplyConfig := ToRBGApplyConfigurationForStatus(rbg) + // Only patch the conditions, not the RoleStatuses. + // RoleStatuses should be managed by RBG controller only. + rbgApplyConfig := toRBGApplyConfigurationForConditionsOnly(rbg) return utils.PatchObjectApplyConfiguration(ctx, r.client, rbgApplyConfig, utils.PatchStatus) } @@ -169,6 +171,21 @@ func ToRBGApplyConfigurationForStatus(rbg *workloadsv1alpha1.RoleBasedGroup) *ap return rbgApplyConfig } +// toRBGApplyConfigurationForConditionsOnly creates an apply configuration that only updates conditions. +// This is used by pod_controller to avoid overwriting RoleStatuses managed by RBG controller. +func toRBGApplyConfigurationForConditionsOnly(rbg *workloadsv1alpha1.RoleBasedGroup) *applyconfiguration.RoleBasedGroupApplyConfiguration { + if rbg == nil { + return nil + } + gkv := utils.GetRbgGVK() + rbgApplyConfig := applyconfiguration.RoleBasedGroup(rbg.Name, rbg.Namespace). + WithKind(gkv.Kind). + WithAPIVersion(gkv.GroupVersion().String()). + WithStatus(applyconfiguration.RoleBasedGroupStatus(). + WithConditions(ToConditionApplyConfigurations(rbg.Status.Conditions)...)) + return rbgApplyConfig +} + func ToRoleStatusApplyConfiguration(roleStatus []workloadsv1alpha1.RoleStatus) []*applyconfiguration.RoleStatusApplyConfiguration { out := make([]*applyconfiguration.RoleStatusApplyConfiguration, 0, len(roleStatus)) for _, rs := range roleStatus { diff --git a/internal/controller/workloads/rolebasedgroup_controller.go b/internal/controller/workloads/rolebasedgroup_controller.go index 61af7470..82a84b40 100644 --- a/internal/controller/workloads/rolebasedgroup_controller.go +++ b/internal/controller/workloads/rolebasedgroup_controller.go @@ -147,13 +147,29 @@ func (r *RoleBasedGroupReconciler) Reconcile(ctx context.Context, req ctrl.Reque return ctrl.Result{}, err } - // Step 3: Construct role statuses + // Step 3: Initialize discovery config mode. + // IMPORTANT: This must be done BEFORE constructAndUpdateRoleStatuses, because + // shouldUseLegacyDiscoveryConfig checks rbg.Status.RoleStatuses to determine if + // this is a new RBG. If roleStatuses is populated first, it will incorrectly + // use legacy mode for new RBGs. + if needRequeue, err := r.ensureDiscoveryConfigMode(ctx, rbg); err != nil || needRequeue { + return ctrl.Result{Requeue: true}, err + } + + // Step 4: Reconcile refined discovery ConfigMap. + // This must happen before reconcileRoles to ensure ConfigMap exists before workloads are created. + if err := r.reconcileRefinedDiscoveryConfigMap(ctx, rbg); err != nil { + r.recorder.Event(rbg, corev1.EventTypeWarning, FailedReconcileDiscoveryConfigMap, err.Error()) + return ctrl.Result{}, err + } + + // Step 5: Construct role statuses roleStatuses, err := r.constructAndUpdateRoleStatuses(ctx, rbg) if err != nil { return ctrl.Result{}, err } - // Step 4: Calculate coordination strategies for scaling and rolling update + // Step 6: Calculate coordination strategies for scaling and rolling update // TODO: This is a simple method consolidation for now. // The coordination logic will be refactored later to improve extensibility and readability. scalingTargets, rollingUpdateStrategies, err := r.handleCoordinationStrategies(ctx, rbg, roleStatuses) @@ -161,17 +177,6 @@ func (r *RoleBasedGroupReconciler) Reconcile(ctx context.Context, req ctrl.Reque return ctrl.Result{}, err } - // Step 5: Initialize discovery config mode. - if needRequeue, err := r.ensureDiscoveryConfigMode(ctx, rbg); err != nil || needRequeue { - return ctrl.Result{Requeue: true}, err - } - - // Step 6: Reconcile refined discovery ConfigMap. - if err := r.reconcileRefinedDiscoveryConfigMap(ctx, rbg); err != nil { - r.recorder.Event(rbg, corev1.EventTypeWarning, FailedReconcileDiscoveryConfigMap, err.Error()) - return ctrl.Result{}, err - } - // Step 7: Reconcile roles, do create/update actions for roles. if err := r.reconcileRoles(ctx, rbg, expectedRolesRevisionHash, scalingTargets, rollingUpdateStrategies); err != nil { return ctrl.Result{}, err @@ -273,7 +278,7 @@ func (r *RoleBasedGroupReconciler) ensureDiscoveryConfigMode( return false, nil } - // determine mode, update annotation and then requeue + // determine mode, update annotation legacy, err := r.shouldUseLegacyDiscoveryConfig(ctx, rbg) if err != nil { return true, err @@ -291,7 +296,9 @@ func (r *RoleBasedGroupReconciler) ensureDiscoveryConfigMode( } log.FromContext(ctx).Info("Initialized discovery config mode", "mode", mode) - return true, nil + // Don't requeue here - continue to reconcile ConfigMap and workloads in the same loop + // to avoid race condition where workload is created before ConfigMap exists + return false, nil } func (r *RoleBasedGroupReconciler) shouldUseLegacyDiscoveryConfig( diff --git a/test/e2e/framework/rbg_expect.go b/test/e2e/framework/rbg_expect.go index 0a39c7cc..ccdd1ab5 100644 --- a/test/e2e/framework/rbg_expect.go +++ b/test/e2e/framework/rbg_expect.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -94,6 +95,9 @@ func (f *Framework) ExpectRbgEqual(rbg *v1alpha1.RoleBasedGroup) { } func (f *Framework) ExpectRbgDeleted(rbg *v1alpha1.RoleBasedGroup) { + logger := log.FromContext(f.Ctx).WithValues("rbg", rbg.Name) + + // Check RBG is deleted newRbg := &v1alpha1.RoleBasedGroup{} gomega.Eventually( func() bool { @@ -108,6 +112,27 @@ func (f *Framework) ExpectRbgDeleted(rbg *v1alpha1.RoleBasedGroup) { }, utils.Timeout, utils.Interval, ).Should(gomega.BeTrue()) + + // Check all Pods managed by this RBG are deleted + gomega.Eventually( + func() bool { + podList := &corev1.PodList{} + err := f.Client.List( + f.Ctx, podList, + client.InNamespace(rbg.Namespace), + client.MatchingLabels{v1alpha1.SetNameLabelKey: rbg.Name}, + ) + if err != nil { + logger.Error(err, "failed to list pods") + return false + } + if len(podList.Items) > 0 { + logger.V(1).Info("waiting for pods to be deleted", "remainingPods", len(podList.Items)) + return false + } + return true + }, utils.Timeout, utils.Interval, + ).Should(gomega.BeTrue()) } func (f *Framework) ExpectRbgCondition( diff --git a/test/e2e/testcase/debug.go b/test/e2e/testcase/debug.go new file mode 100644 index 00000000..1b9c2c93 --- /dev/null +++ b/test/e2e/testcase/debug.go @@ -0,0 +1,316 @@ +package testcase + +import ( + "fmt" + + "github.com/onsi/ginkgo/v2" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + lwsv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" + workloadsv1alpha1 "sigs.k8s.io/rbgs/api/workloads/v1alpha1" + "sigs.k8s.io/rbgs/test/e2e/framework" +) + +// dumpDebugInfo collects diagnostic information only when test fails +// It prints workload status, pod status, and events for all roles in the RBG +func dumpDebugInfo(f *framework.Framework, rbg *workloadsv1alpha1.RoleBasedGroup) { + if rbg == nil { + return + } + // Only dump debug info when test fails + if !ginkgo.CurrentSpecReport().Failed() { + return + } + fmt.Println("\n========== Debug Info ==========") + fmt.Printf("RBG: %s/%s\n", rbg.Namespace, rbg.Name) + + // Get and print RBG status + currentRBG := &workloadsv1alpha1.RoleBasedGroup{} + if err := f.Client.Get(f.Ctx, client.ObjectKey{Name: rbg.Name, Namespace: rbg.Namespace}, currentRBG); err != nil { + fmt.Printf("[RBG] Failed to get RBG: %v\n", err) + } else { + fmt.Printf("[RBG] Status: TotalRoles=%d, RoleStatuses=%d\n", + len(currentRBG.Spec.Roles), len(currentRBG.Status.RoleStatuses)) + fmt.Printf("[RBG] Conditions:\n") + for _, cond := range currentRBG.Status.Conditions { + fmt.Printf(" - Type=%s, Status=%s, Reason=%s, Message=%s\n", + cond.Type, cond.Status, cond.Reason, cond.Message) + } + } + + for _, role := range rbg.Spec.Roles { + workloadName := rbg.GetWorkloadName(&role) + fmt.Printf("\n--- Role: %s, Workload: %s ---\n", role.Name, workloadName) + + // Determine workload type and dump accordingly + switch { + case role.Workload.Kind == "LeaderWorkerSet" || role.LeaderWorkerSet != nil: + dumpLWSWorkload(f, rbg.Namespace, workloadName) + case role.Workload.Kind == "Deployment": + dumpDeploymentWorkload(f, rbg.Namespace, workloadName) + case role.Workload.Kind == "InstanceSet": + dumpInstanceSetWorkload(f, rbg.Namespace, workloadName, role.Name) + default: // StatefulSet is default + dumpStatefulSetWorkload(f, rbg.Namespace, workloadName) + } + + // Print pods for this role + dumpRolePods(f, rbg, role.Name, workloadName) + } + + fmt.Println("\n========== End Debug Info ==========") +} + +// dumpDebugInfoForRBGSet collects diagnostic information for RoleBasedGroupSet only when test fails +func dumpDebugInfoForRBGSet(f *framework.Framework, rbgset *workloadsv1alpha1.RoleBasedGroupSet) { + if rbgset == nil { + return + } + // Only dump debug info when test fails + if !ginkgo.CurrentSpecReport().Failed() { + return + } + fmt.Println("\n========== RBGSet Debug Info ==========") + fmt.Printf("RBGSet: %s/%s\n", rbgset.Namespace, rbgset.Name) + + // Get and print RBGSet status + currentRBGSet := &workloadsv1alpha1.RoleBasedGroupSet{} + if err := f.Client.Get(f.Ctx, client.ObjectKey{Name: rbgset.Name, Namespace: rbgset.Namespace}, currentRBGSet); err != nil { + fmt.Printf("[RBGSet] Failed to get RBGSet: %v\n", err) + } else { + fmt.Printf("[RBGSet] Status: Replicas=%d, ReadyReplicas=%d\n", + currentRBGSet.Status.Replicas, currentRBGSet.Status.ReadyReplicas) + fmt.Printf("[RBGSet] Conditions:\n") + for _, cond := range currentRBGSet.Status.Conditions { + fmt.Printf(" - Type=%s, Status=%s, Reason=%s, Message=%s\n", + cond.Type, cond.Status, cond.Reason, cond.Message) + } + } + + // List all RBGs owned by this RBGSet + rbgList := &workloadsv1alpha1.RoleBasedGroupList{} + if err := f.Client.List(f.Ctx, rbgList, client.InNamespace(rbgset.Namespace), + client.MatchingLabels{workloadsv1alpha1.SetNameLabelKey: rbgset.Name}); err != nil { + fmt.Printf("[RBGSet] Failed to list RBGs: %v\n", err) + } else { + fmt.Printf("[RBGSet] Found %d RBGs\n", len(rbgList.Items)) + for _, rbg := range rbgList.Items { + fmt.Printf("\n=== Child RBG: %s ===\n", rbg.Name) + dumpDebugInfo(f, &rbg) + } + } + + fmt.Println("\n========== End RBGSet Debug Info ==========") +} + +func dumpLWSWorkload(f *framework.Framework, namespace, lwsName string) { + // Get LWS status + lws := &lwsv1.LeaderWorkerSet{} + if err := f.Client.Get(f.Ctx, client.ObjectKey{Name: lwsName, Namespace: namespace}, lws); err != nil { + fmt.Printf("[LWS] Failed to get LWS: %v\n", err) + } else { + fmt.Printf("[LWS] Status: Replicas=%d, ReadyReplicas=%d, UpdatedReplicas=%d\n", + lws.Status.Replicas, lws.Status.ReadyReplicas, lws.Status.UpdatedReplicas) + fmt.Printf("[LWS] Conditions:\n") + for _, cond := range lws.Status.Conditions { + fmt.Printf(" - Type=%s, Status=%s, Reason=%s, Message=%s\n", + cond.Type, cond.Status, cond.Reason, cond.Message) + } + } + + // Get StatefulSets created by LWS + stsList := &appsv1.StatefulSetList{} + if err := f.Client.List(f.Ctx, stsList, client.InNamespace(namespace), + client.MatchingLabels{"leaderworkerset.sigs.k8s.io/name": lwsName}); err != nil { + fmt.Printf("[STS] Failed to list StatefulSets: %v\n", err) + } else { + fmt.Printf("[STS] Found %d StatefulSets\n", len(stsList.Items)) + for _, sts := range stsList.Items { + fmt.Printf(" - Name=%s, Replicas=%d, ReadyReplicas=%d, CurrentReplicas=%d, UpdatedReplicas=%d\n", + sts.Name, *sts.Spec.Replicas, sts.Status.ReadyReplicas, sts.Status.CurrentReplicas, sts.Status.UpdatedReplicas) + for _, cond := range sts.Status.Conditions { + fmt.Printf(" Condition: Type=%s, Status=%s, Reason=%s, Message=%s\n", + cond.Type, cond.Status, cond.Reason, cond.Message) + } + } + } +} + +func dumpDeploymentWorkload(f *framework.Framework, namespace, deployName string) { + deploy := &appsv1.Deployment{} + if err := f.Client.Get(f.Ctx, client.ObjectKey{Name: deployName, Namespace: namespace}, deploy); err != nil { + fmt.Printf("[Deployment] Failed to get Deployment: %v\n", err) + } else { + fmt.Printf("[Deployment] Status: Replicas=%d, ReadyReplicas=%d, AvailableReplicas=%d, UpdatedReplicas=%d\n", + deploy.Status.Replicas, deploy.Status.ReadyReplicas, deploy.Status.AvailableReplicas, deploy.Status.UpdatedReplicas) + fmt.Printf("[Deployment] Conditions:\n") + for _, cond := range deploy.Status.Conditions { + fmt.Printf(" - Type=%s, Status=%s, Reason=%s, Message=%s\n", + cond.Type, cond.Status, cond.Reason, cond.Message) + } + } + + // Get ReplicaSets + rsList := &appsv1.ReplicaSetList{} + if err := f.Client.List(f.Ctx, rsList, client.InNamespace(namespace)); err != nil { + fmt.Printf("[ReplicaSet] Failed to list ReplicaSets: %v\n", err) + } else { + for _, rs := range rsList.Items { + for _, ownerRef := range rs.OwnerReferences { + if ownerRef.Kind == "Deployment" && ownerRef.Name == deployName { + fmt.Printf(" [RS] Name=%s, Replicas=%d, ReadyReplicas=%d, AvailableReplicas=%d\n", + rs.Name, *rs.Spec.Replicas, rs.Status.ReadyReplicas, rs.Status.AvailableReplicas) + } + } + } + } +} + +func dumpStatefulSetWorkload(f *framework.Framework, namespace, stsName string) { + sts := &appsv1.StatefulSet{} + if err := f.Client.Get(f.Ctx, client.ObjectKey{Name: stsName, Namespace: namespace}, sts); err != nil { + fmt.Printf("[StatefulSet] Failed to get StatefulSet: %v\n", err) + } else { + fmt.Printf("[StatefulSet] Status: Replicas=%d, ReadyReplicas=%d, CurrentReplicas=%d, UpdatedReplicas=%d\n", + *sts.Spec.Replicas, sts.Status.ReadyReplicas, sts.Status.CurrentReplicas, sts.Status.UpdatedReplicas) + fmt.Printf("[StatefulSet] CurrentRevision=%s, UpdateRevision=%s\n", + sts.Status.CurrentRevision, sts.Status.UpdateRevision) + fmt.Printf("[StatefulSet] Conditions:\n") + for _, cond := range sts.Status.Conditions { + fmt.Printf(" - Type=%s, Status=%s, Reason=%s, Message=%s\n", + cond.Type, cond.Status, cond.Reason, cond.Message) + } + } +} + +func dumpInstanceSetWorkload(f *framework.Framework, namespace, instanceSetName, roleName string) { + // Get InstanceSet status + instanceSet := &workloadsv1alpha1.InstanceSet{} + if err := f.Client.Get(f.Ctx, client.ObjectKey{Name: instanceSetName, Namespace: namespace}, instanceSet); err != nil { + fmt.Printf("[InstanceSet] Failed to get InstanceSet for role %s: %v\n", roleName, err) + } else { + fmt.Printf("[InstanceSet] Status: Replicas=%d, ReadyReplicas=%d, AvailableReplicas=%d, UpdatedReplicas=%d\n", + instanceSet.Status.Replicas, instanceSet.Status.ReadyReplicas, instanceSet.Status.AvailableReplicas, instanceSet.Status.UpdatedReplicas) + fmt.Printf("[InstanceSet] Conditions:\n") + for _, cond := range instanceSet.Status.Conditions { + fmt.Printf(" - Type=%s, Status=%s, Reason=%s, Message=%s\n", + cond.Type, cond.Status, cond.Reason, cond.Message) + } + } + + // List Instances + instanceList := &workloadsv1alpha1.InstanceList{} + if err := f.Client.List(f.Ctx, instanceList, client.InNamespace(namespace), + client.MatchingLabels{workloadsv1alpha1.SetInstanceNameLabelKey: instanceSetName}); err != nil { + fmt.Printf("[Instance] Failed to list Instances: %v\n", err) + } else { + fmt.Printf("[Instance] Found %d Instances\n", len(instanceList.Items)) + for _, inst := range instanceList.Items { + // Check InstanceReady condition + ready := false + for _, cond := range inst.Status.Conditions { + if cond.Type == workloadsv1alpha1.InstanceReady && cond.Status == corev1.ConditionTrue { + ready = true + break + } + } + fmt.Printf(" - Name=%s, Ready=%v\n", inst.Name, ready) + } + } +} + +func dumpRolePods(f *framework.Framework, rbg *workloadsv1alpha1.RoleBasedGroup, roleName, workloadName string) { + podList := &corev1.PodList{} + + // Try to find pods by different labels based on workload type + selectors := []client.MatchingLabels{ + {workloadsv1alpha1.SetNameLabelKey: rbg.Name, workloadsv1alpha1.SetRoleLabelKey: roleName}, + {"leaderworkerset.sigs.k8s.io/name": workloadName}, + {"app.kubernetes.io/instance": workloadName}, + } + + var foundPods []corev1.Pod + for _, selector := range selectors { + if err := f.Client.List(f.Ctx, podList, client.InNamespace(rbg.Namespace), selector); err == nil && len(podList.Items) > 0 { + foundPods = podList.Items + break + } + } + + if len(foundPods) == 0 { + fmt.Printf("[POD] No pods found for role %s\n", roleName) + return + } + + fmt.Printf("[POD] Found %d Pods\n", len(foundPods)) + for _, pod := range foundPods { + ready := isPodReady(&pod) + fmt.Printf(" - Name=%s, Phase=%s, Ready=%v\n", + pod.Name, pod.Status.Phase, ready) + // Print container statuses + for _, cs := range pod.Status.ContainerStatuses { + fmt.Printf(" Container: %s, Ready=%v, RestartCount=%d\n", + cs.Name, cs.Ready, cs.RestartCount) + if cs.State.Waiting != nil { + fmt.Printf(" Waiting: Reason=%s, Message=%s\n", + cs.State.Waiting.Reason, cs.State.Waiting.Message) + } + if cs.State.Terminated != nil { + fmt.Printf(" Terminated: Reason=%s, ExitCode=%d, Message=%s\n", + cs.State.Terminated.Reason, cs.State.Terminated.ExitCode, cs.State.Terminated.Message) + } + } + // Print conditions + fmt.Printf(" Conditions:\n") + for _, cond := range pod.Status.Conditions { + fmt.Printf(" - Type=%s, Status=%s, Reason=%s, Message=%s\n", + cond.Type, cond.Status, cond.Reason, cond.Message) + } + // Print events for not ready pods to help diagnose issues + if !ready { + dumpPodEvents(f, rbg.Namespace, pod.Name) + } + } +} + +// dumpPodEvents prints recent events for a pod +func dumpPodEvents(f *framework.Framework, namespace, podName string) { + eventList := &corev1.EventList{} + if err := f.Client.List(f.Ctx, eventList, client.InNamespace(namespace), + client.MatchingFields{"involvedObject.name": podName}); err != nil { + // Fallback: list all events and filter manually + if err := f.Client.List(f.Ctx, eventList, client.InNamespace(namespace)); err != nil { + fmt.Printf(" [Events] Failed to list events: %v\n", err) + return + } + } + + // Filter events for this pod + var podEvents []corev1.Event + for _, event := range eventList.Items { + if event.InvolvedObject.Name == podName && event.InvolvedObject.Kind == "Pod" { + podEvents = append(podEvents, event) + } + } + + if len(podEvents) == 0 { + fmt.Printf(" [Events] No events found for pod\n") + return + } + + fmt.Printf(" [Events] Found %d events:\n", len(podEvents)) + for _, event := range podEvents { + fmt.Printf(" - Type=%s, Reason=%s, Message=%s, Count=%d, LastTime=%s\n", + event.Type, event.Reason, event.Message, event.Count, event.LastTimestamp.Format("15:04:05")) + } +} + +func isPodReady(pod *corev1.Pod) bool { + for _, cond := range pod.Status.Conditions { + if cond.Type == corev1.PodReady { + return cond.Status == corev1.ConditionTrue + } + } + return false +} diff --git a/test/e2e/testcase/deploy.go b/test/e2e/testcase/deploy.go index c27f2d67..3b7b076a 100644 --- a/test/e2e/testcase/deploy.go +++ b/test/e2e/testcase/deploy.go @@ -17,6 +17,9 @@ func RunDeploymentWorkloadTestCases(f *framework.Framework) { rbg := wrappers.BuildBasicRoleBasedGroup("e2e-test", f.Namespace).WithRoles([]workloadsv1alpha1.RoleSpec{ wrappers.BuildBasicRole("role-1").WithWorkload(workloadsv1alpha1.DeploymentWorkloadType).Obj(), }).Obj() + + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) @@ -42,6 +45,9 @@ func RunDeploymentWorkloadTestCases(f *framework.Framework) { MaxSurge: ptr.To(intstr.FromInt32(1)), }).Obj(), }).Obj() + + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) @@ -64,6 +70,9 @@ func RunDeploymentWorkloadTestCases(f *framework.Framework) { WithRestartPolicy(workloadsv1alpha1.RecreateRBGOnPodRestart). Obj(), }).Obj() + + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) diff --git a/test/e2e/testcase/lws.go b/test/e2e/testcase/lws.go index bc26c0f9..e96fdbce 100644 --- a/test/e2e/testcase/lws.go +++ b/test/e2e/testcase/lws.go @@ -22,6 +22,8 @@ func RunLeaderWorkerSetWorkloadTestCases(f *framework.Framework) { Obj(), }).Obj() + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(testutils.CreatePatioRuntime(f.Ctx, f.Client)).Should(gomega.Succeed()) gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) @@ -31,6 +33,9 @@ func RunLeaderWorkerSetWorkloadTestCases(f *framework.Framework) { rbg := wrappers.BuildBasicRoleBasedGroup("e2e-test", f.Namespace).WithRoles([]workloadsv1alpha1.RoleSpec{ wrappers.BuildLwsRole("role-1").Obj(), }).Obj() + + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) @@ -49,6 +54,9 @@ func RunLeaderWorkerSetWorkloadTestCases(f *framework.Framework) { rbg := wrappers.BuildBasicRoleBasedGroup("e2e-test", f.Namespace).WithRoles([]workloadsv1alpha1.RoleSpec{ wrappers.BuildLwsRole("role-1").Obj(), }).Obj() + + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) @@ -73,6 +81,9 @@ func RunLeaderWorkerSetWorkloadTestCases(f *framework.Framework) { MaxSurge: ptr.To(intstr.FromInt32(1)), }).Obj(), }).Obj() + + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) @@ -92,6 +103,9 @@ func RunLeaderWorkerSetWorkloadTestCases(f *framework.Framework) { WithRestartPolicy(workloadsv1alpha1.RecreateRBGOnPodRestart). Obj(), }).Obj() + + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) diff --git a/test/e2e/testcase/rbg.go b/test/e2e/testcase/rbg.go index 1a3ac9f6..b7142e74 100644 --- a/test/e2e/testcase/rbg.go +++ b/test/e2e/testcase/rbg.go @@ -26,6 +26,8 @@ func RunRbgControllerTestCases(f *framework.Framework) { }, ).Obj() + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) @@ -46,6 +48,8 @@ func RunRbgControllerTestCases(f *framework.Framework) { }, ).Obj() + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) @@ -66,6 +70,8 @@ func RunRbgControllerTestCases(f *framework.Framework) { }, ).Obj() + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(utils.CreatePatioRuntime(f.Ctx, f.Client)).Should(gomega.Succeed()) gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) @@ -82,6 +88,9 @@ func RunRbgControllerTestCases(f *framework.Framework) { wrappers.BuildBasicRole("role-2").Obj(), }, ).Obj() + + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) @@ -140,6 +149,8 @@ func RunRbgControllerTestCases(f *framework.Framework) { }, ).Obj() + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) podGroupLabel := map[string]string{ @@ -189,6 +200,8 @@ func RunRbgControllerTestCases(f *framework.Framework) { }, ).Obj() + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) podGroupAnnotation := map[string]string{ @@ -252,6 +265,8 @@ func RunRbgControllerTestCases(f *framework.Framework) { }, ).Obj() + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectWorkloadExclusiveTopology(rbg, rbg.Spec.Roles[0], topologyKey) f.ExpectWorkloadExclusiveTopology(rbg, rbg.Spec.Roles[1], topologyKey) diff --git a/test/e2e/testcase/rbgs.go b/test/e2e/testcase/rbgs.go index 05681180..4f2aa266 100644 --- a/test/e2e/testcase/rbgs.go +++ b/test/e2e/testcase/rbgs.go @@ -16,6 +16,9 @@ func RunRbgSetControllerTestCases(f *framework.Framework) { ginkgo.It( "create & delete rbgset", func() { rbgset := wrappers.BuildBasicRoleBasedGroupSet("test", f.Namespace).Obj() + + ginkgo.DeferCleanup(func() { dumpDebugInfoForRBGSet(f, rbgset) }) + gomega.Expect(f.Client.Create(f.Ctx, rbgset)).Should(gomega.Succeed()) f.ExpectRbgSetEqual(rbgset) @@ -28,6 +31,9 @@ func RunRbgSetControllerTestCases(f *framework.Framework) { ginkgo.It( "scaling rbgset", func() { rbgset := wrappers.BuildBasicRoleBasedGroupSet("test", f.Namespace).WithReplicas(1).Obj() + + ginkgo.DeferCleanup(func() { dumpDebugInfoForRBGSet(f, rbgset) }) + gomega.Expect(f.Client.Create(f.Ctx, rbgset)).Should(gomega.Succeed()) f.ExpectRbgSetEqual(rbgset) @@ -57,6 +63,9 @@ func RunRbgSetControllerTestCases(f *framework.Framework) { WithAnnotations( map[string]string{workloadsv1alpha1.ExclusiveKeyAnnotationKey: topologyKey}, ).Obj() + + ginkgo.DeferCleanup(func() { dumpDebugInfoForRBGSet(f, rbgset) }) + gomega.Expect(f.Client.Create(f.Ctx, rbgset)).Should(gomega.Succeed()) f.ExpectRbgAnnotation( rbgset, map[string]string{workloadsv1alpha1.ExclusiveKeyAnnotationKey: topologyKey}, diff --git a/test/e2e/testcase/revision.go b/test/e2e/testcase/revision.go index 90960923..8ba12f43 100644 --- a/test/e2e/testcase/revision.go +++ b/test/e2e/testcase/revision.go @@ -30,6 +30,8 @@ func RunControllerRevisionTestCases(f *framework.Framework) { }, ).Obj() + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) old, err := pkgutils.NewRevision(f.Ctx, f.Client, rbg, nil) gomega.Expect(err).ToNot(gomega.HaveOccurred()) @@ -84,6 +86,8 @@ func RunControllerRevisionTestCases(f *framework.Framework) { }, ).Obj() + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(utils.CreatePatioRuntime(f.Ctx, f.Client)).Should(gomega.Succeed()) gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) @@ -120,6 +124,9 @@ func RunControllerRevisionTestCases(f *framework.Framework) { rbg := wrappers.BuildBasicRoleBasedGroup("e2e-test", f.Namespace).WithRoles([]v1alpha1.RoleSpec{ wrappers.BuildLwsRole("role-1").Obj(), }).Obj() + + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) diff --git a/test/e2e/testcase/roletemplate.go b/test/e2e/testcase/roletemplate.go index 90be9697..81198e4c 100644 --- a/test/e2e/testcase/roletemplate.go +++ b/test/e2e/testcase/roletemplate.go @@ -80,6 +80,8 @@ func RunRoleTemplateTestCases(f *framework.Framework) { Obj(), }).Obj() + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) @@ -145,6 +147,8 @@ func RunRoleTemplateTestCases(f *framework.Framework) { Obj(), }).Obj() + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) @@ -284,6 +288,8 @@ func RunRoleTemplateTestCases(f *framework.Framework) { Obj(), }).Obj() + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) @@ -345,6 +351,8 @@ func RunRoleTemplateTestCases(f *framework.Framework) { Obj(), }).Obj() + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) diff --git a/test/e2e/testcase/scaling_adapter.go b/test/e2e/testcase/scaling_adapter.go index 9b056abd..880e483a 100644 --- a/test/e2e/testcase/scaling_adapter.go +++ b/test/e2e/testcase/scaling_adapter.go @@ -27,6 +27,9 @@ func RunRbgScalingAdapterControllerTestCases(f *framework.Framework) { wrappers.BuildBasicRole("role-3").Obj(), }, ).Obj() + + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) @@ -50,6 +53,9 @@ func RunRbgScalingAdapterControllerTestCases(f *framework.Framework) { wrappers.BuildBasicRole("role-3").Obj(), }, ).Obj() + + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) @@ -85,6 +91,9 @@ func RunRbgScalingAdapterControllerTestCases(f *framework.Framework) { wrappers.BuildBasicRole("role-1").Obj(), }, ).Obj() + + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) @@ -128,6 +137,9 @@ func RunRbgScalingAdapterControllerTestCases(f *framework.Framework) { wrappers.BuildBasicRole("role-3").Obj(), }, ).Obj() + + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) @@ -181,6 +193,9 @@ func RunRbgScalingAdapterControllerTestCases(f *framework.Framework) { WithScalingAdapter(true).Obj(), }, ).Obj() + + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) diff --git a/test/e2e/testcase/sts.go b/test/e2e/testcase/sts.go index 1d6175c0..6ba5e88d 100644 --- a/test/e2e/testcase/sts.go +++ b/test/e2e/testcase/sts.go @@ -15,6 +15,9 @@ import ( func RunStatefulSetWorkloadTestCases(f *framework.Framework) { ginkgo.It("update sts role.replicas & role.Template", func() { rbg := wrappers.BuildBasicRoleBasedGroup("e2e-test", f.Namespace).Obj() + + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) @@ -42,6 +45,9 @@ func RunStatefulSetWorkloadTestCases(f *framework.Framework) { MaxSurge: ptr.To(intstr.FromInt32(1)), }).Obj(), }).Obj() + + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) @@ -64,6 +70,9 @@ func RunStatefulSetWorkloadTestCases(f *framework.Framework) { WithRestartPolicy(workloadsv1alpha1.RecreateRBGOnPodRestart). Obj(), }).Obj() + + ginkgo.DeferCleanup(func() { dumpDebugInfo(f, rbg) }) + gomega.Expect(f.Client.Create(f.Ctx, rbg)).Should(gomega.Succeed()) f.ExpectRbgEqual(rbg) diff --git a/test/utils/utils.go b/test/utils/utils.go index 871b58dd..bddd7e6c 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -31,7 +31,7 @@ import ( ) const ( - Timeout = 60 * time.Second + Timeout = 150 * time.Second Interval = time.Millisecond * 250 DefaultImage = "registry.cn-hangzhou.aliyuncs.com/acs-sample/nginx:latest"