diff --git a/cli/cmd/smoketest_codesphere.go b/cli/cmd/smoketest_codesphere.go index c20b7f16..5f1462d7 100644 --- a/cli/cmd/smoketest_codesphere.go +++ b/cli/cmd/smoketest_codesphere.go @@ -28,7 +28,7 @@ var availableSteps = []teststeps.SmokeTestStep{ &teststeps.SetEnvVarStep{}, &teststeps.CreateFilesStep{}, &teststeps.SyncLandscapeStep{}, - &teststeps.StartPipelineStep{}, + &teststeps.ExecuteRunStageStep{}, &teststeps.DeleteWorkspaceStep{}, } diff --git a/cli/cmd/smoketest_codesphere_test.go b/cli/cmd/smoketest_codesphere_test.go index 2f953b1a..a90a00c5 100644 --- a/cli/cmd/smoketest_codesphere_test.go +++ b/cli/cmd/smoketest_codesphere_test.go @@ -23,7 +23,7 @@ import ( func mockFullTestRun(mockClient *codesphere.MockClient, teamId, planId, workspaceId int) { // Expect the rest of the steps to run with the fetched plan ID mockClient.EXPECT().CreateWorkspace( - teamId, // teamID + teamId, planId, // fetched planID mock.AnythingOfType("string"), // workspace name is timestamped (*string)(nil), // empty workspace @@ -60,6 +60,13 @@ func mockFullTestRun(mockClient *codesphere.MockClient, teamId, planId, workspac "run", ).Return(nil).Once() + mockClient.EXPECT().GetPipelineState( + workspaceId, + "run", + ).Return([]api.PipelineStatus{ + {State: "running", Server: "customer-server", Replica: "replica-0"}, + }, nil).Once() + mockClient.EXPECT().DeleteWorkspace( workspaceId, ).Return(nil).Once() @@ -180,8 +187,8 @@ var _ = Describe("SmoketestCodesphereCmd", func() { It("deletes workspace even on CreateWorkspace failure", func() { mockClient.EXPECT().CreateWorkspace( - teamIdInt, // teamID - planIdInt, // planID + teamIdInt, + planIdInt, mock.AnythingOfType("string"), (*string)(nil), // empty workspace ).Return(0, fmt.Errorf("create failed")).Once() @@ -194,8 +201,8 @@ var _ = Describe("SmoketestCodesphereCmd", func() { workspaceID := 789 mockClient.EXPECT().CreateWorkspace( - teamIdInt, // teamID - planIdInt, // planID + teamIdInt, + planIdInt, mock.AnythingOfType("string"), (*string)(nil), // empty workspace ).Return(workspaceID, nil).Once() @@ -216,8 +223,8 @@ var _ = Describe("SmoketestCodesphereCmd", func() { It("deletes workspace on ExecuteCommand failure", func() { mockClient.EXPECT().CreateWorkspace( - teamIdInt, // teamID - planIdInt, // planID + teamIdInt, + planIdInt, mock.AnythingOfType("string"), (*string)(nil), // empty workspace ).Return(workspaceId, nil).Once() @@ -246,8 +253,8 @@ var _ = Describe("SmoketestCodesphereCmd", func() { It("deletes workspace on SyncLandscape failure", func() { mockClient.EXPECT().CreateWorkspace( - teamIdInt, // teamID - planIdInt, // planID + teamIdInt, + planIdInt, mock.AnythingOfType("string"), (*string)(nil), // empty workspace ).Return(workspaceId, nil).Once() @@ -289,8 +296,8 @@ var _ = Describe("SmoketestCodesphereCmd", func() { It("deletes workspace on StartPipeline failure", func() { mockClient.EXPECT().CreateWorkspace( - teamIdInt, // teamID - planIdInt, // planID + teamIdInt, + planIdInt, mock.AnythingOfType("string"), (*string)(nil), // empty workspace ).Return(workspaceId, nil).Once() @@ -336,10 +343,228 @@ var _ = Describe("SmoketestCodesphereCmd", func() { Expect(err).To(MatchError(ContainSubstring("failed to start pipeline"))) }) + It("deletes workspace when run stage reaches failure state", func() { + mockClient.EXPECT().CreateWorkspace( + teamIdInt, + planIdInt, + mock.AnythingOfType("string"), + (*string)(nil), // empty workspace + ).Return(workspaceId, nil).Once() + + mockClient.EXPECT().SetEnvVar( + workspaceId, + "TEST_VAR", + "smoketest", + ).Return(nil).Once() + + mockClient.EXPECT().ExecuteCommand( + workspaceId, + mock.MatchedBy(func(cmd string) bool { + return strings.Contains(cmd, "> ci.yml") + }), + ).Return(nil).Once() + + mockClient.EXPECT().ExecuteCommand( + workspaceId, + mock.MatchedBy(func(cmd string) bool { + return strings.Contains(cmd, "> index.html") + }), + ).Return(nil).Once() + + mockClient.EXPECT().SyncLandscape( + workspaceId, + "ci.yml", + ).Return(nil).Once() + + mockClient.EXPECT().StartPipeline( + workspaceId, + "ci.yml", + "run", + ).Return(nil).Once() + + mockClient.EXPECT().GetPipelineState( + workspaceId, + "run", + ).Return([]api.PipelineStatus{ + {State: "failure", Server: "customer-server", Replica: "replica-0"}, + }, nil).Once() + + mockClient.EXPECT().DeleteWorkspace( + workspaceId, + ).Return(nil).Once() + + err := c.RunSmoketest() + Expect(err).To(MatchError(ContainSubstring("unexpected state"))) + }) + + It("deletes workspace when run stage reaches aborted state", func() { + mockClient.EXPECT().CreateWorkspace( + teamIdInt, + planIdInt, + mock.AnythingOfType("string"), + (*string)(nil), // empty workspace + ).Return(workspaceId, nil).Once() + + mockClient.EXPECT().SetEnvVar( + workspaceId, + "TEST_VAR", + "smoketest", + ).Return(nil).Once() + + mockClient.EXPECT().ExecuteCommand( + workspaceId, + mock.MatchedBy(func(cmd string) bool { + return strings.Contains(cmd, "> ci.yml") + }), + ).Return(nil).Once() + + mockClient.EXPECT().ExecuteCommand( + workspaceId, + mock.MatchedBy(func(cmd string) bool { + return strings.Contains(cmd, "> index.html") + }), + ).Return(nil).Once() + + mockClient.EXPECT().SyncLandscape( + workspaceId, + "ci.yml", + ).Return(nil).Once() + + mockClient.EXPECT().StartPipeline( + workspaceId, + "ci.yml", + "run", + ).Return(nil).Once() + + mockClient.EXPECT().GetPipelineState( + workspaceId, + "run", + ).Return([]api.PipelineStatus{ + {State: "aborted", Server: "customer-server", Replica: "replica-0"}, + }, nil).Once() + + mockClient.EXPECT().DeleteWorkspace( + workspaceId, + ).Return(nil).Once() + + err := c.RunSmoketest() + Expect(err).To(MatchError(ContainSubstring("unexpected state"))) + }) + + It("retries GetPipelineState on error and times out including the last error", func() { + mockClient.EXPECT().CreateWorkspace( + teamIdInt, + planIdInt, + mock.AnythingOfType("string"), + (*string)(nil), // empty workspace + ).Return(workspaceId, nil).Once() + + mockClient.EXPECT().SetEnvVar( + workspaceId, + "TEST_VAR", + "smoketest", + ).Return(nil).Once() + + mockClient.EXPECT().ExecuteCommand( + workspaceId, + mock.MatchedBy(func(cmd string) bool { + return strings.Contains(cmd, "> ci.yml") + }), + ).Return(nil).Once() + + mockClient.EXPECT().ExecuteCommand( + workspaceId, + mock.MatchedBy(func(cmd string) bool { + return strings.Contains(cmd, "> index.html") + }), + ).Return(nil).Once() + + mockClient.EXPECT().SyncLandscape( + workspaceId, + "ci.yml", + ).Return(nil).Once() + + mockClient.EXPECT().StartPipeline( + workspaceId, + "ci.yml", + "run", + ).Return(nil).Once() + + mockClient.EXPECT().GetPipelineState( + workspaceId, + "run", + ).Return(nil, fmt.Errorf("connection refused")).Once() + + mockClient.EXPECT().DeleteWorkspace( + workspaceId, + ).Return(nil).Once() + + opts.Timeout = 100 * time.Millisecond + err := c.RunSmoketest() + Expect(err).To(MatchError(ContainSubstring("timed out"))) + Expect(err).To(MatchError(ContainSubstring("connection refused"))) + }) + + It("does not return success when only IDE server is in running state", func() { + mockClient.EXPECT().CreateWorkspace( + teamIdInt, + planIdInt, + mock.AnythingOfType("string"), + (*string)(nil), + ).Return(workspaceId, nil).Once() + + mockClient.EXPECT().SetEnvVar( + workspaceId, + "TEST_VAR", + "smoketest", + ).Return(nil).Once() + + mockClient.EXPECT().ExecuteCommand( + workspaceId, + mock.MatchedBy(func(cmd string) bool { + return strings.Contains(cmd, "> ci.yml") + }), + ).Return(nil).Once() + + mockClient.EXPECT().ExecuteCommand( + workspaceId, + mock.MatchedBy(func(cmd string) bool { + return strings.Contains(cmd, "> index.html") + }), + ).Return(nil).Once() + + mockClient.EXPECT().SyncLandscape( + workspaceId, + "ci.yml", + ).Return(nil).Once() + + mockClient.EXPECT().StartPipeline( + workspaceId, + "ci.yml", + "run", + ).Return(nil).Once() + + // Only IDE server running — must NOT be treated as success + mockClient.EXPECT().GetPipelineState( + workspaceId, + "run", + ).Return([]api.PipelineStatus{ + {State: "running", Server: "codesphere-ide", Replica: "replica-0"}, + }, nil).Once() + + mockClient.EXPECT().DeleteWorkspace( + workspaceId, + ).Return(nil).Once() + + opts.Timeout = 100 * time.Millisecond + err := c.RunSmoketest() + Expect(err).To(MatchError(ContainSubstring("timed out"))) + }) + It("returns cleanup error when DeleteWorkspace fails", func() { mockClient.EXPECT().CreateWorkspace( - teamIdInt, // teamID - planIdInt, // planID + teamIdInt, + planIdInt, mock.AnythingOfType("string"), (*string)(nil), // empty workspace ).Return(workspaceId, nil).Once() @@ -377,6 +602,13 @@ var _ = Describe("SmoketestCodesphereCmd", func() { "run", ).Return(nil).Once() + mockClient.EXPECT().GetPipelineState( + workspaceId, + "run", + ).Return([]api.PipelineStatus{ + {State: "running", Server: "customer-server", Replica: "replica-0"}, + }, nil).Once() + mockClient.EXPECT().DeleteWorkspace( workspaceId, ).Return(fmt.Errorf("delete failed")).Once() @@ -389,8 +621,8 @@ var _ = Describe("SmoketestCodesphereCmd", func() { opts.Steps = []string{"createWorkspace", "setEnvVar"} mockClient.EXPECT().CreateWorkspace( - teamIdInt, // teamID - planIdInt, // planID + teamIdInt, + planIdInt, mock.AnythingOfType("string"), (*string)(nil), ).Return(workspaceId, nil).Once() diff --git a/docs/oms_smoketest_codesphere.md b/docs/oms_smoketest_codesphere.md index dbe9e901..7cdd0b55 100644 --- a/docs/oms_smoketest_codesphere.md +++ b/docs/oms_smoketest_codesphere.md @@ -46,7 +46,7 @@ $ oms smoketest codesphere --baseurl https://codesphere.example.com/api --token --plan-id string Plan ID for workspace creation --profile string CI profile to use for landscape and pipeline (default "ci.yml") -q, --quiet Suppress progress logging - --steps strings Comma-separated list of steps to run (createWorkspace,setEnvVar,createFiles,syncLandscape,startPipeline,deleteWorkspace). If empty, all steps including deleteWorkspace are run. If specified without deleteWorkspace, the workspace will be kept for manual inspection. + --steps strings Comma-separated list of steps to run (createWorkspace,setEnvVar,createFiles,syncLandscape,executeRunStage,deleteWorkspace). If empty, all steps including deleteWorkspace are run. If specified without deleteWorkspace, the workspace will be kept for manual inspection. --team-id string Team ID for workspace creation --timeout duration Timeout for the entire smoke test (default 10m0s) --token string API token for authentication diff --git a/internal/codesphere/codesphere.go b/internal/codesphere/codesphere.go index b59fb2a1..30d8af8f 100644 --- a/internal/codesphere/codesphere.go +++ b/internal/codesphere/codesphere.go @@ -21,6 +21,7 @@ type Client interface { ExecuteCommand(workspaceID int, command string) error SyncLandscape(workspaceID int, profile string) error StartPipeline(workspaceID int, profile, stage string) error + GetPipelineState(workspaceID int, stage string) ([]api.PipelineStatus, error) DeleteWorkspace(workspaceID int) error ListWorkspacePlans() ([]api.WorkspacePlan, error) ListTeams() ([]api.Team, error) @@ -108,6 +109,15 @@ func (c *APIClient) StartPipeline(workspaceID int, profile, stage string) error return nil } +// GetPipelineState returns the current state of a pipeline stage +func (c *APIClient) GetPipelineState(workspaceID int, stage string) ([]api.PipelineStatus, error) { + states, err := c.client.GetPipelineState(workspaceID, stage) + if err != nil { + return nil, fmt.Errorf("failed to get pipeline state: %w", err) + } + return states, nil +} + // DeleteWorkspace deletes a workspace func (c *APIClient) DeleteWorkspace(workspaceID int) error { err := c.client.DeleteWorkspace(workspaceID) diff --git a/internal/codesphere/mocks.go b/internal/codesphere/mocks.go index f236d994..6e38480f 100644 --- a/internal/codesphere/mocks.go +++ b/internal/codesphere/mocks.go @@ -222,6 +222,74 @@ func (_c *MockClient_ExecuteCommand_Call) RunAndReturn(run func(workspaceID int, return _c } +// GetPipelineState provides a mock function for the type MockClient +func (_mock *MockClient) GetPipelineState(workspaceID int, stage string) ([]api.PipelineStatus, error) { + ret := _mock.Called(workspaceID, stage) + + if len(ret) == 0 { + panic("no return value specified for GetPipelineState") + } + + var r0 []api.PipelineStatus + var r1 error + if returnFunc, ok := ret.Get(0).(func(int, string) ([]api.PipelineStatus, error)); ok { + return returnFunc(workspaceID, stage) + } + if returnFunc, ok := ret.Get(0).(func(int, string) []api.PipelineStatus); ok { + r0 = returnFunc(workspaceID, stage) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]api.PipelineStatus) + } + } + if returnFunc, ok := ret.Get(1).(func(int, string) error); ok { + r1 = returnFunc(workspaceID, stage) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockClient_GetPipelineState_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetPipelineState' +type MockClient_GetPipelineState_Call struct { + *mock.Call +} + +// GetPipelineState is a helper method to define mock.On call +// - workspaceID int +// - stage string +func (_e *MockClient_Expecter) GetPipelineState(workspaceID interface{}, stage interface{}) *MockClient_GetPipelineState_Call { + return &MockClient_GetPipelineState_Call{Call: _e.mock.On("GetPipelineState", workspaceID, stage)} +} + +func (_c *MockClient_GetPipelineState_Call) Run(run func(workspaceID int, stage string)) *MockClient_GetPipelineState_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 int + if args[0] != nil { + arg0 = args[0].(int) + } + var arg1 string + if args[1] != nil { + arg1 = args[1].(string) + } + run( + arg0, + arg1, + ) + }) + return _c +} + +func (_c *MockClient_GetPipelineState_Call) Return(vs []api.PipelineStatus, err error) *MockClient_GetPipelineState_Call { + _c.Call.Return(vs, err) + return _c +} + +func (_c *MockClient_GetPipelineState_Call) RunAndReturn(run func(workspaceID int, stage string) ([]api.PipelineStatus, error)) *MockClient_GetPipelineState_Call { + _c.Call.Return(run) + return _c +} + // ListTeams provides a mock function for the type MockClient func (_mock *MockClient) ListTeams() ([]api.Team, error) { ret := _mock.Called() diff --git a/internal/codesphere/teststeps/csgo.go b/internal/codesphere/teststeps/csgo.go index b10a9690..3e85658b 100644 --- a/internal/codesphere/teststeps/csgo.go +++ b/internal/codesphere/teststeps/csgo.go @@ -6,6 +6,7 @@ package teststeps import ( "context" "fmt" + "log" "strconv" "time" ) @@ -19,9 +20,15 @@ const ( stepNameSetEnvVar = "setEnvVar" stepNameCreateFiles = "createFiles" stepNameSyncLandscape = "syncLandscape" - stepNameStartPipeline = "startPipeline" + stepNameExecuteRunStage = "executeRunStage" stepNameDeleteWorkspace = "deleteWorkspace" + ideServer = "codesphere-ide" + pipelineStateRunning = "running" + pipelineStateFailure = "failure" + pipelineStateAborted = "aborted" + pipelineStatePollDelay = 5 * time.Second + ciYmlContent = `schemaVersion: v0.2 prepare: steps: [] @@ -172,18 +179,72 @@ func (s *SyncLandscapeStep) Run(ctx context.Context, c *SmoketestCodesphereOpts, return nil } -type StartPipelineStep struct{} +type ExecuteRunStageStep struct{} -func (s *StartPipelineStep) Name() string { return stepNameStartPipeline } +func (s *ExecuteRunStageStep) Name() string { return stepNameExecuteRunStage } -func (s *StartPipelineStep) Run(ctx context.Context, c *SmoketestCodesphereOpts, workspaceID *int) error { - c.logStep(fmt.Sprintf("Starting '%s' pipeline stage", smoketestPipelineStage)) +func (s *ExecuteRunStageStep) Run(ctx context.Context, c *SmoketestCodesphereOpts, workspaceID *int) error { + c.logStep(fmt.Sprintf("Executing '%s' pipeline stage", smoketestPipelineStage)) if err := c.Client.StartPipeline(*workspaceID, c.Profile, smoketestPipelineStage); err != nil { c.logFailure() return fmt.Errorf("failed to start pipeline: %w", err) } - c.logSuccess() - return nil + var lastErr error + for { + select { + case <-ctx.Done(): + c.logFailure() + if lastErr != nil { + return fmt.Errorf("timed out waiting for workspace to be running: %w", lastErr) + } + return fmt.Errorf("timed out waiting for workspace to be running") + default: + } + + states, err := c.Client.GetPipelineState(*workspaceID, smoketestPipelineStage) + if err != nil { + lastErr = err + log.Printf("failed to get pipeline state, retrying: %s", err) + select { + case <-ctx.Done(): + c.logFailure() + return fmt.Errorf("timed out waiting for workspace to be running: %w", lastErr) + case <-time.After(pipelineStatePollDelay): + continue + } + } + + for _, st := range states { + if st.State == pipelineStateFailure || st.State == pipelineStateAborted { + c.logFailure() + return fmt.Errorf("workspace run stage reached unexpected state: %s (server: %s, replica: %s)", st.State, st.Server, st.Replica) + } + } + + hasNonIdeServer := false + allNonIdeRunning := true + for _, st := range states { + if st.Server == ideServer { + continue + } + hasNonIdeServer = true + if st.State != pipelineStateRunning { + allNonIdeRunning = false + break + } + } + if hasNonIdeServer && allNonIdeRunning { + c.logSuccess() + return nil + } + + select { + case <-ctx.Done(): + c.logFailure() + return fmt.Errorf("timed out waiting for workspace to be running") + case <-time.After(pipelineStatePollDelay): + } + } } type DeleteWorkspaceStep struct{}