From e518dc7f41792be2ff4371337154612911c7bbab Mon Sep 17 00:00:00 2001 From: Deepak Bhatt Date: Fri, 28 Mar 2025 13:15:20 +0000 Subject: [PATCH] feat: [CDE-697]: Add async flow when gitspace agent fails to perform action (#3602) * feat: [CDE-697]: fix stop async flow * feat: [CDE-697]: Add async flow when gitspace agent fails to perform action --- app/events/gitspaceoperations/events.go | 2 +- .../container/devcontainer_container_utils.go | 1 + .../embedded_docker_container_orchestrator.go | 18 +++++++---- .../orchestrator/container/response/delete.go | 4 ++- .../orchestrator/container/response/start.go | 2 ++ .../orchestrator/container/response/status.go | 22 ++++++++++++++ .../orchestrator/container/response/stop.go | 20 +++++++++++++ .../orchestrator/orchestrator_resume.go | 14 ++++++++- .../orchestrator/orchestrator_trigger.go | 30 ++++++++++++++++--- .../gitspaceoperationsevent/handler.go | 16 ++++++++-- events/reporter.go | 1 + 11 files changed, 116 insertions(+), 14 deletions(-) create mode 100644 app/gitspace/orchestrator/container/response/status.go create mode 100644 app/gitspace/orchestrator/container/response/stop.go diff --git a/app/events/gitspaceoperations/events.go b/app/events/gitspaceoperations/events.go index 9bc9ecaa7..09e2ddb8c 100644 --- a/app/events/gitspaceoperations/events.go +++ b/app/events/gitspaceoperations/events.go @@ -47,7 +47,7 @@ func (r *Reporter) EmitGitspaceOperationsEvent( } eventID, err := events.ReporterSendEvent(r.innerReporter, ctx, event, payload) if err != nil { - return fmt.Errorf("failed to send %+v event", event) + return fmt.Errorf("failed to send %s event: %w", event, err) } log.Ctx(ctx).Debug().Msgf("reported %v event with id '%s'", event, eventID) diff --git a/app/gitspace/orchestrator/container/devcontainer_container_utils.go b/app/gitspace/orchestrator/container/devcontainer_container_utils.go index f11aadbed..ec575fa9d 100644 --- a/app/gitspace/orchestrator/container/devcontainer_container_utils.go +++ b/app/gitspace/orchestrator/container/devcontainer_container_utils.go @@ -805,6 +805,7 @@ func GetContainerResponse( codeRepoDir := filepath.Join(homeDir, repoName) return &response.StartResponse{ + Status: response.SuccessStatus, ContainerID: id, ContainerName: containerName, PublishedPorts: ports, diff --git a/app/gitspace/orchestrator/container/embedded_docker_container_orchestrator.go b/app/gitspace/orchestrator/container/embedded_docker_container_orchestrator.go index baea7179a..8ca319f16 100644 --- a/app/gitspace/orchestrator/container/embedded_docker_container_orchestrator.go +++ b/app/gitspace/orchestrator/container/embedded_docker_container_orchestrator.go @@ -318,12 +318,17 @@ func (e *EmbeddedDockerOrchestrator) StopGitspace( return fmt.Errorf("gitspace %s is in a bad state: %s", containerName, state) } + stopResponse := &response.StopResponse{ + Status: response.SuccessStatus, + } + err = e.eventReporter.EmitGitspaceOperationsEvent( ctx, events.GitspaceOperationsEvent, &events.GitspaceOperationsEventPayload{ - Type: enum.GitspaceOperationsEventStop, - Infra: infra, + Type: enum.GitspaceOperationsEventStop, + Infra: infra, + Response: stopResponse, }, ) logger.Debug().Msg("stopped gitspace") @@ -411,9 +416,12 @@ func (e *EmbeddedDockerOrchestrator) StopAndRemoveGitspace( ctx, events.GitspaceOperationsEvent, &events.GitspaceOperationsEventPayload{ - Type: enum.GitspaceOperationsEventDelete, - Infra: infra, - Response: &response.DeleteResponse{CanDeleteUserData: canDeleteUserData}, + Type: enum.GitspaceOperationsEventDelete, + Infra: infra, + Response: &response.DeleteResponse{ + Status: response.SuccessStatus, + CanDeleteUserData: canDeleteUserData, + }, }, ) logger.Debug().Msg("removed gitspace") diff --git a/app/gitspace/orchestrator/container/response/delete.go b/app/gitspace/orchestrator/container/response/delete.go index e35abac87..ce95cf870 100644 --- a/app/gitspace/orchestrator/container/response/delete.go +++ b/app/gitspace/orchestrator/container/response/delete.go @@ -15,5 +15,7 @@ package response type DeleteResponse struct { - CanDeleteUserData bool `json:"can_delete_user_data"` + Status Status `json:"status"` + ErrMessage string `json:"err_message"` + CanDeleteUserData bool `json:"can_delete_user_data"` } diff --git a/app/gitspace/orchestrator/container/response/start.go b/app/gitspace/orchestrator/container/response/start.go index 95eb37839..7327e6392 100644 --- a/app/gitspace/orchestrator/container/response/start.go +++ b/app/gitspace/orchestrator/container/response/start.go @@ -16,6 +16,8 @@ package response type StartResponse struct { ContainerID string `json:"container_id"` + Status Status `json:"status"` + ErrMessage string `json:"err_message"` ContainerName string `json:"container_name"` PublishedPorts map[int]string `json:"published_ports"` AbsoluteRepoPath string `json:"absolute_repo_path"` diff --git a/app/gitspace/orchestrator/container/response/status.go b/app/gitspace/orchestrator/container/response/status.go new file mode 100644 index 000000000..0069d5dff --- /dev/null +++ b/app/gitspace/orchestrator/container/response/status.go @@ -0,0 +1,22 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package response + +type Status string + +const ( + SuccessStatus Status = "success" + FailureStatus Status = "failure" +) diff --git a/app/gitspace/orchestrator/container/response/stop.go b/app/gitspace/orchestrator/container/response/stop.go new file mode 100644 index 000000000..7ecfffef3 --- /dev/null +++ b/app/gitspace/orchestrator/container/response/stop.go @@ -0,0 +1,20 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package response + +type StopResponse struct { + Status Status `json:"status"` + ErrMessage string `json:"err_message"` +} diff --git a/app/gitspace/orchestrator/orchestrator_resume.go b/app/gitspace/orchestrator/orchestrator_resume.go index 5326a9e97..ded964fa6 100644 --- a/app/gitspace/orchestrator/orchestrator_resume.go +++ b/app/gitspace/orchestrator/orchestrator_resume.go @@ -149,8 +149,20 @@ func (o Orchestrator) FinishResumeStartGitspace( provisionedInfra types.Infrastructure, startResponse *response.StartResponse, ) (types.GitspaceInstance, *types.GitspaceError) { - o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentGitspaceCreationCompleted) gitspaceInstance := gitspaceConfig.GitspaceInstance + if startResponse == nil || startResponse.Status == response.FailureStatus { + gitspaceInstance.State = enum.GitspaceInstanceStateError + err := fmt.Errorf("gitspace agent does not specify the error for failure") + if startResponse != nil && startResponse.ErrMessage != "" { + err = fmt.Errorf("%s", startResponse.ErrMessage) + } + return *gitspaceInstance, &types.GitspaceError{ + Error: err, + ErrorMessage: ptr.String(err.Error()), + } + } + + o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentGitspaceCreationCompleted) ideSvc, err := o.ideFactory.GetIDE(gitspaceConfig.IDE) if err != nil { diff --git a/app/gitspace/orchestrator/orchestrator_trigger.go b/app/gitspace/orchestrator/orchestrator_trigger.go index 498ad4c4d..9639f8479 100644 --- a/app/gitspace/orchestrator/orchestrator_trigger.go +++ b/app/gitspace/orchestrator/orchestrator_trigger.go @@ -22,6 +22,7 @@ import ( events "github.com/harness/gitness/app/events/gitspace" "github.com/harness/gitness/app/gitspace/infrastructure" "github.com/harness/gitness/app/gitspace/orchestrator/container" + "github.com/harness/gitness/app/gitspace/orchestrator/container/response" "github.com/harness/gitness/app/gitspace/orchestrator/ide" "github.com/harness/gitness/app/gitspace/platformconnector" "github.com/harness/gitness/app/gitspace/scm" @@ -206,7 +207,18 @@ func (o Orchestrator) FinishStopGitspaceContainer( ctx context.Context, gitspaceConfig types.GitspaceConfig, infra types.Infrastructure, + stopResponse *response.StopResponse, ) *types.GitspaceError { + if stopResponse == nil || stopResponse.Status == response.FailureStatus { + err := fmt.Errorf("gitspace agent does not specify the error for failure") + if stopResponse != nil && stopResponse.ErrMessage != "" { + err = fmt.Errorf("%s", stopResponse.ErrMessage) + } + return &types.GitspaceError{ + Error: err, + ErrorMessage: ptr.String(err.Error()), + } + } o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentGitspaceStopCompleted) o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraStopStart) @@ -257,19 +269,29 @@ func (o Orchestrator) FinishStopAndRemoveGitspaceContainer( ctx context.Context, gitspaceConfig types.GitspaceConfig, infra types.Infrastructure, - canDeleteUserData bool, + deleteResponse *response.DeleteResponse, ) *types.GitspaceError { + if deleteResponse == nil || deleteResponse.Status == response.FailureStatus { + err := fmt.Errorf("gitspace agent does not specify the error for failure") + if deleteResponse != nil && deleteResponse.ErrMessage != "" { + err = fmt.Errorf("%s", deleteResponse.ErrMessage) + } + return &types.GitspaceError{ + Error: err, + ErrorMessage: ptr.String(err.Error()), + } + } o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentGitspaceDeletionCompleted) - if canDeleteUserData { + if deleteResponse.CanDeleteUserData { o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraDeprovisioningStart) } else { o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraResetStart) } - opts := infrastructure.InfraEventOpts{CanDeleteUserData: canDeleteUserData} + opts := infrastructure.InfraEventOpts{CanDeleteUserData: deleteResponse.CanDeleteUserData} err := o.infraProvisioner.TriggerInfraEventWithOpts(ctx, enum.InfraEventDeprovision, gitspaceConfig, &infra, opts) if err != nil { - if canDeleteUserData { + if deleteResponse.CanDeleteUserData { o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraDeprovisioningFailed) } else { o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraResetFailed) diff --git a/app/services/gitspaceoperationsevent/handler.go b/app/services/gitspaceoperationsevent/handler.go index f2996a4bd..d0386d9a6 100644 --- a/app/services/gitspaceoperationsevent/handler.go +++ b/app/services/gitspaceoperationsevent/handler.go @@ -86,14 +86,25 @@ func (s *Service) handleGitspaceOperationsEvent( ) if handleResumeStartErr != nil { s.emitGitspaceConfigEvent(ctxWithTimedOut, config, enum.GitspaceEventTypeGitspaceActionStartFailed) + instance.State = enum.GitspaceInstanceStateError updatedInstance.ErrorMessage = handleResumeStartErr.ErrorMessage err = fmt.Errorf("failed to finish resume start gitspace: %w", handleResumeStartErr.Error) } instance = &updatedInstance case enum.GitspaceOperationsEventStop: - finishStopErr := s.orchestrator.FinishStopGitspaceContainer(ctxWithTimedOut, *config, payload.Infra) + stopResponse, ok := payload.Response.(*response.StopResponse) + if !ok { + return fmt.Errorf("failed to cast stop response") + } + finishStopErr := s.orchestrator.FinishStopGitspaceContainer( + ctxWithTimedOut, + *config, + payload.Infra, + stopResponse, + ) if finishStopErr != nil { s.emitGitspaceConfigEvent(ctxWithTimedOut, config, enum.GitspaceEventTypeGitspaceActionStopFailed) + instance.State = enum.GitspaceInstanceStateError instance.ErrorMessage = finishStopErr.ErrorMessage err = fmt.Errorf("failed to finish trigger start gitspace: %w", finishStopErr.Error) } @@ -106,10 +117,11 @@ func (s *Service) handleGitspaceOperationsEvent( ctxWithTimedOut, *config, payload.Infra, - deleteResponse.CanDeleteUserData, + deleteResponse, ) if finishStopAndRemoveErr != nil { s.emitGitspaceConfigEvent(ctxWithTimedOut, config, enum.GitspaceEventTypeGitspaceActionStopFailed) + instance.State = enum.GitspaceInstanceStateError instance.ErrorMessage = finishStopAndRemoveErr.ErrorMessage err = fmt.Errorf("failed to finish trigger start gitspace: %w", finishStopAndRemoveErr.Error) } diff --git a/events/reporter.go b/events/reporter.go index 470f40bf0..1e00f581b 100644 --- a/events/reporter.go +++ b/events/reporter.go @@ -49,6 +49,7 @@ func ReporterSendEvent[T interface{}](reporter *GenericReporter, ctx context.Con buff := &bytes.Buffer{} encoder := gob.NewEncoder(buff) gob.Register((*response.StartResponse)(nil)) + gob.Register((*response.StopResponse)(nil)) gob.Register((*response.DeleteResponse)(nil)) if err := encoder.Encode(&event); err != nil {