mirror of
https://github.com/harness/drone.git
synced 2025-05-05 15:32:56 +00:00
feat: [CDE-753]: Fix backend handling of gitspace deletion requests (#3745)
* feat: [CDE-753]: Fix lint * feat: [CDE-753]: Fix backend handling of gitspace deletion requests
This commit is contained in:
parent
8311ae2c2f
commit
8d13a4beb3
@ -100,7 +100,7 @@ func (i InfraProvisioner) TriggerInfraEventWithOpts(
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
_, configMetadata, err := i.getAllParamsFromDB(ctx, gitspaceConfig.InfraProviderResource, infraProvider)
|
allParams, configMetadata, err := i.getAllParamsFromDB(ctx, gitspaceConfig.InfraProviderResource, infraProvider)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("could not get all params from DB while provisioning: %w", err)
|
return fmt.Errorf("could not get all params from DB while provisioning: %w", err)
|
||||||
}
|
}
|
||||||
@ -126,6 +126,8 @@ func (i InfraProvisioner) TriggerInfraEventWithOpts(
|
|||||||
gitspaceConfig,
|
gitspaceConfig,
|
||||||
opts.RequiredGitspacePorts,
|
opts.RequiredGitspacePorts,
|
||||||
stoppedInfra,
|
stoppedInfra,
|
||||||
|
configMetadata,
|
||||||
|
allParams,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
return i.provisionExistingInfrastructure(
|
return i.provisionExistingInfrastructure(
|
||||||
@ -145,9 +147,10 @@ func (i InfraProvisioner) TriggerInfraEventWithOpts(
|
|||||||
*infra,
|
*infra,
|
||||||
opts.CanDeleteUserData,
|
opts.CanDeleteUserData,
|
||||||
configMetadata,
|
configMetadata,
|
||||||
|
allParams,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
return infraProvider.Deprovision(ctx, *infra, opts.CanDeleteUserData, configMetadata)
|
return infraProvider.Deprovision(ctx, *infra, opts.CanDeleteUserData, configMetadata, allParams)
|
||||||
|
|
||||||
case enum.InfraEventCleanup:
|
case enum.InfraEventCleanup:
|
||||||
return infraProvider.CleanupInstanceResources(ctx, *infra)
|
return infraProvider.CleanupInstanceResources(ctx, *infra)
|
||||||
@ -167,6 +170,8 @@ func (i InfraProvisioner) provisionNewInfrastructure(
|
|||||||
gitspaceConfig types.GitspaceConfig,
|
gitspaceConfig types.GitspaceConfig,
|
||||||
requiredGitspacePorts []types.GitspacePort,
|
requiredGitspacePorts []types.GitspacePort,
|
||||||
stoppedInfra types.Infrastructure,
|
stoppedInfra types.Infrastructure,
|
||||||
|
configMetadata map[string]any,
|
||||||
|
allParams []types.InfraProviderParameter,
|
||||||
) error {
|
) error {
|
||||||
// Logic for new provisioning...
|
// Logic for new provisioning...
|
||||||
infraProvisionedLatest, _ := i.infraProvisionedStore.FindLatestByGitspaceInstanceID(
|
infraProvisionedLatest, _ := i.infraProvisionedStore.FindLatestByGitspaceInstanceID(
|
||||||
@ -185,12 +190,8 @@ func (i InfraProvisioner) provisionNewInfrastructure(
|
|||||||
}
|
}
|
||||||
|
|
||||||
infraProviderResource := gitspaceConfig.InfraProviderResource
|
infraProviderResource := gitspaceConfig.InfraProviderResource
|
||||||
allParams, configMetadata, err := i.getAllParamsFromDB(ctx, infraProviderResource, infraProvider)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("could not get all params from DB while provisioning: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = infraProvider.ValidateParams(allParams)
|
err := infraProvider.ValidateParams(allParams)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("invalid provisioning params %v: %w", infraProviderResource.Metadata, err)
|
return fmt.Errorf("invalid provisioning params %v: %w", infraProviderResource.Metadata, err)
|
||||||
}
|
}
|
||||||
@ -210,6 +211,7 @@ func (i InfraProvisioner) provisionNewInfrastructure(
|
|||||||
InputParameters: allParams,
|
InputParameters: allParams,
|
||||||
ConfigMetadata: configMetadata,
|
ConfigMetadata: configMetadata,
|
||||||
InstanceInfo: stoppedInfra.InstanceInfo,
|
InstanceInfo: stoppedInfra.InstanceInfo,
|
||||||
|
Status: enum.InfraStatusPending,
|
||||||
}
|
}
|
||||||
responseMetadata, err := json.Marshal(infrastructure)
|
responseMetadata, err := json.Marshal(infrastructure)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -315,6 +317,7 @@ func (i InfraProvisioner) deprovisionNewInfrastructure(
|
|||||||
infra types.Infrastructure,
|
infra types.Infrastructure,
|
||||||
canDeleteUserData bool,
|
canDeleteUserData bool,
|
||||||
configMetadata map[string]any,
|
configMetadata map[string]any,
|
||||||
|
params []types.InfraProviderParameter,
|
||||||
) error {
|
) error {
|
||||||
infraProvisionedLatest, err := i.infraProvisionedStore.FindLatestByGitspaceInstanceID(
|
infraProvisionedLatest, err := i.infraProvisionedStore.FindLatestByGitspaceInstanceID(
|
||||||
ctx, gitspaceConfig.GitspaceInstance.ID)
|
ctx, gitspaceConfig.GitspaceInstance.ID)
|
||||||
@ -328,7 +331,7 @@ func (i InfraProvisioner) deprovisionNewInfrastructure(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
err = infraProvider.Deprovision(ctx, infra, canDeleteUserData, configMetadata)
|
err = infraProvider.Deprovision(ctx, infra, canDeleteUserData, configMetadata, params)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("unable to trigger deprovision infra %+v: %w", infra, err)
|
return fmt.Errorf("unable to trigger deprovision infra %+v: %w", infra, err)
|
||||||
}
|
}
|
||||||
|
@ -344,17 +344,18 @@ func (o Orchestrator) TriggerCleanupInstanceResources(ctx context.Context, gitsp
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// TriggerDeleteGitspace removes the Gitspace container and triggers infra deprovisioning to deprovision
|
// TriggerStopAndDeleteGitspace removes the Gitspace container and triggers infra deprovisioning to deprovision
|
||||||
// the infra resources.
|
// the infra resources.
|
||||||
// canDeleteUserData = false -> trigger deprovision of all resources except storage associated to user data.
|
// canDeleteUserData = false -> trigger deprovision of all resources except storage associated to user data.
|
||||||
// canDeleteUserData = true -> trigger deprovision of all resources.
|
// canDeleteUserData = true -> trigger deprovision of all resources.
|
||||||
func (o Orchestrator) TriggerDeleteGitspace(
|
func (o Orchestrator) TriggerStopAndDeleteGitspace(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
gitspaceConfig types.GitspaceConfig,
|
gitspaceConfig types.GitspaceConfig,
|
||||||
canDeleteUserData bool,
|
canDeleteUserData bool,
|
||||||
) error {
|
) error {
|
||||||
infra, err := o.getProvisionedInfra(ctx, gitspaceConfig,
|
infra, err := o.getProvisionedInfra(ctx, gitspaceConfig,
|
||||||
[]enum.InfraStatus{
|
[]enum.InfraStatus{
|
||||||
|
enum.InfraStatusPending,
|
||||||
enum.InfraStatusProvisioned,
|
enum.InfraStatusProvisioned,
|
||||||
enum.InfraStatusStopped,
|
enum.InfraStatusStopped,
|
||||||
enum.InfraStatusDestroyed,
|
enum.InfraStatusDestroyed,
|
||||||
@ -368,6 +369,8 @@ func (o Orchestrator) TriggerDeleteGitspace(
|
|||||||
}
|
}
|
||||||
if err = o.stopAndRemoveGitspaceContainer(ctx, gitspaceConfig, *infra, canDeleteUserData); err != nil {
|
if err = o.stopAndRemoveGitspaceContainer(ctx, gitspaceConfig, *infra, canDeleteUserData); err != nil {
|
||||||
log.Warn().Msgf("error stopping and removing gitspace container: %v", err)
|
log.Warn().Msgf("error stopping and removing gitspace container: %v", err)
|
||||||
|
// If stop fails, delete the gitspace anyway
|
||||||
|
return o.triggerDeleteGitspace(ctx, gitspaceConfig, infra, canDeleteUserData)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: Add a job for cleanup of infra if stop fails
|
// TODO: Add a job for cleanup of infra if stop fails
|
||||||
@ -375,22 +378,56 @@ func (o Orchestrator) TriggerDeleteGitspace(
|
|||||||
"Checking and force deleting the infra if required for gitspace instance %s",
|
"Checking and force deleting the infra if required for gitspace instance %s",
|
||||||
gitspaceConfig.GitspaceInstance.Identifier,
|
gitspaceConfig.GitspaceInstance.Identifier,
|
||||||
)
|
)
|
||||||
ticker := time.NewTicker(60 * time.Second)
|
if err = o.waitForGitspaceCleanupOrTimeout(ctx, gitspaceConfig, 15*time.Minute, 60*time.Second); err != nil {
|
||||||
timeout := time.After(15 * time.Minute)
|
return o.triggerDeleteGitspace(ctx, gitspaceConfig, infra, canDeleteUserData)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o Orchestrator) triggerDeleteGitspace(
|
||||||
|
ctx context.Context,
|
||||||
|
gitspaceConfig types.GitspaceConfig,
|
||||||
|
infra *types.Infrastructure,
|
||||||
|
canDeleteUserData bool,
|
||||||
|
) error {
|
||||||
|
opts := infrastructure.InfraEventOpts{CanDeleteUserData: true}
|
||||||
|
err := o.infraProvisioner.TriggerInfraEventWithOpts(
|
||||||
|
ctx,
|
||||||
|
enum.InfraEventDeprovision,
|
||||||
|
gitspaceConfig,
|
||||||
|
infra,
|
||||||
|
opts,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
if canDeleteUserData {
|
||||||
|
o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraDeprovisioningFailed)
|
||||||
|
} else {
|
||||||
|
o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraResetFailed)
|
||||||
|
}
|
||||||
|
return fmt.Errorf(
|
||||||
|
"cannot trigger deprovision infrastructure with gitspace identifier %s: %w",
|
||||||
|
gitspaceConfig.GitspaceInstance.Identifier,
|
||||||
|
err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o Orchestrator) waitForGitspaceCleanupOrTimeout(
|
||||||
|
ctx context.Context,
|
||||||
|
gitspaceConfig types.GitspaceConfig,
|
||||||
|
timeoutDuration time.Duration,
|
||||||
|
tickInterval time.Duration,
|
||||||
|
) error {
|
||||||
|
ticker := time.NewTicker(tickInterval)
|
||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
ch := make(chan error)
|
|
||||||
|
timeout := time.After(timeoutDuration)
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case msg := <-ch:
|
|
||||||
if msg == nil {
|
|
||||||
return msg
|
|
||||||
}
|
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
instance, err := o.gitspaceInstanceStore.Find(
|
instance, err := o.gitspaceInstanceStore.Find(ctx, gitspaceConfig.GitspaceInstance.ID)
|
||||||
ctx,
|
|
||||||
gitspaceConfig.GitspaceInstance.ID,
|
|
||||||
)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"failed to find gitspace instance %s: %w",
|
"failed to find gitspace instance %s: %w",
|
||||||
@ -403,27 +440,10 @@ func (o Orchestrator) TriggerDeleteGitspace(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
case <-timeout:
|
case <-timeout:
|
||||||
opts := infrastructure.InfraEventOpts{CanDeleteUserData: true}
|
return fmt.Errorf(
|
||||||
err := o.infraProvisioner.TriggerInfraEventWithOpts(
|
"timeout waiting for gitspace cleanup for instance %s",
|
||||||
ctx,
|
gitspaceConfig.GitspaceInstance.Identifier,
|
||||||
enum.InfraEventDeprovision,
|
|
||||||
gitspaceConfig,
|
|
||||||
infra,
|
|
||||||
opts,
|
|
||||||
)
|
)
|
||||||
if err != nil {
|
|
||||||
if canDeleteUserData {
|
|
||||||
o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraDeprovisioningFailed)
|
|
||||||
} else {
|
|
||||||
o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraResetFailed)
|
|
||||||
}
|
|
||||||
return fmt.Errorf(
|
|
||||||
"cannot trigger deprovision infrastructure with gitspace identifier %s: %w",
|
|
||||||
gitspaceConfig.GitspaceInstance.Identifier,
|
|
||||||
err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -89,7 +89,7 @@ func (c *Service) RemoveGitspace(ctx context.Context, config types.GitspaceConfi
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.orchestrator.TriggerDeleteGitspace(ctx, config, canDeleteUserData); err != nil {
|
if err := c.orchestrator.TriggerStopAndDeleteGitspace(ctx, config, canDeleteUserData); err != nil {
|
||||||
log.Ctx(ctx).Err(err).Msgf("error during triggering delete for gitspace instance %s",
|
log.Ctx(ctx).Err(err).Msgf("error during triggering delete for gitspace instance %s",
|
||||||
config.GitspaceInstance.Identifier)
|
config.GitspaceInstance.Identifier)
|
||||||
config.GitspaceInstance.State = enum.GitspaceInstanceStateError
|
config.GitspaceInstance.State = enum.GitspaceInstanceStateError
|
||||||
|
@ -208,6 +208,7 @@ func (d DockerProvider) Deprovision(
|
|||||||
infra types.Infrastructure,
|
infra types.Infrastructure,
|
||||||
canDeleteUserData bool,
|
canDeleteUserData bool,
|
||||||
_ map[string]any,
|
_ map[string]any,
|
||||||
|
_ []types.InfraProviderParameter,
|
||||||
) error {
|
) error {
|
||||||
if canDeleteUserData {
|
if canDeleteUserData {
|
||||||
err := d.deleteVolume(ctx, infra)
|
err := d.deleteVolume(ctx, infra)
|
||||||
|
@ -67,6 +67,7 @@ type InfraProvider interface {
|
|||||||
infra types.Infrastructure,
|
infra types.Infrastructure,
|
||||||
canDeleteUserData bool,
|
canDeleteUserData bool,
|
||||||
configMetadata map[string]any,
|
configMetadata map[string]any,
|
||||||
|
params []types.InfraProviderParameter,
|
||||||
) error
|
) error
|
||||||
|
|
||||||
// AvailableParams provides a schema to define the infrastructure.
|
// AvailableParams provides a schema to define the infrastructure.
|
||||||
|
Loading…
x
Reference in New Issue
Block a user