Sean monitioring manual probes (#1128)

* add a monitoring probe

* add in untracked files

* don't need the security thing

* revert this change

* go generate

* remove the namespace stuff

* cleanup

* clean up workflow afterwards, turn off logs for client and worker wait for tasks to finish so we don't cause errors

* clean up workflow afterwards, turn off logs for client and worker wait for tasks to finish so we don't cause errors

* extend the delay a little

* use the tenant from the authed token

* remove static field

* PR feedback add polling for workflow state before deletion, check the step runs finish in the right order

* PR review comments

* tweak the timeout so we don't see errors in dev and fix description

* fix the open api build

* modify return type, return if we delete or do it 10 times, 10 second timeout for deletion attempts

---------

Co-authored-by: Sean Reilly <sean@hatchet.run>
This commit is contained in:
Sean Reilly
2024-12-17 14:37:30 -08:00
committed by GitHub
parent cbc2526c0b
commit 53d5347ef4

View File

@@ -24,7 +24,8 @@ import (
func (m *MonitoringService) MonitoringPostRunProbe(ctx echo.Context, request gen.MonitoringPostRunProbeRequestObject) (gen.MonitoringPostRunProbeResponseObject, error) {
if !m.enabled {
return nil, fmt.Errorf("monitoring is not enabled")
m.l.Error().Msg("monitoring is not enabled")
return gen.MonitoringPostRunProbe403JSONResponse{}, nil
}
tenant := ctx.Get("tenant").(*db.TenantModel)
@@ -259,7 +260,8 @@ func (m *MonitoringService) run(ctx context.Context, cf clientconfig.ClientConfi
return
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
for i := 0; i < 10; i++ {
@@ -276,11 +278,14 @@ func (m *MonitoringService) run(ctx context.Context, cf clientconfig.ClientConfi
_, err = m.config.APIRepository.Workflow().DeleteWorkflow(ctx, cf.TenantId, workflowId)
m.l.Info().Msgf("deleted workflow %s", workflowId)
if err != nil {
m.l.Error().Msgf("error deleting workflow: %s", err)
} else {
m.l.Info().Msgf("deleted workflow %s", workflowId)
return
}
return
}
time.Sleep(200 * time.Millisecond)