diff --git a/api-contracts/workflows/workflows.proto b/api-contracts/workflows/workflows.proto index b9fe11192..edda4f1a9 100644 --- a/api-contracts/workflows/workflows.proto +++ b/api-contracts/workflows/workflows.proto @@ -60,6 +60,7 @@ message CreateWorkflowStepOpts { string inputs = 4; // (optional) the step inputs, assuming string representation of JSON repeated string parents = 5; // (optional) the step parents. if none are passed in, this is a root step string user_data = 6; // (optional) the custom step user data, assuming string representation of JSON + int32 retries = 7; // (optional) the number of retries for the step, default 0 } // ListWorkflowsRequest is the request for ListWorkflows. diff --git a/api/v1/server/handlers/step-runs/rerun.go b/api/v1/server/handlers/step-runs/rerun.go index 98d10b336..b054c4a11 100644 --- a/api/v1/server/handlers/step-runs/rerun.go +++ b/api/v1/server/handlers/step-runs/rerun.go @@ -13,7 +13,6 @@ import ( "github.com/hatchet-dev/hatchet/internal/datautils" "github.com/hatchet-dev/hatchet/internal/repository" "github.com/hatchet-dev/hatchet/internal/repository/prisma/db" - "github.com/hatchet-dev/hatchet/internal/schema" "github.com/hatchet-dev/hatchet/internal/services/shared/tasktypes" "github.com/hatchet-dev/hatchet/internal/taskqueue" ) @@ -38,12 +37,6 @@ func (t *StepRunService) StepRunUpdateRerun(ctx echo.Context, request gen.StepRu ), nil } - err = t.config.Repository.StepRun().ArchiveStepRunResult(tenant.ID, stepRun.ID) - - if err != nil { - return nil, fmt.Errorf("could not archive step run result: %w", err) - } - // make sure input can be marshalled and unmarshalled to input type inputBytes, err := json.Marshal(request.Body.Input) @@ -69,52 +62,31 @@ func (t *StepRunService) StepRunUpdateRerun(ctx echo.Context, request gen.StepRu ), nil } - // update the input schema for the step run based on the new input - jsonSchemaBytes, err := schema.SchemaBytesFromBytes(inputBytes) - - if err != nil { - return nil, err - } - - _, err = t.config.Repository.StepRun().UpdateStepRunInputSchema(tenant.ID, stepRun.ID, jsonSchemaBytes) - - if err != nil { - return nil, err - } - - // update step run - _, _, err = t.config.Repository.StepRun().UpdateStepRun(tenant.ID, stepRun.ID, &repository.UpdateStepRunOpts{ - Input: inputBytes, - Status: repository.StepRunStatusPtr(db.StepRunStatusPending), - IsRerun: true, - }) - - if err != nil { - return nil, fmt.Errorf("could not update step run: %w", err) - } - - // requeue the step run in the task queue - jobRun, err := t.config.Repository.JobRun().GetJobRunById(tenant.ID, stepRun.JobRunID) - - if err != nil { - return nil, fmt.Errorf("could not get job run: %w", err) - } - // send a task to the taskqueue err = t.config.TaskQueue.AddTask( ctx.Request().Context(), taskqueue.JOB_PROCESSING_QUEUE, - tasktypes.StepRunQueuedToTask(jobRun.Job(), stepRun), + tasktypes.StepRunRetryToTask(stepRun, inputBytes), ) if err != nil { return nil, fmt.Errorf("could not add step queued task to task queue: %w", err) } - stepRun, err = t.config.Repository.StepRun().GetStepRunById(tenant.ID, stepRun.ID) + // wait for a short period of time + for i := 0; i < 5; i++ { + newStepRun, err := t.config.Repository.StepRun().GetStepRunById(tenant.ID, stepRun.ID) - if err != nil { - return nil, fmt.Errorf("could not get step run: %w", err) + if err != nil { + return nil, fmt.Errorf("could not get step run: %w", err) + } + + if newStepRun.Status != stepRun.Status { + stepRun = newStepRun + break + } + + time.Sleep(100 * time.Millisecond) } res, err := transformers.ToStepRun(stepRun) diff --git a/examples/cron/main.go b/examples/cron/main.go index c64bc8ef4..2d6253626 100644 --- a/examples/cron/main.go +++ b/examples/cron/main.go @@ -4,10 +4,11 @@ import ( "context" "fmt" + "github.com/joho/godotenv" + "github.com/hatchet-dev/hatchet/pkg/client" "github.com/hatchet-dev/hatchet/pkg/cmdutils" "github.com/hatchet-dev/hatchet/pkg/worker" - "github.com/joho/godotenv" ) type printOutput struct{} @@ -31,8 +32,6 @@ func main() { panic(err) } - // Create a worker. This automatically reads in a TemporalClient from .env and workflow files from the .hatchet - // directory, but this can be customized with the `worker.WithTemporalClient` and `worker.WithWorkflowFiles` options. w, err := worker.NewWorker( worker.WithClient( client, diff --git a/examples/dag/main.go b/examples/dag/main.go index b58547224..b0962978b 100644 --- a/examples/dag/main.go +++ b/examples/dag/main.go @@ -42,8 +42,6 @@ func run(ch <-chan interface{}, events chan<- string) error { return fmt.Errorf("error creating client: %w", err) } - // Create a worker. This automatically reads in a TemporalClient from .env and workflow files from the .hatchet - // directory, but this can be customized with the `worker.WithTemporalClient` and `worker.WithWorkflowFiles` options. w, err := worker.NewWorker( worker.WithClient( c, diff --git a/examples/deprecated/requeue/main.go b/examples/deprecated/requeue/main.go index 3dfb98228..dce9eb2d7 100644 --- a/examples/deprecated/requeue/main.go +++ b/examples/deprecated/requeue/main.go @@ -4,10 +4,11 @@ import ( "context" "time" + "github.com/joho/godotenv" + "github.com/hatchet-dev/hatchet/pkg/client" "github.com/hatchet-dev/hatchet/pkg/cmdutils" "github.com/hatchet-dev/hatchet/pkg/worker" - "github.com/joho/godotenv" ) type sampleEvent struct{} @@ -29,8 +30,6 @@ func main() { panic(err) } - // Create a worker. This automatically reads in a TemporalClient from .env and workflow files from the .hatchet - // directory, but this can be customized with the `worker.WithTemporalClient` and `worker.WithWorkflowFiles` options. worker, err := worker.NewWorker( worker.WithClient( client, diff --git a/examples/deprecated/slack/main.go b/examples/deprecated/slack/main.go index ad330bb20..02a015795 100644 --- a/examples/deprecated/slack/main.go +++ b/examples/deprecated/slack/main.go @@ -8,12 +8,13 @@ import ( "context" "time" + "github.com/joho/godotenv" + "github.com/hatchet-dev/hatchet/pkg/client" "github.com/hatchet-dev/hatchet/pkg/client/types" "github.com/hatchet-dev/hatchet/pkg/cmdutils" "github.com/hatchet-dev/hatchet/pkg/integrations/slack" "github.com/hatchet-dev/hatchet/pkg/worker" - "github.com/joho/godotenv" ) type teamCreateEvent struct { @@ -75,8 +76,6 @@ func main() { panic(err) } - // Create a worker. This automatically reads in a TemporalClient from .env and workflow files from the .hatchet - // directory, but this can be customized with the `worker.WithTemporalClient` and `worker.WithWorkflowFiles` options. worker, err := worker.NewWorker( worker.WithClient( client, diff --git a/examples/deprecated/timeout/main.go b/examples/deprecated/timeout/main.go index 95faec9c9..4e77f1a67 100644 --- a/examples/deprecated/timeout/main.go +++ b/examples/deprecated/timeout/main.go @@ -5,10 +5,11 @@ import ( "fmt" "time" + "github.com/joho/godotenv" + "github.com/hatchet-dev/hatchet/pkg/client" "github.com/hatchet-dev/hatchet/pkg/cmdutils" "github.com/hatchet-dev/hatchet/pkg/worker" - "github.com/joho/godotenv" ) type sampleEvent struct{} @@ -30,8 +31,6 @@ func main() { panic(err) } - // Create a worker. This automatically reads in a TemporalClient from .env and workflow files from the .hatchet - // directory, but this can be customized with the `worker.WithTemporalClient` and `worker.WithWorkflowFiles` options. worker, err := worker.NewWorker( worker.WithClient( client, diff --git a/examples/deprecated/yaml/main.go b/examples/deprecated/yaml/main.go index 5bbb8dc17..49b6ff8f0 100644 --- a/examples/deprecated/yaml/main.go +++ b/examples/deprecated/yaml/main.go @@ -4,10 +4,11 @@ import ( "context" "time" + "github.com/joho/godotenv" + "github.com/hatchet-dev/hatchet/pkg/client" "github.com/hatchet-dev/hatchet/pkg/cmdutils" "github.com/hatchet-dev/hatchet/pkg/worker" - "github.com/joho/godotenv" ) type userCreateEvent struct { @@ -49,8 +50,6 @@ func main() { panic(err) } - // Create a worker. This automatically reads in a TemporalClient from .env and workflow files from the .hatchet - // directory, but this can be customized with the `worker.WithTemporalClient` and `worker.WithWorkflowFiles` options. worker, err := worker.NewWorker( worker.WithClient( client, diff --git a/examples/errors-test/main.go b/examples/errors-test/main.go index 0994b8b9f..92401c467 100644 --- a/examples/errors-test/main.go +++ b/examples/errors-test/main.go @@ -50,8 +50,6 @@ func main() { panic(err) } - // Create a worker. This automatically reads in a TemporalClient from .env and workflow files from the .hatchet - // directory, but this can be customized with the `worker.WithTemporalClient` and `worker.WithWorkflowFiles` options. w, err := worker.NewWorker( worker.WithClient( client, diff --git a/examples/limit-concurrency/main.go b/examples/limit-concurrency/main.go index f05eba38a..02a7ba01c 100644 --- a/examples/limit-concurrency/main.go +++ b/examples/limit-concurrency/main.go @@ -43,8 +43,6 @@ func run(ch <-chan interface{}, events chan<- string) error { return fmt.Errorf("error creating client: %w", err) } - // Create a worker. This automatically reads in a TemporalClient from .env and workflow files from the .hatchet - // directory, but this can be customized with the `worker.WithTemporalClient` and `worker.WithWorkflowFiles` options. w, err := worker.NewWorker( worker.WithClient( c, diff --git a/examples/loadtest/worker/main.go b/examples/loadtest/worker/main.go index 7a73b8129..156c18478 100644 --- a/examples/loadtest/worker/main.go +++ b/examples/loadtest/worker/main.go @@ -49,8 +49,6 @@ func main() { panic(err) } - // Create a worker. This automatically reads in a TemporalClient from .env and workflow files from the .hatchet - // directory, but this can be customized with the `worker.WithTemporalClient` and `worker.WithWorkflowFiles` options. w, err := worker.NewWorker( worker.WithClient( client, diff --git a/examples/manual-trigger/worker/main.go b/examples/manual-trigger/worker/main.go index 4e72bd72e..3e79a8533 100644 --- a/examples/manual-trigger/worker/main.go +++ b/examples/manual-trigger/worker/main.go @@ -40,8 +40,6 @@ func run(ch <-chan interface{}, events chan<- string) error { return fmt.Errorf("error creating client: %w", err) } - // Create a worker. This automatically reads in a TemporalClient from .env and workflow files from the .hatchet - // directory, but this can be customized with the `worker.WithTemporalClient` and `worker.WithWorkflowFiles` options. w, err := worker.NewWorker( worker.WithClient( c, diff --git a/examples/middleware/main.go b/examples/middleware/main.go index 7cee75ebd..3fd4e0a25 100644 --- a/examples/middleware/main.go +++ b/examples/middleware/main.go @@ -42,8 +42,6 @@ func run(interruptCtx context.Context, events chan<- string) error { return fmt.Errorf("error creating client: %w", err) } - // Create a worker. This automatically reads in a TemporalClient from .env and workflow files from the .hatchet - // directory, but this can be customized with the `worker.WithTemporalClient` and `worker.WithWorkflowFiles` options. w, err := worker.NewWorker( worker.WithClient( c, diff --git a/examples/register-action/main.go b/examples/register-action/main.go index 0ec9647e8..09ffc00be 100644 --- a/examples/register-action/main.go +++ b/examples/register-action/main.go @@ -50,8 +50,6 @@ func main() { panic(err) } - // Create a worker. This automatically reads in a TemporalClient from .env and workflow files from the .hatchet - // directory, but this can be customized with the `worker.WithTemporalClient` and `worker.WithWorkflowFiles` options. w, err := worker.NewWorker( worker.WithClient( client, diff --git a/examples/retries/main.go b/examples/retries/main.go new file mode 100644 index 000000000..cf48dc16d --- /dev/null +++ b/examples/retries/main.go @@ -0,0 +1,143 @@ +package main + +import ( + "context" + "fmt" + "log" + "time" + + "github.com/joho/godotenv" + + "github.com/hatchet-dev/hatchet/pkg/client" + "github.com/hatchet-dev/hatchet/pkg/cmdutils" + "github.com/hatchet-dev/hatchet/pkg/worker" +) + +type userCreateEvent struct { + Username string `json:"username"` + UserID string `json:"user_id"` + Data map[string]string `json:"data"` +} + +type stepOneOutput struct { + Message string `json:"message"` +} + +func main() { + err := godotenv.Load() + if err != nil { + panic(err) + } + + events := make(chan string, 50) + if err := run(cmdutils.InterruptChan(), events); err != nil { + panic(err) + } +} + +func getConcurrencyKey(ctx worker.HatchetContext) (string, error) { + return "user-create", nil +} + +type retryWorkflow struct { + retries int +} + +func (r *retryWorkflow) StepOne(ctx worker.HatchetContext) (result *stepOneOutput, err error) { + input := &userCreateEvent{} + + err = ctx.WorkflowInput(input) + + if err != nil { + return nil, err + } + + if r.retries < 5 { + r.retries++ + return nil, fmt.Errorf("error") + } + + log.Printf("finished step-one") + return &stepOneOutput{ + Message: "Username is: " + input.Username, + }, nil +} + +func run(ch <-chan interface{}, events chan<- string) error { + c, err := client.New() + + if err != nil { + return fmt.Errorf("error creating client: %w", err) + } + + w, err := worker.NewWorker( + worker.WithClient( + c, + ), + ) + if err != nil { + return fmt.Errorf("error creating worker: %w", err) + } + + testSvc := w.NewService("test") + + wk := &retryWorkflow{} + + err = testSvc.On( + worker.Events("user:create:simple"), + &worker.WorkflowJob{ + Name: "simple", + Description: "This runs after an update to the user model.", + Concurrency: worker.Concurrency(getConcurrencyKey), + Steps: []*worker.WorkflowStep{ + worker.Fn(wk.StepOne).SetName("step-one").SetRetries(3), + }, + }, + ) + if err != nil { + return fmt.Errorf("error registering workflow: %w", err) + } + + interruptCtx, cancel := cmdutils.InterruptContextFromChan(ch) + defer cancel() + + go func() { + err = w.Start(interruptCtx) + + if err != nil { + panic(err) + } + + cancel() + }() + + testEvent := userCreateEvent{ + Username: "echo-test", + UserID: "1234", + Data: map[string]string{ + "test": "test", + }, + } + + log.Printf("pushing event user:create:simple") + + // push an event + err = c.Event().Push( + context.Background(), + "user:create:simple", + testEvent, + ) + + if err != nil { + return fmt.Errorf("error pushing event: %w", err) + } + + for { + select { + case <-interruptCtx.Done(): + return nil + default: + time.Sleep(time.Second) + } + } +} diff --git a/examples/scheduled/main.go b/examples/scheduled/main.go index 4a5fcbe23..340c9b657 100644 --- a/examples/scheduled/main.go +++ b/examples/scheduled/main.go @@ -50,8 +50,6 @@ func main() { panic(err) } - // Create a worker. This automatically reads in a TemporalClient from .env and workflow files from the .hatchet - // directory, but this can be customized with the `worker.WithTemporalClient` and `worker.WithWorkflowFiles` options. w, err := worker.NewWorker( worker.WithClient( c, diff --git a/examples/simple/main.go b/examples/simple/main.go index 5e9b6992e..80899b6c1 100644 --- a/examples/simple/main.go +++ b/examples/simple/main.go @@ -47,8 +47,6 @@ func run(interruptCtx context.Context, events chan<- string) error { return fmt.Errorf("error creating client: %w", err) } - // Create a worker. This automatically reads in a TemporalClient from .env and workflow files from the .hatchet - // directory, but this can be customized with the `worker.WithTemporalClient` and `worker.WithWorkflowFiles` options. w, err := worker.NewWorker( worker.WithClient( c, diff --git a/frontend/docs/pages/home/go-sdk/creating-a-worker.mdx b/frontend/docs/pages/home/go-sdk/creating-a-worker.mdx index 102fec622..b431e5f60 100644 --- a/frontend/docs/pages/home/go-sdk/creating-a-worker.mdx +++ b/frontend/docs/pages/home/go-sdk/creating-a-worker.mdx @@ -88,8 +88,7 @@ func main() { panic(err) } - // Create a worker. This automatically reads in a TemporalClient from .env and workflow files from the .hatchet - // directory, but this can be customized with the `worker.WithTemporalClient` and `worker.WithWorkflowFiles` options. + w, err := worker.NewWorker( worker.WithClient( client, diff --git a/internal/repository/prisma/dbsqlc/models.go b/internal/repository/prisma/dbsqlc/models.go index f77e86a5c..2ff70dee9 100644 --- a/internal/repository/prisma/dbsqlc/models.go +++ b/internal/repository/prisma/dbsqlc/models.go @@ -571,6 +571,7 @@ type Step struct { ActionId string `json:"actionId"` Timeout pgtype.Text `json:"timeout"` CustomUserData []byte `json:"customUserData"` + Retries int32 `json:"retries"` } type StepOrder struct { @@ -604,6 +605,7 @@ type StepRun struct { InputSchema []byte `json:"inputSchema"` CallerFiles []byte `json:"callerFiles"` GitRepoBranch pgtype.Text `json:"gitRepoBranch"` + RetryCount int32 `json:"retryCount"` } type StepRunOrder struct { diff --git a/internal/repository/prisma/dbsqlc/schema.sql b/internal/repository/prisma/dbsqlc/schema.sql index d056dc08f..808b0a1eb 100644 --- a/internal/repository/prisma/dbsqlc/schema.sql +++ b/internal/repository/prisma/dbsqlc/schema.sql @@ -251,6 +251,7 @@ CREATE TABLE "Step" ( "actionId" TEXT NOT NULL, "timeout" TEXT, "customUserData" JSONB, + "retries" INTEGER NOT NULL DEFAULT 0, CONSTRAINT "Step_pkey" PRIMARY KEY ("id") ); @@ -282,6 +283,7 @@ CREATE TABLE "StepRun" ( "inputSchema" JSONB, "callerFiles" JSONB, "gitRepoBranch" TEXT, + "retryCount" INTEGER NOT NULL DEFAULT 0, CONSTRAINT "StepRun_pkey" PRIMARY KEY ("id") ); diff --git a/internal/repository/prisma/dbsqlc/step_runs.sql b/internal/repository/prisma/dbsqlc/step_runs.sql index 1f7bc5092..97e96f9e0 100644 --- a/internal/repository/prisma/dbsqlc/step_runs.sql +++ b/internal/repository/prisma/dbsqlc/step_runs.sql @@ -46,7 +46,8 @@ SET -- if this is a rerun, we clear the cancelledReason WHEN sqlc.narg('rerun')::boolean THEN NULL ELSE COALESCE(sqlc.narg('cancelledReason')::text, "cancelledReason") - END + END, + "retryCount" = COALESCE(sqlc.narg('retryCount')::int, "retryCount") WHERE "id" = @id::uuid AND "tenantId" = @tenantId::uuid diff --git a/internal/repository/prisma/dbsqlc/step_runs.sql.go b/internal/repository/prisma/dbsqlc/step_runs.sql.go index 7b858c961..9509c1470 100644 --- a/internal/repository/prisma/dbsqlc/step_runs.sql.go +++ b/internal/repository/prisma/dbsqlc/step_runs.sql.go @@ -97,7 +97,7 @@ func (q *Queries) ArchiveStepRunResultFromStepRun(ctx context.Context, db DBTX, const getStepRun = `-- name: GetStepRun :one SELECT - "StepRun".id, "StepRun"."createdAt", "StepRun"."updatedAt", "StepRun"."deletedAt", "StepRun"."tenantId", "StepRun"."jobRunId", "StepRun"."stepId", "StepRun"."order", "StepRun"."workerId", "StepRun"."tickerId", "StepRun".status, "StepRun".input, "StepRun".output, "StepRun"."requeueAfter", "StepRun"."scheduleTimeoutAt", "StepRun".error, "StepRun"."startedAt", "StepRun"."finishedAt", "StepRun"."timeoutAt", "StepRun"."cancelledAt", "StepRun"."cancelledReason", "StepRun"."cancelledError", "StepRun"."inputSchema", "StepRun"."callerFiles", "StepRun"."gitRepoBranch" + "StepRun".id, "StepRun"."createdAt", "StepRun"."updatedAt", "StepRun"."deletedAt", "StepRun"."tenantId", "StepRun"."jobRunId", "StepRun"."stepId", "StepRun"."order", "StepRun"."workerId", "StepRun"."tickerId", "StepRun".status, "StepRun".input, "StepRun".output, "StepRun"."requeueAfter", "StepRun"."scheduleTimeoutAt", "StepRun".error, "StepRun"."startedAt", "StepRun"."finishedAt", "StepRun"."timeoutAt", "StepRun"."cancelledAt", "StepRun"."cancelledReason", "StepRun"."cancelledError", "StepRun"."inputSchema", "StepRun"."callerFiles", "StepRun"."gitRepoBranch", "StepRun"."retryCount" FROM "StepRun" WHERE @@ -139,13 +139,14 @@ func (q *Queries) GetStepRun(ctx context.Context, db DBTX, arg GetStepRunParams) &i.InputSchema, &i.CallerFiles, &i.GitRepoBranch, + &i.RetryCount, ) return &i, err } const resolveLaterStepRuns = `-- name: ResolveLaterStepRuns :many WITH currStepRun AS ( - SELECT id, "createdAt", "updatedAt", "deletedAt", "tenantId", "jobRunId", "stepId", "order", "workerId", "tickerId", status, input, output, "requeueAfter", "scheduleTimeoutAt", error, "startedAt", "finishedAt", "timeoutAt", "cancelledAt", "cancelledReason", "cancelledError", "inputSchema", "callerFiles", "gitRepoBranch" + SELECT id, "createdAt", "updatedAt", "deletedAt", "tenantId", "jobRunId", "stepId", "order", "workerId", "tickerId", status, input, output, "requeueAfter", "scheduleTimeoutAt", error, "startedAt", "finishedAt", "timeoutAt", "cancelledAt", "cancelledReason", "cancelledError", "inputSchema", "callerFiles", "gitRepoBranch", "retryCount" FROM "StepRun" WHERE "id" = $1::uuid AND @@ -178,7 +179,7 @@ WHERE WHERE "id" = $1::uuid ) AND sr."tenantId" = $2::uuid -RETURNING sr.id, sr."createdAt", sr."updatedAt", sr."deletedAt", sr."tenantId", sr."jobRunId", sr."stepId", sr."order", sr."workerId", sr."tickerId", sr.status, sr.input, sr.output, sr."requeueAfter", sr."scheduleTimeoutAt", sr.error, sr."startedAt", sr."finishedAt", sr."timeoutAt", sr."cancelledAt", sr."cancelledReason", sr."cancelledError", sr."inputSchema", sr."callerFiles", sr."gitRepoBranch" +RETURNING sr.id, sr."createdAt", sr."updatedAt", sr."deletedAt", sr."tenantId", sr."jobRunId", sr."stepId", sr."order", sr."workerId", sr."tickerId", sr.status, sr.input, sr.output, sr."requeueAfter", sr."scheduleTimeoutAt", sr.error, sr."startedAt", sr."finishedAt", sr."timeoutAt", sr."cancelledAt", sr."cancelledReason", sr."cancelledError", sr."inputSchema", sr."callerFiles", sr."gitRepoBranch", sr."retryCount" ` type ResolveLaterStepRunsParams struct { @@ -221,6 +222,7 @@ func (q *Queries) ResolveLaterStepRuns(ctx context.Context, db DBTX, arg Resolve &i.InputSchema, &i.CallerFiles, &i.GitRepoBranch, + &i.RetryCount, ); err != nil { return nil, err } @@ -271,11 +273,12 @@ SET -- if this is a rerun, we clear the cancelledReason WHEN $4::boolean THEN NULL ELSE COALESCE($11::text, "cancelledReason") - END + END, + "retryCount" = COALESCE($12::int, "retryCount") WHERE - "id" = $12::uuid AND - "tenantId" = $13::uuid -RETURNING "StepRun".id, "StepRun"."createdAt", "StepRun"."updatedAt", "StepRun"."deletedAt", "StepRun"."tenantId", "StepRun"."jobRunId", "StepRun"."stepId", "StepRun"."order", "StepRun"."workerId", "StepRun"."tickerId", "StepRun".status, "StepRun".input, "StepRun".output, "StepRun"."requeueAfter", "StepRun"."scheduleTimeoutAt", "StepRun".error, "StepRun"."startedAt", "StepRun"."finishedAt", "StepRun"."timeoutAt", "StepRun"."cancelledAt", "StepRun"."cancelledReason", "StepRun"."cancelledError", "StepRun"."inputSchema", "StepRun"."callerFiles", "StepRun"."gitRepoBranch" + "id" = $13::uuid AND + "tenantId" = $14::uuid +RETURNING "StepRun".id, "StepRun"."createdAt", "StepRun"."updatedAt", "StepRun"."deletedAt", "StepRun"."tenantId", "StepRun"."jobRunId", "StepRun"."stepId", "StepRun"."order", "StepRun"."workerId", "StepRun"."tickerId", "StepRun".status, "StepRun".input, "StepRun".output, "StepRun"."requeueAfter", "StepRun"."scheduleTimeoutAt", "StepRun".error, "StepRun"."startedAt", "StepRun"."finishedAt", "StepRun"."timeoutAt", "StepRun"."cancelledAt", "StepRun"."cancelledReason", "StepRun"."cancelledError", "StepRun"."inputSchema", "StepRun"."callerFiles", "StepRun"."gitRepoBranch", "StepRun"."retryCount" ` type UpdateStepRunParams struct { @@ -290,6 +293,7 @@ type UpdateStepRunParams struct { Error pgtype.Text `json:"error"` CancelledAt pgtype.Timestamp `json:"cancelledAt"` CancelledReason pgtype.Text `json:"cancelledReason"` + RetryCount pgtype.Int4 `json:"retryCount"` ID pgtype.UUID `json:"id"` Tenantid pgtype.UUID `json:"tenantid"` } @@ -307,6 +311,7 @@ func (q *Queries) UpdateStepRun(ctx context.Context, db DBTX, arg UpdateStepRunP arg.Error, arg.CancelledAt, arg.CancelledReason, + arg.RetryCount, arg.ID, arg.Tenantid, ) @@ -337,6 +342,7 @@ func (q *Queries) UpdateStepRun(ctx context.Context, db DBTX, arg UpdateStepRunP &i.InputSchema, &i.CallerFiles, &i.GitRepoBranch, + &i.RetryCount, ) return &i, err } diff --git a/internal/repository/prisma/dbsqlc/workflow_runs.sql.go b/internal/repository/prisma/dbsqlc/workflow_runs.sql.go index 0164f7d67..1d9cc6884 100644 --- a/internal/repository/prisma/dbsqlc/workflow_runs.sql.go +++ b/internal/repository/prisma/dbsqlc/workflow_runs.sql.go @@ -330,7 +330,7 @@ INSERT INTO "StepRun" ( NULL, NULL, '{}' -) RETURNING id, "createdAt", "updatedAt", "deletedAt", "tenantId", "jobRunId", "stepId", "order", "workerId", "tickerId", status, input, output, "requeueAfter", "scheduleTimeoutAt", error, "startedAt", "finishedAt", "timeoutAt", "cancelledAt", "cancelledReason", "cancelledError", "inputSchema", "callerFiles", "gitRepoBranch" +) RETURNING id, "createdAt", "updatedAt", "deletedAt", "tenantId", "jobRunId", "stepId", "order", "workerId", "tickerId", status, input, output, "requeueAfter", "scheduleTimeoutAt", error, "startedAt", "finishedAt", "timeoutAt", "cancelledAt", "cancelledReason", "cancelledError", "inputSchema", "callerFiles", "gitRepoBranch", "retryCount" ` type CreateStepRunParams struct { @@ -378,6 +378,7 @@ func (q *Queries) CreateStepRun(ctx context.Context, db DBTX, arg CreateStepRunP &i.InputSchema, &i.CallerFiles, &i.GitRepoBranch, + &i.RetryCount, ) return &i, err } @@ -524,7 +525,7 @@ func (q *Queries) LinkStepRunParents(ctx context.Context, db DBTX, jobrunid pgty const listStartableStepRuns = `-- name: ListStartableStepRuns :many SELECT - child_run.id, child_run."createdAt", child_run."updatedAt", child_run."deletedAt", child_run."tenantId", child_run."jobRunId", child_run."stepId", child_run."order", child_run."workerId", child_run."tickerId", child_run.status, child_run.input, child_run.output, child_run."requeueAfter", child_run."scheduleTimeoutAt", child_run.error, child_run."startedAt", child_run."finishedAt", child_run."timeoutAt", child_run."cancelledAt", child_run."cancelledReason", child_run."cancelledError", child_run."inputSchema", child_run."callerFiles", child_run."gitRepoBranch" + child_run.id, child_run."createdAt", child_run."updatedAt", child_run."deletedAt", child_run."tenantId", child_run."jobRunId", child_run."stepId", child_run."order", child_run."workerId", child_run."tickerId", child_run.status, child_run.input, child_run.output, child_run."requeueAfter", child_run."scheduleTimeoutAt", child_run.error, child_run."startedAt", child_run."finishedAt", child_run."timeoutAt", child_run."cancelledAt", child_run."cancelledReason", child_run."cancelledError", child_run."inputSchema", child_run."callerFiles", child_run."gitRepoBranch", child_run."retryCount" FROM "StepRun" AS child_run JOIN @@ -585,6 +586,7 @@ func (q *Queries) ListStartableStepRuns(ctx context.Context, db DBTX, arg ListSt &i.InputSchema, &i.CallerFiles, &i.GitRepoBranch, + &i.RetryCount, ); err != nil { return nil, err } diff --git a/internal/repository/prisma/dbsqlc/workflows.sql b/internal/repository/prisma/dbsqlc/workflows.sql index 46bcb46e1..51afe9d43 100644 --- a/internal/repository/prisma/dbsqlc/workflows.sql +++ b/internal/repository/prisma/dbsqlc/workflows.sql @@ -218,7 +218,8 @@ INSERT INTO "Step" ( "jobId", "actionId", "timeout", - "customUserData" + "customUserData", + "retries" ) VALUES ( @id::uuid, coalesce(sqlc.narg('createdAt')::timestamp, CURRENT_TIMESTAMP), @@ -229,7 +230,8 @@ INSERT INTO "Step" ( @jobId::uuid, @actionId::text, @timeout::text, - coalesce(sqlc.narg('customUserData')::jsonb, '{}') + coalesce(sqlc.narg('customUserData')::jsonb, '{}'), + coalesce(sqlc.narg('retries')::integer, 0) ) RETURNING *; -- name: AddStepParents :exec diff --git a/internal/repository/prisma/dbsqlc/workflows.sql.go b/internal/repository/prisma/dbsqlc/workflows.sql.go index 49fa2cf54..6145488d1 100644 --- a/internal/repository/prisma/dbsqlc/workflows.sql.go +++ b/internal/repository/prisma/dbsqlc/workflows.sql.go @@ -170,7 +170,8 @@ INSERT INTO "Step" ( "jobId", "actionId", "timeout", - "customUserData" + "customUserData", + "retries" ) VALUES ( $1::uuid, coalesce($2::timestamp, CURRENT_TIMESTAMP), @@ -181,8 +182,9 @@ INSERT INTO "Step" ( $7::uuid, $8::text, $9::text, - coalesce($10::jsonb, '{}') -) RETURNING id, "createdAt", "updatedAt", "deletedAt", "readableId", "tenantId", "jobId", "actionId", timeout, "customUserData" + coalesce($10::jsonb, '{}'), + coalesce($11::integer, 0) +) RETURNING id, "createdAt", "updatedAt", "deletedAt", "readableId", "tenantId", "jobId", "actionId", timeout, "customUserData", retries ` type CreateStepParams struct { @@ -196,6 +198,7 @@ type CreateStepParams struct { Actionid string `json:"actionid"` Timeout string `json:"timeout"` CustomUserData []byte `json:"customUserData"` + Retries pgtype.Int4 `json:"retries"` } func (q *Queries) CreateStep(ctx context.Context, db DBTX, arg CreateStepParams) (*Step, error) { @@ -210,6 +213,7 @@ func (q *Queries) CreateStep(ctx context.Context, db DBTX, arg CreateStepParams) arg.Actionid, arg.Timeout, arg.CustomUserData, + arg.Retries, ) var i Step err := row.Scan( @@ -223,6 +227,7 @@ func (q *Queries) CreateStep(ctx context.Context, db DBTX, arg CreateStepParams) &i.ActionId, &i.Timeout, &i.CustomUserData, + &i.Retries, ) return &i, err } diff --git a/internal/repository/prisma/step_run.go b/internal/repository/prisma/step_run.go index b5715bc3b..2003762be 100644 --- a/internal/repository/prisma/step_run.go +++ b/internal/repository/prisma/step_run.go @@ -466,6 +466,13 @@ func getUpdateParams( updateParams.CancelledReason = sqlchelpers.TextFromStr(*opts.CancelledReason) } + if opts.RetryCount != nil { + updateParams.RetryCount = pgtype.Int4{ + Valid: true, + Int32: int32(*opts.RetryCount), + } + } + return updateParams, updateJobRunLookupDataParams, resolveJobRunParams, resolveLaterStepRunsParams, nil } diff --git a/internal/repository/prisma/workflow.go b/internal/repository/prisma/workflow.go index a131b5439..cddd31050 100644 --- a/internal/repository/prisma/workflow.go +++ b/internal/repository/prisma/workflow.go @@ -563,6 +563,7 @@ func (r *workflowRepository) createWorkflowVersionTxs(ctx context.Context, tx pg var ( timeout string customUserData []byte + retries pgtype.Int4 ) if stepOpts.Timeout != nil { @@ -573,6 +574,13 @@ func (r *workflowRepository) createWorkflowVersionTxs(ctx context.Context, tx pg customUserData = []byte(*stepOpts.UserData) } + if stepOpts.Retries != nil { + retries = pgtype.Int4{ + Valid: true, + Int32: int32(*stepOpts.Retries), + } + } + // upsert the action _, err := r.queries.UpsertAction( context.Background(), @@ -598,6 +606,7 @@ func (r *workflowRepository) createWorkflowVersionTxs(ctx context.Context, tx pg Timeout: timeout, Readableid: stepOpts.ReadableId, CustomUserData: customUserData, + Retries: retries, }, ) diff --git a/internal/repository/step_run.go b/internal/repository/step_run.go index bf1934e2f..892742e76 100644 --- a/internal/repository/step_run.go +++ b/internal/repository/step_run.go @@ -50,6 +50,8 @@ type UpdateStepRunOpts struct { Input []byte Output []byte + + RetryCount *int } type UpdateStepRunOverridesDataOpts struct { diff --git a/internal/repository/workflow.go b/internal/repository/workflow.go index b44642703..7dce0cb1a 100644 --- a/internal/repository/workflow.go +++ b/internal/repository/workflow.go @@ -109,6 +109,9 @@ type CreateWorkflowStepOpts struct { // (optional) the custom user data for the step, serialized as a json string UserData *string `validate:"omitnil,json"` + + // (optional) the step retry max + Retries *int `validate:"omitempty,min=0"` } type ListWorkflowsOpts struct { diff --git a/internal/services/admin/contracts/workflows.pb.go b/internal/services/admin/contracts/workflows.pb.go index 6fd9e712c..8ed87724b 100644 --- a/internal/services/admin/contracts/workflows.pb.go +++ b/internal/services/admin/contracts/workflows.pb.go @@ -369,6 +369,7 @@ type CreateWorkflowStepOpts struct { Inputs string `protobuf:"bytes,4,opt,name=inputs,proto3" json:"inputs,omitempty"` // (optional) the step inputs, assuming string representation of JSON Parents []string `protobuf:"bytes,5,rep,name=parents,proto3" json:"parents,omitempty"` // (optional) the step parents. if none are passed in, this is a root step UserData string `protobuf:"bytes,6,opt,name=user_data,json=userData,proto3" json:"user_data,omitempty"` // (optional) the custom step user data, assuming string representation of JSON + Retries int32 `protobuf:"varint,7,opt,name=retries,proto3" json:"retries,omitempty"` // (optional) the number of retries for the step, default 0 } func (x *CreateWorkflowStepOpts) Reset() { @@ -445,6 +446,13 @@ func (x *CreateWorkflowStepOpts) GetUserData() string { return "" } +func (x *CreateWorkflowStepOpts) GetRetries() int32 { + if x != nil { + return x.Retries + } + return 0 +} + // ListWorkflowsRequest is the request for ListWorkflows. type ListWorkflowsRequest struct { state protoimpl.MessageState @@ -1535,7 +1543,7 @@ var file_workflows_proto_rawDesc = []byte{ 0x12, 0x2d, 0x0a, 0x05, 0x73, 0x74, 0x65, 0x70, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x53, 0x74, 0x65, 0x70, 0x4f, 0x70, 0x74, 0x73, 0x52, 0x05, 0x73, 0x74, 0x65, 0x70, 0x73, 0x22, - 0xba, 0x01, 0x0a, 0x16, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, + 0xd4, 0x01, 0x0a, 0x16, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x53, 0x74, 0x65, 0x70, 0x4f, 0x70, 0x74, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x72, 0x65, 0x61, 0x64, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x61, 0x64, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x61, @@ -1546,95 +1554,28 @@ var file_workflows_proto_rawDesc = []byte{ 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x22, 0x16, 0x0a, 0x14, - 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x22, 0x8a, 0x01, 0x0a, 0x17, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, - 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x1f, 0x0a, 0x0b, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x49, - 0x64, 0x12, 0x38, 0x0a, 0x09, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x09, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x69, - 0x6e, 0x70, 0x75, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, - 0x74, 0x22, 0x40, 0x0a, 0x15, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, - 0x77, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x27, 0x0a, 0x09, 0x77, 0x6f, - 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x09, 0x2e, - 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, - 0x6f, 0x77, 0x73, 0x22, 0x3b, 0x0a, 0x1c, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, - 0x6c, 0x6f, 0x77, 0x73, 0x46, 0x6f, 0x72, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, - 0x22, 0xaf, 0x02, 0x0a, 0x08, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x0e, 0x0a, - 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x39, 0x0a, - 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, - 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x39, 0x0a, 0x0a, 0x75, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x64, 0x41, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x5f, 0x69, 0x64, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x49, 0x64, - 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3e, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, - 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2c, 0x0a, 0x08, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, - 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, - 0x77, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x73, 0x22, 0xb1, 0x02, 0x0a, 0x0f, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x56, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x39, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, - 0x64, 0x5f, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, - 0x74, 0x12, 0x39, 0x0a, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x52, 0x09, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x18, 0x0a, 0x07, - 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x12, 0x1f, 0x0a, 0x0b, - 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0a, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x49, 0x64, 0x12, 0x2d, 0x0a, - 0x08, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x11, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, - 0x72, 0x73, 0x52, 0x08, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x73, 0x12, 0x18, 0x0a, 0x04, - 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x04, 0x2e, 0x4a, 0x6f, 0x62, - 0x52, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x22, 0xc6, 0x02, 0x0a, 0x10, 0x57, 0x6f, 0x72, 0x6b, 0x66, - 0x6c, 0x6f, 0x77, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x39, 0x0a, 0x0a, 0x63, - 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x39, 0x0a, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x64, 0x5f, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, - 0x74, 0x12, 0x2e, 0x0a, 0x13, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, - 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, - 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x30, - 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, - 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, - 0x12, 0x2d, 0x0a, 0x05, 0x63, 0x72, 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x17, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, - 0x72, 0x43, 0x72, 0x6f, 0x6e, 0x52, 0x65, 0x66, 0x52, 0x05, 0x63, 0x72, 0x6f, 0x6e, 0x73, 0x22, - 0x53, 0x0a, 0x17, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x72, 0x69, 0x67, 0x67, - 0x65, 0x72, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x61, - 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, - 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, - 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, 0x76, 0x65, 0x6e, - 0x74, 0x4b, 0x65, 0x79, 0x22, 0x49, 0x0a, 0x16, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, - 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x43, 0x72, 0x6f, 0x6e, 0x52, 0x65, 0x66, 0x12, 0x1b, - 0x0a, 0x09, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x63, - 0x72, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x72, 0x6f, 0x6e, 0x22, - 0x81, 0x03, 0x0a, 0x03, 0x4a, 0x6f, 0x62, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x12, 0x18, 0x0a, 0x07, + 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x72, + 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x22, 0x16, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, + 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x8a, + 0x01, 0x0a, 0x17, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, + 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x77, 0x6f, + 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0a, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x49, 0x64, 0x12, 0x38, 0x0a, 0x09, 0x73, + 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x63, 0x68, 0x65, + 0x64, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x22, 0x40, 0x0a, 0x15, 0x4c, + 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x27, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, + 0x6f, 0x77, 0x52, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x22, 0x3b, 0x0a, + 0x1c, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x46, 0x6f, + 0x72, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, + 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x08, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x22, 0xaf, 0x02, 0x0a, 0x08, 0x57, + 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x39, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, @@ -1644,99 +1585,168 @@ var file_workflows_proto_rawDesc = []byte{ 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x08, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x2e, 0x0a, 0x13, 0x77, 0x6f, - 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, - 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, - 0x77, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3e, - 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, + 0x52, 0x08, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3e, + 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, - 0x65, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1b, - 0x0a, 0x05, 0x73, 0x74, 0x65, 0x70, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x05, 0x2e, - 0x53, 0x74, 0x65, 0x70, 0x52, 0x05, 0x73, 0x74, 0x65, 0x70, 0x73, 0x12, 0x36, 0x0a, 0x07, 0x74, - 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, - 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, 0x74, 0x69, 0x6d, 0x65, - 0x6f, 0x75, 0x74, 0x22, 0x85, 0x03, 0x0a, 0x04, 0x53, 0x74, 0x65, 0x70, 0x12, 0x0e, 0x0a, 0x02, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x39, 0x0a, 0x0a, - 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x65, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2c, + 0x0a, 0x08, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x10, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x52, 0x08, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xb1, 0x02, 0x0a, + 0x0f, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, + 0x12, 0x39, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x39, 0x0a, 0x0a, 0x75, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x75, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x12, 0x14, 0x0a, 0x05, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x05, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, + 0x6f, 0x77, 0x5f, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x77, 0x6f, 0x72, + 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x49, 0x64, 0x12, 0x2d, 0x0a, 0x08, 0x74, 0x72, 0x69, 0x67, 0x67, + 0x65, 0x72, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x57, 0x6f, 0x72, 0x6b, + 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x73, 0x52, 0x08, 0x74, 0x72, + 0x69, 0x67, 0x67, 0x65, 0x72, 0x73, 0x12, 0x18, 0x0a, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x09, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x04, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x04, 0x6a, 0x6f, 0x62, 0x73, + 0x22, 0xc6, 0x02, 0x0a, 0x10, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x72, 0x69, + 0x67, 0x67, 0x65, 0x72, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x39, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, + 0x5f, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, + 0x12, 0x39, 0x0a, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x09, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x2e, 0x0a, 0x13, 0x77, + 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x5f, + 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, + 0x6f, 0x77, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x74, + 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, + 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, + 0x6c, 0x6f, 0x77, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, + 0x65, 0x66, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x2d, 0x0a, 0x05, 0x63, 0x72, + 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x57, 0x6f, 0x72, 0x6b, + 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x43, 0x72, 0x6f, 0x6e, 0x52, + 0x65, 0x66, 0x52, 0x05, 0x63, 0x72, 0x6f, 0x6e, 0x73, 0x22, 0x53, 0x0a, 0x17, 0x57, 0x6f, 0x72, + 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x52, 0x65, 0x66, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, + 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x22, 0x49, + 0x0a, 0x16, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, + 0x72, 0x43, 0x72, 0x6f, 0x6e, 0x52, 0x65, 0x66, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x65, + 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x61, 0x72, + 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x72, 0x6f, 0x6e, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x72, 0x6f, 0x6e, 0x22, 0x81, 0x03, 0x0a, 0x03, 0x4a, 0x6f, + 0x62, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, + 0x64, 0x12, 0x39, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x39, 0x0a, 0x0a, + 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x39, 0x0a, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, - 0x41, 0x74, 0x12, 0x3d, 0x0a, 0x0b, 0x72, 0x65, 0x61, 0x64, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x69, - 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0a, 0x72, 0x65, 0x61, 0x64, 0x61, 0x62, 0x6c, 0x65, 0x49, - 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x15, - 0x0a, 0x06, 0x6a, 0x6f, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x6a, 0x6f, 0x62, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, - 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x36, 0x0a, - 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, 0x74, 0x69, - 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, - 0x18, 0x0a, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x12, - 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x18, 0x0b, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x08, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x22, 0x38, 0x0a, 0x15, 0x44, - 0x65, 0x6c, 0x65, 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x77, 0x6f, 0x72, 0x6b, 0x66, - 0x6c, 0x6f, 0x77, 0x49, 0x64, 0x22, 0x2e, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x57, 0x6f, 0x72, 0x6b, - 0x66, 0x6c, 0x6f, 0x77, 0x42, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x42, 0x0a, 0x16, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, - 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x22, 0x41, 0x0a, 0x17, 0x54, 0x72, 0x69, - 0x67, 0x67, 0x65, 0x72, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x0f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, - 0x5f, 0x72, 0x75, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x77, - 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x75, 0x6e, 0x49, 0x64, 0x2a, 0x55, 0x0a, 0x18, - 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, - 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x12, 0x16, 0x0a, 0x12, 0x43, 0x41, 0x4e, 0x43, - 0x45, 0x4c, 0x5f, 0x49, 0x4e, 0x5f, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45, 0x53, 0x53, 0x10, 0x00, - 0x12, 0x0f, 0x0a, 0x0b, 0x44, 0x52, 0x4f, 0x50, 0x5f, 0x4e, 0x45, 0x57, 0x45, 0x53, 0x54, 0x10, - 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x51, 0x55, 0x45, 0x55, 0x45, 0x5f, 0x4e, 0x45, 0x57, 0x45, 0x53, - 0x54, 0x10, 0x02, 0x32, 0xcd, 0x03, 0x0a, 0x0f, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x3e, 0x0a, 0x0d, 0x4c, 0x69, 0x73, 0x74, 0x57, - 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x12, 0x15, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, - 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x16, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x0b, 0x50, 0x75, 0x74, 0x57, 0x6f, - 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x13, 0x2e, 0x50, 0x75, 0x74, 0x57, 0x6f, 0x72, 0x6b, - 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x57, 0x6f, - 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x3e, 0x0a, - 0x10, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, - 0x77, 0x12, 0x18, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x57, 0x6f, 0x72, 0x6b, - 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x57, 0x6f, - 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x44, 0x0a, - 0x0f, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, - 0x12, 0x17, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, - 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x54, 0x72, 0x69, 0x67, - 0x67, 0x65, 0x72, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, - 0x6f, 0x77, 0x42, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x19, 0x2e, 0x47, 0x65, 0x74, 0x57, 0x6f, - 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x42, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x09, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x4e, - 0x0a, 0x15, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x46, - 0x6f, 0x72, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x1d, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, - 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x46, 0x6f, 0x72, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, - 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, - 0x0a, 0x0e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, - 0x12, 0x16, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, - 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x09, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, - 0x6c, 0x6f, 0x77, 0x42, 0x42, 0x5a, 0x40, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x68, 0x61, 0x74, 0x63, 0x68, 0x65, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x68, 0x61, - 0x74, 0x63, 0x68, 0x65, 0x74, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2f, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x75, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x65, 0x6e, 0x61, 0x6e, + 0x74, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x65, 0x6e, 0x61, + 0x6e, 0x74, 0x49, 0x64, 0x12, 0x2e, 0x0a, 0x13, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, + 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x11, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3e, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, + 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0b, 0x64, 0x65, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1b, 0x0a, 0x05, 0x73, 0x74, 0x65, 0x70, + 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x05, 0x2e, 0x53, 0x74, 0x65, 0x70, 0x52, 0x05, + 0x73, 0x74, 0x65, 0x70, 0x73, 0x12, 0x36, 0x0a, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, + 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x22, 0x85, 0x03, + 0x0a, 0x04, 0x53, 0x74, 0x65, 0x70, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x39, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x64, 0x5f, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, + 0x74, 0x12, 0x39, 0x0a, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x52, 0x09, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x3d, 0x0a, 0x0b, + 0x72, 0x65, 0x61, 0x64, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, + 0x0a, 0x72, 0x65, 0x61, 0x64, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x74, + 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x15, 0x0a, 0x06, 0x6a, 0x6f, 0x62, 0x5f, + 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6a, 0x6f, 0x62, 0x49, 0x64, 0x12, + 0x16, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x36, 0x0a, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, + 0x75, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, + 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, + 0x18, 0x0a, 0x07, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x07, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x69, + 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x63, 0x68, 0x69, + 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x22, 0x38, 0x0a, 0x15, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x57, + 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, + 0x0a, 0x0b, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0a, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x49, 0x64, 0x22, + 0x2e, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x42, 0x79, + 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, + 0x42, 0x0a, 0x16, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, + 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, + 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6e, + 0x70, 0x75, 0x74, 0x22, 0x41, 0x0a, 0x17, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x57, 0x6f, + 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, + 0x0a, 0x0f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x72, 0x75, 0x6e, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, + 0x77, 0x52, 0x75, 0x6e, 0x49, 0x64, 0x2a, 0x55, 0x0a, 0x18, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, + 0x72, 0x65, 0x6e, 0x63, 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x53, 0x74, 0x72, 0x61, 0x74, 0x65, + 0x67, 0x79, 0x12, 0x16, 0x0a, 0x12, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x5f, 0x49, 0x4e, 0x5f, + 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x44, 0x52, + 0x4f, 0x50, 0x5f, 0x4e, 0x45, 0x57, 0x45, 0x53, 0x54, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x51, + 0x55, 0x45, 0x55, 0x45, 0x5f, 0x4e, 0x45, 0x57, 0x45, 0x53, 0x54, 0x10, 0x02, 0x32, 0xcd, 0x03, + 0x0a, 0x0f, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x12, 0x3e, 0x0a, 0x0d, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, + 0x77, 0x73, 0x12, 0x15, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, + 0x77, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x4c, 0x69, 0x73, 0x74, + 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x34, 0x0a, 0x0b, 0x50, 0x75, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, + 0x12, 0x13, 0x2e, 0x50, 0x75, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, + 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x3e, 0x0a, 0x10, 0x53, 0x63, 0x68, 0x65, 0x64, + 0x75, 0x6c, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x18, 0x2e, 0x53, 0x63, + 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, + 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x44, 0x0a, 0x0f, 0x54, 0x72, 0x69, 0x67, 0x67, + 0x65, 0x72, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x17, 0x2e, 0x54, 0x72, 0x69, + 0x67, 0x67, 0x65, 0x72, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x57, 0x6f, 0x72, + 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, + 0x11, 0x47, 0x65, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x42, 0x79, 0x4e, 0x61, + 0x6d, 0x65, 0x12, 0x19, 0x2e, 0x47, 0x65, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, + 0x42, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x09, 0x2e, + 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x4e, 0x0a, 0x15, 0x4c, 0x69, 0x73, 0x74, + 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x46, 0x6f, 0x72, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x12, 0x1d, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, + 0x73, 0x46, 0x6f, 0x72, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x16, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x0e, 0x44, 0x65, 0x6c, 0x65, + 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x12, 0x16, 0x2e, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x09, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x42, 0x42, 0x5a, + 0x40, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x74, 0x63, + 0x68, 0x65, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x68, 0x61, 0x74, 0x63, 0x68, 0x65, 0x74, 0x2f, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x2f, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, + 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/internal/services/admin/server.go b/internal/services/admin/server.go index f30265874..547189b25 100644 --- a/internal/services/admin/server.go +++ b/internal/services/admin/server.go @@ -570,11 +570,14 @@ func getCreateWorkflowOpts(req *contracts.PutWorkflowRequest) (*repository.Creat return nil, err } + retries := int(stepCp.Retries) + steps[j] = repository.CreateWorkflowStepOpts{ ReadableId: stepCp.ReadableId, Action: parsedAction.String(), Timeout: &stepCp.Timeout, Parents: stepCp.Parents, + Retries: &retries, } if stepCp.UserData != "" { diff --git a/internal/services/controllers/jobs/controller.go b/internal/services/controllers/jobs/controller.go index 2a79f94f9..6d6de0875 100644 --- a/internal/services/controllers/jobs/controller.go +++ b/internal/services/controllers/jobs/controller.go @@ -129,6 +129,8 @@ func (ec *JobsControllerImpl) handleTask(ctx context.Context, task *taskqueue.Ta return ec.handleJobRunQueued(ctx, task) case "job-run-timed-out": return ec.handleJobRunTimedOut(ctx, task) + case "step-run-retry": + return ec.handleStepRunRetry(ctx, task) case "step-run-queued": return ec.handleStepRunQueued(ctx, task) case "step-run-requeue-ticker": @@ -339,25 +341,100 @@ func (ec *JobsControllerImpl) handleJobRunTimedOut(ctx context.Context, task *ta } } - // // cancel all step runs - // err = ec.repo.StepRun().CancelPendingStepRuns(metadata.TenantId, jobRun.ID, "JOB_RUN_TIMED_OUT") - - // if err != nil { - // return fmt.Errorf("could not cancel pending step runs: %w", err) - // } - - // // update the job run in the database - // _, err = ec.repo.JobRun().UpdateJobRun(metadata.TenantId, jobRun.ID, &repository.UpdateJobRunOpts{ - // Status: repository.JobRunStatusPtr(db.JobRunStatusCANCELLED), - // }) - - // if err != nil { - // return fmt.Errorf("could not update job run: %w", err) - // } - return nil } +func (ec *JobsControllerImpl) handleStepRunRetry(ctx context.Context, task *taskqueue.Task) error { + ctx, span := telemetry.NewSpan(ctx, "handle-step-run-retry") + defer span.End() + + payload := tasktypes.StepRunRetryTaskPayload{} + metadata := tasktypes.StepRunRetryTaskMetadata{} + + err := ec.dv.DecodeAndValidate(task.Payload, &payload) + + if err != nil { + return fmt.Errorf("could not decode job task payload: %w", err) + } + + err = ec.dv.DecodeAndValidate(task.Metadata, &metadata) + + if err != nil { + return fmt.Errorf("could not decode job task metadata: %w", err) + } + + err = ec.repo.StepRun().ArchiveStepRunResult(metadata.TenantId, payload.StepRunId) + + if err != nil { + return fmt.Errorf("could not archive step run result: %w", err) + } + + ec.l.Error().Err(fmt.Errorf("starting step run retry")) + + stepRun, err := ec.repo.StepRun().GetStepRunById(metadata.TenantId, payload.StepRunId) + + if err != nil { + return fmt.Errorf("could not get step run: %w", err) + } + + var inputBytes []byte + var retryCount = stepRun.RetryCount + 1 + + if len(payload.InputData) > 0 { + // update the input schema for the step run based on the new input + jsonSchemaBytes, err := schema.SchemaBytesFromBytes([]byte(payload.InputData)) + + if err != nil { + return err + } + + _, err = ec.repo.StepRun().UpdateStepRunInputSchema(metadata.TenantId, stepRun.ID, jsonSchemaBytes) + + if err != nil { + return err + } + + inputBytes = []byte(payload.InputData) + + // if the input data has been manually set, we reset the retry count as this is a user-triggered retry + retryCount = 0 + } else { + var ok bool + + inputBytes, ok = stepRun.Input() + + if !ok { + return fmt.Errorf("could not get step run input: %w", err) + } + } + + // update step run + _, _, err = ec.repo.StepRun().UpdateStepRun(metadata.TenantId, stepRun.ID, &repository.UpdateStepRunOpts{ + Input: inputBytes, + Status: repository.StepRunStatusPtr(db.StepRunStatusPending), + IsRerun: true, + RetryCount: &retryCount, + }) + + if err != nil { + return fmt.Errorf("could not update step run: %w", err) + } + + // requeue the step run in the task queue + jobRun, err := ec.repo.JobRun().GetJobRunById(metadata.TenantId, stepRun.JobRunID) + + if err != nil { + return fmt.Errorf("could not get job run: %w", err) + } + + // send a task to the taskqueue + return ec.tq.AddTask( + ctx, + taskqueue.JOB_PROCESSING_QUEUE, + tasktypes.StepRunQueuedToTask(jobRun.Job(), stepRun), + ) +} + func (ec *JobsControllerImpl) handleStepRunQueued(ctx context.Context, task *taskqueue.Task) error { ctx, span := telemetry.NewSpan(ctx, "handle-step-run-queued") defer span.End() @@ -834,10 +911,25 @@ func (ec *JobsControllerImpl) handleStepRunFailed(ctx context.Context, task *tas return fmt.Errorf("could not parse started at: %w", err) } + stepRun, err := ec.repo.StepRun().GetStepRunById(metadata.TenantId, payload.StepRunId) + + if err != nil { + return fmt.Errorf("could not get step run: %w", err) + } + + // determine if step run should be retried or not + shouldRetry := stepRun.RetryCount < stepRun.Step().Retries + + status := db.StepRunStatusFailed + + if shouldRetry { + status = db.StepRunStatusPending + } + stepRun, updateInfo, err := ec.repo.StepRun().UpdateStepRun(metadata.TenantId, payload.StepRunId, &repository.UpdateStepRunOpts{ FinishedAt: &failedAt, Error: &payload.Error, - Status: repository.StepRunStatusPtr(db.StepRunStatusFailed), + Status: repository.StepRunStatusPtr(status), }) if err != nil { @@ -863,6 +955,15 @@ func (ec *JobsControllerImpl) handleStepRunFailed(ctx context.Context, task *tas } } + if shouldRetry { + // send a task to the taskqueue + return ec.tq.AddTask( + ctx, + taskqueue.JOB_PROCESSING_QUEUE, + tasktypes.StepRunRetryToTask(stepRun, nil), + ) + } + return nil } @@ -958,6 +1059,20 @@ func (ec *JobsControllerImpl) cancelStepRun(ctx context.Context, tenantId, stepR } func (ec *JobsControllerImpl) handleStepRunUpdateInfo(stepRun *db.StepRunModel, updateInfo *repository.StepRunUpdateInfo) { + defer func() { + if r := recover(); r != nil { + err, ok := r.(error) + + if !ok { + err = fmt.Errorf("%v", r) + } + + ec.l.Error().Err(err).Msg("recovered from panic") + + return + } + }() + if updateInfo.WorkflowRunFinalState { err := ec.tq.AddTask( context.Background(), diff --git a/internal/services/shared/tasktypes/step.go b/internal/services/shared/tasktypes/step.go index deef06081..cf31525c5 100644 --- a/internal/services/shared/tasktypes/step.go +++ b/internal/services/shared/tasktypes/step.go @@ -97,6 +97,18 @@ type StepRunTimedOutTaskMetadata struct { TenantId string `json:"tenant_id" validate:"required,uuid"` } +type StepRunRetryTaskPayload struct { + StepRunId string `json:"step_run_id" validate:"required,uuid"` + JobRunId string `json:"job_run_id" validate:"required,uuid"` + + // optional - if not provided, the step run will be retried with the same input + InputData string `json:"omitempty,input_data"` +} + +type StepRunRetryTaskMetadata struct { + TenantId string `json:"tenant_id" validate:"required,uuid"` +} + func TenantToStepRunRequeueTask(tenant db.TenantModel) *taskqueue.Task { payload, _ := datautils.ToJSONMap(StepRunRequeueTaskPayload{ TenantId: tenant.ID, @@ -113,6 +125,24 @@ func TenantToStepRunRequeueTask(tenant db.TenantModel) *taskqueue.Task { } } +func StepRunRetryToTask(stepRun *db.StepRunModel, inputData []byte) *taskqueue.Task { + payload, _ := datautils.ToJSONMap(StepRunRetryTaskPayload{ + JobRunId: stepRun.JobRunID, + StepRunId: stepRun.ID, + InputData: string(inputData), + }) + + metadata, _ := datautils.ToJSONMap(StepRunRetryTaskMetadata{ + TenantId: stepRun.TenantID, + }) + + return &taskqueue.Task{ + ID: "step-run-retry", + Payload: payload, + Metadata: metadata, + } +} + func StepRunQueuedToTask(job *db.JobModel, stepRun *db.StepRunModel) *taskqueue.Task { payload, _ := datautils.ToJSONMap(StepRunTaskPayload{ JobRunId: stepRun.JobRunID, diff --git a/pkg/client/admin.go b/pkg/client/admin.go index f38300ca8..66fb10951 100644 --- a/pkg/client/admin.go +++ b/pkg/client/admin.go @@ -211,6 +211,7 @@ func (a *adminClientImpl) getPutRequest(workflow *types.Workflow) (*admincontrac Timeout: step.Timeout, Inputs: string(inputBytes), Parents: step.Parents, + Retries: int32(step.Retries), } stepOpts[i] = stepOpt diff --git a/pkg/client/loader/loader.go b/pkg/client/loader/loader.go index fa24fbf83..112192c36 100644 --- a/pkg/client/loader/loader.go +++ b/pkg/client/loader/loader.go @@ -81,7 +81,7 @@ func GetClientConfigFromConfigFile(cf *client.ClientConfigFile) (res *client.Cli // if the tls server name is empty, parse the domain from the host:port if tlsServerName == "" { // parse the domain from the host:port - domain, err := parseDomain(cf.HostPort) + domain, err := parseDomain(grpcBroadcastAddress) if err != nil { return nil, fmt.Errorf("could not parse domain: %w", err) diff --git a/pkg/client/types/file.go b/pkg/client/types/file.go index 11cfe7b2e..2bd0bdc86 100644 --- a/pkg/client/types/file.go +++ b/pkg/client/types/file.go @@ -74,6 +74,7 @@ type WorkflowStep struct { Timeout string `yaml:"timeout,omitempty"` With map[string]interface{} `yaml:"with,omitempty"` Parents []string `yaml:"parents,omitempty"` + Retries int `yaml:"retries"` } func ParseYAML(ctx context.Context, yamlBytes []byte) (Workflow, error) { diff --git a/pkg/worker/workflow.go b/pkg/worker/workflow.go index 63f24b30e..8143d8445 100644 --- a/pkg/worker/workflow.go +++ b/pkg/worker/workflow.go @@ -253,6 +253,8 @@ type WorkflowStep struct { // The ids of the parents Parents []string + + Retries int } func Fn(f any) *WorkflowStep { @@ -272,6 +274,11 @@ func (w *WorkflowStep) SetTimeout(timeout string) *WorkflowStep { return w } +func (w *WorkflowStep) SetRetries(retries int) *WorkflowStep { + w.Retries = retries + return w +} + func (w *WorkflowStep) AddParents(parents ...string) *WorkflowStep { w.Parents = append(w.Parents, parents...) return w @@ -326,6 +333,7 @@ func (w *WorkflowStep) ToWorkflowStep(svcName string, index int) (*Step, error) Timeout: w.Timeout, ActionID: w.GetActionId(svcName, index), Parents: []string{}, + Retries: w.Retries, } inputs, err := decodeFnArgTypes(fnType) diff --git a/prisma/migrations/20240215171232_v0_11_0/migration.sql b/prisma/migrations/20240216133745_v0_11_0/migration.sql similarity index 98% rename from prisma/migrations/20240215171232_v0_11_0/migration.sql rename to prisma/migrations/20240216133745_v0_11_0/migration.sql index 6e7479f30..66e80d4f8 100644 --- a/prisma/migrations/20240215171232_v0_11_0/migration.sql +++ b/prisma/migrations/20240216133745_v0_11_0/migration.sql @@ -1,9 +1,13 @@ -- CreateEnum CREATE TYPE "VcsProvider" AS ENUM ('GITHUB'); +-- AlterTable +ALTER TABLE "Step" ADD COLUMN "retries" INTEGER NOT NULL DEFAULT 0; + -- AlterTable ALTER TABLE "StepRun" ADD COLUMN "callerFiles" JSONB, -ADD COLUMN "gitRepoBranch" TEXT; +ADD COLUMN "gitRepoBranch" TEXT, +ADD COLUMN "retryCount" INTEGER NOT NULL DEFAULT 0; -- AlterTable ALTER TABLE "WorkflowRun" ADD COLUMN "gitRepoBranch" TEXT; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 24d828462..154e23331 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -519,6 +519,8 @@ model Step { timeout String? + retries Int @default(0) + // customUserData is a JSON object that can be used to store arbitrary data for the step customUserData Json? @@ -821,6 +823,9 @@ model StepRun { // when the step run times out due to a scheduling timeout (no workers available) scheduleTimeoutAt DateTime? + // which retry we're on for this step run + retryCount Int @default(0) + // the run error error String? diff --git a/python-sdk/hatchet_sdk/dispatcher_pb2_grpc.py b/python-sdk/hatchet_sdk/dispatcher_pb2_grpc.py index 2daffb0f9..856af8d05 100644 --- a/python-sdk/hatchet_sdk/dispatcher_pb2_grpc.py +++ b/python-sdk/hatchet_sdk/dispatcher_pb2_grpc.py @@ -4,7 +4,6 @@ import grpc from . import dispatcher_pb2 as dispatcher__pb2 - class DispatcherStub(object): """Missing associated documentation comment in .proto file.""" diff --git a/python-sdk/hatchet_sdk/hatchet.py b/python-sdk/hatchet_sdk/hatchet.py index e11b4787e..29860833e 100644 --- a/python-sdk/hatchet_sdk/hatchet.py +++ b/python-sdk/hatchet_sdk/hatchet.py @@ -35,11 +35,12 @@ class Hatchet: return inner - def step(self, name : str='', timeout : str='', parents : List[str] = []): + def step(self, name : str='', timeout : str='', parents : List[str] = [], retries : int = 0): def inner(func): func._step_name = name or func.__name__ func._step_parents = parents func._step_timeout = timeout + func._step_retries = retries return func diff --git a/python-sdk/hatchet_sdk/workflow.py b/python-sdk/hatchet_sdk/workflow.py index c8c03a601..b3e0d4e7a 100644 --- a/python-sdk/hatchet_sdk/workflow.py +++ b/python-sdk/hatchet_sdk/workflow.py @@ -48,8 +48,9 @@ class WorkflowMeta(type): action=serviceName + ":" + func_name, timeout=func._step_timeout or "60s", inputs='{}', - parents=[x for x in func._step_parents] - ) + parents=[x for x in func._step_parents], + retries=func._step_retries, + ) for func_name, func in attrs.items() if hasattr(func, '_step_name') ] diff --git a/python-sdk/hatchet_sdk/workflows_pb2.py b/python-sdk/hatchet_sdk/workflows_pb2.py index fc79d5752..c7d42b882 100644 --- a/python-sdk/hatchet_sdk/workflows_pb2.py +++ b/python-sdk/hatchet_sdk/workflows_pb2.py @@ -16,7 +16,7 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__ from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0fworkflows.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/wrappers.proto\">\n\x12PutWorkflowRequest\x12(\n\x04opts\x18\x01 \x01(\x0b\x32\x1a.CreateWorkflowVersionOpts\"\x8b\x02\n\x19\x43reateWorkflowVersionOpts\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12\x16\n\x0e\x65vent_triggers\x18\x04 \x03(\t\x12\x15\n\rcron_triggers\x18\x05 \x03(\t\x12\x36\n\x12scheduled_triggers\x18\x06 \x03(\x0b\x32\x1a.google.protobuf.Timestamp\x12$\n\x04jobs\x18\x07 \x03(\x0b\x32\x16.CreateWorkflowJobOpts\x12-\n\x0b\x63oncurrency\x18\x08 \x01(\x0b\x32\x18.WorkflowConcurrencyOpts\"n\n\x17WorkflowConcurrencyOpts\x12\x0e\n\x06\x61\x63tion\x18\x01 \x01(\t\x12\x10\n\x08max_runs\x18\x02 \x01(\x05\x12\x31\n\x0elimit_strategy\x18\x03 \x01(\x0e\x32\x19.ConcurrencyLimitStrategy\"s\n\x15\x43reateWorkflowJobOpts\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0f\n\x07timeout\x18\x03 \x01(\t\x12&\n\x05steps\x18\x04 \x03(\x0b\x32\x17.CreateWorkflowStepOpts\"\x82\x01\n\x16\x43reateWorkflowStepOpts\x12\x13\n\x0breadable_id\x18\x01 \x01(\t\x12\x0e\n\x06\x61\x63tion\x18\x02 \x01(\t\x12\x0f\n\x07timeout\x18\x03 \x01(\t\x12\x0e\n\x06inputs\x18\x04 \x01(\t\x12\x0f\n\x07parents\x18\x05 \x03(\t\x12\x11\n\tuser_data\x18\x06 \x01(\t\"\x16\n\x14ListWorkflowsRequest\"l\n\x17ScheduleWorkflowRequest\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12-\n\tschedules\x18\x02 \x03(\x0b\x32\x1a.google.protobuf.Timestamp\x12\r\n\x05input\x18\x03 \x01(\t\"5\n\x15ListWorkflowsResponse\x12\x1c\n\tworkflows\x18\x01 \x03(\x0b\x32\t.Workflow\"1\n\x1cListWorkflowsForEventRequest\x12\x11\n\tevent_key\x18\x01 \x01(\t\"\xee\x01\n\x08Workflow\x12\n\n\x02id\x18\x01 \x01(\t\x12.\n\ncreated_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12.\n\nupdated_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x11\n\ttenant_id\x18\x05 \x01(\t\x12\x0c\n\x04name\x18\x06 \x01(\t\x12\x31\n\x0b\x64\x65scription\x18\x07 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\"\n\x08versions\x18\x08 \x03(\x0b\x32\x10.WorkflowVersion\"\xeb\x01\n\x0fWorkflowVersion\x12\n\n\x02id\x18\x01 \x01(\t\x12.\n\ncreated_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12.\n\nupdated_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07version\x18\x05 \x01(\t\x12\r\n\x05order\x18\x06 \x01(\x05\x12\x13\n\x0bworkflow_id\x18\x07 \x01(\t\x12#\n\x08triggers\x18\x08 \x01(\x0b\x32\x11.WorkflowTriggers\x12\x12\n\x04jobs\x18\t \x03(\x0b\x32\x04.Job\"\x80\x02\n\x10WorkflowTriggers\x12\n\n\x02id\x18\x01 \x01(\t\x12.\n\ncreated_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12.\n\nupdated_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1b\n\x13workflow_version_id\x18\x05 \x01(\t\x12\x11\n\ttenant_id\x18\x06 \x01(\t\x12(\n\x06\x65vents\x18\x07 \x03(\x0b\x32\x18.WorkflowTriggerEventRef\x12&\n\x05\x63rons\x18\x08 \x03(\x0b\x32\x17.WorkflowTriggerCronRef\"?\n\x17WorkflowTriggerEventRef\x12\x11\n\tparent_id\x18\x01 \x01(\t\x12\x11\n\tevent_key\x18\x02 \x01(\t\"9\n\x16WorkflowTriggerCronRef\x12\x11\n\tparent_id\x18\x01 \x01(\t\x12\x0c\n\x04\x63ron\x18\x02 \x01(\t\"\xa7\x02\n\x03Job\x12\n\n\x02id\x18\x01 \x01(\t\x12.\n\ncreated_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12.\n\nupdated_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x11\n\ttenant_id\x18\x05 \x01(\t\x12\x1b\n\x13workflow_version_id\x18\x06 \x01(\t\x12\x0c\n\x04name\x18\x07 \x01(\t\x12\x31\n\x0b\x64\x65scription\x18\x08 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x14\n\x05steps\x18\t \x03(\x0b\x32\x05.Step\x12-\n\x07timeout\x18\n \x01(\x0b\x32\x1c.google.protobuf.StringValue\"\xaa\x02\n\x04Step\x12\n\n\x02id\x18\x01 \x01(\t\x12.\n\ncreated_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12.\n\nupdated_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x31\n\x0breadable_id\x18\x05 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x11\n\ttenant_id\x18\x06 \x01(\t\x12\x0e\n\x06job_id\x18\x07 \x01(\t\x12\x0e\n\x06\x61\x63tion\x18\x08 \x01(\t\x12-\n\x07timeout\x18\t \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x0f\n\x07parents\x18\n \x03(\t\x12\x10\n\x08\x63hildren\x18\x0b \x03(\t\",\n\x15\x44\x65leteWorkflowRequest\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\"(\n\x18GetWorkflowByNameRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"5\n\x16TriggerWorkflowRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05input\x18\x02 \x01(\t\"2\n\x17TriggerWorkflowResponse\x12\x17\n\x0fworkflow_run_id\x18\x01 \x01(\t*U\n\x18\x43oncurrencyLimitStrategy\x12\x16\n\x12\x43\x41NCEL_IN_PROGRESS\x10\x00\x12\x0f\n\x0b\x44ROP_NEWEST\x10\x01\x12\x10\n\x0cQUEUE_NEWEST\x10\x02\x32\xcd\x03\n\x0fWorkflowService\x12>\n\rListWorkflows\x12\x15.ListWorkflowsRequest\x1a\x16.ListWorkflowsResponse\x12\x34\n\x0bPutWorkflow\x12\x13.PutWorkflowRequest\x1a\x10.WorkflowVersion\x12>\n\x10ScheduleWorkflow\x12\x18.ScheduleWorkflowRequest\x1a\x10.WorkflowVersion\x12\x44\n\x0fTriggerWorkflow\x12\x17.TriggerWorkflowRequest\x1a\x18.TriggerWorkflowResponse\x12\x39\n\x11GetWorkflowByName\x12\x19.GetWorkflowByNameRequest\x1a\t.Workflow\x12N\n\x15ListWorkflowsForEvent\x12\x1d.ListWorkflowsForEventRequest\x1a\x16.ListWorkflowsResponse\x12\x33\n\x0e\x44\x65leteWorkflow\x12\x16.DeleteWorkflowRequest\x1a\t.WorkflowBBZ@github.com/hatchet-dev/hatchet/internal/services/admin/contractsb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0fworkflows.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/wrappers.proto\">\n\x12PutWorkflowRequest\x12(\n\x04opts\x18\x01 \x01(\x0b\x32\x1a.CreateWorkflowVersionOpts\"\x8b\x02\n\x19\x43reateWorkflowVersionOpts\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12\x16\n\x0e\x65vent_triggers\x18\x04 \x03(\t\x12\x15\n\rcron_triggers\x18\x05 \x03(\t\x12\x36\n\x12scheduled_triggers\x18\x06 \x03(\x0b\x32\x1a.google.protobuf.Timestamp\x12$\n\x04jobs\x18\x07 \x03(\x0b\x32\x16.CreateWorkflowJobOpts\x12-\n\x0b\x63oncurrency\x18\x08 \x01(\x0b\x32\x18.WorkflowConcurrencyOpts\"n\n\x17WorkflowConcurrencyOpts\x12\x0e\n\x06\x61\x63tion\x18\x01 \x01(\t\x12\x10\n\x08max_runs\x18\x02 \x01(\x05\x12\x31\n\x0elimit_strategy\x18\x03 \x01(\x0e\x32\x19.ConcurrencyLimitStrategy\"s\n\x15\x43reateWorkflowJobOpts\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0f\n\x07timeout\x18\x03 \x01(\t\x12&\n\x05steps\x18\x04 \x03(\x0b\x32\x17.CreateWorkflowStepOpts\"\x93\x01\n\x16\x43reateWorkflowStepOpts\x12\x13\n\x0breadable_id\x18\x01 \x01(\t\x12\x0e\n\x06\x61\x63tion\x18\x02 \x01(\t\x12\x0f\n\x07timeout\x18\x03 \x01(\t\x12\x0e\n\x06inputs\x18\x04 \x01(\t\x12\x0f\n\x07parents\x18\x05 \x03(\t\x12\x11\n\tuser_data\x18\x06 \x01(\t\x12\x0f\n\x07retries\x18\x07 \x01(\x05\"\x16\n\x14ListWorkflowsRequest\"l\n\x17ScheduleWorkflowRequest\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12-\n\tschedules\x18\x02 \x03(\x0b\x32\x1a.google.protobuf.Timestamp\x12\r\n\x05input\x18\x03 \x01(\t\"5\n\x15ListWorkflowsResponse\x12\x1c\n\tworkflows\x18\x01 \x03(\x0b\x32\t.Workflow\"1\n\x1cListWorkflowsForEventRequest\x12\x11\n\tevent_key\x18\x01 \x01(\t\"\xee\x01\n\x08Workflow\x12\n\n\x02id\x18\x01 \x01(\t\x12.\n\ncreated_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12.\n\nupdated_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x11\n\ttenant_id\x18\x05 \x01(\t\x12\x0c\n\x04name\x18\x06 \x01(\t\x12\x31\n\x0b\x64\x65scription\x18\x07 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\"\n\x08versions\x18\x08 \x03(\x0b\x32\x10.WorkflowVersion\"\xeb\x01\n\x0fWorkflowVersion\x12\n\n\x02id\x18\x01 \x01(\t\x12.\n\ncreated_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12.\n\nupdated_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07version\x18\x05 \x01(\t\x12\r\n\x05order\x18\x06 \x01(\x05\x12\x13\n\x0bworkflow_id\x18\x07 \x01(\t\x12#\n\x08triggers\x18\x08 \x01(\x0b\x32\x11.WorkflowTriggers\x12\x12\n\x04jobs\x18\t \x03(\x0b\x32\x04.Job\"\x80\x02\n\x10WorkflowTriggers\x12\n\n\x02id\x18\x01 \x01(\t\x12.\n\ncreated_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12.\n\nupdated_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1b\n\x13workflow_version_id\x18\x05 \x01(\t\x12\x11\n\ttenant_id\x18\x06 \x01(\t\x12(\n\x06\x65vents\x18\x07 \x03(\x0b\x32\x18.WorkflowTriggerEventRef\x12&\n\x05\x63rons\x18\x08 \x03(\x0b\x32\x17.WorkflowTriggerCronRef\"?\n\x17WorkflowTriggerEventRef\x12\x11\n\tparent_id\x18\x01 \x01(\t\x12\x11\n\tevent_key\x18\x02 \x01(\t\"9\n\x16WorkflowTriggerCronRef\x12\x11\n\tparent_id\x18\x01 \x01(\t\x12\x0c\n\x04\x63ron\x18\x02 \x01(\t\"\xa7\x02\n\x03Job\x12\n\n\x02id\x18\x01 \x01(\t\x12.\n\ncreated_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12.\n\nupdated_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x11\n\ttenant_id\x18\x05 \x01(\t\x12\x1b\n\x13workflow_version_id\x18\x06 \x01(\t\x12\x0c\n\x04name\x18\x07 \x01(\t\x12\x31\n\x0b\x64\x65scription\x18\x08 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x14\n\x05steps\x18\t \x03(\x0b\x32\x05.Step\x12-\n\x07timeout\x18\n \x01(\x0b\x32\x1c.google.protobuf.StringValue\"\xaa\x02\n\x04Step\x12\n\n\x02id\x18\x01 \x01(\t\x12.\n\ncreated_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12.\n\nupdated_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x31\n\x0breadable_id\x18\x05 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x11\n\ttenant_id\x18\x06 \x01(\t\x12\x0e\n\x06job_id\x18\x07 \x01(\t\x12\x0e\n\x06\x61\x63tion\x18\x08 \x01(\t\x12-\n\x07timeout\x18\t \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x0f\n\x07parents\x18\n \x03(\t\x12\x10\n\x08\x63hildren\x18\x0b \x03(\t\",\n\x15\x44\x65leteWorkflowRequest\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\"(\n\x18GetWorkflowByNameRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"5\n\x16TriggerWorkflowRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05input\x18\x02 \x01(\t\"2\n\x17TriggerWorkflowResponse\x12\x17\n\x0fworkflow_run_id\x18\x01 \x01(\t*U\n\x18\x43oncurrencyLimitStrategy\x12\x16\n\x12\x43\x41NCEL_IN_PROGRESS\x10\x00\x12\x0f\n\x0b\x44ROP_NEWEST\x10\x01\x12\x10\n\x0cQUEUE_NEWEST\x10\x02\x32\xcd\x03\n\x0fWorkflowService\x12>\n\rListWorkflows\x12\x15.ListWorkflowsRequest\x1a\x16.ListWorkflowsResponse\x12\x34\n\x0bPutWorkflow\x12\x13.PutWorkflowRequest\x1a\x10.WorkflowVersion\x12>\n\x10ScheduleWorkflow\x12\x18.ScheduleWorkflowRequest\x1a\x10.WorkflowVersion\x12\x44\n\x0fTriggerWorkflow\x12\x17.TriggerWorkflowRequest\x1a\x18.TriggerWorkflowResponse\x12\x39\n\x11GetWorkflowByName\x12\x19.GetWorkflowByNameRequest\x1a\t.Workflow\x12N\n\x15ListWorkflowsForEvent\x12\x1d.ListWorkflowsForEventRequest\x1a\x16.ListWorkflowsResponse\x12\x33\n\x0e\x44\x65leteWorkflow\x12\x16.DeleteWorkflowRequest\x1a\t.WorkflowBBZ@github.com/hatchet-dev/hatchet/internal/services/admin/contractsb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -24,8 +24,8 @@ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'workflows_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: _globals['DESCRIPTOR']._options = None _globals['DESCRIPTOR']._serialized_options = b'Z@github.com/hatchet-dev/hatchet/internal/services/admin/contracts' - _globals['_CONCURRENCYLIMITSTRATEGY']._serialized_start=2676 - _globals['_CONCURRENCYLIMITSTRATEGY']._serialized_end=2761 + _globals['_CONCURRENCYLIMITSTRATEGY']._serialized_start=2693 + _globals['_CONCURRENCYLIMITSTRATEGY']._serialized_end=2778 _globals['_PUTWORKFLOWREQUEST']._serialized_start=84 _globals['_PUTWORKFLOWREQUEST']._serialized_end=146 _globals['_CREATEWORKFLOWVERSIONOPTS']._serialized_start=149 @@ -35,37 +35,37 @@ if _descriptor._USE_C_DESCRIPTORS == False: _globals['_CREATEWORKFLOWJOBOPTS']._serialized_start=530 _globals['_CREATEWORKFLOWJOBOPTS']._serialized_end=645 _globals['_CREATEWORKFLOWSTEPOPTS']._serialized_start=648 - _globals['_CREATEWORKFLOWSTEPOPTS']._serialized_end=778 - _globals['_LISTWORKFLOWSREQUEST']._serialized_start=780 - _globals['_LISTWORKFLOWSREQUEST']._serialized_end=802 - _globals['_SCHEDULEWORKFLOWREQUEST']._serialized_start=804 - _globals['_SCHEDULEWORKFLOWREQUEST']._serialized_end=912 - _globals['_LISTWORKFLOWSRESPONSE']._serialized_start=914 - _globals['_LISTWORKFLOWSRESPONSE']._serialized_end=967 - _globals['_LISTWORKFLOWSFOREVENTREQUEST']._serialized_start=969 - _globals['_LISTWORKFLOWSFOREVENTREQUEST']._serialized_end=1018 - _globals['_WORKFLOW']._serialized_start=1021 - _globals['_WORKFLOW']._serialized_end=1259 - _globals['_WORKFLOWVERSION']._serialized_start=1262 - _globals['_WORKFLOWVERSION']._serialized_end=1497 - _globals['_WORKFLOWTRIGGERS']._serialized_start=1500 - _globals['_WORKFLOWTRIGGERS']._serialized_end=1756 - _globals['_WORKFLOWTRIGGEREVENTREF']._serialized_start=1758 - _globals['_WORKFLOWTRIGGEREVENTREF']._serialized_end=1821 - _globals['_WORKFLOWTRIGGERCRONREF']._serialized_start=1823 - _globals['_WORKFLOWTRIGGERCRONREF']._serialized_end=1880 - _globals['_JOB']._serialized_start=1883 - _globals['_JOB']._serialized_end=2178 - _globals['_STEP']._serialized_start=2181 - _globals['_STEP']._serialized_end=2479 - _globals['_DELETEWORKFLOWREQUEST']._serialized_start=2481 - _globals['_DELETEWORKFLOWREQUEST']._serialized_end=2525 - _globals['_GETWORKFLOWBYNAMEREQUEST']._serialized_start=2527 - _globals['_GETWORKFLOWBYNAMEREQUEST']._serialized_end=2567 - _globals['_TRIGGERWORKFLOWREQUEST']._serialized_start=2569 - _globals['_TRIGGERWORKFLOWREQUEST']._serialized_end=2622 - _globals['_TRIGGERWORKFLOWRESPONSE']._serialized_start=2624 - _globals['_TRIGGERWORKFLOWRESPONSE']._serialized_end=2674 - _globals['_WORKFLOWSERVICE']._serialized_start=2764 - _globals['_WORKFLOWSERVICE']._serialized_end=3225 + _globals['_CREATEWORKFLOWSTEPOPTS']._serialized_end=795 + _globals['_LISTWORKFLOWSREQUEST']._serialized_start=797 + _globals['_LISTWORKFLOWSREQUEST']._serialized_end=819 + _globals['_SCHEDULEWORKFLOWREQUEST']._serialized_start=821 + _globals['_SCHEDULEWORKFLOWREQUEST']._serialized_end=929 + _globals['_LISTWORKFLOWSRESPONSE']._serialized_start=931 + _globals['_LISTWORKFLOWSRESPONSE']._serialized_end=984 + _globals['_LISTWORKFLOWSFOREVENTREQUEST']._serialized_start=986 + _globals['_LISTWORKFLOWSFOREVENTREQUEST']._serialized_end=1035 + _globals['_WORKFLOW']._serialized_start=1038 + _globals['_WORKFLOW']._serialized_end=1276 + _globals['_WORKFLOWVERSION']._serialized_start=1279 + _globals['_WORKFLOWVERSION']._serialized_end=1514 + _globals['_WORKFLOWTRIGGERS']._serialized_start=1517 + _globals['_WORKFLOWTRIGGERS']._serialized_end=1773 + _globals['_WORKFLOWTRIGGEREVENTREF']._serialized_start=1775 + _globals['_WORKFLOWTRIGGEREVENTREF']._serialized_end=1838 + _globals['_WORKFLOWTRIGGERCRONREF']._serialized_start=1840 + _globals['_WORKFLOWTRIGGERCRONREF']._serialized_end=1897 + _globals['_JOB']._serialized_start=1900 + _globals['_JOB']._serialized_end=2195 + _globals['_STEP']._serialized_start=2198 + _globals['_STEP']._serialized_end=2496 + _globals['_DELETEWORKFLOWREQUEST']._serialized_start=2498 + _globals['_DELETEWORKFLOWREQUEST']._serialized_end=2542 + _globals['_GETWORKFLOWBYNAMEREQUEST']._serialized_start=2544 + _globals['_GETWORKFLOWBYNAMEREQUEST']._serialized_end=2584 + _globals['_TRIGGERWORKFLOWREQUEST']._serialized_start=2586 + _globals['_TRIGGERWORKFLOWREQUEST']._serialized_end=2639 + _globals['_TRIGGERWORKFLOWRESPONSE']._serialized_start=2641 + _globals['_TRIGGERWORKFLOWRESPONSE']._serialized_end=2691 + _globals['_WORKFLOWSERVICE']._serialized_start=2781 + _globals['_WORKFLOWSERVICE']._serialized_end=3242 # @@protoc_insertion_point(module_scope) diff --git a/python-sdk/hatchet_sdk/workflows_pb2.pyi b/python-sdk/hatchet_sdk/workflows_pb2.pyi index eddfa996a..27fcedd00 100644 --- a/python-sdk/hatchet_sdk/workflows_pb2.pyi +++ b/python-sdk/hatchet_sdk/workflows_pb2.pyi @@ -66,20 +66,22 @@ class CreateWorkflowJobOpts(_message.Message): def __init__(self, name: _Optional[str] = ..., description: _Optional[str] = ..., timeout: _Optional[str] = ..., steps: _Optional[_Iterable[_Union[CreateWorkflowStepOpts, _Mapping]]] = ...) -> None: ... class CreateWorkflowStepOpts(_message.Message): - __slots__ = ("readable_id", "action", "timeout", "inputs", "parents", "user_data") + __slots__ = ("readable_id", "action", "timeout", "inputs", "parents", "user_data", "retries") READABLE_ID_FIELD_NUMBER: _ClassVar[int] ACTION_FIELD_NUMBER: _ClassVar[int] TIMEOUT_FIELD_NUMBER: _ClassVar[int] INPUTS_FIELD_NUMBER: _ClassVar[int] PARENTS_FIELD_NUMBER: _ClassVar[int] USER_DATA_FIELD_NUMBER: _ClassVar[int] + RETRIES_FIELD_NUMBER: _ClassVar[int] readable_id: str action: str timeout: str inputs: str parents: _containers.RepeatedScalarFieldContainer[str] user_data: str - def __init__(self, readable_id: _Optional[str] = ..., action: _Optional[str] = ..., timeout: _Optional[str] = ..., inputs: _Optional[str] = ..., parents: _Optional[_Iterable[str]] = ..., user_data: _Optional[str] = ...) -> None: ... + retries: int + def __init__(self, readable_id: _Optional[str] = ..., action: _Optional[str] = ..., timeout: _Optional[str] = ..., inputs: _Optional[str] = ..., parents: _Optional[_Iterable[str]] = ..., user_data: _Optional[str] = ..., retries: _Optional[int] = ...) -> None: ... class ListWorkflowsRequest(_message.Message): __slots__ = () diff --git a/python-sdk/pyproject.toml b/python-sdk/pyproject.toml index 3557852f8..1c2869a33 100644 --- a/python-sdk/pyproject.toml +++ b/python-sdk/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "hatchet-sdk" -version = "0.9.4" +version = "0.10.0" description = "" authors = ["Alexander Belanger "] readme = "README.md" diff --git a/typescript-sdk/examples/retries-worker.ts b/typescript-sdk/examples/retries-worker.ts new file mode 100644 index 000000000..c8116f7b3 --- /dev/null +++ b/typescript-sdk/examples/retries-worker.ts @@ -0,0 +1,53 @@ +import Hatchet from '../src/sdk'; +import { Workflow } from '../src/workflow'; + +const hatchet = Hatchet.init(); + +const sleep = (ms: number) => + new Promise((resolve) => { + setTimeout(resolve, ms); + }); + +let numRetries = 0; + +const workflow: Workflow = { + id: 'retries-workflow', + description: 'test', + on: { + event: 'user:create', + }, + steps: [ + { + name: 'step1', + run: async (ctx) => { + if (numRetries < 3) { + numRetries += 1; + throw new Error('step1 failed'); + } + + console.log('starting step1 with the following input', ctx.workflowInput()); + console.log('waiting 5 seconds...'); + await sleep(5000); + console.log('executed step1!'); + return { step1: 'step1 results!' }; + }, + retries: 3, + }, + { + name: 'step2', + parents: ['step1'], + run: (ctx) => { + console.log('executed step2 after step1 returned ', ctx.stepOutput('step1')); + return { step2: 'step2 results!' }; + }, + }, + ], +}; + +async function main() { + const worker = await hatchet.worker('example-worker'); + await worker.registerWorkflow(workflow); + worker.start(); +} + +main(); diff --git a/typescript-sdk/package.json b/typescript-sdk/package.json index 166669506..cc362f6dc 100644 --- a/typescript-sdk/package.json +++ b/typescript-sdk/package.json @@ -1,6 +1,6 @@ { "name": "@hatchet-dev/typescript-sdk", - "version": "0.1.13", + "version": "0.1.14", "description": "Background task orchestration & visibility for developers", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -33,6 +33,7 @@ "worker:dag": "npm run exec -- ./examples/dag-worker.ts", "worker:concurrency": "npm run exec -- ./examples/concurrency/concurrency-worker.ts", "event:concurrency": "npm run exec -- ./examples/concurrency/concurrency-event.ts", + "worker:retries": "npm run exec -- ./examples/retries-worker.ts", "api": "npm run exec -- ./examples/api.ts", "prepublish": "cp package.json dist/package.json;", "publish:ci": "rm -rf ./dist && npm run tsc:build && npm run prepublish && cd dist && npm publish --access public --no-git-checks", diff --git a/typescript-sdk/src/clients/worker/worker.ts b/typescript-sdk/src/clients/worker/worker.ts index c2fb13f41..056d48536 100644 --- a/typescript-sdk/src/clients/worker/worker.ts +++ b/typescript-sdk/src/clients/worker/worker.ts @@ -80,6 +80,7 @@ export class Worker { inputs: '{}', parents: step.parents ?? [], userData: '{}', + retries: step.retries || 0, })), }, ], diff --git a/typescript-sdk/src/protoc/dispatcher/dispatcher.ts b/typescript-sdk/src/protoc/dispatcher/dispatcher.ts index b28b9d244..ed6bed879 100644 --- a/typescript-sdk/src/protoc/dispatcher/dispatcher.ts +++ b/typescript-sdk/src/protoc/dispatcher/dispatcher.ts @@ -352,6 +352,25 @@ export interface WorkflowEvent { | undefined; /** the event payload */ eventPayload: string; + /** + * whether this is the last event for the workflow run - server + * will hang up the connection but clients might want to case + */ + hangup: boolean; +} + +export interface OverridesData { + /** the step run id */ + stepRunId: string; + /** the path of the data to set */ + path: string; + /** the value to set */ + value: string; + /** the filename of the caller */ + callerFilename: string; +} + +export interface OverridesDataResponse { } function createBaseWorkerRegisterRequest(): WorkerRegisterRequest { @@ -1426,6 +1445,7 @@ function createBaseWorkflowEvent(): WorkflowEvent { resourceId: "", eventTimestamp: undefined, eventPayload: "", + hangup: false, }; } @@ -1449,6 +1469,9 @@ export const WorkflowEvent = { if (message.eventPayload !== "") { writer.uint32(50).string(message.eventPayload); } + if (message.hangup === true) { + writer.uint32(56).bool(message.hangup); + } return writer; }, @@ -1501,6 +1524,13 @@ export const WorkflowEvent = { message.eventPayload = reader.string(); continue; + case 7: + if (tag !== 56) { + break; + } + + message.hangup = reader.bool(); + continue; } if ((tag & 7) === 4 || tag === 0) { break; @@ -1518,6 +1548,7 @@ export const WorkflowEvent = { resourceId: isSet(object.resourceId) ? globalThis.String(object.resourceId) : "", eventTimestamp: isSet(object.eventTimestamp) ? fromJsonTimestamp(object.eventTimestamp) : undefined, eventPayload: isSet(object.eventPayload) ? globalThis.String(object.eventPayload) : "", + hangup: isSet(object.hangup) ? globalThis.Boolean(object.hangup) : false, }; }, @@ -1541,6 +1572,9 @@ export const WorkflowEvent = { if (message.eventPayload !== "") { obj.eventPayload = message.eventPayload; } + if (message.hangup === true) { + obj.hangup = message.hangup; + } return obj; }, @@ -1555,6 +1589,154 @@ export const WorkflowEvent = { message.resourceId = object.resourceId ?? ""; message.eventTimestamp = object.eventTimestamp ?? undefined; message.eventPayload = object.eventPayload ?? ""; + message.hangup = object.hangup ?? false; + return message; + }, +}; + +function createBaseOverridesData(): OverridesData { + return { stepRunId: "", path: "", value: "", callerFilename: "" }; +} + +export const OverridesData = { + encode(message: OverridesData, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { + if (message.stepRunId !== "") { + writer.uint32(10).string(message.stepRunId); + } + if (message.path !== "") { + writer.uint32(18).string(message.path); + } + if (message.value !== "") { + writer.uint32(26).string(message.value); + } + if (message.callerFilename !== "") { + writer.uint32(34).string(message.callerFilename); + } + return writer; + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): OverridesData { + const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseOverridesData(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: + if (tag !== 10) { + break; + } + + message.stepRunId = reader.string(); + continue; + case 2: + if (tag !== 18) { + break; + } + + message.path = reader.string(); + continue; + case 3: + if (tag !== 26) { + break; + } + + message.value = reader.string(); + continue; + case 4: + if (tag !== 34) { + break; + } + + message.callerFilename = reader.string(); + continue; + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skipType(tag & 7); + } + return message; + }, + + fromJSON(object: any): OverridesData { + return { + stepRunId: isSet(object.stepRunId) ? globalThis.String(object.stepRunId) : "", + path: isSet(object.path) ? globalThis.String(object.path) : "", + value: isSet(object.value) ? globalThis.String(object.value) : "", + callerFilename: isSet(object.callerFilename) ? globalThis.String(object.callerFilename) : "", + }; + }, + + toJSON(message: OverridesData): unknown { + const obj: any = {}; + if (message.stepRunId !== "") { + obj.stepRunId = message.stepRunId; + } + if (message.path !== "") { + obj.path = message.path; + } + if (message.value !== "") { + obj.value = message.value; + } + if (message.callerFilename !== "") { + obj.callerFilename = message.callerFilename; + } + return obj; + }, + + create(base?: DeepPartial): OverridesData { + return OverridesData.fromPartial(base ?? {}); + }, + fromPartial(object: DeepPartial): OverridesData { + const message = createBaseOverridesData(); + message.stepRunId = object.stepRunId ?? ""; + message.path = object.path ?? ""; + message.value = object.value ?? ""; + message.callerFilename = object.callerFilename ?? ""; + return message; + }, +}; + +function createBaseOverridesDataResponse(): OverridesDataResponse { + return {}; +} + +export const OverridesDataResponse = { + encode(_: OverridesDataResponse, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { + return writer; + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): OverridesDataResponse { + const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseOverridesDataResponse(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skipType(tag & 7); + } + return message; + }, + + fromJSON(_: any): OverridesDataResponse { + return {}; + }, + + toJSON(_: OverridesDataResponse): unknown { + const obj: any = {}; + return obj; + }, + + create(base?: DeepPartial): OverridesDataResponse { + return OverridesDataResponse.fromPartial(base ?? {}); + }, + fromPartial(_: DeepPartial): OverridesDataResponse { + const message = createBaseOverridesDataResponse(); return message; }, }; @@ -1604,6 +1786,14 @@ export const DispatcherDefinition = { responseStream: false, options: {}, }, + putOverridesData: { + name: "PutOverridesData", + requestType: OverridesData, + requestStream: false, + responseType: OverridesDataResponse, + responseStream: false, + options: {}, + }, unsubscribe: { name: "Unsubscribe", requestType: WorkerUnsubscribeRequest, @@ -1636,6 +1826,10 @@ export interface DispatcherServiceImplementation { request: GroupKeyActionEvent, context: CallContext & CallContextExt, ): Promise>; + putOverridesData( + request: OverridesData, + context: CallContext & CallContextExt, + ): Promise>; unsubscribe( request: WorkerUnsubscribeRequest, context: CallContext & CallContextExt, @@ -1663,6 +1857,10 @@ export interface DispatcherClient { request: DeepPartial, options?: CallOptions & CallOptionsExt, ): Promise; + putOverridesData( + request: DeepPartial, + options?: CallOptions & CallOptionsExt, + ): Promise; unsubscribe( request: DeepPartial, options?: CallOptions & CallOptionsExt, diff --git a/typescript-sdk/src/protoc/workflows/workflows.ts b/typescript-sdk/src/protoc/workflows/workflows.ts index 3a714e408..ffd476784 100644 --- a/typescript-sdk/src/protoc/workflows/workflows.ts +++ b/typescript-sdk/src/protoc/workflows/workflows.ts @@ -104,6 +104,8 @@ export interface CreateWorkflowStepOpts { parents: string[]; /** (optional) the custom step user data, assuming string representation of JSON */ userData: string; + /** (optional) the number of retries for the step, default 0 */ + retries: number; } /** ListWorkflowsRequest is the request for ListWorkflows. */ @@ -663,7 +665,7 @@ export const CreateWorkflowJobOpts = { }; function createBaseCreateWorkflowStepOpts(): CreateWorkflowStepOpts { - return { readableId: "", action: "", timeout: "", inputs: "", parents: [], userData: "" }; + return { readableId: "", action: "", timeout: "", inputs: "", parents: [], userData: "", retries: 0 }; } export const CreateWorkflowStepOpts = { @@ -686,6 +688,9 @@ export const CreateWorkflowStepOpts = { if (message.userData !== "") { writer.uint32(50).string(message.userData); } + if (message.retries !== 0) { + writer.uint32(56).int32(message.retries); + } return writer; }, @@ -738,6 +743,13 @@ export const CreateWorkflowStepOpts = { message.userData = reader.string(); continue; + case 7: + if (tag !== 56) { + break; + } + + message.retries = reader.int32(); + continue; } if ((tag & 7) === 4 || tag === 0) { break; @@ -755,6 +767,7 @@ export const CreateWorkflowStepOpts = { inputs: isSet(object.inputs) ? globalThis.String(object.inputs) : "", parents: globalThis.Array.isArray(object?.parents) ? object.parents.map((e: any) => globalThis.String(e)) : [], userData: isSet(object.userData) ? globalThis.String(object.userData) : "", + retries: isSet(object.retries) ? globalThis.Number(object.retries) : 0, }; }, @@ -778,6 +791,9 @@ export const CreateWorkflowStepOpts = { if (message.userData !== "") { obj.userData = message.userData; } + if (message.retries !== 0) { + obj.retries = Math.round(message.retries); + } return obj; }, @@ -792,6 +808,7 @@ export const CreateWorkflowStepOpts = { message.inputs = object.inputs ?? ""; message.parents = object.parents?.map((e) => e) || []; message.userData = object.userData ?? ""; + message.retries = object.retries ?? 0; return message; }, }; diff --git a/typescript-sdk/src/step.ts b/typescript-sdk/src/step.ts index bdb8cf1d6..22b37d6dd 100644 --- a/typescript-sdk/src/step.ts +++ b/typescript-sdk/src/step.ts @@ -6,6 +6,7 @@ export const CreateStepSchema = z.object({ name: z.string(), parents: z.array(z.string()).optional(), timeout: HatchetTimeoutSchema.optional(), + retries: z.number().optional(), }); export type NextStep = { [key: string]: string };