mirror of
https://github.com/hatchet-dev/hatchet.git
synced 2026-05-06 18:09:49 -05:00
add a monitoring probe (#1108)
* add a monitoring probe --------- Co-authored-by: Sean Reilly <sean@hatchet.run>
This commit is contained in:
Sean Reilly
@@ -178,3 +178,5 @@ paths:
|
||||
$ref: "./paths/webhook-worker/webhook-worker.yaml#/webhookworkerRequests"
|
||||
/api/v1/tenants/{tenant}/workflow-runs/{workflow-run}/input:
|
||||
$ref: "./paths/workflow-run/workflow-run.yaml#/getWorkflowRunInput"
|
||||
/api/v1/monitoring/{tenant}/probe:
|
||||
$ref: "./paths/monitoring/monitoring.yaml#/probe"
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
probe:
|
||||
post:
|
||||
x-resources: ["tenant"]
|
||||
description: Triggers a workflow to check the status of the instance
|
||||
summary: Detailed Health Probe For the Instance
|
||||
operationId: monitoring:post:runProbe
|
||||
parameters:
|
||||
- description: The tenant id
|
||||
in: path
|
||||
name: tenant
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
format: uuid
|
||||
minLength: 36
|
||||
maxLength: 36
|
||||
responses:
|
||||
"200":
|
||||
description: Successfully executed the probe.
|
||||
"403":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "../../components/schemas/_index.yaml#/APIErrors"
|
||||
description: Not authorized to perform this action
|
||||
@@ -0,0 +1,313 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/labstack/echo/v4"
|
||||
"golang.org/x/exp/rand"
|
||||
|
||||
"github.com/hatchet-dev/hatchet/api/v1/server/oas/gen"
|
||||
"github.com/hatchet-dev/hatchet/pkg/client"
|
||||
clientconfig "github.com/hatchet-dev/hatchet/pkg/config/client"
|
||||
"github.com/hatchet-dev/hatchet/pkg/config/shared"
|
||||
"github.com/hatchet-dev/hatchet/pkg/repository/prisma/db"
|
||||
"github.com/hatchet-dev/hatchet/pkg/repository/prisma/dbsqlc"
|
||||
"github.com/hatchet-dev/hatchet/pkg/repository/prisma/sqlchelpers"
|
||||
"github.com/hatchet-dev/hatchet/pkg/worker"
|
||||
)
|
||||
|
||||
func (m *MonitoringService) MonitoringPostRunProbe(ctx echo.Context, request gen.MonitoringPostRunProbeRequestObject) (gen.MonitoringPostRunProbeResponseObject, error) {
|
||||
if !m.enabled {
|
||||
return nil, fmt.Errorf("monitoring is not enabled")
|
||||
}
|
||||
|
||||
tenant := ctx.Get("tenant").(*db.TenantModel)
|
||||
|
||||
if !slices.Contains[[]string](m.permittedTenants, tenant.ID) {
|
||||
|
||||
err := fmt.Errorf("tenant is not a monitoring tenant for this instance")
|
||||
|
||||
if len(m.permittedTenants) > 0 {
|
||||
m.l.Error().Err(err).Msgf("monitoring tenants are %v", m.permittedTenants)
|
||||
} else {
|
||||
m.l.Error().Err(err).Msg("no monitoring tenants are configured")
|
||||
}
|
||||
|
||||
return gen.MonitoringPostRunProbe403JSONResponse{}, nil
|
||||
}
|
||||
|
||||
token, err := getBearerTokenFromRequest(ctx.Request())
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
events := make(chan string, 50)
|
||||
stepChan := make(chan string, 50)
|
||||
errorChan := make(chan error, 50)
|
||||
|
||||
cancellableContext, cancel := context.WithTimeout(ctx.Request().Context(), m.probeTimeout)
|
||||
|
||||
defer cancel()
|
||||
|
||||
cf := clientconfig.ClientConfigFile{
|
||||
Token: token,
|
||||
TenantId: tenant.ID,
|
||||
Namespace: randomNamespace(),
|
||||
TLS: clientconfig.ClientTLSConfigFile{
|
||||
Base: shared.TLSConfigFile{
|
||||
TLSRootCAFile: m.tlsRootCAFile,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
cleanup, err := m.run(cancellableContext, cf, m.workflowName, events, stepChan, errorChan)
|
||||
if err != nil {
|
||||
m.l.Error().Msgf("error running probe: %s", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer cleanup()
|
||||
// Stream events are not necessarily received in order so we don't distinguish between them
|
||||
messages := []string{"This is a stream event", "This is a stream event"}
|
||||
messageIndex := 0
|
||||
stepMessages := []string{"step-one", "step-two"}
|
||||
for {
|
||||
|
||||
select {
|
||||
case <-cancellableContext.Done():
|
||||
|
||||
if cancellableContext.Err() == context.DeadlineExceeded {
|
||||
m.l.Error().Msg("timed out waiting for probe to complete")
|
||||
return nil, fmt.Errorf("timed out waiting for probe to complete")
|
||||
}
|
||||
|
||||
case err := <-errorChan:
|
||||
m.l.Error().Msgf("error during probe: %s", err)
|
||||
return nil, err
|
||||
|
||||
case e := <-events:
|
||||
if !strings.HasPrefix(e, messages[messageIndex]) {
|
||||
return nil, fmt.Errorf("expected message %s, to start with %s", messages[messageIndex], e)
|
||||
}
|
||||
messageIndex++
|
||||
|
||||
if messageIndex == len(messages) {
|
||||
for i := range stepMessages {
|
||||
stepMessage := <-stepChan
|
||||
if stepMessage != stepMessages[i] {
|
||||
|
||||
return nil, fmt.Errorf("probe did not complete successfully - step messages failed")
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
type probeEvent struct {
|
||||
UniqueStreamId string
|
||||
}
|
||||
|
||||
type stepOneOutput struct {
|
||||
Message string `json:"message"`
|
||||
UniqueStreamId string
|
||||
}
|
||||
|
||||
func (m *MonitoringService) run(ctx context.Context, cf clientconfig.ClientConfigFile, workflowName string, events chan<- string, stepChan chan<- string, errors chan<- error) (func(), error) {
|
||||
|
||||
c, err := client.NewFromConfigFile(
|
||||
&cf, client.WithLogLevel(m.l.GetLevel().String()),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating client: %w", err)
|
||||
}
|
||||
|
||||
w, err := worker.NewWorker(
|
||||
|
||||
worker.WithClient(
|
||||
c,
|
||||
), worker.WithLogLevel(m.l.GetLevel().String()),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating worker: %w", err)
|
||||
}
|
||||
streamKey := "streamKey"
|
||||
streamValue := fmt.Sprintf("stream-event-%d", rand.Intn(100)+1)
|
||||
var wfrId string
|
||||
|
||||
err = w.RegisterWorkflow(
|
||||
&worker.WorkflowJob{
|
||||
On: worker.Events(m.eventName),
|
||||
Name: workflowName,
|
||||
Description: "This is part of the monitoring system for testing the readiness of this Hatchet installation.",
|
||||
Steps: []*worker.WorkflowStep{
|
||||
worker.Fn(func(ctx worker.HatchetContext) (result *stepOneOutput, err error) {
|
||||
|
||||
input := &probeEvent{}
|
||||
|
||||
err = ctx.WorkflowInput(input)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
wfrId = ctx.WorkflowRunId()
|
||||
|
||||
if input.UniqueStreamId == "" {
|
||||
return nil, fmt.Errorf("uniqueStreamId is required")
|
||||
}
|
||||
|
||||
if input.UniqueStreamId != streamValue {
|
||||
return nil, fmt.Errorf("uniqueStreamId does not match stream value")
|
||||
}
|
||||
|
||||
ctx.StreamEvent([]byte("This is a stream event for step-one"))
|
||||
|
||||
stepChan <- "step-one"
|
||||
|
||||
return &stepOneOutput{
|
||||
Message: "This is a message for step-one",
|
||||
UniqueStreamId: streamValue,
|
||||
}, nil
|
||||
},
|
||||
).SetName("step-one"),
|
||||
worker.Fn(func(ctx worker.HatchetContext) (result *stepOneOutput, err error) {
|
||||
input := &probeEvent{}
|
||||
err = ctx.StepOutput("step-one", input)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if input.UniqueStreamId == "" {
|
||||
return nil, fmt.Errorf("uniqueStreamId is required")
|
||||
}
|
||||
|
||||
if input.UniqueStreamId != streamValue {
|
||||
return nil, fmt.Errorf("uniqueStreamId does not match stream value")
|
||||
}
|
||||
|
||||
ctx.StreamEvent([]byte("This is a stream event for step-two"))
|
||||
stepChan <- "step-two"
|
||||
|
||||
return &stepOneOutput{
|
||||
Message: "This is a message for step-two",
|
||||
UniqueStreamId: streamValue,
|
||||
}, nil
|
||||
}).SetName("step-two").AddParents("step-one"),
|
||||
},
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error registering workflow: %w", err)
|
||||
}
|
||||
|
||||
go func() {
|
||||
err = c.Subscribe().StreamByAdditionalMetadata(ctx, streamKey, streamValue, func(event client.StreamEvent) error {
|
||||
events <- string(event.Message)
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
errors <- fmt.Errorf("error subscribing to stream: %w", err)
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
testEvent := probeEvent{
|
||||
UniqueStreamId: streamValue,
|
||||
}
|
||||
|
||||
err := c.Event().Push(
|
||||
ctx,
|
||||
m.eventName,
|
||||
testEvent,
|
||||
client.WithEventMetadata(map[string]string{
|
||||
streamKey: streamValue,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
errors <- fmt.Errorf("error pushing event: %w", err)
|
||||
}
|
||||
}()
|
||||
|
||||
cleanupWorker, err := w.Start()
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error starting worker: %w", err)
|
||||
}
|
||||
|
||||
cleanup := func() {
|
||||
err := cleanupWorker()
|
||||
if err != nil {
|
||||
m.l.Error().Msgf("error cleaning up worker: %s", err)
|
||||
}
|
||||
defer func() {
|
||||
if wfrId == "" {
|
||||
m.l.Warn().Msg("workflow run id was never set for probe")
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
wrfRow, err := m.config.APIRepository.WorkflowRun().GetWorkflowRunById(ctx, cf.TenantId, wfrId)
|
||||
|
||||
if err != nil {
|
||||
m.l.Error().Msgf("error getting workflow run: %s", err)
|
||||
|
||||
}
|
||||
|
||||
if wrfRow.Status != dbsqlc.WorkflowRunStatusRUNNING {
|
||||
|
||||
workflowId := sqlchelpers.UUIDToStr(wrfRow.Workflow.ID)
|
||||
|
||||
_, err = m.config.APIRepository.Workflow().DeleteWorkflow(ctx, cf.TenantId, workflowId)
|
||||
|
||||
m.l.Info().Msgf("deleted workflow %s", workflowId)
|
||||
if err != nil {
|
||||
m.l.Error().Msgf("error deleting workflow: %s", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
||||
}
|
||||
|
||||
m.l.Error().Msg("could not clean up workflow after 10 attempts")
|
||||
|
||||
}()
|
||||
}
|
||||
|
||||
return cleanup, nil
|
||||
}
|
||||
|
||||
func getBearerTokenFromRequest(r *http.Request) (string, error) {
|
||||
reqToken := r.Header.Get("Authorization")
|
||||
splitToken := strings.Split(reqToken, "Bearer")
|
||||
|
||||
if len(splitToken) != 2 {
|
||||
return "", fmt.Errorf("invalid token")
|
||||
}
|
||||
|
||||
reqToken = strings.TrimSpace(splitToken[1])
|
||||
|
||||
return reqToken, nil
|
||||
}
|
||||
|
||||
func randomNamespace() string {
|
||||
return "ns_" + uuid.New().String()[0:8]
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
|
||||
"github.com/hatchet-dev/hatchet/pkg/config/server"
|
||||
)
|
||||
|
||||
type MonitoringService struct {
|
||||
enabled bool
|
||||
permittedTenants []string
|
||||
eventName string
|
||||
workflowName string
|
||||
probeTimeout time.Duration
|
||||
config *server.ServerConfig
|
||||
l *zerolog.Logger
|
||||
tlsRootCAFile string
|
||||
}
|
||||
|
||||
func NewMonitoringService(config *server.ServerConfig) *MonitoringService {
|
||||
return &MonitoringService{
|
||||
enabled: config.Runtime.Monitoring.Enabled,
|
||||
l: config.Logger,
|
||||
permittedTenants: config.Runtime.Monitoring.PermittedTenants,
|
||||
eventName: "monitoring:probe",
|
||||
workflowName: "probe-workflow",
|
||||
probeTimeout: config.Runtime.Monitoring.ProbeTimeout,
|
||||
tlsRootCAFile: config.Runtime.Monitoring.TLSRootCAFile,
|
||||
config: config,
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,7 @@ func (t *WorkflowService) WorkflowDelete(ctx echo.Context, request gen.WorkflowD
|
||||
tenant := ctx.Get("tenant").(*db.TenantModel)
|
||||
workflow := ctx.Get("workflow").(*dbsqlc.GetWorkflowByIdRow)
|
||||
|
||||
_, err := t.config.APIRepository.Workflow().DeleteWorkflow(tenant.ID, sqlchelpers.UUIDToStr(workflow.Workflow.ID))
|
||||
_, err := t.config.APIRepository.Workflow().DeleteWorkflow(ctx.Request().Context(), tenant.ID, sqlchelpers.UUIDToStr(workflow.Workflow.ID))
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -1842,6 +1842,9 @@ type ServerInterface interface {
|
||||
// List integrations
|
||||
// (GET /api/v1/meta/integrations)
|
||||
MetadataListIntegrations(ctx echo.Context) error
|
||||
// Detailed Health Probe For the Instance
|
||||
// (POST /api/v1/monitoring/{tenant}/probe)
|
||||
MonitoringPostRunProbe(ctx echo.Context, tenant openapi_types.UUID) error
|
||||
// Delete Slack webhook
|
||||
// (DELETE /api/v1/slack/{slack})
|
||||
SlackWebhookDelete(ctx echo.Context, slack openapi_types.UUID) error
|
||||
@@ -2244,6 +2247,26 @@ func (w *ServerInterfaceWrapper) MetadataListIntegrations(ctx echo.Context) erro
|
||||
return err
|
||||
}
|
||||
|
||||
// MonitoringPostRunProbe converts echo context to params.
|
||||
func (w *ServerInterfaceWrapper) MonitoringPostRunProbe(ctx echo.Context) error {
|
||||
var err error
|
||||
// ------------- Path parameter "tenant" -------------
|
||||
var tenant openapi_types.UUID
|
||||
|
||||
err = runtime.BindStyledParameterWithLocation("simple", false, "tenant", runtime.ParamLocationPath, ctx.Param("tenant"), &tenant)
|
||||
if err != nil {
|
||||
return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter tenant: %s", err))
|
||||
}
|
||||
|
||||
ctx.Set(BearerAuthScopes, []string{})
|
||||
|
||||
ctx.Set(CookieAuthScopes, []string{})
|
||||
|
||||
// Invoke the callback with all the unmarshaled arguments
|
||||
err = w.Handler.MonitoringPostRunProbe(ctx, tenant)
|
||||
return err
|
||||
}
|
||||
|
||||
// SlackWebhookDelete converts echo context to params.
|
||||
func (w *ServerInterfaceWrapper) SlackWebhookDelete(ctx echo.Context) error {
|
||||
var err error
|
||||
@@ -4461,6 +4484,7 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL
|
||||
router.GET(baseURL+"/api/v1/events/:event/data", wrapper.EventDataGet)
|
||||
router.GET(baseURL+"/api/v1/meta", wrapper.MetadataGet)
|
||||
router.GET(baseURL+"/api/v1/meta/integrations", wrapper.MetadataListIntegrations)
|
||||
router.POST(baseURL+"/api/v1/monitoring/:tenant/probe", wrapper.MonitoringPostRunProbe)
|
||||
router.DELETE(baseURL+"/api/v1/slack/:slack", wrapper.SlackWebhookDelete)
|
||||
router.DELETE(baseURL+"/api/v1/sns/:sns", wrapper.SnsDelete)
|
||||
router.POST(baseURL+"/api/v1/sns/:tenant/:event", wrapper.SnsUpdate)
|
||||
@@ -4841,6 +4865,31 @@ func (response MetadataListIntegrations400JSONResponse) VisitMetadataListIntegra
|
||||
return json.NewEncoder(w).Encode(response)
|
||||
}
|
||||
|
||||
type MonitoringPostRunProbeRequestObject struct {
|
||||
Tenant openapi_types.UUID `json:"tenant"`
|
||||
}
|
||||
|
||||
type MonitoringPostRunProbeResponseObject interface {
|
||||
VisitMonitoringPostRunProbeResponse(w http.ResponseWriter) error
|
||||
}
|
||||
|
||||
type MonitoringPostRunProbe200Response struct {
|
||||
}
|
||||
|
||||
func (response MonitoringPostRunProbe200Response) VisitMonitoringPostRunProbeResponse(w http.ResponseWriter) error {
|
||||
w.WriteHeader(200)
|
||||
return nil
|
||||
}
|
||||
|
||||
type MonitoringPostRunProbe403JSONResponse APIErrors
|
||||
|
||||
func (response MonitoringPostRunProbe403JSONResponse) VisitMonitoringPostRunProbeResponse(w http.ResponseWriter) error {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(403)
|
||||
|
||||
return json.NewEncoder(w).Encode(response)
|
||||
}
|
||||
|
||||
type SlackWebhookDeleteRequestObject struct {
|
||||
Slack openapi_types.UUID `json:"slack"`
|
||||
}
|
||||
@@ -8044,6 +8093,8 @@ type StrictServerInterface interface {
|
||||
|
||||
MetadataListIntegrations(ctx echo.Context, request MetadataListIntegrationsRequestObject) (MetadataListIntegrationsResponseObject, error)
|
||||
|
||||
MonitoringPostRunProbe(ctx echo.Context, request MonitoringPostRunProbeRequestObject) (MonitoringPostRunProbeResponseObject, error)
|
||||
|
||||
SlackWebhookDelete(ctx echo.Context, request SlackWebhookDeleteRequestObject) (SlackWebhookDeleteResponseObject, error)
|
||||
|
||||
SnsDelete(ctx echo.Context, request SnsDeleteRequestObject) (SnsDeleteResponseObject, error)
|
||||
@@ -8466,6 +8517,31 @@ func (sh *strictHandler) MetadataListIntegrations(ctx echo.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// MonitoringPostRunProbe operation middleware
|
||||
func (sh *strictHandler) MonitoringPostRunProbe(ctx echo.Context, tenant openapi_types.UUID) error {
|
||||
var request MonitoringPostRunProbeRequestObject
|
||||
|
||||
request.Tenant = tenant
|
||||
|
||||
handler := func(ctx echo.Context, request interface{}) (interface{}, error) {
|
||||
return sh.ssi.MonitoringPostRunProbe(ctx, request.(MonitoringPostRunProbeRequestObject))
|
||||
}
|
||||
for _, middleware := range sh.middlewares {
|
||||
handler = middleware(handler, "MonitoringPostRunProbe")
|
||||
}
|
||||
|
||||
response, err := handler(ctx, request)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
} else if validResponse, ok := response.(MonitoringPostRunProbeResponseObject); ok {
|
||||
return validResponse.VisitMonitoringPostRunProbeResponse(ctx.Response())
|
||||
} else if response != nil {
|
||||
return fmt.Errorf("Unexpected response type: %T", response)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SlackWebhookDelete operation middleware
|
||||
func (sh *strictHandler) SlackWebhookDelete(ctx echo.Context, slack openapi_types.UUID) error {
|
||||
var request SlackWebhookDeleteRequestObject
|
||||
@@ -10790,84 +10866,86 @@ var swaggerSpec = []string{
|
||||
"IE4aZmFbW0NgNVo7R7D6LCe2/ofyIPO8J4fYi2L+pi1ONGW/uXH14Cf772vVflMpxVrtlzaU2Vj5RtZK",
|
||||
"IjaEUTlhXzcqhFa32aJ8RM3hnUCSIPgoxBrHBtuxVrYVSFzBTE7eHMUVUo3Tz52Zwg/qxBrblkyq1dD8",
|
||||
"WSbA3jvdnzESbml/u2h/Bhc+w42n9+YObpG6vglNZUfijhzkqzjC6RgHSuFYbNzxc4TpBShwCq1NG0xb",
|
||||
"94sN17bbdC6x48qUDTdfZi0orG6bCCHberYRc5tQ3n91k1kZ4oOf7D8W5lVnqJYtLm2xWova3ppaGNN4",
|
||||
"lDEQt9JsWsTJNp05R5sB4yYEKZlGCfpf6POJP21mYp7Ag+VBAkEQPUFfb6qdp1rJE+z3qrOPE12RY0J8",
|
||||
"8BOH2IpbiqW3y/wS4gZsMlfH28goQqRuHZvMIaNllC1klBLBZqxyMaxklBBr2IR/flXNAHqDJZ1X3lVK",
|
||||
"LNL40cLEGRm062KOjvmG9gBfFr2iKTAcf/pUAOLI+kpWwaBxEtF/QL89w7aINU3aPSLTdOSAOJbUXj7W",
|
||||
"eJs5fiQw3ktSdniJP18PAC+yW6fZi1YyrlMknimzKo/XYDq3HNiCaeV45gNNwLtpxhVRrSRy8AOKJWx/",
|
||||
"pTB5yYGLxmPMbqwaUFBIfvmoDXCtno5XUhm9GKZknxvOuE5DjaaE9AIWG/zOrTV01o+bmbXAdU8AM+Ez",
|
||||
"jtLQ190nC+yvMH+mGdCfBmnlq1DGwvUyKXfLNksk3qaBPOrxQVtp9G6kUV4DvJVFfx9ZpDD++iVREE2q",
|
||||
"5RB2gmjiBCgs6Ubld53zaHKOQn46tmJoO8RQx1zhKoCPMMB0Xp6npGJi1rIwc6VFWtAB7cUD7g0rx5Ae",
|
||||
"vA6bTYFjHCUGQHiHpoAMeS8NELesGm7kMNd68/ojNXlAw8kLiQcMeODT+1mGg0oozpRmi0CS91/vIaVK",
|
||||
"g7rziZJkezgZnjXZqZBJYeUsOI8mzY8B/hmb7VQ8WT12ACueY3Cm4+5+vKm7Hk9VPnixply1ayqJRGrz",
|
||||
"N3BErSVxkWhE8Txt/UwzEud7nRNbnVepjqIzUyzPblPhXc5cU54RJiicVBP47phlN+AubseEeZjZmzqG",
|
||||
"t/y4Mr/vBl7elXypj4Gq9rEBmbZq8kHHdfEgtteRreDgTQZLLGA5MG9CyzsFda2KWu2ZqdNARWseKJVp",
|
||||
"b+/1cFM1zNXFQlmroEdvHAtVPgHbWChbHXWpWCi7U/IAQ0L/i+vjpmUXR3apjoRSyAWFk6HoY+mM/U6O",
|
||||
"SQUxS5yR6p60rFRw3zWiaWV8lAUUVj+0ZfF92C5+sNUnM59jhg+cZ4VtxCcyeUhr65tXHrMgRNwsMrFO",
|
||||
"YVwgWLbVERkCJK0rauE6TRjzk7b8tSr+EoywYOhv9YFj4dWBWQhJwbWD9zYEye3KWfOen1Ef4IvVIypt",
|
||||
"V5jVKqMeIwOWpKqckNUMk1IFxQq2XFY0BlApx7IYiEkainRP0ApW2db6+VOfwviNnqTZfr7NgzSbegue",
|
||||
"o1U41MfoCmLJQi0f4IuoWxcDlJToJcug/gdlt6MT1vSIF6s75v86puJdtx5Nln4tM9TmSTYvQwYyW9G5",
|
||||
"SFZtYMnV5nZee4xz6wWwkpsBlD6elpHNtibkqkD99grAECCSIVeahTl/v40bgl0KDdXmy6Ny3r0X6PG/",
|
||||
"NzOrTFwr1FP47EHol4LUxAVFRkxZ83n9xeRglAYPZrefz2nwIMgD5zIBVwoF2ucdCwa6/IbCAb+ldMDN",
|
||||
"xUPrJb5l8oGxqSok8IqlhMfKjVS4B7Lv3JChVEIsqLgmqcHdSvgI71mhYAiwVyjEhSGBcQBeVi423qyc",
|
||||
"zHwW8BrRxJAG/ZzoWiG1rUJqwCh1PfKJmdEsbazcNmdhZ/0OX9pnvdzYuNBtnSG7vbHrbuyOsP2ukg/E",
|
||||
"aVCRH5d+x82O5oE8Yt7r0cwRsC1H82rMahy4Vqt/bwcmCh8RgU0drGUvvdNYn31tz0rpK6bgYyEvMYnt",
|
||||
"1jdM5z6d0+KafKb5BJW03pq/FS9pjhI752iO2zf1iObgLuIILQijZUu993PGN6tx1RR8Ln/Y4/9uVgrJ",
|
||||
"gpUbFz/aLn+aIl9Vw7aXoWPXz9Za7tVUdtoy7tVlIcz2xxS9XdzHJhWTLDhhx9MNbiEnrDf0drFz982C",
|
||||
"by05V1OMaZs5VwTFNubcqpNvBmcjUT22wR1N9tKz+A/2tb2jSWpU8LHQHU1iu1UGdXe0nBZXowuK8Q5+",
|
||||
"8j9sUlADAYQzTqJZXdgbp4a/hyoolm2CjX/efKLslfPuIjrg++DaLcpyd2FIapcxaWFjViYv/kphCvdm",
|
||||
"VHB7uLY6EWvtiNbZK3KlwPgKyX9prx9iil2UGTsVGbBLzt7r114KtLdYBFhW2l5ySSsT31gmUnGU7c4s",
|
||||
"EyxSIkrOWVQmJoDAPfbgZOMqQVvz56k6X4kBIPCcNmzj0rY1Lm1VMUy1mFxnpFJGZ1sQrTQPy6bSZxZ5",
|
||||
"rYEzjsLOrTfO3J1VxU0ubimqnXP+66ISV/TYi6MAeS/1KVtkB4d3sEnYIl0JrliPNl3LgQ4ti5l45naj",
|
||||
"NfVsPOsRr0JWmailUOEMVxbma42fPEeLipMmt4c5VLe1kraojJnCC4YymDUl/ywY8QATkBAjOw7pV36O",
|
||||
"XXZTMnXYZWWeIW8wTPibCQPokiKU9dxFzvxweFxTYoyhTBwrBaxMIfDFG08QcYIp0sr83K9zxbEo2UUP",
|
||||
"CNJBWfLjQrUshtLijJIQ6A4sTAd1ebPm6uhhXVm7Vg4LOXwxLJQDbiCJ57HcyuKtk8VlRrCqKFmbrsui",
|
||||
"tGrrncgQUOSvyixdq6PZ4qTWXoZtjdgtZmgj51lydOWJKupx7G3iyUqUCNu1l6v1mwt0iGlmM8jqVhV2",
|
||||
"pn1U2YZHlWxvyo8qS9onNNXTKlk3L5TmjF44Q2lLN+6IHa+zrRXcNlBncUH50EqErSuwqIqIlRRVtJIT",
|
||||
"tTk1uoTAWSySw7C2FjVfdy2ZRitBqhzYEGbu/UKEcCIItu+C8MaPeHWMsimGTiDtWBF7z5KU2PIwa96y",
|
||||
"8DZmA0jSUGxVTfAFCuOU+UPwx13dcl+3QlNpcwFUyBe24W8hUPI1VdoCeDPLovBfIRnyYVvR8nbaQbMs",
|
||||
"VwZLgxiuvVBs84VC7tJapIZ4i997ipKHqoCx3K3T6CjR+kjkLuocFbcMqRQhVbU2KDIyN3re0ZHb0Rrx",
|
||||
"t+1VTiH/xVOFiEFMLPTuX98K/MOxsaESOZqZ/UaJPuTWtpy7fc9vKuMtYqznUrnaPE9PSC68q31v87Ph",
|
||||
"3R+WOSbaSlRLXzVlCFAxdprjeNFHKolofr1sniFSrcmjSRSpFNJp00Uq6SIVvOAaM1Gh6tHbJY/UwW1d",
|
||||
"ZE6xIBUIpr2ebmVSyeIelYMMqy+oTQTOT/Wfda/jBU6oPYEFme7yY/kc6+tBUzG4w2qC2K5F45Xbx3Nz",
|
||||
"tHDRLl0fKdwp0tTi/HzAnjhqTdT8IYQztAr0fg1f99noLXO/PXPnuRGulNIQHMZlrNlFHLHtbg3aGzJo",
|
||||
"36q4D22yEuSb1FRlWJ3EwVMQwzXpEUM2ditvdkaZ4BvWahR/I40i84i3KJ1dqJodBNmrG9boGlWsz8Kx",
|
||||
"+AN5T6bbb2XAygE8B5g4/TOWtHIKnQDIHTQlPwGY9H1j9pMPx7rsJxvw3GtSZkOVPK1vzZa+2C8gS+yf",
|
||||
"8+1kIbZ6mWAt7TSad5mOyYdjkAbEPTnsFETFJhIzZXN/WmRyXv7dGb04bAL9pOKTOUp8E2pX+9izen1r",
|
||||
"lYnesjEty3Y6wBkB4k1Ljz1VGtO7r9epvpNwZNg6Awsf9fJTybsu4hm0r0c1SZc42Wzi5QYfeEkU1msk",
|
||||
"tJXzZzTKgSIJmkxq3SdOkyh812rKzmSNzDYW+XTaCSSZSrxfkxzYdHFbdfLiXcoMXJGrcvTijEU+zJWl",
|
||||
"zFT5DNunzRy9rC9zpnJsbjh3ZgEZS+iw7cGk0WNLJ8GaFFp6LB38pP/Zk7/aFYMoH1XWTwOUcHa8NES2",
|
||||
"ehNYBYxuvjiEZRUH7Sa2eTnnqyro0dTMml8kiLvXTtVz25LMtcsOPFvMWWs6OttjcxdM340O6xXIB7vz",
|
||||
"m9GArZ1bNb7Xv96398htvkfKwvi2l0jWfr03yK2+3lLgYpBQpBledOfA4o1vVRvfhuDTxGNrYRNvp5sy",
|
||||
"CxTQhgkgKYZWxY1k20WutEPWV1wubYB7QKFvBRVr2Bik7yj066HZeQsKQTPogDEFtORT+ASwDPFTl+Ae",
|
||||
"Hx4f7R3S/10fHp6w//1fA+5F9y6dQE+8PiBwj0Lh2tbqoxCP4DhK4DpB/sxmWCXMFVgeoxDh6eIwy/4b",
|
||||
"xfOqgF4pptdnESyb396tPXBed2yvNWvxIlyPIZA5DtokywWOAI0edEX2V7PnWvoH73K5x1YNb9Xwzavh",
|
||||
"rW7Z6pZvEhmAlyyPygRQm8a7/nxfQ6nS/JynoPppQI/HGqth1nIR++FQdm6tiNtsRVzfvSgjgJ1yl2iV",
|
||||
"qVaZ2hllKl9GLqpXYpu1qjufMXhmpd1w4fayhGmtDqvVSgwawHr1koOf2Z97pUwntV5JepAb6iw77puk",
|
||||
"wYExs68W1VvrrqTf3dZfad5fyYCnZg4JBtqo8VxaCQPudLWeneK+dR7H7VG8635N65UjdopBlszgNY+h",
|
||||
"qaznCZwQPpkjaewDaa55h91JP1x9e1WjYPXZCypB22ilUc02NKkMYtz8jaZ/bObkqWZNNsPfisXNlz/c",
|
||||
"upSTQtBVUfl6ghgVWVywI+vlsdQIhES21wdLqsQgDVspvEkpLHdA2YAm8teoN2ywVFNzdVSVwO/yptmK",
|
||||
"XyvxKxSSOp145SKX5zHf86I0JDUuOqyNzAolE/CDR4ACMAogk76KuNHfxr9CwvOk41M2486L3rrkXTue",
|
||||
"vK+wWQtevTmpcPJpreGGN/oCkhZL6Vdk/xTDBB94aZLAas7G/HbAGzq0W4l7bzBMvkJyKgZbI93RmRrS",
|
||||
"GYO4LQXz9qVgoJcmiLwwMe5F0QOC3ZTKrj/uqKiaC24rkpskd7b9GjKeIDJNRwceCIIR8B6M5HwazeIA",
|
||||
"Eshp+pLO72jPIzoRL4TxlQ19SXF5KoefI/APh8c17wmemNcvzzuFwBdV34KIb4a2ymAm1l/nkFnAnVxg",
|
||||
"cQ5L9GECErMoGNKviyGOdW2ONQbP+nHGoGuIsCiaBHA99MaG/pvTG0ffiuktR9zfjt5Q+IgItCkNKbVh",
|
||||
"3oEp3VbHNx3hmvXti7nWeIqrE1n5TwQIy40pLrDVF62PVZYddQ57OeVda26IBdo7AJ4HY2K2vHXZd5xZ",
|
||||
"2MQkJWpTN5/3cddjT+KD84nqSxdWUB9fuY7+Wi+AvH4/Q1Jp7+3pK4Esz2BFTTP6vRl98T7uuiqE0cFX",
|
||||
"QF985S191dRvp0hagL6CaIJCM1mdRxPsoNAB7Gzcr1AwztlA66EldgTT8TdUY9XqHh1Ekwn0HRS21+et",
|
||||
"uj4Xj3VKNbb35CCaRCmpYYYoJXbcEKVvb+sRNBptWcWhlkhrlFFGPbZkO4OzEUzwFMUNrkBKJ7trED9C",
|
||||
"fuTdRBjRWglcP2nz+5CKovZOtMidSMVgPUnGAOOnKKnwROBiUkhSR7avEqlXcsz16RinUxBOsom2Sdnw",
|
||||
"GGR+hqhWnO+QOOdkVaR0CyZK4IQKsqTq0sdb4EqNJPPTWRfbSDC2iWEk8tpnrp3Q0yUJ2eo8OADew1pe",
|
||||
"GIZ05C1+YKgRNQ1fHJ7gaBpFD3vCIeXgp/jBIrSLCh3Ruuywwn+3j9oSA5kdQrKJNuwPYhkGJeFrRczb",
|
||||
"i5j50CuVTI1eIKKFHXMcCDzb3LdkU1lhrZpjxBGKbXM0bC3frMaPikPP3agEaihmBmJCk+drloJSYCfb",
|
||||
"rpY9t4g92fWytEVNeTTjTfbHq0XRZI1xg1OYZYyjcDar8l3UhLXsjudiYx8yseLWsFJyTiwFflD9q9oX",
|
||||
"kWlolAqJN60wm1QSMm+1M7S8hlspQ0Dh3DCdFQIDqUTZ5uIhLHmNQ9Zymp7TBEMsw2xzp8m8k79VkovM",
|
||||
"E9kqqr7BvWgrPeWbJIjIAGwDdTYfqKO7DikUs6CffKdOw7LnhAYq13sIGFkwSKTlrbfmLTUaZRnGslH7",
|
||||
"7LmrmR64FQy2viLGHBm2MbNc6ypy2aaVQyuJMK8etvLAqCAux5w1aqJVpna6ScWU7BnjPcIE86yZxpOy",
|
||||
"QWb2beBnTXZEnttwBaVrFi9cowdskkRpzFJO5iDIjTKCwjp9hy9ubTqANQuJJdNAC9JrM0FvozaxUOrp",
|
||||
"RoJLpigxuhnI6PqmSUMWyhWylZLrWsMu+05/zKzbOKXUAf0O46oAEIhJxlMIO2NIvCn0TYmJc8G/5YqU",
|
||||
"IIMFE5C8WdoRBd5G+UbaLCNtlpE1ZBlpJJqFbMAWr1qFk9xKLP/GG++QCebvIJfXLOXEpi6pCrbybqtU",
|
||||
"wJwUF1UB533IRhAkMMl8yDparzKYPEp5kCaBe+K6r3ev/y8AAP//mzw37rn+AQA=",
|
||||
"94sN17bbdC6x48qUDTdfZi0orG6bCCHberYRc5tQ3v/CJkchIhGV5gc/Oce/HsRJNILmy6V8pXNA/hBM",
|
||||
"IofZdUUhWTWi1szw2dRXESaDNLxi89rbpkyHXia5NnzqVRCUiD7n9MTwu7/RU+EiIg5IyTRK0P9SKCKZ",
|
||||
"h4LHyYtKvfNmTgJQAH2H2+0dtj3OFyHP+/m26g+OApmxatcHP9l/LKz4zlCtjl2iHLXkub3RvjCmkXgY",
|
||||
"iFtpnS/iZJtUm6PNgHET5iTMJ/60mYl5nhiWbgsEQfQEff2LwDzVStHLfq9SsTjRFTkmxAc/cYituKVY",
|
||||
"4b3MLyFuwCZz5eKNjCJO7q1jkzlktIyyhYxSItiMVS6GlYwSYg2bSMVFsTbpVRc6r7wSl1ik8dvYm+kf",
|
||||
"HbMh4AG+LGoJUGA4/vSpAMTRKnSgOInoP6DfnmFbxJqmSyQi03TkgDiW1F4+1nibOX4kMN5LUnZ4iT9f",
|
||||
"DwCv5Vx3gRStZPiwyG9UZlUeFsSudnJgC6aV45kPNAHvphlXBE+TyMEPKJaw/ZXC5CUHLhqPMTOMaEBB",
|
||||
"IfnlozaOuno6XrBn9GKYkn1uOOM67YGaSuULGAbxOzcK0lk/bmbWAtc9AcyEzzhKQ19ntiiwv8L8mWZA",
|
||||
"fxqklY+PGQvXy6Tc+98skXibBvKoxwdtpdG7kUZ5qflWFv19ZJHC+OuXREE0qZZD2AmiiROgsKQblZ8P",
|
||||
"z6PJOQr56diKoe0QQx1zIbUAPsIA03l5OpyKiVnLwsyVDx+CDmgvntfBsHIM6cHrsNkUOMZRYgCEd2gK",
|
||||
"yJD30gBxy4ouRw6L4DCvP1JzVDScvJDfwoAHPr2fJdKohOJMabYIJHn/9R5SqjSoO58oSbaHk+H1nJ0K",
|
||||
"mRRWzoLzaNL8GOCfsdlOxWsiYAewGk0Gn03uVcqbuutxiOaDF0sXVntAk0hk0H8Df+daEhf5bBQH59ad",
|
||||
"OSNxvtc5sdU5L+soOjPF8iRKFUEMzAPqGWGCwkk1ge+OWXYDUQl2TJhHM75p/EHLjysLL2gQTFDJl/pQ",
|
||||
"u2pXLpBpq6ZQB1wXdmR7HdlSx471xeQsYDkwb0LLOwV1rYpa7Zmp00BFax6Pl2lv7/VwUzXM1YXcWaug",
|
||||
"R28cclc+AduQO1sddamQO7tT8gBDQv+L68PzZRdHdqkOuFPIBYWToehj6fP/To5JBTFLnJHqnrSsVPAS",
|
||||
"N6JpZXyUxa1WP7RlYaTYLky11Scz13aGD5wnH27EJ9J/u7X1zSuPWawrbhYAW6cwLhCT3eqIDAGS1hW1",
|
||||
"cJ0mjPlJW/5aFX8JRlgwwrz6wLHw6sAsUqng2sF7G2Ixd+Wsec/PqA/wxeoRlbYrzGqVuJGRAcuFVs77",
|
||||
"a4ZJKbZjBVsuKxoDqFT9WQzEJA1F1Ba0glW2tX7+1GfKfqMnabafb/MgzabegudoFQ71MbqCWLKI3gf4",
|
||||
"IsojxgAlJXrJEvX/Qdnt6IQ1PeI1EY/5v46peNetR1MMQssMtem4zcuQ8fJWdC5yohtYcrUpxNceSt96",
|
||||
"AazkZgClj6dlAL2tCbkqH0R7BWAIEDm3K83CnL/fxg3BLlOLavPlUTnv3gv0+N+bmVXmRxbqKXz2IPRL",
|
||||
"QWrigiIjpqz5vP5icjBKgwez28/nNHgQ5IFzmYArhQLt844FA11+Q+GA31I64ObiofUS3zL5wNhUFRJ4",
|
||||
"xVLCY1VtKtwD2XduyFAKbhZUXJPU4G4lfIT3rFAwBNgrFOLCkMA4AC8rFxtvVrVoPtl8jWhiSIN+TnSt",
|
||||
"kNpWITVglLoe+cTMaJY2Vm6bs7Czfocv7bNebmxc6LbOkN3e2HU3dkfYflfJB+I0qEjDTL/jZkfzQB4x",
|
||||
"7/Vo5gjYlqN5NWY1Dlyr1b+3AxOFj4jApg7WspfeaazPvrZnpfQVU/CxkJeYxHbrG6Zzn85pcU0+03yC",
|
||||
"Slpvzd+KlzRHiZ1zNMftm3pEc3AXcYQWhNGypd77OeOb1bhqCj6XP+zxfzeruGXByo1rbG2XP02Rr6ph",
|
||||
"28vQsetnay33agqIbRn36rIQZvtjit4u7mOTwlwWnLDj6Qa3kBPWG3q72Ln7ZsG3lpyrqfm1zZwrgmIb",
|
||||
"c27VyTeDs5EoUtzgjiZ76Vn8B/va3tEkNSr4WOiOJrHdKoO6O1pOi6vRBcV4Bz/5HzYpqIEAwhkn0awu",
|
||||
"7I1Tw99DFRTLNsHGP28+UfbKeXcRHfB9cO0WZbm7MCS1y5i0sDErkxd/pTCFezMquD1cWwSLtXZE6+wV",
|
||||
"uVJgfIXkv7TXDzHFLsqMnYoM2CVn7/VrLwXaWywCzBFF8CXdtzLxrWUiFUfZ7swywSIlouScRWViAgjc",
|
||||
"Yw9ONq4StDV/nqrzlRgAAs9pwzYubVvj0lYVw1SLyXVGKmV0tgXRSvOwbCp9ZpHXGjjjKOzceuPM3VlV",
|
||||
"3OTilqLaOee/LipxRY+9OAqQ91KfskV2cHgHm4Qt0pXgivVo07Uc6NCymIlnbjdaU8/Gsx7xKmSViVoK",
|
||||
"Fc5wZWG+1vjJc7SoOGlye5hDdVsraYvKmCm8YKi2WlPyz4IRDzABCTGy45B+5efYZTclU4ddVuYZ8gbD",
|
||||
"hL+ZMIAuKUJZz13kzA+HxzUlxhjKxLFSwMoUAl+88QQRJ5girczP/TpXHIuSXfSAIB2UJT8uVMtiKC3O",
|
||||
"KAmB7sDCdFCXN2uujh7WlbVr5bCQwxfDQtXpBpJ4HsutLN46WVxmBKuKkrXpuixKq7beiQwBRf6qzNK1",
|
||||
"OpotTmrtZdjWiN1ihjZyniVHV56ooh7H3iaerESJsF17uVq/uUCHmGY2g6xuVWFn2keVbXhUyfam/Kiy",
|
||||
"pH1CUz2tknXzQmnO6IUzlLZ0447Y8TrbWsFtA3UWF5QPrUTYugKLqohYSVFFKzlRm1OjSwicxSI5DGtr",
|
||||
"UfN115JptBKkyoENYebeL0QIJ4Jg+y4Ib/yIV8com2LoBNKOFbH3LEmJLQ+z5i0Lb2M2gCQNxVbVBF+g",
|
||||
"ME6ZPwR/3NUt93UrNJU2F0CFfGEb/hYCJV9TpS2AN7MsCv8VkiEfthUtb6cdNMtyZbA0iOHaC8U2Xyjk",
|
||||
"Lq1Faoi3+L2nKHmoChjL3TqNjhKtj0Tuos5RccuQShFSVWuDIiNzo+cdHbkdrRF/217lFPJfPFWIGMTE",
|
||||
"Qu/+9a3APxwbGyqRo5nZb5ToQ25ty7nb9/ymMt4ixnoulavN8/SE5MK72vc2Pxve/WGZY6KtRLX0VVOG",
|
||||
"ABVjpzmOF32kkojm18vmGSLVmjyaRJFKIZ02XaSSLlLBC64xExWqHr1d8kgd3NZF5hQLUoFg2uvpViaV",
|
||||
"LO5ROciw+oLaROD8VP9Z9zpe4ITaE1iQ6S4/ls+xvh40FYM7rCaI7Vo0Xrl9PDdHCxft0vWRwp0iTS3O",
|
||||
"zwfsiaPWRM0fQjhDq0Dv1/B1n43eMvfbM3eeG+FKKQ3BYVzGml3EEdvu1qC9IYP2rYr70CYrQb5JTVWG",
|
||||
"1UkcPAUxXJMeMWRjt/JmZ5QJvmGtRvE30igyj3iL0tmFqtlBkL26YY2uUcX6LByLP5D3ZLr9VgasHMBz",
|
||||
"gInTP2NJK6fQCYDcQVPyE4BJ3zdmP/lwrMt+sgHPvSZlNlTJ0/rWbOmL/QKyxP45304WYquXCdbSTqN5",
|
||||
"l+mYfDgGaUDck8NOQVRsIjFTNvenRSbn5d+d0YvDJtBPKj6Zo8Q3oXa1jz2r17dWmegtG9OybKcDnBEg",
|
||||
"3rT02FOlMb37ep3qOwlHhq0zsPBRLz+VvOsinkH7elSTdImTzSZebvCBl0RhvUZCWzl/RqMcKJKgyaTW",
|
||||
"feI0icJ3rabsTNbIbGORT6edQJKpxPs1yYFNF7dVJy/epczAFbkqRy/OWOTDXFnKTJXPsH3azNHL+jJn",
|
||||
"KsfmhnNnFpCxhA7bHkwaPbZ0EqxJoaXH0sFP+p89+atdMYjyUWX9NEAJZ8dLQ2SrN4FVwOjmi0NYVnHQ",
|
||||
"bmKbl3O+qoIeTc2s+UWCuHvtVD23Lclcu+zAs8Wctaajsz02d8H03eiwXoF8sDu/GQ3Y2rlV43v96317",
|
||||
"j9zme6QsjG97iWTt13uD3OrrLQUuBglFmuFFdw4s3vhWtfFtCD5NPLYWNvF2uimzQAFtmACSYmhV3Ei2",
|
||||
"XeRKO2R9xeXSBrgHFPpWULGGjUH6jkK/Hpqdt6AQNIMOGFNASz6FTwDLED91Ce7x4fHR3iH93/Xh4Qn7",
|
||||
"3/814F5079IJ9MTrAwL3KBSuba0+CvEIjqMErhPkz2yGVcJcgeUxChGeLg6z7L9RPK8K6JVien0WwbL5",
|
||||
"7d3aA+d1x/ZasxYvwvUYApnjoE2yXOAI0OhBV2R/NXuupX/wLpd7bNXwVg3fvBre6patbvkmkQF4yfKo",
|
||||
"TAC1abzrz/c1lCrNz3kKqp8G9HissRpmLRexHw5l59aKuM1WxPXdizIC2Cl3iVaZapWpnVGm8mXkonol",
|
||||
"tlmruvMZg2dW2g0Xbi9LmNbqsFqtxKABrFcvOfiZ/blXynRS65WkB7mhzrLjvkkaHBgz+2pRvbXuSvrd",
|
||||
"bf2V5v2VDHhq5pBgoI0az6WVMOBOV+vZKe5b53HcHsW77te0XjlipxhkyQxe8xiaynqewAnhkzmSxj6Q",
|
||||
"5pp32J30w9W3VzUKVp+9oBK0jVYa1WxDk8ogxs3faPrHZk6eatZkM/ytWNx8+cOtSzkpBF0Vla8niFGR",
|
||||
"xQU7sl4eS41ASGR7fbCkSgzSsJXCm5TCcgeUDWgif416wwZLNTVXR1UJ/C5vmq34tRK/QiGp04lXLnJ5",
|
||||
"HvM9L0pDUuOiw9rIrFAyAT94BCgAowAy6auIG/1t/CskPE86PmUz7rzorUvetePJ+wqbteDVm5MKJ5/W",
|
||||
"Gm54oy8gabGUfkX2TzFM8IGXJgms5mzMbwe8oUO7lbj3BsPkKySnYrA10h2dqSGdMYjbUjBvXwoGemmC",
|
||||
"yAsT414UPSDYTans+uOOiqq54LYiuUlyZ9uvIeMJItN0dOCBIBgB78FIzqfRLA4ggZymL+n8jvY8ohPx",
|
||||
"Qhhf2dCXFJencvg5Av9weFzznuCJef3yvFMIfFH1LYj4ZmirDGZi/XUOmQXcyQUW57BEHyYgMYuCIf26",
|
||||
"GOJY1+ZYY/CsH2cMuoYIi6JJANdDb2zovzm9cfStmN5yxP3t6A2Fj4hAm9KQUhvmHZjSbXV80xGuWd++",
|
||||
"mGuNp7g6kZX/RICw3JjiAlt90fpYZdlR57CXU9615oZYoL0D4HkwJmbLW5d9x5mFTUxSojZ183kfdz32",
|
||||
"JD44n6i+dGEF9fGV6+iv9QLI6/czJJX23p6+EsjyDFbUNKPfm9EX7+Ouq0IYHXwF9MVX3tJXTf12iqQF",
|
||||
"6CuIJig0k9V5NMEOCh3Azsb9CgXjnA20HlpiRzAdf0M1Vq3u0UE0mUDfQWF7fd6q63PxWKdUY3tPDqJJ",
|
||||
"lJIaZohSYscNUfr2th5Bo9GWVRxqibRGGWXUY0u2MzgbwQRPUdzgCqR0srsG8SPkR95NhBGtlcD1kza/",
|
||||
"D6koau9Ei9yJVAzWk2QMMH6KkgpPBC4mhSR1ZPsqkXolx1yfjnE6BeEkm2iblA2PQeZniGrF+Q6Jc05W",
|
||||
"RUq3YKIETqggS6oufbwFrtRIMj+ddbGNBGObGEYir33m2gk9XZKQrc6DA+A9rOWFYUhH3uIHhhpR0/DF",
|
||||
"4QmOplH0sCccUg5+ih8sQruo0BGtyw4r/Hf7qC0xkNkhJJtow/4glmFQEr5WxLy9iJkPvVLJ1OgFIlrY",
|
||||
"MceBwLPNfUs2lRXWqjlGHKHYNkfD1vLNavyoOPTcjUqghmJmICY0eb5mKSgFdrLtatlzi9iTXS9LW9SU",
|
||||
"RzPeZH+8WhRN1hg3OIVZxjgKZ7Mq30VNWMvueC429iETK24NKyXnxFLgB9W/qn0RmYZGqZB40wqzSSUh",
|
||||
"81Y7Q8truJUyBBTODdNZITCQSpRtLh7Cktc4ZC2n6TlNMMQyzDZ3msw7+Vslucg8ka2i6hvci7bSU75J",
|
||||
"gogMwDZQZ/OBOrrrkEIxC/rJd+o0LHtOaKByvYeAkQWDRFreemveUqNRlmEsG7XPnrua6YFbwWDrK2LM",
|
||||
"kWEbM8u1riKXbVo5tJII8+phKw+MCuJyzFmjJlplaqebVEzJnjHeI0wwz5ppPCkbZGbfBn7WZEfkuQ1X",
|
||||
"ULpm8cI1esAmSZTGLOVkDoLcKCMorNN3+OLWpgNYs5BYMg20IL02E/Q2ahMLpZ5uJLhkihKjm4GMrm+a",
|
||||
"NGShXCFbKbmuNeyy7/THzLqNU0od0O8wrgoAgZhkPIWwM4bEm0LflJg4F/xbrkgJMlgwAcmbpR1R4G2U",
|
||||
"b6TNMtJmGVlDlpFGolnIBmzxqlU4ya3E8m+88Q6ZYP4OcnnNUk5s6pKqYCvvtkoFzElxURVw3odsBEEC",
|
||||
"k8yHrKP1KoPJo5QHaRK4J677evf6/wIAAP//MiYyXyABAgA=",
|
||||
}
|
||||
|
||||
// GetSwagger returns the content of the embedded swagger specification file
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/hatchet-dev/hatchet/api/v1/server/handlers/ingestors"
|
||||
"github.com/hatchet-dev/hatchet/api/v1/server/handlers/logs"
|
||||
"github.com/hatchet-dev/hatchet/api/v1/server/handlers/metadata"
|
||||
"github.com/hatchet-dev/hatchet/api/v1/server/handlers/monitoring"
|
||||
rate_limits "github.com/hatchet-dev/hatchet/api/v1/server/handlers/rate-limits"
|
||||
slackapp "github.com/hatchet-dev/hatchet/api/v1/server/handlers/slack-app"
|
||||
stepruns "github.com/hatchet-dev/hatchet/api/v1/server/handlers/step-runs"
|
||||
@@ -49,6 +50,7 @@ type apiService struct {
|
||||
*slackapp.SlackAppService
|
||||
*webhookworker.WebhookWorkersService
|
||||
*workflowruns.WorkflowRunsService
|
||||
*monitoring.MonitoringService
|
||||
}
|
||||
|
||||
func newAPIService(config *server.ServerConfig) *apiService {
|
||||
@@ -67,6 +69,7 @@ func newAPIService(config *server.ServerConfig) *apiService {
|
||||
IngestorsService: ingestors.NewIngestorsService(config),
|
||||
SlackAppService: slackapp.NewSlackAppService(config),
|
||||
WebhookWorkersService: webhookworker.NewWebhookWorkersService(config),
|
||||
MonitoringService: monitoring.NewMonitoringService(config),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2115,4 +2115,19 @@ export class Api<SecurityDataType = unknown> extends HttpClient<SecurityDataType
|
||||
format: 'json',
|
||||
...params,
|
||||
});
|
||||
/**
|
||||
* @description Triggers a workflow to check the status of the instance
|
||||
*
|
||||
* @name MonitoringPostRunProbe
|
||||
* @summary Detailed Health Probe For the Instance
|
||||
* @request POST:/api/v1/monitoring/{tenant}/probe
|
||||
* @secure
|
||||
*/
|
||||
monitoringPostRunProbe = (tenant: string, params: RequestParams = {}) =>
|
||||
this.request<void, APIErrors>({
|
||||
path: `/api/v1/monitoring/${tenant}/probe`,
|
||||
method: 'POST',
|
||||
secure: true,
|
||||
...params,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -297,7 +297,7 @@ func (wc *WorkflowsControllerImpl) handleWorkflowRunFinished(ctx context.Context
|
||||
workflowRun, err := wc.repo.WorkflowRun().GetWorkflowRunById(ctx, metadata.TenantId, payload.WorkflowRunId)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not get job run: %w", err)
|
||||
return fmt.Errorf("handleWorkflowRunFinished - could not get job run: %w", err)
|
||||
}
|
||||
|
||||
workflowRunId := sqlchelpers.UUIDToStr(workflowRun.WorkflowRun.ID)
|
||||
|
||||
@@ -1912,6 +1912,9 @@ type ClientInterface interface {
|
||||
// MetadataListIntegrations request
|
||||
MetadataListIntegrations(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error)
|
||||
|
||||
// MonitoringPostRunProbe request
|
||||
MonitoringPostRunProbe(ctx context.Context, tenant openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error)
|
||||
|
||||
// SlackWebhookDelete request
|
||||
SlackWebhookDelete(ctx context.Context, slack openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error)
|
||||
|
||||
@@ -2341,6 +2344,18 @@ func (c *Client) MetadataListIntegrations(ctx context.Context, reqEditors ...Req
|
||||
return c.Client.Do(req)
|
||||
}
|
||||
|
||||
func (c *Client) MonitoringPostRunProbe(ctx context.Context, tenant openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) {
|
||||
req, err := NewMonitoringPostRunProbeRequest(c.Server, tenant)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req = req.WithContext(ctx)
|
||||
if err := c.applyEditors(ctx, req, reqEditors); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c.Client.Do(req)
|
||||
}
|
||||
|
||||
func (c *Client) SlackWebhookDelete(ctx context.Context, slack openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) {
|
||||
req, err := NewSlackWebhookDeleteRequest(c.Server, slack)
|
||||
if err != nil {
|
||||
@@ -3943,6 +3958,40 @@ func NewMetadataListIntegrationsRequest(server string) (*http.Request, error) {
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// NewMonitoringPostRunProbeRequest generates requests for MonitoringPostRunProbe
|
||||
func NewMonitoringPostRunProbeRequest(server string, tenant openapi_types.UUID) (*http.Request, error) {
|
||||
var err error
|
||||
|
||||
var pathParam0 string
|
||||
|
||||
pathParam0, err = runtime.StyleParamWithLocation("simple", false, "tenant", runtime.ParamLocationPath, tenant)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
serverURL, err := url.Parse(server)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
operationPath := fmt.Sprintf("/api/v1/monitoring/%s/probe", pathParam0)
|
||||
if operationPath[0] == '/' {
|
||||
operationPath = "." + operationPath
|
||||
}
|
||||
|
||||
queryURL, err := serverURL.Parse(operationPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", queryURL.String(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// NewSlackWebhookDeleteRequest generates requests for SlackWebhookDelete
|
||||
func NewSlackWebhookDeleteRequest(server string, slack openapi_types.UUID) (*http.Request, error) {
|
||||
var err error
|
||||
@@ -8401,6 +8450,9 @@ type ClientWithResponsesInterface interface {
|
||||
// MetadataListIntegrationsWithResponse request
|
||||
MetadataListIntegrationsWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*MetadataListIntegrationsResponse, error)
|
||||
|
||||
// MonitoringPostRunProbeWithResponse request
|
||||
MonitoringPostRunProbeWithResponse(ctx context.Context, tenant openapi_types.UUID, reqEditors ...RequestEditorFn) (*MonitoringPostRunProbeResponse, error)
|
||||
|
||||
// SlackWebhookDeleteWithResponse request
|
||||
SlackWebhookDeleteWithResponse(ctx context.Context, slack openapi_types.UUID, reqEditors ...RequestEditorFn) (*SlackWebhookDeleteResponse, error)
|
||||
|
||||
@@ -8927,6 +8979,28 @@ func (r MetadataListIntegrationsResponse) StatusCode() int {
|
||||
return 0
|
||||
}
|
||||
|
||||
type MonitoringPostRunProbeResponse struct {
|
||||
Body []byte
|
||||
HTTPResponse *http.Response
|
||||
JSON403 *APIErrors
|
||||
}
|
||||
|
||||
// Status returns HTTPResponse.Status
|
||||
func (r MonitoringPostRunProbeResponse) Status() string {
|
||||
if r.HTTPResponse != nil {
|
||||
return r.HTTPResponse.Status
|
||||
}
|
||||
return http.StatusText(0)
|
||||
}
|
||||
|
||||
// StatusCode returns HTTPResponse.StatusCode
|
||||
func (r MonitoringPostRunProbeResponse) StatusCode() int {
|
||||
if r.HTTPResponse != nil {
|
||||
return r.HTTPResponse.StatusCode
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type SlackWebhookDeleteResponse struct {
|
||||
Body []byte
|
||||
HTTPResponse *http.Response
|
||||
@@ -11009,6 +11083,15 @@ func (c *ClientWithResponses) MetadataListIntegrationsWithResponse(ctx context.C
|
||||
return ParseMetadataListIntegrationsResponse(rsp)
|
||||
}
|
||||
|
||||
// MonitoringPostRunProbeWithResponse request returning *MonitoringPostRunProbeResponse
|
||||
func (c *ClientWithResponses) MonitoringPostRunProbeWithResponse(ctx context.Context, tenant openapi_types.UUID, reqEditors ...RequestEditorFn) (*MonitoringPostRunProbeResponse, error) {
|
||||
rsp, err := c.MonitoringPostRunProbe(ctx, tenant, reqEditors...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ParseMonitoringPostRunProbeResponse(rsp)
|
||||
}
|
||||
|
||||
// SlackWebhookDeleteWithResponse request returning *SlackWebhookDeleteResponse
|
||||
func (c *ClientWithResponses) SlackWebhookDeleteWithResponse(ctx context.Context, slack openapi_types.UUID, reqEditors ...RequestEditorFn) (*SlackWebhookDeleteResponse, error) {
|
||||
rsp, err := c.SlackWebhookDelete(ctx, slack, reqEditors...)
|
||||
@@ -12264,6 +12347,32 @@ func ParseMetadataListIntegrationsResponse(rsp *http.Response) (*MetadataListInt
|
||||
return response, nil
|
||||
}
|
||||
|
||||
// ParseMonitoringPostRunProbeResponse parses an HTTP response from a MonitoringPostRunProbeWithResponse call
|
||||
func ParseMonitoringPostRunProbeResponse(rsp *http.Response) (*MonitoringPostRunProbeResponse, error) {
|
||||
bodyBytes, err := io.ReadAll(rsp.Body)
|
||||
defer func() { _ = rsp.Body.Close() }()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
response := &MonitoringPostRunProbeResponse{
|
||||
Body: bodyBytes,
|
||||
HTTPResponse: rsp,
|
||||
}
|
||||
|
||||
switch {
|
||||
case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 403:
|
||||
var dest APIErrors
|
||||
if err := json.Unmarshal(bodyBytes, &dest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
response.JSON403 = &dest
|
||||
|
||||
}
|
||||
|
||||
return response, nil
|
||||
}
|
||||
|
||||
// ParseSlackWebhookDeleteResponse parses an HTTP response from a SlackWebhookDeleteWithResponse call
|
||||
func ParseSlackWebhookDeleteResponse(rsp *http.Response) (*SlackWebhookDeleteResponse, error) {
|
||||
bodyBytes, err := io.ReadAll(rsp.Body)
|
||||
|
||||
@@ -60,7 +60,6 @@ func LoadServerConfigFile(files ...[]byte) (*server.ServerConfigFile, error) {
|
||||
f := server.BindAllEnv
|
||||
|
||||
_, err := loaderutils.LoadConfigFromViper(f, configFile, files...)
|
||||
|
||||
return configFile, err
|
||||
}
|
||||
|
||||
@@ -473,6 +472,10 @@ func GetServerConfigFromConfigfile(dc *database.Config, cf *server.ServerConfigF
|
||||
services = strings.Split(cf.ServicesString, " ")
|
||||
}
|
||||
|
||||
if cf.Runtime.Monitoring.TLSRootCAFile == "" {
|
||||
cf.Runtime.Monitoring.TLSRootCAFile = cf.TLS.TLSRootCAFile
|
||||
}
|
||||
|
||||
return cleanup, &server.ServerConfig{
|
||||
Alerter: alerter,
|
||||
Analytics: analyticsEmitter,
|
||||
|
||||
@@ -61,6 +61,8 @@ type ServerConfigFile struct {
|
||||
TenantAlerting ConfigFileTenantAlerting `mapstructure:"tenantAlerting" json:"tenantAlerting,omitempty"`
|
||||
|
||||
Email ConfigFileEmail `mapstructure:"email" json:"email,omitempty"`
|
||||
|
||||
Monitoring ConfigFileMonitoring `mapstructure:"monitoring" json:"monitoring,omitempty"`
|
||||
}
|
||||
|
||||
type ConfigFileAdditionalLoggers struct {
|
||||
@@ -143,7 +145,7 @@ type ConfigFileRuntime struct {
|
||||
MaxInternalRetryCount int32 `mapstructure:"maxInternalRetryCount" json:"maxInternalRetryCount,omitempty" default:"3"`
|
||||
|
||||
// WaitForFlush is the time to wait for the buffer to flush used for exerting some back pressure on writers
|
||||
WaitForFlush time.Duration `mapstructure:"waitForFlush" json:"waitForFlush,omitempty" default:"1ms"`
|
||||
WaitForFlush time.Duration `mapstructure:"waitForFlush" json:"waitForFlush,omitempty" default:"1"`
|
||||
|
||||
// MaxConcurrent is the maximum number of concurrent flushes
|
||||
MaxConcurrent int `mapstructure:"maxConcurrent" json:"maxConcurrent,omitempty" default:"50"`
|
||||
@@ -168,6 +170,8 @@ type ConfigFileRuntime struct {
|
||||
|
||||
// QueueStepRunBuffer represents the buffer settings for inserting step runs into the queue
|
||||
QueueStepRunBuffer buffer.ConfigFileBuffer `mapstructure:"queueStepRunBuffer" json:"queueStepRunBuffer,omitempty"`
|
||||
|
||||
Monitoring ConfigFileMonitoring `mapstructure:"monitoring" json:"monitoring,omitempty"`
|
||||
}
|
||||
|
||||
type SecurityCheckConfigFile struct {
|
||||
@@ -348,6 +352,20 @@ type ConfigFileEmail struct {
|
||||
Postmark PostmarkConfigFile `mapstructure:"postmark" json:"postmark,omitempty"`
|
||||
}
|
||||
|
||||
type ConfigFileMonitoring struct {
|
||||
// Enabled controls whether the monitoring service is enabled for this Hatchet instance.
|
||||
Enabled bool `mapstructure:"enabled" json:"enabled,omitempty" default:"true"`
|
||||
|
||||
// PermittedTenants is a list of tenant IDs that are allowed to use the monitoring service.
|
||||
PermittedTenants []string `mapstructure:"permittedTenants" json:"permittedTenants"`
|
||||
|
||||
// ProbeTimeout is the time to wait for the probe to complete
|
||||
ProbeTimeout time.Duration `mapstructure:"probeTimeout" json:"probeTimeout,omitempty" default:"30s"`
|
||||
|
||||
// TLSRootCAFile is the path to the root CA file for the monitoring service
|
||||
TLSRootCAFile string `mapstructure:"tlsRootCAFile" json:"tlsRootCAFile,omitempty"`
|
||||
}
|
||||
|
||||
type PostmarkConfigFile struct {
|
||||
Enabled bool `mapstructure:"enabled" json:"enabled,omitempty"`
|
||||
|
||||
@@ -619,4 +637,11 @@ func BindAllEnv(v *viper.Viper) {
|
||||
_ = v.BindEnv("email.postmark.fromName", "SERVER_EMAIL_POSTMARK_FROM_NAME")
|
||||
_ = v.BindEnv("email.postmark.supportEmail", "SERVER_EMAIL_POSTMARK_SUPPORT_EMAIL")
|
||||
|
||||
// monitoring options
|
||||
_ = v.BindEnv("runtime.monitoring.enabled", "SERVER_MONITORING_ENABLED")
|
||||
_ = v.BindEnv("runtime.monitoring.permittedTenants", "SERVER_MONITORING_PERMITTED_TENANTS")
|
||||
_ = v.BindEnv("runtime.monitoring.probeTimeout", "SERVER_MONITORING_PROBE_TIMEOUT")
|
||||
// we will fill this in from the server config if it is not set
|
||||
_ = v.BindEnv("runtime.monitoring.tlsRootCAFile", "SERVER_MONITORING_TLS_ROOT_CA_FILE")
|
||||
|
||||
}
|
||||
|
||||
@@ -311,7 +311,7 @@ WHERE
|
||||
|
||||
-- name: ListStepRuns :many
|
||||
SELECT
|
||||
DISTINCT ON ("StepRun"."id")
|
||||
DISTINCT ON ("StepRun"."order","StepRun"."id")
|
||||
"StepRun"."id"
|
||||
FROM
|
||||
"StepRun"
|
||||
@@ -339,7 +339,9 @@ WHERE
|
||||
AND (
|
||||
sqlc.narg('tickerId')::uuid IS NULL OR
|
||||
"StepRun"."tickerId" = sqlc.narg('tickerId')::uuid
|
||||
);
|
||||
)
|
||||
|
||||
ORDER BY "StepRun"."order" ASC ;
|
||||
|
||||
-- name: ListStepRunsToCancel :many
|
||||
SELECT
|
||||
|
||||
@@ -1967,7 +1967,7 @@ func (q *Queries) ListStepRunExpressionEvals(ctx context.Context, db DBTX, stepr
|
||||
|
||||
const listStepRuns = `-- name: ListStepRuns :many
|
||||
SELECT
|
||||
DISTINCT ON ("StepRun"."id")
|
||||
DISTINCT ON ("StepRun"."order","StepRun"."id")
|
||||
"StepRun"."id"
|
||||
FROM
|
||||
"StepRun"
|
||||
@@ -1996,6 +1996,8 @@ WHERE
|
||||
$5::uuid IS NULL OR
|
||||
"StepRun"."tickerId" = $5::uuid
|
||||
)
|
||||
|
||||
ORDER BY "StepRun"."order" ASC
|
||||
`
|
||||
|
||||
type ListStepRunsParams struct {
|
||||
|
||||
@@ -231,8 +231,8 @@ func (r *workflowAPIRepository) GetWorkflowVersionById(tenantId, workflowVersion
|
||||
return row, crons, events, scheduled, nil
|
||||
}
|
||||
|
||||
func (r *workflowAPIRepository) DeleteWorkflow(tenantId, workflowId string) (*dbsqlc.Workflow, error) {
|
||||
return r.queries.SoftDeleteWorkflow(context.Background(), r.pool, sqlchelpers.UUIDFromStr(workflowId))
|
||||
func (r *workflowAPIRepository) DeleteWorkflow(ctx context.Context, tenantId, workflowId string) (*dbsqlc.Workflow, error) {
|
||||
return r.queries.SoftDeleteWorkflow(ctx, r.pool, sqlchelpers.UUIDFromStr(workflowId))
|
||||
}
|
||||
|
||||
func (r *workflowAPIRepository) GetWorkflowMetrics(tenantId, workflowId string, opts *repository.GetWorkflowMetricsOpts) (*repository.WorkflowMetrics, error) {
|
||||
|
||||
@@ -286,7 +286,7 @@ type WorkflowAPIRepository interface {
|
||||
error)
|
||||
|
||||
// DeleteWorkflow deletes a workflow for a given tenant.
|
||||
DeleteWorkflow(tenantId, workflowId string) (*dbsqlc.Workflow, error)
|
||||
DeleteWorkflow(ctx context.Context, tenantId, workflowId string) (*dbsqlc.Workflow, error)
|
||||
|
||||
// GetWorkflowVersionMetrics returns the metrics for a given workflow version.
|
||||
GetWorkflowMetrics(tenantId, workflowId string, opts *GetWorkflowMetricsOpts) (*WorkflowMetrics, error)
|
||||
|
||||
Reference in New Issue
Block a user