mirror/hatchet
1
0
Fork 0
mirror of https://github.com/hatchet-dev/hatchet.git synced 2025-12-30 05:09:44 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix: make stripped payload size configurable (#1685)

Browse Source
This commit is contained in:
abelanger5
2025-05-07 09:13:07 -04:00
committed by GitHub
parent db246df476
commit 8f9ae4ecf2
4 changed files with 47 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/hatchet-engine/engine/run.go
View File

@@ -431,6 +431,7 @@ func runV0Config(ctx context.Context, sc *server.ServerConfig) ([]Teardown, erro
dispatcher.WithLogger(sc.Logger),
dispatcher.WithEntitlementsRepository(sc.EntitlementRepository),
dispatcher.WithCache(cacheInstance),
dispatcher.WithPayloadSizeThreshold(sc.Runtime.GRPCMaxMsgSize),
)
if err != nil {
@@ -879,6 +880,7 @@ func runV1Config(ctx context.Context, sc *server.ServerConfig) ([]Teardown, erro
dispatcher.WithLogger(sc.Logger),
dispatcher.WithEntitlementsRepository(sc.EntitlementRepository),
dispatcher.WithCache(cacheInstance),
dispatcher.WithPayloadSizeThreshold(sc.Runtime.GRPCMaxMsgSize),
)
if err != nil {

66
internal/services/dispatcher/dispatcher.go
View File

@@ -54,6 +54,7 @@ type DispatcherImpl struct {
repo repository.EngineRepository
repov1 v1.Repository
cache cache.Cacheable
payloadSizeThreshold int
entitlements repository.EntitlementsRepository
@@ -126,16 +127,17 @@ func (w *workers) Delete(workerId string) {
type DispatcherOpt func(*DispatcherOpts)
type DispatcherOpts struct {
mq msgqueue.MessageQueue
mqv1 msgqueuev1.MessageQueue
l *zerolog.Logger
dv datautils.DataDecoderValidator
repo repository.EngineRepository
repov1 v1.Repository
entitlements repository.EntitlementsRepository
dispatcherId string
alerter hatcheterrors.Alerter
cache cache.Cacheable
mq msgqueue.MessageQueue
mqv1 msgqueuev1.MessageQueue
l *zerolog.Logger
dv datautils.DataDecoderValidator
repo repository.EngineRepository
repov1 v1.Repository
entitlements repository.EntitlementsRepository
dispatcherId string
alerter hatcheterrors.Alerter
cache cache.Cacheable
payloadSizeThreshold int
}
func defaultDispatcherOpts() *DispatcherOpts {
@@ -143,10 +145,11 @@ func defaultDispatcherOpts() *DispatcherOpts {
alerter := hatcheterrors.NoOpAlerter{}
return &DispatcherOpts{
l: &logger,
dv: datautils.NewDataDecoderValidator(),
dispatcherId: uuid.New().String(),
alerter: alerter,
l: &logger,
dv: datautils.NewDataDecoderValidator(),
dispatcherId: uuid.New().String(),
alerter: alerter,
payloadSizeThreshold: 3 * 1024 * 1024,
}
}
@@ -210,6 +213,12 @@ func WithCache(cache cache.Cacheable) DispatcherOpt {
}
}
func WithPayloadSizeThreshold(threshold int) DispatcherOpt {
return func(opts *DispatcherOpts) {
opts.payloadSizeThreshold = threshold
}
}
func New(fs ...DispatcherOpt) (*DispatcherImpl, error) {
opts := defaultDispatcherOpts()
@@ -257,20 +266,21 @@ func New(fs ...DispatcherOpt) (*DispatcherImpl, error) {
pubBuffer := msgqueuev1.NewMQPubBuffer(opts.mqv1)
return &DispatcherImpl{
mq: opts.mq,
mqv1: opts.mqv1,
pubBuffer: pubBuffer,
l: opts.l,
dv: opts.dv,
v: validator.NewDefaultValidator(),
repo: opts.repo,
repov1: opts.repov1,
entitlements: opts.entitlements,
dispatcherId: opts.dispatcherId,
workers: &workers{},
s: s,
a: a,
cache: opts.cache,
mq: opts.mq,
mqv1: opts.mqv1,
pubBuffer: pubBuffer,
l: opts.l,
dv: opts.dv,
v: validator.NewDefaultValidator(),
repo: opts.repo,
repov1: opts.repov1,
entitlements: opts.entitlements,
dispatcherId: opts.dispatcherId,
workers: &workers{},
s: s,
a: a,
cache: opts.cache,
payloadSizeThreshold: opts.payloadSizeThreshold,
}, nil
}

14
internal/services/dispatcher/server.go
View File

@@ -865,23 +865,21 @@ func (s *sendTimeFilter) canSend() bool {
return true
}
const payloadSizeThreshold = 3 * 1024 * 1024
func cleanResults(results []*contracts.StepRunResult) []*contracts.StepRunResult {
func (d *DispatcherImpl) cleanResults(results []*contracts.StepRunResult) []*contracts.StepRunResult {
totalSize, sizeOfOutputs, _ := calculateResultsSize(results)
if totalSize < payloadSizeThreshold {
if totalSize < d.payloadSizeThreshold {
return results
}
if sizeOfOutputs >= payloadSizeThreshold {
if sizeOfOutputs >= d.payloadSizeThreshold {
return nil
}
// otherwise, attempt to clean the results by removing large error fields
cleanedResults := make([]*contracts.StepRunResult, 0, len(results))
fieldThreshold := (payloadSizeThreshold - sizeOfOutputs) / len(results) // how much overhead we'd have per result or error field, in the worst case
fieldThreshold := (d.payloadSizeThreshold - sizeOfOutputs) / len(results) // how much overhead we'd have per result or error field, in the worst case
for _, result := range results {
if result == nil {
@@ -897,7 +895,7 @@ func cleanResults(results []*contracts.StepRunResult) []*contracts.StepRunResult
}
// if we are still over the limit, we just return nil
if totalSize, _, _ := calculateResultsSize(cleanedResults); totalSize > payloadSizeThreshold {
if totalSize, _, _ := calculateResultsSize(cleanedResults); totalSize > d.payloadSizeThreshold {
return nil
}
@@ -951,7 +949,7 @@ func (s *DispatcherImpl) subscribeToWorkflowRunsV0(server contracts.Dispatcher_S
sendMu := sync.Mutex{}
sendEvent := func(e *contracts.WorkflowRunEvent) error {
results := cleanResults(e.Results)
results := s.cleanResults(e.Results)
if results == nil {
s.l.Warn().Msgf("results size for workflow run %s exceeds 3MB and cannot be reduced", e.WorkflowRunId)

2
internal/services/dispatcher/server_v1.go
View File

@@ -47,7 +47,7 @@ func (s *DispatcherImpl) subscribeToWorkflowRunsV1(server contracts.Dispatcher_S
iterMu := sync.Mutex{}
sendEvent := func(e *contracts.WorkflowRunEvent) error {
results := cleanResults(e.Results)
results := s.cleanResults(e.Results)
if results == nil {
s.l.Warn().Msgf("results size for workflow run %s exceeds 3MB and cannot be reduced", e.WorkflowRunId)