mirror of
https://github.com/hatchet-dev/hatchet.git
synced 2026-04-29 05:59:52 -05:00
Sean Reilly
b5de6e26ff
* add a dynamic strategy for flushing where we make the trigger for flush a funciton of the depth of the concurrency * default value for tests and New for FlushStrategy * clean up the currently flushing locking and add deadlock.Mutex * don't wait as long for the buffer * lets see if this 2ms thing is what is causing things to break * lets error for this to see if we are actually hitting these limits * put a really short deadline on the lock timeout to see if github actions will blow up * lets use RW mutexs se we don't block as much * lets extend this out to 100ms * lets just do fewer locks * add a lock to prevent a queue behind the semaphore * deal with potential data races * a simpler loop fib and now locks * lets get rid of the wait for flush * remove the deadlock stuff * mod tidy --------- Co-authored-by: Sean Reilly <sean@hatchet.run>
26 lines
1.2 KiB
Go
26 lines
1.2 KiB
Go
package buffer
|
|
|
|
import "time"
|
|
|
|
// ConfigFileBuffer is the configuration for the buffer. We store it here to prevent circular dependencies
|
|
type ConfigFileBuffer struct {
|
|
|
|
// WaitForFlush is the time to wait for the buffer to flush used for backpressure on writers
|
|
WaitForFlush time.Duration `mapstructure:"waitForFlush" json:"waitForFlush,omitempty" default:"1ms"`
|
|
|
|
// MaxConcurrent is the maximum number of concurrent flushes
|
|
MaxConcurrent int `mapstructure:"maxConcurrent" json:"maxConcurrent,omitempty" default:"50"`
|
|
|
|
// FlushPeriodMilliseconds is the number of milliseconds before flush
|
|
FlushPeriodMilliseconds int `mapstructure:"flushPeriodMilliseconds" json:"flushPeriodMilliseconds,omitempty" default:"10"`
|
|
|
|
// FlushItemsThreshold is the number of items to hold in memory until flushing to the database
|
|
FlushItemsThreshold int `mapstructure:"flushItemsThreshold" json:"flushItemsThreshold,omitempty" default:"100"`
|
|
|
|
// SerialBuffer is a flag to determine if the buffer should be serial or bulk
|
|
SerialBuffer bool `mapstructure:"serialBuffer" json:"serialBuffer,omitempty" default:"false"`
|
|
|
|
// FlushStrategy is the strategy to use for flushing the buffer
|
|
FlushStrategy BuffStrategy `mapstructure:"flushStrategy" json:"flushStrategy" default:"DYNAMIC"`
|
|
}
|