mirror/hatchet
1
0
Fork 0
mirror of https://github.com/hatchet-dev/hatchet.git synced 2026-01-05 16:19:43 -06:00
Code Issues Packages Projects Releases Wiki Activity

feat: support maxRuns parameter on workers (#195)

Browse Source 
* feat: round robin queueing

* feat: configurable max runs per worker

* fix: address PR review

* docs for max runs and group round robin
This commit is contained in:
abelanger5
2024-02-25 21:48:46 -08:00
committed by GitHub
parent 2d625fec81
commit 6ea38a99f2
46 changed files with 659 additions and 373 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
python-sdk/examples/concurrency-limit-rr/worker.py
View File

@@ -23,7 +23,7 @@ class ConcurrencyDemoWorkflowRR:
pass
workflow = ConcurrencyDemoWorkflowRR()
worker = hatchet.worker('concurrency-demo-worker-rr', max_threads=1)
worker = hatchet.worker('concurrency-demo-worker-rr', max_runs=1)
worker.register_workflow(workflow)
worker.start()

2
python-sdk/examples/concurrency-limit/worker.py
View File

@@ -26,7 +26,7 @@ class ConcurrencyDemoWorkflow:
print("finished step2")
workflow = ConcurrencyDemoWorkflow()
worker = hatchet.worker('concurrency-demo-worker', max_threads=4)
worker = hatchet.worker('concurrency-demo-worker', max_runs=4)
worker.register_workflow(workflow)
worker.start()

4
python-sdk/examples/dag/worker.py
View File

@@ -1,3 +1,4 @@
import time
from hatchet_sdk import Hatchet, Context
from dotenv import load_dotenv
@@ -13,7 +14,7 @@ class MyWorkflow:
@hatchet.step()
def step1(self, context : Context):
overrideValue = context.playground("prompt", "You are an AI assistant...")
time.sleep(5)
print("executed step1", context.workflow_input())
return {
"step1": overrideValue,
@@ -22,6 +23,7 @@ class MyWorkflow:
@hatchet.step()
def step2(self, context : Context):
print("executed step2", context.workflow_input())
time.sleep(5)
return {
"step2": "step2",
}

2
python-sdk/examples/manual_trigger/worker.py
View File

@@ -40,7 +40,7 @@ class ManualTriggerWorkflow:
workflow = ManualTriggerWorkflow()
worker = hatchet.worker('manual-worker', max_threads=4)
worker = hatchet.worker('manual-worker', max_runs=4)
worker.register_workflow(workflow)
worker.start()

2
python-sdk/examples/simple/worker.py
View File

@@ -29,7 +29,7 @@ class MyWorkflow:
print("finished step2")
workflow = MyWorkflow()
worker = hatchet.worker('test-worker', max_threads=4)
worker = hatchet.worker('test-worker', max_runs=4)
worker.register_workflow(workflow)
worker.start()

2
python-sdk/examples/timeout/worker.py
View File

@@ -21,7 +21,7 @@ class TimeoutWorkflow:
raise e
workflow = TimeoutWorkflow()
worker = hatchet.worker('timeout-worker', max_threads=4)
worker = hatchet.worker('timeout-worker', max_runs=4)
worker.register_workflow(workflow)
worker.start()

6
python-sdk/hatchet_sdk/clients/dispatcher.py
View File

@@ -33,10 +33,11 @@ DEFAULT_REGISTER_TIMEOUT = 5
class GetActionListenerRequest:
def __init__(self, worker_name: str, services: List[str], actions: List[str]):
def __init__(self, worker_name: str, services: List[str], actions: List[str], max_runs: int | None = None):
self.worker_name = worker_name
self.services = services
self.actions = actions
self.max_runs = max_runs
class Action:
def __init__(self, worker_id: str, tenant_id: str, workflow_run_id: str, get_group_key_run_id: str, job_id: str, job_name: str, job_run_id: str, step_id: str, step_run_id: str, action_id: str, action_payload: str, action_type: ActionType):
@@ -196,7 +197,8 @@ class DispatcherClientImpl(DispatcherClient):
response : WorkerRegisterResponse = self.client.Register(WorkerRegisterRequest(
workerName=req.worker_name,
actions=req.actions,
services=req.services
services=req.services,
maxRuns=req.max_runs
), timeout=DEFAULT_REGISTER_TIMEOUT, metadata=get_metadata(self.token))
return ActionListenerImpl(self.client, self.token, response.workerId)

76
python-sdk/hatchet_sdk/dispatcher_pb2.py
View File

@@ -15,7 +15,7 @@ _sym_db = _symbol_database.Default()
from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x10\x64ispatcher.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"N\n\x15WorkerRegisterRequest\x12\x12\n\nworkerName\x18\x01 \x01(\t\x12\x0f\n\x07\x61\x63tions\x18\x02 \x03(\t\x12\x10\n\x08services\x18\x03 \x03(\t\"P\n\x16WorkerRegisterResponse\x12\x10\n\x08tenantId\x18\x01 \x01(\t\x12\x10\n\x08workerId\x18\x02 \x01(\t\x12\x12\n\nworkerName\x18\x03 \x01(\t\"\xf2\x01\n\x0e\x41ssignedAction\x12\x10\n\x08tenantId\x18\x01 \x01(\t\x12\x15\n\rworkflowRunId\x18\x02 \x01(\t\x12\x18\n\x10getGroupKeyRunId\x18\x03 \x01(\t\x12\r\n\x05jobId\x18\x04 \x01(\t\x12\x0f\n\x07jobName\x18\x05 \x01(\t\x12\x10\n\x08jobRunId\x18\x06 \x01(\t\x12\x0e\n\x06stepId\x18\x07 \x01(\t\x12\x11\n\tstepRunId\x18\x08 \x01(\t\x12\x10\n\x08\x61\x63tionId\x18\t \x01(\t\x12\x1f\n\nactionType\x18\n \x01(\x0e\x32\x0b.ActionType\x12\x15\n\ractionPayload\x18\x0b \x01(\t\"\'\n\x13WorkerListenRequest\x12\x10\n\x08workerId\x18\x01 \x01(\t\",\n\x18WorkerUnsubscribeRequest\x12\x10\n\x08workerId\x18\x01 \x01(\t\"?\n\x19WorkerUnsubscribeResponse\x12\x10\n\x08tenantId\x18\x01 \x01(\t\x12\x10\n\x08workerId\x18\x02 \x01(\t\"\xe1\x01\n\x13GroupKeyActionEvent\x12\x10\n\x08workerId\x18\x01 \x01(\t\x12\x15\n\rworkflowRunId\x18\x02 \x01(\t\x12\x18\n\x10getGroupKeyRunId\x18\x03 \x01(\t\x12\x10\n\x08\x61\x63tionId\x18\x04 \x01(\t\x12\x32\n\x0e\x65ventTimestamp\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12+\n\teventType\x18\x06 \x01(\x0e\x32\x18.GroupKeyActionEventType\x12\x14\n\x0c\x65ventPayload\x18\x07 \x01(\t\"\xec\x01\n\x0fStepActionEvent\x12\x10\n\x08workerId\x18\x01 \x01(\t\x12\r\n\x05jobId\x18\x02 \x01(\t\x12\x10\n\x08jobRunId\x18\x03 \x01(\t\x12\x0e\n\x06stepId\x18\x04 \x01(\t\x12\x11\n\tstepRunId\x18\x05 \x01(\t\x12\x10\n\x08\x61\x63tionId\x18\x06 \x01(\t\x12\x32\n\x0e\x65ventTimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\'\n\teventType\x18\x08 \x01(\x0e\x32\x14.StepActionEventType\x12\x14\n\x0c\x65ventPayload\x18\t \x01(\t\"9\n\x13\x41\x63tionEventResponse\x12\x10\n\x08tenantId\x18\x01 \x01(\t\x12\x10\n\x08workerId\x18\x02 \x01(\t\"9\n SubscribeToWorkflowEventsRequest\x12\x15\n\rworkflowRunId\x18\x01 \x01(\t\"\xe0\x01\n\rWorkflowEvent\x12\x15\n\rworkflowRunId\x18\x01 \x01(\t\x12#\n\x0cresourceType\x18\x02 \x01(\x0e\x32\r.ResourceType\x12%\n\teventType\x18\x03 \x01(\x0e\x32\x12.ResourceEventType\x12\x12\n\nresourceId\x18\x04 \x01(\t\x12\x32\n\x0e\x65ventTimestamp\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0c\x65ventPayload\x18\x06 \x01(\t\x12\x0e\n\x06hangup\x18\x07 \x01(\x08\"W\n\rOverridesData\x12\x11\n\tstepRunId\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\x12\r\n\x05value\x18\x03 \x01(\t\x12\x16\n\x0e\x63\x61llerFilename\x18\x04 \x01(\t\"\x17\n\x15OverridesDataResponse*N\n\nActionType\x12\x12\n\x0eSTART_STEP_RUN\x10\x00\x12\x13\n\x0f\x43\x41NCEL_STEP_RUN\x10\x01\x12\x17\n\x13START_GET_GROUP_KEY\x10\x02*\xa2\x01\n\x17GroupKeyActionEventType\x12 \n\x1cGROUP_KEY_EVENT_TYPE_UNKNOWN\x10\x00\x12 \n\x1cGROUP_KEY_EVENT_TYPE_STARTED\x10\x01\x12\"\n\x1eGROUP_KEY_EVENT_TYPE_COMPLETED\x10\x02\x12\x1f\n\x1bGROUP_KEY_EVENT_TYPE_FAILED\x10\x03*\x8a\x01\n\x13StepActionEventType\x12\x1b\n\x17STEP_EVENT_TYPE_UNKNOWN\x10\x00\x12\x1b\n\x17STEP_EVENT_TYPE_STARTED\x10\x01\x12\x1d\n\x19STEP_EVENT_TYPE_COMPLETED\x10\x02\x12\x1a\n\x16STEP_EVENT_TYPE_FAILED\x10\x03*e\n\x0cResourceType\x12\x19\n\x15RESOURCE_TYPE_UNKNOWN\x10\x00\x12\x1a\n\x16RESOURCE_TYPE_STEP_RUN\x10\x01\x12\x1e\n\x1aRESOURCE_TYPE_WORKFLOW_RUN\x10\x02*\xde\x01\n\x11ResourceEventType\x12\x1f\n\x1bRESOURCE_EVENT_TYPE_UNKNOWN\x10\x00\x12\x1f\n\x1bRESOURCE_EVENT_TYPE_STARTED\x10\x01\x12!\n\x1dRESOURCE_EVENT_TYPE_COMPLETED\x10\x02\x12\x1e\n\x1aRESOURCE_EVENT_TYPE_FAILED\x10\x03\x12!\n\x1dRESOURCE_EVENT_TYPE_CANCELLED\x10\x04\x12!\n\x1dRESOURCE_EVENT_TYPE_TIMED_OUT\x10\x05\x32\xe4\x03\n\nDispatcher\x12=\n\x08Register\x12\x16.WorkerRegisterRequest\x1a\x17.WorkerRegisterResponse\"\x00\x12\x33\n\x06Listen\x12\x14.WorkerListenRequest\x1a\x0f.AssignedAction\"\x00\x30\x01\x12R\n\x19SubscribeToWorkflowEvents\x12!.SubscribeToWorkflowEventsRequest\x1a\x0e.WorkflowEvent\"\x00\x30\x01\x12?\n\x13SendStepActionEvent\x12\x10.StepActionEvent\x1a\x14.ActionEventResponse\"\x00\x12G\n\x17SendGroupKeyActionEvent\x12\x14.GroupKeyActionEvent\x1a\x14.ActionEventResponse\"\x00\x12<\n\x10PutOverridesData\x12\x0e.OverridesData\x1a\x16.OverridesDataResponse\"\x00\x12\x46\n\x0bUnsubscribe\x12\x19.WorkerUnsubscribeRequest\x1a\x1a.WorkerUnsubscribeResponse\"\x00\x42GZEgithub.com/hatchet-dev/hatchet/internal/services/dispatcher/contractsb\x06proto3')
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x10\x64ispatcher.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"p\n\x15WorkerRegisterRequest\x12\x12\n\nworkerName\x18\x01 \x01(\t\x12\x0f\n\x07\x61\x63tions\x18\x02 \x03(\t\x12\x10\n\x08services\x18\x03 \x03(\t\x12\x14\n\x07maxRuns\x18\x04 \x01(\x05H\x00\x88\x01\x01\x42\n\n\x08_maxRuns\"P\n\x16WorkerRegisterResponse\x12\x10\n\x08tenantId\x18\x01 \x01(\t\x12\x10\n\x08workerId\x18\x02 \x01(\t\x12\x12\n\nworkerName\x18\x03 \x01(\t\"\xf2\x01\n\x0e\x41ssignedAction\x12\x10\n\x08tenantId\x18\x01 \x01(\t\x12\x15\n\rworkflowRunId\x18\x02 \x01(\t\x12\x18\n\x10getGroupKeyRunId\x18\x03 \x01(\t\x12\r\n\x05jobId\x18\x04 \x01(\t\x12\x0f\n\x07jobName\x18\x05 \x01(\t\x12\x10\n\x08jobRunId\x18\x06 \x01(\t\x12\x0e\n\x06stepId\x18\x07 \x01(\t\x12\x11\n\tstepRunId\x18\x08 \x01(\t\x12\x10\n\x08\x61\x63tionId\x18\t \x01(\t\x12\x1f\n\nactionType\x18\n \x01(\x0e\x32\x0b.ActionType\x12\x15\n\ractionPayload\x18\x0b \x01(\t\"\'\n\x13WorkerListenRequest\x12\x10\n\x08workerId\x18\x01 \x01(\t\",\n\x18WorkerUnsubscribeRequest\x12\x10\n\x08workerId\x18\x01 \x01(\t\"?\n\x19WorkerUnsubscribeResponse\x12\x10\n\x08tenantId\x18\x01 \x01(\t\x12\x10\n\x08workerId\x18\x02 \x01(\t\"\xe1\x01\n\x13GroupKeyActionEvent\x12\x10\n\x08workerId\x18\x01 \x01(\t\x12\x15\n\rworkflowRunId\x18\x02 \x01(\t\x12\x18\n\x10getGroupKeyRunId\x18\x03 \x01(\t\x12\x10\n\x08\x61\x63tionId\x18\x04 \x01(\t\x12\x32\n\x0e\x65ventTimestamp\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12+\n\teventType\x18\x06 \x01(\x0e\x32\x18.GroupKeyActionEventType\x12\x14\n\x0c\x65ventPayload\x18\x07 \x01(\t\"\xec\x01\n\x0fStepActionEvent\x12\x10\n\x08workerId\x18\x01 \x01(\t\x12\r\n\x05jobId\x18\x02 \x01(\t\x12\x10\n\x08jobRunId\x18\x03 \x01(\t\x12\x0e\n\x06stepId\x18\x04 \x01(\t\x12\x11\n\tstepRunId\x18\x05 \x01(\t\x12\x10\n\x08\x61\x63tionId\x18\x06 \x01(\t\x12\x32\n\x0e\x65ventTimestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\'\n\teventType\x18\x08 \x01(\x0e\x32\x14.StepActionEventType\x12\x14\n\x0c\x65ventPayload\x18\t \x01(\t\"9\n\x13\x41\x63tionEventResponse\x12\x10\n\x08tenantId\x18\x01 \x01(\t\x12\x10\n\x08workerId\x18\x02 \x01(\t\"9\n SubscribeToWorkflowEventsRequest\x12\x15\n\rworkflowRunId\x18\x01 \x01(\t\"\xe0\x01\n\rWorkflowEvent\x12\x15\n\rworkflowRunId\x18\x01 \x01(\t\x12#\n\x0cresourceType\x18\x02 \x01(\x0e\x32\r.ResourceType\x12%\n\teventType\x18\x03 \x01(\x0e\x32\x12.ResourceEventType\x12\x12\n\nresourceId\x18\x04 \x01(\t\x12\x32\n\x0e\x65ventTimestamp\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0c\x65ventPayload\x18\x06 \x01(\t\x12\x0e\n\x06hangup\x18\x07 \x01(\x08\"W\n\rOverridesData\x12\x11\n\tstepRunId\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\x12\r\n\x05value\x18\x03 \x01(\t\x12\x16\n\x0e\x63\x61llerFilename\x18\x04 \x01(\t\"\x17\n\x15OverridesDataResponse*N\n\nActionType\x12\x12\n\x0eSTART_STEP_RUN\x10\x00\x12\x13\n\x0f\x43\x41NCEL_STEP_RUN\x10\x01\x12\x17\n\x13START_GET_GROUP_KEY\x10\x02*\xa2\x01\n\x17GroupKeyActionEventType\x12 \n\x1cGROUP_KEY_EVENT_TYPE_UNKNOWN\x10\x00\x12 \n\x1cGROUP_KEY_EVENT_TYPE_STARTED\x10\x01\x12\"\n\x1eGROUP_KEY_EVENT_TYPE_COMPLETED\x10\x02\x12\x1f\n\x1bGROUP_KEY_EVENT_TYPE_FAILED\x10\x03*\x8a\x01\n\x13StepActionEventType\x12\x1b\n\x17STEP_EVENT_TYPE_UNKNOWN\x10\x00\x12\x1b\n\x17STEP_EVENT_TYPE_STARTED\x10\x01\x12\x1d\n\x19STEP_EVENT_TYPE_COMPLETED\x10\x02\x12\x1a\n\x16STEP_EVENT_TYPE_FAILED\x10\x03*e\n\x0cResourceType\x12\x19\n\x15RESOURCE_TYPE_UNKNOWN\x10\x00\x12\x1a\n\x16RESOURCE_TYPE_STEP_RUN\x10\x01\x12\x1e\n\x1aRESOURCE_TYPE_WORKFLOW_RUN\x10\x02*\xde\x01\n\x11ResourceEventType\x12\x1f\n\x1bRESOURCE_EVENT_TYPE_UNKNOWN\x10\x00\x12\x1f\n\x1bRESOURCE_EVENT_TYPE_STARTED\x10\x01\x12!\n\x1dRESOURCE_EVENT_TYPE_COMPLETED\x10\x02\x12\x1e\n\x1aRESOURCE_EVENT_TYPE_FAILED\x10\x03\x12!\n\x1dRESOURCE_EVENT_TYPE_CANCELLED\x10\x04\x12!\n\x1dRESOURCE_EVENT_TYPE_TIMED_OUT\x10\x05\x32\xe4\x03\n\nDispatcher\x12=\n\x08Register\x12\x16.WorkerRegisterRequest\x1a\x17.WorkerRegisterResponse\"\x00\x12\x33\n\x06Listen\x12\x14.WorkerListenRequest\x1a\x0f.AssignedAction\"\x00\x30\x01\x12R\n\x19SubscribeToWorkflowEvents\x12!.SubscribeToWorkflowEventsRequest\x1a\x0e.WorkflowEvent\"\x00\x30\x01\x12?\n\x13SendStepActionEvent\x12\x10.StepActionEvent\x1a\x14.ActionEventResponse\"\x00\x12G\n\x17SendGroupKeyActionEvent\x12\x14.GroupKeyActionEvent\x1a\x14.ActionEventResponse\"\x00\x12<\n\x10PutOverridesData\x12\x0e.OverridesData\x1a\x16.OverridesDataResponse\"\x00\x12\x46\n\x0bUnsubscribe\x12\x19.WorkerUnsubscribeRequest\x1a\x1a.WorkerUnsubscribeResponse\"\x00\x42GZEgithub.com/hatchet-dev/hatchet/internal/services/dispatcher/contractsb\x06proto3')
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -23,42 +23,42 @@ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'dispatcher_pb2', _globals)
if _descriptor._USE_C_DESCRIPTORS == False:
_globals['DESCRIPTOR']._options = None
_globals['DESCRIPTOR']._serialized_options = b'ZEgithub.com/hatchet-dev/hatchet/internal/services/dispatcher/contracts'
_globals['_ACTIONTYPE']._serialized_start=1538
_globals['_ACTIONTYPE']._serialized_end=1616
_globals['_GROUPKEYACTIONEVENTTYPE']._serialized_start=1619
_globals['_GROUPKEYACTIONEVENTTYPE']._serialized_end=1781
_globals['_STEPACTIONEVENTTYPE']._serialized_start=1784
_globals['_STEPACTIONEVENTTYPE']._serialized_end=1922
_globals['_RESOURCETYPE']._serialized_start=1924
_globals['_RESOURCETYPE']._serialized_end=2025
_globals['_RESOURCEEVENTTYPE']._serialized_start=2028
_globals['_RESOURCEEVENTTYPE']._serialized_end=2250
_globals['_ACTIONTYPE']._serialized_start=1572
_globals['_ACTIONTYPE']._serialized_end=1650
_globals['_GROUPKEYACTIONEVENTTYPE']._serialized_start=1653
_globals['_GROUPKEYACTIONEVENTTYPE']._serialized_end=1815
_globals['_STEPACTIONEVENTTYPE']._serialized_start=1818
_globals['_STEPACTIONEVENTTYPE']._serialized_end=1956
_globals['_RESOURCETYPE']._serialized_start=1958
_globals['_RESOURCETYPE']._serialized_end=2059
_globals['_RESOURCEEVENTTYPE']._serialized_start=2062
_globals['_RESOURCEEVENTTYPE']._serialized_end=2284
_globals['_WORKERREGISTERREQUEST']._serialized_start=53
_globals['_WORKERREGISTERREQUEST']._serialized_end=131
_globals['_WORKERREGISTERRESPONSE']._serialized_start=133
_globals['_WORKERREGISTERRESPONSE']._serialized_end=213
_globals['_ASSIGNEDACTION']._serialized_start=216
_globals['_ASSIGNEDACTION']._serialized_end=458
_globals['_WORKERLISTENREQUEST']._serialized_start=460
_globals['_WORKERLISTENREQUEST']._serialized_end=499
_globals['_WORKERUNSUBSCRIBEREQUEST']._serialized_start=501
_globals['_WORKERUNSUBSCRIBEREQUEST']._serialized_end=545
_globals['_WORKERUNSUBSCRIBERESPONSE']._serialized_start=547
_globals['_WORKERUNSUBSCRIBERESPONSE']._serialized_end=610
_globals['_GROUPKEYACTIONEVENT']._serialized_start=613
_globals['_GROUPKEYACTIONEVENT']._serialized_end=838
_globals['_STEPACTIONEVENT']._serialized_start=841
_globals['_STEPACTIONEVENT']._serialized_end=1077
_globals['_ACTIONEVENTRESPONSE']._serialized_start=1079
_globals['_ACTIONEVENTRESPONSE']._serialized_end=1136
_globals['_SUBSCRIBETOWORKFLOWEVENTSREQUEST']._serialized_start=1138
_globals['_SUBSCRIBETOWORKFLOWEVENTSREQUEST']._serialized_end=1195
_globals['_WORKFLOWEVENT']._serialized_start=1198
_globals['_WORKFLOWEVENT']._serialized_end=1422
_globals['_OVERRIDESDATA']._serialized_start=1424
_globals['_OVERRIDESDATA']._serialized_end=1511
_globals['_OVERRIDESDATARESPONSE']._serialized_start=1513
_globals['_OVERRIDESDATARESPONSE']._serialized_end=1536
_globals['_DISPATCHER']._serialized_start=2253
_globals['_DISPATCHER']._serialized_end=2737
_globals['_WORKERREGISTERREQUEST']._serialized_end=165
_globals['_WORKERREGISTERRESPONSE']._serialized_start=167
_globals['_WORKERREGISTERRESPONSE']._serialized_end=247
_globals['_ASSIGNEDACTION']._serialized_start=250
_globals['_ASSIGNEDACTION']._serialized_end=492
_globals['_WORKERLISTENREQUEST']._serialized_start=494
_globals['_WORKERLISTENREQUEST']._serialized_end=533
_globals['_WORKERUNSUBSCRIBEREQUEST']._serialized_start=535
_globals['_WORKERUNSUBSCRIBEREQUEST']._serialized_end=579
_globals['_WORKERUNSUBSCRIBERESPONSE']._serialized_start=581
_globals['_WORKERUNSUBSCRIBERESPONSE']._serialized_end=644
_globals['_GROUPKEYACTIONEVENT']._serialized_start=647
_globals['_GROUPKEYACTIONEVENT']._serialized_end=872
_globals['_STEPACTIONEVENT']._serialized_start=875
_globals['_STEPACTIONEVENT']._serialized_end=1111
_globals['_ACTIONEVENTRESPONSE']._serialized_start=1113
_globals['_ACTIONEVENTRESPONSE']._serialized_end=1170
_globals['_SUBSCRIBETOWORKFLOWEVENTSREQUEST']._serialized_start=1172
_globals['_SUBSCRIBETOWORKFLOWEVENTSREQUEST']._serialized_end=1229
_globals['_WORKFLOWEVENT']._serialized_start=1232
_globals['_WORKFLOWEVENT']._serialized_end=1456
_globals['_OVERRIDESDATA']._serialized_start=1458
_globals['_OVERRIDESDATA']._serialized_end=1545
_globals['_OVERRIDESDATARESPONSE']._serialized_start=1547
_globals['_OVERRIDESDATARESPONSE']._serialized_end=1570
_globals['_DISPATCHER']._serialized_start=2287
_globals['_DISPATCHER']._serialized_end=2771
# @@protoc_insertion_point(module_scope)

6
python-sdk/hatchet_sdk/dispatcher_pb2.pyi
View File

@@ -63,14 +63,16 @@ RESOURCE_EVENT_TYPE_CANCELLED: ResourceEventType
RESOURCE_EVENT_TYPE_TIMED_OUT: ResourceEventType
class WorkerRegisterRequest(_message.Message):
__slots__ = ("workerName", "actions", "services")
__slots__ = ("workerName", "actions", "services", "maxRuns")
WORKERNAME_FIELD_NUMBER: _ClassVar[int]
ACTIONS_FIELD_NUMBER: _ClassVar[int]
SERVICES_FIELD_NUMBER: _ClassVar[int]
MAXRUNS_FIELD_NUMBER: _ClassVar[int]
workerName: str
actions: _containers.RepeatedScalarFieldContainer[str]
services: _containers.RepeatedScalarFieldContainer[str]
def __init__(self, workerName: _Optional[str] = ..., actions: _Optional[_Iterable[str]] = ..., services: _Optional[_Iterable[str]] = ...) -> None: ...
maxRuns: int
def __init__(self, workerName: _Optional[str] = ..., actions: _Optional[_Iterable[str]] = ..., services: _Optional[_Iterable[str]] = ..., maxRuns: _Optional[int] = ...) -> None: ...
class WorkerRegisterResponse(_message.Message):
__slots__ = ("tenantId", "workerId", "workerName")

4
python-sdk/hatchet_sdk/hatchet.py
View File

@@ -48,5 +48,5 @@ class Hatchet:
return inner
def worker(self, name: str, max_threads: int = 200):
return Worker(name, max_threads)
def worker(self, name: str, max_runs: int | None = None):
return Worker(name=name, max_runs=max_runs)

7
python-sdk/hatchet_sdk/worker.py
View File

@@ -19,10 +19,11 @@ from .logger import logger
class Worker:
def __init__(self, name: str, max_threads: int = 200, debug=False, handle_kill=True):
def __init__(self, name: str, max_runs: int | None = None, debug=False, handle_kill=True):
self.name = name
self.threads: Dict[str, Thread] = {} # Store step run ids and threads
self.thread_pool = ThreadPoolExecutor(max_workers=max_threads)
self.max_runs = max_runs
self.thread_pool = ThreadPoolExecutor(max_workers=max_runs)
self.futures: Dict[str, Future] = {} # Store step run ids and futures
self.contexts: Dict[str, Context] = {} # Store step run ids and contexts
self.action_registry : dict[str, Callable[..., Any]] = {}
@@ -138,7 +139,6 @@ class Worker:
if not errored:
# Create an action event
try:
print(action)
event = self.get_group_key_action_finished_event(action, output)
except Exception as e:
logger.error(f"Could not get action finished event: {e}")
@@ -335,6 +335,7 @@ class Worker:
worker_name=self.name,
services=["default"],
actions=self.action_registry.keys(),
max_runs=self.max_runs,
))
generator = self.listener.actions()

2
python-sdk/pyproject.toml
View File

@@ -1,6 +1,6 @@
[tool.poetry]
name = "hatchet-sdk"
version = "0.11.0"
version = "0.12.0"
description = ""
authors = ["Alexander Belanger <alexander@hatchet.run>"]
readme = "README.md"