From b3e5630e1a4e0460077990f4f78aace65851b32a Mon Sep 17 00:00:00 2001 From: Matt Kaye Date: Thu, 3 Apr 2025 09:00:18 -0400 Subject: [PATCH] Fix: Move Non-Retryable to `hatchet_sdk.exceptions` (#1483) * feat: move non-retry to exceptions * fix: client config default * chore: ver * cleanup: fix some dependencies * feat: add retries to pytest in CI --- .github/workflows/sdk-python.yml | 2 +- sdks/python/examples/non_retryable/worker.py | 3 +- sdks/python/hatchet_sdk/__init__.py | 3 +- sdks/python/hatchet_sdk/exceptions.py | 2 + sdks/python/hatchet_sdk/hatchet.py | 6 +- sdks/python/hatchet_sdk/runnables/task.py | 4 -- .../hatchet_sdk/worker/runner/runner.py | 3 +- sdks/python/poetry.lock | 63 ++++++++++--------- sdks/python/pyproject.toml | 19 +++--- 9 files changed, 55 insertions(+), 50 deletions(-) create mode 100644 sdks/python/hatchet_sdk/exceptions.py diff --git a/.github/workflows/sdk-python.yml b/.github/workflows/sdk-python.yml index a0b9423ca..a09ef94bc 100644 --- a/.github/workflows/sdk-python.yml +++ b/.github/workflows/sdk-python.yml @@ -135,7 +135,7 @@ jobs: run: | echo "Using HATCHET_CLIENT_NAMESPACE: $HATCHET_CLIENT_NAMESPACE" - poetry run pytest -s -vvv --maxfail=5 --capture=no -n 5 --timeout 60 + poetry run pytest -s -vvv --maxfail=5 --capture=no -n 5 --timeout 60 --retries 2 --retry-delay 5 - name: Upload engine logs if: always() diff --git a/sdks/python/examples/non_retryable/worker.py b/sdks/python/examples/non_retryable/worker.py index d0483969d..55e35c681 100644 --- a/sdks/python/examples/non_retryable/worker.py +++ b/sdks/python/examples/non_retryable/worker.py @@ -1,4 +1,5 @@ -from hatchet_sdk import Context, EmptyModel, Hatchet, NonRetryableException +from hatchet_sdk import Context, EmptyModel, Hatchet +from hatchet_sdk.exceptions import NonRetryableException hatchet = Hatchet(debug=True) diff --git a/sdks/python/hatchet_sdk/__init__.py b/sdks/python/hatchet_sdk/__init__.py index e35f29daf..abf233445 100644 --- a/sdks/python/hatchet_sdk/__init__.py +++ b/sdks/python/hatchet_sdk/__init__.py @@ -138,7 +138,7 @@ from hatchet_sdk.contracts.workflows_pb2 import ( ) from hatchet_sdk.features.runs import BulkCancelReplayOpts, RunFilter from hatchet_sdk.hatchet import Hatchet -from hatchet_sdk.runnables.task import NonRetryableException, Task +from hatchet_sdk.runnables.task import Task from hatchet_sdk.runnables.types import ( ConcurrencyExpression, ConcurrencyLimitStrategy, @@ -269,5 +269,4 @@ __all__ = [ "BulkCancelReplayOpts", "RunFilter", "V1TaskStatus", - "NonRetryableException", ] diff --git a/sdks/python/hatchet_sdk/exceptions.py b/sdks/python/hatchet_sdk/exceptions.py new file mode 100644 index 000000000..87952e378 --- /dev/null +++ b/sdks/python/hatchet_sdk/exceptions.py @@ -0,0 +1,2 @@ +class NonRetryableException(Exception): + pass diff --git a/sdks/python/hatchet_sdk/hatchet.py b/sdks/python/hatchet_sdk/hatchet.py index ffff37113..44f86e970 100644 --- a/sdks/python/hatchet_sdk/hatchet.py +++ b/sdks/python/hatchet_sdk/hatchet.py @@ -56,7 +56,7 @@ class Hatchet: self, debug: bool = False, client: Client | None = None, - config: ClientConfig = ClientConfig(), + config: ClientConfig | None = None, ): """ Initialize a new Hatchet instance. @@ -74,7 +74,9 @@ class Hatchet: if debug: logger.setLevel(logging.DEBUG) - self._client = client if client else Client(config=config, debug=debug) + self._client = ( + client if client else Client(config=config or ClientConfig(), debug=debug) + ) @property def cron(self) -> CronClient: diff --git a/sdks/python/hatchet_sdk/runnables/task.py b/sdks/python/hatchet_sdk/runnables/task.py index 4a6b19c8a..c995b0b8c 100644 --- a/sdks/python/hatchet_sdk/runnables/task.py +++ b/sdks/python/hatchet_sdk/runnables/task.py @@ -44,10 +44,6 @@ def fall_back_to_default(value: T, default: T, fallback_value: T) -> T: return fallback_value -class NonRetryableException(Exception): - pass - - class Task(Generic[TWorkflowInput, R]): def __init__( self, diff --git a/sdks/python/hatchet_sdk/worker/runner/runner.py b/sdks/python/hatchet_sdk/worker/runner/runner.py index db2390a79..416901352 100644 --- a/sdks/python/hatchet_sdk/worker/runner/runner.py +++ b/sdks/python/hatchet_sdk/worker/runner/runner.py @@ -30,6 +30,7 @@ from hatchet_sdk.contracts.dispatcher_pb2 import ( STEP_EVENT_TYPE_FAILED, STEP_EVENT_TYPE_STARTED, ) +from hatchet_sdk.exceptions import NonRetryableException from hatchet_sdk.logger import logger from hatchet_sdk.runnables.contextvars import ( ctx_step_run_id, @@ -38,7 +39,7 @@ from hatchet_sdk.runnables.contextvars import ( spawn_index_lock, workflow_spawn_indices, ) -from hatchet_sdk.runnables.task import NonRetryableException, Task +from hatchet_sdk.runnables.task import Task from hatchet_sdk.runnables.types import R, TWorkflowInput from hatchet_sdk.utils.typing import WorkflowValidator from hatchet_sdk.worker.action_listener_process import ActionEvent diff --git a/sdks/python/poetry.lock b/sdks/python/poetry.lock index 6346dd5bc..2ccea345b 100644 --- a/sdks/python/poetry.lock +++ b/sdks/python/poetry.lock @@ -410,12 +410,12 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main", "dev", "lint", "test"] +groups = ["lint", "test"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -markers = {main = "sys_platform == \"win32\"", dev = "sys_platform == \"win32\"", lint = "platform_system == \"Windows\"", test = "sys_platform == \"win32\""} +markers = {lint = "platform_system == \"Windows\"", test = "sys_platform == \"win32\""} [[package]] name = "deprecated" @@ -442,7 +442,7 @@ version = "1.2.2" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" -groups = ["main", "dev", "test"] +groups = ["test"] markers = "python_version < \"3.11\"" files = [ {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, @@ -458,7 +458,7 @@ version = "2.1.1" description = "execnet: rapid multi-Python deployment" optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["test"] files = [ {file = "execnet-2.1.1-py3-none-any.whl", hash = "sha256:26dee51f1b80cebd6d0ca8e74dd8745419761d3bef34163928cbebbdc4749fdc"}, {file = "execnet-2.1.1.tar.gz", hash = "sha256:5189b52c6121c24feae288166ab41b32549c7e2348652736540b9e6e7d4e72e3"}, @@ -594,7 +594,7 @@ version = "1.53.0.5" description = "Mypy stubs for gRPC" optional = false python-versions = ">=3.6" -groups = ["dev"] +groups = ["lint"] files = [ {file = "grpc-stubs-1.53.0.5.tar.gz", hash = "sha256:3e1b642775cbc3e0c6332cfcedfccb022176db87e518757bef3a1241397be406"}, {file = "grpc_stubs-1.53.0.5-py3-none-any.whl", hash = "sha256:04183fb65a1b166a1febb9627e3d9647d3926ccc2dfe049fe7b6af243428dbe1"}, @@ -609,7 +609,7 @@ version = "1.71.0" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main", "lint"] files = [ {file = "grpcio-1.71.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:c200cb6f2393468142eb50ab19613229dcc7829b5ccee8b658a36005f6669fdd"}, {file = "grpcio-1.71.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:b2266862c5ad664a380fbbcdbdb8289d71464c42a8c29053820ee78ba0119e5d"}, @@ -779,7 +779,7 @@ version = "2.1.0" description = "brain-dead simple config-ini parsing" optional = false python-versions = ">=3.8" -groups = ["main", "dev", "test"] +groups = ["test"] files = [ {file = "iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760"}, {file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"}, @@ -1000,18 +1000,6 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] -[[package]] -name = "nest-asyncio" -version = "1.6.0" -description = "Patch asyncio to allow nested event loops" -optional = false -python-versions = ">=3.5" -groups = ["main"] -files = [ - {file = "nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c"}, - {file = "nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe"}, -] - [[package]] name = "opentelemetry-api" version = "1.31.1" @@ -1206,11 +1194,12 @@ version = "24.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" -groups = ["main", "dev", "lint", "test"] +groups = ["main", "lint", "test"] files = [ {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, ] +markers = {main = "extra == \"otel\""} [[package]] name = "pathspec" @@ -1247,7 +1236,7 @@ version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" -groups = ["main", "dev", "test"] +groups = ["test"] files = [ {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, @@ -1407,7 +1396,7 @@ version = "6.1.1" description = "Cross-platform lib for process and system monitoring in Python." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" -groups = ["dev"] +groups = ["lint"] files = [ {file = "psutil-6.1.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:9ccc4316f24409159897799b83004cb1e24f9819b0dcf9c0b68bdcb6cefee6a8"}, {file = "psutil-6.1.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ca9609c77ea3b8481ab005da74ed894035936223422dc591d6772b147421f777"}, @@ -1593,7 +1582,7 @@ version = "8.3.5" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" -groups = ["main", "dev", "test"] +groups = ["test"] files = [ {file = "pytest-8.3.5-py3-none-any.whl", hash = "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820"}, {file = "pytest-8.3.5.tar.gz", hash = "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845"}, @@ -1616,7 +1605,7 @@ version = "0.25.3" description = "Pytest support for asyncio" optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["test"] files = [ {file = "pytest_asyncio-0.25.3-py3-none-any.whl", hash = "sha256:9e89518e0f9bd08928f97a3482fdc4e244df17529460bc038291ccaf8f85c7c3"}, {file = "pytest_asyncio-0.25.3.tar.gz", hash = "sha256:fc1da2cf9f125ada7e710b4ddad05518d4cee187ae9412e9ac9271003497f07a"}, @@ -1648,13 +1637,31 @@ tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""} [package.extras] testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "pytest-mock (>=3.14)"] +[[package]] +name = "pytest-retry" +version = "1.7.0" +description = "Adds the ability to retry flaky tests in CI environments" +optional = false +python-versions = ">=3.9" +groups = ["test"] +files = [ + {file = "pytest_retry-1.7.0-py3-none-any.whl", hash = "sha256:a2dac85b79a4e2375943f1429479c65beb6c69553e7dae6b8332be47a60954f4"}, + {file = "pytest_retry-1.7.0.tar.gz", hash = "sha256:f8d52339f01e949df47c11ba9ee8d5b362f5824dff580d3870ec9ae0057df80f"}, +] + +[package.dependencies] +pytest = ">=7.0.0" + +[package.extras] +dev = ["black", "flake8", "isort", "mypy"] + [[package]] name = "pytest-timeout" version = "2.3.1" description = "pytest plugin to abort hanging tests" optional = false python-versions = ">=3.7" -groups = ["main"] +groups = ["test"] files = [ {file = "pytest-timeout-2.3.1.tar.gz", hash = "sha256:12397729125c6ecbdaca01035b9e5239d4db97352320af155b3f5de1ba5165d9"}, {file = "pytest_timeout-2.3.1-py3-none-any.whl", hash = "sha256:68188cb703edfc6a18fad98dc25a3c61e9f24d644b0b70f33af545219fc7813e"}, @@ -1669,7 +1676,7 @@ version = "3.6.1" description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs" optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["test"] files = [ {file = "pytest_xdist-3.6.1-py3-none-any.whl", hash = "sha256:9ed4adfb68a016610848639bb7e02c9352d5d9f03d04809919e2dafc3be4cca7"}, {file = "pytest_xdist-3.6.1.tar.gz", hash = "sha256:ead156a4db231eec769737f57668ef58a2084a34b2e55c4a8fa20d861107300d"}, @@ -1883,7 +1890,7 @@ version = "2.2.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" -groups = ["main", "dev", "lint", "test"] +groups = ["lint", "test"] markers = "python_version < \"3.11\"" files = [ {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, @@ -2227,4 +2234,4 @@ otel = ["opentelemetry-api", "opentelemetry-distro", "opentelemetry-exporter-otl [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "12c7598b8b1ece6a5567e67ad2a1eb230abc73c1ebd4995e729d9471849fdd19" +content-hash = "aefaf08d9d452c523435af00962423d803d14a0d069a7946466d62468ef24e94" diff --git a/sdks/python/pyproject.toml b/sdks/python/pyproject.toml index edd7b17d4..c4a820714 100644 --- a/sdks/python/pyproject.toml +++ b/sdks/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "hatchet-sdk" -version = "1.2.1" +version = "1.2.2" description = "" authors = ["Alexander Belanger "] readme = "README.md" @@ -17,12 +17,10 @@ grpcio-tools = [ { version = ">=1.69.0", markers = "python_version >= '3.13'" }, ] protobuf = "^5.29.1" -pyyaml = "^6.0.1" pydantic = "^2.6.3" python-dateutil = "^2.9.0.post0" urllib3 = ">=1.26.20" aiostream = "^0.5.2" -nest-asyncio = "^1.6.0" aiohttp = "^3.10.5" aiohttp-retry = "^2.8.3" tenacity = ">=8.4.1" @@ -35,14 +33,6 @@ opentelemetry-exporter-otlp = { version = "^1.28.0", optional = true } opentelemetry-exporter-otlp-proto-http = { version = "^1.28.0", optional = true } prometheus-client = "^0.21.1" pydantic-settings = "^2.7.1" -pytest-timeout = "^2.3.1" - -[tool.poetry.group.dev.dependencies] -pytest = "^8.3.5" -pytest-asyncio = "^0.25.3" -psutil = "^6.0.0" -grpc-stubs = "^1.53.0.5" -pytest-xdist = "^3.6.1" [tool.poetry.group.lint.dependencies] mypy = "^1.14.0" @@ -52,9 +42,16 @@ isort = "^5.13.2" types-psutil = "^6.1.0.20241221" ruff = "^0.9.7" types-requests = "^2.32.0.20241016" +psutil = "^6.0.0" +grpc-stubs = "^1.53.0.5" [tool.poetry.group.test.dependencies] +pytest = "^8.3.5" +pytest-asyncio = "^0.25.3" pytest-env = "^1.1.5" +pytest-xdist = "^3.6.1" +pytest-retry = "^1.7.0" +pytest-timeout = "^2.3.1" [tool.poetry.extras] otel = [