mirror/hatchet
1
0
Fork 0
mirror of https://github.com/hatchet-dev/hatchet.git synced 2026-04-24 02:59:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

Fix: Don't reset offset if a new process acquires lease (#2628)

Browse Source 
* fix: don't reset offset if a new process acquires the lease

* fix: copy paste

* feat: migration, fix queries

* fix: more queries

* fix: down migration

* fix: comment

* feat: finish wiring up everything else

* fix: placeholder initial type

* fix: zero values everywhere

* fix: param ordering

* fix: handle no rows

* fix: zero values

* fix: limit

* fix: simplify

* fix: better defaults
This commit is contained in:
matt
2025-12-09 19:01:51 -05:00
committed by GitHub
parent 65bd347e94
commit 3ff672ebe4
12 changed files with 577 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sql/schema/v1-core.sql
+13 -6
View File
@@ -1758,10 +1758,14 @@ $$;
CREATE TABLE v1_payload_cutover_job_offset (
key DATE PRIMARY KEY,
last_offset BIGINT NOT NULL,
is_completed BOOLEAN NOT NULL DEFAULT FALSE,
lease_process_id UUID NOT NULL DEFAULT gen_random_uuid(),
lease_expires_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
lease_expires_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
last_tenant_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000'::UUID,
last_inserted_at TIMESTAMPTZ NOT NULL DEFAULT '1970-01-01 00:00:00+00',
last_id BIGINT NOT NULL DEFAULT 0,
last_type v1_payload_type NOT NULL DEFAULT 'TASK_INPUT'
);
CREATE OR REPLACE FUNCTION copy_v1_payload_partition_structure(
@@ -1875,7 +1879,10 @@ $$;
CREATE OR REPLACE FUNCTION list_paginated_payloads_for_offload(
partition_date date,
limit_param int,
offset_param bigint
last_tenant_id uuid,
last_inserted_at timestamptz,
last_id bigint,
last_type v1_payload_type
) RETURNS TABLE (
tenant_id UUID,
id BIGINT,
@@ -1909,12 +1916,12 @@ BEGIN
SELECT tenant_id, id, inserted_at, external_id, type, location,
external_location_key, inline_content, updated_at
FROM %I
WHERE (tenant_id, inserted_at, id, type) > ($1, $2, $3, $4)
ORDER BY tenant_id, inserted_at, id, type
LIMIT $1
OFFSET $2
LIMIT $5
', source_partition_name);
RETURN QUERY EXECUTE query USING limit_param, offset_param;
RETURN QUERY EXECUTE query USING last_tenant_id, last_inserted_at, last_id, last_type, limit_param;
END;
$$;
sql/schema/v1-olap.sql
+11 -6
View File
@@ -801,10 +801,13 @@ EXECUTE FUNCTION v1_events_lookup_table_olap_insert_function();
CREATE TABLE v1_payloads_olap_cutover_job_offset (
key DATE PRIMARY KEY,
last_offset BIGINT NOT NULL,
is_completed BOOLEAN NOT NULL DEFAULT FALSE,
lease_process_id UUID NOT NULL DEFAULT gen_random_uuid(),
lease_expires_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
lease_expires_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
last_tenant_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000'::UUID,
last_external_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000'::UUID,
last_inserted_at TIMESTAMPTZ NOT NULL DEFAULT '1970-01-01 00:00:00+00'
);
CREATE OR REPLACE FUNCTION copy_v1_payloads_olap_partition_structure(
@@ -918,7 +921,9 @@ $$;
CREATE OR REPLACE FUNCTION list_paginated_olap_payloads_for_offload(
partition_date date,
limit_param int,
offset_param bigint
last_tenant_id uuid,
last_external_id uuid,
last_inserted_at timestamptz
) RETURNS TABLE (
tenant_id UUID,
external_id UUID,
@@ -949,12 +954,12 @@ BEGIN
query := format('
SELECT tenant_id, external_id, location, external_location_key, inline_content, inserted_at, updated_at
FROM %I
WHERE (tenant_id, external_id, inserted_at) > ($1, $2, $3)
ORDER BY tenant_id, external_id, inserted_at
LIMIT $1
OFFSET $2
LIMIT $4
', source_partition_name);
RETURN QUERY EXECUTE query USING limit_param, offset_param;
RETURN QUERY EXECUTE query USING last_tenant_id, last_external_id, last_inserted_at, limit_param;
END;
$$;