Files
hatchet/pkg/repository/v1/sqlcv1/payload-store-overwrite.sql.go
matt 18940869ae Feat: Job for payload cutovers to external (#2586)
* feat: initial payload cutover job

* refactor: fix a couple things

* feat: start wiring up writes

* feat: only run job if external store is enabled

* fix: add some notes, add loop

* feat: function for reading out payloads

* fix: date handling, logging

* feat: remove wal and immediate offloads

* feat: advisory lock

* feat: partition swap logic

* fix: rm debug

* fix: add todo

* fix: sql cleanup

* fix: sql cleanup, ii

* chore: nuke a bunch of WAL stuff

* chore: more wal

* feat: trigger for crud opts

* feat: drop trigger + function in swapover

* feat: move autovac to later

* feat: use unlogged table initially

* feat: update migration

* fix: drop trigger

* fix: use insert + on conflict

* fix: types

* refactor: clean up a bit

* fix: panic

* fix: detach partition before dropping

* feat: configurable batch size

* feat: offset tracking in the db

* feat: explicitly lock

* fix: down migration

* fix: bug

* fix: offset handling

* fix: try explicit ordering of the insert

* fix: lock location

* fix: do less stuff after locking

* fix: ordering

* fix: dont drop and recreate if temp table exists

* fix: explicitly track completed status

* fix: table name

* fix: dont use unlogged table

* fix: rm todos

* chore: lint

* feat: configurable delay

* fix: use date as pk instead of varchar

* fix: daily job

* fix: hack check constraint to speed up partition attach

* fix: syntax

* fix: syntax

* fix: drop constraint after attaching

* fix: syntax

* fix: drop triggers properly

* fix: factor out insert logic

* refactor: factor out loop logic

* refactor: factor out job preparation work

* fix: ordering

* fix: run the job more often

* fix: use `WithSingletonMode`

* fix: singleton mode sig

* fix: env var cleanup

* fix: overwrite sig

* fix: re-enable immediate offloads with a flag

* fix: order, offload at logic

* feat: add count query to compare

* fix: row-level triggers, partition time bug

* fix: rm todo

* fix: for true

* fix: handle lock not acquired

* fix: handle error

* fix: comment
2025-12-05 10:54:26 -05:00

116 lines
3.0 KiB
Go

package sqlcv1
import (
"context"
"fmt"
"github.com/jackc/pgx/v5/pgtype"
)
type CutoverPayloadToInsert struct {
TenantID pgtype.UUID
ID int64
InsertedAt pgtype.Timestamptz
ExternalID pgtype.UUID
Type V1PayloadType
ExternalLocationKey string
}
func InsertCutOverPayloadsIntoTempTable(ctx context.Context, tx DBTX, tableName string, payloads []CutoverPayloadToInsert) (int64, error) {
tenantIds := make([]pgtype.UUID, 0, len(payloads))
ids := make([]int64, 0, len(payloads))
insertedAts := make([]pgtype.Timestamptz, 0, len(payloads))
externalIds := make([]pgtype.UUID, 0, len(payloads))
types := make([]string, 0, len(payloads))
locations := make([]string, 0, len(payloads))
externalLocationKeys := make([]string, 0, len(payloads))
for _, payload := range payloads {
externalIds = append(externalIds, payload.ExternalID)
tenantIds = append(tenantIds, payload.TenantID)
ids = append(ids, payload.ID)
insertedAts = append(insertedAts, payload.InsertedAt)
types = append(types, string(payload.Type))
locations = append(locations, string(V1PayloadLocationEXTERNAL))
externalLocationKeys = append(externalLocationKeys, string(payload.ExternalLocationKey))
}
row := tx.QueryRow(
ctx,
fmt.Sprintf(
// we unfortunately need to use `INSERT INTO` instead of `COPY` here
// because we can't have conflict resolution with `COPY`.
`
WITH inputs AS (
SELECT
UNNEST($1::UUID[]) AS tenant_id,
UNNEST($2::BIGINT[]) AS id,
UNNEST($3::TIMESTAMPTZ[]) AS inserted_at,
UNNEST($4::UUID[]) AS external_id,
UNNEST($5::TEXT[]) AS type,
UNNEST($6::TEXT[]) AS location,
UNNEST($7::TEXT[]) AS external_location_key
), inserts AS (
INSERT INTO %s (tenant_id, id, inserted_at, external_id, type, location, external_location_key, inline_content, updated_at)
SELECT
tenant_id,
id,
inserted_at,
external_id,
type::v1_payload_type,
location::v1_payload_location,
external_location_key,
NULL,
NOW()
FROM inputs
ORDER BY tenant_id, inserted_at, id, type
ON CONFLICT(tenant_id, id, inserted_at, type) DO NOTHING
RETURNING *
)
SELECT COUNT(*)
FROM inserts
`,
tableName,
),
tenantIds,
ids,
insertedAts,
externalIds,
types,
locations,
externalLocationKeys,
)
var copyCount int64
err := row.Scan(&copyCount)
return copyCount, err
}
func ComparePartitionRowCounts(ctx context.Context, tx DBTX, tempPartitionName, sourcePartitionName string) (bool, error) {
row := tx.QueryRow(
ctx,
fmt.Sprintf(
`
SELECT
(SELECT COUNT(*) FROM %s) AS temp_partition_count,
(SELECT COUNT(*) FROM %s) AS source_partition_count
`,
tempPartitionName,
sourcePartitionName,
),
)
var tempPartitionCount int64
var sourcePartitionCount int64
err := row.Scan(&tempPartitionCount, &sourcePartitionCount)
if err != nil {
return false, err
}
return tempPartitionCount == sourcePartitionCount, nil
}