mirror of
https://github.com/hatchet-dev/hatchet.git
synced 2025-12-20 08:10:26 -06:00
* feat: initial payload cutover job * refactor: fix a couple things * feat: start wiring up writes * feat: only run job if external store is enabled * fix: add some notes, add loop * feat: function for reading out payloads * fix: date handling, logging * feat: remove wal and immediate offloads * feat: advisory lock * feat: partition swap logic * fix: rm debug * fix: add todo * fix: sql cleanup * fix: sql cleanup, ii * chore: nuke a bunch of WAL stuff * chore: more wal * feat: trigger for crud opts * feat: drop trigger + function in swapover * feat: move autovac to later * feat: use unlogged table initially * feat: update migration * fix: drop trigger * fix: use insert + on conflict * fix: types * refactor: clean up a bit * fix: panic * fix: detach partition before dropping * feat: configurable batch size * feat: offset tracking in the db * feat: explicitly lock * fix: down migration * fix: bug * fix: offset handling * fix: try explicit ordering of the insert * fix: lock location * fix: do less stuff after locking * fix: ordering * fix: dont drop and recreate if temp table exists * fix: explicitly track completed status * fix: table name * fix: dont use unlogged table * fix: rm todos * chore: lint * feat: configurable delay * fix: use date as pk instead of varchar * fix: daily job * fix: hack check constraint to speed up partition attach * fix: syntax * fix: syntax * fix: drop constraint after attaching * fix: syntax * fix: drop triggers properly * fix: factor out insert logic * refactor: factor out loop logic * refactor: factor out job preparation work * fix: ordering * fix: run the job more often * fix: use `WithSingletonMode` * fix: singleton mode sig * fix: env var cleanup * fix: overwrite sig * fix: re-enable immediate offloads with a flag * fix: order, offload at logic * feat: add count query to compare * fix: row-level triggers, partition time bug * fix: rm todo * fix: for true * fix: handle lock not acquired * fix: handle error * fix: comment
116 lines
3.0 KiB
Go
116 lines
3.0 KiB
Go
package sqlcv1
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
)
|
|
|
|
type CutoverPayloadToInsert struct {
|
|
TenantID pgtype.UUID
|
|
ID int64
|
|
InsertedAt pgtype.Timestamptz
|
|
ExternalID pgtype.UUID
|
|
Type V1PayloadType
|
|
ExternalLocationKey string
|
|
}
|
|
|
|
func InsertCutOverPayloadsIntoTempTable(ctx context.Context, tx DBTX, tableName string, payloads []CutoverPayloadToInsert) (int64, error) {
|
|
tenantIds := make([]pgtype.UUID, 0, len(payloads))
|
|
ids := make([]int64, 0, len(payloads))
|
|
insertedAts := make([]pgtype.Timestamptz, 0, len(payloads))
|
|
externalIds := make([]pgtype.UUID, 0, len(payloads))
|
|
types := make([]string, 0, len(payloads))
|
|
locations := make([]string, 0, len(payloads))
|
|
externalLocationKeys := make([]string, 0, len(payloads))
|
|
|
|
for _, payload := range payloads {
|
|
externalIds = append(externalIds, payload.ExternalID)
|
|
tenantIds = append(tenantIds, payload.TenantID)
|
|
ids = append(ids, payload.ID)
|
|
insertedAts = append(insertedAts, payload.InsertedAt)
|
|
types = append(types, string(payload.Type))
|
|
locations = append(locations, string(V1PayloadLocationEXTERNAL))
|
|
externalLocationKeys = append(externalLocationKeys, string(payload.ExternalLocationKey))
|
|
}
|
|
|
|
row := tx.QueryRow(
|
|
ctx,
|
|
fmt.Sprintf(
|
|
// we unfortunately need to use `INSERT INTO` instead of `COPY` here
|
|
// because we can't have conflict resolution with `COPY`.
|
|
`
|
|
WITH inputs AS (
|
|
SELECT
|
|
UNNEST($1::UUID[]) AS tenant_id,
|
|
UNNEST($2::BIGINT[]) AS id,
|
|
UNNEST($3::TIMESTAMPTZ[]) AS inserted_at,
|
|
UNNEST($4::UUID[]) AS external_id,
|
|
UNNEST($5::TEXT[]) AS type,
|
|
UNNEST($6::TEXT[]) AS location,
|
|
UNNEST($7::TEXT[]) AS external_location_key
|
|
), inserts AS (
|
|
INSERT INTO %s (tenant_id, id, inserted_at, external_id, type, location, external_location_key, inline_content, updated_at)
|
|
SELECT
|
|
tenant_id,
|
|
id,
|
|
inserted_at,
|
|
external_id,
|
|
type::v1_payload_type,
|
|
location::v1_payload_location,
|
|
external_location_key,
|
|
NULL,
|
|
NOW()
|
|
FROM inputs
|
|
ORDER BY tenant_id, inserted_at, id, type
|
|
ON CONFLICT(tenant_id, id, inserted_at, type) DO NOTHING
|
|
RETURNING *
|
|
)
|
|
|
|
SELECT COUNT(*)
|
|
FROM inserts
|
|
`,
|
|
tableName,
|
|
),
|
|
tenantIds,
|
|
ids,
|
|
insertedAts,
|
|
externalIds,
|
|
types,
|
|
locations,
|
|
externalLocationKeys,
|
|
)
|
|
|
|
var copyCount int64
|
|
err := row.Scan(©Count)
|
|
|
|
return copyCount, err
|
|
}
|
|
|
|
func ComparePartitionRowCounts(ctx context.Context, tx DBTX, tempPartitionName, sourcePartitionName string) (bool, error) {
|
|
row := tx.QueryRow(
|
|
ctx,
|
|
fmt.Sprintf(
|
|
`
|
|
SELECT
|
|
(SELECT COUNT(*) FROM %s) AS temp_partition_count,
|
|
(SELECT COUNT(*) FROM %s) AS source_partition_count
|
|
`,
|
|
tempPartitionName,
|
|
sourcePartitionName,
|
|
),
|
|
)
|
|
|
|
var tempPartitionCount int64
|
|
var sourcePartitionCount int64
|
|
|
|
err := row.Scan(&tempPartitionCount, &sourcePartitionCount)
|
|
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
return tempPartitionCount == sourcePartitionCount, nil
|
|
}
|