feat: limits (#559)

* feat: workflow run limits

* fix: resource exhausted 429

* feat: event limit

* feat: worker limit

* fix: sensible error

* fix: pb

* feat: expose limits api

* feat: default limits

* feat: add enable alert option

* feat: slack and email alerts

* fix: cron interval

* feat: make metered util

* wip: schedules and crons

* chore: squash migration

* fix: select or insert

* fix: remove unfinished meter

* chore: atlas migration

* fix: template format

* fix: shared ErrResourceExhausted

* feat: cache

* fix: limit can be nil

* fix: clarification

* fix: close meter ticker

* fix: friendly error for child workflows

api-contracts/openapi/components/schemas/_index.yaml

@@ -36,6 +36,12 @@ TenantMemberList:
   $ref: "./tenant.yaml#/TenantMemberList"
 TenantMemberRole:
   $ref: "./tenant.yaml#/TenantMemberRole"
+TenantResource:
+  $ref: "./tenant.yaml#/TenantResource"
+TenantResourceLimit:
+  $ref: "./tenant.yaml#/TenantResourceLimit"
+TenantResourcePolicy:
+  $ref: "./tenant.yaml#/TenantResourcePolicy"
 CreateTenantInviteRequest:
   $ref: "./tenant.yaml#/CreateTenantInviteRequest"
 UpdateTenantInviteRequest:

api-contracts/openapi/components/schemas/tenant.yaml

@@ -33,6 +33,9 @@ TenantAlertingSettings:
     enableExpiringTokenAlerts:
       type: boolean
       description: Whether to enable alerts when tokens are approaching expiration.
+    enableTenantResourceLimitAlerts:
+      type: boolean
+      description: Whether to enable alerts when tenant resources are approaching limits.
     maxAlertingFrequency:
       type: string
       description: The max frequency at which to alert.
@@ -79,6 +82,9 @@ UpdateTenantRequest:
     enableExpiringTokenAlerts:
       type: boolean
       description: Whether to enable alerts when tokens are approaching expiration.
+    enableTenantResourceLimitAlerts:
+      type: boolean
+      description: Whether to enable alerts when tenant resources are approaching limits.
     maxAlertingFrequency:
       type: string
       description: The max frequency at which to alert.
@@ -86,6 +92,57 @@ UpdateTenantRequest:
         validate: "omitnil,duration"
   type: object
 
+TenantResource:
+  enum:
+    - "WORKER"
+    - "EVENT"
+    - "WORKFLOW_RUN"
+    - "CRON"
+    - "SCHEDULE"
+  type: string
+
+TenantResourceLimit:
+  properties:
+    metadata:
+      $ref: "./metadata.yaml#/APIResourceMeta"
+    resource:
+      $ref: "#/TenantResource"
+      description: The resource associated with this limit.
+    limitValue:
+      type: integer
+      description: The limit associated with this limit.
+    alarmValue:
+      type: integer
+      description: The alarm value associated with this limit to warn of approaching limit value.
+    value:
+      type: integer
+      description: The current value associated with this limit.
+    window:
+      type: string
+      description: The meter window for the limit. (i.e. 1 day, 1 week, 1 month)
+    lastRefill:
+      type: string
+      description: The last time the limit was refilled.
+      format: date-time
+  required:
+    - metadata
+    - tenantId
+    - resource
+    - limitValue
+    - value
+  type: object
+
+TenantResourcePolicy:
+  properties:
+    limits:
+      type: array
+      items:
+        $ref: "#/TenantResourceLimit"
+      description: A list of resource limits for the tenant.
+  required:
+    - limits
+  type: object
+
 TenantMember:
   properties:
     metadata:

api-contracts/openapi/openapi.yaml

@@ -56,6 +56,8 @@ paths:
     $ref: "./paths/ingestors/ingestors.yaml#/snsIntegration"
   /api/v1/tenants/{tenant}/alerting-email-groups:
     $ref: "./paths/tenant/tenant.yaml#/tenantAlertEmailGroups"
+  /api/v1/tenants/{tenant}/resource-policy:
+    $ref: "./paths/tenant/tenant.yaml#/tenantResourcePolicy"
   /api/v1/alerting-email-groups/{alert-email-group}:
     $ref: "./paths/tenant/tenant.yaml#/alertEmailGroup"
   /api/v1/sns/{sns}:

api-contracts/openapi/paths/event/event.yaml

@@ -216,6 +216,12 @@ replayEvents:
             schema:
               $ref: "../../components/schemas/_index.yaml#/APIErrors"
         description: Forbidden
+      "429":
+        content:
+          application/json:
+            schema:
+              $ref: "../../components/schemas/_index.yaml#/APIErrors"
+        description: Resource limit exceeded
     summary: Replay events
     tags:
       - Event

api-contracts/openapi/paths/tenant/tenant.yaml

@@ -269,6 +269,45 @@ tenantAlertEmailGroups:
     summary: List tenant alert email groups
     tags:
       - Tenant
+
+tenantResourcePolicy:
+  get:
+    x-resources: ["tenant"]
+    description: Gets the resource policy for a tenant
+    operationId: tenant-resource-policy:get
+    parameters:
+      - description: The tenant id
+        in: path
+        name: tenant
+        required: true
+        schema:
+          type: string
+          format: uuid
+          minLength: 36
+          maxLength: 36
+    responses:
+      "200":
+        content:
+          application/json:
+            schema:
+              $ref: "../../components/schemas/_index.yaml#/TenantResourcePolicy"
+        description: Successfully retrieved the tenant resource policy
+      "400":
+        content:
+          application/json:
+            schema:
+              $ref: "../../components/schemas/_index.yaml#/APIErrors"
+        description: A malformed or bad request
+      "403":
+        content:
+          application/json:
+            schema:
+              $ref: "../../components/schemas/_index.yaml#/APIError"
+        description: Forbidden
+    summary: Create tenant alert email group
+    tags:
+      - Tenant
+
 invites:
   post:
     x-resources: ["tenant"]

api-contracts/openapi/paths/workflow/workflow.yaml

@@ -258,6 +258,12 @@ triggerWorkflow:
             schema:
               $ref: "../../components/schemas/_index.yaml#/APIErrors"
         description: A malformed or bad request
+      "429":
+        content:
+          application/json:
+            schema:
+              $ref: "../../components/schemas/_index.yaml#/APIErrors"
+        description: Resource limit exceeded
       "403":
         content:
           application/json: