Introduce tenant Prometheus metrics (#1875)

* introduce tenant workflow completed metric

* expose tenant prom metrics via handler

* fix workflow and worker id in metrics

* correctly add workflow metrics from workflow controller

* use olap DB to gather information for workflow completion

* fix prom metrics endpoint for tenant

* workflow name from external id

* simplify tenant registry based metrics

* add docs for prometheus metrics

* fix docs lint

* run prettier fix

* WIP metrics work

* use federate prom server URL to proxy metrics

* implement workflow duration histogram metric

* separate prom stack docker compose

* fix duration metrics calls

* move scheduler metrics to prom tenant specific file

* update docs for prom metrics

* fix lint

* use proper indices to query for durations

* reorg tenant metrics

* fix lint for doc

* update docs with promql examples and casing around prom metrics enabled

* update prom server url

* fix lint

* enabled prom metrics for v1 only from controller

api-contracts/openapi/openapi.yaml

@@ -220,3 +220,5 @@ paths:
     $ref: "./paths/monitoring/monitoring.yaml#/probe"
   /api/v1/version:
     $ref: "./paths/info/version.yaml#/version"
+  /api/v1/tenants/{tenant}/prometheus-metrics:
+    $ref: "./paths/tenant/tenant.yaml#/getPrometheusMetrics"

api-contracts/openapi/paths/tenant/tenant.yaml

@@ -707,3 +707,47 @@ getStepRunQueueMetrics:
     summary: Get step run metrics
     tags:
       - Tenant
+
+getPrometheusMetrics:
+  get:
+    x-resources: ["tenant"]
+    description: Get the prometheus metrics for the tenant
+    operationId: tenant:get:prometheus-metrics
+    parameters:
+      - description: The tenant id
+        in: path
+        name: tenant
+        required: true
+        schema:
+          type: string
+          format: uuid
+          minLength: 36
+          maxLength: 36
+    responses:
+      "200":
+        content:
+          text/plain:
+            schema:
+              type: string
+        description: Successfully retrieved the prometheus metrics
+      "400":
+        content:
+          application/json:
+            schema:
+              $ref: "../../components/schemas/_index.yaml#/APIErrors"
+        description: A malformed or bad request
+      "403":
+        content:
+          application/json:
+            schema:
+              $ref: "../../components/schemas/_index.yaml#/APIErrors"
+        description: Forbidden
+      "404":
+        content:
+          application/json:
+            schema:
+              $ref: "../../components/schemas/_index.yaml#/APIErrors"
+        description: Not found
+    summary: Get prometheus metrics
+    tags:
+      - Tenant