From 13402ef940ce7b3a02cf6be65e98a1a27b7d55ba Mon Sep 17 00:00:00 2001
From: matt <mrkaye97@gmail.com>
Date: Fri, 14 Nov 2025 13:49:40 -0500
Subject: [PATCH] Feat: REST API Instrumentation (#2529)

* feat: instrument the api

* fix: fmt

* fix: paths

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
---
 .../server/middleware/telemetry/telemetry.go  | 32 +++++++++++++++++++
 api/v1/server/run/run.go                      |  3 ++
 go.mod                                        |  1 +
 go.sum                                        |  2 ++
 4 files changed, 38 insertions(+)
 create mode 100644 api/v1/server/middleware/telemetry/telemetry.go

diff --git a/api/v1/server/middleware/telemetry/telemetry.go b/api/v1/server/middleware/telemetry/telemetry.go
new file mode 100644
index 000000000..bc5b640fb
--- /dev/null
+++ b/api/v1/server/middleware/telemetry/telemetry.go
@@ -0,0 +1,32 @@
+package telemetry
+
+import (
+	"github.com/labstack/echo/v4"
+	"go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho"
+	"go.opentelemetry.io/otel"
+
+	"github.com/hatchet-dev/hatchet/pkg/config/server"
+)
+
+type OTelMiddleware struct {
+	config *server.ServerConfig
+}
+
+func NewOTelMiddleware(config *server.ServerConfig) *OTelMiddleware {
+	return &OTelMiddleware{
+		config: config,
+	}
+}
+
+func (m *OTelMiddleware) Middleware() echo.MiddlewareFunc {
+	serviceName := m.config.OpenTelemetry.ServiceName
+	tracerProvider := otel.GetTracerProvider()
+
+	return otelecho.Middleware(serviceName,
+		otelecho.WithSkipper(func(c echo.Context) bool {
+			path := c.Path()
+			return path == "/api/ready" || path == "/api/live"
+		}),
+		otelecho.WithTracerProvider(tracerProvider),
+	)
+}
diff --git a/api/v1/server/run/run.go b/api/v1/server/run/run.go
index f782ce53c..54c204922 100644
--- a/api/v1/server/run/run.go
+++ b/api/v1/server/run/run.go
@@ -43,6 +43,7 @@ import (
 	hatchetmiddleware "github.com/hatchet-dev/hatchet/api/v1/server/middleware"
 	"github.com/hatchet-dev/hatchet/api/v1/server/middleware/populator"
 	"github.com/hatchet-dev/hatchet/api/v1/server/middleware/ratelimit"
+	"github.com/hatchet-dev/hatchet/api/v1/server/middleware/telemetry"
 	"github.com/hatchet-dev/hatchet/api/v1/server/oas/gen"
 	"github.com/hatchet-dev/hatchet/pkg/config/server"
 	"github.com/hatchet-dev/hatchet/pkg/repository/postgres/dbsqlc"
@@ -563,12 +564,14 @@ func (t *APIServer) registerSpec(g *echo.Group, spec *openapi3.T) (*populator.Po
 	})
 
 	rateLimitMW := ratelimit.NewRateLimitMiddleware(t.config, spec)
+	otelMW := telemetry.NewOTelMiddleware(t.config)
 
 	// register echo middleware
 	g.Use(
 		loggerMiddleware,
 		middleware.Recover(),
 		rateLimitMW.Middleware(),
+		otelMW.Middleware(),
 		allHatchetMiddleware,
 	)
 
diff --git a/go.mod b/go.mod
index 147dcaa00..4f915a3c2 100644
--- a/go.mod
+++ b/go.mod
@@ -154,6 +154,7 @@ require (
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.63.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect
 	go.opentelemetry.io/otel/metric v1.38.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.7.1 // indirect
diff --git a/go.sum b/go.sum
index 28000e4b8..76a08f203 100644
--- a/go.sum
+++ b/go.sum
@@ -403,6 +403,8 @@ github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
 go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.63.0 h1:6YeICKmGrvgJ5th4+OMNpcuoB6q/Xs8gt0YCO7MUv1k=
+go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.63.0/go.mod h1:ZEA7j2B35siNV0T00aapacNzjz4tvOlNoHp0ncCfwNQ=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 h1:Hf9xI/XLML9ElpiHVDNwvqI0hIFlzV8dgIr35kV1kRU=