feat(rfdetr): add object detection API (#5923)

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

pkg/grpc/backend.go

@@ -9,7 +9,7 @@ import (
 
 var embeds = map[string]*embedBackend{}
 
-func Provide(addr string, llm LLM) {
+func Provide(addr string, llm AIModel) {
 	embeds[addr] = &embedBackend{s: &server{llm: llm}}
 }
 
@@ -42,6 +42,7 @@ type Backend interface {
 	GenerateVideo(ctx context.Context, in *pb.GenerateVideoRequest, opts ...grpc.CallOption) (*pb.Result, error)
 	TTS(ctx context.Context, in *pb.TTSRequest, opts ...grpc.CallOption) (*pb.Result, error)
 	SoundGeneration(ctx context.Context, in *pb.SoundGenerationRequest, opts ...grpc.CallOption) (*pb.Result, error)
+	Detect(ctx context.Context, in *pb.DetectOptions, opts ...grpc.CallOption) (*pb.DetectResponse, error)
 	AudioTranscription(ctx context.Context, in *pb.TranscriptRequest, opts ...grpc.CallOption) (*pb.TranscriptResult, error)
 	TokenizeString(ctx context.Context, in *pb.PredictOptions, opts ...grpc.CallOption) (*pb.TokenizationResponse, error)
 	Status(ctx context.Context) (*pb.StatusResponse, error)

pkg/grpc/base/base.go

@@ -69,6 +69,10 @@ func (llm *Base) SoundGeneration(*pb.SoundGenerationRequest) error {
 	return fmt.Errorf("unimplemented")
 }
 
+func (llm *Base) Detect(*pb.DetectOptions) (pb.DetectResponse, error) {
+	return pb.DetectResponse{}, fmt.Errorf("unimplemented")
+}
+
 func (llm *Base) TokenizeString(opts *pb.PredictOptions) (pb.TokenizationResponse, error) {
 	return pb.TokenizationResponse{}, fmt.Errorf("unimplemented")
 }

pkg/grpc/client.go

@@ -504,3 +504,25 @@ func (c *Client) VAD(ctx context.Context, in *pb.VADRequest, opts ...grpc.CallOp
 	client := pb.NewBackendClient(conn)
 	return client.VAD(ctx, in, opts...)
 }
+
+func (c *Client) Detect(ctx context.Context, in *pb.DetectOptions, opts ...grpc.CallOption) (*pb.DetectResponse, error) {
+	if !c.parallel {
+		c.opMutex.Lock()
+		defer c.opMutex.Unlock()
+	}
+	c.setBusy(true)
+	defer c.setBusy(false)
+	c.wdMark()
+	defer c.wdUnMark()
+	conn, err := grpc.Dial(c.address, grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithDefaultCallOptions(
+			grpc.MaxCallRecvMsgSize(50*1024*1024), // 50MB
+			grpc.MaxCallSendMsgSize(50*1024*1024), // 50MB
+		))
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+	client := pb.NewBackendClient(conn)
+	return client.Detect(ctx, in, opts...)
+}

pkg/grpc/embed.go

@@ -59,6 +59,10 @@ func (e *embedBackend) SoundGeneration(ctx context.Context, in *pb.SoundGenerati
 	return e.s.SoundGeneration(ctx, in)
 }
 
+func (e *embedBackend) Detect(ctx context.Context, in *pb.DetectOptions, opts ...grpc.CallOption) (*pb.DetectResponse, error) {
+	return e.s.Detect(ctx, in)
+}
+
 func (e *embedBackend) AudioTranscription(ctx context.Context, in *pb.TranscriptRequest, opts ...grpc.CallOption) (*pb.TranscriptResult, error) {
 	return e.s.AudioTranscription(ctx, in)
 }

pkg/grpc/interface.go

@@ -4,7 +4,7 @@ import (
 	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
 )
 
-type LLM interface {
+type AIModel interface {
 	Busy() bool
 	Lock()
 	Unlock()
@@ -15,6 +15,7 @@ type LLM interface {
 	Embeddings(*pb.PredictOptions) ([]float32, error)
 	GenerateImage(*pb.GenerateImageRequest) error
 	GenerateVideo(*pb.GenerateVideoRequest) error
+	Detect(*pb.DetectOptions) (pb.DetectResponse, error)
 	AudioTranscription(*pb.TranscriptRequest) (pb.TranscriptResult, error)
 	TTS(*pb.TTSRequest) error
 	SoundGeneration(*pb.SoundGenerationRequest) error

pkg/grpc/server.go

@@ -22,7 +22,7 @@ import (
 // server is used to implement helloworld.GreeterServer.
 type server struct {
 	pb.UnimplementedBackendServer
-	llm LLM
+	llm AIModel
 }
 
 func (s *server) Health(ctx context.Context, in *pb.HealthMessage) (*pb.Reply, error) {
@@ -111,6 +111,18 @@ func (s *server) SoundGeneration(ctx context.Context, in *pb.SoundGenerationRequ
 	return &pb.Result{Message: "Sound Generation audio generated", Success: true}, nil
 }
 
+func (s *server) Detect(ctx context.Context, in *pb.DetectOptions) (*pb.DetectResponse, error) {
+	if s.llm.Locking() {
+		s.llm.Lock()
+		defer s.llm.Unlock()
+	}
+	res, err := s.llm.Detect(in)
+	if err != nil {
+		return nil, err
+	}
+	return &res, nil
+}
+
 func (s *server) AudioTranscription(ctx context.Context, in *pb.TranscriptRequest) (*pb.TranscriptResult, error) {
 	if s.llm.Locking() {
 		s.llm.Lock()
@@ -251,7 +263,7 @@ func (s *server) VAD(ctx context.Context, in *pb.VADRequest) (*pb.VADResponse, e
 	return &res, nil
 }
 
-func StartServer(address string, model LLM) error {
+func StartServer(address string, model AIModel) error {
 	lis, err := net.Listen("tcp", address)
 	if err != nil {
 		return err
@@ -269,7 +281,7 @@ func StartServer(address string, model LLM) error {
 	return nil
 }
 
-func RunServer(address string, model LLM) (func() error, error) {
+func RunServer(address string, model AIModel) (func() error, error) {
 	lis, err := net.Listen("tcp", address)
 	if err != nil {
 		return nil, err

pkg/utils/base64.go

@@ -20,7 +20,7 @@ var dataURIPattern = regexp.MustCompile(`^data:([^;]+);base64,`)
 
 // GetContentURIAsBase64 checks if the string is an URL, if it's an URL downloads the content in memory encodes it in base64 and returns the base64 string, otherwise returns the string by stripping base64 data headers
 func GetContentURIAsBase64(s string) (string, error) {
-	if strings.HasPrefix(s, "http") {
+	if strings.HasPrefix(s, "http") || strings.HasPrefix(s, "https") {
 		// download the image
 		resp, err := base64DownloadClient.Get(s)
 		if err != nil {