feat(neutts): add backend (#6404)

* feat(neutts): add backend

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* chore(ci): add images to CI

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* chore(gallery): add Neutts

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Make it work with quantized versions

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Fixups

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Docs

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Fixups

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Apply suggestion from @mudler

Signed-off-by: Ettore Di Giacinto <mudler@users.noreply.github.com>

* Apply suggestion from @mudler

Signed-off-by: Ettore Di Giacinto <mudler@users.noreply.github.com>

* Apply suggestion from @mudler

Signed-off-by: Ettore Di Giacinto <mudler@users.noreply.github.com>

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Signed-off-by: Ettore Di Giacinto <mudler@users.noreply.github.com>

backend/python/neutts/Makefile

@@ -0,0 +1,23 @@
+.PHONY: neutts
+neutts:
+	bash install.sh
+
+.PHONY: run
+run: neutts
+	@echo "Running neutts..."
+	bash run.sh
+	@echo "neutts run."
+
+.PHONY: test
+test: neutts
+	@echo "Testing neutts..."
+	bash test.sh
+	@echo "neutts tested."
+
+.PHONY: protogen-clean
+protogen-clean:
+	$(RM) backend_pb2_grpc.py backend_pb2.py
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf venv __pycache__

backend/python/neutts/backend.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+"""
+This is an extra gRPC server of LocalAI for NeuTTSAir
+"""
+from concurrent import futures
+import time
+import argparse
+import signal
+import sys
+import os
+import backend_pb2
+import backend_pb2_grpc
+import torch
+from neuttsair.neutts import NeuTTSAir
+import soundfile as sf
+
+import grpc
+
+def is_float(s):
+    """Check if a string can be converted to float."""
+    try:
+        float(s)
+        return True
+    except ValueError:
+        return False
+def is_int(s):
+    """Check if a string can be converted to int."""
+    try:
+        int(s)
+        return True
+    except ValueError:
+        return False
+
+_ONE_DAY_IN_SECONDS = 60 * 60 * 24
+
+# If MAX_WORKERS are specified in the environment use it, otherwise default to 1
+MAX_WORKERS = int(os.environ.get('PYTHON_GRPC_MAX_WORKERS', '1'))
+
+# Implement the BackendServicer class with the service methods
+class BackendServicer(backend_pb2_grpc.BackendServicer):
+    """
+    BackendServicer is the class that implements the gRPC service
+    """
+    def Health(self, request, context):
+        return backend_pb2.Reply(message=bytes("OK", 'utf-8'))
+    def LoadModel(self, request, context):
+
+        # Get device
+        # device = "cuda" if request.CUDA else "cpu"
+        if torch.cuda.is_available():
+            print("CUDA is available", file=sys.stderr)
+            device = "cuda"
+        else:
+            print("CUDA is not available", file=sys.stderr)
+            device = "cpu"
+        mps_available = hasattr(torch.backends, "mps") and torch.backends.mps.is_available()
+        if mps_available:
+            device = "mps"
+        if not torch.cuda.is_available() and request.CUDA:
+            return backend_pb2.Result(success=False, message="CUDA is not available")
+
+
+        options = request.Options
+
+        # empty dict
+        self.options = {}
+        self.ref_text = None
+
+        # The options are a list of strings in this form optname:optvalue
+        # We are storing all the options in a dict so we can use it later when
+        # generating the images
+        for opt in options:
+            if ":" not in opt:
+                continue
+            key, value = opt.split(":")
+            # if value is a number, convert it to the appropriate type
+            if is_float(value):
+                value = float(value)
+            elif is_int(value):
+                value = int(value)
+            elif value.lower() in ["true", "false"]:
+                value = value.lower() == "true"
+            self.options[key] = value
+
+        codec_repo = "neuphonic/neucodec"
+        if "codec_repo" in self.options:
+            codec_repo = self.options["codec_repo"]
+            del self.options["codec_repo"]
+        if "ref_text" in self.options:
+            self.ref_text = self.options["ref_text"]
+            del self.options["ref_text"]
+
+        self.AudioPath = None
+
+        if os.path.isabs(request.AudioPath):
+            self.AudioPath = request.AudioPath
+        elif request.AudioPath and request.ModelFile != "" and not os.path.isabs(request.AudioPath):
+            # get base path of modelFile
+            modelFileBase = os.path.dirname(request.ModelFile)
+            # modify LoraAdapter to be relative to modelFileBase
+            self.AudioPath = os.path.join(modelFileBase, request.AudioPath)
+        try:
+            print("Preparing models, please wait", file=sys.stderr)
+            self.model = NeuTTSAir(backbone_repo=request.Model, backbone_device=device, codec_repo=codec_repo, codec_device=device)
+        except Exception as err:
+            return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
+        # Implement your logic here for the LoadModel service
+        # Replace this with your desired response
+        return backend_pb2.Result(message="Model loaded successfully", success=True)
+
+    def TTS(self, request, context):
+        try:
+            kwargs = {}
+
+            # add options to kwargs
+            kwargs.update(self.options)
+
+            ref_codes = self.model.encode_reference(self.AudioPath)
+
+            wav = self.model.infer(request.text, ref_codes, self.ref_text)
+
+            sf.write(request.dst, wav, 24000)            
+        except Exception as err:
+            return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
+        return backend_pb2.Result(success=True)
+
+def serve(address):
+    server = grpc.server(futures.ThreadPoolExecutor(max_workers=MAX_WORKERS),
+        options=[
+            ('grpc.max_message_length', 50 * 1024 * 1024),  # 50MB
+            ('grpc.max_send_message_length', 50 * 1024 * 1024),  # 50MB
+            ('grpc.max_receive_message_length', 50 * 1024 * 1024),  # 50MB
+        ])
+    backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
+    server.add_insecure_port(address)
+    server.start()
+    print("Server started. Listening on: " + address, file=sys.stderr)
+
+    # Define the signal handler function
+    def signal_handler(sig, frame):
+        print("Received termination signal. Shutting down...")
+        server.stop(0)
+        sys.exit(0)
+
+    # Set the signal handlers for SIGINT and SIGTERM
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTERM, signal_handler)
+
+    try:
+        while True:
+            time.sleep(_ONE_DAY_IN_SECONDS)
+    except KeyboardInterrupt:
+        server.stop(0)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run the gRPC server.")
+    parser.add_argument(
+        "--addr", default="localhost:50051", help="The address to bind the server to."
+    )
+    args = parser.parse_args()
+
+    serve(args.addr)

backend/python/neutts/install.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -e
+
+backend_dir=$(dirname $0)
+if [ -d $backend_dir/common ]; then
+    source $backend_dir/common/libbackend.sh
+else
+    source $backend_dir/../common/libbackend.sh
+fi
+
+# This is here because the Intel pip index is broken and returns 200 status codes for every package name, it just doesn't return any package links.
+# This makes uv think that the package exists in the Intel pip index, and by default it stops looking at other pip indexes once it finds a match.
+# We need uv to continue falling through to the pypi default index to find optimum[openvino] in the pypi index
+# the --upgrade actually allows us to *downgrade* torch to the version provided in the Intel pip index
+if [ "x${BUILD_PROFILE}" == "xintel" ]; then
+    EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match"
+fi
+
+if [ "x${BUILD_TYPE}" == "xcublas" ] || [ "x${BUILD_TYPE}" == "xl4t" ]; then
+    export CMAKE_ARGS="-DGGML_CUDA=on"
+fi
+
+if [ "x${BUILD_TYPE}" == "xhipblas" ]; then
+    export CMAKE_ARGS="-DGGML_HIPBLAS=on"
+fi
+
+EXTRA_PIP_INSTALL_FLAGS+=" --no-build-isolation"
+
+git clone https://github.com/neuphonic/neutts-air neutts-air
+
+cp -rfv neutts-air/neuttsair ./
+
+installRequirements

backend/python/neutts/requirements-after.txt

@@ -0,0 +1,2 @@
+datasets==4.1.1
+torchtune==0.6.1

backend/python/neutts/requirements-cpu.txt

@@ -0,0 +1,10 @@
+--extra-index-url https://download.pytorch.org/whl/cpu
+accelerate
+torch==2.8.0
+transformers==4.56.1
+librosa==0.11.0
+neucodec>=0.0.4
+phonemizer==3.3.0
+soundfile==0.13.1
+resemble-perth==1.0.1
+llama-cpp-python

backend/python/neutts/requirements-cublas12.txt

@@ -0,0 +1,8 @@
+librosa==0.11.0
+neucodec>=0.0.4
+phonemizer==3.3.0
+soundfile==0.13.1
+torch==2.8.0
+transformers==4.56.1
+resemble-perth==1.0.1
+accelerate

backend/python/neutts/requirements-hipblas.txt

@@ -0,0 +1,10 @@
+--extra-index-url https://download.pytorch.org/whl/rocm6.3
+torch==2.8.0+rocm6.3
+transformers==4.56.1
+accelerate
+librosa==0.11.0
+neucodec>=0.0.4
+phonemizer==3.3.0
+soundfile==0.13.1
+resemble-perth==1.0.1
+llama-cpp-python

backend/python/neutts/requirements-l4t.txt

@@ -0,0 +1,10 @@
+--extra-index-url https://pypi.jetson-ai-lab.io/jp6/cu126/
+torch
+transformers
+accelerate
+librosa==0.11.0
+neucodec>=0.0.4
+phonemizer==3.3.0
+soundfile==0.13.1
+resemble-perth==1.0.1
+llama-cpp-python

backend/python/neutts/requirements.txt

@@ -0,0 +1,7 @@
+grpcio==1.71.0
+protobuf
+certifi
+packaging
+setuptools
+numpy==2.2.6
+scikit_build_core

backend/python/neutts/run.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+backend_dir=$(dirname $0)
+if [ -d $backend_dir/common ]; then
+    source $backend_dir/common/libbackend.sh
+else
+    source $backend_dir/../common/libbackend.sh
+fi
+
+
+startBackend $@

backend/python/neutts/test.py

@@ -0,0 +1,82 @@
+"""
+A test script to test the gRPC service
+"""
+import unittest
+import subprocess
+import time
+import backend_pb2
+import backend_pb2_grpc
+
+import grpc
+
+
+class TestBackendServicer(unittest.TestCase):
+    """
+    TestBackendServicer is the class that tests the gRPC service
+    """
+    def setUp(self):
+        """
+        This method sets up the gRPC service by starting the server
+        """
+        self.service = subprocess.Popen(["python3", "backend.py", "--addr", "localhost:50051"])
+        time.sleep(30)
+
+    def tearDown(self) -> None:
+        """
+        This method tears down the gRPC service by terminating the server
+        """
+        self.service.terminate()
+        self.service.wait()
+
+    def test_server_startup(self):
+        """
+        This method tests if the server starts up successfully
+        """
+        try:
+            self.setUp()
+            with grpc.insecure_channel("localhost:50051") as channel:
+                stub = backend_pb2_grpc.BackendStub(channel)
+                response = stub.Health(backend_pb2.HealthMessage())
+                self.assertEqual(response.message, b'OK')
+        except Exception as err:
+            print(err)
+            self.fail("Server failed to start")
+        finally:
+            self.tearDown()
+
+    def test_load_model(self):
+        """
+        This method tests if the model is loaded successfully
+        """
+        try:
+            self.setUp()
+            with grpc.insecure_channel("localhost:50051") as channel:
+                stub = backend_pb2_grpc.BackendStub(channel)
+                response = stub.LoadModel(backend_pb2.ModelOptions())
+                print(response)
+                self.assertTrue(response.success)
+                self.assertEqual(response.message, "Model loaded successfully")
+        except Exception as err:
+            print(err)
+            self.fail("LoadModel service failed")
+        finally:
+            self.tearDown()
+
+    def test_tts(self):
+        """
+        This method tests if the embeddings are generated successfully
+        """
+        try:
+            self.setUp()
+            with grpc.insecure_channel("localhost:50051") as channel:
+                stub = backend_pb2_grpc.BackendStub(channel)
+                response = stub.LoadModel(backend_pb2.ModelOptions())
+                self.assertTrue(response.success)
+                tts_request = backend_pb2.TTSRequest(text="80s TV news production music hit for tonight's biggest story")
+                tts_response = stub.TTS(tts_request)
+                self.assertIsNotNone(tts_response)
+        except Exception as err:
+            print(err)
+            self.fail("TTS service failed")
+        finally:
+            self.tearDown()

backend/python/neutts/test.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+backend_dir=$(dirname $0)
+if [ -d $backend_dir/common ]; then
+    source $backend_dir/common/libbackend.sh
+else
+    source $backend_dir/../common/libbackend.sh
+fi
+
+runUnittests