feat(mlx-audio): Add mlx-audio backend (#6138)

* feat(mlx-audio): Add mlx-audio backend

Signed-off-by: Ettore Di Giacinto <[email protected]>

* improve loading

Signed-off-by: Ettore Di Giacinto <[email protected]>

* CI tests

Signed-off-by: Ettore Di Giacinto <[email protected]>

* fix: set C_INCLUDE_PATH to point to python install

Signed-off-by: Ettore Di Giacinto <[email protected]>

---------

Signed-off-by: Ettore Di Giacinto <[email protected]>

Author: mudler

.github/workflows/backend.yml

@@ -388,6 +388,10 @@ backends/mlx-vlm:
 	BACKEND=mlx-vlm $(MAKE) build-darwin-python-backend
 	./local-ai backends install "ocifile://$(abspath ./backend-images/mlx-vlm.tar)"
 
+backends/mlx-audio:
+	BACKEND=mlx-audio $(MAKE) build-darwin-python-backend
+	./local-ai backends install "ocifile://$(abspath ./backend-images/mlx-audio.tar)"
+
 backend-images:
 	mkdir -p backend-images
 

Makefile

@@ -159,6 +159,23 @@
     - vision-language
     - LLM
     - MLX
+- &mlx-audio
+  name: "mlx-audio"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-mlx-audio"
+  icon: https://avatars.githubusercontent.com/u/102832242?s=200&v=4
+  urls:
+    - https://github.com/Blaizzy/mlx-audio
+  mirrors:
+    - localai/localai-backends:latest-metal-darwin-arm64-mlx-audio
+  license: MIT
+  description: |
+      Run Audio Models with MLX
+  tags:
+    - audio-to-text
+    - audio-generation
+    - text-to-audio
+    - LLM
+    - MLX
 - &rerankers
   name: "rerankers"
   alias: "rerankers"
@@ -415,6 +432,11 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-mlx-vlm"
   mirrors:
     - localai/localai-backends:master-metal-darwin-arm64-mlx-vlm
+- !!merge <<: *mlx-audio
+  name: "mlx-audio-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-mlx-audio"
+  mirrors:
+    - localai/localai-backends:master-metal-darwin-arm64-mlx-audio
 - !!merge <<: *kitten-tts
   name: "kitten-tts-development"
   uri: "quay.io/go-skynet/local-ai-backends:master-kitten-tts"

backend/index.yaml

@@ -384,6 +384,11 @@ function installRequirements() {
         requirementFiles+=("${EDIR}/requirements-${BUILD_PROFILE}-after.txt")
     fi
 
+    # This is needed to build wheels that e.g. depends on Python.h
+    if [ "x${PORTABLE_PYTHON}" == "xtrue" ]; then
+        export C_INCLUDE_PATH="${C_INCLUDE_PATH:-}:$(_portable_dir)/include/python${PYTHON_VERSION}"
+    fi
+
     for reqFile in ${requirementFiles[@]}; do
         if [ -f "${reqFile}" ]; then
             echo "starting requirements install for ${reqFile}"

backend/python/common/libbackend.sh

@@ -0,0 +1,23 @@
+.PHONY: mlx-audio
+mlx-audio:
+	bash install.sh
+
+.PHONY: run
+run: mlx-audio
+	@echo "Running mlx-audio..."
+	bash run.sh
+	@echo "mlx run."
+
+.PHONY: test
+test: mlx-audio
+	@echo "Testing mlx-audio..."
+	bash test.sh
+	@echo "mlx tested."
+
+.PHONY: protogen-clean
+protogen-clean:
+	$(RM) backend_pb2_grpc.py backend_pb2.py
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf venv __pycache__