Move inputs to model's same device, previously hardcoded to CUDA

2026-05-03 18:50:37 -05:00 · 2025-08-03 13:02:15 +02:00
parent 5b5f17af8e
commit 9781810451
1 changed files with 1 additions and 2 deletions
@@ -141,8 +141,7 @@ class HuggingFaceLocalAdapter(CustomLLM):
        )
        
        # Move inputs to the same device as model
-        if torch.cuda.is_available() and self.device != "cpu":
-            inputs = inputs.to("cuda")
+        inputs = inputs.to(model.device)
        
        # Generate response
        with torch.no_grad():