1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161

using UnityEngine;
using UnityEngine.Networking;
using System.Collections;
using System.Text;
using Whisper;

// Classi per gestire il JSON di LM Studio
[System.Serializable]
public class LMRequest
{
    public LMMessage[] messages;
    public float temperature = 0.7f;
    public int max_tokens = 200;
}
[System.Serializable]
public class LMMessage { public string role; public string content; }
[System.Serializable]
public class LMResponse { public LMChoice[] choices; }
[System.Serializable]
public class LMChoice { public LMMessage message; }

public class LocalAvatarController : MonoBehaviour
{
    [Header(" --- Unity Connections --- ")]
    public WhisperManager whisperManager;
    public AudioSource mouthAudioSource;
    public Animator avatarAnimator;

    [Header(" --- AI Prompt (Multi) --- ")]
    [TextArea(3, 10)]
    public string systemPrompt = "Sei un assistente virtuale. Rispondi in italiano in modo breve.";
    
    // LM Studio API
    private string lmStudioUrl = "http://localhost:1234/v1/chat/completions";

    // Piper TTS Server (Voice)
    private string piperServerUrl = "http://localhost:5000/"; 

    // Private Variables
    private AudioClip _recordingClip;
    private string _micDevice;
    private bool _isRecording = false;

    void Start()
    {
        if (Microphone.devices.Length > 0)
        {
            _micDevice = Microphone.devices[0];
            Debug.Log($"Microphone active: {_micDevice}");
        }
        else Debug.LogError("No microphone found!");
    }

    void Update()
    
        // Press SPACE to talk
        if (Input.GetKeyDown(KeyCode.Space) && !_isRecording) StartRecording();
        if (Input.GetKeyUp(KeyCode.Space) && _isRecording) StopRecordingAndProcess();

        // Talking Animation
        if (avatarAnimator != null)
        {
            avatarAnimator.SetBool("Talk", mouthAudioSource.isPlaying);
        }
    }

    // 1. Listen (Whisper)
    void StartRecording()
    {
        _isRecording = true;
        _recordingClip = Microphone.Start(_micDevice, false, 30, 16000);
        Debug.Log(" Listening...");
    }

    async void StopRecordingAndProcess()
    {
        _isRecording = false;
        Microphone.End(_micDevice);
        
        var result = await whisperManager.GetTextAsync(_recordingClip);
        string userText = result.Result;

        if (string.IsNullOrWhiteSpace(userText)) return;

        Debug.Log($"User: {userText}");
        StartCoroutine(AskLMStudio(userText));
    }

    // 2. Thinking (LM Studio)
    IEnumerator AskLMStudio(string prompt)
    {
        Debug.Log("Thinking...");

        var msgs = new LMMessage[] {
            new LMMessage { role = "system", content = systemPrompt },
            new LMMessage { role = "user", content = prompt }
        };
        
        string json = JsonUtility.ToJson(new LMRequest { messages = msgs });

        using (UnityWebRequest req = new UnityWebRequest(lmStudioUrl, "POST"))
        {
            byte[] bodyRaw = Encoding.UTF8.GetBytes(json);
            req.uploadHandler = new UploadHandlerRaw(bodyRaw);
            req.downloadHandler = new DownloadHandlerBuffer();
            req.SetRequestHeader("Content-Type", "application/json");

            yield return req.SendWebRequest();

            if (req.result == UnityWebRequest.Result.Success)
            {
                LMResponse response = JsonUtility.FromJson<LMResponse>(req.downloadHandler.text);
                string aiText = response.choices[0].message.content;
                Debug.Log($"🤖 AI: {aiText}");
                
                // Chiamata al server Piper
                StartCoroutine(GetAudioFromPiper(aiText));
            }
            else Debug.LogError($"ERROR LM Studio: {req.error}");
        }
    }

    //  3. Voice (Piper HTTP Server)
    IEnumerator GetAudioFromPiper(string text)
    {
        Debug.Log("Generate audio (Piper)...");

        // Clean Text to assure json is ok
        string cleanText = text.Replace("\"", "'").Replace("\n", " ");
        
        // JSON Creation
        string jsonPayload = "{\"text\": \"" + cleanText + "\"}";

        using (UnityWebRequest req = new UnityWebRequest(piperServerUrl, "POST"))
        {
            byte[] bodyRaw = Encoding.UTF8.GetBytes(jsonPayload);
            req.uploadHandler = new UploadHandlerRaw(bodyRaw);
            
            // Unity download WAV file directly
            req.downloadHandler = new DownloadHandlerAudioClip(piperServerUrl, AudioType.WAV);
            req.SetRequestHeader("Content-Type", "application/json");

            yield return req.SendWebRequest();

            if (req.result == UnityWebRequest.Result.Success)
            {
                AudioClip clip = DownloadHandlerAudioClip.GetContent(req);
                if (clip != null)
                {
                    mouthAudioSource.clip = clip;
                    mouthAudioSource.Play();
                }
                else Debug.LogError("Recived Empty Audio!");
            }
            else
            {
                Debug.LogError($"ERROR Piper Server: {req.error} - {req.downloadHandler.text}");
            }
        }
    }
}