import React, { useState, useEffect, useRef, useCallback } from 'react'; import { X, MessageSquare, Mic, Send, Loader2, AudioWaveform, StopCircle, Radio, Cpu, AlertCircle, RefreshCw, ShieldCheck, GripVertical, MapPin, Navigation, Sparkles, Globe } from 'lucide-react'; import { Logo } from './ui/Logo'; import { StorageService } from '../services/storageService'; import { connectToGuruLive, encodeAudio, decodeAudio, decodeAudioData, requestMicPermission, translateText } from '../services/geminiService'; import { ChatMessage, TaskStatus } from '../types'; import { Modality, LiveServerMessage, Blob, GoogleGenAI } from '@google/genai'; import { useNavigate } from 'react-router-dom'; import { GET_ROBOTIC_SYSTEM_INSTRUCTION } from '../ai-voice-model/ai'; import { AI_LANGUAGES } from '../ai-voice-model/languages'; import { ALL_RUDRA_TOOLS, executeRudraTool } from '../ai-voice-model/tool-registry'; import { useLanguage } from '../contexts/LanguageContext'; const RudraAI: React.FC = () => { const navigate = useNavigate(); const { t, language } = useLanguage(); const [isOpen, setIsOpen] = useState(false); const [mode, setMode] = useState<'chat' | 'voice'>('voice'); const [messages, setMessages] = useState([]); const [input, setInput] = useState(''); const [isTyping, setIsTyping] = useState(false); const [isTranslating, setIsTranslating] = useState(false); // Position & Drag State - Responsive defaults const [pos, setPos] = useState({ x: window.innerWidth - 80, y: window.innerHeight - 140 }); const [dragging, setDragging] = useState(false); const dragOffset = useRef({ x: 0, y: 0 }); const dragStartPos = useRef({ x: 0, y: 0 }); const hasMovedRef = useRef(false); // Voice Session State const [hasMicPermission, setHasMicPermission] = useState(null); const hasMicPermissionRef = useRef(null); const [isLiveActive, setIsLiveActive] = useState(false); const [liveStatus, setLiveStatus] = useState<'Idle' | 'Connecting' | 'Listening' | 'Speaking' | 'Error'>('Idle'); const [errorMsg, setErrorMsg] = useState(null); const [userTranscript, setUserTranscript] = useState(''); const [aiTranscript, setAiTranscript] = useState(''); // Navigation Overlay State const [navOverlay, setNavOverlay] = useState<{destination: string, mode: string} | null>(null); // Refs const liveSessionPromiseRef = useRef | null>(null); const inputAudioContextRef = useRef(null); const outputAudioContextRef = useRef(null); const nextStartTimeRef = useRef(0); const audioSourcesRef = useRef>(new Set()); const micStreamRef = useRef(null); const scrollRef = useRef(null); // Background Wake Word Engine const wakeWordRecognitionRef = useRef(null); const isMicLockedRef = useRef(false); // State Refs for Event Handlers const isLiveActiveRef = useRef(isLiveActive); useEffect(() => { isLiveActiveRef.current = isLiveActive; }, [isLiveActive]); useEffect(() => { hasMicPermissionRef.current = hasMicPermission; }, [hasMicPermission]); // Load Persistence useEffect(() => { const history = StorageService.getGlobalChatHistory(); if (history && history.length > 0) { setMessages(history); } }, []); // Save Persistence useEffect(() => { if (messages.length > 0) { StorageService.saveGlobalChatHistory(messages); } if (scrollRef.current) scrollRef.current.scrollTop = scrollRef.current.scrollHeight; }, [messages, isTyping, isTranslating]); // Auto-Translate Logic on Language Toggle useEffect(() => { const handleLanguageSwitch = async () => { if (messages.length === 0 || isTyping || isLiveActive) return; const lastMsg = messages[messages.length - 1]; // Only translate if the last message was from the bot and isn't a system action if (lastMsg.role === 'model' && !lastMsg.text.startsWith('[SYSTEM')) { setIsTranslating(true); try { const translated = await translateText(lastMsg.text, language); setMessages(prev => { const newMsgs = [...prev]; newMsgs[newMsgs.length - 1] = { ...lastMsg, text: translated }; return newMsgs; }); } catch (e) { console.error("Translation failed", e); } finally { setIsTranslating(false); } } }; handleLanguageSwitch(); }, [language]); useEffect(() => { // Initial permission check against App Settings const checkPermissions = async () => { const settings = await StorageService.getSettings(); if (settings.permissions.microphone) { if (navigator.permissions && (navigator.permissions as any).query) { (navigator.permissions as any).query({ name: 'microphone' }).then((result: any) => { if (result.state === 'granted') { setHasMicPermission(true); initWakeWord(); } else if (result.state === 'denied') { setHasMicPermission(false); } }); } else { setHasMicPermission(true); initWakeWord(); } } else { setHasMicPermission(false); } }; checkPermissions(); const handleResize = () => { setPos(prev => ({ x: Math.min(prev.x, window.innerWidth - 70), y: Math.min(prev.y, window.innerHeight - 120) })); }; window.addEventListener('resize', handleResize); // Listen for navigation commands const handleNavStart = (e: any) => { setNavOverlay(e.detail); setIsOpen(true); }; window.addEventListener('rudraksha-nav-start', handleNavStart); return () => { window.removeEventListener('resize', handleResize); window.removeEventListener('rudraksha-nav-start', handleNavStart); }; }, []); const onMouseDown = (e: React.MouseEvent | React.TouchEvent) => { setDragging(true); const clientX = 'touches' in e ? e.touches[0].clientX : e.clientX; const clientY = 'touches' in e ? e.touches[0].clientY : e.clientY; dragOffset.current = { x: clientX - pos.x, y: clientY - pos.y }; }; useEffect(() => { const onMouseMove = (e: MouseEvent | TouchEvent) => { if (!dragging) return; hasMovedRef.current = true; const clientX = 'touches' in e ? e.touches[0].clientX : (e as MouseEvent).clientX; const clientY = 'touches' in e ? e.touches[0].clientY : (e as MouseEvent).clientY; const newX = Math.max(10, Math.min(window.innerWidth - 60, clientX - dragOffset.current.x)); const newY = Math.max(10, Math.min(window.innerHeight - 110, clientY - dragOffset.current.y)); setPos({ x: newX, y: newY }); }; const onMouseUp = () => setDragging(false); if (dragging) { window.addEventListener('mousemove', onMouseMove); window.addEventListener('mouseup', onMouseUp); window.addEventListener('touchmove', onMouseMove); window.addEventListener('touchend', onMouseUp); } return () => { window.removeEventListener('mousemove', onMouseMove); window.removeEventListener('mouseup', onMouseUp); window.removeEventListener('touchmove', onMouseMove); window.removeEventListener('touchend', onMouseUp); }; }, [dragging]); // --- WAKE WORD LOGIC --- const startLiveConversationRef = useRef<(context?: string) => Promise>(async () => {}); const stopWakeWord = useCallback(() => { if (wakeWordRecognitionRef.current) { try { wakeWordRecognitionRef.current.onend = null; wakeWordRecognitionRef.current.stop(); wakeWordRecognitionRef.current = null; } catch (e) {} } }, []); const initWakeWord = useCallback(() => { if (isMicLockedRef.current || isLiveActiveRef.current) return; const SpeechRecognition = (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition; if (!SpeechRecognition) return; if (wakeWordRecognitionRef.current) return; try { const recognition = new SpeechRecognition(); recognition.continuous = true; recognition.interimResults = true; recognition.lang = 'en-US'; recognition.onresult = (event: any) => { if (isLiveActiveRef.current || isMicLockedRef.current) return; for (let i = event.resultIndex; i < event.results.length; ++i) { const text = event.results[i][0].transcript.toLowerCase().trim(); const isSecretTrigger = text.includes("oi baiman baccha") || text.includes("oi baiman bacha") || text.includes("ओई बेईमान बच्चा"); const isStandardTrigger = text.includes("hey rudra") || text.includes("rudra") || text.includes("ai babu"); if (isSecretTrigger || isStandardTrigger) { stopWakeWord(); setIsOpen(true); setMode('voice'); const audio = new Audio('https://assets.mixkit.co/active_storage/sfx/2869/2869-preview.mp3'); audio.volume = 0.5; audio.play().catch(() => {}); const triggerContext = isSecretTrigger ? " [SYSTEM EVENT: User activated via secret phrase 'Oi Baiman Baccha'. Respond ONLY with 'Jwajalapa Ama!' immediately.]" : undefined; setTimeout(() => { if (startLiveConversationRef.current) { startLiveConversationRef.current(triggerContext); } }, 100); return; } } }; recognition.onend = () => { wakeWordRecognitionRef.current = null; if (!isLiveActiveRef.current && !isMicLockedRef.current) { try { setTimeout(() => initWakeWord(), 1000); } catch (e) {} } }; recognition.start(); wakeWordRecognitionRef.current = recognition; } catch (e) { } }, []); useEffect(() => { const handleMicLock = (e: any) => { const { state } = e.detail; isMicLockedRef.current = state; if (state) { stopWakeWord(); } else { setTimeout(() => initWakeWord(), 1500); } }; window.addEventListener('rudraksha-mic-lock', handleMicLock); return () => { stopWakeWord(); window.removeEventListener('rudraksha-mic-lock', handleMicLock); }; }, [initWakeWord, stopWakeWord]); const handleGrantPermission = async () => { const settings = await StorageService.getSettings(); await StorageService.saveSettings({ ...settings, permissions: { ...settings.permissions, microphone: true } }); const success = await requestMicPermission(); setHasMicPermission(success); if (success) { setErrorMsg(null); initWakeWord(); } else { setErrorMsg("Microphone access is required for voice features."); } return success; }; // --- GEMINI LIVE LOGIC --- const startLiveConversation = async (initialContext?: string) => { if (isLiveActive) return; const settings = await StorageService.getSettings(); if (!settings.permissions.microphone) { setHasMicPermission(false); setErrorMsg("Privacy Mode Active. Microphone access disabled in Settings."); return; } if (!hasMicPermissionRef.current) { const granted = await handleGrantPermission(); if (!granted) return; } stopWakeWord(); window.dispatchEvent(new CustomEvent('rudraksha-mic-lock', { detail: { state: true } })); setLiveStatus('Connecting'); setIsLiveActive(true); setUserTranscript(''); setAiTranscript(''); setErrorMsg(null); await new Promise(r => setTimeout(r, 800)); try { let stream: MediaStream | null = null; for (let i = 0; i < 3; i++) { try { stream = await navigator.mediaDevices.getUserMedia({ audio: true }); break; } catch (err) { if (i === 2) throw new Error("Microphone busy."); await new Promise(r => setTimeout(r, 1000)); } } micStreamRef.current = stream; inputAudioContextRef.current = new (window.AudioContext || (window as any).webkitAudioContext)({ sampleRate: 16000 }); outputAudioContextRef.current = new (window.AudioContext || (window as any).webkitAudioContext)({ sampleRate: 24000 }); await inputAudioContextRef.current.resume(); await outputAudioContextRef.current.resume(); // Fetch Mood Log const today = new Date().toISOString().split('T')[0]; const healthLog = await StorageService.getHealthLog(today); const moodContext = healthLog ? `[USER CONTEXT: Current recorded mood is ${healthLog.mood}. Adjust tone accordingly.]` : ""; const systemInstructionWithContext = GET_ROBOTIC_SYSTEM_INSTRUCTION(language) + moodContext + (initialContext || ""); const sessionPromise = connectToGuruLive({ model: 'gemini-2.5-flash-native-audio-preview-12-2025', config: { responseModalities: [Modality.AUDIO], systemInstruction: systemInstructionWithContext, tools: ALL_RUDRA_TOOLS, speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: AI_LANGUAGES[language].voiceName } } }, inputAudioTranscription: {}, outputAudioTranscription: {} }, callbacks: { onopen: () => { if (!inputAudioContextRef.current || !micStreamRef.current) return; const source = inputAudioContextRef.current.createMediaStreamSource(micStreamRef.current); const scriptProcessor = inputAudioContextRef.current.createScriptProcessor(4096, 1, 1); scriptProcessor.onaudioprocess = (e) => { const inputData = e.inputBuffer.getChannelData(0); const pcmBlob = createPcmBlob(inputData); sessionPromise.then(session => { if (session) session.sendRealtimeInput({ media: pcmBlob }); }).catch(() => {}); }; source.connect(scriptProcessor); scriptProcessor.connect(inputAudioContextRef.current.destination); setLiveStatus('Listening'); }, onmessage: async (message: LiveServerMessage) => { if (message.toolCall) { for (const fc of message.toolCall.functionCalls) { try { const result = await executeRudraTool(fc.name, fc.args, navigate); // Handle special UI signals if (result === "TERMINATE_SIGNAL") { stopLiveConversation(); return; } if (result === "LOGOUT_SIGNAL") { stopLiveConversation(); await StorageService.logout(); navigate('/auth'); return; } if (result === "HANDOFF_TO_CHEF") { // Automatically terminate voice session and close UI overlay stopLiveConversation(); setIsOpen(false); return; } sessionPromise.then(session => { if (session) { session.sendToolResponse({ functionResponses: { id: fc.id, name: fc.name, response: { result: result } } }); } }); // Add system response to chat history setMessages(prev => [...prev, { id: Date.now().toString(), role: 'model', text: `[SYSTEM ACTION] ${result}`, timestamp: Date.now() }]); } catch (toolError) { console.error("Tool execution failed:", toolError); } } } const audioData = message.serverContent?.modelTurn?.parts?.[0]?.inlineData?.data; if (audioData && outputAudioContextRef.current) { setLiveStatus('Speaking'); nextStartTimeRef.current = Math.max(nextStartTimeRef.current, outputAudioContextRef.current.currentTime); const buffer = await decodeAudioData(decodeAudio(audioData), outputAudioContextRef.current, 24000, 1); const source = outputAudioContextRef.current.createBufferSource(); source.buffer = buffer; source.connect(outputAudioContextRef.current.destination); source.onended = () => { audioSourcesRef.current.delete(source); if (audioSourcesRef.current.size === 0) setLiveStatus('Listening'); }; source.start(nextStartTimeRef.current); nextStartTimeRef.current += buffer.duration; audioSourcesRef.current.add(source); } // Transcripts are captured but NOT displayed in UI as per request if (message.serverContent?.inputTranscription) { const text = message.serverContent.inputTranscription.text || ''; setUserTranscript(text); } if (message.serverContent?.outputTranscription) { setAiTranscript(prev => `${prev}${message.serverContent?.outputTranscription?.text}`); } if (message.serverContent?.interrupted) { audioSourcesRef.current.forEach(s => { try { s.stop(); } catch(e) {} }); audioSourcesRef.current.clear(); nextStartTimeRef.current = 0; setLiveStatus('Listening'); } }, onerror: (e) => { console.error("Neural Voice Error:", e); setErrorMsg("Hardware link interrupted. Network or API error."); stopLiveConversation(); }, onclose: () => stopLiveConversation(), } }); liveSessionPromiseRef.current = sessionPromise; } catch (err: any) { console.error("Neural Voice Link Failure:", err); setLiveStatus('Error'); setErrorMsg(err.message || "Network Error."); setTimeout(() => stopLiveConversation(), 3000); } }; useEffect(() => { startLiveConversationRef.current = startLiveConversation; }); const stopLiveConversation = () => { setIsLiveActive(false); setLiveStatus('Idle'); liveSessionPromiseRef.current?.then(s => { try { s.close(); } catch(e) {} }); if (micStreamRef.current) { micStreamRef.current.getTracks().forEach(t => t.stop()); micStreamRef.current = null; } audioSourcesRef.current.forEach(s => { try { s.stop(); } catch(e) {} }); audioSourcesRef.current.clear(); if (inputAudioContextRef.current && inputAudioContextRef.current.state !== 'closed') { inputAudioContextRef.current.close().catch(() => {}); } if (outputAudioContextRef.current && outputAudioContextRef.current.state !== 'closed') { outputAudioContextRef.current.close().catch(() => {}); } inputAudioContextRef.current = null; outputAudioContextRef.current = null; nextStartTimeRef.current = 0; window.dispatchEvent(new CustomEvent('rudraksha-mic-lock', { detail: { state: false } })); setTimeout(() => initWakeWord(), 1500); }; const createPcmBlob = (data: Float32Array): Blob => { const l = data.length; const int16 = new Int16Array(l); for (let i = 0; i < l; i++) int16[i] = data[i] * 32768; return { data: encodeAudio(new Uint8Array(int16.buffer)), mimeType: 'audio/pcm;rate=16000', }; }; // --- UI INTERACTION --- const onTriggerStart = (e: React.MouseEvent | React.TouchEvent) => { if (e.type === 'mousedown' && (e as React.MouseEvent).button !== 0) return; setDragging(true); hasMovedRef.current = false; const clientX = 'touches' in e ? e.touches[0].clientX : (e as React.MouseEvent).clientX; const clientY = 'touches' in e ? e.touches[0].clientY : (e as React.MouseEvent).clientY; dragStartPos.current = { x: clientX, y: clientY }; dragOffset.current = { x: clientX - pos.x, y: clientY - pos.y }; }; const handleOpen = () => { if (!hasMovedRef.current) { setIsOpen(!isOpen); } }; const handleChatSubmit = async (e: React.FormEvent) => { e.preventDefault(); if (!input.trim() || isTyping) return; const userMsg: ChatMessage = { id: Date.now().toString(), role: 'user', text: input, timestamp: Date.now() }; setMessages(prev => [...prev, userMsg]); const currentInput = input; setInput(''); setIsTyping(true); try { const aiClient = new GoogleGenAI({ apiKey: process.env.API_KEY }); const res = await aiClient.models.generateContent({ model: 'gemini-3-flash-preview', contents: currentInput, config: { systemInstruction: GET_ROBOTIC_SYSTEM_INSTRUCTION(language), tools: ALL_RUDRA_TOOLS } }); if (res.functionCalls) { for (const fc of res.functionCalls) { const result = await executeRudraTool(fc.name, fc.args, navigate); setMessages(prev => [...prev, { id: Date.now().toString(), role: 'model', text: `Action Taken: ${result}`, timestamp: Date.now() }]); } } else { setMessages(prev => [...prev, { id: Date.now().toString(), role: 'model', text: res.text || "Command accepted.", timestamp: Date.now() }]); } } catch { setMessages(prev => [...prev, { id: 'err', role: 'model', text: "Signal Interrupted.", timestamp: Date.now() }]); } finally { setIsTyping(false); } }; const getMapEmbedUrl = (dest: string, mode: string) => { const m = mode.toLowerCase(); return `https://www.google.com/maps?saddr=Current+Location&daddr=${encodeURIComponent(dest)}&dirflg=${m.charAt(0)}&output=embed`; }; return (
{isOpen && (
e.stopPropagation()}> {/* HEADER */}

Rudra Core

{isLiveActive ? liveStatus : 'System Ready'}
{navOverlay ? (
{navOverlay.destination}
{navOverlay.mode}