feat: voice recording + TTS read-aloud for assistant messages

Audio recording:
- Mic button wired to MediaRecorder (AAC/M4A, 44.1kHz/128kbps)
- Tap to start, tap again to stop — auto-uploads as audio attachment
- RECORD_AUDIO permission with runtime request
- Recording state: red stop-square on mic button

TTS read-aloud:
- Volume button on assistant messages → POST /api/v1/audio-gen
- Gemini-TTS generates speech server-side, MediaPlayer streams it
- Tap again to stop playback, accent-colored icon when playing
- Graceful error handling (Toast on failure)

Supporting changes:
- KaizenApi.generateSpeech() for /api/v1/audio-gen endpoint
- KaizenIcons.Volume2 added (Lucide)
- DE/EN strings for recording, permission, TTS states
This commit is contained in:
Bruno Deanoz 2026-06-23 00:27:02 +02:00
parent edd8ccaa05
commit 9e1f30de1f
7 changed files with 198 additions and 2 deletions

View file

@ -8,6 +8,9 @@
<!-- Required to talk to the Kaizen backend (login + chat streaming) --> <!-- Required to talk to the Kaizen backend (login + chat streaming) -->
<uses-permission android:name="android.permission.INTERNET" /> <uses-permission android:name="android.permission.INTERNET" />
<!-- Required for voice recording (mic button in chat) -->
<uses-permission android:name="android.permission.RECORD_AUDIO" />
<application <application
android:allowBackup="true" android:allowBackup="true"
android:dataExtractionRules="@xml/data_extraction_rules" android:dataExtractionRules="@xml/data_extraction_rules"

View file

@ -331,6 +331,8 @@ fun MessageRow(
onCopy: (String) -> Unit = {}, onCopy: (String) -> Unit = {},
onDelete: ((String) -> Unit)? = null, onDelete: ((String) -> Unit)? = null,
onRegenerate: ((String) -> Unit)? = null, onRegenerate: ((String) -> Unit)? = null,
onReadAloud: ((String) -> Unit)? = null,
isPlayingTts: Boolean = false,
) { ) {
val cs = MaterialTheme.colorScheme val cs = MaterialTheme.colorScheme
val isDark = isSystemInDarkTheme() val isDark = isSystemInDarkTheme()
@ -461,6 +463,14 @@ fun MessageRow(
copied = false copied = false
} }
} }
onReadAloud?.let { readAloud ->
MessageActionButton(
icon = KaizenIcons.Volume2,
tint = if (isPlayingTts) accent.primary else cs.onSurfaceVariant.copy(alpha = 0.35f),
) {
haptics.tick(); readAloud(message.content)
}
}
onRegenerate?.let { regen -> onRegenerate?.let { regen ->
MessageActionButton(KaizenIcons.RefreshCw, cs.onSurfaceVariant.copy(alpha = 0.35f)) { MessageActionButton(KaizenIcons.RefreshCw, cs.onSurfaceVariant.copy(alpha = 0.35f)) {
haptics.tick(); regen(message.wireId) haptics.tick(); regen(message.wireId)
@ -527,6 +537,8 @@ fun ChatInput(
enabled: Boolean, enabled: Boolean,
isStreaming: Boolean = false, isStreaming: Boolean = false,
onStop: () -> Unit = {}, onStop: () -> Unit = {},
isRecording: Boolean = false,
onMicClick: () -> Unit = {},
pendingFiles: List<PendingFile> = emptyList(), pendingFiles: List<PendingFile> = emptyList(),
onAddClick: () -> Unit = {}, onAddClick: () -> Unit = {},
onRemoveFile: (String) -> Unit = {}, onRemoveFile: (String) -> Unit = {},
@ -584,10 +596,16 @@ fun ChatInput(
} }
Spacer(Modifier.width(8.dp)) Spacer(Modifier.width(8.dp))
Box( Box(
Modifier.size(38.dp).clip(KaizenShapes.circle).background(iconBg), Modifier.size(38.dp).clip(KaizenShapes.circle)
.background(if (isRecording) Color(0xFFEF4444).copy(alpha = if (isDark) 0.85f else 0.80f) else iconBg)
.clickable(onClick = onMicClick),
contentAlignment = Alignment.Center, contentAlignment = Alignment.Center,
) { ) {
Icon(KaizenIcons.Mic, stringResource(R.string.chat_attachment), tint = cs.onSurfaceVariant, modifier = Modifier.size(19.dp)) if (isRecording) {
Icon(KaizenIcons.Square, stringResource(R.string.chat_recording), tint = Color.White, modifier = Modifier.size(14.dp))
} else {
Icon(KaizenIcons.Mic, stringResource(R.string.chat_attachment), tint = cs.onSurfaceVariant, modifier = Modifier.size(19.dp))
}
} }
Spacer(Modifier.weight(1f)) Spacer(Modifier.weight(1f))

View file

@ -82,10 +82,17 @@ import androidx.activity.result.contract.ActivityResultContracts
import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.res.stringResource import androidx.compose.ui.res.stringResource
import androidx.lifecycle.compose.LifecycleResumeEffect import androidx.lifecycle.compose.LifecycleResumeEffect
import android.Manifest
import android.content.pm.PackageManager
import android.media.MediaPlayer
import android.media.MediaRecorder
import android.widget.Toast
import androidx.core.content.ContextCompat
import kotlinx.coroutines.Job import kotlinx.coroutines.Job
import kotlinx.coroutines.async import kotlinx.coroutines.async
import kotlinx.coroutines.launch import kotlinx.coroutines.launch
import java.io.ByteArrayOutputStream import java.io.ByteArrayOutputStream
import java.io.File
// Screen enumeration for modular state-driven navigation (Zero Technical Debt) // Screen enumeration for modular state-driven navigation (Zero Technical Debt)
enum class AppScreen { Chat, Settings } enum class AppScreen { Chat, Settings }
@ -111,6 +118,15 @@ fun ChatScreen(
var chatModel by remember { mutableStateOf<String?>(null) } var chatModel by remember { mutableStateOf<String?>(null) }
var streamJob by remember { mutableStateOf<Job?>(null) } var streamJob by remember { mutableStateOf<Job?>(null) }
// Audio recording state
var isRecording by remember { mutableStateOf(false) }
val recorderRef = remember { mutableStateOf<MediaRecorder?>(null) }
val audioFileRef = remember { mutableStateOf<File?>(null) }
// TTS playback state
var playingTtsForId by remember { mutableStateOf<Long?>(null) }
val ttsPlayerRef = remember { mutableStateOf<MediaPlayer?>(null) }
// Navigation State // Navigation State
var currentScreen by remember { mutableStateOf(AppScreen.Chat) } var currentScreen by remember { mutableStateOf(AppScreen.Chat) }
@ -191,6 +207,67 @@ fun ChatScreen(
uploadPickedFile("photo_${System.currentTimeMillis()}.jpg", "image/jpeg", stream.toByteArray()) uploadPickedFile("photo_${System.currentTimeMillis()}.jpg", "image/jpeg", stream.toByteArray())
} }
val micPermission = rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted ->
if (granted) startRecording(context, recorderRef, audioFileRef) { isRecording = true }
else Toast.makeText(context, context.getString(R.string.chat_mic_permission), Toast.LENGTH_SHORT).show()
}
fun toggleRecording() {
if (isRecording) {
stopRecording(recorderRef, audioFileRef) { name, mime, bytes ->
isRecording = false
uploadPickedFile(name, mime, bytes)
}
} else {
if (ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) {
startRecording(context, recorderRef, audioFileRef) { isRecording = true }
} else {
micPermission.launch(Manifest.permission.RECORD_AUDIO)
}
}
}
fun readAloud(content: String, messageId: Long) {
val cfg = session.config ?: return
if (playingTtsForId == messageId) {
ttsPlayerRef.value?.release()
ttsPlayerRef.value = null
playingTtsForId = null
return
}
ttsPlayerRef.value?.release()
playingTtsForId = messageId
scope.launch {
val url = KaizenApi.generateSpeech(cfg.baseUrl, cfg.token, content)
if (url == null) {
playingTtsForId = null
Toast.makeText(context, context.getString(R.string.chat_tts_error), Toast.LENGTH_SHORT).show()
return@launch
}
try {
val player = MediaPlayer().apply {
setDataSource(url)
prepareAsync()
setOnPreparedListener { start() }
setOnCompletionListener {
release()
ttsPlayerRef.value = null
playingTtsForId = null
}
setOnErrorListener { _, _, _ ->
release()
ttsPlayerRef.value = null
playingTtsForId = null
true
}
}
ttsPlayerRef.value = player
} catch (_: Exception) {
playingTtsForId = null
}
}
}
LaunchedEffect(session.config?.baseUrl, session.config?.token) { LaunchedEffect(session.config?.baseUrl, session.config?.token) {
val cfg = session.config ?: return@LaunchedEffect val cfg = session.config ?: return@LaunchedEffect
loadError = null loadError = null
@ -659,6 +736,10 @@ fun ChatScreen(
send(userContent) send(userContent)
} }
} else null, } else null,
onReadAloud = if (message.role == Role.Assistant && !message.streaming && message.content.isNotBlank()) { content ->
readAloud(content, message.id)
} else null,
isPlayingTts = playingTtsForId == message.id,
) )
} }
} }
@ -858,6 +939,8 @@ fun ChatScreen(
enabled = !isStreaming, enabled = !isStreaming,
isStreaming = isStreaming, isStreaming = isStreaming,
onStop = { streamJob?.cancel() }, onStop = { streamJob?.cancel() },
isRecording = isRecording,
onMicClick = { toggleRecording() },
pendingFiles = pendingFiles, pendingFiles = pendingFiles,
onAddClick = { showAttachMenu = !showAttachMenu }, onAddClick = { showAttachMenu = !showAttachMenu },
onRemoveFile = { id -> pendingFiles.removeAll { it.id == id } }, onRemoveFile = { id -> pendingFiles.removeAll { it.id == id } },
@ -950,3 +1033,53 @@ private fun chatErrorText(e: Throwable, ctx: android.content.Context): String =
e is ChatHttpException -> ctx.getString(R.string.error_server, e.code) e is ChatHttpException -> ctx.getString(R.string.error_server, e.code)
else -> ctx.getString(R.string.error_no_connection) else -> ctx.getString(R.string.error_no_connection)
} }
@Suppress("DEPRECATION")
private fun startRecording(
context: android.content.Context,
recorderRef: androidx.compose.runtime.MutableState<MediaRecorder?>,
audioFileRef: androidx.compose.runtime.MutableState<File?>,
onStarted: () -> Unit,
) {
try {
val file = File(context.cacheDir, "voice_${System.currentTimeMillis()}.m4a")
val recorder = MediaRecorder(context).apply {
setAudioSource(MediaRecorder.AudioSource.MIC)
setOutputFormat(MediaRecorder.OutputFormat.MPEG_4)
setAudioEncoder(MediaRecorder.AudioEncoder.AAC)
setAudioSamplingRate(44100)
setAudioEncodingBitRate(128000)
setOutputFile(file.absolutePath)
prepare()
start()
}
recorderRef.value = recorder
audioFileRef.value = file
onStarted()
} catch (_: Exception) {
recorderRef.value = null
audioFileRef.value = null
}
}
private fun stopRecording(
recorderRef: androidx.compose.runtime.MutableState<MediaRecorder?>,
audioFileRef: androidx.compose.runtime.MutableState<File?>,
onRecorded: (String, String, ByteArray) -> Unit,
) {
val recorder = recorderRef.value ?: return
val file = audioFileRef.value ?: return
try {
recorder.stop()
recorder.release()
} catch (_: Exception) {
recorder.release()
}
recorderRef.value = null
audioFileRef.value = null
if (file.exists() && file.length() > 0) {
val bytes = file.readBytes()
file.delete()
onRecorded("voice_${System.currentTimeMillis()}.m4a", "audio/mp4", bytes)
}
}

View file

@ -572,6 +572,35 @@ object KaizenApi {
} }
} }
@Serializable private data class SpeechRequest(val prompt: String, val kind: String = "speech", val voice: String? = null)
@Serializable private data class SpeechResponse(val url: String = "", val cost: Double? = null)
/** POST /api/v1/audio-gen — generate speech from text via Gemini-TTS. Returns the audio URL or null. */
suspend fun generateSpeech(baseUrl: String, token: String, text: String, voice: String? = null): String? =
withContext(Dispatchers.IO) {
val body = json.encodeToString(SpeechRequest(
prompt = text.take(2000),
voice = voice,
)).toRequestBody(jsonMedia)
val req = Request.Builder()
.url("$baseUrl/api/v1/audio-gen")
.post(body)
.header("Authorization", "Bearer $token")
.build()
try {
client.newCall(req).execute().use { resp ->
if (!resp.isSuccessful) {
Log.w(TAG, "generateSpeech: HTTP ${resp.code}")
return@use null
}
json.decodeFromString<SpeechResponse>(resp.body!!.string()).url.takeIf { it.isNotBlank() }
}
} catch (e: Exception) {
Log.w(TAG, "generateSpeech: ${e.javaClass.simpleName}: ${e.message}")
null
}
}
suspend fun prewarm(baseUrl: String) = withContext(Dispatchers.IO) { suspend fun prewarm(baseUrl: String) = withContext(Dispatchers.IO) {
try { try {
val req = Request.Builder().url("$baseUrl/api/v1/meta").build() val req = Request.Builder().url("$baseUrl/api/v1/meta").build()

View file

@ -91,6 +91,9 @@ object KaizenIcons {
val ArrowDown: ImageVector get() = Lucide.ArrowDown val ArrowDown: ImageVector get() = Lucide.ArrowDown
val ArrowUp: ImageVector get() = Lucide.ArrowUp val ArrowUp: ImageVector get() = Lucide.ArrowUp
// ── Audio ──────────────────────────────────────────────────────────────
val Volume2: ImageVector get() = Lucide.Volume2
// ── Custom (brand overrides) ──────────────────────────────────────────── // ── Custom (brand overrides) ────────────────────────────────────────────
// Add hand-drawn SVGs here as ImageVector literals when ready. // Add hand-drawn SVGs here as ImageVector literals when ready.
// Example: // Example:

View file

@ -19,6 +19,11 @@
<string name="chat_error">Error</string> <string name="chat_error">Error</string>
<string name="chat_open_menu">Open menu</string> <string name="chat_open_menu">Open menu</string>
<string name="chat_stop">Stop response</string> <string name="chat_stop">Stop response</string>
<string name="chat_recording">Recording …</string>
<string name="chat_mic_permission">Microphone access required</string>
<string name="chat_read_aloud">Read aloud</string>
<string name="chat_tts_error">Read aloud failed</string>
<string name="chat_tts_unavailable">Read aloud not available (Vertex required)</string>
<string name="attach_camera">Take Photo</string> <string name="attach_camera">Take Photo</string>
<string name="attach_gallery">Photo &amp; Video</string> <string name="attach_gallery">Photo &amp; Video</string>
<string name="attach_file">Choose File</string> <string name="attach_file">Choose File</string>

View file

@ -21,6 +21,11 @@
<string name="chat_error">Fehler</string> <string name="chat_error">Fehler</string>
<string name="chat_open_menu">Menü öffnen</string> <string name="chat_open_menu">Menü öffnen</string>
<string name="chat_stop">Antwort stoppen</string> <string name="chat_stop">Antwort stoppen</string>
<string name="chat_recording">Aufnahme läuft …</string>
<string name="chat_mic_permission">Mikrofonzugriff wird benötigt</string>
<string name="chat_read_aloud">Vorlesen</string>
<string name="chat_tts_error">Vorlesen fehlgeschlagen</string>
<string name="chat_tts_unavailable">Vorlesen nicht verfügbar (Vertex benötigt)</string>
<string name="attach_camera">Foto aufnehmen</string> <string name="attach_camera">Foto aufnehmen</string>
<string name="attach_gallery">Foto &amp; Video</string> <string name="attach_gallery">Foto &amp; Video</string>
<string name="attach_file">Datei auswählen</string> <string name="attach_file">Datei auswählen</string>