krypto/kaizen-app
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: voice recording + TTS read-aloud for assistant messages

Browse source 
Audio recording:
- Mic button wired to MediaRecorder (AAC/M4A, 44.1kHz/128kbps)
- Tap to start, tap again to stop — auto-uploads as audio attachment
- RECORD_AUDIO permission with runtime request
- Recording state: red stop-square on mic button

TTS read-aloud:
- Volume button on assistant messages → POST /api/v1/audio-gen
- Gemini-TTS generates speech server-side, MediaPlayer streams it
- Tap again to stop playback, accent-colored icon when playing
- Graceful error handling (Toast on failure)

Supporting changes:
- KaizenApi.generateSpeech() for /api/v1/audio-gen endpoint
- KaizenIcons.Volume2 added (Lucide)
- DE/EN strings for recording, permission, TTS states
This commit is contained in:
Bruno Deanoz 2026-06-23 00:27:02 +02:00
parent edd8ccaa05
commit 9e1f30de1f
7 changed files with 198 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
app/src/main/AndroidManifest.xml
View file

@ -8,6 +8,9 @@
<!-- Required to talk to the Kaizen backend (login + chat streaming) -->
<uses-permission android:name="android.permission.INTERNET" />
<!-- Required for voice recording (mic button in chat) -->
<uses-permission android:name="android.permission.RECORD_AUDIO" />
<application
android:allowBackup="true"
android:dataExtractionRules="@xml/data_extraction_rules"

22
app/src/main/java/dev/kaizen/app/chat/ChatComponents.kt
View file

@ -331,6 +331,8 @@ fun MessageRow(
onCopy: (String) -> Unit = {},
onDelete: ((String) -> Unit)? = null,
onRegenerate: ((String) -> Unit)? = null,
onReadAloud: ((String) -> Unit)? = null,
isPlayingTts: Boolean = false,
) {
val cs = MaterialTheme.colorScheme
val isDark = isSystemInDarkTheme()
@ -461,6 +463,14 @@ fun MessageRow(
copied = false
}
}
onReadAloud?.let { readAloud ->
MessageActionButton(
icon = KaizenIcons.Volume2,
tint = if (isPlayingTts) accent.primary else cs.onSurfaceVariant.copy(alpha = 0.35f),
) {
haptics.tick(); readAloud(message.content)
}
}
onRegenerate?.let { regen ->
MessageActionButton(KaizenIcons.RefreshCw, cs.onSurfaceVariant.copy(alpha = 0.35f)) {
haptics.tick(); regen(message.wireId)
@ -527,6 +537,8 @@ fun ChatInput(
enabled: Boolean,
isStreaming: Boolean = false,
onStop: () -> Unit = {},
isRecording: Boolean = false,
onMicClick: () -> Unit = {},
pendingFiles: List<PendingFile> = emptyList(),
onAddClick: () -> Unit = {},
onRemoveFile: (String) -> Unit = {},
@ -584,10 +596,16 @@ fun ChatInput(
}
Spacer(Modifier.width(8.dp))
Box(
Modifier.size(38.dp).clip(KaizenShapes.circle).background(iconBg),
Modifier.size(38.dp).clip(KaizenShapes.circle)
.background(if (isRecording) Color(0xFFEF4444).copy(alpha = if (isDark) 0.85f else 0.80f) else iconBg)
.clickable(onClick = onMicClick),
contentAlignment = Alignment.Center,
) {
Icon(KaizenIcons.Mic, stringResource(R.string.chat_attachment), tint = cs.onSurfaceVariant, modifier = Modifier.size(19.dp))
if (isRecording) {
Icon(KaizenIcons.Square, stringResource(R.string.chat_recording), tint = Color.White, modifier = Modifier.size(14.dp))
} else {
Icon(KaizenIcons.Mic, stringResource(R.string.chat_attachment), tint = cs.onSurfaceVariant, modifier = Modifier.size(19.dp))
}
}
Spacer(Modifier.weight(1f))

133
app/src/main/java/dev/kaizen/app/chat/ChatScreen.kt
View file

@ -82,10 +82,17 @@ import androidx.activity.result.contract.ActivityResultContracts
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.res.stringResource
import androidx.lifecycle.compose.LifecycleResumeEffect
import android.Manifest
import android.content.pm.PackageManager
import android.media.MediaPlayer
import android.media.MediaRecorder
import android.widget.Toast
import androidx.core.content.ContextCompat
import kotlinx.coroutines.Job
import kotlinx.coroutines.async
import kotlinx.coroutines.launch
import java.io.ByteArrayOutputStream
import java.io.File
// Screen enumeration for modular state-driven navigation (Zero Technical Debt)
enum class AppScreen { Chat, Settings }
@ -111,6 +118,15 @@ fun ChatScreen(
var chatModel by remember { mutableStateOf<String?>(null) }
var streamJob by remember { mutableStateOf<Job?>(null) }
// Audio recording state
var isRecording by remember { mutableStateOf(false) }
val recorderRef = remember { mutableStateOf<MediaRecorder?>(null) }
val audioFileRef = remember { mutableStateOf<File?>(null) }
// TTS playback state
var playingTtsForId by remember { mutableStateOf<Long?>(null) }
val ttsPlayerRef = remember { mutableStateOf<MediaPlayer?>(null) }
// Navigation State
var currentScreen by remember { mutableStateOf(AppScreen.Chat) }
@ -191,6 +207,67 @@ fun ChatScreen(
uploadPickedFile("photo_${System.currentTimeMillis()}.jpg", "image/jpeg", stream.toByteArray())
}
val micPermission = rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted ->
if (granted) startRecording(context, recorderRef, audioFileRef) { isRecording = true }
else Toast.makeText(context, context.getString(R.string.chat_mic_permission), Toast.LENGTH_SHORT).show()
}
fun toggleRecording() {
if (isRecording) {
stopRecording(recorderRef, audioFileRef) { name, mime, bytes ->
isRecording = false
uploadPickedFile(name, mime, bytes)
}
} else {
if (ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) {
startRecording(context, recorderRef, audioFileRef) { isRecording = true }
} else {
micPermission.launch(Manifest.permission.RECORD_AUDIO)
}
}
}
fun readAloud(content: String, messageId: Long) {
val cfg = session.config ?: return
if (playingTtsForId == messageId) {
ttsPlayerRef.value?.release()
ttsPlayerRef.value = null
playingTtsForId = null
return
}
ttsPlayerRef.value?.release()
playingTtsForId = messageId
scope.launch {
val url = KaizenApi.generateSpeech(cfg.baseUrl, cfg.token, content)
if (url == null) {
playingTtsForId = null
Toast.makeText(context, context.getString(R.string.chat_tts_error), Toast.LENGTH_SHORT).show()
return@launch
}
try {
val player = MediaPlayer().apply {
setDataSource(url)
prepareAsync()
setOnPreparedListener { start() }
setOnCompletionListener {
release()
ttsPlayerRef.value = null
playingTtsForId = null
}
setOnErrorListener { _, _, _ ->
release()
ttsPlayerRef.value = null
playingTtsForId = null
true
}
}
ttsPlayerRef.value = player
} catch (_: Exception) {
playingTtsForId = null
}
}
}
LaunchedEffect(session.config?.baseUrl, session.config?.token) {
val cfg = session.config ?: return@LaunchedEffect
loadError = null
@ -659,6 +736,10 @@ fun ChatScreen(
send(userContent)
}
} else null,
onReadAloud = if (message.role == Role.Assistant && !message.streaming && message.content.isNotBlank()) { content ->
readAloud(content, message.id)
} else null,
isPlayingTts = playingTtsForId == message.id,
)
}
}
@ -858,6 +939,8 @@ fun ChatScreen(
enabled = !isStreaming,
isStreaming = isStreaming,
onStop = { streamJob?.cancel() },
isRecording = isRecording,
onMicClick = { toggleRecording() },
pendingFiles = pendingFiles,
onAddClick = { showAttachMenu = !showAttachMenu },
onRemoveFile = { id -> pendingFiles.removeAll { it.id == id } },
@ -950,3 +1033,53 @@ private fun chatErrorText(e: Throwable, ctx: android.content.Context): String =
e is ChatHttpException -> ctx.getString(R.string.error_server, e.code)
else -> ctx.getString(R.string.error_no_connection)
}
@Suppress("DEPRECATION")
private fun startRecording(
context: android.content.Context,
recorderRef: androidx.compose.runtime.MutableState<MediaRecorder?>,
audioFileRef: androidx.compose.runtime.MutableState<File?>,
onStarted: () -> Unit,
) {
try {
val file = File(context.cacheDir, "voice_${System.currentTimeMillis()}.m4a")
val recorder = MediaRecorder(context).apply {
setAudioSource(MediaRecorder.AudioSource.MIC)
setOutputFormat(MediaRecorder.OutputFormat.MPEG_4)
setAudioEncoder(MediaRecorder.AudioEncoder.AAC)
setAudioSamplingRate(44100)
setAudioEncodingBitRate(128000)
setOutputFile(file.absolutePath)
prepare()
start()
}
recorderRef.value = recorder
audioFileRef.value = file
onStarted()
} catch (_: Exception) {
recorderRef.value = null
audioFileRef.value = null
}
}
private fun stopRecording(
recorderRef: androidx.compose.runtime.MutableState<MediaRecorder?>,
audioFileRef: androidx.compose.runtime.MutableState<File?>,
onRecorded: (String, String, ByteArray) -> Unit,
) {
val recorder = recorderRef.value ?: return
val file = audioFileRef.value ?: return
try {
recorder.stop()
recorder.release()
} catch (_: Exception) {
recorder.release()
}
recorderRef.value = null
audioFileRef.value = null
if (file.exists() && file.length() > 0) {
val bytes = file.readBytes()
file.delete()
onRecorded("voice_${System.currentTimeMillis()}.m4a", "audio/mp4", bytes)
}
}

29
app/src/main/java/dev/kaizen/app/net/KaizenApi.kt
View file

@ -572,6 +572,35 @@ object KaizenApi {
}
}
@Serializable private data class SpeechRequest(val prompt: String, val kind: String = "speech", val voice: String? = null)
@Serializable private data class SpeechResponse(val url: String = "", val cost: Double? = null)
/** POST /api/v1/audio-gen — generate speech from text via Gemini-TTS. Returns the audio URL or null. */
suspend fun generateSpeech(baseUrl: String, token: String, text: String, voice: String? = null): String? =
withContext(Dispatchers.IO) {
val body = json.encodeToString(SpeechRequest(
prompt = text.take(2000),
voice = voice,
)).toRequestBody(jsonMedia)
val req = Request.Builder()
.url("$baseUrl/api/v1/audio-gen")
.post(body)
.header("Authorization", "Bearer $token")
.build()
try {
client.newCall(req).execute().use { resp ->
if (!resp.isSuccessful) {
Log.w(TAG, "generateSpeech: HTTP ${resp.code}")
return@use null
}
json.decodeFromString<SpeechResponse>(resp.body!!.string()).url.takeIf { it.isNotBlank() }
}
} catch (e: Exception) {
Log.w(TAG, "generateSpeech: ${e.javaClass.simpleName}: ${e.message}")
null
}
}
suspend fun prewarm(baseUrl: String) = withContext(Dispatchers.IO) {
try {
val req = Request.Builder().url("$baseUrl/api/v1/meta").build()

3
app/src/main/java/dev/kaizen/app/ui/icon/KaizenIcons.kt
View file

@ -91,6 +91,9 @@ object KaizenIcons {
val ArrowDown: ImageVector get() = Lucide.ArrowDown
val ArrowUp: ImageVector get() = Lucide.ArrowUp
// ── Audio ──────────────────────────────────────────────────────────────
val Volume2: ImageVector get() = Lucide.Volume2
// ── Custom (brand overrides) ────────────────────────────────────────────
// Add hand-drawn SVGs here as ImageVector literals when ready.
// Example:

5
app/src/main/res/values-en/strings.xml
View file

@ -19,6 +19,11 @@
<string name="chat_error">Error</string>
<string name="chat_open_menu">Open menu</string>
<string name="chat_stop">Stop response</string>
<string name="chat_recording">Recording …</string>
<string name="chat_mic_permission">Microphone access required</string>
<string name="chat_read_aloud">Read aloud</string>
<string name="chat_tts_error">Read aloud failed</string>
<string name="chat_tts_unavailable">Read aloud not available (Vertex required)</string>
<string name="attach_camera">Take Photo</string>
<string name="attach_gallery">Photo &amp; Video</string>
<string name="attach_file">Choose File</string>

5
app/src/main/res/values/strings.xml
View file

@ -21,6 +21,11 @@
<string name="chat_error">Fehler</string>
<string name="chat_open_menu">Menü öffnen</string>
<string name="chat_stop">Antwort stoppen</string>
<string name="chat_recording">Aufnahme läuft …</string>
<string name="chat_mic_permission">Mikrofonzugriff wird benötigt</string>
<string name="chat_read_aloud">Vorlesen</string>
<string name="chat_tts_error">Vorlesen fehlgeschlagen</string>
<string name="chat_tts_unavailable">Vorlesen nicht verfügbar (Vertex benötigt)</string>
<string name="attach_camera">Foto aufnehmen</string>
<string name="attach_gallery">Foto &amp; Video</string>
<string name="attach_file">Datei auswählen</string>