From a30341516f73008944d0e34cc43e40e7b209821d Mon Sep 17 00:00:00 2001 From: Somasundaram Mahesh Date: Fri, 20 Feb 2026 06:54:00 +0530 Subject: [PATCH] feat(android): auto-connect, assistant invocation, suggestion cards, and onboarding assistant step - Auto-connect to server on app open when API key is configured - Show error card below top bar on connection errors - Fix VoiceInteractionService registration with RecognitionService stub - Voice session triggers overlay command panel instead of separate UI - Add suggestion cards (recent goals + defaults) to HomeScreen empty state - Add digital assistant setup step to onboarding with skip option - Add ACTION_SHOW_COMMAND_PANEL to ConnectionService and AgentOverlay Co-Authored-By: Claude Opus 4.6 --- android/app/src/main/AndroidManifest.xml | 9 ++ .../java/com/thisux/droidclaw/MainActivity.kt | 60 +++++++++- .../droidclaw/connection/ConnectionService.kt | 4 + .../thisux/droidclaw/overlay/AgentOverlay.kt | 5 + .../thisux/droidclaw/ui/screens/HomeScreen.kt | 90 +++++++++++++- .../droidclaw/ui/screens/OnboardingScreen.kt | 111 +++++++++++++++++- .../voice/DroidClawRecognitionService.kt | 10 ++ .../droidclaw/voice/DroidClawVoiceSession.kt | 59 +--------- .../res/xml/voice_interaction_service.xml | 1 + 9 files changed, 280 insertions(+), 69 deletions(-) create mode 100644 android/app/src/main/java/com/thisux/droidclaw/voice/DroidClawRecognitionService.kt diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index cd6001f..2515df5 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -76,6 +76,15 @@ android:permission="android.permission.BIND_VOICE_INTERACTION" android:exported="true" /> + + + + + + \ No newline at end of file diff --git a/android/app/src/main/java/com/thisux/droidclaw/MainActivity.kt b/android/app/src/main/java/com/thisux/droidclaw/MainActivity.kt index a661562..e4cee38 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/MainActivity.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/MainActivity.kt @@ -5,6 +5,9 @@ import android.content.Intent import android.os.Bundle import android.provider.Settings import androidx.activity.ComponentActivity +import androidx.lifecycle.lifecycleScope +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch import androidx.activity.compose.setContent import androidx.activity.enableEdgeToEdge import androidx.activity.result.contract.ActivityResultContracts @@ -37,12 +40,22 @@ import androidx.navigation.compose.composable import androidx.navigation.compose.currentBackStackEntryAsState import androidx.navigation.compose.rememberNavController import com.thisux.droidclaw.ui.components.PermissionStatusBar +import com.thisux.droidclaw.model.ConnectionState import com.thisux.droidclaw.ui.screens.HomeScreen import com.thisux.droidclaw.ui.screens.LogsScreen import com.thisux.droidclaw.ui.screens.OnboardingScreen import com.thisux.droidclaw.ui.screens.SettingsScreen import com.thisux.droidclaw.ui.theme.DroidClawTheme import com.thisux.droidclaw.ui.theme.InstrumentSerif +import com.thisux.droidclaw.ui.theme.StatusRed +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.ui.draw.clip +import androidx.compose.ui.unit.dp sealed class Screen(val route: String, val label: String) { data object Home : Screen("home", "Home") @@ -69,6 +82,7 @@ class MainActivity : ComponentActivity() { if (intent?.getBooleanExtra("request_audio_permission", false) == true) { audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) } + autoConnectIfNeeded() } override fun onNewIntent(intent: Intent) { @@ -85,6 +99,20 @@ class MainActivity : ComponentActivity() { service.overlay?.show() } } + + private fun autoConnectIfNeeded() { + if (ConnectionService.connectionState.value != com.thisux.droidclaw.model.ConnectionState.Disconnected) return + val app = application as DroidClawApp + lifecycleScope.launch { + val apiKey = app.settingsStore.apiKey.first() + if (apiKey.isNotBlank()) { + val intent = Intent(this@MainActivity, ConnectionService::class.java).apply { + action = ConnectionService.ACTION_CONNECT + } + startForegroundService(intent) + } + } + } } @OptIn(ExperimentalMaterial3Api::class) @@ -183,11 +211,32 @@ fun MainNavigation() { ) { innerPadding -> val startDestination = if (hasOnboarded) Screen.Home.route else Screen.Onboarding.route - NavHost( - navController = navController, - startDestination = startDestination, - modifier = Modifier.padding(innerPadding) - ) { + val connectionState by ConnectionService.connectionState.collectAsState() + val errorMessage by ConnectionService.errorMessage.collectAsState() + + Column(modifier = Modifier.padding(innerPadding)) { + if (showChrome && connectionState == ConnectionState.Error) { + Box( + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 16.dp, vertical = 4.dp) + .clip(RoundedCornerShape(8.dp)) + .background(StatusRed.copy(alpha = 0.15f)) + .padding(horizontal = 12.dp, vertical = 8.dp) + ) { + Text( + text = errorMessage ?: "Connection error", + style = MaterialTheme.typography.bodySmall, + color = StatusRed + ) + } + } + + NavHost( + navController = navController, + startDestination = startDestination, + modifier = Modifier.weight(1f) + ) { composable(Screen.Onboarding.route) { OnboardingScreen( onComplete = { @@ -201,5 +250,6 @@ fun MainNavigation() { composable(Screen.Settings.route) { SettingsScreen() } composable(Screen.Logs.route) { LogsScreen() } } + } } } diff --git a/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt b/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt index 22cefae..ea920d4 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt @@ -58,6 +58,7 @@ class ConnectionService : LifecycleService() { const val ACTION_CONNECT = "com.thisux.droidclaw.CONNECT" const val ACTION_DISCONNECT = "com.thisux.droidclaw.DISCONNECT" const val ACTION_SEND_GOAL = "com.thisux.droidclaw.SEND_GOAL" + const val ACTION_SHOW_COMMAND_PANEL = "com.thisux.droidclaw.SHOW_COMMAND_PANEL" const val EXTRA_GOAL = "goal_text" } @@ -108,6 +109,9 @@ class ConnectionService : LifecycleService() { val goal = intent.getStringExtra(EXTRA_GOAL) ?: return START_NOT_STICKY sendGoal(goal) } + ACTION_SHOW_COMMAND_PANEL -> { + overlay?.showCommandPanel() + } } return START_NOT_STICKY diff --git a/android/app/src/main/java/com/thisux/droidclaw/overlay/AgentOverlay.kt b/android/app/src/main/java/com/thisux/droidclaw/overlay/AgentOverlay.kt index 68bf974..b08cab7 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/overlay/AgentOverlay.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/overlay/AgentOverlay.kt @@ -177,6 +177,11 @@ class AgentOverlay(private val service: LifecycleService) { mode.value = OverlayMode.Idle } + fun showCommandPanel() { + hide() + commandPanel.show() + } + // ── Private: Pill overlay ─────────────────────────────── private fun showPill() { diff --git a/android/app/src/main/java/com/thisux/droidclaw/ui/screens/HomeScreen.kt b/android/app/src/main/java/com/thisux/droidclaw/ui/screens/HomeScreen.kt index 7ad924f..e12482b 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/ui/screens/HomeScreen.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/ui/screens/HomeScreen.kt @@ -45,16 +45,26 @@ import androidx.compose.ui.graphics.Color import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.unit.dp +import com.thisux.droidclaw.DroidClawApp import com.thisux.droidclaw.connection.ConnectionService import com.thisux.droidclaw.model.AgentStep import com.thisux.droidclaw.model.ConnectionState import com.thisux.droidclaw.model.GoalStatus import com.thisux.droidclaw.ui.theme.StatusGreen import com.thisux.droidclaw.ui.theme.StatusRed +import androidx.compose.material3.Card +import androidx.compose.material3.CardDefaults import java.text.SimpleDateFormat import java.util.Date import java.util.Locale +private val DEFAULT_SUGGESTIONS = listOf( + "Open WhatsApp and reply to the last message", + "Take a screenshot and save it", + "Turn on Do Not Disturb", + "Search for nearby restaurants on Maps" +) + // Represents a message in the chat timeline private sealed class ChatItem { data class GoalMessage(val text: String) : ChatItem() @@ -65,13 +75,28 @@ private sealed class ChatItem { @Composable fun HomeScreen() { val context = LocalContext.current + val app = context.applicationContext as DroidClawApp val connectionState by ConnectionService.connectionState.collectAsState() val goalStatus by ConnectionService.currentGoalStatus.collectAsState() val steps by ConnectionService.currentSteps.collectAsState() val currentGoal by ConnectionService.currentGoal.collectAsState() + val recentGoals by app.settingsStore.recentGoals.collectAsState(initial = emptyList()) var goalInput by remember { mutableStateOf("") } + val isConnected = connectionState == ConnectionState.Connected + val canSend = isConnected && goalStatus != GoalStatus.Running + + val suggestions = remember(recentGoals) { + val combined = mutableListOf() + combined.addAll(recentGoals.take(4)) + for (default in DEFAULT_SUGGESTIONS) { + if (combined.size >= 4) break + if (default !in combined) combined.add(default) + } + combined.take(4) + } + // Build chat items: goal bubble → step bubbles → status bubble val chatItems = remember(currentGoal, steps, goalStatus) { buildList { @@ -105,7 +130,10 @@ fun HomeScreen() { .fillMaxWidth(), contentAlignment = Alignment.Center ) { - Column(horizontalAlignment = Alignment.CenterHorizontally) { + Column( + horizontalAlignment = Alignment.CenterHorizontally, + modifier = Modifier.padding(horizontal = 20.dp) + ) { Text( text = "What should I do?", style = MaterialTheme.typography.headlineSmall, @@ -117,6 +145,33 @@ fun HomeScreen() { style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant.copy(alpha = 0.35f) ) + Spacer(modifier = Modifier.height(24.dp)) + Column(verticalArrangement = Arrangement.spacedBy(8.dp)) { + for (row in suggestions.chunked(2)) { + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.spacedBy(8.dp) + ) { + for (suggestion in row) { + SuggestionCard( + text = suggestion, + enabled = canSend, + onClick = { + val intent = Intent(context, ConnectionService::class.java).apply { + action = ConnectionService.ACTION_SEND_GOAL + putExtra(ConnectionService.EXTRA_GOAL, suggestion) + } + context.startService(intent) + }, + modifier = Modifier.weight(1f) + ) + } + if (row.size < 2) { + Spacer(modifier = Modifier.weight(1f)) + } + } + } + } } } } else { @@ -366,6 +421,39 @@ private fun InputBar( } } +@Composable +private fun SuggestionCard( + text: String, + enabled: Boolean, + onClick: () -> Unit, + modifier: Modifier = Modifier +) { + Card( + onClick = onClick, + modifier = modifier.height(72.dp), + enabled = enabled, + shape = RoundedCornerShape(16.dp), + colors = CardDefaults.cardColors( + containerColor = MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.4f) + ) + ) { + Box( + modifier = Modifier + .fillMaxSize() + .padding(12.dp), + contentAlignment = Alignment.CenterStart + ) { + Text( + text = text, + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurface, + maxLines = 3, + overflow = TextOverflow.Ellipsis + ) + } + } +} + private fun formatTime(timestamp: Long): String { val sdf = SimpleDateFormat("HH:mm", Locale.getDefault()) return sdf.format(Date(timestamp)) diff --git a/android/app/src/main/java/com/thisux/droidclaw/ui/screens/OnboardingScreen.kt b/android/app/src/main/java/com/thisux/droidclaw/ui/screens/OnboardingScreen.kt index 7dd4782..6f13528 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/ui/screens/OnboardingScreen.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/ui/screens/OnboardingScreen.kt @@ -1,10 +1,12 @@ package com.thisux.droidclaw.ui.screens import android.app.Activity +import android.app.role.RoleManager import android.content.Context import android.content.Intent import android.media.projection.MediaProjectionManager import android.net.Uri +import android.os.Build import android.provider.Settings import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.result.contract.ActivityResultContracts @@ -32,6 +34,7 @@ import androidx.compose.material3.MaterialTheme import androidx.compose.material3.OutlinedButton import androidx.compose.material3.OutlinedTextField import androidx.compose.material3.Text +import androidx.compose.material3.TextButton import androidx.compose.runtime.Composable import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.collectAsState @@ -99,10 +102,22 @@ fun OnboardingScreen(onComplete: () -> Unit) { } ) 1 -> OnboardingStepTwo( - onGetStarted = { + onContinue = { currentStep = 2 } + ) + 2 -> OnboardingStepAssistant( + onContinue = { + scope.launch { + app.settingsStore.setHasOnboarded(true) + val intent = Intent(context, ConnectionService::class.java).apply { + action = ConnectionService.ACTION_CONNECT + } + context.startForegroundService(intent) + onComplete() + } + }, + onSkip = { scope.launch { app.settingsStore.setHasOnboarded(true) - // Auto-connect val intent = Intent(context, ConnectionService::class.java).apply { action = ConnectionService.ACTION_CONNECT } @@ -191,7 +206,7 @@ private fun OnboardingStepOne( } @Composable -private fun OnboardingStepTwo(onGetStarted: () -> Unit) { +private fun OnboardingStepTwo(onContinue: () -> Unit) { val context = LocalContext.current val isCaptureAvailable by ScreenCaptureManager.isAvailable.collectAsState() @@ -312,14 +327,14 @@ private fun OnboardingStepTwo(onGetStarted: () -> Unit) { Spacer(modifier = Modifier.height(32.dp)) Button( - onClick = onGetStarted, + onClick = onContinue, enabled = allGranted, modifier = Modifier .fillMaxWidth() .height(52.dp), shape = RoundedCornerShape(12.dp) ) { - Text("Get Started", style = MaterialTheme.typography.labelLarge) + Text("Continue", style = MaterialTheme.typography.labelLarge) } if (!allGranted) { @@ -335,6 +350,92 @@ private fun OnboardingStepTwo(onGetStarted: () -> Unit) { } } +@Composable +private fun OnboardingStepAssistant( + onContinue: () -> Unit, + onSkip: () -> Unit +) { + val context = LocalContext.current + + var isDefaultAssistant by remember { + mutableStateOf( + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + val rm = context.getSystemService(Context.ROLE_SERVICE) as RoleManager + rm.isRoleHeld(RoleManager.ROLE_ASSISTANT) + } else false + ) + } + + val lifecycleOwner = LocalLifecycleOwner.current + DisposableEffect(lifecycleOwner) { + val observer = LifecycleEventObserver { _, event -> + if (event == Lifecycle.Event.ON_RESUME) { + isDefaultAssistant = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + val rm = context.getSystemService(Context.ROLE_SERVICE) as RoleManager + rm.isRoleHeld(RoleManager.ROLE_ASSISTANT) + } else false + } + } + lifecycleOwner.lifecycle.addObserver(observer) + onDispose { lifecycleOwner.lifecycle.removeObserver(observer) } + } + + Column( + modifier = Modifier + .fillMaxSize() + .padding(horizontal = 24.dp, vertical = 48.dp) + .verticalScroll(rememberScrollState()), + horizontalAlignment = Alignment.CenterHorizontally + ) { + Spacer(modifier = Modifier.height(32.dp)) + + Text( + text = "Digital Assistant", + style = MaterialTheme.typography.headlineMedium, + color = MaterialTheme.colorScheme.onBackground + ) + + Spacer(modifier = Modifier.height(8.dp)) + + Text( + text = "Set DroidClaw as your default digital assistant to invoke it with a long-press on the home button", + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant, + textAlign = TextAlign.Center + ) + + Spacer(modifier = Modifier.height(32.dp)) + + OnboardingChecklistItem( + label = "Default Digital Assistant", + description = "Long-press home to open DroidClaw command panel", + isOk = isDefaultAssistant, + actionLabel = "Set", + onAction = { + context.startActivity(Intent(Settings.ACTION_VOICE_INPUT_SETTINGS)) + } + ) + + Spacer(modifier = Modifier.height(32.dp)) + + Button( + onClick = onContinue, + modifier = Modifier + .fillMaxWidth() + .height(52.dp), + shape = RoundedCornerShape(12.dp) + ) { + Text("Get Started", style = MaterialTheme.typography.labelLarge) + } + + Spacer(modifier = Modifier.height(8.dp)) + + TextButton(onClick = onSkip) { + Text("Skip for now") + } + } +} + @Composable private fun OnboardingChecklistItem( label: String, diff --git a/android/app/src/main/java/com/thisux/droidclaw/voice/DroidClawRecognitionService.kt b/android/app/src/main/java/com/thisux/droidclaw/voice/DroidClawRecognitionService.kt new file mode 100644 index 0000000..e6dc870 --- /dev/null +++ b/android/app/src/main/java/com/thisux/droidclaw/voice/DroidClawRecognitionService.kt @@ -0,0 +1,10 @@ +package com.thisux.droidclaw.voice + +import android.content.Intent +import android.speech.RecognitionService + +class DroidClawRecognitionService : RecognitionService() { + override fun onStartListening(intent: Intent?, callback: Callback?) {} + override fun onCancel(callback: Callback?) {} + override fun onStopListening(callback: Callback?) {} +} diff --git a/android/app/src/main/java/com/thisux/droidclaw/voice/DroidClawVoiceSession.kt b/android/app/src/main/java/com/thisux/droidclaw/voice/DroidClawVoiceSession.kt index a7e3d64..90c039f 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/voice/DroidClawVoiceSession.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/voice/DroidClawVoiceSession.kt @@ -4,71 +4,14 @@ import android.content.Context import android.content.Intent import android.os.Bundle import android.service.voice.VoiceInteractionSession -import android.view.View -import androidx.compose.ui.platform.ComposeView -import androidx.lifecycle.Lifecycle -import androidx.lifecycle.LifecycleOwner -import androidx.lifecycle.LifecycleRegistry -import androidx.lifecycle.setViewTreeLifecycleOwner -import androidx.savedstate.SavedStateRegistry -import androidx.savedstate.SavedStateRegistryController -import androidx.savedstate.SavedStateRegistryOwner -import androidx.savedstate.setViewTreeSavedStateRegistryOwner import com.thisux.droidclaw.connection.ConnectionService -import com.thisux.droidclaw.ui.theme.DroidClawTheme class DroidClawVoiceSession(context: Context) : VoiceInteractionSession(context) { - private val lifecycleOwner = object : LifecycleOwner { - val registry = LifecycleRegistry(this) - override val lifecycle: Lifecycle get() = registry - } - - private val savedStateOwner = object : SavedStateRegistryOwner { - private val controller = SavedStateRegistryController.create(this) - override val lifecycle: Lifecycle get() = lifecycleOwner.lifecycle - override val savedStateRegistry: SavedStateRegistry get() = controller.savedStateRegistry - init { controller.performRestore(null) } - } - - override fun onCreateContentView(): View { - lifecycleOwner.registry.currentState = Lifecycle.State.CREATED - - return ComposeView(context).apply { - setViewTreeLifecycleOwner(lifecycleOwner) - setViewTreeSavedStateRegistryOwner(savedStateOwner) - setContent { - DroidClawTheme { - GoalInputSheet( - onSubmit = { goal -> submitGoal(goal) }, - onDismiss = { hide() } - ) - } - } - } - } - override fun onShow(args: Bundle?, showFlags: Int) { super.onShow(args, showFlags) - lifecycleOwner.registry.currentState = Lifecycle.State.STARTED - lifecycleOwner.registry.currentState = Lifecycle.State.RESUMED - } - - override fun onHide() { - lifecycleOwner.registry.currentState = Lifecycle.State.STARTED - lifecycleOwner.registry.currentState = Lifecycle.State.CREATED - super.onHide() - } - - override fun onDestroy() { - lifecycleOwner.registry.currentState = Lifecycle.State.DESTROYED - super.onDestroy() - } - - private fun submitGoal(goal: String) { val intent = Intent(context, ConnectionService::class.java).apply { - action = ConnectionService.ACTION_SEND_GOAL - putExtra(ConnectionService.EXTRA_GOAL, goal) + action = ConnectionService.ACTION_SHOW_COMMAND_PANEL } context.startService(intent) hide() diff --git a/android/app/src/main/res/xml/voice_interaction_service.xml b/android/app/src/main/res/xml/voice_interaction_service.xml index ef25a6f..8b4afee 100644 --- a/android/app/src/main/res/xml/voice_interaction_service.xml +++ b/android/app/src/main/res/xml/voice_interaction_service.xml @@ -1,6 +1,7 @@