diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index 8111fbf..52ab4f5 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -11,6 +11,7 @@ + >(emptyList()) var instance: DroidClawAccessibilityService? = null + + fun isEnabledOnDevice(context: Context): Boolean { + val am = context.getSystemService(Context.ACCESSIBILITY_SERVICE) as AccessibilityManager + val ourComponent = ComponentName(context, DroidClawAccessibilityService::class.java) + return am.getEnabledAccessibilityServiceList(AccessibilityEvent.TYPES_ALL_MASK) + .any { it.resolveInfo.serviceInfo.let { si -> + ComponentName(si.packageName, si.name) == ourComponent + }} + } } override fun onServiceConnected() { diff --git a/android/app/src/main/java/com/thisux/droidclaw/accessibility/ScreenTreeBuilder.kt b/android/app/src/main/java/com/thisux/droidclaw/accessibility/ScreenTreeBuilder.kt index 918f9ab..3f28b58 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/accessibility/ScreenTreeBuilder.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/accessibility/ScreenTreeBuilder.kt @@ -21,6 +21,9 @@ object ScreenTreeBuilder { parentDesc: String ) { try { + // Skip DroidClaw's own overlay nodes so the agent never sees them + if (node.packageName?.toString() == "com.thisux.droidclaw") return + val rect = Rect() node.getBoundsInScreen(rect) diff --git a/android/app/src/main/java/com/thisux/droidclaw/capture/ScreenCaptureManager.kt b/android/app/src/main/java/com/thisux/droidclaw/capture/ScreenCaptureManager.kt index d006d42..67386f5 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/capture/ScreenCaptureManager.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/capture/ScreenCaptureManager.kt @@ -22,6 +22,9 @@ class ScreenCaptureManager(private val context: Context) { companion object { private const val TAG = "ScreenCapture" + private const val PREFS_NAME = "screen_capture" + private const val KEY_RESULT_CODE = "consent_result_code" + private const val KEY_CONSENT_URI = "consent_data_uri" val isAvailable = MutableStateFlow(false) // Stores MediaProjection consent for use by ConnectionService @@ -37,6 +40,37 @@ class ScreenCaptureManager(private val context: Context) { hasConsentState.value = (resultCode == Activity.RESULT_OK && data != null) } + fun storeConsent(context: Context, resultCode: Int, data: Intent?) { + storeConsent(resultCode, data) + if (resultCode == Activity.RESULT_OK && data != null) { + context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE).edit() + .putInt(KEY_RESULT_CODE, resultCode) + .putString(KEY_CONSENT_URI, data.toUri(0)) + .apply() + } + } + + fun restoreConsent(context: Context) { + if (consentResultCode != null && consentData != null) return + val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) + val code = prefs.getInt(KEY_RESULT_CODE, 0) + val uri = prefs.getString(KEY_CONSENT_URI, null) + if (code == Activity.RESULT_OK && uri != null) { + consentResultCode = code + consentData = Intent.parseUri(uri, 0) + hasConsentState.value = true + } + } + + fun clearConsent(context: Context) { + consentResultCode = null + consentData = null + hasConsentState.value = false + context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE).edit() + .clear() + .apply() + } + fun hasConsent(): Boolean = consentResultCode != null && consentData != null } diff --git a/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt b/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt index 04468a5..a267066 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt @@ -27,6 +27,9 @@ import com.thisux.droidclaw.model.InstalledAppInfo import com.thisux.droidclaw.util.DeviceInfoHelper import android.content.pm.PackageManager import android.net.Uri +import android.provider.Settings +import com.thisux.droidclaw.model.StopGoalMessage +import com.thisux.droidclaw.overlay.AgentOverlay import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.first @@ -56,11 +59,13 @@ class ConnectionService : LifecycleService() { private var commandRouter: CommandRouter? = null private var captureManager: ScreenCaptureManager? = null private var wakeLock: PowerManager.WakeLock? = null + private var overlay: AgentOverlay? = null override fun onCreate() { super.onCreate() instance = this createNotificationChannel() + overlay = AgentOverlay(this) } override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { @@ -97,6 +102,7 @@ class ConnectionService : LifecycleService() { return@launch } + ScreenCaptureManager.restoreConsent(this@ConnectionService) captureManager = ScreenCaptureManager(this@ConnectionService).also { mgr -> if (ScreenCaptureManager.hasConsent()) { try { @@ -105,7 +111,8 @@ class ConnectionService : LifecycleService() { ScreenCaptureManager.consentData!! ) } catch (e: SecurityException) { - Log.w(TAG, "Screen capture unavailable (needs mediaProjection service type): ${e.message}") + Log.w(TAG, "Screen capture unavailable: ${e.message}") + ScreenCaptureManager.clearConsent(this@ConnectionService) } } } @@ -131,6 +138,9 @@ class ConnectionService : LifecycleService() { ) // Send installed apps list once connected if (state == ConnectionState.Connected) { + if (Settings.canDrawOverlays(this@ConnectionService)) { + overlay?.show() + } val apps = getInstalledApps() webSocket?.sendTyped(AppsMessage(apps = apps)) Log.i(TAG, "Sent ${apps.size} installed apps to server") @@ -167,7 +177,12 @@ class ConnectionService : LifecycleService() { webSocket?.sendTyped(GoalMessage(text = text)) } + fun stopGoal() { + webSocket?.sendTyped(StopGoalMessage()) + } + private fun disconnect() { + overlay?.hide() webSocket?.disconnect() webSocket = null commandRouter?.reset() @@ -179,6 +194,8 @@ class ConnectionService : LifecycleService() { } override fun onDestroy() { + overlay?.destroy() + overlay = null disconnect() instance = null super.onDestroy() diff --git a/android/app/src/main/java/com/thisux/droidclaw/model/Protocol.kt b/android/app/src/main/java/com/thisux/droidclaw/model/Protocol.kt index 0c9b4ba..a68ecc6 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/model/Protocol.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/model/Protocol.kt @@ -71,6 +71,11 @@ data class AppsMessage( val apps: List ) +@Serializable +data class StopGoalMessage( + val type: String = "stop_goal" +) + @Serializable data class ServerMessage( val type: String, diff --git a/android/app/src/main/java/com/thisux/droidclaw/overlay/AgentOverlay.kt b/android/app/src/main/java/com/thisux/droidclaw/overlay/AgentOverlay.kt new file mode 100644 index 0000000..2818101 --- /dev/null +++ b/android/app/src/main/java/com/thisux/droidclaw/overlay/AgentOverlay.kt @@ -0,0 +1,92 @@ +package com.thisux.droidclaw.overlay + +import android.graphics.PixelFormat +import android.view.Gravity +import android.view.MotionEvent +import android.view.View +import android.view.WindowManager +import androidx.compose.ui.platform.ComposeView +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.LifecycleService +import androidx.lifecycle.setViewTreeLifecycleOwner +import androidx.savedstate.SavedStateRegistry +import androidx.savedstate.SavedStateRegistryController +import androidx.savedstate.SavedStateRegistryOwner +import androidx.savedstate.setViewTreeSavedStateRegistryOwner + +class AgentOverlay(private val service: LifecycleService) { + + private val windowManager = service.getSystemService(WindowManager::class.java) + private var composeView: ComposeView? = null + + private val savedStateOwner = object : SavedStateRegistryOwner { + private val controller = SavedStateRegistryController.create(this) + override val lifecycle: Lifecycle get() = service.lifecycle + override val savedStateRegistry: SavedStateRegistry get() = controller.savedStateRegistry + init { controller.performRestore(null) } + } + + private val layoutParams = WindowManager.LayoutParams( + WindowManager.LayoutParams.WRAP_CONTENT, + WindowManager.LayoutParams.WRAP_CONTENT, + WindowManager.LayoutParams.TYPE_APPLICATION_OVERLAY, + WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE, + PixelFormat.TRANSLUCENT + ).apply { + gravity = Gravity.TOP or Gravity.START + x = 0 + y = 200 + } + + fun show() { + if (composeView != null) return + + val view = ComposeView(service).apply { + importantForAccessibility = View.IMPORTANT_FOR_ACCESSIBILITY_NO_HIDE_DESCENDANTS + setViewTreeLifecycleOwner(service) + setViewTreeSavedStateRegistryOwner(savedStateOwner) + setContent { OverlayContent() } + setupDrag(this) + } + + composeView = view + windowManager.addView(view, layoutParams) + } + + fun hide() { + composeView?.let { + windowManager.removeView(it) + } + composeView = null + } + + fun destroy() { + hide() + } + + private fun setupDrag(view: View) { + var initialX = 0 + var initialY = 0 + var initialTouchX = 0f + var initialTouchY = 0f + + view.setOnTouchListener { _, event -> + when (event.action) { + MotionEvent.ACTION_DOWN -> { + initialX = layoutParams.x + initialY = layoutParams.y + initialTouchX = event.rawX + initialTouchY = event.rawY + true + } + MotionEvent.ACTION_MOVE -> { + layoutParams.x = initialX + (event.rawX - initialTouchX).toInt() + layoutParams.y = initialY + (event.rawY - initialTouchY).toInt() + windowManager.updateViewLayout(view, layoutParams) + true + } + else -> false + } + } + } +} diff --git a/android/app/src/main/java/com/thisux/droidclaw/overlay/OverlayContent.kt b/android/app/src/main/java/com/thisux/droidclaw/overlay/OverlayContent.kt new file mode 100644 index 0000000..6a0e3a9 --- /dev/null +++ b/android/app/src/main/java/com/thisux/droidclaw/overlay/OverlayContent.kt @@ -0,0 +1,165 @@ +package com.thisux.droidclaw.overlay + +import androidx.compose.animation.animateColorAsState +import androidx.compose.animation.core.LinearEasing +import androidx.compose.animation.core.RepeatMode +import androidx.compose.animation.core.animateFloat +import androidx.compose.animation.core.infiniteRepeatable +import androidx.compose.animation.core.rememberInfiniteTransition +import androidx.compose.animation.core.tween +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.widthIn +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.IconButtonDefaults +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Close +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.alpha +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import com.thisux.droidclaw.connection.ConnectionService +import com.thisux.droidclaw.model.ConnectionState +import com.thisux.droidclaw.model.GoalStatus +import com.thisux.droidclaw.ui.theme.DroidClawTheme +import kotlinx.coroutines.delay + +private val Green = Color(0xFF4CAF50) +private val Blue = Color(0xFF2196F3) +private val Red = Color(0xFFF44336) +private val Gray = Color(0xFF9E9E9E) +private val PillBackground = Color(0xE6212121) + +@Composable +fun OverlayContent() { + DroidClawTheme { + val connectionState by ConnectionService.connectionState.collectAsState() + val goalStatus by ConnectionService.currentGoalStatus.collectAsState() + val steps by ConnectionService.currentSteps.collectAsState() + + // Auto-reset Completed/Failed back to Idle after 3s + var displayStatus by remember { mutableStateOf(goalStatus) } + LaunchedEffect(goalStatus) { + displayStatus = goalStatus + if (goalStatus == GoalStatus.Completed || goalStatus == GoalStatus.Failed) { + delay(3000) + displayStatus = GoalStatus.Idle + } + } + + val isConnected = connectionState == ConnectionState.Connected + + val dotColor by animateColorAsState( + targetValue = when { + !isConnected -> Gray + displayStatus == GoalStatus.Running -> Blue + displayStatus == GoalStatus.Failed -> Red + else -> Green + }, + label = "dotColor" + ) + + val statusText = when { + !isConnected -> "Offline" + displayStatus == GoalStatus.Running -> { + val last = steps.lastOrNull() + if (last != null) "Step ${last.step}: ${last.action}" else "Running..." + } + displayStatus == GoalStatus.Completed -> "Done" + displayStatus == GoalStatus.Failed -> "Stopped" + else -> "Ready" + } + + Row( + modifier = Modifier + .clip(RoundedCornerShape(24.dp)) + .background(PillBackground) + .height(48.dp) + .widthIn(min = 100.dp, max = 220.dp) + .padding(horizontal = 12.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(8.dp) + ) { + StatusDot( + color = dotColor, + pulse = isConnected && displayStatus == GoalStatus.Running + ) + + Text( + text = statusText, + color = Color.White, + fontSize = 13.sp, + maxLines = 1, + overflow = TextOverflow.Ellipsis, + modifier = Modifier.weight(1f, fill = false) + ) + + if (isConnected && displayStatus == GoalStatus.Running) { + IconButton( + onClick = { ConnectionService.instance?.stopGoal() }, + modifier = Modifier.size(28.dp), + colors = IconButtonDefaults.iconButtonColors( + contentColor = Color.White.copy(alpha = 0.8f) + ) + ) { + Icon( + imageVector = Icons.Default.Close, + contentDescription = "Stop goal", + modifier = Modifier.size(16.dp) + ) + } + } + } + } +} + +@Composable +private fun StatusDot(color: Color, pulse: Boolean) { + if (pulse) { + val transition = rememberInfiniteTransition(label = "pulse") + val alpha by transition.animateFloat( + initialValue = 1f, + targetValue = 0.3f, + animationSpec = infiniteRepeatable( + animation = tween(800, easing = LinearEasing), + repeatMode = RepeatMode.Reverse + ), + label = "pulseAlpha" + ) + Box( + modifier = Modifier + .size(10.dp) + .alpha(alpha) + .clip(CircleShape) + .background(color) + ) + } else { + Box( + modifier = Modifier + .size(10.dp) + .clip(CircleShape) + .background(color) + ) + } +} diff --git a/android/app/src/main/java/com/thisux/droidclaw/ui/screens/SettingsScreen.kt b/android/app/src/main/java/com/thisux/droidclaw/ui/screens/SettingsScreen.kt index 2696d84..2cc0c40 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/ui/screens/SettingsScreen.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/ui/screens/SettingsScreen.kt @@ -2,7 +2,10 @@ package com.thisux.droidclaw.ui.screens import android.app.Activity import android.content.Context +import android.content.Intent import android.media.projection.MediaProjectionManager +import android.net.Uri +import android.provider.Settings import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.result.contract.ActivityResultContracts import androidx.compose.foundation.layout.Arrangement @@ -62,18 +65,27 @@ fun SettingsScreen() { var editingServerUrl by remember { mutableStateOf(null) } val displayServerUrl = editingServerUrl ?: serverUrl - val isAccessibilityEnabled by DroidClawAccessibilityService.isRunning.collectAsState() val isCaptureAvailable by ScreenCaptureManager.isAvailable.collectAsState() - val hasConsent by ScreenCaptureManager.hasConsentState.collectAsState() - val hasCaptureConsent = isCaptureAvailable || hasConsent + var isAccessibilityEnabled by remember { + mutableStateOf(DroidClawAccessibilityService.isEnabledOnDevice(context)) + } + var hasCaptureConsent by remember { + ScreenCaptureManager.restoreConsent(context) + mutableStateOf(isCaptureAvailable || ScreenCaptureManager.hasConsent()) + } var isBatteryExempt by remember { mutableStateOf(BatteryOptimization.isIgnoringBatteryOptimizations(context)) } + var hasOverlayPermission by remember { mutableStateOf(Settings.canDrawOverlays(context)) } val lifecycleOwner = LocalLifecycleOwner.current DisposableEffect(lifecycleOwner) { val observer = LifecycleEventObserver { _, event -> if (event == Lifecycle.Event.ON_RESUME) { + isAccessibilityEnabled = DroidClawAccessibilityService.isEnabledOnDevice(context) + ScreenCaptureManager.restoreConsent(context) + hasCaptureConsent = isCaptureAvailable || ScreenCaptureManager.hasConsent() isBatteryExempt = BatteryOptimization.isIgnoringBatteryOptimizations(context) + hasOverlayPermission = Settings.canDrawOverlays(context) } } lifecycleOwner.lifecycle.addObserver(observer) @@ -84,7 +96,8 @@ fun SettingsScreen() { ActivityResultContracts.StartActivityForResult() ) { result -> if (result.resultCode == Activity.RESULT_OK && result.data != null) { - ScreenCaptureManager.storeConsent(result.resultCode, result.data) + ScreenCaptureManager.storeConsent(context, result.resultCode, result.data) + hasCaptureConsent = true } } @@ -172,6 +185,20 @@ fun SettingsScreen() { actionLabel = "Disable", onAction = { BatteryOptimization.requestExemption(context) } ) + + ChecklistItem( + label = "Overlay permission", + isOk = hasOverlayPermission, + actionLabel = "Grant", + onAction = { + context.startActivity( + Intent( + Settings.ACTION_MANAGE_OVERLAY_PERMISSION, + Uri.parse("package:${context.packageName}") + ) + ) + } + ) } } diff --git a/packages/shared/src/protocol.ts b/packages/shared/src/protocol.ts index 38f3755..5aeda72 100644 --- a/packages/shared/src/protocol.ts +++ b/packages/shared/src/protocol.ts @@ -7,7 +7,8 @@ export type DeviceMessage = | { type: "goal"; text: string } | { type: "pong" } | { type: "heartbeat"; batteryLevel: number; isCharging: boolean } - | { type: "apps"; apps: InstalledApp[] }; + | { type: "apps"; apps: InstalledApp[] } + | { type: "stop_goal" }; export type ServerToDeviceMessage = | { type: "auth_ok"; deviceId: string } diff --git a/server/src/ws/device.ts b/server/src/ws/device.ts index 6016d90..ba8a5bd 100644 --- a/server/src/ws/device.ts +++ b/server/src/ws/device.ts @@ -22,7 +22,7 @@ async function hashApiKey(key: string): Promise { } /** Track running agent sessions to prevent duplicates per device */ -const activeSessions = new Map(); +const activeSessions = new Map(); /** * Send a JSON message to a device WebSocket (safe — catches send errors). @@ -252,7 +252,8 @@ export async function handleDeviceMessage( } console.log(`[Pipeline] Starting goal for device ${deviceId}: ${goal}`); - activeSessions.set(deviceId, goal); + const abortController = new AbortController(); + activeSessions.set(deviceId, { goal, abort: abortController }); sendToDevice(ws, { type: "goal_started", sessionId: deviceId, goal }); @@ -262,6 +263,7 @@ export async function handleDeviceMessage( userId, goal, llmConfig: userLlmConfig, + signal: abortController.signal, onStep(step) { sendToDevice(ws, { type: "step", @@ -294,6 +296,23 @@ export async function handleDeviceMessage( break; } + case "stop_goal": { + const deviceId = ws.data.deviceId!; + const active = activeSessions.get(deviceId); + if (active) { + console.log(`[Pipeline] Stop requested for device ${deviceId}`); + active.abort.abort(); + activeSessions.delete(deviceId); + sendToDevice(ws, { + type: "goal_completed", + sessionId: deviceId, + success: false, + stepsUsed: 0, + }); + } + break; + } + case "apps": { const persistentDeviceId = ws.data.persistentDeviceId; if (persistentDeviceId) { @@ -360,7 +379,11 @@ export function handleDeviceClose( const { deviceId, userId, persistentDeviceId } = ws.data; if (!deviceId) return; - activeSessions.delete(deviceId); + const active = activeSessions.get(deviceId); + if (active) { + active.abort.abort(); + activeSessions.delete(deviceId); + } sessions.removeDevice(deviceId); // Update device status in DB