thomas/droidclaw
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: agent overlay, stop-goal support, and state persistence across app kill

Browse Source 
- Add draggable agent overlay pill (status dot + step text + stop button)
  that shows over other apps while connected. Fix ComposeView rendering
  in service context by providing a SavedStateRegistryOwner.
- Add stop_goal protocol message so the overlay/client can abort a
  running agent session; server aborts via AbortController.
- Persist screen-capture consent to SharedPreferences so it survives
  process death; restore on ConnectionService connect and Settings resume.
- Query AccessibilityManager for real service state instead of relying
  on in-process MutableStateFlow that resets on restart.
- Add overlay permission checklist item and SYSTEM_ALERT_WINDOW manifest
  entry.
- Filter DroidClaw's own overlay nodes from the accessibility tree so the
  agent never interacts with them.
This commit is contained in:
Somasundaram Mahesh
2026-02-18 18:49:13 +05:30
parent 88af77ddc7
commit 011e2be291
11 changed files with 389 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
android/app/src/main/AndroidManifest.xml
View File

@@ -11,6 +11,7 @@
<uses-permission android:name="android.permission.CHANGE_NETWORK_STATE" />
<uses-permission android:name="android.permission.QUERY_ALL_PACKAGES"
tools:ignore="QueryAllPackagesPermission" />
<uses-permission android:name="android.permission.SYSTEM_ALERT_WINDOW" />
<application
android:name=".DroidClawApp"

12
android/app/src/main/java/com/thisux/droidclaw/accessibility/DroidClawAccessibilityService.kt
View File

@@ -1,8 +1,11 @@
package com.thisux.droidclaw.accessibility
import android.accessibilityservice.AccessibilityService
import android.content.ComponentName
import android.content.Context
import android.util.Log
import android.view.accessibility.AccessibilityEvent
import android.view.accessibility.AccessibilityManager
import android.view.accessibility.AccessibilityNodeInfo
import com.thisux.droidclaw.model.UIElement
import kotlinx.coroutines.delay
@@ -16,6 +19,15 @@ class DroidClawAccessibilityService : AccessibilityService() {
val isRunning = MutableStateFlow(false)
val lastScreenTree = MutableStateFlow<List<UIElement>>(emptyList())
var instance: DroidClawAccessibilityService? = null
fun isEnabledOnDevice(context: Context): Boolean {
val am = context.getSystemService(Context.ACCESSIBILITY_SERVICE) as AccessibilityManager
val ourComponent = ComponentName(context, DroidClawAccessibilityService::class.java)
return am.getEnabledAccessibilityServiceList(AccessibilityEvent.TYPES_ALL_MASK)
.any { it.resolveInfo.serviceInfo.let { si ->
ComponentName(si.packageName, si.name) == ourComponent
}}
}
}
override fun onServiceConnected() {

3
android/app/src/main/java/com/thisux/droidclaw/accessibility/ScreenTreeBuilder.kt
View File

@@ -21,6 +21,9 @@ object ScreenTreeBuilder {
parentDesc: String
) {
try {
// Skip DroidClaw's own overlay nodes so the agent never sees them
if (node.packageName?.toString() == "com.thisux.droidclaw") return
val rect = Rect()
node.getBoundsInScreen(rect)

34
android/app/src/main/java/com/thisux/droidclaw/capture/ScreenCaptureManager.kt
View File

@@ -22,6 +22,9 @@ class ScreenCaptureManager(private val context: Context) {
companion object {
private const val TAG = "ScreenCapture"
private const val PREFS_NAME = "screen_capture"
private const val KEY_RESULT_CODE = "consent_result_code"
private const val KEY_CONSENT_URI = "consent_data_uri"
val isAvailable = MutableStateFlow(false)
// Stores MediaProjection consent for use by ConnectionService
@@ -37,6 +40,37 @@ class ScreenCaptureManager(private val context: Context) {
hasConsentState.value = (resultCode == Activity.RESULT_OK && data != null)
}
fun storeConsent(context: Context, resultCode: Int, data: Intent?) {
storeConsent(resultCode, data)
if (resultCode == Activity.RESULT_OK && data != null) {
context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE).edit()
.putInt(KEY_RESULT_CODE, resultCode)
.putString(KEY_CONSENT_URI, data.toUri(0))
.apply()
}
}
fun restoreConsent(context: Context) {
if (consentResultCode != null && consentData != null) return
val prefs = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
val code = prefs.getInt(KEY_RESULT_CODE, 0)
val uri = prefs.getString(KEY_CONSENT_URI, null)
if (code == Activity.RESULT_OK && uri != null) {
consentResultCode = code
consentData = Intent.parseUri(uri, 0)
hasConsentState.value = true
}
}
fun clearConsent(context: Context) {
consentResultCode = null
consentData = null
hasConsentState.value = false
context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE).edit()
.clear()
.apply()
}
fun hasConsent(): Boolean = consentResultCode != null && consentData != null
}

19
android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt
View File

@@ -27,6 +27,9 @@ import com.thisux.droidclaw.model.InstalledAppInfo
import com.thisux.droidclaw.util.DeviceInfoHelper
import android.content.pm.PackageManager
import android.net.Uri
import android.provider.Settings
import com.thisux.droidclaw.model.StopGoalMessage
import com.thisux.droidclaw.overlay.AgentOverlay
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.first
@@ -56,11 +59,13 @@ class ConnectionService : LifecycleService() {
private var commandRouter: CommandRouter? = null
private var captureManager: ScreenCaptureManager? = null
private var wakeLock: PowerManager.WakeLock? = null
private var overlay: AgentOverlay? = null
override fun onCreate() {
super.onCreate()
instance = this
createNotificationChannel()
overlay = AgentOverlay(this)
}
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
@@ -97,6 +102,7 @@ class ConnectionService : LifecycleService() {
return@launch
}
ScreenCaptureManager.restoreConsent(this@ConnectionService)
captureManager = ScreenCaptureManager(this@ConnectionService).also { mgr ->
if (ScreenCaptureManager.hasConsent()) {
try {
@@ -105,7 +111,8 @@ class ConnectionService : LifecycleService() {
ScreenCaptureManager.consentData!!
)
} catch (e: SecurityException) {
Log.w(TAG, "Screen capture unavailable (needs mediaProjection service type): ${e.message}")
Log.w(TAG, "Screen capture unavailable: ${e.message}")
ScreenCaptureManager.clearConsent(this@ConnectionService)
}
}
}
@@ -131,6 +138,9 @@ class ConnectionService : LifecycleService() {
)
// Send installed apps list once connected
if (state == ConnectionState.Connected) {
if (Settings.canDrawOverlays(this@ConnectionService)) {
overlay?.show()
}
val apps = getInstalledApps()
webSocket?.sendTyped(AppsMessage(apps = apps))
Log.i(TAG, "Sent ${apps.size} installed apps to server")
@@ -167,7 +177,12 @@ class ConnectionService : LifecycleService() {
webSocket?.sendTyped(GoalMessage(text = text))
}
fun stopGoal() {
webSocket?.sendTyped(StopGoalMessage())
}
private fun disconnect() {
overlay?.hide()
webSocket?.disconnect()
webSocket = null
commandRouter?.reset()
@@ -179,6 +194,8 @@ class ConnectionService : LifecycleService() {
}
override fun onDestroy() {
overlay?.destroy()
overlay = null
disconnect()
instance = null
super.onDestroy()

5
android/app/src/main/java/com/thisux/droidclaw/model/Protocol.kt
View File

@@ -71,6 +71,11 @@ data class AppsMessage(
val apps: List<InstalledAppInfo>
)
@Serializable
data class StopGoalMessage(
val type: String = "stop_goal"
)
@Serializable
data class ServerMessage(
val type: String,

92
android/app/src/main/java/com/thisux/droidclaw/overlay/AgentOverlay.kt Normal file
View File

@@ -0,0 +1,92 @@
package com.thisux.droidclaw.overlay
import android.graphics.PixelFormat
import android.view.Gravity
import android.view.MotionEvent
import android.view.View
import android.view.WindowManager
import androidx.compose.ui.platform.ComposeView
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.LifecycleService
import androidx.lifecycle.setViewTreeLifecycleOwner
import androidx.savedstate.SavedStateRegistry
import androidx.savedstate.SavedStateRegistryController
import androidx.savedstate.SavedStateRegistryOwner
import androidx.savedstate.setViewTreeSavedStateRegistryOwner
class AgentOverlay(private val service: LifecycleService) {
private val windowManager = service.getSystemService(WindowManager::class.java)
private var composeView: ComposeView? = null
private val savedStateOwner = object : SavedStateRegistryOwner {
private val controller = SavedStateRegistryController.create(this)
override val lifecycle: Lifecycle get() = service.lifecycle
override val savedStateRegistry: SavedStateRegistry get() = controller.savedStateRegistry
init { controller.performRestore(null) }
}
private val layoutParams = WindowManager.LayoutParams(
WindowManager.LayoutParams.WRAP_CONTENT,
WindowManager.LayoutParams.WRAP_CONTENT,
WindowManager.LayoutParams.TYPE_APPLICATION_OVERLAY,
WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE,
PixelFormat.TRANSLUCENT
).apply {
gravity = Gravity.TOP or Gravity.START
x = 0
y = 200
}
fun show() {
if (composeView != null) return
val view = ComposeView(service).apply {
importantForAccessibility = View.IMPORTANT_FOR_ACCESSIBILITY_NO_HIDE_DESCENDANTS
setViewTreeLifecycleOwner(service)
setViewTreeSavedStateRegistryOwner(savedStateOwner)
setContent { OverlayContent() }
setupDrag(this)
}
composeView = view
windowManager.addView(view, layoutParams)
}
fun hide() {
composeView?.let {
windowManager.removeView(it)
}
composeView = null
}
fun destroy() {
hide()
}
private fun setupDrag(view: View) {
var initialX = 0
var initialY = 0
var initialTouchX = 0f
var initialTouchY = 0f
view.setOnTouchListener { _, event ->
when (event.action) {
MotionEvent.ACTION_DOWN -> {
initialX = layoutParams.x
initialY = layoutParams.y
initialTouchX = event.rawX
initialTouchY = event.rawY
true
}
MotionEvent.ACTION_MOVE -> {
layoutParams.x = initialX + (event.rawX - initialTouchX).toInt()
layoutParams.y = initialY + (event.rawY - initialTouchY).toInt()
windowManager.updateViewLayout(view, layoutParams)
true
}
else -> false
}
}
}
}

165
android/app/src/main/java/com/thisux/droidclaw/overlay/OverlayContent.kt Normal file
View File

@@ -0,0 +1,165 @@
package com.thisux.droidclaw.overlay
import androidx.compose.animation.animateColorAsState
import androidx.compose.animation.core.LinearEasing
import androidx.compose.animation.core.RepeatMode
import androidx.compose.animation.core.animateFloat
import androidx.compose.animation.core.infiniteRepeatable
import androidx.compose.animation.core.rememberInfiniteTransition
import androidx.compose.animation.core.tween
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.widthIn
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButton
import androidx.compose.material3.IconButtonDefaults
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Text
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.Close
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.alpha
import androidx.compose.ui.draw.clip
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import com.thisux.droidclaw.connection.ConnectionService
import com.thisux.droidclaw.model.ConnectionState
import com.thisux.droidclaw.model.GoalStatus
import com.thisux.droidclaw.ui.theme.DroidClawTheme
import kotlinx.coroutines.delay
private val Green = Color(0xFF4CAF50)
private val Blue = Color(0xFF2196F3)
private val Red = Color(0xFFF44336)
private val Gray = Color(0xFF9E9E9E)
private val PillBackground = Color(0xE6212121)
@Composable
fun OverlayContent() {
DroidClawTheme {
val connectionState by ConnectionService.connectionState.collectAsState()
val goalStatus by ConnectionService.currentGoalStatus.collectAsState()
val steps by ConnectionService.currentSteps.collectAsState()
// Auto-reset Completed/Failed back to Idle after 3s
var displayStatus by remember { mutableStateOf(goalStatus) }
LaunchedEffect(goalStatus) {
displayStatus = goalStatus
if (goalStatus == GoalStatus.Completed || goalStatus == GoalStatus.Failed) {
delay(3000)
displayStatus = GoalStatus.Idle
}
}
val isConnected = connectionState == ConnectionState.Connected
val dotColor by animateColorAsState(
targetValue = when {
!isConnected -> Gray
displayStatus == GoalStatus.Running -> Blue
displayStatus == GoalStatus.Failed -> Red
else -> Green
},
label = "dotColor"
)
val statusText = when {
!isConnected -> "Offline"
displayStatus == GoalStatus.Running -> {
val last = steps.lastOrNull()
if (last != null) "Step ${last.step}: ${last.action}" else "Running..."
}
displayStatus == GoalStatus.Completed -> "Done"
displayStatus == GoalStatus.Failed -> "Stopped"
else -> "Ready"
}
Row(
modifier = Modifier
.clip(RoundedCornerShape(24.dp))
.background(PillBackground)
.height(48.dp)
.widthIn(min = 100.dp, max = 220.dp)
.padding(horizontal = 12.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp)
) {
StatusDot(
color = dotColor,
pulse = isConnected && displayStatus == GoalStatus.Running
)
Text(
text = statusText,
color = Color.White,
fontSize = 13.sp,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
modifier = Modifier.weight(1f, fill = false)
)
if (isConnected && displayStatus == GoalStatus.Running) {
IconButton(
onClick = { ConnectionService.instance?.stopGoal() },
modifier = Modifier.size(28.dp),
colors = IconButtonDefaults.iconButtonColors(
contentColor = Color.White.copy(alpha = 0.8f)
)
) {
Icon(
imageVector = Icons.Default.Close,
contentDescription = "Stop goal",
modifier = Modifier.size(16.dp)
)
}
}
}
}
}
@Composable
private fun StatusDot(color: Color, pulse: Boolean) {
if (pulse) {
val transition = rememberInfiniteTransition(label = "pulse")
val alpha by transition.animateFloat(
initialValue = 1f,
targetValue = 0.3f,
animationSpec = infiniteRepeatable(
animation = tween(800, easing = LinearEasing),
repeatMode = RepeatMode.Reverse
),
label = "pulseAlpha"
)
Box(
modifier = Modifier
.size(10.dp)
.alpha(alpha)
.clip(CircleShape)
.background(color)
)
} else {
Box(
modifier = Modifier
.size(10.dp)
.clip(CircleShape)
.background(color)
)
}
}

35
android/app/src/main/java/com/thisux/droidclaw/ui/screens/SettingsScreen.kt
View File

@@ -2,7 +2,10 @@ package com.thisux.droidclaw.ui.screens
import android.app.Activity
import android.content.Context
import android.content.Intent
import android.media.projection.MediaProjectionManager
import android.net.Uri
import android.provider.Settings
import androidx.activity.compose.rememberLauncherForActivityResult
import androidx.activity.result.contract.ActivityResultContracts
import androidx.compose.foundation.layout.Arrangement
@@ -62,18 +65,27 @@ fun SettingsScreen() {
var editingServerUrl by remember { mutableStateOf<String?>(null) }
val displayServerUrl = editingServerUrl ?: serverUrl
val isAccessibilityEnabled by DroidClawAccessibilityService.isRunning.collectAsState()
val isCaptureAvailable by ScreenCaptureManager.isAvailable.collectAsState()
val hasConsent by ScreenCaptureManager.hasConsentState.collectAsState()
val hasCaptureConsent = isCaptureAvailable || hasConsent
var isAccessibilityEnabled by remember {
mutableStateOf(DroidClawAccessibilityService.isEnabledOnDevice(context))
}
var hasCaptureConsent by remember {
ScreenCaptureManager.restoreConsent(context)
mutableStateOf(isCaptureAvailable || ScreenCaptureManager.hasConsent())
}
var isBatteryExempt by remember { mutableStateOf(BatteryOptimization.isIgnoringBatteryOptimizations(context)) }
var hasOverlayPermission by remember { mutableStateOf(Settings.canDrawOverlays(context)) }
val lifecycleOwner = LocalLifecycleOwner.current
DisposableEffect(lifecycleOwner) {
val observer = LifecycleEventObserver { _, event ->
if (event == Lifecycle.Event.ON_RESUME) {
isAccessibilityEnabled = DroidClawAccessibilityService.isEnabledOnDevice(context)
ScreenCaptureManager.restoreConsent(context)
hasCaptureConsent = isCaptureAvailable || ScreenCaptureManager.hasConsent()
isBatteryExempt = BatteryOptimization.isIgnoringBatteryOptimizations(context)
hasOverlayPermission = Settings.canDrawOverlays(context)
}
}
lifecycleOwner.lifecycle.addObserver(observer)
@@ -84,7 +96,8 @@ fun SettingsScreen() {
ActivityResultContracts.StartActivityForResult()
) { result ->
if (result.resultCode == Activity.RESULT_OK && result.data != null) {
ScreenCaptureManager.storeConsent(result.resultCode, result.data)
ScreenCaptureManager.storeConsent(context, result.resultCode, result.data)
hasCaptureConsent = true
}
}
@@ -172,6 +185,20 @@ fun SettingsScreen() {
actionLabel = "Disable",
onAction = { BatteryOptimization.requestExemption(context) }
)
ChecklistItem(
label = "Overlay permission",
isOk = hasOverlayPermission,
actionLabel = "Grant",
onAction = {
context.startActivity(
Intent(
Settings.ACTION_MANAGE_OVERLAY_PERMISSION,
Uri.parse("package:${context.packageName}")
)
)
}
)
}
}

3
packages/shared/src/protocol.ts
View File

@@ -7,7 +7,8 @@ export type DeviceMessage =
| { type: "goal"; text: string }
| { type: "pong" }
| { type: "heartbeat"; batteryLevel: number; isCharging: boolean }
| { type: "apps"; apps: InstalledApp[] };
| { type: "apps"; apps: InstalledApp[] }
| { type: "stop_goal" };
export type ServerToDeviceMessage =
| { type: "auth_ok"; deviceId: string }

27
server/src/ws/device.ts
View File

@@ -22,7 +22,7 @@ async function hashApiKey(key: string): Promise<string> {
}
/** Track running agent sessions to prevent duplicates per device */
const activeSessions = new Map<string, string>();
const activeSessions = new Map<string, { goal: string; abort: AbortController }>();
/**
* Send a JSON message to a device WebSocket (safe — catches send errors).
@@ -252,7 +252,8 @@ export async function handleDeviceMessage(
}
console.log(`[Pipeline] Starting goal for device ${deviceId}: ${goal}`);
activeSessions.set(deviceId, goal);
const abortController = new AbortController();
activeSessions.set(deviceId, { goal, abort: abortController });
sendToDevice(ws, { type: "goal_started", sessionId: deviceId, goal });
@@ -262,6 +263,7 @@ export async function handleDeviceMessage(
userId,
goal,
llmConfig: userLlmConfig,
signal: abortController.signal,
onStep(step) {
sendToDevice(ws, {
type: "step",
@@ -294,6 +296,23 @@ export async function handleDeviceMessage(
break;
}
case "stop_goal": {
const deviceId = ws.data.deviceId!;
const active = activeSessions.get(deviceId);
if (active) {
console.log(`[Pipeline] Stop requested for device ${deviceId}`);
active.abort.abort();
activeSessions.delete(deviceId);
sendToDevice(ws, {
type: "goal_completed",
sessionId: deviceId,
success: false,
stepsUsed: 0,
});
}
break;
}
case "apps": {
const persistentDeviceId = ws.data.persistentDeviceId;
if (persistentDeviceId) {
@@ -360,7 +379,11 @@ export function handleDeviceClose(
const { deviceId, userId, persistentDeviceId } = ws.data;
if (!deviceId) return;
const active = activeSessions.get(deviceId);
if (active) {
active.abort.abort();
activeSessions.delete(deviceId);
}
sessions.removeDevice(deviceId);
// Update device status in DB