From 516b83bd0f6ae306bc34ef1b753227c2aba49fba Mon Sep 17 00:00:00 2001 From: Sanju Sivalingam Date: Tue, 17 Feb 2026 17:51:19 +0530 Subject: [PATCH] feat(android): add ReliableWebSocket, CommandRouter, ConnectionService Co-Authored-By: Claude Opus 4.6 --- .../droidclaw/connection/CommandRouter.kt | 129 +++++++++++ .../droidclaw/connection/ConnectionService.kt | 211 +++++++++++++++++- .../droidclaw/connection/ReliableWebSocket.kt | 168 ++++++++++++++ .../com/thisux/droidclaw/util/DeviceInfo.kt | 21 ++ 4 files changed, 522 insertions(+), 7 deletions(-) create mode 100644 android/app/src/main/java/com/thisux/droidclaw/connection/CommandRouter.kt create mode 100644 android/app/src/main/java/com/thisux/droidclaw/connection/ReliableWebSocket.kt create mode 100644 android/app/src/main/java/com/thisux/droidclaw/util/DeviceInfo.kt diff --git a/android/app/src/main/java/com/thisux/droidclaw/connection/CommandRouter.kt b/android/app/src/main/java/com/thisux/droidclaw/connection/CommandRouter.kt new file mode 100644 index 0000000..b19bc4a --- /dev/null +++ b/android/app/src/main/java/com/thisux/droidclaw/connection/CommandRouter.kt @@ -0,0 +1,129 @@ +package com.thisux.droidclaw.connection + +import android.util.Base64 +import android.util.Log +import com.thisux.droidclaw.accessibility.DroidClawAccessibilityService +import com.thisux.droidclaw.accessibility.GestureExecutor +import com.thisux.droidclaw.capture.ScreenCaptureManager +import com.thisux.droidclaw.model.AgentStep +import com.thisux.droidclaw.model.GoalStatus +import com.thisux.droidclaw.model.PongMessage +import com.thisux.droidclaw.model.ResultResponse +import com.thisux.droidclaw.model.ScreenResponse +import com.thisux.droidclaw.model.ServerMessage +import kotlinx.coroutines.flow.MutableStateFlow + +class CommandRouter( + private val webSocket: ReliableWebSocket, + private val captureManager: ScreenCaptureManager? +) { + companion object { + private const val TAG = "CommandRouter" + } + + val currentGoalStatus = MutableStateFlow(GoalStatus.Idle) + val currentSteps = MutableStateFlow>(emptyList()) + val currentGoal = MutableStateFlow("") + val currentSessionId = MutableStateFlow(null) + + private var gestureExecutor: GestureExecutor? = null + + fun updateGestureExecutor() { + val svc = DroidClawAccessibilityService.instance + gestureExecutor = if (svc != null) GestureExecutor(svc) else null + } + + suspend fun handleMessage(msg: ServerMessage) { + Log.d(TAG, "Handling: ${msg.type}") + + when (msg.type) { + "get_screen" -> handleGetScreen(msg.requestId!!) + "ping" -> webSocket.sendTyped(PongMessage()) + + "tap", "type", "enter", "back", "home", "notifications", + "longpress", "swipe", "launch", "clear", "clipboard_set", + "clipboard_get", "paste", "open_url", "switch_app", + "keyevent", "open_settings", "wait" -> handleAction(msg) + + "goal_started" -> { + currentSessionId.value = msg.sessionId + currentGoal.value = msg.goal ?: "" + currentGoalStatus.value = GoalStatus.Running + currentSteps.value = emptyList() + Log.i(TAG, "Goal started: ${msg.goal}") + } + "step" -> { + val step = AgentStep( + step = msg.step ?: 0, + action = msg.action?.toString() ?: "", + reasoning = msg.reasoning ?: "" + ) + currentSteps.value = currentSteps.value + step + Log.d(TAG, "Step ${step.step}: ${step.reasoning}") + } + "goal_completed" -> { + currentGoalStatus.value = if (msg.success == true) GoalStatus.Completed else GoalStatus.Failed + Log.i(TAG, "Goal completed: success=${msg.success}, steps=${msg.stepsUsed}") + } + + else -> Log.w(TAG, "Unknown message type: ${msg.type}") + } + } + + private fun handleGetScreen(requestId: String) { + updateGestureExecutor() + val svc = DroidClawAccessibilityService.instance + val elements = svc?.getScreenTree() ?: emptyList() + val packageName = try { + svc?.rootInActiveWindow?.packageName?.toString() + } catch (_: Exception) { null } + + var screenshot: String? = null + if (elements.isEmpty()) { + val bytes = captureManager?.capture() + if (bytes != null) { + screenshot = Base64.encodeToString(bytes, Base64.NO_WRAP) + } + } + + val response = ScreenResponse( + requestId = requestId, + elements = elements, + screenshot = screenshot, + packageName = packageName + ) + webSocket.sendTyped(response) + } + + private suspend fun handleAction(msg: ServerMessage) { + updateGestureExecutor() + val executor = gestureExecutor + if (executor == null) { + webSocket.sendTyped( + ResultResponse( + requestId = msg.requestId!!, + success = false, + error = "Accessibility service not running" + ) + ) + return + } + + val result = executor.execute(msg) + webSocket.sendTyped( + ResultResponse( + requestId = msg.requestId!!, + success = result.success, + error = result.error, + data = result.data + ) + ) + } + + fun reset() { + currentGoalStatus.value = GoalStatus.Idle + currentSteps.value = emptyList() + currentGoal.value = "" + currentSessionId.value = null + } +} diff --git a/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt b/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt index 93c717f..37e0721 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt @@ -1,18 +1,215 @@ package com.thisux.droidclaw.connection -import android.app.Service +import android.app.Notification +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.PendingIntent +import android.content.Context import android.content.Intent +import android.os.Build import android.os.IBinder +import android.os.PowerManager +import android.util.Log +import androidx.core.app.NotificationCompat +import androidx.lifecycle.LifecycleService +import androidx.lifecycle.lifecycleScope +import com.thisux.droidclaw.DroidClawApp +import com.thisux.droidclaw.MainActivity +import com.thisux.droidclaw.R +import com.thisux.droidclaw.capture.ScreenCaptureManager +import com.thisux.droidclaw.model.ConnectionState +import com.thisux.droidclaw.model.GoalMessage +import com.thisux.droidclaw.model.GoalStatus +import com.thisux.droidclaw.model.AgentStep +import com.thisux.droidclaw.util.DeviceInfoHelper +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch -/** - * Foreground service for maintaining the WebSocket connection to the DroidClaw server. - * Full implementation will be added in Task 9. - */ -class ConnectionService : Service() { +class ConnectionService : LifecycleService() { - override fun onBind(intent: Intent?): IBinder? = null + companion object { + private const val TAG = "ConnectionSvc" + private const val CHANNEL_ID = "droidclaw_connection" + private const val NOTIFICATION_ID = 1 + + val connectionState = MutableStateFlow(ConnectionState.Disconnected) + val currentSteps = MutableStateFlow>(emptyList()) + val currentGoalStatus = MutableStateFlow(GoalStatus.Idle) + val currentGoal = MutableStateFlow("") + val errorMessage = MutableStateFlow(null) + var instance: ConnectionService? = null + + const val ACTION_CONNECT = "com.thisux.droidclaw.CONNECT" + const val ACTION_DISCONNECT = "com.thisux.droidclaw.DISCONNECT" + const val ACTION_SEND_GOAL = "com.thisux.droidclaw.SEND_GOAL" + const val EXTRA_GOAL = "goal_text" + } + + private var webSocket: ReliableWebSocket? = null + private var commandRouter: CommandRouter? = null + private var captureManager: ScreenCaptureManager? = null + private var wakeLock: PowerManager.WakeLock? = null + + override fun onCreate() { + super.onCreate() + instance = this + createNotificationChannel() + } override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + super.onStartCommand(intent, flags, startId) + + when (intent?.action) { + ACTION_CONNECT -> { + startForeground(NOTIFICATION_ID, buildNotification("Connecting...")) + connect() + } + ACTION_DISCONNECT -> { + disconnect() + stopSelf() + } + ACTION_SEND_GOAL -> { + val goal = intent.getStringExtra(EXTRA_GOAL) ?: return START_NOT_STICKY + sendGoal(goal) + } + } + return START_NOT_STICKY } + + private fun connect() { + lifecycleScope.launch { + val app = application as DroidClawApp + val apiKey = app.settingsStore.apiKey.first() + val serverUrl = app.settingsStore.serverUrl.first() + + if (apiKey.isBlank() || serverUrl.isBlank()) { + connectionState.value = ConnectionState.Error + errorMessage.value = "API key or server URL not configured" + stopSelf() + return@launch + } + + captureManager = ScreenCaptureManager(this@ConnectionService) + + val ws = ReliableWebSocket(lifecycleScope) { msg -> + commandRouter?.handleMessage(msg) + } + webSocket = ws + + val router = CommandRouter(ws, captureManager) + commandRouter = router + + launch { + ws.state.collect { state -> + connectionState.value = state + updateNotification( + when (state) { + ConnectionState.Connected -> "Connected to server" + ConnectionState.Connecting -> "Connecting..." + ConnectionState.Error -> "Connection error" + ConnectionState.Disconnected -> "Disconnected" + } + ) + } + } + launch { ws.errorMessage.collect { errorMessage.value = it } } + launch { router.currentSteps.collect { currentSteps.value = it } } + launch { router.currentGoalStatus.collect { currentGoalStatus.value = it } } + launch { router.currentGoal.collect { currentGoal.value = it } } + + acquireWakeLock() + + val deviceInfo = DeviceInfoHelper.get(this@ConnectionService) + ws.connect(serverUrl, apiKey, deviceInfo) + } + } + + private fun sendGoal(text: String) { + webSocket?.sendTyped(GoalMessage(text = text)) + } + + private fun disconnect() { + webSocket?.disconnect() + webSocket = null + commandRouter?.reset() + commandRouter = null + captureManager?.release() + captureManager = null + releaseWakeLock() + connectionState.value = ConnectionState.Disconnected + } + + override fun onDestroy() { + disconnect() + instance = null + super.onDestroy() + } + + override fun onBind(intent: Intent): IBinder? { + super.onBind(intent) + return null + } + + private fun createNotificationChannel() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + val channel = NotificationChannel( + CHANNEL_ID, + "DroidClaw Connection", + NotificationManager.IMPORTANCE_LOW + ).apply { + description = "Shows when DroidClaw is connected to the server" + } + val nm = getSystemService(NotificationManager::class.java) + nm.createNotificationChannel(channel) + } + } + + private fun buildNotification(text: String): Notification { + val openIntent = PendingIntent.getActivity( + this, 0, + Intent(this, MainActivity::class.java), + PendingIntent.FLAG_IMMUTABLE + ) + + val disconnectIntent = PendingIntent.getService( + this, 1, + Intent(this, ConnectionService::class.java).apply { + action = ACTION_DISCONNECT + }, + PendingIntent.FLAG_IMMUTABLE + ) + + return NotificationCompat.Builder(this, CHANNEL_ID) + .setContentTitle("DroidClaw") + .setContentText(text) + .setSmallIcon(R.drawable.ic_launcher_foreground) + .setOngoing(true) + .setContentIntent(openIntent) + .addAction(0, "Disconnect", disconnectIntent) + .build() + } + + private fun updateNotification(text: String) { + val nm = getSystemService(NotificationManager::class.java) + nm.notify(NOTIFICATION_ID, buildNotification(text)) + } + + private fun acquireWakeLock() { + val pm = getSystemService(Context.POWER_SERVICE) as PowerManager + wakeLock = pm.newWakeLock( + PowerManager.PARTIAL_WAKE_LOCK, + "DroidClaw::ConnectionWakeLock" + ).apply { + acquire(10 * 60 * 1000L) + } + } + + private fun releaseWakeLock() { + wakeLock?.let { + if (it.isHeld) it.release() + } + wakeLock = null + } } diff --git a/android/app/src/main/java/com/thisux/droidclaw/connection/ReliableWebSocket.kt b/android/app/src/main/java/com/thisux/droidclaw/connection/ReliableWebSocket.kt new file mode 100644 index 0000000..742b23c --- /dev/null +++ b/android/app/src/main/java/com/thisux/droidclaw/connection/ReliableWebSocket.kt @@ -0,0 +1,168 @@ +package com.thisux.droidclaw.connection + +import android.util.Log +import com.thisux.droidclaw.model.AuthMessage +import com.thisux.droidclaw.model.ConnectionState +import com.thisux.droidclaw.model.DeviceInfoMsg +import com.thisux.droidclaw.model.ServerMessage +import io.ktor.client.HttpClient +import io.ktor.client.engine.cio.CIO +import io.ktor.client.plugins.websocket.WebSockets +import io.ktor.client.plugins.websocket.webSocket +import io.ktor.websocket.Frame +import io.ktor.websocket.close +import io.ktor.websocket.readText +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json + +class ReliableWebSocket( + private val scope: CoroutineScope, + private val onMessage: suspend (ServerMessage) -> Unit +) { + companion object { + private const val TAG = "ReliableWS" + private const val MAX_BACKOFF_MS = 30_000L + } + + @PublishedApi + internal val json = Json { ignoreUnknownKeys = true; encodeDefaults = true } + + private val _state = MutableStateFlow(ConnectionState.Disconnected) + val state: StateFlow = _state + + private val _errorMessage = MutableStateFlow(null) + val errorMessage: StateFlow = _errorMessage + + private val outbound = Channel(Channel.BUFFERED) + private var connectionJob: Job? = null + private var client: HttpClient? = null + private var backoffMs = 1000L + private var shouldReconnect = true + + var deviceId: String? = null + private set + + fun connect(serverUrl: String, apiKey: String, deviceInfo: DeviceInfoMsg) { + shouldReconnect = true + connectionJob?.cancel() + connectionJob = scope.launch { + while (shouldReconnect && isActive) { + try { + _state.value = ConnectionState.Connecting + _errorMessage.value = null + connectOnce(serverUrl, apiKey, deviceInfo) + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + Log.e(TAG, "Connection failed: ${e.message}") + _state.value = ConnectionState.Error + _errorMessage.value = e.message + } + if (shouldReconnect && isActive) { + Log.i(TAG, "Reconnecting in ${backoffMs}ms") + delay(backoffMs) + backoffMs = (backoffMs * 2).coerceAtMost(MAX_BACKOFF_MS) + } + } + } + } + + private suspend fun connectOnce(serverUrl: String, apiKey: String, deviceInfo: DeviceInfoMsg) { + val httpClient = HttpClient(CIO) { + install(WebSockets) { + pingIntervalMillis = 30_000 + } + } + client = httpClient + + val wsUrl = serverUrl.trimEnd('/') + "/ws/device" + + httpClient.webSocket(wsUrl) { + // Auth handshake + val authMsg = AuthMessage(apiKey = apiKey, deviceInfo = deviceInfo) + send(Frame.Text(json.encodeToString(authMsg))) + Log.i(TAG, "Sent auth message") + + // Wait for auth response + val authFrame = incoming.receive() as? Frame.Text + ?: throw Exception("Expected text frame for auth response") + + val authResponse = json.decodeFromString(authFrame.readText()) + when (authResponse.type) { + "auth_ok" -> { + deviceId = authResponse.deviceId + _state.value = ConnectionState.Connected + _errorMessage.value = null + backoffMs = 1000L + Log.i(TAG, "Authenticated, deviceId=$deviceId") + } + "auth_error" -> { + shouldReconnect = false + _state.value = ConnectionState.Error + _errorMessage.value = authResponse.message ?: "Authentication failed" + close() + return@webSocket + } + else -> { + throw Exception("Unexpected auth response: ${authResponse.type}") + } + } + + // Launch outbound sender + val senderJob = launch { + for (msg in outbound) { + send(Frame.Text(msg)) + } + } + + // Read incoming messages + try { + for (frame in incoming) { + if (frame is Frame.Text) { + val text = frame.readText() + try { + val msg = json.decodeFromString(text) + onMessage(msg) + } catch (e: Exception) { + Log.e(TAG, "Failed to parse message: ${e.message}") + } + } + } + } finally { + senderJob.cancel() + } + } + + httpClient.close() + client = null + _state.value = ConnectionState.Disconnected + } + + fun send(message: String) { + outbound.trySend(message) + } + + inline fun sendTyped(message: T) { + send(json.encodeToString(message)) + } + + fun disconnect() { + shouldReconnect = false + connectionJob?.cancel() + connectionJob = null + client?.close() + client = null + _state.value = ConnectionState.Disconnected + _errorMessage.value = null + deviceId = null + } +} diff --git a/android/app/src/main/java/com/thisux/droidclaw/util/DeviceInfo.kt b/android/app/src/main/java/com/thisux/droidclaw/util/DeviceInfo.kt new file mode 100644 index 0000000..5e51171 --- /dev/null +++ b/android/app/src/main/java/com/thisux/droidclaw/util/DeviceInfo.kt @@ -0,0 +1,21 @@ +package com.thisux.droidclaw.util + +import android.content.Context +import android.util.DisplayMetrics +import android.view.WindowManager +import com.thisux.droidclaw.model.DeviceInfoMsg + +object DeviceInfoHelper { + fun get(context: Context): DeviceInfoMsg { + val wm = context.getSystemService(Context.WINDOW_SERVICE) as WindowManager + val metrics = DisplayMetrics() + @Suppress("DEPRECATION") + wm.defaultDisplay.getRealMetrics(metrics) + return DeviceInfoMsg( + model = android.os.Build.MODEL, + androidVersion = android.os.Build.VERSION.RELEASE, + screenWidth = metrics.widthPixels, + screenHeight = metrics.heightPixels + ) + } +}