diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index 0f4c0ac..852b990 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -2,6 +2,13 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/android/app/src/main/java/com/thisux/droidclaw/accessibility/DroidClawAccessibilityService.kt b/android/app/src/main/java/com/thisux/droidclaw/accessibility/DroidClawAccessibilityService.kt new file mode 100644 index 0000000..0878cd2 --- /dev/null +++ b/android/app/src/main/java/com/thisux/droidclaw/accessibility/DroidClawAccessibilityService.kt @@ -0,0 +1,96 @@ +package com.thisux.droidclaw.accessibility + +import android.accessibilityservice.AccessibilityService +import android.util.Log +import android.view.accessibility.AccessibilityEvent +import android.view.accessibility.AccessibilityNodeInfo +import com.thisux.droidclaw.model.UIElement +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.runBlocking + +class DroidClawAccessibilityService : AccessibilityService() { + + companion object { + private const val TAG = "DroidClawA11y" + val isRunning = MutableStateFlow(false) + val lastScreenTree = MutableStateFlow>(emptyList()) + var instance: DroidClawAccessibilityService? = null + } + + override fun onServiceConnected() { + super.onServiceConnected() + Log.i(TAG, "Accessibility service connected") + instance = this + isRunning.value = true + } + + override fun onAccessibilityEvent(event: AccessibilityEvent?) { + // We capture on-demand via getScreenTree(), not on every event + } + + override fun onInterrupt() { + Log.w(TAG, "Accessibility service interrupted") + } + + override fun onDestroy() { + super.onDestroy() + Log.i(TAG, "Accessibility service destroyed") + instance = null + isRunning.value = false + } + + fun getScreenTree(): List { + val delays = longArrayOf(50, 100, 200) + for (delayMs in delays) { + val root = rootInActiveWindow + if (root != null) { + try { + val elements = ScreenTreeBuilder.capture(root) + lastScreenTree.value = elements + return elements + } finally { + root.recycle() + } + } + runBlocking { delay(delayMs) } + } + Log.w(TAG, "rootInActiveWindow null after retries") + return emptyList() + } + + fun findNodeAt(x: Int, y: Int): AccessibilityNodeInfo? { + val root = rootInActiveWindow ?: return null + return findNodeAtRecursive(root, x, y) + } + + private fun findNodeAtRecursive( + node: AccessibilityNodeInfo, + x: Int, + y: Int + ): AccessibilityNodeInfo? { + val rect = android.graphics.Rect() + node.getBoundsInScreen(rect) + + if (!rect.contains(x, y)) { + node.recycle() + return null + } + + for (i in 0 until node.childCount) { + val child = node.getChild(i) ?: continue + val found = findNodeAtRecursive(child, x, y) + if (found != null) { + node.recycle() + return found + } + } + + return if (node.isClickable || node.isLongClickable || node.isEditable) { + node + } else { + node.recycle() + null + } + } +} diff --git a/android/app/src/main/java/com/thisux/droidclaw/accessibility/ScreenTreeBuilder.kt b/android/app/src/main/java/com/thisux/droidclaw/accessibility/ScreenTreeBuilder.kt new file mode 100644 index 0000000..918f9ab --- /dev/null +++ b/android/app/src/main/java/com/thisux/droidclaw/accessibility/ScreenTreeBuilder.kt @@ -0,0 +1,95 @@ +package com.thisux.droidclaw.accessibility + +import android.graphics.Rect +import android.view.accessibility.AccessibilityNodeInfo +import com.thisux.droidclaw.model.UIElement +import java.security.MessageDigest + +object ScreenTreeBuilder { + + fun capture(rootNode: AccessibilityNodeInfo?): List { + if (rootNode == null) return emptyList() + val elements = mutableListOf() + walkTree(rootNode, elements, depth = 0, parentDesc = "") + return elements + } + + private fun walkTree( + node: AccessibilityNodeInfo, + elements: MutableList, + depth: Int, + parentDesc: String + ) { + try { + val rect = Rect() + node.getBoundsInScreen(rect) + + val text = node.text?.toString() ?: "" + val contentDesc = node.contentDescription?.toString() ?: "" + val viewId = node.viewIdResourceName ?: "" + val className = node.className?.toString() ?: "" + val displayText = text.ifEmpty { contentDesc } + + val isInteractive = node.isClickable || node.isLongClickable || + node.isEditable || node.isScrollable || node.isFocusable + + if (isInteractive || displayText.isNotEmpty()) { + val centerX = (rect.left + rect.right) / 2 + val centerY = (rect.top + rect.bottom) / 2 + val width = rect.width() + val height = rect.height() + + val action = when { + node.isEditable -> "type" + node.isScrollable -> "scroll" + node.isLongClickable -> "longpress" + node.isClickable -> "tap" + else -> "read" + } + + elements.add( + UIElement( + id = viewId, + text = displayText, + type = className.substringAfterLast("."), + bounds = "[${rect.left},${rect.top}][${rect.right},${rect.bottom}]", + center = listOf(centerX, centerY), + size = listOf(width, height), + clickable = node.isClickable, + editable = node.isEditable, + enabled = node.isEnabled, + checked = node.isChecked, + focused = node.isFocused, + selected = node.isSelected, + scrollable = node.isScrollable, + longClickable = node.isLongClickable, + password = node.isPassword, + hint = node.hintText?.toString() ?: "", + action = action, + parent = parentDesc, + depth = depth + ) + ) + } + + for (i in 0 until node.childCount) { + val child = node.getChild(i) ?: continue + try { + walkTree(child, elements, depth + 1, className) + } finally { + child.recycle() + } + } + } catch (_: Exception) { + // Node may have been recycled during traversal + } + } + + fun computeScreenHash(elements: List): String { + val digest = MessageDigest.getInstance("MD5") + for (el in elements) { + digest.update("${el.id}|${el.text}|${el.center}".toByteArray()) + } + return digest.digest().joinToString("") { "%02x".format(it) }.take(12) + } +} diff --git a/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt b/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt new file mode 100644 index 0000000..93c717f --- /dev/null +++ b/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt @@ -0,0 +1,18 @@ +package com.thisux.droidclaw.connection + +import android.app.Service +import android.content.Intent +import android.os.IBinder + +/** + * Foreground service for maintaining the WebSocket connection to the DroidClaw server. + * Full implementation will be added in Task 9. + */ +class ConnectionService : Service() { + + override fun onBind(intent: Intent?): IBinder? = null + + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + return START_NOT_STICKY + } +} diff --git a/android/app/src/main/res/xml/accessibility_config.xml b/android/app/src/main/res/xml/accessibility_config.xml new file mode 100644 index 0000000..54547f0 --- /dev/null +++ b/android/app/src/main/res/xml/accessibility_config.xml @@ -0,0 +1,9 @@ + +