feat: workflow automation via NotificationListenerService

Add workflow system that lets users describe automations in natural
language through the same input field. The server LLM classifies
input as either an immediate goal or a workflow rule, then:

- Parses workflow descriptions into structured trigger conditions
- Stores workflows per-user in Postgres
- Syncs workflows to device via WebSocket
- NotificationListenerService monitors notifications and triggers
  matching workflows as agent goals

Also cleans up overlay text and adds network security config.
This commit is contained in:
Somasundaram Mahesh
2026-02-18 18:49:14 +05:30
parent 45766621f2
commit 4d4b7059e4
18 changed files with 921 additions and 9 deletions

View File

@@ -22,7 +22,8 @@
android:label="@string/app_name"
android:roundIcon="@mipmap/ic_launcher_round"
android:supportsRtl="true"
android:theme="@style/Theme.DroidClaw">
android:theme="@style/Theme.DroidClaw"
android:networkSecurityConfig="@xml/network_security_config">
<activity
android:name=".MainActivity"
android:exported="true"
@@ -51,6 +52,15 @@
android:name=".connection.ConnectionService"
android:foregroundServiceType="connectedDevice"
android:exported="false" />
<service
android:name=".workflow.WorkflowNotificationService"
android:permission="android.permission.BIND_NOTIFICATION_LISTENER_SERVICE"
android:exported="false">
<intent-filter>
<action android:name="android.service.notification.NotificationListenerService" />
</intent-filter>
</service>
</application>
</manifest>

View File

@@ -2,13 +2,17 @@ package com.thisux.droidclaw
import android.app.Application
import com.thisux.droidclaw.data.SettingsStore
import com.thisux.droidclaw.data.WorkflowStore
class DroidClawApp : Application() {
lateinit var settingsStore: SettingsStore
private set
lateinit var workflowStore: WorkflowStore
private set
override fun onCreate() {
super.onCreate()
settingsStore = SettingsStore(this)
workflowStore = WorkflowStore(this)
}
}

View File

@@ -12,14 +12,20 @@ import com.thisux.droidclaw.model.PongMessage
import com.thisux.droidclaw.model.ResultResponse
import com.thisux.droidclaw.model.ScreenResponse
import com.thisux.droidclaw.model.ServerMessage
import com.thisux.droidclaw.model.Workflow
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.serialization.json.Json
class CommandRouter(
private val webSocket: ReliableWebSocket,
private val captureManager: ScreenCaptureManager?
private val captureManager: ScreenCaptureManager?,
private val onWorkflowSync: (suspend (List<Workflow>) -> Unit)? = null,
private val onWorkflowCreated: (suspend (Workflow) -> Unit)? = null,
private val onWorkflowDeleted: (suspend (String) -> Unit)? = null
) {
companion object {
private const val TAG = "CommandRouter"
private val json = Json { ignoreUnknownKeys = true; encodeDefaults = true }
}
val currentGoalStatus = MutableStateFlow(GoalStatus.Idle)
@@ -71,6 +77,38 @@ class CommandRouter(
Log.i(TAG, "Goal failed: ${msg.message}")
}
"workflow_created" -> {
val wfJson = msg.workflowJson ?: return
try {
val wf = json.decodeFromString<Workflow>(wfJson)
onWorkflowCreated?.invoke(wf)
Log.i(TAG, "Workflow created: ${wf.name}")
} catch (e: Exception) {
Log.e(TAG, "Failed to parse workflow_created: ${e.message}")
}
}
"workflow_synced" -> {
val wfsJson = msg.workflowsJson ?: return
try {
val wfs = json.decodeFromString<List<Workflow>>(wfsJson)
onWorkflowSync?.invoke(wfs)
Log.i(TAG, "Workflows synced: ${wfs.size} workflows")
} catch (e: Exception) {
Log.e(TAG, "Failed to parse workflow_synced: ${e.message}")
}
}
"workflow_deleted" -> {
val id = msg.workflowId ?: return
onWorkflowDeleted?.invoke(id)
Log.i(TAG, "Workflow deleted: $id")
}
"workflow_goal" -> {
val goal = msg.goal ?: return
Log.i(TAG, "Workflow-triggered goal: $goal")
// Submit as a regular goal via the WebSocket
webSocket.sendTyped(com.thisux.droidclaw.model.GoalMessage(text = goal))
}
else -> Log.w(TAG, "Unknown message type: ${msg.type}")
}
}

View File

@@ -29,6 +29,11 @@ import android.content.pm.PackageManager
import android.net.Uri
import android.provider.Settings
import com.thisux.droidclaw.model.StopGoalMessage
import com.thisux.droidclaw.model.WorkflowCreateMessage
import com.thisux.droidclaw.model.WorkflowDeleteMessage
import com.thisux.droidclaw.model.WorkflowSyncMessage
import com.thisux.droidclaw.model.WorkflowTriggerMessage
import com.thisux.droidclaw.model.WorkflowUpdateMessage
import com.thisux.droidclaw.overlay.AgentOverlay
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
@@ -122,7 +127,12 @@ class ConnectionService : LifecycleService() {
}
webSocket = ws
val router = CommandRouter(ws, captureManager)
val router = CommandRouter(
ws, captureManager,
onWorkflowSync = { workflows -> app.workflowStore.replaceAll(workflows) },
onWorkflowCreated = { workflow -> app.workflowStore.save(workflow) },
onWorkflowDeleted = { id -> app.workflowStore.delete(id) }
)
commandRouter = router
launch {
@@ -144,6 +154,8 @@ class ConnectionService : LifecycleService() {
val apps = getInstalledApps()
webSocket?.sendTyped(AppsMessage(apps = apps))
Log.i(TAG, "Sent ${apps.size} installed apps to server")
// Sync workflows from server
webSocket?.sendTyped(WorkflowSyncMessage())
}
}
}
@@ -181,6 +193,26 @@ class ConnectionService : LifecycleService() {
webSocket?.sendTyped(StopGoalMessage())
}
fun sendWorkflowCreate(description: String) {
webSocket?.sendTyped(WorkflowCreateMessage(description = description))
}
fun sendWorkflowUpdate(workflowId: String, enabled: Boolean?) {
webSocket?.sendTyped(WorkflowUpdateMessage(workflowId = workflowId, enabled = enabled))
}
fun sendWorkflowDelete(workflowId: String) {
webSocket?.sendTyped(WorkflowDeleteMessage(workflowId = workflowId))
}
fun sendWorkflowSync() {
webSocket?.sendTyped(WorkflowSyncMessage())
}
fun sendWorkflowTrigger(msg: WorkflowTriggerMessage) {
webSocket?.sendTyped(msg)
}
private fun disconnect() {
overlay?.hide()
webSocket?.disconnect()

View File

@@ -0,0 +1,48 @@
package com.thisux.droidclaw.data
import android.content.Context
import androidx.datastore.preferences.core.edit
import androidx.datastore.preferences.core.stringPreferencesKey
import com.thisux.droidclaw.model.Workflow
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.map
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
private val WORKFLOWS_KEY = stringPreferencesKey("workflows_json")
private val json = Json { ignoreUnknownKeys = true; encodeDefaults = true }
class WorkflowStore(private val context: Context) {
val workflows: Flow<List<Workflow>> = context.dataStore.data.map { prefs ->
val raw = prefs[WORKFLOWS_KEY] ?: "[]"
try { json.decodeFromString<List<Workflow>>(raw) } catch (_: Exception) { emptyList() }
}
suspend fun save(workflow: Workflow) {
context.dataStore.edit { prefs ->
val list = currentList(prefs).toMutableList()
val idx = list.indexOfFirst { it.id == workflow.id }
if (idx >= 0) list[idx] = workflow else list.add(workflow)
prefs[WORKFLOWS_KEY] = json.encodeToString(list)
}
}
suspend fun delete(workflowId: String) {
context.dataStore.edit { prefs ->
val list = currentList(prefs).filter { it.id != workflowId }
prefs[WORKFLOWS_KEY] = json.encodeToString(list)
}
}
suspend fun replaceAll(workflows: List<Workflow>) {
context.dataStore.edit { prefs ->
prefs[WORKFLOWS_KEY] = json.encodeToString(workflows)
}
}
private fun currentList(prefs: androidx.datastore.preferences.core.Preferences): List<Workflow> {
val raw = prefs[WORKFLOWS_KEY] ?: "[]"
return try { json.decodeFromString<List<Workflow>>(raw) } catch (_: Exception) { emptyList() }
}
}

View File

@@ -76,6 +76,39 @@ data class StopGoalMessage(
val type: String = "stop_goal"
)
@Serializable
data class WorkflowCreateMessage(
val type: String = "workflow_create",
val description: String // natural-language workflow description
)
@Serializable
data class WorkflowUpdateMessage(
val type: String = "workflow_update",
val workflowId: String,
val enabled: Boolean? = null
)
@Serializable
data class WorkflowDeleteMessage(
val type: String = "workflow_delete",
val workflowId: String
)
@Serializable
data class WorkflowSyncMessage(
val type: String = "workflow_sync"
)
@Serializable
data class WorkflowTriggerMessage(
val type: String = "workflow_trigger",
val workflowId: String,
val notificationApp: String? = null,
val notificationTitle: String? = null,
val notificationText: String? = null
)
@Serializable
data class ServerMessage(
val type: String,
@@ -106,5 +139,9 @@ data class ServerMessage(
val intentUri: String? = null,
val intentType: String? = null,
val intentExtras: Map<String, String>? = null,
val setting: String? = null
val setting: String? = null,
// Workflow fields
val workflowId: String? = null,
val workflowJson: String? = null, // single workflow as JSON
val workflowsJson: String? = null // array of workflows as JSON (for sync)
)

View File

@@ -0,0 +1,32 @@
package com.thisux.droidclaw.model
import kotlinx.serialization.Serializable
@Serializable
enum class TriggerType {
notification
}
@Serializable
enum class MatchMode {
contains, exact, regex
}
@Serializable
data class TriggerCondition(
val field: String, // "app_package", "title", "text"
val matchMode: MatchMode,
val value: String
)
@Serializable
data class Workflow(
val id: String,
val name: String,
val description: String, // original natural-language input
val triggerType: TriggerType = TriggerType.notification,
val conditions: List<TriggerCondition> = emptyList(),
val goalTemplate: String, // sent to agent as a goal
val enabled: Boolean = true,
val createdAt: Long = System.currentTimeMillis()
)

View File

@@ -84,7 +84,13 @@ fun OverlayContent() {
!isConnected -> "Offline"
displayStatus == GoalStatus.Running -> {
val last = steps.lastOrNull()
if (last != null) "Step ${last.step}: ${last.action}" else "Running..."
if (last != null) {
val label = last.reasoning.ifBlank {
// Extract just the action name from the JSON string
Regex("""action[=:]?\s*(\w+)""").find(last.action)?.groupValues?.get(1) ?: "working"
}
"${last.step}: $label"
} else "Running..."
}
displayStatus == GoalStatus.Completed -> "Done"
displayStatus == GoalStatus.Failed -> "Stopped"

View File

@@ -1,7 +1,9 @@
package com.thisux.droidclaw.ui.screens
import android.content.Intent
import androidx.compose.animation.AnimatedVisibility
import androidx.compose.foundation.background
import androidx.compose.foundation.clickable
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
@@ -15,35 +17,49 @@ import androidx.compose.foundation.layout.size
import androidx.compose.foundation.lazy.LazyColumn
import androidx.compose.foundation.lazy.items
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.Delete
import androidx.compose.material.icons.filled.ExpandLess
import androidx.compose.material.icons.filled.ExpandMore
import androidx.compose.material3.Button
import androidx.compose.material3.Card
import androidx.compose.material3.CardDefaults
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButton
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.OutlinedTextField
import androidx.compose.material3.Switch
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.saveable.rememberSaveable
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import com.thisux.droidclaw.DroidClawApp
import com.thisux.droidclaw.connection.ConnectionService
import com.thisux.droidclaw.model.ConnectionState
import com.thisux.droidclaw.model.GoalStatus
import com.thisux.droidclaw.model.Workflow
@Composable
fun HomeScreen() {
val context = LocalContext.current
val app = context.applicationContext as DroidClawApp
val connectionState by ConnectionService.connectionState.collectAsState()
val goalStatus by ConnectionService.currentGoalStatus.collectAsState()
val steps by ConnectionService.currentSteps.collectAsState()
val currentGoal by ConnectionService.currentGoal.collectAsState()
val errorMessage by ConnectionService.errorMessage.collectAsState()
val workflows by app.workflowStore.workflows.collectAsState(initial = emptyList())
var goalInput by remember { mutableStateOf("") }
@@ -108,7 +124,7 @@ fun HomeScreen() {
Spacer(modifier = Modifier.height(16.dp))
// Goal Input
// Goal Input — same field for goals and workflows
Row(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.spacedBy(8.dp)
@@ -116,7 +132,7 @@ fun HomeScreen() {
OutlinedTextField(
value = goalInput,
onValueChange = { goalInput = it },
label = { Text("Enter a goal...") },
label = { Text("Goal or workflow...") },
modifier = Modifier.weight(1f),
enabled = connectionState == ConnectionState.Connected && goalStatus != GoalStatus.Running,
singleLine = true
@@ -136,7 +152,7 @@ fun HomeScreen() {
&& goalStatus != GoalStatus.Running
&& goalInput.isNotBlank()
) {
Text("Run")
Text("Send")
}
}
@@ -192,5 +208,103 @@ fun HomeScreen() {
}
)
}
// Saved Workflows section
if (workflows.isNotEmpty()) {
Spacer(modifier = Modifier.height(8.dp))
WorkflowsSection(workflows)
}
}
}
@Composable
private fun WorkflowsSection(workflows: List<Workflow>) {
var expanded by rememberSaveable { mutableStateOf(false) }
Column {
Row(
modifier = Modifier
.fillMaxWidth()
.clickable { expanded = !expanded }
.padding(vertical = 4.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.SpaceBetween
) {
Text(
text = "Saved Workflows (${workflows.size})",
style = MaterialTheme.typography.titleSmall
)
Icon(
imageVector = if (expanded) Icons.Default.ExpandLess else Icons.Default.ExpandMore,
contentDescription = if (expanded) "Collapse" else "Expand"
)
}
AnimatedVisibility(visible = expanded) {
Column(verticalArrangement = Arrangement.spacedBy(6.dp)) {
workflows.forEach { wf ->
WorkflowChip(wf)
}
}
}
}
}
@Composable
private fun WorkflowChip(workflow: Workflow) {
Card(
modifier = Modifier.fillMaxWidth(),
colors = CardDefaults.cardColors(
containerColor = if (workflow.enabled) {
MaterialTheme.colorScheme.secondaryContainer
} else {
MaterialTheme.colorScheme.surfaceVariant
}
)
) {
Row(
modifier = Modifier
.fillMaxWidth()
.padding(horizontal = 12.dp, vertical = 8.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.SpaceBetween
) {
Column(modifier = Modifier.weight(1f)) {
Text(
text = workflow.name,
style = MaterialTheme.typography.bodyMedium,
maxLines = 1,
overflow = TextOverflow.Ellipsis
)
Text(
text = workflow.description,
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
maxLines = 1,
overflow = TextOverflow.Ellipsis
)
}
Row(verticalAlignment = Alignment.CenterVertically) {
Switch(
checked = workflow.enabled,
onCheckedChange = { enabled ->
ConnectionService.instance?.sendWorkflowUpdate(workflow.id, enabled)
}
)
IconButton(
onClick = {
ConnectionService.instance?.sendWorkflowDelete(workflow.id)
},
modifier = Modifier.size(32.dp)
) {
Icon(
Icons.Default.Delete,
contentDescription = "Delete",
tint = MaterialTheme.colorScheme.error,
modifier = Modifier.size(18.dp)
)
}
}
}
}
}

View File

@@ -49,6 +49,7 @@ import com.thisux.droidclaw.DroidClawApp
import com.thisux.droidclaw.accessibility.DroidClawAccessibilityService
import com.thisux.droidclaw.capture.ScreenCaptureManager
import com.thisux.droidclaw.util.BatteryOptimization
import com.thisux.droidclaw.workflow.WorkflowNotificationService
import kotlinx.coroutines.launch
@Composable
@@ -76,6 +77,9 @@ fun SettingsScreen() {
}
var isBatteryExempt by remember { mutableStateOf(BatteryOptimization.isIgnoringBatteryOptimizations(context)) }
var hasOverlayPermission by remember { mutableStateOf(Settings.canDrawOverlays(context)) }
var isNotificationListenerEnabled by remember {
mutableStateOf(WorkflowNotificationService.isEnabled(context))
}
val lifecycleOwner = LocalLifecycleOwner.current
DisposableEffect(lifecycleOwner) {
@@ -86,6 +90,7 @@ fun SettingsScreen() {
hasCaptureConsent = isCaptureAvailable || ScreenCaptureManager.hasConsent()
isBatteryExempt = BatteryOptimization.isIgnoringBatteryOptimizations(context)
hasOverlayPermission = Settings.canDrawOverlays(context)
isNotificationListenerEnabled = WorkflowNotificationService.isEnabled(context)
}
}
lifecycleOwner.lifecycle.addObserver(observer)
@@ -199,6 +204,17 @@ fun SettingsScreen() {
)
}
)
ChecklistItem(
label = "Notification listener (for workflows)",
isOk = isNotificationListenerEnabled,
actionLabel = "Enable",
onAction = {
context.startActivity(
Intent("android.settings.ACTION_NOTIFICATION_LISTENER_SETTINGS")
)
}
)
}
}

View File

@@ -0,0 +1,115 @@
package com.thisux.droidclaw.workflow
import android.content.ComponentName
import android.content.Context
import android.provider.Settings
import android.service.notification.NotificationListenerService
import android.service.notification.StatusBarNotification
import android.util.Log
import com.thisux.droidclaw.DroidClawApp
import com.thisux.droidclaw.connection.ConnectionService
import com.thisux.droidclaw.model.ConnectionState
import com.thisux.droidclaw.model.MatchMode
import com.thisux.droidclaw.model.TriggerCondition
import com.thisux.droidclaw.model.Workflow
import com.thisux.droidclaw.model.WorkflowTriggerMessage
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.launch
class WorkflowNotificationService : NotificationListenerService() {
companion object {
private const val TAG = "WorkflowNotifSvc"
fun isEnabled(context: Context): Boolean {
val flat = Settings.Secure.getString(
context.contentResolver,
"enabled_notification_listeners"
) ?: return false
val ourComponent = ComponentName(context, WorkflowNotificationService::class.java)
return flat.contains(ourComponent.flattenToString())
}
}
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
override fun onNotificationPosted(sbn: StatusBarNotification?) {
sbn ?: return
val pkg = sbn.packageName ?: return
// Ignore our own notifications
if (pkg == packageName) return
val extras = sbn.notification?.extras ?: return
val title = extras.getCharSequence("android.title")?.toString() ?: ""
val text = extras.getCharSequence("android.text")?.toString() ?: ""
Log.d(TAG, "Notification from=$pkg title=$title text=$text")
scope.launch {
try {
val app = application as DroidClawApp
val workflows = app.workflowStore.workflows.first()
val enabled = workflows.filter { it.enabled }
for (wf in enabled) {
if (matchesWorkflow(wf, pkg, title, text)) {
Log.i(TAG, "Workflow '${wf.name}' matched notification from $pkg")
triggerWorkflow(wf, pkg, title, text)
}
}
} catch (e: Exception) {
Log.e(TAG, "Failed to process notification for workflows: ${e.message}")
}
}
}
private fun matchesWorkflow(
wf: Workflow,
pkg: String,
title: String,
text: String
): Boolean {
if (wf.conditions.isEmpty()) return false
return wf.conditions.all { cond -> matchesCondition(cond, pkg, title, text) }
}
private fun matchesCondition(
cond: TriggerCondition,
pkg: String,
title: String,
text: String
): Boolean {
val actual = when (cond.field) {
"app_package" -> pkg
"title" -> title
"text" -> text
else -> return false
}
return when (cond.matchMode) {
MatchMode.contains -> actual.contains(cond.value, ignoreCase = true)
MatchMode.exact -> actual.equals(cond.value, ignoreCase = true)
MatchMode.regex -> try {
Regex(cond.value, RegexOption.IGNORE_CASE).containsMatchIn(actual)
} catch (_: Exception) { false }
}
}
private fun triggerWorkflow(wf: Workflow, pkg: String, title: String, text: String) {
val svc = ConnectionService.instance ?: return
if (ConnectionService.connectionState.value != ConnectionState.Connected) {
Log.w(TAG, "Cannot trigger workflow '${wf.name}': not connected")
return
}
svc.sendWorkflowTrigger(
WorkflowTriggerMessage(
workflowId = wf.id,
notificationApp = pkg,
notificationTitle = title,
notificationText = text
)
)
}
}

View File

@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<network-security-config>
<base-config cleartextTrafficPermitted="true" />
</network-security-config>