diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index f23f06f..9382be0 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -53,14 +53,6 @@ android:foregroundServiceType="connectedDevice" android:exported="false" /> - - - - - \ No newline at end of file diff --git a/android/app/src/main/java/com/thisux/droidclaw/DroidClawApp.kt b/android/app/src/main/java/com/thisux/droidclaw/DroidClawApp.kt index 87fb727..31752b6 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/DroidClawApp.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/DroidClawApp.kt @@ -2,17 +2,13 @@ package com.thisux.droidclaw import android.app.Application import com.thisux.droidclaw.data.SettingsStore -import com.thisux.droidclaw.data.WorkflowStore class DroidClawApp : Application() { lateinit var settingsStore: SettingsStore private set - lateinit var workflowStore: WorkflowStore - private set override fun onCreate() { super.onCreate() settingsStore = SettingsStore(this) - workflowStore = WorkflowStore(this) } } diff --git a/android/app/src/main/java/com/thisux/droidclaw/connection/CommandRouter.kt b/android/app/src/main/java/com/thisux/droidclaw/connection/CommandRouter.kt index 84b9421..6558e68 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/connection/CommandRouter.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/connection/CommandRouter.kt @@ -12,20 +12,14 @@ import com.thisux.droidclaw.model.PongMessage import com.thisux.droidclaw.model.ResultResponse import com.thisux.droidclaw.model.ScreenResponse import com.thisux.droidclaw.model.ServerMessage -import com.thisux.droidclaw.model.Workflow import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.serialization.json.Json class CommandRouter( private val webSocket: ReliableWebSocket, - private val captureManager: ScreenCaptureManager?, - private val onWorkflowSync: (suspend (List) -> Unit)? = null, - private val onWorkflowCreated: (suspend (Workflow) -> Unit)? = null, - private val onWorkflowDeleted: (suspend (String) -> Unit)? = null + private val captureManager: ScreenCaptureManager? ) { companion object { private const val TAG = "CommandRouter" - private val json = Json { ignoreUnknownKeys = true; encodeDefaults = true } } val currentGoalStatus = MutableStateFlow(GoalStatus.Idle) @@ -77,38 +71,6 @@ class CommandRouter( Log.i(TAG, "Goal failed: ${msg.message}") } - "workflow_created" -> { - val wfJson = msg.workflowJson ?: return - try { - val wf = json.decodeFromString(wfJson) - onWorkflowCreated?.invoke(wf) - Log.i(TAG, "Workflow created: ${wf.name}") - } catch (e: Exception) { - Log.e(TAG, "Failed to parse workflow_created: ${e.message}") - } - } - "workflow_synced" -> { - val wfsJson = msg.workflowsJson ?: return - try { - val wfs = json.decodeFromString>(wfsJson) - onWorkflowSync?.invoke(wfs) - Log.i(TAG, "Workflows synced: ${wfs.size} workflows") - } catch (e: Exception) { - Log.e(TAG, "Failed to parse workflow_synced: ${e.message}") - } - } - "workflow_deleted" -> { - val id = msg.workflowId ?: return - onWorkflowDeleted?.invoke(id) - Log.i(TAG, "Workflow deleted: $id") - } - "workflow_goal" -> { - val goal = msg.goal ?: return - Log.i(TAG, "Workflow-triggered goal: $goal") - // Submit as a regular goal via the WebSocket - webSocket.sendTyped(com.thisux.droidclaw.model.GoalMessage(text = goal)) - } - else -> Log.w(TAG, "Unknown message type: ${msg.type}") } } diff --git a/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt b/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt index ee08d12..a267066 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt @@ -29,11 +29,6 @@ import android.content.pm.PackageManager import android.net.Uri import android.provider.Settings import com.thisux.droidclaw.model.StopGoalMessage -import com.thisux.droidclaw.model.WorkflowCreateMessage -import com.thisux.droidclaw.model.WorkflowDeleteMessage -import com.thisux.droidclaw.model.WorkflowSyncMessage -import com.thisux.droidclaw.model.WorkflowTriggerMessage -import com.thisux.droidclaw.model.WorkflowUpdateMessage import com.thisux.droidclaw.overlay.AgentOverlay import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow @@ -127,12 +122,7 @@ class ConnectionService : LifecycleService() { } webSocket = ws - val router = CommandRouter( - ws, captureManager, - onWorkflowSync = { workflows -> app.workflowStore.replaceAll(workflows) }, - onWorkflowCreated = { workflow -> app.workflowStore.save(workflow) }, - onWorkflowDeleted = { id -> app.workflowStore.delete(id) } - ) + val router = CommandRouter(ws, captureManager) commandRouter = router launch { @@ -154,8 +144,6 @@ class ConnectionService : LifecycleService() { val apps = getInstalledApps() webSocket?.sendTyped(AppsMessage(apps = apps)) Log.i(TAG, "Sent ${apps.size} installed apps to server") - // Sync workflows from server - webSocket?.sendTyped(WorkflowSyncMessage()) } } } @@ -193,26 +181,6 @@ class ConnectionService : LifecycleService() { webSocket?.sendTyped(StopGoalMessage()) } - fun sendWorkflowCreate(description: String) { - webSocket?.sendTyped(WorkflowCreateMessage(description = description)) - } - - fun sendWorkflowUpdate(workflowId: String, enabled: Boolean?) { - webSocket?.sendTyped(WorkflowUpdateMessage(workflowId = workflowId, enabled = enabled)) - } - - fun sendWorkflowDelete(workflowId: String) { - webSocket?.sendTyped(WorkflowDeleteMessage(workflowId = workflowId)) - } - - fun sendWorkflowSync() { - webSocket?.sendTyped(WorkflowSyncMessage()) - } - - fun sendWorkflowTrigger(msg: WorkflowTriggerMessage) { - webSocket?.sendTyped(msg) - } - private fun disconnect() { overlay?.hide() webSocket?.disconnect() diff --git a/android/app/src/main/java/com/thisux/droidclaw/data/WorkflowStore.kt b/android/app/src/main/java/com/thisux/droidclaw/data/WorkflowStore.kt deleted file mode 100644 index e96a0fd..0000000 --- a/android/app/src/main/java/com/thisux/droidclaw/data/WorkflowStore.kt +++ /dev/null @@ -1,48 +0,0 @@ -package com.thisux.droidclaw.data - -import android.content.Context -import androidx.datastore.preferences.core.edit -import androidx.datastore.preferences.core.stringPreferencesKey -import com.thisux.droidclaw.model.Workflow -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.map -import kotlinx.serialization.encodeToString -import kotlinx.serialization.json.Json - -private val WORKFLOWS_KEY = stringPreferencesKey("workflows_json") -private val json = Json { ignoreUnknownKeys = true; encodeDefaults = true } - -class WorkflowStore(private val context: Context) { - - val workflows: Flow> = context.dataStore.data.map { prefs -> - val raw = prefs[WORKFLOWS_KEY] ?: "[]" - try { json.decodeFromString>(raw) } catch (_: Exception) { emptyList() } - } - - suspend fun save(workflow: Workflow) { - context.dataStore.edit { prefs -> - val list = currentList(prefs).toMutableList() - val idx = list.indexOfFirst { it.id == workflow.id } - if (idx >= 0) list[idx] = workflow else list.add(workflow) - prefs[WORKFLOWS_KEY] = json.encodeToString(list) - } - } - - suspend fun delete(workflowId: String) { - context.dataStore.edit { prefs -> - val list = currentList(prefs).filter { it.id != workflowId } - prefs[WORKFLOWS_KEY] = json.encodeToString(list) - } - } - - suspend fun replaceAll(workflows: List) { - context.dataStore.edit { prefs -> - prefs[WORKFLOWS_KEY] = json.encodeToString(workflows) - } - } - - private fun currentList(prefs: androidx.datastore.preferences.core.Preferences): List { - val raw = prefs[WORKFLOWS_KEY] ?: "[]" - return try { json.decodeFromString>(raw) } catch (_: Exception) { emptyList() } - } -} diff --git a/android/app/src/main/java/com/thisux/droidclaw/model/Protocol.kt b/android/app/src/main/java/com/thisux/droidclaw/model/Protocol.kt index 3397879..a68ecc6 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/model/Protocol.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/model/Protocol.kt @@ -76,39 +76,6 @@ data class StopGoalMessage( val type: String = "stop_goal" ) -@Serializable -data class WorkflowCreateMessage( - val type: String = "workflow_create", - val description: String // natural-language workflow description -) - -@Serializable -data class WorkflowUpdateMessage( - val type: String = "workflow_update", - val workflowId: String, - val enabled: Boolean? = null -) - -@Serializable -data class WorkflowDeleteMessage( - val type: String = "workflow_delete", - val workflowId: String -) - -@Serializable -data class WorkflowSyncMessage( - val type: String = "workflow_sync" -) - -@Serializable -data class WorkflowTriggerMessage( - val type: String = "workflow_trigger", - val workflowId: String, - val notificationApp: String? = null, - val notificationTitle: String? = null, - val notificationText: String? = null -) - @Serializable data class ServerMessage( val type: String, @@ -139,9 +106,5 @@ data class ServerMessage( val intentUri: String? = null, val intentType: String? = null, val intentExtras: Map? = null, - val setting: String? = null, - // Workflow fields - val workflowId: String? = null, - val workflowJson: String? = null, // single workflow as JSON - val workflowsJson: String? = null // array of workflows as JSON (for sync) + val setting: String? = null ) diff --git a/android/app/src/main/java/com/thisux/droidclaw/model/Workflow.kt b/android/app/src/main/java/com/thisux/droidclaw/model/Workflow.kt deleted file mode 100644 index 730e2db..0000000 --- a/android/app/src/main/java/com/thisux/droidclaw/model/Workflow.kt +++ /dev/null @@ -1,32 +0,0 @@ -package com.thisux.droidclaw.model - -import kotlinx.serialization.Serializable - -@Serializable -enum class TriggerType { - notification -} - -@Serializable -enum class MatchMode { - contains, exact, regex -} - -@Serializable -data class TriggerCondition( - val field: String, // "app_package", "title", "text" - val matchMode: MatchMode, - val value: String -) - -@Serializable -data class Workflow( - val id: String, - val name: String, - val description: String, // original natural-language input - val triggerType: TriggerType = TriggerType.notification, - val conditions: List = emptyList(), - val goalTemplate: String, // sent to agent as a goal - val enabled: Boolean = true, - val createdAt: Long = System.currentTimeMillis() -) diff --git a/android/app/src/main/java/com/thisux/droidclaw/ui/screens/HomeScreen.kt b/android/app/src/main/java/com/thisux/droidclaw/ui/screens/HomeScreen.kt index 5e04328..47bea8f 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/ui/screens/HomeScreen.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/ui/screens/HomeScreen.kt @@ -1,9 +1,7 @@ package com.thisux.droidclaw.ui.screens import android.content.Intent -import androidx.compose.animation.AnimatedVisibility import androidx.compose.foundation.background -import androidx.compose.foundation.clickable import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column @@ -17,49 +15,35 @@ import androidx.compose.foundation.layout.size import androidx.compose.foundation.lazy.LazyColumn import androidx.compose.foundation.lazy.items import androidx.compose.foundation.shape.CircleShape -import androidx.compose.material.icons.Icons -import androidx.compose.material.icons.filled.Delete -import androidx.compose.material.icons.filled.ExpandLess -import androidx.compose.material.icons.filled.ExpandMore import androidx.compose.material3.Button import androidx.compose.material3.Card -import androidx.compose.material3.CardDefaults -import androidx.compose.material3.Icon -import androidx.compose.material3.IconButton import androidx.compose.material3.MaterialTheme import androidx.compose.material3.OutlinedTextField -import androidx.compose.material3.Switch import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember -import androidx.compose.runtime.saveable.rememberSaveable import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip import androidx.compose.ui.graphics.Color import androidx.compose.ui.platform.LocalContext -import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.unit.dp -import com.thisux.droidclaw.DroidClawApp import com.thisux.droidclaw.connection.ConnectionService import com.thisux.droidclaw.model.ConnectionState import com.thisux.droidclaw.model.GoalStatus -import com.thisux.droidclaw.model.Workflow @Composable fun HomeScreen() { val context = LocalContext.current - val app = context.applicationContext as DroidClawApp val connectionState by ConnectionService.connectionState.collectAsState() val goalStatus by ConnectionService.currentGoalStatus.collectAsState() val steps by ConnectionService.currentSteps.collectAsState() val currentGoal by ConnectionService.currentGoal.collectAsState() val errorMessage by ConnectionService.errorMessage.collectAsState() - val workflows by app.workflowStore.workflows.collectAsState(initial = emptyList()) var goalInput by remember { mutableStateOf("") } @@ -124,7 +108,7 @@ fun HomeScreen() { Spacer(modifier = Modifier.height(16.dp)) - // Goal Input — same field for goals and workflows + // Goal Input Row( modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.spacedBy(8.dp) @@ -132,7 +116,7 @@ fun HomeScreen() { OutlinedTextField( value = goalInput, onValueChange = { goalInput = it }, - label = { Text("Goal or workflow...") }, + label = { Text("Enter a goal...") }, modifier = Modifier.weight(1f), enabled = connectionState == ConnectionState.Connected && goalStatus != GoalStatus.Running, singleLine = true @@ -209,102 +193,5 @@ fun HomeScreen() { ) } - // Saved Workflows section - if (workflows.isNotEmpty()) { - Spacer(modifier = Modifier.height(8.dp)) - WorkflowsSection(workflows) - } - } -} - -@Composable -private fun WorkflowsSection(workflows: List) { - var expanded by rememberSaveable { mutableStateOf(false) } - - Column { - Row( - modifier = Modifier - .fillMaxWidth() - .clickable { expanded = !expanded } - .padding(vertical = 4.dp), - verticalAlignment = Alignment.CenterVertically, - horizontalArrangement = Arrangement.SpaceBetween - ) { - Text( - text = "Saved Workflows (${workflows.size})", - style = MaterialTheme.typography.titleSmall - ) - Icon( - imageVector = if (expanded) Icons.Default.ExpandLess else Icons.Default.ExpandMore, - contentDescription = if (expanded) "Collapse" else "Expand" - ) - } - - AnimatedVisibility(visible = expanded) { - Column(verticalArrangement = Arrangement.spacedBy(6.dp)) { - workflows.forEach { wf -> - WorkflowChip(wf) - } - } - } - } -} - -@Composable -private fun WorkflowChip(workflow: Workflow) { - Card( - modifier = Modifier.fillMaxWidth(), - colors = CardDefaults.cardColors( - containerColor = if (workflow.enabled) { - MaterialTheme.colorScheme.secondaryContainer - } else { - MaterialTheme.colorScheme.surfaceVariant - } - ) - ) { - Row( - modifier = Modifier - .fillMaxWidth() - .padding(horizontal = 12.dp, vertical = 8.dp), - verticalAlignment = Alignment.CenterVertically, - horizontalArrangement = Arrangement.SpaceBetween - ) { - Column(modifier = Modifier.weight(1f)) { - Text( - text = workflow.name, - style = MaterialTheme.typography.bodyMedium, - maxLines = 1, - overflow = TextOverflow.Ellipsis - ) - Text( - text = workflow.description, - style = MaterialTheme.typography.bodySmall, - color = MaterialTheme.colorScheme.onSurfaceVariant, - maxLines = 1, - overflow = TextOverflow.Ellipsis - ) - } - Row(verticalAlignment = Alignment.CenterVertically) { - Switch( - checked = workflow.enabled, - onCheckedChange = { enabled -> - ConnectionService.instance?.sendWorkflowUpdate(workflow.id, enabled) - } - ) - IconButton( - onClick = { - ConnectionService.instance?.sendWorkflowDelete(workflow.id) - }, - modifier = Modifier.size(32.dp) - ) { - Icon( - Icons.Default.Delete, - contentDescription = "Delete", - tint = MaterialTheme.colorScheme.error, - modifier = Modifier.size(18.dp) - ) - } - } - } } } diff --git a/android/app/src/main/java/com/thisux/droidclaw/ui/screens/SettingsScreen.kt b/android/app/src/main/java/com/thisux/droidclaw/ui/screens/SettingsScreen.kt index c5d21d4..aa9444a 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/ui/screens/SettingsScreen.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/ui/screens/SettingsScreen.kt @@ -49,7 +49,6 @@ import com.thisux.droidclaw.DroidClawApp import com.thisux.droidclaw.accessibility.DroidClawAccessibilityService import com.thisux.droidclaw.capture.ScreenCaptureManager import com.thisux.droidclaw.util.BatteryOptimization -import com.thisux.droidclaw.workflow.WorkflowNotificationService import kotlinx.coroutines.launch @Composable @@ -77,9 +76,6 @@ fun SettingsScreen() { } var isBatteryExempt by remember { mutableStateOf(BatteryOptimization.isIgnoringBatteryOptimizations(context)) } var hasOverlayPermission by remember { mutableStateOf(Settings.canDrawOverlays(context)) } - var isNotificationListenerEnabled by remember { - mutableStateOf(WorkflowNotificationService.isEnabled(context)) - } val lifecycleOwner = LocalLifecycleOwner.current DisposableEffect(lifecycleOwner) { @@ -90,7 +86,6 @@ fun SettingsScreen() { hasCaptureConsent = isCaptureAvailable || ScreenCaptureManager.hasConsent() isBatteryExempt = BatteryOptimization.isIgnoringBatteryOptimizations(context) hasOverlayPermission = Settings.canDrawOverlays(context) - isNotificationListenerEnabled = WorkflowNotificationService.isEnabled(context) } } lifecycleOwner.lifecycle.addObserver(observer) @@ -205,16 +200,6 @@ fun SettingsScreen() { } ) - ChecklistItem( - label = "Notification listener (for workflows)", - isOk = isNotificationListenerEnabled, - actionLabel = "Enable", - onAction = { - context.startActivity( - Intent("android.settings.ACTION_NOTIFICATION_LISTENER_SETTINGS") - ) - } - ) } } diff --git a/android/app/src/main/java/com/thisux/droidclaw/workflow/WorkflowNotificationService.kt b/android/app/src/main/java/com/thisux/droidclaw/workflow/WorkflowNotificationService.kt deleted file mode 100644 index 8508c28..0000000 --- a/android/app/src/main/java/com/thisux/droidclaw/workflow/WorkflowNotificationService.kt +++ /dev/null @@ -1,115 +0,0 @@ -package com.thisux.droidclaw.workflow - -import android.content.ComponentName -import android.content.Context -import android.provider.Settings -import android.service.notification.NotificationListenerService -import android.service.notification.StatusBarNotification -import android.util.Log -import com.thisux.droidclaw.DroidClawApp -import com.thisux.droidclaw.connection.ConnectionService -import com.thisux.droidclaw.model.ConnectionState -import com.thisux.droidclaw.model.MatchMode -import com.thisux.droidclaw.model.TriggerCondition -import com.thisux.droidclaw.model.Workflow -import com.thisux.droidclaw.model.WorkflowTriggerMessage -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.SupervisorJob -import kotlinx.coroutines.flow.first -import kotlinx.coroutines.launch - -class WorkflowNotificationService : NotificationListenerService() { - - companion object { - private const val TAG = "WorkflowNotifSvc" - - fun isEnabled(context: Context): Boolean { - val flat = Settings.Secure.getString( - context.contentResolver, - "enabled_notification_listeners" - ) ?: return false - val ourComponent = ComponentName(context, WorkflowNotificationService::class.java) - return flat.contains(ourComponent.flattenToString()) - } - } - - private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) - - override fun onNotificationPosted(sbn: StatusBarNotification?) { - sbn ?: return - val pkg = sbn.packageName ?: return - // Ignore our own notifications - if (pkg == packageName) return - - val extras = sbn.notification?.extras ?: return - val title = extras.getCharSequence("android.title")?.toString() ?: "" - val text = extras.getCharSequence("android.text")?.toString() ?: "" - - Log.d(TAG, "Notification from=$pkg title=$title text=$text") - - scope.launch { - try { - val app = application as DroidClawApp - val workflows = app.workflowStore.workflows.first() - val enabled = workflows.filter { it.enabled } - - for (wf in enabled) { - if (matchesWorkflow(wf, pkg, title, text)) { - Log.i(TAG, "Workflow '${wf.name}' matched notification from $pkg") - triggerWorkflow(wf, pkg, title, text) - } - } - } catch (e: Exception) { - Log.e(TAG, "Failed to process notification for workflows: ${e.message}") - } - } - } - - private fun matchesWorkflow( - wf: Workflow, - pkg: String, - title: String, - text: String - ): Boolean { - if (wf.conditions.isEmpty()) return false - return wf.conditions.all { cond -> matchesCondition(cond, pkg, title, text) } - } - - private fun matchesCondition( - cond: TriggerCondition, - pkg: String, - title: String, - text: String - ): Boolean { - val actual = when (cond.field) { - "app_package" -> pkg - "title" -> title - "text" -> text - else -> return false - } - return when (cond.matchMode) { - MatchMode.contains -> actual.contains(cond.value, ignoreCase = true) - MatchMode.exact -> actual.equals(cond.value, ignoreCase = true) - MatchMode.regex -> try { - Regex(cond.value, RegexOption.IGNORE_CASE).containsMatchIn(actual) - } catch (_: Exception) { false } - } - } - - private fun triggerWorkflow(wf: Workflow, pkg: String, title: String, text: String) { - val svc = ConnectionService.instance ?: return - if (ConnectionService.connectionState.value != ConnectionState.Connected) { - Log.w(TAG, "Cannot trigger workflow '${wf.name}': not connected") - return - } - svc.sendWorkflowTrigger( - WorkflowTriggerMessage( - workflowId = wf.id, - notificationApp = pkg, - notificationTitle = title, - notificationText = text - ) - ) - } -} diff --git a/docs/plans/workflow-automation-v2.md b/docs/plans/workflow-automation-v2.md new file mode 100644 index 0000000..3a3e9e1 --- /dev/null +++ b/docs/plans/workflow-automation-v2.md @@ -0,0 +1,119 @@ +# Workflow Automation v2 + +## Context + +msomu's PR #6 added a workflow automation system (notification-triggered agent goals). The concept is valuable but the implementation had issues, so the workflow code was removed while keeping the overlay, stop_goal, and AbortSignal changes. + +This doc captures what was good, what was wrong, and how to ship it properly. + +## The Core Idea + +Turn DroidClaw from a manual remote-control into a persistent automation engine. Users describe rules in plain English like: + +- "When I get a WhatsApp message saying 'where are you', reply with 'Bangalore'" +- "Whenever someone messages me on Telegram, auto-reply with 'I'm busy'" +- "When boss emails me, open it and mark as important" + +Notifications trigger the agent to execute goals automatically. + +## What Was Built (and removed) + +- **Input classifier** — LLM call on every goal to detect "goal" vs "workflow" +- **Workflow parser** — LLM converts natural language to structured trigger conditions (app, title, text matching with contains/exact/regex) +- **Workflow table** — Postgres storage for parsed workflows +- **NotificationListenerService** — Android service capturing all notifications, matching against synced workflows +- **Workflow CRUD** — create/update/delete/sync/trigger via WebSocket +- **Auto-execution** — matched workflows send goals to the agent pipeline with no confirmation + +## Problems With v1 + +### 1. Classifier tax on every goal +Every goal got an extra LLM round-trip to decide if it's a goal or workflow. Adds latency and cost on 100% of requests to benefit ~5% of inputs. + +### 2. LLM-generated regex is fragile +The parser asks the LLM to produce regex match conditions. LLMs are bad at regex. One wrong pattern = workflow never triggers or triggers on everything. + +### 3. No guardrails on auto-execution +A notification match runs the full agent pipeline automatically — tapping, typing, navigating with zero human confirmation. One bad match = replying to your boss with the wrong message. + +### 4. No observability +No execution history, no "workflow X triggered 5 times today", no way to preview what a workflow would match before enabling it. + +### 5. Powerful permission for a v1 +`NotificationListenerService` reads ALL notifications. Users will hesitate to grant that without clear value. + +## v2 Design + +### Explicit creation, not auto-classification +- A dedicated "Create Workflow" button/screen, not auto-detection of every goal input +- Remove the classifier entirely — goals are goals, workflows are created intentionally +- Web dashboard should also support workflow creation/management + +### Confirmation mode (default) +- When a workflow matches a notification, show a confirmation notification: "Workflow 'Auto-reply busy' matched! Run this goal?" +- User taps to confirm, agent executes +- Power users can toggle to "auto-execute" per workflow after they trust it +- Three modes: `confirm` (default), `auto`, `disabled` + +### Simple server-side conditions +- Don't use LLM to generate regex — use simple string matching configured via UI +- Fields: app name (dropdown from installed apps), title contains, text contains +- AND logic between conditions +- Let the LLM help draft the goal template, but conditions should be human-configured + +### Execution log +- Record every trigger: timestamp, workflow name, matched notification, goal sent, result (success/fail/skipped) +- Show in web dashboard and in-app +- Rate limiting: max N triggers per workflow per hour (prevent notification storms) + +### Scoped notification access +- Only listen for notifications from apps the user explicitly selects in workflow conditions +- Show exactly which apps are being monitored in settings +- Easy one-tap disable-all + +### Goal template improvements +- Preview: show what the expanded goal would look like with sample notification data +- Variables: `{{app}}`, `{{title}}`, `{{text}}`, `{{time}}` +- Test button: "Simulate this workflow with a fake notification" + +## Implementation Order + +1. **Web dashboard workflow CRUD** — create/edit/delete workflows with simple condition builder +2. **Confirmation mode** — notification-based confirm-before-execute +3. **Execution log** — record and display trigger history +4. **Scoped notification listener** — only monitor selected apps +5. **Auto-execute toggle** — per-workflow setting for trusted workflows +6. **Rate limiting** — prevent runaway triggers + +## Schema (revised) + +```sql +CREATE TABLE workflow ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL REFERENCES "user"(id) ON DELETE CASCADE, + name TEXT NOT NULL, + description TEXT NOT NULL, + execution_mode TEXT NOT NULL DEFAULT 'confirm', -- confirm | auto | disabled + conditions JSONB NOT NULL DEFAULT '[]', + goal_template TEXT NOT NULL, + max_triggers_per_hour INT DEFAULT 5, + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() +); + +CREATE TABLE workflow_execution ( + id TEXT PRIMARY KEY, + workflow_id TEXT NOT NULL REFERENCES workflow(id) ON DELETE CASCADE, + notification_app TEXT, + notification_title TEXT, + notification_text TEXT, + expanded_goal TEXT NOT NULL, + status TEXT NOT NULL, -- confirmed | auto_executed | skipped | failed + agent_session_id TEXT REFERENCES agent_session(id), + triggered_at TIMESTAMP DEFAULT NOW() +); +``` + +## Key Principle + +The agent taking autonomous action on a user's phone is powerful and dangerous. Default to safety: confirm before executing, log everything, let users build trust gradually before enabling auto-mode. diff --git a/packages/shared/src/protocol.ts b/packages/shared/src/protocol.ts index 7b93e57..5aeda72 100644 --- a/packages/shared/src/protocol.ts +++ b/packages/shared/src/protocol.ts @@ -8,12 +8,7 @@ export type DeviceMessage = | { type: "pong" } | { type: "heartbeat"; batteryLevel: number; isCharging: boolean } | { type: "apps"; apps: InstalledApp[] } - | { type: "stop_goal" } - | { type: "workflow_create"; description: string } - | { type: "workflow_update"; workflowId: string; enabled?: boolean } - | { type: "workflow_delete"; workflowId: string } - | { type: "workflow_sync" } - | { type: "workflow_trigger"; workflowId: string; notificationApp?: string; notificationTitle?: string; notificationText?: string }; + | { type: "stop_goal" }; export type ServerToDeviceMessage = | { type: "auth_ok"; deviceId: string } diff --git a/server/src/agent/input-classifier.ts b/server/src/agent/input-classifier.ts deleted file mode 100644 index f340294..0000000 --- a/server/src/agent/input-classifier.ts +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Classifies user input as either an immediate goal or a workflow (automation rule). - * - * Uses the user's LLM to determine intent. Workflows describe recurring - * automations ("when X happens, do Y"), goals are one-time tasks ("open WhatsApp"). - */ - -import type { LLMConfig } from "./llm.js"; -import { getLlmProvider, parseJsonResponse } from "./llm.js"; - -export type InputType = "goal" | "workflow"; - -export interface ClassificationResult { - type: InputType; -} - -const CLASSIFIER_PROMPT = `You classify user input for an Android automation agent. - -Decide if the input is: -- "goal": A one-time task to execute right now (e.g. "open WhatsApp", "search for pizza", "take a screenshot", "reply to John with hello") -- "workflow": An automation rule that should be saved and triggered later when a condition is met (e.g. "when I get a notification from WhatsApp saying where are you, reply with Bangalore", "whenever someone messages me on Telegram, auto-reply with I'm busy", "reply to all notifications that have a reply button") - -Key signals for "workflow": -- Uses words like "when", "whenever", "if", "every time", "automatically", "always" -- Describes a trigger condition + a response action -- Refers to future/recurring events - -Key signals for "goal": -- Describes a single task to do now -- Imperative commands ("open", "send", "search", "go to") -- No conditional/temporal trigger - -Respond with ONLY: {"type": "goal"} or {"type": "workflow"}`; - -export async function classifyInput( - text: string, - llmConfig: LLMConfig -): Promise { - const provider = getLlmProvider(llmConfig); - - try { - const raw = await provider.getAction(CLASSIFIER_PROMPT, text); - const parsed = parseJsonResponse(raw); - - if (parsed?.type === "workflow") { - return { type: "workflow" }; - } - } catch (err) { - console.error(`[Classifier] Failed to classify input, defaulting to goal:`, err); - } - - // Default to goal — safer to execute once than to accidentally create a rule - return { type: "goal" }; -} diff --git a/server/src/agent/workflow-parser.ts b/server/src/agent/workflow-parser.ts deleted file mode 100644 index 6bfd5dc..0000000 --- a/server/src/agent/workflow-parser.ts +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Parses a natural-language workflow description into structured - * trigger conditions and a goal template using the user's LLM. - */ - -import type { LLMConfig } from "./llm.js"; -import { getLlmProvider, parseJsonResponse } from "./llm.js"; - -export interface ParsedWorkflow { - name: string; - triggerType: "notification"; - conditions: Array<{ - field: "app_package" | "title" | "text"; - matchMode: "contains" | "exact" | "regex"; - value: string; - }>; - goalTemplate: string; -} - -const PARSER_PROMPT = `You are a workflow parser for an Android automation agent. - -The user describes an automation rule in plain English. Parse it into a structured workflow. - -A workflow has: -1. **name**: A short human-readable name (3-6 words). -2. **triggerType**: Always "notification" for now. -3. **conditions**: An array of matching rules for incoming notifications. Each condition has: - - "field": one of "app_package", "title", or "text" - - "matchMode": one of "contains", "exact", or "regex" - - "value": the string or regex to match -4. **goalTemplate**: The goal string to send to the agent when triggered. Use {{title}}, {{text}}, {{app}} as placeholders that get filled from the notification. - -Example input: "When I get a WhatsApp message saying 'where are you', reply with 'Bangalore'" -Example output: -{ - "name": "Auto-reply where are you", - "triggerType": "notification", - "conditions": [ - {"field": "app_package", "matchMode": "contains", "value": "whatsapp"}, - {"field": "text", "matchMode": "contains", "value": "where are you"} - ], - "goalTemplate": "Open the WhatsApp notification from {{title}} and reply with 'Bangalore'" -} - -Example input: "Reply to all notifications that have a reply button with 'I am busy'" -Example output: -{ - "name": "Auto-reply I am busy", - "triggerType": "notification", - "conditions": [], - "goalTemplate": "Open the notification '{{title}}' from {{app}} and reply with 'I am busy'" -} - -Respond with ONLY a valid JSON object. No explanation.`; - -export async function parseWorkflowDescription( - description: string, - llmConfig: LLMConfig -): Promise { - const provider = getLlmProvider(llmConfig); - - const raw = await provider.getAction(PARSER_PROMPT, description); - const parsed = parseJsonResponse(raw); - - if (!parsed || !parsed.name || !parsed.goalTemplate) { - throw new Error("Failed to parse workflow description into structured format"); - } - - return { - name: parsed.name as string, - triggerType: "notification", - conditions: (parsed.conditions as ParsedWorkflow["conditions"]) ?? [], - goalTemplate: parsed.goalTemplate as string, - }; -} diff --git a/server/src/schema.ts b/server/src/schema.ts index ebd6fc0..dd8fb24 100644 --- a/server/src/schema.ts +++ b/server/src/schema.ts @@ -128,24 +128,6 @@ export const agentSession = pgTable("agent_session", { completedAt: timestamp("completed_at"), }); -export const workflow = pgTable("workflow", { - id: text("id").primaryKey(), - userId: text("user_id") - .notNull() - .references(() => user.id, { onDelete: "cascade" }), - name: text("name").notNull(), - description: text("description").notNull(), - triggerType: text("trigger_type").notNull().default("notification"), - conditions: jsonb("conditions").notNull().default("[]"), - goalTemplate: text("goal_template").notNull(), - enabled: boolean("enabled").default(true).notNull(), - createdAt: timestamp("created_at").defaultNow().notNull(), - updatedAt: timestamp("updated_at") - .defaultNow() - .$onUpdate(() => new Date()) - .notNull(), -}); - export const agentStep = pgTable("agent_step", { id: text("id").primaryKey(), sessionId: text("session_id") diff --git a/server/src/ws/device.ts b/server/src/ws/device.ts index b8981cf..ba8a5bd 100644 --- a/server/src/ws/device.ts +++ b/server/src/ws/device.ts @@ -6,14 +6,6 @@ import { apikey, llmConfig, device } from "../schema.js"; import { sessions, type WebSocketData } from "./sessions.js"; import { runPipeline } from "../agent/pipeline.js"; import type { LLMConfig } from "../agent/llm.js"; -import { - handleWorkflowCreate, - handleWorkflowUpdate, - handleWorkflowDelete, - handleWorkflowSync, - handleWorkflowTrigger, -} from "./workflow-handlers.js"; -import { classifyInput } from "../agent/input-classifier.js"; /** * Hash an API key the same way better-auth does: @@ -259,20 +251,6 @@ export async function handleDeviceMessage( break; } - // Classify: is this an immediate goal or a workflow? - try { - const classification = await classifyInput(goal, userLlmConfig); - if (classification.type === "workflow") { - console.log(`[Classifier] Input classified as workflow: ${goal}`); - handleWorkflowCreate(ws, goal).catch((err) => - console.error(`[Workflow] Auto-create error:`, err) - ); - break; - } - } catch (err) { - console.warn(`[Classifier] Classification failed, treating as goal:`, err); - } - console.log(`[Pipeline] Starting goal for device ${deviceId}: ${goal}`); const abortController = new AbortController(); activeSessions.set(deviceId, { goal, abort: abortController }); @@ -383,59 +361,6 @@ export async function handleDeviceMessage( break; } - case "workflow_create": { - const description = (msg as unknown as { description: string }).description; - if (description) { - handleWorkflowCreate(ws, description).catch((err) => - console.error(`[Workflow] Create error:`, err) - ); - } - break; - } - - case "workflow_update": { - const { workflowId, enabled } = msg as unknown as { workflowId: string; enabled?: boolean }; - if (workflowId) { - handleWorkflowUpdate(ws, workflowId, enabled).catch((err) => - console.error(`[Workflow] Update error:`, err) - ); - } - break; - } - - case "workflow_delete": { - const { workflowId } = msg as unknown as { workflowId: string }; - if (workflowId) { - handleWorkflowDelete(ws, workflowId).catch((err) => - console.error(`[Workflow] Delete error:`, err) - ); - } - break; - } - - case "workflow_sync": { - handleWorkflowSync(ws).catch((err) => - console.error(`[Workflow] Sync error:`, err) - ); - break; - } - - case "workflow_trigger": { - const { workflowId, notificationApp, notificationTitle, notificationText } = - msg as unknown as { - workflowId: string; - notificationApp?: string; - notificationTitle?: string; - notificationText?: string; - }; - if (workflowId) { - handleWorkflowTrigger(ws, workflowId, notificationApp, notificationTitle, notificationText).catch( - (err) => console.error(`[Workflow] Trigger error:`, err) - ); - } - break; - } - default: { console.warn( `Unknown message type from device ${ws.data.deviceId}:`, diff --git a/server/src/ws/workflow-handlers.ts b/server/src/ws/workflow-handlers.ts deleted file mode 100644 index 271f8d3..0000000 --- a/server/src/ws/workflow-handlers.ts +++ /dev/null @@ -1,229 +0,0 @@ -/** - * Server-side handlers for workflow CRUD and trigger messages - * from the Android device WebSocket. - */ - -import type { ServerWebSocket } from "bun"; -import { eq, and } from "drizzle-orm"; -import { db } from "../db.js"; -import { workflow, llmConfig } from "../schema.js"; -import { parseWorkflowDescription } from "../agent/workflow-parser.js"; -import type { LLMConfig } from "../agent/llm.js"; -import type { WebSocketData } from "./sessions.js"; - -function sendToDevice(ws: ServerWebSocket, msg: Record) { - try { - ws.send(JSON.stringify(msg)); - } catch { - // device disconnected - } -} - -async function getUserLlmConfig(userId: string): Promise { - const configs = await db - .select() - .from(llmConfig) - .where(eq(llmConfig.userId, userId)) - .limit(1); - - if (configs.length === 0) return null; - - const cfg = configs[0]; - return { - provider: cfg.provider, - apiKey: cfg.apiKey, - model: cfg.model ?? undefined, - }; -} - -function workflowToJson(wf: typeof workflow.$inferSelect): string { - return JSON.stringify({ - id: wf.id, - name: wf.name, - description: wf.description, - triggerType: wf.triggerType, - conditions: wf.conditions, - goalTemplate: wf.goalTemplate, - enabled: wf.enabled, - createdAt: new Date(wf.createdAt).getTime(), - }); -} - -export async function handleWorkflowCreate( - ws: ServerWebSocket, - description: string -): Promise { - const userId = ws.data.userId!; - - const userLlm = await getUserLlmConfig(userId); - if (!userLlm) { - sendToDevice(ws, { - type: "error", - message: "No LLM provider configured. Set it up in the web dashboard.", - }); - return; - } - - try { - const parsed = await parseWorkflowDescription(description, userLlm); - - // Validate regexes before persisting - for (const cond of parsed.conditions) { - if (cond.matchMode === "regex") { - try { - new RegExp(cond.value, "i"); - } catch { - throw new Error(`Invalid regex in condition: ${cond.value}`); - } - } - } - - const id = crypto.randomUUID(); - const now = new Date(); - - await db.insert(workflow).values({ - id, - userId, - name: parsed.name, - description, - triggerType: parsed.triggerType, - conditions: parsed.conditions, - goalTemplate: parsed.goalTemplate, - enabled: true, - createdAt: now, - updatedAt: now, - }); - - const inserted = await db - .select() - .from(workflow) - .where(eq(workflow.id, id)) - .limit(1); - - if (inserted.length > 0) { - sendToDevice(ws, { - type: "workflow_created", - workflowId: id, - workflowJson: workflowToJson(inserted[0]), - }); - } - - console.log(`[Workflow] Created '${parsed.name}' for user ${userId}`); - } catch (err) { - console.error(`[Workflow] Failed to create workflow:`, err); - sendToDevice(ws, { - type: "error", - message: `Failed to parse workflow: ${err}`, - }); - } -} - -export async function handleWorkflowUpdate( - ws: ServerWebSocket, - workflowId: string, - enabled?: boolean -): Promise { - const userId = ws.data.userId!; - - const updates: Record = {}; - if (enabled !== undefined) updates.enabled = enabled; - - await db - .update(workflow) - .set(updates) - .where(and(eq(workflow.id, workflowId), eq(workflow.userId, userId))); - - console.log(`[Workflow] Updated ${workflowId}: enabled=${enabled}`); -} - -export async function handleWorkflowDelete( - ws: ServerWebSocket, - workflowId: string -): Promise { - const userId = ws.data.userId!; - - await db - .delete(workflow) - .where(and(eq(workflow.id, workflowId), eq(workflow.userId, userId))); - - sendToDevice(ws, { - type: "workflow_deleted", - workflowId, - }); - - console.log(`[Workflow] Deleted ${workflowId}`); -} - -export async function handleWorkflowSync( - ws: ServerWebSocket -): Promise { - const userId = ws.data.userId!; - - const workflows = await db - .select() - .from(workflow) - .where(eq(workflow.userId, userId)); - - const workflowsJson = JSON.stringify( - workflows.map((wf) => ({ - id: wf.id, - name: wf.name, - description: wf.description, - triggerType: wf.triggerType, - conditions: wf.conditions, - goalTemplate: wf.goalTemplate, - enabled: wf.enabled, - createdAt: new Date(wf.createdAt).getTime(), - })) - ); - - sendToDevice(ws, { - type: "workflow_synced", - workflowsJson, - }); - - console.log(`[Workflow] Synced ${workflows.length} workflows for user ${userId}`); -} - -export async function handleWorkflowTrigger( - ws: ServerWebSocket, - workflowId: string, - notificationApp?: string, - notificationTitle?: string, - notificationText?: string -): Promise { - const userId = ws.data.userId!; - - const workflows = await db - .select() - .from(workflow) - .where(and(eq(workflow.id, workflowId), eq(workflow.userId, userId))) - .limit(1); - - if (workflows.length === 0) { - console.warn(`[Workflow] Trigger for unknown workflow ${workflowId}`); - return; - } - - const wf = workflows[0]; - if (!wf.enabled) return; - - // Expand goal template placeholders - let goal = wf.goalTemplate; - goal = goal.replace(/\{\{app\}\}/g, notificationApp ?? "unknown app"); - goal = goal.replace(/\{\{title\}\}/g, notificationTitle ?? ""); - goal = goal.replace(/\{\{text\}\}/g, notificationText ?? ""); - - console.log(`[Workflow] Triggering '${wf.name}' with goal: ${goal}`); - - // Send as a goal — reuse existing goal handling by injecting a goal message - sendToDevice(ws, { type: "ping" }); // keep-alive before goal injection - - // The device will receive this as a workflow-triggered goal - // We send the goal text back to the device to be submitted as a regular goal - sendToDevice(ws, { - type: "workflow_goal", - workflowId: wf.id, - goal, - }); -}