feat: workflow automation via NotificationListenerService
Add workflow system that lets users describe automations in natural language through the same input field. The server LLM classifies input as either an immediate goal or a workflow rule, then: - Parses workflow descriptions into structured trigger conditions - Stores workflows per-user in Postgres - Syncs workflows to device via WebSocket - NotificationListenerService monitors notifications and triggers matching workflows as agent goals Also cleans up overlay text and adds network security config.
This commit is contained in:
@@ -22,7 +22,8 @@
|
|||||||
android:label="@string/app_name"
|
android:label="@string/app_name"
|
||||||
android:roundIcon="@mipmap/ic_launcher_round"
|
android:roundIcon="@mipmap/ic_launcher_round"
|
||||||
android:supportsRtl="true"
|
android:supportsRtl="true"
|
||||||
android:theme="@style/Theme.DroidClaw">
|
android:theme="@style/Theme.DroidClaw"
|
||||||
|
android:networkSecurityConfig="@xml/network_security_config">
|
||||||
<activity
|
<activity
|
||||||
android:name=".MainActivity"
|
android:name=".MainActivity"
|
||||||
android:exported="true"
|
android:exported="true"
|
||||||
@@ -51,6 +52,15 @@
|
|||||||
android:name=".connection.ConnectionService"
|
android:name=".connection.ConnectionService"
|
||||||
android:foregroundServiceType="connectedDevice"
|
android:foregroundServiceType="connectedDevice"
|
||||||
android:exported="false" />
|
android:exported="false" />
|
||||||
|
|
||||||
|
<service
|
||||||
|
android:name=".workflow.WorkflowNotificationService"
|
||||||
|
android:permission="android.permission.BIND_NOTIFICATION_LISTENER_SERVICE"
|
||||||
|
android:exported="false">
|
||||||
|
<intent-filter>
|
||||||
|
<action android:name="android.service.notification.NotificationListenerService" />
|
||||||
|
</intent-filter>
|
||||||
|
</service>
|
||||||
</application>
|
</application>
|
||||||
|
|
||||||
</manifest>
|
</manifest>
|
||||||
@@ -2,13 +2,17 @@ package com.thisux.droidclaw
|
|||||||
|
|
||||||
import android.app.Application
|
import android.app.Application
|
||||||
import com.thisux.droidclaw.data.SettingsStore
|
import com.thisux.droidclaw.data.SettingsStore
|
||||||
|
import com.thisux.droidclaw.data.WorkflowStore
|
||||||
|
|
||||||
class DroidClawApp : Application() {
|
class DroidClawApp : Application() {
|
||||||
lateinit var settingsStore: SettingsStore
|
lateinit var settingsStore: SettingsStore
|
||||||
private set
|
private set
|
||||||
|
lateinit var workflowStore: WorkflowStore
|
||||||
|
private set
|
||||||
|
|
||||||
override fun onCreate() {
|
override fun onCreate() {
|
||||||
super.onCreate()
|
super.onCreate()
|
||||||
settingsStore = SettingsStore(this)
|
settingsStore = SettingsStore(this)
|
||||||
|
workflowStore = WorkflowStore(this)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,14 +12,20 @@ import com.thisux.droidclaw.model.PongMessage
|
|||||||
import com.thisux.droidclaw.model.ResultResponse
|
import com.thisux.droidclaw.model.ResultResponse
|
||||||
import com.thisux.droidclaw.model.ScreenResponse
|
import com.thisux.droidclaw.model.ScreenResponse
|
||||||
import com.thisux.droidclaw.model.ServerMessage
|
import com.thisux.droidclaw.model.ServerMessage
|
||||||
|
import com.thisux.droidclaw.model.Workflow
|
||||||
import kotlinx.coroutines.flow.MutableStateFlow
|
import kotlinx.coroutines.flow.MutableStateFlow
|
||||||
|
import kotlinx.serialization.json.Json
|
||||||
|
|
||||||
class CommandRouter(
|
class CommandRouter(
|
||||||
private val webSocket: ReliableWebSocket,
|
private val webSocket: ReliableWebSocket,
|
||||||
private val captureManager: ScreenCaptureManager?
|
private val captureManager: ScreenCaptureManager?,
|
||||||
|
private val onWorkflowSync: (suspend (List<Workflow>) -> Unit)? = null,
|
||||||
|
private val onWorkflowCreated: (suspend (Workflow) -> Unit)? = null,
|
||||||
|
private val onWorkflowDeleted: (suspend (String) -> Unit)? = null
|
||||||
) {
|
) {
|
||||||
companion object {
|
companion object {
|
||||||
private const val TAG = "CommandRouter"
|
private const val TAG = "CommandRouter"
|
||||||
|
private val json = Json { ignoreUnknownKeys = true; encodeDefaults = true }
|
||||||
}
|
}
|
||||||
|
|
||||||
val currentGoalStatus = MutableStateFlow(GoalStatus.Idle)
|
val currentGoalStatus = MutableStateFlow(GoalStatus.Idle)
|
||||||
@@ -71,6 +77,38 @@ class CommandRouter(
|
|||||||
Log.i(TAG, "Goal failed: ${msg.message}")
|
Log.i(TAG, "Goal failed: ${msg.message}")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
"workflow_created" -> {
|
||||||
|
val wfJson = msg.workflowJson ?: return
|
||||||
|
try {
|
||||||
|
val wf = json.decodeFromString<Workflow>(wfJson)
|
||||||
|
onWorkflowCreated?.invoke(wf)
|
||||||
|
Log.i(TAG, "Workflow created: ${wf.name}")
|
||||||
|
} catch (e: Exception) {
|
||||||
|
Log.e(TAG, "Failed to parse workflow_created: ${e.message}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"workflow_synced" -> {
|
||||||
|
val wfsJson = msg.workflowsJson ?: return
|
||||||
|
try {
|
||||||
|
val wfs = json.decodeFromString<List<Workflow>>(wfsJson)
|
||||||
|
onWorkflowSync?.invoke(wfs)
|
||||||
|
Log.i(TAG, "Workflows synced: ${wfs.size} workflows")
|
||||||
|
} catch (e: Exception) {
|
||||||
|
Log.e(TAG, "Failed to parse workflow_synced: ${e.message}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"workflow_deleted" -> {
|
||||||
|
val id = msg.workflowId ?: return
|
||||||
|
onWorkflowDeleted?.invoke(id)
|
||||||
|
Log.i(TAG, "Workflow deleted: $id")
|
||||||
|
}
|
||||||
|
"workflow_goal" -> {
|
||||||
|
val goal = msg.goal ?: return
|
||||||
|
Log.i(TAG, "Workflow-triggered goal: $goal")
|
||||||
|
// Submit as a regular goal via the WebSocket
|
||||||
|
webSocket.sendTyped(com.thisux.droidclaw.model.GoalMessage(text = goal))
|
||||||
|
}
|
||||||
|
|
||||||
else -> Log.w(TAG, "Unknown message type: ${msg.type}")
|
else -> Log.w(TAG, "Unknown message type: ${msg.type}")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,6 +29,11 @@ import android.content.pm.PackageManager
|
|||||||
import android.net.Uri
|
import android.net.Uri
|
||||||
import android.provider.Settings
|
import android.provider.Settings
|
||||||
import com.thisux.droidclaw.model.StopGoalMessage
|
import com.thisux.droidclaw.model.StopGoalMessage
|
||||||
|
import com.thisux.droidclaw.model.WorkflowCreateMessage
|
||||||
|
import com.thisux.droidclaw.model.WorkflowDeleteMessage
|
||||||
|
import com.thisux.droidclaw.model.WorkflowSyncMessage
|
||||||
|
import com.thisux.droidclaw.model.WorkflowTriggerMessage
|
||||||
|
import com.thisux.droidclaw.model.WorkflowUpdateMessage
|
||||||
import com.thisux.droidclaw.overlay.AgentOverlay
|
import com.thisux.droidclaw.overlay.AgentOverlay
|
||||||
import kotlinx.coroutines.delay
|
import kotlinx.coroutines.delay
|
||||||
import kotlinx.coroutines.flow.MutableStateFlow
|
import kotlinx.coroutines.flow.MutableStateFlow
|
||||||
@@ -122,7 +127,12 @@ class ConnectionService : LifecycleService() {
|
|||||||
}
|
}
|
||||||
webSocket = ws
|
webSocket = ws
|
||||||
|
|
||||||
val router = CommandRouter(ws, captureManager)
|
val router = CommandRouter(
|
||||||
|
ws, captureManager,
|
||||||
|
onWorkflowSync = { workflows -> app.workflowStore.replaceAll(workflows) },
|
||||||
|
onWorkflowCreated = { workflow -> app.workflowStore.save(workflow) },
|
||||||
|
onWorkflowDeleted = { id -> app.workflowStore.delete(id) }
|
||||||
|
)
|
||||||
commandRouter = router
|
commandRouter = router
|
||||||
|
|
||||||
launch {
|
launch {
|
||||||
@@ -144,6 +154,8 @@ class ConnectionService : LifecycleService() {
|
|||||||
val apps = getInstalledApps()
|
val apps = getInstalledApps()
|
||||||
webSocket?.sendTyped(AppsMessage(apps = apps))
|
webSocket?.sendTyped(AppsMessage(apps = apps))
|
||||||
Log.i(TAG, "Sent ${apps.size} installed apps to server")
|
Log.i(TAG, "Sent ${apps.size} installed apps to server")
|
||||||
|
// Sync workflows from server
|
||||||
|
webSocket?.sendTyped(WorkflowSyncMessage())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -181,6 +193,26 @@ class ConnectionService : LifecycleService() {
|
|||||||
webSocket?.sendTyped(StopGoalMessage())
|
webSocket?.sendTyped(StopGoalMessage())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fun sendWorkflowCreate(description: String) {
|
||||||
|
webSocket?.sendTyped(WorkflowCreateMessage(description = description))
|
||||||
|
}
|
||||||
|
|
||||||
|
fun sendWorkflowUpdate(workflowId: String, enabled: Boolean?) {
|
||||||
|
webSocket?.sendTyped(WorkflowUpdateMessage(workflowId = workflowId, enabled = enabled))
|
||||||
|
}
|
||||||
|
|
||||||
|
fun sendWorkflowDelete(workflowId: String) {
|
||||||
|
webSocket?.sendTyped(WorkflowDeleteMessage(workflowId = workflowId))
|
||||||
|
}
|
||||||
|
|
||||||
|
fun sendWorkflowSync() {
|
||||||
|
webSocket?.sendTyped(WorkflowSyncMessage())
|
||||||
|
}
|
||||||
|
|
||||||
|
fun sendWorkflowTrigger(msg: WorkflowTriggerMessage) {
|
||||||
|
webSocket?.sendTyped(msg)
|
||||||
|
}
|
||||||
|
|
||||||
private fun disconnect() {
|
private fun disconnect() {
|
||||||
overlay?.hide()
|
overlay?.hide()
|
||||||
webSocket?.disconnect()
|
webSocket?.disconnect()
|
||||||
|
|||||||
@@ -0,0 +1,48 @@
|
|||||||
|
package com.thisux.droidclaw.data
|
||||||
|
|
||||||
|
import android.content.Context
|
||||||
|
import androidx.datastore.preferences.core.edit
|
||||||
|
import androidx.datastore.preferences.core.stringPreferencesKey
|
||||||
|
import com.thisux.droidclaw.model.Workflow
|
||||||
|
import kotlinx.coroutines.flow.Flow
|
||||||
|
import kotlinx.coroutines.flow.map
|
||||||
|
import kotlinx.serialization.encodeToString
|
||||||
|
import kotlinx.serialization.json.Json
|
||||||
|
|
||||||
|
private val WORKFLOWS_KEY = stringPreferencesKey("workflows_json")
|
||||||
|
private val json = Json { ignoreUnknownKeys = true; encodeDefaults = true }
|
||||||
|
|
||||||
|
class WorkflowStore(private val context: Context) {
|
||||||
|
|
||||||
|
val workflows: Flow<List<Workflow>> = context.dataStore.data.map { prefs ->
|
||||||
|
val raw = prefs[WORKFLOWS_KEY] ?: "[]"
|
||||||
|
try { json.decodeFromString<List<Workflow>>(raw) } catch (_: Exception) { emptyList() }
|
||||||
|
}
|
||||||
|
|
||||||
|
suspend fun save(workflow: Workflow) {
|
||||||
|
context.dataStore.edit { prefs ->
|
||||||
|
val list = currentList(prefs).toMutableList()
|
||||||
|
val idx = list.indexOfFirst { it.id == workflow.id }
|
||||||
|
if (idx >= 0) list[idx] = workflow else list.add(workflow)
|
||||||
|
prefs[WORKFLOWS_KEY] = json.encodeToString(list)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
suspend fun delete(workflowId: String) {
|
||||||
|
context.dataStore.edit { prefs ->
|
||||||
|
val list = currentList(prefs).filter { it.id != workflowId }
|
||||||
|
prefs[WORKFLOWS_KEY] = json.encodeToString(list)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
suspend fun replaceAll(workflows: List<Workflow>) {
|
||||||
|
context.dataStore.edit { prefs ->
|
||||||
|
prefs[WORKFLOWS_KEY] = json.encodeToString(workflows)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun currentList(prefs: androidx.datastore.preferences.core.Preferences): List<Workflow> {
|
||||||
|
val raw = prefs[WORKFLOWS_KEY] ?: "[]"
|
||||||
|
return try { json.decodeFromString<List<Workflow>>(raw) } catch (_: Exception) { emptyList() }
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -76,6 +76,39 @@ data class StopGoalMessage(
|
|||||||
val type: String = "stop_goal"
|
val type: String = "stop_goal"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@Serializable
|
||||||
|
data class WorkflowCreateMessage(
|
||||||
|
val type: String = "workflow_create",
|
||||||
|
val description: String // natural-language workflow description
|
||||||
|
)
|
||||||
|
|
||||||
|
@Serializable
|
||||||
|
data class WorkflowUpdateMessage(
|
||||||
|
val type: String = "workflow_update",
|
||||||
|
val workflowId: String,
|
||||||
|
val enabled: Boolean? = null
|
||||||
|
)
|
||||||
|
|
||||||
|
@Serializable
|
||||||
|
data class WorkflowDeleteMessage(
|
||||||
|
val type: String = "workflow_delete",
|
||||||
|
val workflowId: String
|
||||||
|
)
|
||||||
|
|
||||||
|
@Serializable
|
||||||
|
data class WorkflowSyncMessage(
|
||||||
|
val type: String = "workflow_sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
@Serializable
|
||||||
|
data class WorkflowTriggerMessage(
|
||||||
|
val type: String = "workflow_trigger",
|
||||||
|
val workflowId: String,
|
||||||
|
val notificationApp: String? = null,
|
||||||
|
val notificationTitle: String? = null,
|
||||||
|
val notificationText: String? = null
|
||||||
|
)
|
||||||
|
|
||||||
@Serializable
|
@Serializable
|
||||||
data class ServerMessage(
|
data class ServerMessage(
|
||||||
val type: String,
|
val type: String,
|
||||||
@@ -106,5 +139,9 @@ data class ServerMessage(
|
|||||||
val intentUri: String? = null,
|
val intentUri: String? = null,
|
||||||
val intentType: String? = null,
|
val intentType: String? = null,
|
||||||
val intentExtras: Map<String, String>? = null,
|
val intentExtras: Map<String, String>? = null,
|
||||||
val setting: String? = null
|
val setting: String? = null,
|
||||||
|
// Workflow fields
|
||||||
|
val workflowId: String? = null,
|
||||||
|
val workflowJson: String? = null, // single workflow as JSON
|
||||||
|
val workflowsJson: String? = null // array of workflows as JSON (for sync)
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
package com.thisux.droidclaw.model
|
||||||
|
|
||||||
|
import kotlinx.serialization.Serializable
|
||||||
|
|
||||||
|
@Serializable
|
||||||
|
enum class TriggerType {
|
||||||
|
notification
|
||||||
|
}
|
||||||
|
|
||||||
|
@Serializable
|
||||||
|
enum class MatchMode {
|
||||||
|
contains, exact, regex
|
||||||
|
}
|
||||||
|
|
||||||
|
@Serializable
|
||||||
|
data class TriggerCondition(
|
||||||
|
val field: String, // "app_package", "title", "text"
|
||||||
|
val matchMode: MatchMode,
|
||||||
|
val value: String
|
||||||
|
)
|
||||||
|
|
||||||
|
@Serializable
|
||||||
|
data class Workflow(
|
||||||
|
val id: String,
|
||||||
|
val name: String,
|
||||||
|
val description: String, // original natural-language input
|
||||||
|
val triggerType: TriggerType = TriggerType.notification,
|
||||||
|
val conditions: List<TriggerCondition> = emptyList(),
|
||||||
|
val goalTemplate: String, // sent to agent as a goal
|
||||||
|
val enabled: Boolean = true,
|
||||||
|
val createdAt: Long = System.currentTimeMillis()
|
||||||
|
)
|
||||||
@@ -84,7 +84,13 @@ fun OverlayContent() {
|
|||||||
!isConnected -> "Offline"
|
!isConnected -> "Offline"
|
||||||
displayStatus == GoalStatus.Running -> {
|
displayStatus == GoalStatus.Running -> {
|
||||||
val last = steps.lastOrNull()
|
val last = steps.lastOrNull()
|
||||||
if (last != null) "Step ${last.step}: ${last.action}" else "Running..."
|
if (last != null) {
|
||||||
|
val label = last.reasoning.ifBlank {
|
||||||
|
// Extract just the action name from the JSON string
|
||||||
|
Regex("""action[=:]?\s*(\w+)""").find(last.action)?.groupValues?.get(1) ?: "working"
|
||||||
|
}
|
||||||
|
"${last.step}: $label"
|
||||||
|
} else "Running..."
|
||||||
}
|
}
|
||||||
displayStatus == GoalStatus.Completed -> "Done"
|
displayStatus == GoalStatus.Completed -> "Done"
|
||||||
displayStatus == GoalStatus.Failed -> "Stopped"
|
displayStatus == GoalStatus.Failed -> "Stopped"
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
package com.thisux.droidclaw.ui.screens
|
package com.thisux.droidclaw.ui.screens
|
||||||
|
|
||||||
import android.content.Intent
|
import android.content.Intent
|
||||||
|
import androidx.compose.animation.AnimatedVisibility
|
||||||
import androidx.compose.foundation.background
|
import androidx.compose.foundation.background
|
||||||
|
import androidx.compose.foundation.clickable
|
||||||
import androidx.compose.foundation.layout.Arrangement
|
import androidx.compose.foundation.layout.Arrangement
|
||||||
import androidx.compose.foundation.layout.Box
|
import androidx.compose.foundation.layout.Box
|
||||||
import androidx.compose.foundation.layout.Column
|
import androidx.compose.foundation.layout.Column
|
||||||
@@ -15,35 +17,49 @@ import androidx.compose.foundation.layout.size
|
|||||||
import androidx.compose.foundation.lazy.LazyColumn
|
import androidx.compose.foundation.lazy.LazyColumn
|
||||||
import androidx.compose.foundation.lazy.items
|
import androidx.compose.foundation.lazy.items
|
||||||
import androidx.compose.foundation.shape.CircleShape
|
import androidx.compose.foundation.shape.CircleShape
|
||||||
|
import androidx.compose.material.icons.Icons
|
||||||
|
import androidx.compose.material.icons.filled.Delete
|
||||||
|
import androidx.compose.material.icons.filled.ExpandLess
|
||||||
|
import androidx.compose.material.icons.filled.ExpandMore
|
||||||
import androidx.compose.material3.Button
|
import androidx.compose.material3.Button
|
||||||
import androidx.compose.material3.Card
|
import androidx.compose.material3.Card
|
||||||
|
import androidx.compose.material3.CardDefaults
|
||||||
|
import androidx.compose.material3.Icon
|
||||||
|
import androidx.compose.material3.IconButton
|
||||||
import androidx.compose.material3.MaterialTheme
|
import androidx.compose.material3.MaterialTheme
|
||||||
import androidx.compose.material3.OutlinedTextField
|
import androidx.compose.material3.OutlinedTextField
|
||||||
|
import androidx.compose.material3.Switch
|
||||||
import androidx.compose.material3.Text
|
import androidx.compose.material3.Text
|
||||||
import androidx.compose.runtime.Composable
|
import androidx.compose.runtime.Composable
|
||||||
import androidx.compose.runtime.collectAsState
|
import androidx.compose.runtime.collectAsState
|
||||||
import androidx.compose.runtime.getValue
|
import androidx.compose.runtime.getValue
|
||||||
import androidx.compose.runtime.mutableStateOf
|
import androidx.compose.runtime.mutableStateOf
|
||||||
import androidx.compose.runtime.remember
|
import androidx.compose.runtime.remember
|
||||||
|
import androidx.compose.runtime.saveable.rememberSaveable
|
||||||
import androidx.compose.runtime.setValue
|
import androidx.compose.runtime.setValue
|
||||||
import androidx.compose.ui.Alignment
|
import androidx.compose.ui.Alignment
|
||||||
import androidx.compose.ui.Modifier
|
import androidx.compose.ui.Modifier
|
||||||
import androidx.compose.ui.draw.clip
|
import androidx.compose.ui.draw.clip
|
||||||
import androidx.compose.ui.graphics.Color
|
import androidx.compose.ui.graphics.Color
|
||||||
import androidx.compose.ui.platform.LocalContext
|
import androidx.compose.ui.platform.LocalContext
|
||||||
|
import androidx.compose.ui.text.style.TextOverflow
|
||||||
import androidx.compose.ui.unit.dp
|
import androidx.compose.ui.unit.dp
|
||||||
|
import com.thisux.droidclaw.DroidClawApp
|
||||||
import com.thisux.droidclaw.connection.ConnectionService
|
import com.thisux.droidclaw.connection.ConnectionService
|
||||||
import com.thisux.droidclaw.model.ConnectionState
|
import com.thisux.droidclaw.model.ConnectionState
|
||||||
import com.thisux.droidclaw.model.GoalStatus
|
import com.thisux.droidclaw.model.GoalStatus
|
||||||
|
import com.thisux.droidclaw.model.Workflow
|
||||||
|
|
||||||
@Composable
|
@Composable
|
||||||
fun HomeScreen() {
|
fun HomeScreen() {
|
||||||
val context = LocalContext.current
|
val context = LocalContext.current
|
||||||
|
val app = context.applicationContext as DroidClawApp
|
||||||
val connectionState by ConnectionService.connectionState.collectAsState()
|
val connectionState by ConnectionService.connectionState.collectAsState()
|
||||||
val goalStatus by ConnectionService.currentGoalStatus.collectAsState()
|
val goalStatus by ConnectionService.currentGoalStatus.collectAsState()
|
||||||
val steps by ConnectionService.currentSteps.collectAsState()
|
val steps by ConnectionService.currentSteps.collectAsState()
|
||||||
val currentGoal by ConnectionService.currentGoal.collectAsState()
|
val currentGoal by ConnectionService.currentGoal.collectAsState()
|
||||||
val errorMessage by ConnectionService.errorMessage.collectAsState()
|
val errorMessage by ConnectionService.errorMessage.collectAsState()
|
||||||
|
val workflows by app.workflowStore.workflows.collectAsState(initial = emptyList())
|
||||||
|
|
||||||
var goalInput by remember { mutableStateOf("") }
|
var goalInput by remember { mutableStateOf("") }
|
||||||
|
|
||||||
@@ -108,7 +124,7 @@ fun HomeScreen() {
|
|||||||
|
|
||||||
Spacer(modifier = Modifier.height(16.dp))
|
Spacer(modifier = Modifier.height(16.dp))
|
||||||
|
|
||||||
// Goal Input
|
// Goal Input — same field for goals and workflows
|
||||||
Row(
|
Row(
|
||||||
modifier = Modifier.fillMaxWidth(),
|
modifier = Modifier.fillMaxWidth(),
|
||||||
horizontalArrangement = Arrangement.spacedBy(8.dp)
|
horizontalArrangement = Arrangement.spacedBy(8.dp)
|
||||||
@@ -116,7 +132,7 @@ fun HomeScreen() {
|
|||||||
OutlinedTextField(
|
OutlinedTextField(
|
||||||
value = goalInput,
|
value = goalInput,
|
||||||
onValueChange = { goalInput = it },
|
onValueChange = { goalInput = it },
|
||||||
label = { Text("Enter a goal...") },
|
label = { Text("Goal or workflow...") },
|
||||||
modifier = Modifier.weight(1f),
|
modifier = Modifier.weight(1f),
|
||||||
enabled = connectionState == ConnectionState.Connected && goalStatus != GoalStatus.Running,
|
enabled = connectionState == ConnectionState.Connected && goalStatus != GoalStatus.Running,
|
||||||
singleLine = true
|
singleLine = true
|
||||||
@@ -136,7 +152,7 @@ fun HomeScreen() {
|
|||||||
&& goalStatus != GoalStatus.Running
|
&& goalStatus != GoalStatus.Running
|
||||||
&& goalInput.isNotBlank()
|
&& goalInput.isNotBlank()
|
||||||
) {
|
) {
|
||||||
Text("Run")
|
Text("Send")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -192,5 +208,103 @@ fun HomeScreen() {
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Saved Workflows section
|
||||||
|
if (workflows.isNotEmpty()) {
|
||||||
|
Spacer(modifier = Modifier.height(8.dp))
|
||||||
|
WorkflowsSection(workflows)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
private fun WorkflowsSection(workflows: List<Workflow>) {
|
||||||
|
var expanded by rememberSaveable { mutableStateOf(false) }
|
||||||
|
|
||||||
|
Column {
|
||||||
|
Row(
|
||||||
|
modifier = Modifier
|
||||||
|
.fillMaxWidth()
|
||||||
|
.clickable { expanded = !expanded }
|
||||||
|
.padding(vertical = 4.dp),
|
||||||
|
verticalAlignment = Alignment.CenterVertically,
|
||||||
|
horizontalArrangement = Arrangement.SpaceBetween
|
||||||
|
) {
|
||||||
|
Text(
|
||||||
|
text = "Saved Workflows (${workflows.size})",
|
||||||
|
style = MaterialTheme.typography.titleSmall
|
||||||
|
)
|
||||||
|
Icon(
|
||||||
|
imageVector = if (expanded) Icons.Default.ExpandLess else Icons.Default.ExpandMore,
|
||||||
|
contentDescription = if (expanded) "Collapse" else "Expand"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
AnimatedVisibility(visible = expanded) {
|
||||||
|
Column(verticalArrangement = Arrangement.spacedBy(6.dp)) {
|
||||||
|
workflows.forEach { wf ->
|
||||||
|
WorkflowChip(wf)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
private fun WorkflowChip(workflow: Workflow) {
|
||||||
|
Card(
|
||||||
|
modifier = Modifier.fillMaxWidth(),
|
||||||
|
colors = CardDefaults.cardColors(
|
||||||
|
containerColor = if (workflow.enabled) {
|
||||||
|
MaterialTheme.colorScheme.secondaryContainer
|
||||||
|
} else {
|
||||||
|
MaterialTheme.colorScheme.surfaceVariant
|
||||||
|
}
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
Row(
|
||||||
|
modifier = Modifier
|
||||||
|
.fillMaxWidth()
|
||||||
|
.padding(horizontal = 12.dp, vertical = 8.dp),
|
||||||
|
verticalAlignment = Alignment.CenterVertically,
|
||||||
|
horizontalArrangement = Arrangement.SpaceBetween
|
||||||
|
) {
|
||||||
|
Column(modifier = Modifier.weight(1f)) {
|
||||||
|
Text(
|
||||||
|
text = workflow.name,
|
||||||
|
style = MaterialTheme.typography.bodyMedium,
|
||||||
|
maxLines = 1,
|
||||||
|
overflow = TextOverflow.Ellipsis
|
||||||
|
)
|
||||||
|
Text(
|
||||||
|
text = workflow.description,
|
||||||
|
style = MaterialTheme.typography.bodySmall,
|
||||||
|
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||||
|
maxLines = 1,
|
||||||
|
overflow = TextOverflow.Ellipsis
|
||||||
|
)
|
||||||
|
}
|
||||||
|
Row(verticalAlignment = Alignment.CenterVertically) {
|
||||||
|
Switch(
|
||||||
|
checked = workflow.enabled,
|
||||||
|
onCheckedChange = { enabled ->
|
||||||
|
ConnectionService.instance?.sendWorkflowUpdate(workflow.id, enabled)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
IconButton(
|
||||||
|
onClick = {
|
||||||
|
ConnectionService.instance?.sendWorkflowDelete(workflow.id)
|
||||||
|
},
|
||||||
|
modifier = Modifier.size(32.dp)
|
||||||
|
) {
|
||||||
|
Icon(
|
||||||
|
Icons.Default.Delete,
|
||||||
|
contentDescription = "Delete",
|
||||||
|
tint = MaterialTheme.colorScheme.error,
|
||||||
|
modifier = Modifier.size(18.dp)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,6 +49,7 @@ import com.thisux.droidclaw.DroidClawApp
|
|||||||
import com.thisux.droidclaw.accessibility.DroidClawAccessibilityService
|
import com.thisux.droidclaw.accessibility.DroidClawAccessibilityService
|
||||||
import com.thisux.droidclaw.capture.ScreenCaptureManager
|
import com.thisux.droidclaw.capture.ScreenCaptureManager
|
||||||
import com.thisux.droidclaw.util.BatteryOptimization
|
import com.thisux.droidclaw.util.BatteryOptimization
|
||||||
|
import com.thisux.droidclaw.workflow.WorkflowNotificationService
|
||||||
import kotlinx.coroutines.launch
|
import kotlinx.coroutines.launch
|
||||||
|
|
||||||
@Composable
|
@Composable
|
||||||
@@ -76,6 +77,9 @@ fun SettingsScreen() {
|
|||||||
}
|
}
|
||||||
var isBatteryExempt by remember { mutableStateOf(BatteryOptimization.isIgnoringBatteryOptimizations(context)) }
|
var isBatteryExempt by remember { mutableStateOf(BatteryOptimization.isIgnoringBatteryOptimizations(context)) }
|
||||||
var hasOverlayPermission by remember { mutableStateOf(Settings.canDrawOverlays(context)) }
|
var hasOverlayPermission by remember { mutableStateOf(Settings.canDrawOverlays(context)) }
|
||||||
|
var isNotificationListenerEnabled by remember {
|
||||||
|
mutableStateOf(WorkflowNotificationService.isEnabled(context))
|
||||||
|
}
|
||||||
|
|
||||||
val lifecycleOwner = LocalLifecycleOwner.current
|
val lifecycleOwner = LocalLifecycleOwner.current
|
||||||
DisposableEffect(lifecycleOwner) {
|
DisposableEffect(lifecycleOwner) {
|
||||||
@@ -86,6 +90,7 @@ fun SettingsScreen() {
|
|||||||
hasCaptureConsent = isCaptureAvailable || ScreenCaptureManager.hasConsent()
|
hasCaptureConsent = isCaptureAvailable || ScreenCaptureManager.hasConsent()
|
||||||
isBatteryExempt = BatteryOptimization.isIgnoringBatteryOptimizations(context)
|
isBatteryExempt = BatteryOptimization.isIgnoringBatteryOptimizations(context)
|
||||||
hasOverlayPermission = Settings.canDrawOverlays(context)
|
hasOverlayPermission = Settings.canDrawOverlays(context)
|
||||||
|
isNotificationListenerEnabled = WorkflowNotificationService.isEnabled(context)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
lifecycleOwner.lifecycle.addObserver(observer)
|
lifecycleOwner.lifecycle.addObserver(observer)
|
||||||
@@ -199,6 +204,17 @@ fun SettingsScreen() {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
ChecklistItem(
|
||||||
|
label = "Notification listener (for workflows)",
|
||||||
|
isOk = isNotificationListenerEnabled,
|
||||||
|
actionLabel = "Enable",
|
||||||
|
onAction = {
|
||||||
|
context.startActivity(
|
||||||
|
Intent("android.settings.ACTION_NOTIFICATION_LISTENER_SETTINGS")
|
||||||
|
)
|
||||||
|
}
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,115 @@
|
|||||||
|
package com.thisux.droidclaw.workflow
|
||||||
|
|
||||||
|
import android.content.ComponentName
|
||||||
|
import android.content.Context
|
||||||
|
import android.provider.Settings
|
||||||
|
import android.service.notification.NotificationListenerService
|
||||||
|
import android.service.notification.StatusBarNotification
|
||||||
|
import android.util.Log
|
||||||
|
import com.thisux.droidclaw.DroidClawApp
|
||||||
|
import com.thisux.droidclaw.connection.ConnectionService
|
||||||
|
import com.thisux.droidclaw.model.ConnectionState
|
||||||
|
import com.thisux.droidclaw.model.MatchMode
|
||||||
|
import com.thisux.droidclaw.model.TriggerCondition
|
||||||
|
import com.thisux.droidclaw.model.Workflow
|
||||||
|
import com.thisux.droidclaw.model.WorkflowTriggerMessage
|
||||||
|
import kotlinx.coroutines.CoroutineScope
|
||||||
|
import kotlinx.coroutines.Dispatchers
|
||||||
|
import kotlinx.coroutines.SupervisorJob
|
||||||
|
import kotlinx.coroutines.flow.first
|
||||||
|
import kotlinx.coroutines.launch
|
||||||
|
|
||||||
|
class WorkflowNotificationService : NotificationListenerService() {
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
private const val TAG = "WorkflowNotifSvc"
|
||||||
|
|
||||||
|
fun isEnabled(context: Context): Boolean {
|
||||||
|
val flat = Settings.Secure.getString(
|
||||||
|
context.contentResolver,
|
||||||
|
"enabled_notification_listeners"
|
||||||
|
) ?: return false
|
||||||
|
val ourComponent = ComponentName(context, WorkflowNotificationService::class.java)
|
||||||
|
return flat.contains(ourComponent.flattenToString())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||||
|
|
||||||
|
override fun onNotificationPosted(sbn: StatusBarNotification?) {
|
||||||
|
sbn ?: return
|
||||||
|
val pkg = sbn.packageName ?: return
|
||||||
|
// Ignore our own notifications
|
||||||
|
if (pkg == packageName) return
|
||||||
|
|
||||||
|
val extras = sbn.notification?.extras ?: return
|
||||||
|
val title = extras.getCharSequence("android.title")?.toString() ?: ""
|
||||||
|
val text = extras.getCharSequence("android.text")?.toString() ?: ""
|
||||||
|
|
||||||
|
Log.d(TAG, "Notification from=$pkg title=$title text=$text")
|
||||||
|
|
||||||
|
scope.launch {
|
||||||
|
try {
|
||||||
|
val app = application as DroidClawApp
|
||||||
|
val workflows = app.workflowStore.workflows.first()
|
||||||
|
val enabled = workflows.filter { it.enabled }
|
||||||
|
|
||||||
|
for (wf in enabled) {
|
||||||
|
if (matchesWorkflow(wf, pkg, title, text)) {
|
||||||
|
Log.i(TAG, "Workflow '${wf.name}' matched notification from $pkg")
|
||||||
|
triggerWorkflow(wf, pkg, title, text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (e: Exception) {
|
||||||
|
Log.e(TAG, "Failed to process notification for workflows: ${e.message}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun matchesWorkflow(
|
||||||
|
wf: Workflow,
|
||||||
|
pkg: String,
|
||||||
|
title: String,
|
||||||
|
text: String
|
||||||
|
): Boolean {
|
||||||
|
if (wf.conditions.isEmpty()) return false
|
||||||
|
return wf.conditions.all { cond -> matchesCondition(cond, pkg, title, text) }
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun matchesCondition(
|
||||||
|
cond: TriggerCondition,
|
||||||
|
pkg: String,
|
||||||
|
title: String,
|
||||||
|
text: String
|
||||||
|
): Boolean {
|
||||||
|
val actual = when (cond.field) {
|
||||||
|
"app_package" -> pkg
|
||||||
|
"title" -> title
|
||||||
|
"text" -> text
|
||||||
|
else -> return false
|
||||||
|
}
|
||||||
|
return when (cond.matchMode) {
|
||||||
|
MatchMode.contains -> actual.contains(cond.value, ignoreCase = true)
|
||||||
|
MatchMode.exact -> actual.equals(cond.value, ignoreCase = true)
|
||||||
|
MatchMode.regex -> try {
|
||||||
|
Regex(cond.value, RegexOption.IGNORE_CASE).containsMatchIn(actual)
|
||||||
|
} catch (_: Exception) { false }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun triggerWorkflow(wf: Workflow, pkg: String, title: String, text: String) {
|
||||||
|
val svc = ConnectionService.instance ?: return
|
||||||
|
if (ConnectionService.connectionState.value != ConnectionState.Connected) {
|
||||||
|
Log.w(TAG, "Cannot trigger workflow '${wf.name}': not connected")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
svc.sendWorkflowTrigger(
|
||||||
|
WorkflowTriggerMessage(
|
||||||
|
workflowId = wf.id,
|
||||||
|
notificationApp = pkg,
|
||||||
|
notificationTitle = title,
|
||||||
|
notificationText = text
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
4
android/app/src/main/res/xml/network_security_config.xml
Normal file
4
android/app/src/main/res/xml/network_security_config.xml
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<network-security-config>
|
||||||
|
<base-config cleartextTrafficPermitted="true" />
|
||||||
|
</network-security-config>
|
||||||
@@ -8,7 +8,12 @@ export type DeviceMessage =
|
|||||||
| { type: "pong" }
|
| { type: "pong" }
|
||||||
| { type: "heartbeat"; batteryLevel: number; isCharging: boolean }
|
| { type: "heartbeat"; batteryLevel: number; isCharging: boolean }
|
||||||
| { type: "apps"; apps: InstalledApp[] }
|
| { type: "apps"; apps: InstalledApp[] }
|
||||||
| { type: "stop_goal" };
|
| { type: "stop_goal" }
|
||||||
|
| { type: "workflow_create"; description: string }
|
||||||
|
| { type: "workflow_update"; workflowId: string; enabled?: boolean }
|
||||||
|
| { type: "workflow_delete"; workflowId: string }
|
||||||
|
| { type: "workflow_sync" }
|
||||||
|
| { type: "workflow_trigger"; workflowId: string; notificationApp?: string; notificationTitle?: string; notificationText?: string };
|
||||||
|
|
||||||
export type ServerToDeviceMessage =
|
export type ServerToDeviceMessage =
|
||||||
| { type: "auth_ok"; deviceId: string }
|
| { type: "auth_ok"; deviceId: string }
|
||||||
|
|||||||
54
server/src/agent/input-classifier.ts
Normal file
54
server/src/agent/input-classifier.ts
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
/**
|
||||||
|
* Classifies user input as either an immediate goal or a workflow (automation rule).
|
||||||
|
*
|
||||||
|
* Uses the user's LLM to determine intent. Workflows describe recurring
|
||||||
|
* automations ("when X happens, do Y"), goals are one-time tasks ("open WhatsApp").
|
||||||
|
*/
|
||||||
|
|
||||||
|
import type { LLMConfig } from "./llm.js";
|
||||||
|
import { getLlmProvider, parseJsonResponse } from "./llm.js";
|
||||||
|
|
||||||
|
export type InputType = "goal" | "workflow";
|
||||||
|
|
||||||
|
export interface ClassificationResult {
|
||||||
|
type: InputType;
|
||||||
|
}
|
||||||
|
|
||||||
|
const CLASSIFIER_PROMPT = `You classify user input for an Android automation agent.
|
||||||
|
|
||||||
|
Decide if the input is:
|
||||||
|
- "goal": A one-time task to execute right now (e.g. "open WhatsApp", "search for pizza", "take a screenshot", "reply to John with hello")
|
||||||
|
- "workflow": An automation rule that should be saved and triggered later when a condition is met (e.g. "when I get a notification from WhatsApp saying where are you, reply with Bangalore", "whenever someone messages me on Telegram, auto-reply with I'm busy", "reply to all notifications that have a reply button")
|
||||||
|
|
||||||
|
Key signals for "workflow":
|
||||||
|
- Uses words like "when", "whenever", "if", "every time", "automatically", "always"
|
||||||
|
- Describes a trigger condition + a response action
|
||||||
|
- Refers to future/recurring events
|
||||||
|
|
||||||
|
Key signals for "goal":
|
||||||
|
- Describes a single task to do now
|
||||||
|
- Imperative commands ("open", "send", "search", "go to")
|
||||||
|
- No conditional/temporal trigger
|
||||||
|
|
||||||
|
Respond with ONLY: {"type": "goal"} or {"type": "workflow"}`;
|
||||||
|
|
||||||
|
export async function classifyInput(
|
||||||
|
text: string,
|
||||||
|
llmConfig: LLMConfig
|
||||||
|
): Promise<ClassificationResult> {
|
||||||
|
const provider = getLlmProvider(llmConfig);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const raw = await provider.getAction(CLASSIFIER_PROMPT, text);
|
||||||
|
const parsed = parseJsonResponse(raw);
|
||||||
|
|
||||||
|
if (parsed?.type === "workflow") {
|
||||||
|
return { type: "workflow" };
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.error(`[Classifier] Failed to classify input, defaulting to goal:`, err);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default to goal — safer to execute once than to accidentally create a rule
|
||||||
|
return { type: "goal" };
|
||||||
|
}
|
||||||
75
server/src/agent/workflow-parser.ts
Normal file
75
server/src/agent/workflow-parser.ts
Normal file
@@ -0,0 +1,75 @@
|
|||||||
|
/**
|
||||||
|
* Parses a natural-language workflow description into structured
|
||||||
|
* trigger conditions and a goal template using the user's LLM.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import type { LLMConfig } from "./llm.js";
|
||||||
|
import { getLlmProvider, parseJsonResponse } from "./llm.js";
|
||||||
|
|
||||||
|
export interface ParsedWorkflow {
|
||||||
|
name: string;
|
||||||
|
triggerType: "notification";
|
||||||
|
conditions: Array<{
|
||||||
|
field: "app_package" | "title" | "text";
|
||||||
|
matchMode: "contains" | "exact" | "regex";
|
||||||
|
value: string;
|
||||||
|
}>;
|
||||||
|
goalTemplate: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const PARSER_PROMPT = `You are a workflow parser for an Android automation agent.
|
||||||
|
|
||||||
|
The user describes an automation rule in plain English. Parse it into a structured workflow.
|
||||||
|
|
||||||
|
A workflow has:
|
||||||
|
1. **name**: A short human-readable name (3-6 words).
|
||||||
|
2. **triggerType**: Always "notification" for now.
|
||||||
|
3. **conditions**: An array of matching rules for incoming notifications. Each condition has:
|
||||||
|
- "field": one of "app_package", "title", or "text"
|
||||||
|
- "matchMode": one of "contains", "exact", or "regex"
|
||||||
|
- "value": the string or regex to match
|
||||||
|
4. **goalTemplate**: The goal string to send to the agent when triggered. Use {{title}}, {{text}}, {{app}} as placeholders that get filled from the notification.
|
||||||
|
|
||||||
|
Example input: "When I get a WhatsApp message saying 'where are you', reply with 'Bangalore'"
|
||||||
|
Example output:
|
||||||
|
{
|
||||||
|
"name": "Auto-reply where are you",
|
||||||
|
"triggerType": "notification",
|
||||||
|
"conditions": [
|
||||||
|
{"field": "app_package", "matchMode": "contains", "value": "whatsapp"},
|
||||||
|
{"field": "text", "matchMode": "contains", "value": "where are you"}
|
||||||
|
],
|
||||||
|
"goalTemplate": "Open the WhatsApp notification from {{title}} and reply with 'Bangalore'"
|
||||||
|
}
|
||||||
|
|
||||||
|
Example input: "Reply to all notifications that have a reply button with 'I am busy'"
|
||||||
|
Example output:
|
||||||
|
{
|
||||||
|
"name": "Auto-reply I am busy",
|
||||||
|
"triggerType": "notification",
|
||||||
|
"conditions": [],
|
||||||
|
"goalTemplate": "Open the notification '{{title}}' from {{app}} and reply with 'I am busy'"
|
||||||
|
}
|
||||||
|
|
||||||
|
Respond with ONLY a valid JSON object. No explanation.`;
|
||||||
|
|
||||||
|
export async function parseWorkflowDescription(
|
||||||
|
description: string,
|
||||||
|
llmConfig: LLMConfig
|
||||||
|
): Promise<ParsedWorkflow> {
|
||||||
|
const provider = getLlmProvider(llmConfig);
|
||||||
|
|
||||||
|
const raw = await provider.getAction(PARSER_PROMPT, description);
|
||||||
|
const parsed = parseJsonResponse(raw);
|
||||||
|
|
||||||
|
if (!parsed || !parsed.name || !parsed.goalTemplate) {
|
||||||
|
throw new Error("Failed to parse workflow description into structured format");
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
name: parsed.name as string,
|
||||||
|
triggerType: "notification",
|
||||||
|
conditions: (parsed.conditions as ParsedWorkflow["conditions"]) ?? [],
|
||||||
|
goalTemplate: parsed.goalTemplate as string,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -128,6 +128,24 @@ export const agentSession = pgTable("agent_session", {
|
|||||||
completedAt: timestamp("completed_at"),
|
completedAt: timestamp("completed_at"),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
export const workflow = pgTable("workflow", {
|
||||||
|
id: text("id").primaryKey(),
|
||||||
|
userId: text("user_id")
|
||||||
|
.notNull()
|
||||||
|
.references(() => user.id, { onDelete: "cascade" }),
|
||||||
|
name: text("name").notNull(),
|
||||||
|
description: text("description").notNull(),
|
||||||
|
triggerType: text("trigger_type").notNull().default("notification"),
|
||||||
|
conditions: jsonb("conditions").notNull().default("[]"),
|
||||||
|
goalTemplate: text("goal_template").notNull(),
|
||||||
|
enabled: boolean("enabled").default(true).notNull(),
|
||||||
|
createdAt: timestamp("created_at").defaultNow().notNull(),
|
||||||
|
updatedAt: timestamp("updated_at")
|
||||||
|
.defaultNow()
|
||||||
|
.$onUpdate(() => new Date())
|
||||||
|
.notNull(),
|
||||||
|
});
|
||||||
|
|
||||||
export const agentStep = pgTable("agent_step", {
|
export const agentStep = pgTable("agent_step", {
|
||||||
id: text("id").primaryKey(),
|
id: text("id").primaryKey(),
|
||||||
sessionId: text("session_id")
|
sessionId: text("session_id")
|
||||||
|
|||||||
@@ -6,6 +6,14 @@ import { apikey, llmConfig, device } from "../schema.js";
|
|||||||
import { sessions, type WebSocketData } from "./sessions.js";
|
import { sessions, type WebSocketData } from "./sessions.js";
|
||||||
import { runPipeline } from "../agent/pipeline.js";
|
import { runPipeline } from "../agent/pipeline.js";
|
||||||
import type { LLMConfig } from "../agent/llm.js";
|
import type { LLMConfig } from "../agent/llm.js";
|
||||||
|
import {
|
||||||
|
handleWorkflowCreate,
|
||||||
|
handleWorkflowUpdate,
|
||||||
|
handleWorkflowDelete,
|
||||||
|
handleWorkflowSync,
|
||||||
|
handleWorkflowTrigger,
|
||||||
|
} from "./workflow-handlers.js";
|
||||||
|
import { classifyInput } from "../agent/input-classifier.js";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Hash an API key the same way better-auth does:
|
* Hash an API key the same way better-auth does:
|
||||||
@@ -251,6 +259,20 @@ export async function handleDeviceMessage(
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Classify: is this an immediate goal or a workflow?
|
||||||
|
try {
|
||||||
|
const classification = await classifyInput(goal, userLlmConfig);
|
||||||
|
if (classification.type === "workflow") {
|
||||||
|
console.log(`[Classifier] Input classified as workflow: ${goal}`);
|
||||||
|
handleWorkflowCreate(ws, goal).catch((err) =>
|
||||||
|
console.error(`[Workflow] Auto-create error:`, err)
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.warn(`[Classifier] Classification failed, treating as goal:`, err);
|
||||||
|
}
|
||||||
|
|
||||||
console.log(`[Pipeline] Starting goal for device ${deviceId}: ${goal}`);
|
console.log(`[Pipeline] Starting goal for device ${deviceId}: ${goal}`);
|
||||||
const abortController = new AbortController();
|
const abortController = new AbortController();
|
||||||
activeSessions.set(deviceId, { goal, abort: abortController });
|
activeSessions.set(deviceId, { goal, abort: abortController });
|
||||||
@@ -361,6 +383,59 @@ export async function handleDeviceMessage(
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case "workflow_create": {
|
||||||
|
const description = (msg as unknown as { description: string }).description;
|
||||||
|
if (description) {
|
||||||
|
handleWorkflowCreate(ws, description).catch((err) =>
|
||||||
|
console.error(`[Workflow] Create error:`, err)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
case "workflow_update": {
|
||||||
|
const { workflowId, enabled } = msg as unknown as { workflowId: string; enabled?: boolean };
|
||||||
|
if (workflowId) {
|
||||||
|
handleWorkflowUpdate(ws, workflowId, enabled).catch((err) =>
|
||||||
|
console.error(`[Workflow] Update error:`, err)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
case "workflow_delete": {
|
||||||
|
const { workflowId } = msg as unknown as { workflowId: string };
|
||||||
|
if (workflowId) {
|
||||||
|
handleWorkflowDelete(ws, workflowId).catch((err) =>
|
||||||
|
console.error(`[Workflow] Delete error:`, err)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
case "workflow_sync": {
|
||||||
|
handleWorkflowSync(ws).catch((err) =>
|
||||||
|
console.error(`[Workflow] Sync error:`, err)
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
case "workflow_trigger": {
|
||||||
|
const { workflowId, notificationApp, notificationTitle, notificationText } =
|
||||||
|
msg as unknown as {
|
||||||
|
workflowId: string;
|
||||||
|
notificationApp?: string;
|
||||||
|
notificationTitle?: string;
|
||||||
|
notificationText?: string;
|
||||||
|
};
|
||||||
|
if (workflowId) {
|
||||||
|
handleWorkflowTrigger(ws, workflowId, notificationApp, notificationTitle, notificationText).catch(
|
||||||
|
(err) => console.error(`[Workflow] Trigger error:`, err)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
default: {
|
default: {
|
||||||
console.warn(
|
console.warn(
|
||||||
`Unknown message type from device ${ws.data.deviceId}:`,
|
`Unknown message type from device ${ws.data.deviceId}:`,
|
||||||
|
|||||||
229
server/src/ws/workflow-handlers.ts
Normal file
229
server/src/ws/workflow-handlers.ts
Normal file
@@ -0,0 +1,229 @@
|
|||||||
|
/**
|
||||||
|
* Server-side handlers for workflow CRUD and trigger messages
|
||||||
|
* from the Android device WebSocket.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import type { ServerWebSocket } from "bun";
|
||||||
|
import { eq, and } from "drizzle-orm";
|
||||||
|
import { db } from "../db.js";
|
||||||
|
import { workflow, llmConfig } from "../schema.js";
|
||||||
|
import { parseWorkflowDescription } from "../agent/workflow-parser.js";
|
||||||
|
import type { LLMConfig } from "../agent/llm.js";
|
||||||
|
import type { WebSocketData } from "./sessions.js";
|
||||||
|
|
||||||
|
function sendToDevice(ws: ServerWebSocket<WebSocketData>, msg: Record<string, unknown>) {
|
||||||
|
try {
|
||||||
|
ws.send(JSON.stringify(msg));
|
||||||
|
} catch {
|
||||||
|
// device disconnected
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getUserLlmConfig(userId: string): Promise<LLMConfig | null> {
|
||||||
|
const configs = await db
|
||||||
|
.select()
|
||||||
|
.from(llmConfig)
|
||||||
|
.where(eq(llmConfig.userId, userId))
|
||||||
|
.limit(1);
|
||||||
|
|
||||||
|
if (configs.length === 0) return null;
|
||||||
|
|
||||||
|
const cfg = configs[0];
|
||||||
|
return {
|
||||||
|
provider: cfg.provider,
|
||||||
|
apiKey: cfg.apiKey,
|
||||||
|
model: cfg.model ?? undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function workflowToJson(wf: typeof workflow.$inferSelect): string {
|
||||||
|
return JSON.stringify({
|
||||||
|
id: wf.id,
|
||||||
|
name: wf.name,
|
||||||
|
description: wf.description,
|
||||||
|
triggerType: wf.triggerType,
|
||||||
|
conditions: wf.conditions,
|
||||||
|
goalTemplate: wf.goalTemplate,
|
||||||
|
enabled: wf.enabled,
|
||||||
|
createdAt: new Date(wf.createdAt).getTime(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function handleWorkflowCreate(
|
||||||
|
ws: ServerWebSocket<WebSocketData>,
|
||||||
|
description: string
|
||||||
|
): Promise<void> {
|
||||||
|
const userId = ws.data.userId!;
|
||||||
|
|
||||||
|
const userLlm = await getUserLlmConfig(userId);
|
||||||
|
if (!userLlm) {
|
||||||
|
sendToDevice(ws, {
|
||||||
|
type: "error",
|
||||||
|
message: "No LLM provider configured. Set it up in the web dashboard.",
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const parsed = await parseWorkflowDescription(description, userLlm);
|
||||||
|
|
||||||
|
// Validate regexes before persisting
|
||||||
|
for (const cond of parsed.conditions) {
|
||||||
|
if (cond.matchMode === "regex") {
|
||||||
|
try {
|
||||||
|
new RegExp(cond.value, "i");
|
||||||
|
} catch {
|
||||||
|
throw new Error(`Invalid regex in condition: ${cond.value}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const id = crypto.randomUUID();
|
||||||
|
const now = new Date();
|
||||||
|
|
||||||
|
await db.insert(workflow).values({
|
||||||
|
id,
|
||||||
|
userId,
|
||||||
|
name: parsed.name,
|
||||||
|
description,
|
||||||
|
triggerType: parsed.triggerType,
|
||||||
|
conditions: parsed.conditions,
|
||||||
|
goalTemplate: parsed.goalTemplate,
|
||||||
|
enabled: true,
|
||||||
|
createdAt: now,
|
||||||
|
updatedAt: now,
|
||||||
|
});
|
||||||
|
|
||||||
|
const inserted = await db
|
||||||
|
.select()
|
||||||
|
.from(workflow)
|
||||||
|
.where(eq(workflow.id, id))
|
||||||
|
.limit(1);
|
||||||
|
|
||||||
|
if (inserted.length > 0) {
|
||||||
|
sendToDevice(ws, {
|
||||||
|
type: "workflow_created",
|
||||||
|
workflowId: id,
|
||||||
|
workflowJson: workflowToJson(inserted[0]),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`[Workflow] Created '${parsed.name}' for user ${userId}`);
|
||||||
|
} catch (err) {
|
||||||
|
console.error(`[Workflow] Failed to create workflow:`, err);
|
||||||
|
sendToDevice(ws, {
|
||||||
|
type: "error",
|
||||||
|
message: `Failed to parse workflow: ${err}`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function handleWorkflowUpdate(
|
||||||
|
ws: ServerWebSocket<WebSocketData>,
|
||||||
|
workflowId: string,
|
||||||
|
enabled?: boolean
|
||||||
|
): Promise<void> {
|
||||||
|
const userId = ws.data.userId!;
|
||||||
|
|
||||||
|
const updates: Record<string, unknown> = {};
|
||||||
|
if (enabled !== undefined) updates.enabled = enabled;
|
||||||
|
|
||||||
|
await db
|
||||||
|
.update(workflow)
|
||||||
|
.set(updates)
|
||||||
|
.where(and(eq(workflow.id, workflowId), eq(workflow.userId, userId)));
|
||||||
|
|
||||||
|
console.log(`[Workflow] Updated ${workflowId}: enabled=${enabled}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function handleWorkflowDelete(
|
||||||
|
ws: ServerWebSocket<WebSocketData>,
|
||||||
|
workflowId: string
|
||||||
|
): Promise<void> {
|
||||||
|
const userId = ws.data.userId!;
|
||||||
|
|
||||||
|
await db
|
||||||
|
.delete(workflow)
|
||||||
|
.where(and(eq(workflow.id, workflowId), eq(workflow.userId, userId)));
|
||||||
|
|
||||||
|
sendToDevice(ws, {
|
||||||
|
type: "workflow_deleted",
|
||||||
|
workflowId,
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(`[Workflow] Deleted ${workflowId}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function handleWorkflowSync(
|
||||||
|
ws: ServerWebSocket<WebSocketData>
|
||||||
|
): Promise<void> {
|
||||||
|
const userId = ws.data.userId!;
|
||||||
|
|
||||||
|
const workflows = await db
|
||||||
|
.select()
|
||||||
|
.from(workflow)
|
||||||
|
.where(eq(workflow.userId, userId));
|
||||||
|
|
||||||
|
const workflowsJson = JSON.stringify(
|
||||||
|
workflows.map((wf) => ({
|
||||||
|
id: wf.id,
|
||||||
|
name: wf.name,
|
||||||
|
description: wf.description,
|
||||||
|
triggerType: wf.triggerType,
|
||||||
|
conditions: wf.conditions,
|
||||||
|
goalTemplate: wf.goalTemplate,
|
||||||
|
enabled: wf.enabled,
|
||||||
|
createdAt: new Date(wf.createdAt).getTime(),
|
||||||
|
}))
|
||||||
|
);
|
||||||
|
|
||||||
|
sendToDevice(ws, {
|
||||||
|
type: "workflow_synced",
|
||||||
|
workflowsJson,
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(`[Workflow] Synced ${workflows.length} workflows for user ${userId}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function handleWorkflowTrigger(
|
||||||
|
ws: ServerWebSocket<WebSocketData>,
|
||||||
|
workflowId: string,
|
||||||
|
notificationApp?: string,
|
||||||
|
notificationTitle?: string,
|
||||||
|
notificationText?: string
|
||||||
|
): Promise<void> {
|
||||||
|
const userId = ws.data.userId!;
|
||||||
|
|
||||||
|
const workflows = await db
|
||||||
|
.select()
|
||||||
|
.from(workflow)
|
||||||
|
.where(and(eq(workflow.id, workflowId), eq(workflow.userId, userId)))
|
||||||
|
.limit(1);
|
||||||
|
|
||||||
|
if (workflows.length === 0) {
|
||||||
|
console.warn(`[Workflow] Trigger for unknown workflow ${workflowId}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const wf = workflows[0];
|
||||||
|
if (!wf.enabled) return;
|
||||||
|
|
||||||
|
// Expand goal template placeholders
|
||||||
|
let goal = wf.goalTemplate;
|
||||||
|
goal = goal.replace(/\{\{app\}\}/g, notificationApp ?? "unknown app");
|
||||||
|
goal = goal.replace(/\{\{title\}\}/g, notificationTitle ?? "");
|
||||||
|
goal = goal.replace(/\{\{text\}\}/g, notificationText ?? "");
|
||||||
|
|
||||||
|
console.log(`[Workflow] Triggering '${wf.name}' with goal: ${goal}`);
|
||||||
|
|
||||||
|
// Send as a goal — reuse existing goal handling by injecting a goal message
|
||||||
|
sendToDevice(ws, { type: "ping" }); // keep-alive before goal injection
|
||||||
|
|
||||||
|
// The device will receive this as a workflow-triggered goal
|
||||||
|
// We send the goal text back to the device to be submitted as a regular goal
|
||||||
|
sendToDevice(ws, {
|
||||||
|
type: "workflow_goal",
|
||||||
|
workflowId: wf.id,
|
||||||
|
goal,
|
||||||
|
});
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user