From e300f04e139fda7a4a986491e744f60654413783 Mon Sep 17 00:00:00 2001 From: Sanju Sivalingam Date: Tue, 17 Feb 2026 22:50:18 +0530 Subject: [PATCH] feat: installed apps, stop goal, auth fixes, remote commands - Android: fetch installed apps via PackageManager, send to server on connect - Android: add QUERY_ALL_PACKAGES permission for full app visibility - Android: fix duplicate Intent import, increase accessibility retry window - Android: default server URL to ws:// instead of wss:// - Server: store installed apps in device metadata JSONB - Server: inject installed apps context into LLM prompt - Server: preprocessor resolves app names from device's actual installed apps - Server: add POST /goals/stop endpoint with AbortController cancellation - Server: rewrite session middleware to direct DB token lookup - Server: goals route fetches user's saved LLM config from DB - Web: show installed apps in device detail Overview tab with search - Web: add Stop button for running goals - Web: replace API routes with remote commands (submitGoal, stopGoal) - Web: add error display for goal submission failures - Shared: add InstalledApp type and apps message to protocol --- android/app/src/main/AndroidManifest.xml | 2 + .../DroidClawAccessibilityService.kt | 13 ++- .../droidclaw/connection/ConnectionService.kt | 20 ++++ .../thisux/droidclaw/data/SettingsStore.kt | 2 +- .../com/thisux/droidclaw/model/Protocol.kt | 12 +++ .../droidclaw/ui/screens/SettingsScreen.kt | 2 +- packages/shared/src/protocol.ts | 5 +- packages/shared/src/types.ts | 5 + server/src/agent/loop.ts | 34 ++++++- server/src/agent/preprocessor.ts | 68 ++++++++++++-- server/src/middleware/auth.ts | 47 ++++++++-- server/src/routes/goals.ts | 84 ++++++++++++++--- server/src/ws/device.ts | 22 ++++- web/src/hooks.server.ts | 19 ++-- web/src/lib/api/devices.remote.ts | 40 +++++++- web/src/routes/api/goals/+server.ts | 29 ------ .../dashboard/devices/[deviceId]/+page.svelte | 94 ++++++++++++++++--- 17 files changed, 410 insertions(+), 88 deletions(-) delete mode 100644 web/src/routes/api/goals/+server.ts diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index d04afbe..8111fbf 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -9,6 +9,8 @@ + { - val delays = longArrayOf(50, 100, 200) + // Retry with increasing delays — apps like Contacts on Vivo + // can take 500ms+ to render after a cold launch + val delays = longArrayOf(50, 100, 200, 300, 500) for (delayMs in delays) { val root = rootInActiveWindow if (root != null) { try { val elements = ScreenTreeBuilder.capture(root) + // If we got a root but zero elements, the app may still be loading. + // Retry unless this is the last attempt. + if (elements.isEmpty() && delayMs < delays.last()) { + root.recycle() + runBlocking { delay(delayMs) } + continue + } lastScreenTree.value = elements return elements } finally { @@ -55,7 +64,7 @@ class DroidClawAccessibilityService : AccessibilityService() { } runBlocking { delay(delayMs) } } - Log.w(TAG, "rootInActiveWindow null after retries") + Log.w(TAG, "rootInActiveWindow null or empty after retries") return emptyList() } diff --git a/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt b/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt index 4209708..69ba273 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/connection/ConnectionService.kt @@ -22,7 +22,10 @@ import com.thisux.droidclaw.model.GoalMessage import com.thisux.droidclaw.model.GoalStatus import com.thisux.droidclaw.model.AgentStep import com.thisux.droidclaw.model.HeartbeatMessage +import com.thisux.droidclaw.model.AppsMessage +import com.thisux.droidclaw.model.InstalledAppInfo import com.thisux.droidclaw.util.DeviceInfoHelper +import android.content.pm.PackageManager import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.first @@ -125,6 +128,12 @@ class ConnectionService : LifecycleService() { ConnectionState.Disconnected -> "Disconnected" } ) + // Send installed apps list once connected + if (state == ConnectionState.Connected) { + val apps = getInstalledApps() + webSocket?.sendTyped(AppsMessage(apps = apps)) + Log.i(TAG, "Sent ${apps.size} installed apps to server") + } } } launch { ws.errorMessage.collect { errorMessage.value = it } } @@ -179,6 +188,17 @@ class ConnectionService : LifecycleService() { return null } + private fun getInstalledApps(): List { + val pm = packageManager + val intent = Intent(Intent.ACTION_MAIN).addCategory(Intent.CATEGORY_LAUNCHER) + val activities = pm.queryIntentActivities(intent, PackageManager.MATCH_ALL) + return activities.mapNotNull { resolveInfo -> + val pkg = resolveInfo.activityInfo.packageName + val label = resolveInfo.loadLabel(pm).toString() + InstalledAppInfo(packageName = pkg, label = label) + }.distinctBy { it.packageName }.sortedBy { it.label.lowercase() } + } + private fun createNotificationChannel() { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { val channel = NotificationChannel( diff --git a/android/app/src/main/java/com/thisux/droidclaw/data/SettingsStore.kt b/android/app/src/main/java/com/thisux/droidclaw/data/SettingsStore.kt index f9da10b..8bbed4a 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/data/SettingsStore.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/data/SettingsStore.kt @@ -26,7 +26,7 @@ class SettingsStore(private val context: Context) { } val serverUrl: Flow = context.dataStore.data.map { prefs -> - prefs[SettingsKeys.SERVER_URL] ?: "wss://localhost:8080" + prefs[SettingsKeys.SERVER_URL] ?: "ws://localhost:8080" } val deviceName: Flow = context.dataStore.data.map { prefs -> diff --git a/android/app/src/main/java/com/thisux/droidclaw/model/Protocol.kt b/android/app/src/main/java/com/thisux/droidclaw/model/Protocol.kt index df6f5e8..25dbbf7 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/model/Protocol.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/model/Protocol.kt @@ -58,6 +58,18 @@ data class HeartbeatMessage( val isCharging: Boolean ) +@Serializable +data class InstalledAppInfo( + val packageName: String, + val label: String +) + +@Serializable +data class AppsMessage( + val type: String = "apps", + val apps: List +) + @Serializable data class ServerMessage( val type: String, diff --git a/android/app/src/main/java/com/thisux/droidclaw/ui/screens/SettingsScreen.kt b/android/app/src/main/java/com/thisux/droidclaw/ui/screens/SettingsScreen.kt index 6338905..b9edcb0 100644 --- a/android/app/src/main/java/com/thisux/droidclaw/ui/screens/SettingsScreen.kt +++ b/android/app/src/main/java/com/thisux/droidclaw/ui/screens/SettingsScreen.kt @@ -51,7 +51,7 @@ fun SettingsScreen() { val scope = rememberCoroutineScope() val apiKey by app.settingsStore.apiKey.collectAsState(initial = "") - val serverUrl by app.settingsStore.serverUrl.collectAsState(initial = "wss://localhost:8080") + val serverUrl by app.settingsStore.serverUrl.collectAsState(initial = "ws://localhost:8080") var editingApiKey by remember(apiKey) { mutableStateOf(apiKey) } var editingServerUrl by remember(serverUrl) { mutableStateOf(serverUrl) } diff --git a/packages/shared/src/protocol.ts b/packages/shared/src/protocol.ts index 9a7998f..caacd57 100644 --- a/packages/shared/src/protocol.ts +++ b/packages/shared/src/protocol.ts @@ -1,4 +1,4 @@ -import type { UIElement, DeviceInfo } from "./types.js"; +import type { UIElement, DeviceInfo, InstalledApp } from "./types.js"; export type DeviceMessage = | { type: "auth"; apiKey: string; deviceInfo?: DeviceInfo } @@ -6,7 +6,8 @@ export type DeviceMessage = | { type: "result"; requestId: string; success: boolean; error?: string; data?: string } | { type: "goal"; text: string } | { type: "pong" } - | { type: "heartbeat"; batteryLevel: number; isCharging: boolean }; + | { type: "heartbeat"; batteryLevel: number; isCharging: boolean } + | { type: "apps"; apps: InstalledApp[] }; export type ServerToDeviceMessage = | { type: "auth_ok"; deviceId: string } diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index fb994a0..b29ee53 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -61,6 +61,11 @@ export interface DeviceInfo { isCharging: boolean; } +export interface InstalledApp { + packageName: string; + label: string; +} + export interface ScreenState { elements: UIElement[]; screenshot?: string; diff --git a/server/src/agent/loop.ts b/server/src/agent/loop.ts index b9b10af..23d84e1 100644 --- a/server/src/agent/loop.ts +++ b/server/src/agent/loop.ts @@ -26,7 +26,7 @@ import { } from "./llm.js"; import { createStuckDetector } from "./stuck.js"; import { db } from "../db.js"; -import { agentSession, agentStep } from "../schema.js"; +import { agentSession, agentStep, device as deviceTable } from "../schema.js"; import { eq } from "drizzle-orm"; import type { UIElement, ActionDecision } from "@droidclaw/shared"; @@ -42,6 +42,8 @@ export interface AgentLoopOptions { originalGoal?: string; llmConfig: LLMConfig; maxSteps?: number; + /** Abort signal for cancellation */ + signal?: AbortSignal; onStep?: (step: AgentStep) => void; onComplete?: (result: AgentResult) => void; } @@ -224,6 +226,7 @@ export async function runAgentLoop( originalGoal, llmConfig, maxSteps = 30, + signal, onStep, onComplete, } = options; @@ -239,6 +242,28 @@ export async function runAgentLoop( const recentActions: string[] = []; let lastActionFeedback = ""; + // Fetch installed apps from device metadata for LLM context + let installedAppsContext = ""; + if (persistentDeviceId) { + try { + const rows = await db + .select({ info: deviceTable.deviceInfo }) + .from(deviceTable) + .where(eq(deviceTable.id, persistentDeviceId)) + .limit(1); + const info = rows[0]?.info as Record | null; + const apps = info?.installedApps as Array<{ packageName: string; label: string }> | undefined; + if (apps && apps.length > 0) { + installedAppsContext = + `\nINSTALLED_APPS (use exact packageName for "launch" action):\n` + + apps.map((a) => ` ${a.label}: ${a.packageName}`).join("\n") + + "\n"; + } + } catch { + // Non-critical — continue without apps context + } + } + // Persist session to DB if (persistentDeviceId) { try { @@ -268,6 +293,12 @@ export async function runAgentLoop( try { for (let step = 0; step < maxSteps; step++) { + // Check for cancellation + if (signal?.aborted) { + console.log(`[Agent ${sessionId}] Stopped by user at step ${step + 1}`); + break; + } + stepsUsed = step + 1; // ── 1. Get screen state from device ───────────────────── @@ -371,6 +402,7 @@ export async function runAgentLoop( let userPrompt = `GOAL: ${goal}\n\n` + `STEP: ${step + 1}/${maxSteps}\n\n` + + installedAppsContext + foregroundLine + actionFeedbackLine + `SCREEN_CONTEXT:\n${JSON.stringify(elements, null, 2)}` + diff --git a/server/src/agent/preprocessor.ts b/server/src/agent/preprocessor.ts index 1b150e6..f9cddc9 100644 --- a/server/src/agent/preprocessor.ts +++ b/server/src/agent/preprocessor.ts @@ -8,10 +8,21 @@ */ import { sessions } from "../ws/sessions.js"; +import { db } from "../db.js"; +import { device as deviceTable } from "../schema.js"; +import { eq } from "drizzle-orm"; -// ─── App Name → Package Name Map ──────────────────────────── +// ─── Installed App type ────────────────────────────────────── -const APP_PACKAGES: Record = { +interface InstalledApp { + packageName: string; + label: string; +} + +// ─── Fallback App Name → Package Name Map ──────────────────── +// Used only when device has no installed apps data in DB. + +const FALLBACK_PACKAGES: Record = { youtube: "com.google.android.youtube", gmail: "com.google.android.gm", chrome: "com.android.chrome", @@ -79,12 +90,32 @@ interface PreprocessResult { } /** - * Try to find a known app name at the start of a goal string. + * Build a label→packageName lookup from the device's installed apps. + * Keys are lowercase app labels (e.g. "youtube", "play store"). + */ +function buildInstalledAppMap(apps: InstalledApp[]): Record { + const map: Record = {}; + for (const app of apps) { + map[app.label.toLowerCase()] = app.packageName; + } + return map; +} + +/** + * Try to find an app name at the start of a goal string. + * Checks device's installed apps first, then falls back to hardcoded map. * Returns the package name and remaining text, or null. */ -function matchAppName(lower: string): { pkg: string; appName: string; rest: string } | null { +function matchAppName( + lower: string, + installedApps: InstalledApp[] +): { pkg: string; appName: string; rest: string } | null { + // Build combined lookup: installed apps take priority over fallback + const installedMap = buildInstalledAppMap(installedApps); + const combined: Record = { ...FALLBACK_PACKAGES, ...installedMap }; + // Try longest app names first (e.g. "google meet" before "meet") - const sorted = Object.keys(APP_PACKAGES).sort((a, b) => b.length - a.length); + const sorted = Object.keys(combined).sort((a, b) => b.length - a.length); for (const name of sorted) { // Match: "open [app] and " or "open [app]" @@ -93,7 +124,7 @@ function matchAppName(lower: string): { pkg: string; appName: string; rest: stri ); const m = lower.match(pattern); if (m) { - return { pkg: APP_PACKAGES[name], appName: name, rest: m[1]?.trim() ?? "" }; + return { pkg: combined[name], appName: name, rest: m[1]?.trim() ?? "" }; } } return null; @@ -103,6 +134,23 @@ function escapeRegex(s: string): string { return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } +/** + * Fetch installed apps from the device record in DB. + */ +async function fetchInstalledApps(persistentDeviceId: string): Promise { + try { + const rows = await db + .select({ info: deviceTable.deviceInfo }) + .from(deviceTable) + .where(eq(deviceTable.id, persistentDeviceId)) + .limit(1); + const info = rows[0]?.info as Record | null; + return (info?.installedApps as InstalledApp[]) ?? []; + } catch { + return []; + } +} + /** * Attempt to preprocess a goal before the LLM loop. * @@ -113,12 +161,16 @@ function escapeRegex(s: string): string { */ export async function preprocessGoal( deviceId: string, - goal: string + goal: string, + persistentDeviceId?: string ): Promise { const lower = goal.toLowerCase().trim(); + // Fetch device's actual installed apps for accurate package resolution + const installedApps = persistentDeviceId ? await fetchInstalledApps(persistentDeviceId) : []; + // ── Pattern: "open [and ]" ─────────────── - const appMatch = matchAppName(lower); + const appMatch = matchAppName(lower, installedApps); if (appMatch) { try { diff --git a/server/src/middleware/auth.ts b/server/src/middleware/auth.ts index 078c06c..06b6142 100644 --- a/server/src/middleware/auth.ts +++ b/server/src/middleware/auth.ts @@ -1,5 +1,8 @@ import type { Context, Next } from "hono"; -import { auth } from "../auth.js"; +import { db } from "../db.js"; +import { session as sessionTable, user as userTable } from "../schema.js"; +import { eq } from "drizzle-orm"; +import { getCookie } from "hono/cookie"; /** Hono Env type for routes protected by sessionMiddleware */ export type AuthEnv = { @@ -10,15 +13,43 @@ export type AuthEnv = { }; export async function sessionMiddleware(c: Context, next: Next) { - const session = await auth.api.getSession({ - headers: c.req.raw.headers, - }); - - if (!session) { + // Extract session token from cookie (same approach as dashboard WS auth) + const rawCookie = getCookie(c, "better-auth.session_token"); + if (!rawCookie) { return c.json({ error: "unauthorized" }, 401); } - c.set("user", session.user); - c.set("session", session.session); + // Token may have a signature appended after a dot — use only the token part + const token = rawCookie.split(".")[0]; + + // Direct DB lookup (proven to work, unlike auth.api.getSession) + const rows = await db + .select({ + sessionId: sessionTable.id, + userId: sessionTable.userId, + }) + .from(sessionTable) + .where(eq(sessionTable.token, token)) + .limit(1); + + if (rows.length === 0) { + return c.json({ error: "unauthorized" }, 401); + } + + const { sessionId, userId } = rows[0]; + + // Fetch user info + const users = await db + .select({ id: userTable.id, name: userTable.name, email: userTable.email }) + .from(userTable) + .where(eq(userTable.id, userId)) + .limit(1); + + if (users.length === 0) { + return c.json({ error: "unauthorized" }, 401); + } + + c.set("user", users[0]); + c.set("session", { id: sessionId, userId }); await next(); } diff --git a/server/src/routes/goals.ts b/server/src/routes/goals.ts index d79029d..cbbf9d4 100644 --- a/server/src/routes/goals.ts +++ b/server/src/routes/goals.ts @@ -1,14 +1,17 @@ import { Hono } from "hono"; +import { eq } from "drizzle-orm"; import { sessionMiddleware, type AuthEnv } from "../middleware/auth.js"; import { sessions } from "../ws/sessions.js"; import { runAgentLoop, type AgentLoopOptions } from "../agent/loop.js"; import type { LLMConfig } from "../agent/llm.js"; +import { db } from "../db.js"; +import { llmConfig as llmConfigTable } from "../schema.js"; const goals = new Hono(); goals.use("*", sessionMiddleware); -/** Track running agent sessions so we can prevent duplicates */ -const activeSessions = new Map(); +/** Track running agent sessions so we can prevent duplicates and cancel them */ +const activeSessions = new Map(); goals.post("/", async (c) => { const user = c.get("user"); @@ -46,15 +49,39 @@ goals.post("/", async (c) => { ); } - // Build LLM config from request body or environment defaults - const llmConfig: LLMConfig = { - provider: body.llmProvider ?? process.env.LLM_PROVIDER ?? "openai", - apiKey: body.llmApiKey ?? process.env.LLM_API_KEY ?? "", - model: body.llmModel, - }; + // Build LLM config: request body → user's DB config → env defaults + let llmCfg: LLMConfig; - if (!llmConfig.apiKey) { - return c.json({ error: "LLM API key is required (provide llmApiKey or set LLM_API_KEY env var)" }, 400); + if (body.llmApiKey) { + llmCfg = { + provider: body.llmProvider ?? process.env.LLM_PROVIDER ?? "openai", + apiKey: body.llmApiKey, + model: body.llmModel, + }; + } else { + // Fetch user's saved LLM config from DB (same as device WS handler) + const configs = await db + .select() + .from(llmConfigTable) + .where(eq(llmConfigTable.userId, user.id)) + .limit(1); + + if (configs.length > 0) { + const cfg = configs[0]; + llmCfg = { + provider: cfg.provider, + apiKey: cfg.apiKey, + model: body.llmModel ?? cfg.model ?? undefined, + }; + } else if (process.env.LLM_API_KEY) { + llmCfg = { + provider: process.env.LLM_PROVIDER ?? "openai", + apiKey: process.env.LLM_API_KEY, + model: body.llmModel, + }; + } else { + return c.json({ error: "No LLM provider configured. Set it up in the web dashboard Settings." }, 400); + } } const options: AgentLoopOptions = { @@ -62,16 +89,20 @@ goals.post("/", async (c) => { persistentDeviceId: device.persistentDeviceId, userId: user.id, goal: body.goal, - llmConfig, + llmConfig: llmCfg, maxSteps: body.maxSteps, }; + // Create abort controller for this session + const abort = new AbortController(); + options.signal = abort.signal; + // Start the agent loop in the background (fire-and-forget). // The client observes progress via the /ws/dashboard WebSocket. const loopPromise = runAgentLoop(options); // Track as active until it completes - const sessionPlaceholder = { sessionId: "pending", goal: body.goal }; + const sessionPlaceholder = { sessionId: "pending", goal: body.goal, abort }; activeSessions.set(trackingKey, sessionPlaceholder); loopPromise @@ -96,4 +127,33 @@ goals.post("/", async (c) => { }); }); +goals.post("/stop", async (c) => { + const user = c.get("user"); + const body = await c.req.json<{ deviceId: string }>(); + + if (!body.deviceId) { + return c.json({ error: "deviceId is required" }, 400); + } + + // Look up device to verify ownership + const device = sessions.getDevice(body.deviceId) + ?? sessions.getDeviceByPersistentId(body.deviceId); + if (!device) { + return c.json({ error: "device not connected" }, 404); + } + if (device.userId !== user.id) { + return c.json({ error: "device does not belong to you" }, 403); + } + + const trackingKey = device.persistentDeviceId ?? device.deviceId; + const active = activeSessions.get(trackingKey); + if (!active) { + return c.json({ error: "no agent running on this device" }, 404); + } + + active.abort.abort(); + console.log(`[Agent] Stop requested for device ${body.deviceId}`); + return c.json({ status: "stopping" }); +}); + export { goals }; diff --git a/server/src/ws/device.ts b/server/src/ws/device.ts index 7b6c59d..34e429b 100644 --- a/server/src/ws/device.ts +++ b/server/src/ws/device.ts @@ -227,7 +227,7 @@ export async function handleDeviceMessage( // Preprocess: handle simple goals directly, or extract "open X" prefix let effectiveGoal = goal; try { - const preResult = await preprocessGoal(deviceId, goal); + const preResult = await preprocessGoal(deviceId, goal, persistentDeviceId); if (preResult.handled) { await new Promise((r) => setTimeout(r, 1500)); @@ -328,6 +328,26 @@ export async function handleDeviceMessage( break; } + case "apps": { + const persistentDeviceId = ws.data.persistentDeviceId; + if (persistentDeviceId) { + const apps = (msg as unknown as { apps: Array<{ packageName: string; label: string }> }).apps; + // Merge apps into existing deviceInfo + db.update(device) + .set({ + deviceInfo: { + ...(await db.select({ info: device.deviceInfo }).from(device).where(eq(device.id, persistentDeviceId)).limit(1).then(r => (r[0]?.info as Record) ?? {})), + installedApps: apps, + }, + }) + .where(eq(device.id, persistentDeviceId)) + .catch((err) => console.error(`[DB] Failed to store installed apps: ${err}`)); + + console.log(`[Device] Received ${apps.length} installed apps for device ${persistentDeviceId}`); + } + break; + } + case "heartbeat": { const persistentDeviceId = ws.data.persistentDeviceId; const userId = ws.data.userId; diff --git a/web/src/hooks.server.ts b/web/src/hooks.server.ts index 4fb0c34..c082f6f 100644 --- a/web/src/hooks.server.ts +++ b/web/src/hooks.server.ts @@ -4,13 +4,20 @@ import { building } from '$app/environment'; import type { Handle } from '@sveltejs/kit'; export const handle: Handle = async ({ event, resolve }) => { - const session = await auth.api.getSession({ - headers: event.request.headers - }); + try { + const session = await auth.api.getSession({ + headers: event.request.headers + }); - if (session) { - event.locals.session = session.session; - event.locals.user = session.user; + if (session) { + event.locals.session = session.session; + event.locals.user = session.user; + } else if (event.url.pathname.startsWith('/api/')) { + console.log(`[Auth] No session for ${event.request.method} ${event.url.pathname}`); + console.log(`[Auth] Cookie header: ${event.request.headers.get('cookie')?.slice(0, 80) ?? 'NONE'}`); + } + } catch (err) { + console.error(`[Auth] getSession error for ${event.request.method} ${event.url.pathname}:`, err); } return svelteKitHandler({ event, resolve, auth, building }); diff --git a/web/src/lib/api/devices.remote.ts b/web/src/lib/api/devices.remote.ts index e46adc6..06a7025 100644 --- a/web/src/lib/api/devices.remote.ts +++ b/web/src/lib/api/devices.remote.ts @@ -1,5 +1,6 @@ import * as v from 'valibot'; -import { query, getRequestEvent } from '$app/server'; +import { query, command, getRequestEvent } from '$app/server'; +import { env } from '$env/dynamic/private'; import { db } from '$lib/server/db'; import { device, agentSession, agentStep } from '$lib/server/db/schema'; import { eq, desc, and, count, avg, sql, inArray } from 'drizzle-orm'; @@ -85,7 +86,8 @@ export const getDevice = query(v.string(), async (deviceId) => { screenHeight: (info?.screenHeight as number) ?? null, batteryLevel: (info?.batteryLevel as number) ?? null, isCharging: (info?.isCharging as boolean) ?? false, - lastSeen: d.lastSeen?.toISOString() ?? d.createdAt.toISOString() + lastSeen: d.lastSeen?.toISOString() ?? d.createdAt.toISOString(), + installedApps: (info?.installedApps as Array<{ packageName: string; label: string }>) ?? [] }; }); @@ -155,3 +157,37 @@ export const listSessionSteps = query( return steps; } ); + +// ─── Commands (write operations) ───────────────────────────── + +const SERVER_URL = () => env.SERVER_URL || 'http://localhost:8080'; + +/** Forward a request to the DroidClaw server with auth cookies */ +async function serverFetch(path: string, body: Record) { + const { request } = getRequestEvent(); + const res = await fetch(`${SERVER_URL()}${path}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + cookie: request.headers.get('cookie') ?? '' + }, + body: JSON.stringify(body) + }); + const data = await res.json().catch(() => ({ error: 'Unknown error' })); + if (!res.ok) throw new Error(data.error ?? `Error ${res.status}`); + return data; +} + +export const submitGoal = command( + v.object({ deviceId: v.string(), goal: v.string() }), + async ({ deviceId, goal }) => { + return serverFetch('/goals', { deviceId, goal }); + } +); + +export const stopGoal = command( + v.object({ deviceId: v.string() }), + async ({ deviceId }) => { + return serverFetch('/goals/stop', { deviceId }); + } +); diff --git a/web/src/routes/api/goals/+server.ts b/web/src/routes/api/goals/+server.ts deleted file mode 100644 index 34b13e3..0000000 --- a/web/src/routes/api/goals/+server.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { json, error } from '@sveltejs/kit'; -import { env } from '$env/dynamic/private'; -import type { RequestHandler } from './$types'; - -const SERVER_URL = env.SERVER_URL || 'http://localhost:8080'; - -export const POST: RequestHandler = async ({ request, locals }) => { - if (!locals.user) { - return error(401, 'Unauthorized'); - } - - const body = await request.json(); - - const res = await fetch(`${SERVER_URL}/goals`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - cookie: request.headers.get('cookie') ?? '' - }, - body: JSON.stringify(body) - }); - - if (!res.ok) { - const err = await res.json().catch(() => ({ error: 'Unknown error' })); - return error(res.status, err.error ?? 'Failed to submit goal'); - } - - return json(await res.json()); -}; diff --git a/web/src/routes/dashboard/devices/[deviceId]/+page.svelte b/web/src/routes/dashboard/devices/[deviceId]/+page.svelte index 980e3ad..9ddb4d0 100644 --- a/web/src/routes/dashboard/devices/[deviceId]/+page.svelte +++ b/web/src/routes/dashboard/devices/[deviceId]/+page.svelte @@ -4,7 +4,9 @@ getDevice, listDeviceSessions, listSessionSteps, - getDeviceStats + getDeviceStats, + submitGoal as submitGoalCmd, + stopGoal as stopGoalCmd } from '$lib/api/devices.remote'; import { dashboardWs } from '$lib/stores/dashboard-ws.svelte'; import { onMount } from 'svelte'; @@ -27,6 +29,7 @@ batteryLevel: number | null; isCharging: boolean; lastSeen: string; + installedApps: Array<{ packageName: string; label: string }>; } | null; // Device stats @@ -80,23 +83,33 @@ } } + let runError = $state(''); + async function submitGoal() { if (!goal.trim()) return; runStatus = 'running'; + runError = ''; currentGoal = goal; steps = []; - const res = await fetch('/api/goals', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ deviceId, goal }) - }); - - if (!res.ok) { + try { + await submitGoalCmd({ deviceId, goal }); + } catch (e: any) { + runError = e.message ?? String(e); runStatus = 'failed'; } } + async function stopGoal() { + try { + await stopGoalCmd({ deviceId }); + runStatus = 'failed'; + runError = 'Stopped by user'; + } catch { + // ignore + } + } + onMount(() => { const unsub = dashboardWs.subscribe((msg) => { switch (msg.type) { @@ -159,6 +172,16 @@ return `${days}d ago`; } + let appSearch = $state(''); + const filteredApps = $derived( + (deviceData?.installedApps ?? []).filter( + (a) => + !appSearch || + a.label.toLowerCase().includes(appSearch.toLowerCase()) || + a.packageName.toLowerCase().includes(appSearch.toLowerCase()) + ) + ); + const battery = $derived(liveBattery ?? (deviceData?.batteryLevel as number | null)); const charging = $derived(liveCharging || (deviceData?.isCharging as boolean)); @@ -279,6 +302,34 @@ + + {#if deviceData && deviceData.installedApps.length > 0} +
+
+

+ Installed Apps + ({deviceData.installedApps.length}) +

+ +
+
+ {#each filteredApps as app (app.packageName)} +
+ {app.label} + {app.packageName} +
+ {:else} +

No apps match "{appSearch}"

+ {/each} +
+
+ {/if} + {:else if activeTab === 'sessions'} {#if sessions.length === 0} @@ -359,13 +410,21 @@ disabled={runStatus === 'running'} onkeydown={(e) => e.key === 'Enter' && submitGoal()} /> - + {#if runStatus === 'running'} + + {:else} + + {/if} @@ -391,6 +450,11 @@ Failed {/if} + {#if runError} +
+ {runError} +
+ {/if} {#if steps.length > 0}
{#each steps as s (s.step)}