diff --git a/app/src/main/java/com/pixel10/ai/inference/LiteRTModel.kt b/app/src/main/java/com/pixel10/ai/inference/LiteRTModel.kt index a87b5fa..d58d8b9 100644 --- a/app/src/main/java/com/pixel10/ai/inference/LiteRTModel.kt +++ b/app/src/main/java/com/pixel10/ai/inference/LiteRTModel.kt @@ -7,6 +7,8 @@ import com.google.ai.edge.litertlm.Engine import com.google.ai.edge.litertlm.EngineConfig import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.catch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext import java.io.File @@ -29,42 +31,49 @@ class LiteRTModel private constructor( override var isReady: Boolean = true private set + // LiteRT Engine is not thread-safe — serialize all inference calls + private val mutex = Mutex() + override suspend fun generate( prompt: String, maxTokens: Int, temperature: Float - ): String = withContext(Dispatchers.Default) { - val conversation = engine.createConversation() - try { - conversation.sendMessage(prompt).toString() - } catch (e: Exception) { - Log.e(TAG, "LiteRT inference error", e) - throw OnDeviceModel.InferenceException("Generation failed: ${e.message}", e) - } finally { - conversation.close() + ): String = mutex.withLock { + withContext(Dispatchers.Default) { + val conversation = engine.createConversation() + try { + conversation.sendMessage(prompt).toString() + } catch (e: Exception) { + Log.e(TAG, "LiteRT inference error", e) + throw OnDeviceModel.InferenceException("Generation failed: ${e.message}", e) + } finally { + conversation.close() + } } } override suspend fun generateStreaming( prompt: String, onToken: (String) -> Unit - ): String = withContext(Dispatchers.Default) { - val conversation = engine.createConversation() - val sb = StringBuilder() - try { - conversation.sendMessageAsync(prompt) - .catch { e -> - throw OnDeviceModel.InferenceException("Streaming failed: ${e.message}", e) - } - .collect { message -> - val token = message.toString() - sb.append(token) - onToken(token) - } - } finally { - conversation.close() + ): String = mutex.withLock { + withContext(Dispatchers.Default) { + val conversation = engine.createConversation() + val sb = StringBuilder() + try { + conversation.sendMessageAsync(prompt) + .catch { e -> + throw OnDeviceModel.InferenceException("Streaming failed: ${e.message}", e) + } + .collect { message -> + val token = message.toString() + sb.append(token) + onToken(token) + } + } finally { + conversation.close() + } + sb.toString() } - sb.toString() } override fun close() { diff --git a/app/src/main/java/com/pixel10/ai/server/AIApiServer.kt b/app/src/main/java/com/pixel10/ai/server/AIApiServer.kt index 8e4bd05..07661c8 100644 --- a/app/src/main/java/com/pixel10/ai/server/AIApiServer.kt +++ b/app/src/main/java/com/pixel10/ai/server/AIApiServer.kt @@ -27,9 +27,16 @@ import java.util.concurrent.atomic.AtomicLong * -H "Content-Type: application/json" \ * -d '{"messages":[{"role":"user","content":"Hello!"}]}' */ +data class ServerConfig( + val defaultTemperature: Float = 0.7f, + val defaultMaxTokens: Int = 1024, + val autoSystemPrompt: Boolean = true +) + class AIApiServer( port: Int, - private val model: OnDeviceModel + private val model: OnDeviceModel, + private val config: ServerConfig = ServerConfig() ) : NanoHTTPD(port) { private val gson = Gson() @@ -37,6 +44,7 @@ class AIApiServer( val requestCount = AtomicLong(0) var onRequestLogged: ((String) -> Unit)? = null + var onActiveRequest: ((Boolean) -> Unit)? = null override fun serve(session: IHTTPSession): Response { val method = session.method @@ -113,31 +121,48 @@ class AIApiServer( return errorResponse(400, "messages array is required and must not be empty") } - // Auto-inject agent system prompt if the conversation has no system message. - // Auto-inject default tools if the request provides none. - // This makes the server zero-config as a coding agent for any OpenAI-compatible client. - val messages = if (raw.messages.none { it.role == "system" }) { + // Auto-inject agent system prompt if enabled and no system message present + val messages = if (config.autoSystemPrompt && raw.messages.none { it.role == "system" }) { listOf(Message(role = "system", content = AgentConfig.SYSTEM_PROMPT)) + raw.messages } else { raw.messages } val request = raw.copy( messages = messages, - tools = raw.tools.takeUnless { it.isNullOrEmpty() } ?: AgentConfig.DEFAULT_TOOLS + tools = raw.tools.takeUnless { it.isNullOrEmpty() } + ?: if (config.autoSystemPrompt) AgentConfig.DEFAULT_TOOLS else null, + temperature = if (raw.temperature == 0.7f) config.defaultTemperature else raw.temperature, + max_tokens = if (raw.max_tokens == 8192) config.defaultMaxTokens else raw.max_tokens ) val id = "chatcmpl-${UUID.randomUUID().toString().take(8)}" val hasTools = !request.tools.isNullOrEmpty() - log("Chat: ${request.messages.size} messages, tools=${request.tools?.size ?: 0}, stream=${request.stream}") + log("Chat: ${request.messages.size} messages, tools=${request.tools?.size ?: 0}, stream=${request.stream}, temp=${request.temperature}") + + // ── Streaming — always uses flat prompt + generateStreaming ──────────── + if (request.stream) { + val prompt = buildFlatPrompt(request.messages) + onActiveRequest?.invoke(true) + return try { + handleStreamingResponse(id, prompt, request) + } finally { + onActiveRequest?.invoke(false) + } + } // ── Tool calling / multi-turn chat ───────────────────────────────────── if (hasTools || request.messages.size > 1 || request.messages.any { it.role == "system" }) { val convMessages = request.messages.map { it.toConvMessage() } val toolDefs = request.tools?.map { it.toToolDef() } ?: emptyList() - val result = runBlocking { - model.chat(convMessages, toolDefs, request.max_tokens, request.temperature) + onActiveRequest?.invoke(true) + val result = try { + runBlocking { + model.chat(convMessages, toolDefs, request.max_tokens, request.temperature) + } + } finally { + onActiveRequest?.invoke(false) } if (result.toolCalls != null) { diff --git a/app/src/main/java/com/pixel10/ai/server/ApiServerService.kt b/app/src/main/java/com/pixel10/ai/server/ApiServerService.kt index 04ce35e..74a1ffb 100644 --- a/app/src/main/java/com/pixel10/ai/server/ApiServerService.kt +++ b/app/src/main/java/com/pixel10/ai/server/ApiServerService.kt @@ -34,6 +34,7 @@ class ApiServerService : Service() { var onStatusChanged: ((ServerState) -> Unit)? = null var onLog: ((String) -> Unit)? = null + var onActiveRequest: ((Boolean) -> Unit)? = null val isRunning: Boolean get() = server != null val requestCount: Long get() = server?.requestCount?.get() ?: 0 @@ -70,8 +71,15 @@ class ApiServerService : Service() { // Start the HTTP server notifyLog("Starting API server on port $port...") - val apiServer = AIApiServer(port, model!!) + val prefs = getSharedPreferences("pixel10_prefs", MODE_PRIVATE) + val serverConfig = ServerConfig( + defaultTemperature = prefs.getFloat("temperature", 0.7f), + defaultMaxTokens = prefs.getInt("max_tokens", 1024), + autoSystemPrompt = prefs.getBoolean("auto_system_prompt", true) + ) + val apiServer = AIApiServer(port, model!!, serverConfig) apiServer.onRequestLogged = { msg -> notifyLog(msg) } + apiServer.onActiveRequest = { active -> onActiveRequest?.invoke(active) } apiServer.start() server = apiServer diff --git a/app/src/main/java/com/pixel10/ai/ui/MainActivity.kt b/app/src/main/java/com/pixel10/ai/ui/MainActivity.kt index 8794d1a..db50e2d 100644 --- a/app/src/main/java/com/pixel10/ai/ui/MainActivity.kt +++ b/app/src/main/java/com/pixel10/ai/ui/MainActivity.kt @@ -13,6 +13,7 @@ import android.os.Build import android.os.Bundle import android.os.IBinder import android.view.View +import android.widget.SeekBar import androidx.activity.result.contract.ActivityResultContracts import androidx.appcompat.app.AppCompatActivity import androidx.core.content.ContextCompat @@ -53,6 +54,16 @@ class MainActivity : AppCompatActivity() { service?.onLog = { message -> runOnUiThread { appendLog(message) } } + service?.onActiveRequest = { active -> + runOnUiThread { + if (active) { + binding.tvActiveRequest.text = "⚡ Processing request…" + binding.tvActiveRequest.visibility = View.VISIBLE + } else { + binding.tvActiveRequest.visibility = View.GONE + } + } + } if (service?.isRunning == true) { updateStatus(ApiServerService.ServerState.RUNNING) @@ -73,8 +84,21 @@ class MainActivity : AppCompatActivity() { prefs = getSharedPreferences("pixel10_prefs", MODE_PRIVATE) requestNotificationPermission() - // Restore saved HF token + // Restore saved settings binding.etHfToken.setText(prefs.getString("hf_token", "")) + val savedTemp = (prefs.getFloat("temperature", 0.7f) * 100).toInt() + binding.seekTemperature.progress = savedTemp + binding.tvTemperatureValue.text = "%.1f".format(savedTemp / 100f) + binding.etMaxTokens.setText(prefs.getInt("max_tokens", 1024).toString()) + binding.switchSystemPrompt.isChecked = prefs.getBoolean("auto_system_prompt", true) + + binding.seekTemperature.setOnSeekBarChangeListener(object : SeekBar.OnSeekBarChangeListener { + override fun onProgressChanged(seekBar: SeekBar, progress: Int, fromUser: Boolean) { + binding.tvTemperatureValue.text = "%.1f".format(progress / 100f) + } + override fun onStartTrackingTouch(seekBar: SeekBar) {} + override fun onStopTrackingTouch(seekBar: SeekBar) {} + }) binding.btnToggle.setOnClickListener { if (service?.isRunning == true) stopServer() else startServer() @@ -201,7 +225,16 @@ class MainActivity : AppCompatActivity() { } } + private fun saveSettings() { + prefs.edit() + .putFloat("temperature", binding.seekTemperature.progress / 100f) + .putInt("max_tokens", binding.etMaxTokens.text.toString().toIntOrNull() ?: 1024) + .putBoolean("auto_system_prompt", binding.switchSystemPrompt.isChecked) + .apply() + } + private fun startServer() { + saveSettings() val port = binding.etPort.text.toString().toIntOrNull() ?: 8080 val intent = Intent(this, ApiServerService::class.java).apply { action = ApiServerService.ACTION_START @@ -227,6 +260,9 @@ class MainActivity : AppCompatActivity() { val canEdit = state == ApiServerService.ServerState.STOPPED || state == ApiServerService.ServerState.ERROR binding.etPort.isEnabled = canEdit + binding.seekTemperature.isEnabled = canEdit + binding.etMaxTokens.isEnabled = canEdit + binding.switchSystemPrompt.isEnabled = canEdit when (state) { ApiServerService.ServerState.STOPPED -> { diff --git a/app/src/main/res/layout/activity_main.xml b/app/src/main/res/layout/activity_main.xml index 9c6cad0..a1d3ed2 100644 --- a/app/src/main/res/layout/activity_main.xml +++ b/app/src/main/res/layout/activity_main.xml @@ -102,6 +102,16 @@ android:textColor="@color/log_text" android:textSize="13sp" android:layout_marginTop="2dp" /> + + @@ -176,38 +186,143 @@ - - + - + android:orientation="vertical" + android:padding="16dp"> - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + No local model. Enter HuggingFace token and download. Start Server Stop Server - Port: + Port + Temperature + Max tokens + Agent system prompt 8080 Requests served: %d Request Log