mirror of
https://github.com/souravanand001/ai-assistant-android.git
synced 2026-10-21 22:31:14 +00:00
Compare commits
72 commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0f01923057 | ||
|
|
b5f76f5fae | ||
|
|
70d2ea3df4 | ||
|
|
871bc86218 | ||
|
|
a97ae47d5e | ||
|
|
11ca1ea1b3 | ||
|
|
3359adfa4d | ||
|
|
e760bc38f4 | ||
|
|
2646619e9b | ||
|
|
86ccb9ed3e | ||
|
|
fbeb01d633 | ||
|
|
8b82f74c30 | ||
|
|
e5901e5ad1 | ||
|
|
ef9748419e | ||
|
|
2bd7e4beb0 | ||
|
|
59177aa80f | ||
|
|
a21dc95243 | ||
|
|
49eea42363 | ||
|
|
c311c86c42 | ||
|
|
c4399d0659 | ||
|
|
8b26d639d6 | ||
|
|
b6b9b318fd | ||
|
|
be4aa53fa6 | ||
|
|
f2b2bb9bfe | ||
|
|
a1e0001e23 | ||
|
|
dd05760951 | ||
|
|
85cfcc7a62 | ||
|
|
b592f085ae | ||
|
|
90c2219b4b | ||
|
|
3d4d66628a | ||
|
|
ab5465cf41 | ||
|
|
bcecea4276 | ||
|
|
b0227b71b4 | ||
|
|
ccb43f3c96 | ||
|
|
92585e7df1 | ||
|
|
bc02058e0c | ||
|
|
f8c9ac3bf6 | ||
|
|
c3bce8f807 | ||
|
|
04216343f1 | ||
|
|
906659ca4e | ||
|
|
323050a9fa | ||
|
|
871c3c0011 | ||
|
|
6e2f93b758 | ||
|
|
8b0b9b3d98 | ||
|
|
13894b5432 | ||
|
|
6840376aa8 | ||
|
|
1fef463680 | ||
|
|
fae7d23068 | ||
|
|
04f3e52f49 | ||
|
|
75dd48dabe | ||
|
|
e621b7ea15 | ||
|
|
427d1d4b70 | ||
|
|
cfca8f5516 | ||
|
|
b7afeb3899 | ||
|
|
8aec5fb91c | ||
|
|
70c0e575e2 | ||
|
|
e1f4fb469a | ||
|
|
24755c1a0c | ||
|
|
cc38eedc88 | ||
|
|
661894981a | ||
|
|
8426db0d97 | ||
|
|
e23dacce8b | ||
|
|
80a82620e3 | ||
|
|
8c6e583773 | ||
|
|
c07b2c42d9 | ||
|
|
b71a4b0835 | ||
|
|
023a499840 | ||
|
|
8724ad25b7 | ||
|
|
3ba51c50ff | ||
|
|
364438a77c | ||
|
|
5a6d90176d | ||
|
|
a888bead9a |
124 changed files with 18437 additions and 3646 deletions
1
.github/CODEOWNERS
vendored
Normal file
1
.github/CODEOWNERS
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
* @souravanand001
|
||||
81
.github/workflows/ci-cd.yml
vendored
Normal file
81
.github/workflows/ci-cd.yml
vendored
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
name: CI/CD Pipeline
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up JDK 17
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
distribution: 'zulu'
|
||||
java-version: '17'
|
||||
cache: 'gradle'
|
||||
|
||||
- name: Make Gradlew Executable
|
||||
run: chmod +x gradlew
|
||||
|
||||
# - name: Run Unit Tests
|
||||
# run: ./gradlew testDebugUnitTest
|
||||
|
||||
- name: Build Debug APK
|
||||
if: github.event_name == 'pull_request'
|
||||
run: ./gradlew assembleDebug
|
||||
|
||||
- name: Decode Keystore
|
||||
if: github.event_name == 'push'
|
||||
env:
|
||||
KEYSTORE_BASE64: ${{ secrets.KEYSTORE_BASE64 }}
|
||||
run: |
|
||||
if [ -z "$KEYSTORE_BASE64" ]; then
|
||||
echo "Error: KEYSTORE_BASE64 secret is not defined."
|
||||
exit 1
|
||||
fi
|
||||
echo "$KEYSTORE_BASE64" | base64 --decode > "${{ runner.temp }}/keystore.jks"
|
||||
|
||||
- name: Build Release APK
|
||||
if: github.event_name == 'push'
|
||||
env:
|
||||
KEYSTORE_FILE_PATH: ${{ runner.temp }}/keystore.jks
|
||||
KEYSTORE_PASSWORD: ${{ secrets.KEYSTORE_PASSWORD }}
|
||||
KEYSTORE_KEY_ALIAS: ${{ secrets.KEYSTORE_KEY_ALIAS }}
|
||||
KEYSTORE_KEY_PASSWORD: ${{ secrets.KEYSTORE_KEY_PASSWORD }}
|
||||
run: |
|
||||
if [ -z "$KEYSTORE_PASSWORD" ] || [ -z "$KEYSTORE_KEY_ALIAS" ] || [ -z "$KEYSTORE_KEY_PASSWORD" ]; then
|
||||
echo "Error: Keystore signing secrets are missing."
|
||||
exit 1
|
||||
fi
|
||||
./gradlew assembleRelease
|
||||
|
||||
- name: Rename Release APK with Version
|
||||
if: github.event_name == 'push'
|
||||
run: |
|
||||
VERSION_NAME="2.1.${{ github.run_number }}"
|
||||
mv app/build/outputs/apk/release/app-release.apk app/build/outputs/apk/release/assistant-v${VERSION_NAME}.apk
|
||||
echo "VERSION_NAME=${VERSION_NAME}" >> $GITHUB_ENV
|
||||
|
||||
- name: Create GitHub Release and Upload APK
|
||||
if: github.event_name == 'push'
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
tag_name: v${{ env.VERSION_NAME }}
|
||||
name: Release v${{ env.VERSION_NAME }}
|
||||
draft: false
|
||||
prerelease: ${{ github.ref_name != 'main' }}
|
||||
files: app/build/outputs/apk/release/assistant-v${{ env.VERSION_NAME }}.apk
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -18,6 +18,7 @@ out/
|
|||
|
||||
# Gradle files
|
||||
.gradle/
|
||||
.gradle_home/
|
||||
build/
|
||||
|
||||
# Local configuration file (sdk path, etc)
|
||||
|
|
|
|||
552
README.md
552
README.md
|
|
@ -1,173 +1,455 @@
|
|||
# AI Assistant Android App
|
||||
# AI Assistant for Android
|
||||
|
||||

|
||||

|
||||

|
||||

|
||||

|
||||
[](LICENSE) [](https://kotlinlang.org) [](https://developer.android.com/jetpack/compose) [](https://developers.google.com/mediapipe) [](https://www.tensorflow.org/lite) [](https://github.com/k2-fsa/sherpa-onnx)
|
||||
|
||||
## Demo & Screenshots
|
||||
An offline-first Android assistant that runs locally on your device. It supports screen vision, live camera sharing with real-time voice chat, encrypted data storage, and lets you connect your own LLM API keys. You can set it as your default phone assistant.
|
||||
|
||||
<p align="center">
|
||||
<b>App Overview:</b> <br/>
|
||||
<img src="screenshots/gif1.gif" alt="App Overview GIF">
|
||||
<div align="center">
|
||||
|
||||
<p>
|
||||
<img src="screenshots/gif1.gif" alt="App Overview" width="300">
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<b>Screenshots:</b> <br/>
|
||||
</p>
|
||||
<table>
|
||||
<tr align="center">
|
||||
<th width="33%">Conversational Chat UI</th>
|
||||
<th width="33%">Hands Free Floating Orb</th>
|
||||
<th width="33%">System Settings</th>
|
||||
</tr>
|
||||
<tr align="center">
|
||||
<td width="33%"><img src="screenshots/image1.png" width="100%"></td>
|
||||
<td width="33%"><img src="screenshots/image2.png" width="100%"></td>
|
||||
<td width="33%"><img src="screenshots/image3.png" width="100%"></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<table align="center">
|
||||
<tr>
|
||||
<td>
|
||||
<img src="screenshots/image1.png" alt="Screenshot 1" width="250">
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## Demo
|
||||
|
||||
<table width="100%">
|
||||
<tr align="center">
|
||||
<td width="33%">
|
||||
<video src="https://github.com/user-attachments/assets/209f907a-494c-4c7a-b81d-275a61301a7a" width="100%" controls>
|
||||
<a href="https://github.com/user-attachments/assets/209f907a-494c-4c7a-b81d-275a61301a7a">Download/Play video</a>
|
||||
</video>
|
||||
</td>
|
||||
<td>
|
||||
<img src="screenshots/image2.png" alt="Screenshot 2" width="250">
|
||||
<td width="33%">
|
||||
<video src="https://github.com/user-attachments/assets/4b81ecef-11e2-4918-a628-b7f5bca900b0" width="100%" controls>
|
||||
<a href="https://github.com/user-attachments/assets/4b81ecef-11e2-4918-a628-b7f5bca900b0">Download/Play video</a>
|
||||
</video>
|
||||
</td>
|
||||
<td>
|
||||
<img src="screenshots/image3.png" alt="Screenshot 3" width="250">
|
||||
<td width="33%">
|
||||
<video src="https://github.com/user-attachments/assets/6ef9f91c-93fd-4b68-8dd2-980b7f750a72" width="100%" controls>
|
||||
<a href="https://github.com/user-attachments/assets/6ef9f91c-93fd-4b68-8dd2-980b7f750a72">Download/Play video</a>
|
||||
</video>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
---
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Demo](#demo)
|
||||
- [Overview](#overview)
|
||||
- [Features](#features)
|
||||
- [Architecture Overview](#architecture-overview)
|
||||
- [Request Processing Flow](#request-processing-flow)
|
||||
- [Technologies Used](#technologies-used)
|
||||
- [Project Structure](#project-structure)
|
||||
- [Setup](#setup)
|
||||
- [API Configuration](#api-configuration)
|
||||
- [Permissions](#permissions)
|
||||
- [Supported Features Table](#supported-features-table)
|
||||
- [Privacy & Security](#privacy--security)
|
||||
- [Known Issues](#known-issues)
|
||||
- [Roadmap](#roadmap)
|
||||
- [Contributing](#contributing)
|
||||
- [License](#license)
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Assistant is an Android application designed to act as a personal voice assistant. It can understand spoken commands, process them, and perform various actions like making calls, playing music, setting alarms, providing weather information, and more. The app integrates with several external services and uses on-device machine learning for intent recognition.
|
||||
This app is a private voice assistant for Android. It runs offline as much as possible, encrypts your conversations, and reads your screen when asked. You can connect it to any OpenAI-compatible LLM endpoint.
|
||||
|
||||
Unlike standard assistants, it does not send your voice data to external servers. It replaces Android's default assistant, so you can launch it with a long press on the home button.
|
||||
|
||||
---
|
||||
|
||||
## Features
|
||||
|
||||
* **Voice Commands:** Interact with the assistant using natural language.
|
||||
* **Text-to-Speech & Speech-to-Text:** Converts spoken language to text and vice-versa for interaction.
|
||||
* **Intent Classification:** Understands user intent and categorizes requests (e.g., Call, Music, Weather, Navigation, Alarm, Reminder).
|
||||
* **Phone Calls:** Can initiate phone calls to contacts.
|
||||
* **Music Playback:** Plays music via YouTube.
|
||||
* **Alarms & Reminders:** Set alarms and reminders.
|
||||
* **Navigation:** Provides navigation directions using Google Maps.
|
||||
* **Weather Information:** Fetches and displays weather forecasts for specified locations or current location.
|
||||
* **Large Language Model (LLM) Integration:** Uses Groq API for advanced query understanding and responses (can be changed as per requirement).
|
||||
* **Translation:** Supports language translation for input queries and responses.
|
||||
* **Conversation History:** Stores and displays past interactions.
|
||||
* **Customizable UI:** Offers options for UI presentation.
|
||||
### AI & LLM
|
||||
- **Custom LLM Providers:** Works with Groq, OpenAI, Anthropic, Gemini, Ollama, LM Studio, etc.
|
||||
- **Vision Auto-Detection:** Automatically tests if your model supports images and enables screen vision and live camera features if it does.
|
||||
- **Fast Speech Output:** Starts speaking sentences before the model finishes generating the full response.
|
||||
|
||||
## Default Assistant App
|
||||
### Voice Pipeline
|
||||
- **Voice Activation (VAD):** Detects when you start and stop talking locally on your device.
|
||||
- **Interruption Support:** You can speak over the assistant to interrupt it.
|
||||
- **Hands-Free Mode:** Automatically listens for your next response after speaking.
|
||||
- **Audio Routing:** Supports Bluetooth headsets and speakerphone.
|
||||
|
||||
This application can be set as the default assistant app on your Android device. This allows you to launch the app by long-pressing the home button or using other system-level gestures for invoking the assistant.
|
||||
### Screen Vision
|
||||
- **Screen Capture:** Captures screen frames locally in the background.
|
||||
- **Smart Frame Syncing:** Matches the captured frame to the exact moment you started speaking.
|
||||
- **Camera Sharing:** Share your live camera feed to discuss your real-world surroundings with the assistant.
|
||||
- **Privacy Controls:** Easily pause or stop screen capture from a system notification.
|
||||
|
||||
## Process Flow
|
||||
### Privacy & Encryption
|
||||
- **Secure Storage:** Encrypts conversations and attachments using AES-256-GCM.
|
||||
- **Safe Deletion:** Securely deletes files from the device when you delete a message or chat.
|
||||
|
||||
```
|
||||
Start (User Input on ChatScreen: Voice/Text)
|
||||
│
|
||||
V
|
||||
MainViewModel (Input Processing, Permissions, Translation, Cleaning)
|
||||
│
|
||||
├─+------------------------------------─> Lock Active? (e.g., Call/Song specific flow)
|
||||
│ │ (Yes)
|
||||
V (No / General Flow) │
|
||||
Text Classification (Determine User Intent) Process Input based on Active Lock
|
||||
│ │
|
||||
V │
|
||||
Action Execution <──────────────────────────+
|
||||
(Based on Intent or Lock:
|
||||
- Make Call
|
||||
- Play Song
|
||||
- Navigate
|
||||
- Get Weather
|
||||
- Set Alarm/Reminder
|
||||
- Query General AI (Groq)
|
||||
- etc.)
|
||||
│
|
||||
V
|
||||
Process Action Result (e.g., Format AI response, Translate back)
|
||||
│
|
||||
V
|
||||
Update UI (ChatScreen displays conversation)
|
||||
│
|
||||
V
|
||||
Speak Response (TextToSpeech, if enabled)
|
||||
│
|
||||
V
|
||||
End
|
||||
### Offline Tools
|
||||
- **Offline Actions:** Processes commands (like calling or navigation) offline using an on-device model.
|
||||
- **Offline Speech & Voice:** Handles speech-to-text and text-to-speech without internet.
|
||||
- **Offline Translation:** Translates input to English locally using Google ML Kit.
|
||||
|
||||
### Android Integration
|
||||
- **Default Assistant:** Launch by long-pressing the home button or swiping.
|
||||
- **Lock Screen Support:** Answer voice queries without unlocking your phone.
|
||||
- **System Actions:** Set alarms, dial contacts (with fuzzy name matching), search YouTube music, and check the weather.
|
||||
|
||||
---
|
||||
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
The app is built around offline first, modular components. Each layer can be swapped independently.
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
A[User — Voice / Text / File] --> B[MainViewModel]
|
||||
|
||||
subgraph Input Processing
|
||||
B --> C[TranslatorManager\nML Kit Offline]
|
||||
C --> D[TextClassifierHelper\nMediaPipe TFLite]
|
||||
end
|
||||
|
||||
subgraph Screen Context
|
||||
SC[ScreenCaptureService\n~6 FPS] --> VB[VisualBufferManager\nCircular Frame Buffer]
|
||||
VB --> B
|
||||
end
|
||||
|
||||
subgraph Intent Routing
|
||||
D -->|CALL / SONGS / ALARM\nNAVIGATION / WEATHER / REMINDER| E[Native Action Executor]
|
||||
D -->|OTHER / SETTINGS| F[FlexibleLlmAdapter\nStreaming REST]
|
||||
end
|
||||
|
||||
subgraph Lock State Machine
|
||||
E -->|Missing detail| LS[LockState\nMulti-Turn Collector]
|
||||
LS --> B
|
||||
end
|
||||
|
||||
subgraph TTS Layer
|
||||
F -->|Streaming tokens| SB[Sentence Boundary Detector]
|
||||
E --> SB
|
||||
SB --> TTS[TtsEngineSelector]
|
||||
TTS --> T1[EdgeTTS — Free\nNeural WebSocket]
|
||||
TTS --> T2[Google Cloud TTS]
|
||||
TTS --> T3[Offline VITS\nSherpa-ONNX]
|
||||
TTS --> T4[System Native TTS]
|
||||
end
|
||||
|
||||
subgraph Storage
|
||||
B --> DB[(Room DB\nAES-256-GCM\nEncrypted)]
|
||||
end
|
||||
|
||||
subgraph Voice Pipeline
|
||||
MIC[Microphone] --> AH[AudioHygieneProcessor\nAEC · NS · AGC]
|
||||
AH --> VAD[VadIntelligenceProcessor\nSilero VAD]
|
||||
VAD --> VSM[VoiceStateMachine\nIDLE · LISTENING\nPROCESSING · BOT_SPEAKING]
|
||||
VSM --> B
|
||||
end
|
||||
```
|
||||
|
||||
1. **User Input:** The user provides input via voice or text in the `ChatScreen`.
|
||||
2. **Input Handling (`MainViewModel`):**
|
||||
* The input is sent to the `MainViewModel`.
|
||||
* Permissions (like microphone for voice input) are checked.
|
||||
* If the input is not in English, it's translated to English. * The text is cleaned and punctuated.
|
||||
3. **Lock State Management (`MainViewModel`):**
|
||||
* The current lock state is checked.
|
||||
* If a specific lock is active (e.g., "call lock," "song lock"), the input is processed according to that lock.
|
||||
* If no lock is active or the input is negative (e.g., "cancel"), the input proceeds to general classification.
|
||||
4. **Text Classification (`TextClassifierHelper` via `MainViewModel`):**
|
||||
* The input text is classified to determine the user's intent (e.g., make a call, play a song, navigate, get weather, set alarm/reminder, or general query).
|
||||
5. **Action Execution (`CallCommand` & `MainViewModel`):**
|
||||
* Based on the classified category, the `MainViewModel` (often via a `CallCommand` utility) triggers the appropriate action:
|
||||
* **Call:** Initiates a phone call.
|
||||
* **Songs:** Plays a song.
|
||||
* **Navigation:** Starts navigation.
|
||||
* **Weather:** Fetches weather information.
|
||||
* **Alarm:** Sets an alarm.
|
||||
* **Reminder:** Sets a reminder.
|
||||
* **Other/Settings/General AI:** For other queries, or if the intent is unclear or relates to settings, the input is sent to a general AI (Groq API).
|
||||
6. **AI Response Processing (if applicable via `MainViewModel`):**
|
||||
* The response from the Groq API is received.
|
||||
* The response is processed and formatted.
|
||||
* If the original input was not in English, the AI's English response is translated back to the original language.
|
||||
7. **Updating UI (`ChatScreen`):**
|
||||
* The result of the action (e.g., confirmation of a call, song playing, navigation started, weather info, alarm/reminder set, or AI response) is added to the chat list in the `MainViewModel`.
|
||||
* The `ChatScreen` observes changes in the chat list and updates to display the conversation.
|
||||
8. **Speech Output (if enabled via `MainViewModel`):**
|
||||
* The response or confirmation is converted to speech using TextToSpeech.
|
||||
* The synthesized speech is played back to the user.
|
||||
**Design details:**
|
||||
- **Flexible LLM Adapter:** Uses templates to support any OpenAI-compatible API endpoint.
|
||||
- **Intent Verification:** Checks user queries against keyword lists to prevent incorrect system actions.
|
||||
- **Local Encryption:** All conversation history and files are encrypted before saving to disk.
|
||||
|
||||
---
|
||||
|
||||
## Request Processing Flow
|
||||
|
||||
<details>
|
||||
<summary><strong>Voice / Text Input</strong></summary>
|
||||
|
||||
When you speak or type, the app processes the audio locally:
|
||||
1. It records and cleans up the audio.
|
||||
2. It detects when you start and stop speaking.
|
||||
3. If you speak while the assistant is answering, it stops talking and listens to you.
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><strong>Audio Processing & STT</strong></summary>
|
||||
|
||||
The app converts your speech to text using either Android's built-in system, an offline voice model, or a cloud API.
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><strong>Translation & Classification</strong></summary>
|
||||
|
||||
1. If you speak a non-English language, the app translates it to English offline using Google ML Kit.
|
||||
2. It classifies the text to determine your intent (e.g., call, alarm, navigation, or general question).
|
||||
3. It double-checks keywords to make sure it doesn't run a system command by mistake.
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><strong>Native Actions & Lock States</strong></summary>
|
||||
|
||||
When the app identifies a command, it runs the matching phone action:
|
||||
- **Calls:** Searches contacts and dials the number.
|
||||
- **Music:** Searches and plays songs on YouTube.
|
||||
- **Alarms/Reminders:** Sets system alarms using natural time phrasing.
|
||||
- **Navigation:** Launches Google Maps navigation.
|
||||
- **Weather:** Gets your location and displays the forecast.
|
||||
|
||||
If a command is missing information (like a name for a call), the app will ask you for it directly on the next turn.
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><strong>LLM & Vision</strong></summary>
|
||||
|
||||
General questions go to your selected LLM. If you use screen vision or live camera sharing, the app attaches the latest captured frame from your screen or camera when you start speaking, provided your selected LLM supports images.
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><strong>Streaming TTS</strong></summary>
|
||||
|
||||
As the LLM generates a response, the app parses it sentence-by-sentence. Once a sentence is ready, the voice engine starts speaking it immediately so you don't have to wait for the full response to load.
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><strong>Conversation Storage</strong></summary>
|
||||
|
||||
All messages are encrypted and saved locally. Attached files are also encrypted before saving. Deleting a message or chat deletes all associated files permanently.
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
## Technologies Used
|
||||
|
||||
* **Kotlin:** Primary programming language.
|
||||
* **Jetpack Compose:** For building the user interface.
|
||||
* **Android Jetpack:**
|
||||
* ViewModel: For managing UI-related data in a lifecycle-conscious way.
|
||||
* **Speech Recognition & Synthesis:** Android's built-in `SpeechRecognizer` and `TextToSpeech` engines.
|
||||
* **Text Classification:** MediaPipe, utilizing a custom-trained TensorFlow Lite (TFLite) model for on-device text classification.
|
||||
* **Offline Translation:** Google ML Kit for on-device language identification and translation, enabling completely offline functionality.
|
||||
* **Networking:** OkHttp for making API calls.
|
||||
* **APIs:**
|
||||
* Groq API (for LLM, can be changed as per requirement)
|
||||
* YouTube Data API v3 (for music search)
|
||||
* **Coroutines:** For asynchronous programming.
|
||||
| Category | Technology |
|
||||
|---|---|
|
||||
| Language | Kotlin |
|
||||
| UI | Jetpack Compose, Material You |
|
||||
| Architecture | MVVM, ViewModel, Coroutines, Flow |
|
||||
| On-Device ML | MediaPipe Text Classifier, TensorFlow Lite, Sherpa-ONNX (Silero VAD, Parakeet STT, VITS TTS) |
|
||||
| Translation | Google ML Kit On-Device Translation |
|
||||
| Database | Room (encrypted via Android KeyStore + AES-256-GCM) |
|
||||
| Networking | OkHttp, Retrofit |
|
||||
| TTS | Edge TTS (WebSocket), Google Cloud TTS, Sherpa-ONNX VITS, Android TextToSpeech |
|
||||
| APIs | Groq (default LLM), YouTube Data API v3, Open-Meteo (weather) |
|
||||
| Location | GMS Fused Location Provider |
|
||||
|
||||
---
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
app/src/main/java/com/app/assistant/
|
||||
│
|
||||
├── AssistActivity.kt # Entry point for system assistant integration
|
||||
├── MainActivity.kt # Main UI activity and lifecycle management
|
||||
│
|
||||
├── api/ # HTTP clients and API services
|
||||
│
|
||||
├── camera/
|
||||
│ ├── ScreenCaptureService.kt # Captures screenshots periodically
|
||||
│ └── VisualBufferManager.kt # Stores captured frames with timestamps
|
||||
│
|
||||
├── classifier/
|
||||
│ └── TextClassifierHelper.kt # Local TFLite intent classifier
|
||||
│
|
||||
├── db/
|
||||
│ ├── AppDatabase.kt # Room database setup
|
||||
│ ├── EncryptionUtil.kt # Encryption helpers (AES-GCM + KeyStore)
|
||||
│ ├── DynamicConversationRepo # Handles encrypted database operations
|
||||
│ └── *Entity.kt # Database schemas
|
||||
│
|
||||
├── llm/
|
||||
│ ├── FlexibleLlmAdapter.kt # API adapter for various LLMs
|
||||
│ ├── ModelCapabilityProber.kt # Detects model vision capability
|
||||
│ └── LlmMessage.kt # LLM data schemas
|
||||
│
|
||||
├── repository/
|
||||
│ ├── ContactsRepository.kt # Matches contact names offline
|
||||
│ ├── SettingsRepository.kt # App settings manager
|
||||
│ └── WeatherRepository.kt # Location and weather logic
|
||||
│
|
||||
├── speech/
|
||||
│ ├── AudioHygieneProcessor.kt # Audio recording and processing
|
||||
│ ├── VadIntelligenceProcessor.kt # On-device speech detection (VAD)
|
||||
│ ├── VoiceStateMachine.kt # Manages voice states (listening, speaking)
|
||||
│ └── SpeechRecognizerManager.kt # Manages speech-to-text engines
|
||||
│
|
||||
├── translation/
|
||||
│ └── TranslatorManager.kt # Manages offline translation
|
||||
│
|
||||
├── tts/
|
||||
│ ├── TtsEngineSelector.kt # Switches between TTS engines
|
||||
│ ├── EdgeTtsApiManager.kt # Microsoft Edge TTS integration
|
||||
│ ├── GoogleTtsApiManager.kt # Google Cloud TTS integration
|
||||
│ ├── OfflineTtsManager.kt # On-device TTS integration
|
||||
│ └── NativeTtsManager.kt # Default system TTS fallback
|
||||
│
|
||||
└── ui/
|
||||
├── screen/
|
||||
│ ├── ChatScreen.kt # Main chat screen logic
|
||||
│ ├── ChatLayout.kt # Layout definitions for chat
|
||||
│ ├── ConversationItem.kt # Chat item UI elements
|
||||
│ ├── HandsFreeBar.kt # Voice UI overlay
|
||||
│ ├── SettingsScreen.kt # Settings layout and configuration
|
||||
│ └── UserInputField.kt # Text input and file picker UI
|
||||
└── theme/ # Theme, styling, and typography
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Setup
|
||||
|
||||
1. **Clone the repository:**
|
||||
2. **API Keys:**
|
||||
This project requires API keys for YouTube and Groq services.
|
||||
**Prerequisites**
|
||||
- Android Studio
|
||||
- Device or emulator (Android 8.0+ / API 26+)
|
||||
- Groq API key (or another OpenAI-compatible endpoint key)
|
||||
- YouTube Data API v3 key
|
||||
|
||||
**Option A: Using the `local.properties` file (Recommended for development)**
|
||||
You need to create a `local.properties` file in the root directory of your project (if it doesn't already exist) and add your keys there:
|
||||
```properties
|
||||
YOUTUBE_API_KEY=YOUR_YOUTUBE_API_KEY
|
||||
GROQ_API_KEY=YOUR_GROQ_API_KEY
|
||||
```
|
||||
These keys are then accessed via `BuildConfig` fields in the app.
|
||||
**Steps**
|
||||
1. Clone this repository:
|
||||
```bash
|
||||
git clone https://github.com/your-username/ai-assistant-android.git
|
||||
cd ai-assistant-android
|
||||
```
|
||||
2. Open the project in Android Studio and let files sync.
|
||||
3. Configure your API keys (see details in the API Configuration section).
|
||||
4. Run the app on your device.
|
||||
5. (Optional) Set the app as your default assistant under **Settings → Apps → Default Apps → Digital assistant app**.
|
||||
|
||||
**Option B: Entering Keys via the App UI (Recommended for quick testing/user setup)**
|
||||
Alternatively, after installing the app, you can enter the API keys directly within the application. Navigate to the **Menu** and then to the **Settings** page within the UI, and input your `YOUTUBE_API_KEY` and `GROQ_API_KEY` there.
|
||||
3. **Ensure `local.properties` is in `.gitignore`:** The `.gitignore` file should include `local.properties` to prevent your API keys from being checked into version control.
|
||||
---
|
||||
|
||||
## How to Build and Run
|
||||
## API Configuration
|
||||
|
||||
1. Open the project in Android Studio.
|
||||
2. Let Android Studio sync the project and download necessary dependencies.
|
||||
3. Ensure you have an Android device or emulator connected/running.
|
||||
4. Click the "Run" button (green play icon) in Android Studio.
|
||||
Choose one of these methods to configure your API keys:
|
||||
|
||||
## Notes
|
||||
**Option 1: Using `local.properties`**
|
||||
Add the keys to your `local.properties` file:
|
||||
```properties
|
||||
YOUTUBE_API_KEY=YOUR_YOUTUBE_API_KEY
|
||||
GROQ_API_KEY=YOUR_GROQ_API_KEY
|
||||
EDGE_TTS_SUBSCRIPTION_KEY=YOUR_EDGE_TTS_SUBSCRIPTION_KEY
|
||||
```
|
||||
Gradle will load these during the build. (Make sure this file is not committed to git).
|
||||
|
||||
* The app requests various permissions at runtime (e.g., Record Audio, Read Contacts, Call Phone, Location) as needed for its features.
|
||||
* Bluetooth SCO is managed for better compatibility with Bluetooth headsets during speech recognition.
|
||||
**Option 2: In-App Settings**
|
||||
Open the app, go to **Settings**, and paste your keys directly into the input fields. This does not require rebuilding the app.
|
||||
|
||||
**Customizing the LLM Endpoint**
|
||||
In the settings page, you can change the default URL to any OpenAI-compatible server (like Ollama, LM Studio, Anthropic, or Gemini). The app automatically checks if the custom endpoint supports images.
|
||||
|
||||
---
|
||||
|
||||
## Permissions
|
||||
|
||||
| Permission | Required For |
|
||||
|---|---|
|
||||
| `RECORD_AUDIO` | Listening to voice input |
|
||||
| `CAMERA` | Accessing the camera for live camera sharing |
|
||||
| `READ_CONTACTS` | Searching contact names to make calls |
|
||||
| `CALL_PHONE` | Making phone calls |
|
||||
| `ACCESS_FINE_LOCATION` | Checking weather and generating navigation routes |
|
||||
| `MEDIA_PROJECTION` | Recording the screen for screen vision |
|
||||
| `BLUETOOTH` / `BLUETOOTH_CONNECT` | Supporting Bluetooth headsets |
|
||||
| `POST_NOTIFICATIONS` | Showing the screen capture status bar notification |
|
||||
|
||||
---
|
||||
|
||||
## Supported Features Table
|
||||
|
||||
| Feature | Offline | Cloud | Native Android | AI / LLM |
|
||||
|---|:---:|:---:|:---:|:---:|
|
||||
| Voice activity detection (VAD) | ✅ | | | |
|
||||
| Speech-to-text (Parakeet) | ✅ | | | |
|
||||
| Speech-to-text (Cloud) | | ✅ | | |
|
||||
| Intent classification | ✅ | | | |
|
||||
| Language translation | ✅ | | | |
|
||||
| Offline TTS (VITS) | ✅ | | | |
|
||||
| Neural TTS (Edge) | | ✅ | | |
|
||||
| Encrypted storage | ✅ | | | |
|
||||
| Phone calls | | | ✅ | |
|
||||
| Alarms & reminders | | | ✅ | |
|
||||
| Google Maps navigation | | | ✅ | |
|
||||
| YouTube music playback | | ✅ | | |
|
||||
| Weather forecasts | | ✅ | | |
|
||||
| Contact matching | ✅ | | ✅ | |
|
||||
| Screen vision | | | ✅ | ✅ |
|
||||
| Live camera sharing | | | ✅ | ✅ |
|
||||
| General chat | | | | ✅ |
|
||||
| Custom LLM endpoints | | ✅ | | ✅ |
|
||||
| Image & file attachments | | | | ✅ |
|
||||
|
||||
---
|
||||
|
||||
## Privacy & Security
|
||||
|
||||
Your data stays on your device unless it is sent to your selected LLM provider.
|
||||
|
||||
**Database Encryption**
|
||||
The app generates an AES-256 encryption key inside Android's secure hardware Keystore. All messages and chat titles are encrypted before saving. Each record uses a unique initialization vector.
|
||||
|
||||
**File Encryption**
|
||||
All saved images, documents, and screenshots are encrypted before they are written to disk.
|
||||
|
||||
**Secure Deletion**
|
||||
When you delete a message or chat, the app deletes the database records and removes the associated files from disk.
|
||||
|
||||
**Voice Processing**
|
||||
Your raw voice audio is never stored or sent anywhere. Voice detection is handled entirely in memory on the device, and only the transcribed text is processed.
|
||||
|
||||
---
|
||||
|
||||
## Known Issues
|
||||
|
||||
- **Barge-In Interruption:** Sometimes the barge-in/interruption feature does not trigger correctly when speaking over the assistant. Work is underway to refine this.
|
||||
- **Translation Disabled:** The translation feature is currently disabled due to conflicts with streaming text output. A new approach is being developed.
|
||||
- **Settings Classification:** Settings-related commands (unlike Call, Play Song, etc., which work) do not have a separate action yet. Instead, they are classified under `OTHER` and sent directly to the LLM. A new approach is being worked on to support settings actions natively.
|
||||
|
||||
---
|
||||
|
||||
## Roadmap
|
||||
|
||||
- [ ] Android XR support for smart glasses
|
||||
- [ ] Wear OS support for wrist control
|
||||
- [ ] Category for sending text messages through actions
|
||||
- [ ] Automatic summary of older conversations
|
||||
- [ ] Screen capture using Accessibility Services to avoid permission prompts
|
||||
|
||||
---
|
||||
|
||||
## Contributing
|
||||
|
||||
Contributions are welcome. Please open an issue to discuss new features before making changes, or open a pull request directly for bug fixes.
|
||||
|
||||
1. Fork the repository and create a branch.
|
||||
2. Follow the project's coding style (standard Kotlin, MVVM architecture).
|
||||
3. Submit a pull request explaining your changes.
|
||||
|
||||
Make sure your changes do not store unencrypted user data or bypass security features.
|
||||
|
||||
---
|
||||
|
||||
## License
|
||||
|
||||
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
||||
Copyright (c) 2025 Sourav Anand
|
||||
This project is licensed under the [MIT License](LICENSE).
|
||||
|
||||
Copyright © 2026 Sourav Anand.
|
||||
|
|
|
|||
|
|
@ -4,7 +4,8 @@ import java.io.FileInputStream
|
|||
plugins {
|
||||
alias(libs.plugins.android.application)
|
||||
alias(libs.plugins.jetbrains.kotlin.android)
|
||||
kotlin("plugin.serialization") version "2.0.0"
|
||||
alias(libs.plugins.jetbrains.kotlin.serialization)
|
||||
kotlin("kapt")
|
||||
}
|
||||
|
||||
android {
|
||||
|
|
@ -15,8 +16,9 @@ android {
|
|||
applicationId = "com.app.assistant"
|
||||
minSdk = 24
|
||||
targetSdk = 35
|
||||
versionCode = 1
|
||||
versionName = "1.0"
|
||||
val runNumber = System.getenv("GITHUB_RUN_NUMBER")?.toIntOrNull() ?: 0
|
||||
versionCode = 2 + runNumber
|
||||
versionName = if (runNumber > 0) "2.1.$runNumber" else "2.1"
|
||||
|
||||
testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
|
||||
vectorDrawables {
|
||||
|
|
@ -39,6 +41,31 @@ android {
|
|||
buildConfigField("String", "GROQ_API_KEY",
|
||||
"\"${localProperties.getProperty("GROQ_API_KEY") ?: ""}\"" // Add quotes
|
||||
)
|
||||
|
||||
buildConfigField("String", "EDGE_TTS_SUBSCRIPTION_KEY",
|
||||
"\"${localProperties.getProperty("EDGE_TTS_SUBSCRIPTION_KEY") ?: ""}\"" // Add quotes
|
||||
)
|
||||
}
|
||||
|
||||
val keystoreFilePath = System.getenv("KEYSTORE_FILE_PATH")
|
||||
val keystorePassword = System.getenv("KEYSTORE_PASSWORD")
|
||||
val keystoreKeyAlias = System.getenv("KEYSTORE_KEY_ALIAS")
|
||||
val keystoreKeyPassword = System.getenv("KEYSTORE_KEY_PASSWORD")
|
||||
|
||||
val hasSigningConfig = !keystoreFilePath.isNullOrEmpty() &&
|
||||
!keystorePassword.isNullOrEmpty() &&
|
||||
!keystoreKeyAlias.isNullOrEmpty() &&
|
||||
!keystoreKeyPassword.isNullOrEmpty()
|
||||
|
||||
signingConfigs {
|
||||
if (hasSigningConfig) {
|
||||
create("release") {
|
||||
storeFile = file(keystoreFilePath)
|
||||
storePassword = keystorePassword
|
||||
keyAlias = keystoreKeyAlias
|
||||
keyPassword = keystoreKeyPassword
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
buildTypes {
|
||||
|
|
@ -48,8 +75,9 @@ android {
|
|||
getDefaultProguardFile("proguard-android-optimize.txt"),
|
||||
"proguard-rules.pro"
|
||||
)
|
||||
// You might want to also define buildConfigFields for release
|
||||
// if you have different keys for release builds, or if local.properties is not available in CI
|
||||
if (hasSigningConfig) {
|
||||
signingConfig = signingConfigs.getByName("release")
|
||||
}
|
||||
}
|
||||
debug {
|
||||
// BuildConfigFields from defaultConfig are inherited.
|
||||
|
|
@ -69,7 +97,7 @@ android {
|
|||
buildConfig = true // Ensure buildConfig is enabled
|
||||
}
|
||||
composeOptions {
|
||||
kotlinCompilerExtensionVersion = "1.5.1"
|
||||
kotlinCompilerExtensionVersion = libs.versions.androidxComposeCompiler.get()
|
||||
}
|
||||
packaging {
|
||||
resources {
|
||||
|
|
@ -88,29 +116,56 @@ dependencies {
|
|||
implementation(libs.androidx.ui.graphics)
|
||||
implementation(libs.androidx.ui.tooling.preview)
|
||||
implementation(libs.androidx.material3)
|
||||
implementation("androidx.compose.material3:material3:1.2.1")
|
||||
implementation("androidx.compose.material3:material3-window-size-class:1.2.1")
|
||||
implementation("androidx.compose.material3:material3-adaptive-navigation-suite:1.3.0-beta02")
|
||||
implementation("com.squareup.okhttp3:okhttp:4.12.0")
|
||||
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.3")
|
||||
implementation ("com.google.mlkit:translate:17.0.3")
|
||||
implementation ("org.commonmark:commonmark:0.24.0")
|
||||
implementation(libs.androidx.material3.window.size)
|
||||
implementation(libs.androidx.material3.adaptive.navigation.suite)
|
||||
implementation(libs.okhttp)
|
||||
constraints {
|
||||
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.3") {
|
||||
because("Kotlin 1.9.0 compatibility")
|
||||
}
|
||||
implementation("org.jetbrains.kotlinx:kotlinx-serialization-core:1.6.3") {
|
||||
because("Kotlin 1.9.0 compatibility")
|
||||
}
|
||||
}
|
||||
implementation(libs.kotlinx.serialization.json)
|
||||
implementation(libs.mlkit.translate)
|
||||
implementation(libs.mlkit.barcode.scanning)
|
||||
implementation(libs.commonmark)
|
||||
//implementation ("com.github.jeziellago:compose-markdown:0.5.4")
|
||||
implementation("io.coil-kt.coil3:coil-compose:3.0.0-rc02")
|
||||
implementation("io.coil-kt.coil3:coil-network-okhttp:3.0.0-rc02")
|
||||
implementation ("org.jetbrains.kotlin:kotlin-reflect")
|
||||
implementation(libs.coil.compose)
|
||||
implementation(libs.coil.network.okhttp)
|
||||
implementation(libs.androidx.room.runtime)
|
||||
implementation(libs.androidx.room.ktx)
|
||||
kapt(libs.androidx.room.compiler)
|
||||
implementation(libs.play.services.location)
|
||||
implementation(libs.androidx.ui.test.android)
|
||||
implementation("androidx.security:security-crypto:1.1.0")
|
||||
implementation(libs.androidx.security.crypto)
|
||||
implementation(libs.androidx.runtime.saveable)
|
||||
testImplementation(libs.junit)
|
||||
// androidTestImplementation(libs.androidx.junit)
|
||||
// androidTestImplementation(libs.androidx.espresso.core)
|
||||
// androidTestImplementation(platform(libs.androidx.compose.bom))
|
||||
// androidTestImplementation(libs.androidx.ui.test.junit4)
|
||||
androidTestImplementation(libs.androidx.junit)
|
||||
androidTestImplementation(libs.androidx.espresso.core)
|
||||
androidTestImplementation(platform(libs.androidx.compose.bom))
|
||||
androidTestImplementation(libs.androidx.ui.test.junit4)
|
||||
debugImplementation(libs.androidx.ui.tooling)
|
||||
debugImplementation(libs.androidx.ui.test.manifest)
|
||||
|
||||
//MediaPipe library
|
||||
implementation ("com.google.mediapipe:tasks-text:0.10.29")
|
||||
}
|
||||
implementation(libs.mediapipe.tasks.text)
|
||||
|
||||
// Sherpa ONNX Offline STT
|
||||
implementation(files("libs/sherpa-onnx-1.13.2.aar"))
|
||||
|
||||
// CameraX dependencies
|
||||
implementation(libs.androidx.camera.core)
|
||||
implementation(libs.androidx.camera.camera2)
|
||||
implementation(libs.androidx.camera.lifecycle)
|
||||
implementation(libs.androidx.camera.view)
|
||||
}
|
||||
|
||||
configurations.all {
|
||||
resolutionStrategy {
|
||||
force("org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.3")
|
||||
force("org.jetbrains.kotlinx:kotlinx-serialization-core:1.6.3")
|
||||
force("org.jetbrains.kotlinx:kotlinx-serialization-core-jvm:1.6.3")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
BIN
app/libs/sherpa-onnx-1.13.2.aar
Normal file
BIN
app/libs/sherpa-onnx-1.13.2.aar
Normal file
Binary file not shown.
|
|
@ -2,6 +2,10 @@
|
|||
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
xmlns:tools="http://schemas.android.com/tools" >
|
||||
|
||||
<uses-feature
|
||||
android:name="android.hardware.telephony"
|
||||
android:required="false" />
|
||||
|
||||
<uses-permission android:name="android.permission.INTERNET"/>
|
||||
<uses-permission android:name="android.permission.RECORD_AUDIO"/>
|
||||
<uses-permission android:name="android.permission.VIBRATE" />
|
||||
|
|
@ -10,8 +14,12 @@
|
|||
<uses-permission android:name="android.permission.CALL_PHONE" />
|
||||
<uses-permission android:name="android.permission.ACCESS_COARSE_LOCATION" />
|
||||
<uses-permission android:name="com.android.alarm.permission.SET_ALARM"/>
|
||||
<uses-permission android:name="android.permission.READ_CALENDAR"/>
|
||||
<uses-permission android:name="android.permission.WRITE_CALENDAR"/>
|
||||
<uses-permission android:name="android.permission.CAMERA"/>
|
||||
<uses-feature android:name="android.hardware.camera" android:required="false" />
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE"/>
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION"/>
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE"/>
|
||||
<uses-permission android:name="android.permission.POST_NOTIFICATIONS"/>
|
||||
|
||||
<queries>
|
||||
<!-- YouTube Music -->
|
||||
|
|
@ -55,14 +63,28 @@
|
|||
</intent-filter>
|
||||
</activity>
|
||||
|
||||
<!-- MainActivity (now launched from AssistActivity) -->
|
||||
<activity
|
||||
android:name=".MainActivity"
|
||||
android:exported="true"
|
||||
android:exported="false"
|
||||
android:windowSoftInputMode="adjustResize"
|
||||
android:theme="@style/Theme.TransparentActivity"
|
||||
android:showOnLockScreen="true" />
|
||||
|
||||
<provider
|
||||
android:name="androidx.core.content.FileProvider"
|
||||
android:authorities="com.app.assistant.fileprovider"
|
||||
android:exported="false"
|
||||
android:grantUriPermissions="true">
|
||||
<meta-data
|
||||
android:name="android.support.FILE_PROVIDER_PATHS"
|
||||
android:resource="@xml/file_paths" />
|
||||
</provider>
|
||||
|
||||
<service
|
||||
android:name=".camera.ScreenCaptureService"
|
||||
android:foregroundServiceType="mediaProjection|microphone"
|
||||
android:exported="false" />
|
||||
|
||||
</application>
|
||||
|
||||
</manifest>
|
||||
|
|
@ -30,7 +30,7 @@ class AssistActivity : ComponentActivity() {
|
|||
finish()
|
||||
}
|
||||
|
||||
private fun moveToMainActivity(){
|
||||
private fun moveToMainActivity() {
|
||||
val intent = Intent(this, MainActivity::class.java)
|
||||
intent.flags = Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TASK or Intent.FLAG_ACTIVITY_NO_USER_ACTION
|
||||
intent.putExtra("speak", true)
|
||||
|
|
@ -47,8 +47,9 @@ class AssistActivity : ComponentActivity() {
|
|||
} else {
|
||||
this.window.addFlags(
|
||||
WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD or
|
||||
WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED or
|
||||
WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON)
|
||||
WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED or
|
||||
WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,71 +0,0 @@
|
|||
package com.app.assistant.Classifier
|
||||
|
||||
import android.content.Context
|
||||
import android.os.SystemClock
|
||||
import android.util.Log
|
||||
import com.google.mediapipe.tasks.core.BaseOptions
|
||||
import com.google.mediapipe.tasks.text.textclassifier.TextClassifier
|
||||
import com.google.mediapipe.tasks.text.textclassifier.TextClassifierResult
|
||||
import java.util.concurrent.ScheduledThreadPoolExecutor
|
||||
|
||||
class TextClassifierHelper(
|
||||
var currentModel: String = MOBILEBERT,
|
||||
val context: Context,
|
||||
val listener: TextResultsListener
|
||||
) {
|
||||
private lateinit var textClassifier: TextClassifier
|
||||
private lateinit var executor: ScheduledThreadPoolExecutor
|
||||
|
||||
init {
|
||||
initClassifier()
|
||||
}
|
||||
|
||||
fun initClassifier() {
|
||||
val baseOptionsBuilder = BaseOptions.builder()
|
||||
.setModelAssetPath(currentModel)
|
||||
|
||||
try {
|
||||
val baseOptions = baseOptionsBuilder.build()
|
||||
val optionsBuilder = TextClassifier.TextClassifierOptions.builder()
|
||||
.setBaseOptions(baseOptions)
|
||||
val options = optionsBuilder.build()
|
||||
textClassifier = TextClassifier.createFromOptions(context, options)
|
||||
} catch (e: IllegalStateException) {
|
||||
listener.onError(
|
||||
"Text classifier failed to initialize. See error logs for " +
|
||||
"details"
|
||||
)
|
||||
Log.e(
|
||||
TAG, "Text classifier failed to load the task with error: " + e
|
||||
.message
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Run text classification using MediaPipe Text Classifier API
|
||||
fun classify(text: String, itemId: Long, loadingItemId: Long, speak: Boolean) {
|
||||
executor = ScheduledThreadPoolExecutor(1)
|
||||
|
||||
executor.execute {
|
||||
// inferenceTime is the amount of time, in milliseconds, that it takes to
|
||||
// classify the input text.
|
||||
var inferenceTime = SystemClock.uptimeMillis()
|
||||
|
||||
val results = textClassifier.classify(text)
|
||||
|
||||
inferenceTime = SystemClock.uptimeMillis() - inferenceTime
|
||||
|
||||
listener.onResult(results, inferenceTime, text, itemId, loadingItemId, speak)
|
||||
}
|
||||
}
|
||||
|
||||
interface TextResultsListener {
|
||||
fun onError(error: String)
|
||||
fun onResult(results: TextClassifierResult, inferenceTime: Long, inputText: String, itemId: Long, loadingItemId: Long, speak: Boolean)
|
||||
}
|
||||
|
||||
companion object {
|
||||
const val TAG = "TextClassifierHelper"
|
||||
const val MOBILEBERT = "model.tflite"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,30 +1,506 @@
|
|||
package com.app.assistant
|
||||
|
||||
import android.Manifest
|
||||
import android.app.KeyguardManager
|
||||
import android.content.ComponentCallbacks2
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.content.IntentSender
|
||||
import android.content.pm.PackageManager
|
||||
import android.content.res.Configuration
|
||||
import android.location.LocationManager
|
||||
import android.media.AudioDeviceInfo
|
||||
import android.media.AudioManager
|
||||
import android.os.Build
|
||||
import android.os.Bundle
|
||||
import android.os.Looper
|
||||
import android.os.Vibrator
|
||||
import android.os.VibrationEffect
|
||||
import android.speech.RecognitionListener
|
||||
import android.speech.RecognizerIntent
|
||||
import android.speech.SpeechRecognizer
|
||||
import android.speech.tts.TextToSpeech
|
||||
import android.speech.tts.UtteranceProgressListener
|
||||
import android.util.Log
|
||||
import android.view.WindowManager
|
||||
import android.widget.Toast
|
||||
import androidx.activity.ComponentActivity
|
||||
import androidx.activity.compose.setContent
|
||||
import androidx.activity.enableEdgeToEdge
|
||||
import androidx.activity.viewModels
|
||||
import androidx.core.app.ActivityCompat
|
||||
import androidx.core.content.ContextCompat
|
||||
import androidx.lifecycle.Lifecycle
|
||||
import androidx.lifecycle.lifecycleScope
|
||||
import androidx.lifecycle.repeatOnLifecycle
|
||||
import com.app.assistant.ui.screen.SetupUI
|
||||
import com.app.assistant.viewmodel.MainViewModel
|
||||
import com.app.assistant.viewmodel.MainViewModelFactory
|
||||
import com.app.assistant.viewmodel.SettingsViewModel
|
||||
import com.app.assistant.viewmodel.SettingsViewModelFactory
|
||||
import com.app.assistant.viewmodel.UIEvent
|
||||
import com.app.assistant.viewmodel.onLocationFailed
|
||||
import com.app.assistant.viewmodel.onLocationReceived
|
||||
import com.google.android.gms.common.api.ResolvableApiException
|
||||
import com.google.android.gms.location.LocationAvailability
|
||||
import com.google.android.gms.location.LocationCallback
|
||||
import com.google.android.gms.location.LocationRequest
|
||||
import com.google.android.gms.location.LocationResult
|
||||
import com.google.android.gms.location.LocationServices
|
||||
import com.google.android.gms.location.LocationSettingsRequest
|
||||
import com.google.android.gms.location.LocationSettingsResponse
|
||||
import com.google.android.gms.location.Priority
|
||||
import com.google.android.gms.location.SettingsClient
|
||||
import com.google.android.gms.tasks.Task
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.launch
|
||||
import java.util.Locale
|
||||
|
||||
class MainActivity : ComponentActivity() {
|
||||
private val viewModel: MainViewModel by viewModels{
|
||||
private val viewModel: MainViewModel by viewModels {
|
||||
MainViewModelFactory(application, intent.getBooleanExtra("speak", false))
|
||||
}
|
||||
|
||||
private val settingsViewModel: SettingsViewModel by viewModels {
|
||||
SettingsViewModelFactory(
|
||||
com.app.assistant.repository.SettingsRepository(application),
|
||||
com.app.assistant.viewmodel.MainViewModelFactory.okHttpClient
|
||||
)
|
||||
}
|
||||
|
||||
private lateinit var textToSpeechManager: com.app.assistant.tts.TtsManager
|
||||
private lateinit var speechRecognizerManager: com.app.assistant.speech.SpeechRecognizerManager
|
||||
|
||||
private var isScreenServiceRunning = false
|
||||
private var isActivityStarted = false
|
||||
private var activeCollectionJob: kotlinx.coroutines.Job? = null
|
||||
|
||||
private val screenCaptureLauncher = registerForActivityResult(
|
||||
androidx.activity.result.contract.ActivityResultContracts.StartActivityForResult()
|
||||
) { result ->
|
||||
if (result.resultCode == RESULT_OK && result.data != null) {
|
||||
val serviceIntent = Intent(this, com.app.assistant.camera.ScreenCaptureService::class.java).apply {
|
||||
action = com.app.assistant.camera.ScreenCaptureService.ACTION_START
|
||||
putExtra(com.app.assistant.camera.ScreenCaptureService.EXTRA_RESULT_CODE, result.resultCode)
|
||||
putExtra(com.app.assistant.camera.ScreenCaptureService.EXTRA_RESULT_DATA, result.data)
|
||||
}
|
||||
com.app.assistant.camera.ScreenCaptureServiceHelper.isMicMuted = viewModel.isMicMuted.value
|
||||
com.app.assistant.camera.ScreenCaptureServiceHelper.isHandsFreeActive = viewModel.isHandsFreeModeActive.value
|
||||
ContextCompat.startForegroundService(this, serviceIntent)
|
||||
isScreenServiceRunning = true
|
||||
viewModel.setScreenModeActive(true)
|
||||
} else {
|
||||
Toast.makeText(this, "Screen capture permission denied", Toast.LENGTH_SHORT).show()
|
||||
isScreenServiceRunning = false
|
||||
viewModel.setScreenModeActive(false)
|
||||
}
|
||||
}
|
||||
|
||||
private fun startScreenCaptureFlow() {
|
||||
val mediaProjectionManager = getSystemService(Context.MEDIA_PROJECTION_SERVICE) as android.media.projection.MediaProjectionManager
|
||||
try {
|
||||
val intent = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) {
|
||||
val config = android.media.projection.MediaProjectionConfig.createConfigForDefaultDisplay()
|
||||
mediaProjectionManager.createScreenCaptureIntent(config)
|
||||
} else {
|
||||
mediaProjectionManager.createScreenCaptureIntent()
|
||||
}
|
||||
screenCaptureLauncher.launch(intent)
|
||||
} catch (e: Exception) {
|
||||
Log.e("MainActivity", "Failed to launch screen capture intent", e)
|
||||
Toast.makeText(this, "Screen capture not supported or failed to launch", Toast.LENGTH_SHORT).show()
|
||||
}
|
||||
}
|
||||
|
||||
private fun stopScreenCaptureService() {
|
||||
val serviceIntent = Intent(this, com.app.assistant.camera.ScreenCaptureService::class.java).apply {
|
||||
action = com.app.assistant.camera.ScreenCaptureService.ACTION_STOP
|
||||
}
|
||||
startService(serviceIntent)
|
||||
}
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
super.onCreate(savedInstanceState)
|
||||
allowOnLockScreen()
|
||||
enableEdgeToEdge()
|
||||
|
||||
textToSpeechManager = com.app.assistant.tts.TtsEngineSelector(
|
||||
context = this,
|
||||
settingsRepository = com.app.assistant.repository.SettingsRepository(application)
|
||||
) { isSpeaking ->
|
||||
viewModel.setSpeaking(isSpeaking)
|
||||
if (::speechRecognizerManager.isInitialized) {
|
||||
speechRecognizerManager.isTtsSpeaking = isSpeaking
|
||||
}
|
||||
}
|
||||
speechRecognizerManager = com.app.assistant.speech.SpeechRecognizerManager(this, lifecycleScope)
|
||||
speechRecognizerManager.preLoadModelAsync()
|
||||
speechRecognizerManager.shouldIgnoreUiHidden = {
|
||||
viewModel.isScreenModeActive.value
|
||||
}
|
||||
|
||||
com.app.assistant.camera.ScreenCaptureServiceHelper.onServiceStopped = {
|
||||
runOnUiThread {
|
||||
isScreenServiceRunning = false
|
||||
viewModel.setScreenModeActive(false)
|
||||
}
|
||||
}
|
||||
com.app.assistant.camera.ScreenCaptureServiceHelper.onToggleMuteRequested = {
|
||||
runOnUiThread {
|
||||
viewModel.toggleMicMute()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
lifecycleScope.launch {
|
||||
viewModel.isScreenModeActive.collect { active ->
|
||||
if (active) {
|
||||
if (!isScreenServiceRunning) {
|
||||
startScreenCaptureFlow()
|
||||
}
|
||||
} else {
|
||||
if (isScreenServiceRunning) {
|
||||
isScreenServiceRunning = false
|
||||
stopScreenCaptureService()
|
||||
}
|
||||
}
|
||||
updateCollectionState()
|
||||
}
|
||||
}
|
||||
|
||||
setContent {
|
||||
SetupUI(viewModel)
|
||||
SetupUI(
|
||||
viewModel = viewModel,
|
||||
settingsViewModel = settingsViewModel,
|
||||
onToggleVisionMode = {
|
||||
toggleVisionModeWithPermissionCheck()
|
||||
},
|
||||
onToggleScreenMode = {
|
||||
toggleScreenModeWithPermissionCheck()
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override fun onStart() {
|
||||
super.onStart()
|
||||
isActivityStarted = true
|
||||
updateCollectionState()
|
||||
}
|
||||
|
||||
override fun onStop() {
|
||||
super.onStop()
|
||||
isActivityStarted = false
|
||||
updateCollectionState()
|
||||
}
|
||||
|
||||
private fun updateCollectionState() {
|
||||
val shouldCollect = isActivityStarted || viewModel.isScreenModeActive.value
|
||||
if (shouldCollect) {
|
||||
startActiveCollection()
|
||||
} else {
|
||||
stopActiveCollection()
|
||||
}
|
||||
}
|
||||
|
||||
private fun startActiveCollection() {
|
||||
if (activeCollectionJob != null) return
|
||||
|
||||
Log.d("MainActivity", "Starting active flow collection (Activity started: $isActivityStarted, Screen mode active: ${viewModel.isScreenModeActive.value})")
|
||||
activeCollectionJob = lifecycleScope.launch {
|
||||
// uiEvent collection
|
||||
launch {
|
||||
viewModel.uiEvent.collect { event ->
|
||||
handleUiEvent(event)
|
||||
}
|
||||
}
|
||||
|
||||
// isMicMuted collection
|
||||
launch {
|
||||
viewModel.isMicMuted.collect { muted ->
|
||||
if (::speechRecognizerManager.isInitialized) {
|
||||
speechRecognizerManager.setMicMuted(muted)
|
||||
}
|
||||
com.app.assistant.camera.ScreenCaptureServiceHelper.isMicMuted = muted
|
||||
com.app.assistant.camera.ScreenCaptureServiceHelper.onStateChanged?.invoke()
|
||||
}
|
||||
}
|
||||
|
||||
// isHandsFreeModeActive collection
|
||||
launch {
|
||||
var wasHandsFreeActive = false
|
||||
viewModel.isHandsFreeModeActive.collect { active ->
|
||||
com.app.assistant.camera.ScreenCaptureServiceHelper.isHandsFreeActive = active
|
||||
com.app.assistant.camera.ScreenCaptureServiceHelper.onStateChanged?.invoke()
|
||||
|
||||
if (active) {
|
||||
wasHandsFreeActive = true
|
||||
if (::speechRecognizerManager.isInitialized && !speechRecognizerManager.isListeningActive()) {
|
||||
startSpeechRecognition(isHandsFree = true)
|
||||
}
|
||||
} else {
|
||||
if (wasHandsFreeActive) {
|
||||
wasHandsFreeActive = false
|
||||
speechRecognizerManager.stop()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun stopActiveCollection() {
|
||||
if (activeCollectionJob == null) return
|
||||
Log.d("MainActivity", "Stopping active flow collection")
|
||||
activeCollectionJob?.cancel()
|
||||
activeCollectionJob = null
|
||||
}
|
||||
|
||||
private fun handleUiEvent(event: UIEvent) {
|
||||
when (event) {
|
||||
is UIEvent.RequestPermissions -> {
|
||||
ActivityCompat.requestPermissions(this@MainActivity, event.permissions, event.requestCode)
|
||||
}
|
||||
|
||||
is UIEvent.StartIntent -> {
|
||||
try {
|
||||
val intent = event.intent
|
||||
if (intent.action == Intent.ACTION_CALL) {
|
||||
if (!isExternalAudioDeviceConnected()) {
|
||||
intent.putExtra("android.telecom.extra.START_CALL_WITH_SPEAKERPHONE", true)
|
||||
}
|
||||
}
|
||||
startActivity(intent)
|
||||
} catch (e: Exception) {
|
||||
Log.e("MainActivity", "Error starting intent", e)
|
||||
}
|
||||
}
|
||||
|
||||
is UIEvent.ShowToast -> {
|
||||
Toast.makeText(this@MainActivity, event.message, Toast.LENGTH_SHORT).show()
|
||||
}
|
||||
|
||||
is UIEvent.SpeakText -> {
|
||||
if (!viewModel.isListening.value && !viewModel.isVoiceProcessing.value) {
|
||||
textToSpeechManager.speak(event.text, event.queueMode)
|
||||
}
|
||||
}
|
||||
|
||||
is UIEvent.StopSpeaking -> {
|
||||
textToSpeechManager.stop()
|
||||
}
|
||||
|
||||
is UIEvent.StartSpeechRecognition -> {
|
||||
startSpeechRecognition(isHandsFree = viewModel.isHandsFreeModeActive.value)
|
||||
}
|
||||
|
||||
is UIEvent.StopSpeechRecognition -> {
|
||||
speechRecognizerManager.stop()
|
||||
}
|
||||
|
||||
is UIEvent.GetLocationForWeather -> {
|
||||
getCurrentLocationForWeather(event)
|
||||
}
|
||||
|
||||
is UIEvent.ResolveLocationSettings -> {
|
||||
try {
|
||||
event.exception.startResolutionForResult(this@MainActivity, 104)
|
||||
} catch (sendEx: IntentSender.SendIntentException) {
|
||||
Log.e("MainActivity", "Error showing location settings dialog: ${sendEx.message}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun toggleScreenModeWithPermissionCheck() {
|
||||
if (viewModel.isScreenModeActive.value) {
|
||||
viewModel.setScreenModeActive(false)
|
||||
} else {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
|
||||
if (ContextCompat.checkSelfPermission(this, Manifest.permission.POST_NOTIFICATIONS) != PackageManager.PERMISSION_GRANTED) {
|
||||
ActivityCompat.requestPermissions(this, arrayOf(Manifest.permission.POST_NOTIFICATIONS), 103)
|
||||
return
|
||||
}
|
||||
}
|
||||
startScreenCaptureFlow()
|
||||
}
|
||||
}
|
||||
|
||||
private fun startSpeechRecognition(isHandsFree: Boolean = false) {
|
||||
if (ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) {
|
||||
ActivityCompat.requestPermissions(this, arrayOf(Manifest.permission.RECORD_AUDIO), 101)
|
||||
return
|
||||
}
|
||||
|
||||
if (textToSpeechManager.isSpeaking()) {
|
||||
textToSpeechManager.stop()
|
||||
}
|
||||
|
||||
speechRecognizerManager.startListening(
|
||||
isHandsFree = isHandsFree,
|
||||
listener = object : com.app.assistant.speech.SpeechRecognizerManager.SpeechListener {
|
||||
override fun onReadyForSpeech() {
|
||||
viewModel.setMicReady(true)
|
||||
}
|
||||
|
||||
override fun onAudioLevelChanged(level: Float) {
|
||||
viewModel.setAudioAmplitude(level)
|
||||
}
|
||||
|
||||
override fun onBeginningOfSpeech() {
|
||||
viewModel.setListening(true)
|
||||
viewModel.setVoiceProcessing(false)
|
||||
if (textToSpeechManager.isSpeaking()) {
|
||||
textToSpeechManager.stop()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onEndOfSpeech() {
|
||||
viewModel.setListening(false)
|
||||
viewModel.setVoiceProcessing(true)
|
||||
viewModel.setAudioAmplitude(0f)
|
||||
triggerVibration()
|
||||
}
|
||||
|
||||
override fun onError(errorCode: Int) {
|
||||
viewModel.setListening(false)
|
||||
viewModel.setVoiceProcessing(false)
|
||||
viewModel.setMicReady(false)
|
||||
viewModel.setAudioAmplitude(0f)
|
||||
if (viewModel.isHandsFreeModeActive.value) {
|
||||
lifecycleScope.launch {
|
||||
delay(500)
|
||||
if (viewModel.isHandsFreeModeActive.value) {
|
||||
startSpeechRecognition(isHandsFree = true)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun onResults(recognizedText: String) {
|
||||
viewModel.setListening(false)
|
||||
viewModel.setVoiceProcessing(false)
|
||||
viewModel.setAudioAmplitude(0f)
|
||||
if (!viewModel.isHandsFreeModeActive.value) {
|
||||
viewModel.setMicReady(false)
|
||||
}
|
||||
if (recognizedText.isNotBlank()) {
|
||||
if (viewModel.isVisionModeActive.value || viewModel.isScreenModeActive.value) {
|
||||
val startTimestamp = speechRecognizerManager.speechStartTimestamp
|
||||
viewModel.onSpeechRecognizedWithVision(recognizedText, startTimestamp)
|
||||
} else {
|
||||
viewModel.onSpeechRecognized(recognizedText)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun onPartialResults(recognizedText: String) {
|
||||
viewModel.onSpeechPartialResult(recognizedText)
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
private fun getCurrentLocationForWeather(event: UIEvent.GetLocationForWeather) {
|
||||
val locationManager = getSystemService(Context.LOCATION_SERVICE) as LocationManager
|
||||
val gpsEnabled = locationManager.isProviderEnabled(LocationManager.GPS_PROVIDER)
|
||||
|
||||
if (!gpsEnabled) {
|
||||
promptEnableLocation()
|
||||
viewModel.onLocationFailed(event.loadingItemId, event.speak, "GPS_OFF")
|
||||
} else {
|
||||
val fusedLocationClient = LocationServices.getFusedLocationProviderClient(this)
|
||||
val locationRequest =
|
||||
LocationRequest
|
||||
.Builder(Priority.PRIORITY_HIGH_ACCURACY, 1000)
|
||||
.setWaitForAccurateLocation(true)
|
||||
.setMaxUpdates(1)
|
||||
.build()
|
||||
|
||||
val locationCallback =
|
||||
object : LocationCallback() {
|
||||
override fun onLocationResult(locationResult: LocationResult) {
|
||||
val location = locationResult.lastLocation
|
||||
if (location != null) {
|
||||
viewModel.onLocationReceived(
|
||||
location.latitude,
|
||||
location.longitude,
|
||||
event.itemId,
|
||||
event.loadingItemId,
|
||||
event.speak,
|
||||
event.categoryName,
|
||||
event.prompt,
|
||||
)
|
||||
} else {
|
||||
viewModel.onLocationFailed(event.loadingItemId, event.speak, "UNAVAILABLE")
|
||||
}
|
||||
fusedLocationClient.removeLocationUpdates(this)
|
||||
}
|
||||
|
||||
override fun onLocationAvailability(locationAvailability: LocationAvailability) {
|
||||
if (!locationAvailability.isLocationAvailable) {
|
||||
viewModel.onLocationFailed(event.loadingItemId, event.speak, "SUGGEST_CITY")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val permissionGranted =
|
||||
ActivityCompat.checkSelfPermission(
|
||||
this,
|
||||
Manifest.permission.ACCESS_COARSE_LOCATION,
|
||||
) == PackageManager.PERMISSION_GRANTED
|
||||
if (permissionGranted) {
|
||||
fusedLocationClient.requestLocationUpdates(locationRequest, locationCallback, Looper.getMainLooper())
|
||||
} else {
|
||||
viewModel.onLocationFailed(event.loadingItemId, event.speak, "PERMISSION")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun promptEnableLocation() {
|
||||
val locationRequest =
|
||||
LocationRequest
|
||||
.Builder(100, 10000)
|
||||
.setMinUpdateIntervalMillis(5000)
|
||||
.build()
|
||||
|
||||
val builder = LocationSettingsRequest.Builder().addLocationRequest(locationRequest)
|
||||
val settingsClient: SettingsClient = LocationServices.getSettingsClient(this)
|
||||
val task: Task<LocationSettingsResponse> = settingsClient.checkLocationSettings(builder.build())
|
||||
|
||||
task.addOnFailureListener { exception ->
|
||||
if (exception is ResolvableApiException) {
|
||||
try {
|
||||
exception.startResolutionForResult(this@MainActivity, 104)
|
||||
} catch (sendEx: IntentSender.SendIntentException) {
|
||||
Log.d("Location", "Error showing location settings dialog: ${sendEx.message}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
private fun triggerVibration() {
|
||||
try {
|
||||
val vibrator = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
|
||||
val manager = getSystemService(Context.VIBRATOR_MANAGER_SERVICE) as? android.os.VibratorManager
|
||||
manager?.defaultVibrator ?: (getSystemService(Context.VIBRATOR_SERVICE) as? Vibrator)
|
||||
} else {
|
||||
@Suppress("DEPRECATION")
|
||||
getSystemService(Context.VIBRATOR_SERVICE) as? Vibrator
|
||||
}
|
||||
|
||||
if (vibrator?.hasVibrator() == true) {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
|
||||
vibrator.vibrate(VibrationEffect.createOneShot(100, VibrationEffect.DEFAULT_AMPLITUDE))
|
||||
} else {
|
||||
@Suppress("DEPRECATION")
|
||||
vibrator.vibrate(100)
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("MainActivity", "Vibration failed", e)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -35,7 +511,7 @@ class MainActivity : ComponentActivity() {
|
|||
} else {
|
||||
window.addFlags(
|
||||
WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED or
|
||||
WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON
|
||||
WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON,
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -46,13 +522,87 @@ class MainActivity : ComponentActivity() {
|
|||
keyguardManager.requestDismissKeyguard(this, null)
|
||||
}
|
||||
}
|
||||
}catch (ex: Exception){
|
||||
} catch (ex: Exception) {
|
||||
ex.message?.let { Log.d("Exception Occurred", it) }
|
||||
}
|
||||
}
|
||||
|
||||
private fun toggleVisionModeWithPermissionCheck() {
|
||||
if (ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) {
|
||||
ActivityCompat.requestPermissions(this, arrayOf(Manifest.permission.CAMERA), 102)
|
||||
} else {
|
||||
viewModel.toggleVisionMode()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onRequestPermissionsResult(requestCode: Int, permissions: Array<String>, grantResults: IntArray) {
|
||||
super.onRequestPermissionsResult(requestCode, permissions, grantResults)
|
||||
when (requestCode) {
|
||||
101 -> {
|
||||
if (grantResults.isNotEmpty() && grantResults[0] == PackageManager.PERMISSION_GRANTED) {
|
||||
startSpeechRecognition()
|
||||
} else {
|
||||
Toast.makeText(this, "Audio permission is required for speech recognition", Toast.LENGTH_SHORT).show()
|
||||
}
|
||||
}
|
||||
102 -> {
|
||||
if (grantResults.isNotEmpty() && grantResults[0] == PackageManager.PERMISSION_GRANTED) {
|
||||
viewModel.toggleVisionMode()
|
||||
} else {
|
||||
Toast.makeText(this, "Camera permission is required for Vision Mode", Toast.LENGTH_SHORT).show()
|
||||
}
|
||||
}
|
||||
103 -> {
|
||||
if (grantResults.isNotEmpty() && grantResults[0] == PackageManager.PERMISSION_GRANTED) {
|
||||
startScreenCaptureFlow()
|
||||
} else {
|
||||
Toast.makeText(this, "Notification permission is required to show controls in the status bar", Toast.LENGTH_LONG).show()
|
||||
startScreenCaptureFlow()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun isExternalAudioDeviceConnected(): Boolean {
|
||||
val audioManager = getSystemService(Context.AUDIO_SERVICE) as? AudioManager ?: return false
|
||||
val devices = audioManager.getDevices(AudioManager.GET_DEVICES_OUTPUTS)
|
||||
for (device in devices) {
|
||||
val type = device.type
|
||||
if (type == AudioDeviceInfo.TYPE_BLUETOOTH_A2DP ||
|
||||
type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO ||
|
||||
type == AudioDeviceInfo.TYPE_WIRED_HEADSET ||
|
||||
type == AudioDeviceInfo.TYPE_WIRED_HEADPHONES ||
|
||||
type == AudioDeviceInfo.TYPE_USB_HEADSET
|
||||
) {
|
||||
return true
|
||||
}
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
|
||||
if (type == AudioDeviceInfo.TYPE_BLE_HEADSET) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
|
||||
if (type == AudioDeviceInfo.TYPE_BLE_SPEAKER ||
|
||||
type == AudioDeviceInfo.TYPE_BLE_BROADCAST
|
||||
) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
override fun onDestroy() {
|
||||
super.onDestroy()
|
||||
viewModel.shutdownResources()
|
||||
com.app.assistant.camera.ScreenCaptureServiceHelper.onServiceStopped = null
|
||||
com.app.assistant.camera.ScreenCaptureServiceHelper.onToggleMuteRequested = null
|
||||
|
||||
if (isFinishing) {
|
||||
if (isScreenServiceRunning) {
|
||||
stopScreenCaptureService()
|
||||
}
|
||||
}
|
||||
textToSpeechManager.shutdown()
|
||||
speechRecognizerManager.destroy()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,412 @@
|
|||
package com.app.assistant.camera
|
||||
|
||||
import android.app.Notification
|
||||
import android.app.NotificationChannel
|
||||
import android.app.NotificationManager
|
||||
import android.app.Service
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.content.pm.ServiceInfo
|
||||
import android.graphics.Bitmap
|
||||
import android.graphics.PixelFormat
|
||||
import android.hardware.display.DisplayManager
|
||||
import android.hardware.display.VirtualDisplay
|
||||
import android.media.ImageReader
|
||||
import android.media.projection.MediaProjection
|
||||
import android.media.projection.MediaProjectionManager
|
||||
import android.os.Build
|
||||
import android.os.Handler
|
||||
import android.os.HandlerThread
|
||||
import android.os.IBinder
|
||||
import android.app.PendingIntent
|
||||
import android.util.Log
|
||||
import androidx.core.app.NotificationCompat
|
||||
import com.app.assistant.R
|
||||
import android.widget.RemoteViews
|
||||
import android.util.TypedValue
|
||||
import android.view.ContextThemeWrapper
|
||||
import android.content.res.ColorStateList
|
||||
import android.graphics.Color
|
||||
import androidx.core.graphics.ColorUtils
|
||||
import java.io.ByteArrayOutputStream
|
||||
import java.nio.ByteBuffer
|
||||
|
||||
object ScreenCaptureServiceHelper {
|
||||
var onServiceStopped: (() -> Unit)? = null
|
||||
var onToggleMuteRequested: (() -> Unit)? = null
|
||||
|
||||
|
||||
@Volatile
|
||||
var isMicMuted: Boolean = false
|
||||
@Volatile
|
||||
var isHandsFreeActive: Boolean = false
|
||||
|
||||
var onStateChanged: (() -> Unit)? = null
|
||||
}
|
||||
|
||||
class ScreenCaptureService : Service() {
|
||||
|
||||
companion object {
|
||||
const val ACTION_START = "com.app.assistant.action.START_SCREEN_CAPTURE"
|
||||
const val ACTION_STOP = "com.app.assistant.action.STOP_SCREEN_CAPTURE"
|
||||
const val ACTION_TOGGLE_MUTE = "com.app.assistant.action.TOGGLE_MUTE"
|
||||
|
||||
const val EXTRA_RESULT_CODE = "extra_result_code"
|
||||
const val EXTRA_RESULT_DATA = "extra_result_data"
|
||||
|
||||
private const val NOTIFICATION_ID = 5001
|
||||
private const val CHANNEL_ID = "screen_capture_channel"
|
||||
private const val TAG = "ScreenCaptureService"
|
||||
}
|
||||
|
||||
private var mediaProjection: MediaProjection? = null
|
||||
private var virtualDisplay: VirtualDisplay? = null
|
||||
private var imageReader: ImageReader? = null
|
||||
private var handlerThread: HandlerThread? = null
|
||||
private var backgroundHandler: Handler? = null
|
||||
private var lastFrameTime = 0L
|
||||
|
||||
override fun onBind(intent: Intent?): IBinder? = null
|
||||
|
||||
override fun onCreate() {
|
||||
super.onCreate()
|
||||
handlerThread = HandlerThread("ScreenCaptureThread").apply {
|
||||
start()
|
||||
}
|
||||
backgroundHandler = Handler(handlerThread!!.looper)
|
||||
|
||||
ScreenCaptureServiceHelper.onStateChanged = {
|
||||
val manager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
|
||||
manager.notify(NOTIFICATION_ID, buildNotification())
|
||||
}
|
||||
}
|
||||
|
||||
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
|
||||
val action = intent?.action
|
||||
if (action == ACTION_STOP) {
|
||||
stopServiceInternal()
|
||||
return START_NOT_STICKY
|
||||
}
|
||||
|
||||
if (action == ACTION_TOGGLE_MUTE) {
|
||||
ScreenCaptureServiceHelper.onToggleMuteRequested?.invoke()
|
||||
return START_NOT_STICKY
|
||||
}
|
||||
|
||||
|
||||
if (action == ACTION_START) {
|
||||
val resultCode = intent.getIntExtra(EXTRA_RESULT_CODE, 0)
|
||||
val resultData = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
|
||||
intent.getParcelableExtra(EXTRA_RESULT_DATA, Intent::class.java)
|
||||
} else {
|
||||
@Suppress("DEPRECATION")
|
||||
intent.getParcelableExtra(EXTRA_RESULT_DATA)
|
||||
}
|
||||
|
||||
if (resultCode != 0 && resultData != null) {
|
||||
startForegroundNotification()
|
||||
startScreenCapture(resultCode, resultData)
|
||||
} else {
|
||||
Log.e(TAG, "Invalid result code or data, stopping service")
|
||||
stopSelf()
|
||||
}
|
||||
}
|
||||
|
||||
return START_NOT_STICKY
|
||||
}
|
||||
|
||||
private fun startForegroundNotification() {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
|
||||
val channel = NotificationChannel(
|
||||
CHANNEL_ID,
|
||||
"Screen Capture Service",
|
||||
NotificationManager.IMPORTANCE_LOW
|
||||
).apply {
|
||||
description = "Notification for Screen Share Mode"
|
||||
}
|
||||
val manager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
|
||||
manager.createNotificationChannel(channel)
|
||||
}
|
||||
|
||||
val notification = buildNotification()
|
||||
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
|
||||
startForeground(
|
||||
NOTIFICATION_ID,
|
||||
notification,
|
||||
ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE
|
||||
)
|
||||
} else {
|
||||
startForeground(NOTIFICATION_ID, notification)
|
||||
}
|
||||
}
|
||||
|
||||
private fun getThemeColor(attr: Int, defaultColor: Int): Int {
|
||||
val typedValue = TypedValue()
|
||||
val themeContext = ContextThemeWrapper(this, R.style.Theme_Assistant)
|
||||
if (themeContext.theme.resolveAttribute(attr, typedValue, true)) {
|
||||
return typedValue.data
|
||||
}
|
||||
return defaultColor
|
||||
}
|
||||
|
||||
private fun buildNotification(): Notification {
|
||||
val isMuted = ScreenCaptureServiceHelper.isMicMuted
|
||||
val isHandsFree = ScreenCaptureServiceHelper.isHandsFreeActive
|
||||
|
||||
val muteText = if (isMuted) "Unmute" else "Mute"
|
||||
val muteIcon = if (isMuted) R.drawable.ic_mic_off else R.drawable.ic_mic
|
||||
|
||||
|
||||
val statusText = when {
|
||||
isMuted -> "Microphone Muted"
|
||||
isHandsFree -> "Hands-Free Active"
|
||||
else -> "Capturing screen for interactions"
|
||||
}
|
||||
|
||||
val flags = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
|
||||
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
|
||||
} else {
|
||||
PendingIntent.FLAG_UPDATE_CURRENT
|
||||
}
|
||||
|
||||
val stopIntent = Intent(this, ScreenCaptureService::class.java).apply {
|
||||
action = ACTION_STOP
|
||||
}
|
||||
val stopPendingIntent = PendingIntent.getService(this, 1, stopIntent, flags)
|
||||
|
||||
val muteIntent = Intent(this, ScreenCaptureService::class.java).apply {
|
||||
action = ACTION_TOGGLE_MUTE
|
||||
}
|
||||
val mutePendingIntent = PendingIntent.getService(this, 2, muteIntent, flags)
|
||||
|
||||
|
||||
// Resolve wallpaper-based Material You accent or app theme accent
|
||||
val dynamicPrimary = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
|
||||
getColor(android.R.color.system_accent1_600)
|
||||
} else {
|
||||
getThemeColor(android.R.attr.colorAccent, 0xFF6650a4.toInt())
|
||||
}
|
||||
|
||||
// Determine dark mode or light mode from configuration
|
||||
val isDarkMode = (resources.configuration.uiMode and android.content.res.Configuration.UI_MODE_NIGHT_MASK) == android.content.res.Configuration.UI_MODE_NIGHT_YES
|
||||
|
||||
// Determine base text and accent colors depending on dark/light configuration
|
||||
val primaryTextColor = if (isDarkMode) Color.WHITE else Color.BLACK
|
||||
val secondaryTextColor = if (isDarkMode) 0xFFB0B0B0.toInt() else 0xFF666666.toInt()
|
||||
val normalTint = if (isDarkMode) 0x1AFFFFFF else 0x1A000000
|
||||
val activeTint = ColorUtils.setAlphaComponent(dynamicPrimary, if (isDarkMode) 0x66 else 0x4D) // higher opacity in dark mode for better visibility
|
||||
|
||||
val collapsedLayout = RemoteViews(packageName, R.layout.notification_screen_capture_collapsed).apply {
|
||||
setTextViewText(R.id.tv_status_text, statusText)
|
||||
setOnClickPendingIntent(R.id.btn_stop_compact, stopPendingIntent)
|
||||
|
||||
// Apply text and stop icon/background colors dynamically
|
||||
setTextColor(R.id.tv_title, primaryTextColor)
|
||||
setTextColor(R.id.tv_status_text, secondaryTextColor)
|
||||
setInt(R.id.btn_stop_compact, "setColorFilter", primaryTextColor)
|
||||
setColorStateList(R.id.btn_stop_compact, "setBackgroundTintList", ColorStateList.valueOf(normalTint))
|
||||
}
|
||||
|
||||
val expandedLayout = RemoteViews(packageName, R.layout.notification_screen_capture_expanded).apply {
|
||||
setTextViewText(R.id.tv_status_text, statusText)
|
||||
|
||||
// Apply header text colors dynamically
|
||||
setTextColor(R.id.tv_title, primaryTextColor)
|
||||
setTextColor(R.id.tv_status_text, secondaryTextColor)
|
||||
|
||||
// Setup Mute Button
|
||||
setImageViewResource(R.id.iv_mute_icon, muteIcon)
|
||||
setTextViewText(R.id.tv_mute_text, muteText)
|
||||
setOnClickPendingIntent(R.id.btn_mute, mutePendingIntent)
|
||||
|
||||
// Set mute button text, icon, and background colors dynamically
|
||||
setTextColor(R.id.tv_mute_text, primaryTextColor)
|
||||
setInt(R.id.iv_mute_icon, "setColorFilter", primaryTextColor)
|
||||
setColorStateList(
|
||||
R.id.btn_mute,
|
||||
"setBackgroundTintList",
|
||||
ColorStateList.valueOf(if (isMuted) activeTint else normalTint)
|
||||
)
|
||||
|
||||
|
||||
// Setup Stop Button
|
||||
setOnClickPendingIntent(R.id.btn_stop, stopPendingIntent)
|
||||
|
||||
// Set stop button text, icon, and background colors dynamically
|
||||
setTextColor(R.id.tv_stop_text, primaryTextColor)
|
||||
setInt(R.id.iv_stop_icon, "setColorFilter", primaryTextColor)
|
||||
setColorStateList(R.id.btn_stop, "setBackgroundTintList", ColorStateList.valueOf(normalTint))
|
||||
}
|
||||
|
||||
return NotificationCompat.Builder(this, CHANNEL_ID)
|
||||
.setSmallIcon(R.drawable.ic_screen_share)
|
||||
.setPriority(NotificationCompat.PRIORITY_LOW)
|
||||
.setCategory(NotificationCompat.CATEGORY_SERVICE)
|
||||
.setStyle(NotificationCompat.DecoratedCustomViewStyle())
|
||||
.setCustomContentView(collapsedLayout)
|
||||
.setCustomBigContentView(expandedLayout)
|
||||
.build()
|
||||
}
|
||||
|
||||
private fun startScreenCapture(resultCode: Int, resultData: Intent) {
|
||||
val mpManager = getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
|
||||
try {
|
||||
val projection = mpManager.getMediaProjection(resultCode, resultData)
|
||||
mediaProjection = projection
|
||||
|
||||
projection.registerCallback(object : MediaProjection.Callback() {
|
||||
override fun onStop() {
|
||||
Log.d(TAG, "MediaProjection stopped by system/user")
|
||||
stopServiceInternal()
|
||||
}
|
||||
}, backgroundHandler)
|
||||
|
||||
setupCapturePipeline(projection)
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "Failed to start media projection capture", e)
|
||||
stopServiceInternal()
|
||||
}
|
||||
}
|
||||
|
||||
private fun setupCapturePipeline(projection: MediaProjection) {
|
||||
val metrics = resources.displayMetrics
|
||||
val screenWidth = metrics.widthPixels
|
||||
val screenHeight = metrics.heightPixels
|
||||
val densityDpi = metrics.densityDpi
|
||||
|
||||
// Aspect ratio calculations scaled down to width ~720px
|
||||
val targetWidth = 720
|
||||
val targetHeight = (targetWidth * (screenHeight.toFloat() / screenWidth.toFloat())).toInt()
|
||||
val targetDensityDpi = (densityDpi * (targetWidth.toFloat() / screenWidth.toFloat())).toInt()
|
||||
|
||||
Log.d(TAG, "Screen size: ${screenWidth}x${screenHeight}. Scaling capture to ${targetWidth}x${targetHeight} at ${targetDensityDpi}dpi")
|
||||
|
||||
// Max images capacity of 2. Format: RGBA_8888.
|
||||
val reader = ImageReader.newInstance(
|
||||
targetWidth,
|
||||
targetHeight,
|
||||
PixelFormat.RGBA_8888,
|
||||
2
|
||||
)
|
||||
imageReader = reader
|
||||
|
||||
reader.setOnImageAvailableListener({ r ->
|
||||
var image: android.media.Image? = null
|
||||
try {
|
||||
image = r.acquireLatestImage()
|
||||
if (image == null) return@setOnImageAvailableListener
|
||||
|
||||
val now = System.currentTimeMillis()
|
||||
if (now - lastFrameTime < 150) {
|
||||
image.close()
|
||||
image = null
|
||||
return@setOnImageAvailableListener
|
||||
}
|
||||
lastFrameTime = now
|
||||
|
||||
val width = image.width
|
||||
val height = image.height
|
||||
val plane = image.planes[0]
|
||||
val buffer = plane.buffer
|
||||
val pixelStride = plane.pixelStride
|
||||
val rowStride = plane.rowStride
|
||||
val rowPadding = rowStride - pixelStride * width
|
||||
|
||||
val bitmapWidth = width + rowPadding / pixelStride
|
||||
val bitmap = Bitmap.createBitmap(
|
||||
bitmapWidth,
|
||||
height,
|
||||
Bitmap.Config.ARGB_8888
|
||||
)
|
||||
buffer.position(0)
|
||||
bitmap.copyPixelsFromBuffer(buffer)
|
||||
|
||||
// Close image immediately to prevent VirtualDisplay stalling
|
||||
image.close()
|
||||
image = null
|
||||
|
||||
val finalBitmap = if (rowPadding > 0) {
|
||||
val cropped = Bitmap.createBitmap(bitmap, 0, 0, width, height)
|
||||
bitmap.recycle()
|
||||
cropped
|
||||
} else {
|
||||
bitmap
|
||||
}
|
||||
|
||||
val out = ByteArrayOutputStream()
|
||||
finalBitmap.compress(Bitmap.CompressFormat.JPEG, 80, out)
|
||||
finalBitmap.recycle()
|
||||
|
||||
val jpegBytes = out.toByteArray()
|
||||
val cachedFrame = CachedFrame(jpegBytes, now)
|
||||
VisualBufferManager.addFrame(cachedFrame)
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "Error converting image frame", e)
|
||||
try {
|
||||
image?.close()
|
||||
} catch (ex: Exception) {
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
}, backgroundHandler)
|
||||
|
||||
// Create VirtualDisplay using ImageReader surface
|
||||
virtualDisplay = projection.createVirtualDisplay(
|
||||
"ScreenCaptureDisplay",
|
||||
targetWidth,
|
||||
targetHeight,
|
||||
targetDensityDpi,
|
||||
DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,
|
||||
reader.surface,
|
||||
null,
|
||||
backgroundHandler
|
||||
)
|
||||
}
|
||||
|
||||
private fun stopServiceInternal() {
|
||||
Log.d(TAG, "Stopping ScreenCaptureService and cleaning up resources")
|
||||
try {
|
||||
virtualDisplay?.release()
|
||||
virtualDisplay = null
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "Error releasing virtual display", e)
|
||||
}
|
||||
|
||||
try {
|
||||
imageReader?.close()
|
||||
imageReader = null
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "Error closing image reader", e)
|
||||
}
|
||||
|
||||
try {
|
||||
mediaProjection?.stop()
|
||||
mediaProjection = null
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "Error stopping media projection", e)
|
||||
}
|
||||
|
||||
try {
|
||||
handlerThread?.quitSafely()
|
||||
handlerThread = null
|
||||
backgroundHandler = null
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "Error quitting handler thread", e)
|
||||
}
|
||||
|
||||
// Clear visual buffer to prevent leaks
|
||||
VisualBufferManager.clear()
|
||||
|
||||
ScreenCaptureServiceHelper.onStateChanged = null
|
||||
|
||||
stopForeground(true)
|
||||
stopSelf()
|
||||
ScreenCaptureServiceHelper.onServiceStopped?.invoke()
|
||||
}
|
||||
|
||||
override fun onDestroy() {
|
||||
super.onDestroy()
|
||||
stopServiceInternal()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
package com.app.assistant.camera
|
||||
|
||||
import android.graphics.Bitmap
|
||||
import android.graphics.BitmapFactory
|
||||
import android.graphics.ImageFormat
|
||||
import android.graphics.Matrix
|
||||
import android.graphics.Rect
|
||||
import android.graphics.YuvImage
|
||||
import androidx.camera.core.ImageProxy
|
||||
import java.io.ByteArrayOutputStream
|
||||
import java.util.Collections
|
||||
import java.util.ArrayList
|
||||
|
||||
data class CachedFrame(val jpegBytes: ByteArray, val timestamp: Long)
|
||||
|
||||
object VisualBufferManager {
|
||||
private val buffer = Collections.synchronizedList(ArrayList<CachedFrame>())
|
||||
private const val MAX_SIZE = 15
|
||||
|
||||
fun addFrame(frame: CachedFrame) {
|
||||
synchronized(buffer) {
|
||||
buffer.add(frame)
|
||||
if (buffer.size > MAX_SIZE) {
|
||||
buffer.removeAt(0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun getOptimalFrame(speechStartTimestamp: Long): CachedFrame? {
|
||||
synchronized(buffer) {
|
||||
if (buffer.isEmpty()) return null
|
||||
var optimalFrame = buffer[0]
|
||||
var minDiff = Math.abs(optimalFrame.timestamp - speechStartTimestamp)
|
||||
for (i in 1 until buffer.size) {
|
||||
val frame = buffer[i]
|
||||
val diff = Math.abs(frame.timestamp - speechStartTimestamp)
|
||||
if (diff < minDiff) {
|
||||
minDiff = diff
|
||||
optimalFrame = frame
|
||||
}
|
||||
}
|
||||
return optimalFrame
|
||||
}
|
||||
}
|
||||
|
||||
fun clear() {
|
||||
buffer.clear()
|
||||
}
|
||||
}
|
||||
|
||||
fun imageProxyToRotatedJpeg(imageProxy: ImageProxy): ByteArray? {
|
||||
try {
|
||||
val width = imageProxy.width
|
||||
val height = imageProxy.height
|
||||
|
||||
val yBuffer = imageProxy.planes[0].buffer
|
||||
val uBuffer = imageProxy.planes[1].buffer
|
||||
val vBuffer = imageProxy.planes[2].buffer
|
||||
|
||||
// Correct size for NV21 format: width * height * 3 / 2
|
||||
val ySize = width * height
|
||||
val nv21 = ByteArray(ySize + ySize / 2)
|
||||
|
||||
// 1. Copy Y plane, stripping row stride padding if present
|
||||
val yRowStride = imageProxy.planes[0].rowStride
|
||||
val yBufferPos = yBuffer.position()
|
||||
if (yRowStride == width) {
|
||||
yBuffer.get(nv21, 0, ySize)
|
||||
} else {
|
||||
for (row in 0 until height) {
|
||||
yBuffer.position(row * yRowStride)
|
||||
yBuffer.get(nv21, row * width, width)
|
||||
}
|
||||
}
|
||||
yBuffer.position(yBufferPos)
|
||||
|
||||
val uRowStride = imageProxy.planes[1].rowStride
|
||||
val uPixelStride = imageProxy.planes[1].pixelStride
|
||||
val vRowStride = imageProxy.planes[2].rowStride
|
||||
val vPixelStride = imageProxy.planes[2].pixelStride
|
||||
|
||||
// 2. Interleave U and V planes into NV21 format (V, U, V, U...)
|
||||
var pos = ySize
|
||||
for (row in 0 until height / 2) {
|
||||
for (col in 0 until width / 2) {
|
||||
val uIndex = row * uRowStride + col * uPixelStride
|
||||
val vIndex = row * vRowStride + col * vPixelStride
|
||||
|
||||
if (vIndex < vBuffer.capacity() && uIndex < uBuffer.capacity()) {
|
||||
nv21[pos++] = vBuffer.get(vIndex)
|
||||
nv21[pos++] = uBuffer.get(uIndex)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val yuvImage = YuvImage(nv21, ImageFormat.NV21, width, height, null)
|
||||
val out = ByteArrayOutputStream()
|
||||
yuvImage.compressToJpeg(Rect(0, 0, width, height), 80, out)
|
||||
val jpegBytes = out.toByteArray()
|
||||
|
||||
val rotation = imageProxy.imageInfo.rotationDegrees
|
||||
if (rotation == 0) {
|
||||
return jpegBytes
|
||||
}
|
||||
|
||||
val bitmap = BitmapFactory.decodeByteArray(jpegBytes, 0, jpegBytes.size) ?: return null
|
||||
val matrix = Matrix().apply { postRotate(rotation.toFloat()) }
|
||||
val rotatedBitmap = Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true)
|
||||
bitmap.recycle()
|
||||
|
||||
val rotatedOut = ByteArrayOutputStream()
|
||||
rotatedBitmap.compress(Bitmap.CompressFormat.JPEG, 80, rotatedOut)
|
||||
rotatedBitmap.recycle()
|
||||
return rotatedOut.toByteArray()
|
||||
} catch (e: Exception) {
|
||||
android.util.Log.e("VisualBufferManager", "Error converting ImageProxy to rotated JPEG", e)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
package com.app.assistant.classifier
|
||||
|
||||
import android.content.Context
|
||||
import android.os.SystemClock
|
||||
import android.util.Log
|
||||
import com.google.mediapipe.tasks.core.BaseOptions
|
||||
import com.google.mediapipe.tasks.text.textclassifier.TextClassifier
|
||||
import com.google.mediapipe.tasks.text.textclassifier.TextClassifierResult
|
||||
import java.util.concurrent.ScheduledThreadPoolExecutor
|
||||
|
||||
class TextClassifierHelper(
|
||||
var currentModel: String = MOBILEBERT,
|
||||
val context: Context,
|
||||
val listener: TextResultsListener,
|
||||
) {
|
||||
private lateinit var textClassifier: TextClassifier
|
||||
private val executor = ScheduledThreadPoolExecutor(1)
|
||||
|
||||
init {
|
||||
executor.execute {
|
||||
initClassifier()
|
||||
}
|
||||
}
|
||||
|
||||
fun initClassifier() {
|
||||
val baseOptionsBuilder =
|
||||
BaseOptions
|
||||
.builder()
|
||||
.setModelAssetPath(currentModel)
|
||||
|
||||
try {
|
||||
val baseOptions = baseOptionsBuilder.build()
|
||||
val optionsBuilder =
|
||||
TextClassifier.TextClassifierOptions
|
||||
.builder()
|
||||
.setBaseOptions(baseOptions)
|
||||
val options = optionsBuilder.build()
|
||||
textClassifier = TextClassifier.createFromOptions(context, options)
|
||||
} catch (e: Exception) {
|
||||
listener.onError(
|
||||
"Text classifier failed to initialize. See error logs for details"
|
||||
)
|
||||
Log.e(
|
||||
TAG,
|
||||
"Text classifier failed to load the task with error: " + e.message,
|
||||
e
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Run text classification using MediaPipe Text Classifier API
|
||||
fun classify(
|
||||
text: String,
|
||||
itemId: Long,
|
||||
loadingItemId: Long,
|
||||
speak: Boolean,
|
||||
) {
|
||||
executor.execute {
|
||||
if (!::textClassifier.isInitialized) {
|
||||
listener.onError("Text classifier not initialized", itemId, loadingItemId, speak)
|
||||
return@execute
|
||||
}
|
||||
|
||||
// inferenceTime is the amount of time, in milliseconds, that it takes to
|
||||
// classify the input text.
|
||||
var inferenceTime = SystemClock.uptimeMillis()
|
||||
|
||||
// Remove all punctuation before classifying
|
||||
val cleanedText = text.replace(Regex("[.,?!]"), "")
|
||||
try {
|
||||
val results = textClassifier.classify(cleanedText)
|
||||
|
||||
inferenceTime = SystemClock.uptimeMillis() - inferenceTime
|
||||
|
||||
listener.onResult(results, inferenceTime, text, itemId, loadingItemId, speak)
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "Error running classification: ${e.message}", e)
|
||||
listener.onError("Classification failed: ${e.message}", itemId, loadingItemId, speak)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun shutDown() {
|
||||
try {
|
||||
executor.shutdown()
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "Error shutting down executor", e)
|
||||
}
|
||||
}
|
||||
|
||||
interface TextResultsListener {
|
||||
fun onError(error: String)
|
||||
|
||||
fun onError(
|
||||
error: String,
|
||||
itemId: Long,
|
||||
loadingItemId: Long,
|
||||
speak: Boolean
|
||||
) {
|
||||
onError(error)
|
||||
}
|
||||
|
||||
fun onResult(
|
||||
results: TextClassifierResult,
|
||||
inferenceTime: Long,
|
||||
inputText: String,
|
||||
itemId: Long,
|
||||
loadingItemId: Long,
|
||||
speak: Boolean,
|
||||
)
|
||||
}
|
||||
|
||||
companion object {
|
||||
const val TAG = "TextClassifierHelper"
|
||||
const val MOBILEBERT = "model.tflite"
|
||||
}
|
||||
}
|
||||
71
app/src/main/java/com/app/assistant/config/SpeechConfig.kt
Normal file
71
app/src/main/java/com/app/assistant/config/SpeechConfig.kt
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
package com.app.assistant.config
|
||||
|
||||
import com.app.assistant.BuildConfig
|
||||
import com.app.assistant.repository.SettingsRepository
|
||||
|
||||
import com.app.assistant.llm.LlmProvider
|
||||
|
||||
enum class TtsApiProvider {
|
||||
EDGE_TTS,
|
||||
GOOGLE_TTS
|
||||
}
|
||||
|
||||
enum class SttApiProvider {
|
||||
GROQ_WHISPER,
|
||||
GOOGLE_STT
|
||||
}
|
||||
|
||||
object SpeechConfig {
|
||||
// Current Active Providers - change these to switch implementations and rebuild
|
||||
var ACTIVE_TTS_PROVIDER = TtsApiProvider.EDGE_TTS
|
||||
val ACTIVE_STT_PROVIDER = SttApiProvider.GROQ_WHISPER
|
||||
|
||||
// Edge TTS Configurations
|
||||
object EdgeTts {
|
||||
const val WSS_URL = "wss://api.msedgeservices.com/tts/cognitiveservices/websocket/v1"
|
||||
fun getSubscriptionKey(settingsRepository: SettingsRepository): String {
|
||||
val key = settingsRepository.getEdgeTtsSubscriptionKey()
|
||||
return if (key.isNullOrBlank()) BuildConfig.EDGE_TTS_SUBSCRIPTION_KEY else key
|
||||
}
|
||||
const val VOICE = "en-GB-SoniaNeural"
|
||||
const val OUTPUT_FORMAT = "audio-24khz-48kbitrate-mono-mp3"
|
||||
}
|
||||
|
||||
// Google TTS Configurations
|
||||
object GoogleTts {
|
||||
const val BASE_URL = "https://texttospeech.googleapis.com/v1/text:synthesize"
|
||||
const val VOICE = "en-US-Wavenet-D"
|
||||
const val AUDIO_ENCODING = "MP3"
|
||||
|
||||
fun getApiKey(settingsRepository: SettingsRepository): String {
|
||||
return settingsRepository.getChatApiKey() ?: ""
|
||||
}
|
||||
}
|
||||
|
||||
// Groq Whisper Configurations
|
||||
object GroqWhisper {
|
||||
const val BASE_URL = "https://api.groq.com/openai/v1/audio/transcriptions"
|
||||
const val MODEL = "whisper-large-v3"
|
||||
|
||||
fun getApiKey(settingsRepository: SettingsRepository): String {
|
||||
var apiKey = settingsRepository.getChatApiKey()
|
||||
val providerStr = settingsRepository.getLlmProvider()
|
||||
val provider = try {
|
||||
LlmProvider.valueOf(providerStr)
|
||||
} catch (e: Exception) {
|
||||
LlmProvider.GROQ
|
||||
}
|
||||
if (apiKey.isNullOrBlank() && provider == LlmProvider.GROQ) {
|
||||
apiKey = BuildConfig.GROQ_API_KEY
|
||||
}
|
||||
return apiKey ?: ""
|
||||
}
|
||||
}
|
||||
|
||||
// Google STT Configurations
|
||||
object GoogleStt {
|
||||
const val BASE_URL = "https://speech.googleapis.com/v1/speech:recognize"
|
||||
const val ENCODING = "LINEAR16"
|
||||
const val SAMPLE_RATE_HERTZ = 16000
|
||||
}
|
||||
}
|
||||
32
app/src/main/java/com/app/assistant/db/AppDatabase.kt
Normal file
32
app/src/main/java/com/app/assistant/db/AppDatabase.kt
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
package com.app.assistant.db
|
||||
|
||||
import android.content.Context
|
||||
import androidx.room.Database
|
||||
import androidx.room.Room
|
||||
import androidx.room.RoomDatabase
|
||||
import androidx.room.TypeConverters
|
||||
|
||||
@Database(entities = [GroupEntity::class, MessageEntity::class, AttachmentEntity::class], version = 4, exportSchema = false)
|
||||
@TypeConverters(Converters::class)
|
||||
abstract class AppDatabase : RoomDatabase() {
|
||||
abstract fun conversationDao(): ConversationDao
|
||||
|
||||
companion object {
|
||||
@Volatile
|
||||
private var INSTANCE: AppDatabase? = null
|
||||
|
||||
fun getDatabase(context: Context): AppDatabase {
|
||||
return INSTANCE ?: synchronized(this) {
|
||||
val instance = Room.databaseBuilder(
|
||||
context.applicationContext,
|
||||
AppDatabase::class.java,
|
||||
"conversation.db"
|
||||
)
|
||||
.fallbackToDestructiveMigration()
|
||||
.build()
|
||||
INSTANCE = instance
|
||||
instance
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
39
app/src/main/java/com/app/assistant/db/AttachmentEntity.kt
Normal file
39
app/src/main/java/com/app/assistant/db/AttachmentEntity.kt
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
package com.app.assistant.db
|
||||
|
||||
import androidx.room.ColumnInfo
|
||||
import androidx.room.Entity
|
||||
import androidx.room.ForeignKey
|
||||
import androidx.room.Index
|
||||
import androidx.room.PrimaryKey
|
||||
|
||||
@Entity(
|
||||
tableName = "attachments",
|
||||
foreignKeys = [
|
||||
ForeignKey(
|
||||
entity = MessageEntity::class,
|
||||
parentColumns = ["id"],
|
||||
childColumns = ["message_id"],
|
||||
onDelete = ForeignKey.CASCADE
|
||||
)
|
||||
],
|
||||
indices = [Index(value = ["message_id"])]
|
||||
)
|
||||
data class AttachmentEntity(
|
||||
@PrimaryKey(autoGenerate = true)
|
||||
val id: Long = 0,
|
||||
|
||||
@ColumnInfo(name = "message_id")
|
||||
val messageId: Long,
|
||||
|
||||
@ColumnInfo(name = "file_path")
|
||||
val filePath: String,
|
||||
|
||||
@ColumnInfo(name = "mime_type")
|
||||
val mimeType: String,
|
||||
|
||||
@ColumnInfo(name = "file_name")
|
||||
val fileName: String,
|
||||
|
||||
@ColumnInfo(name = "iv")
|
||||
val iv: String
|
||||
)
|
||||
52
app/src/main/java/com/app/assistant/db/ConversationDao.kt
Normal file
52
app/src/main/java/com/app/assistant/db/ConversationDao.kt
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
package com.app.assistant.db
|
||||
|
||||
import androidx.room.Dao
|
||||
import androidx.room.Insert
|
||||
import androidx.room.OnConflictStrategy
|
||||
import androidx.room.Query
|
||||
import androidx.room.Update
|
||||
|
||||
@Dao
|
||||
interface ConversationDao {
|
||||
@Insert(onConflict = OnConflictStrategy.REPLACE)
|
||||
suspend fun insertGroup(group: GroupEntity): Long
|
||||
|
||||
@Query("SELECT * FROM `groups` ORDER BY group_id DESC")
|
||||
suspend fun getAllGroups(): List<GroupEntity>
|
||||
|
||||
@Insert(onConflict = OnConflictStrategy.REPLACE)
|
||||
suspend fun insertMessage(message: MessageEntity)
|
||||
|
||||
@Query("DELETE FROM messages WHERE id = :messageId")
|
||||
suspend fun deleteMessageById(messageId: Long)
|
||||
|
||||
@Update
|
||||
suspend fun updateMessage(message: MessageEntity)
|
||||
|
||||
@Query("DELETE FROM messages WHERE id IN (:ids)")
|
||||
suspend fun deleteMessagesByIds(ids: List<Long>)
|
||||
|
||||
@Query("SELECT file_path FROM attachments WHERE message_id IN (SELECT id FROM messages WHERE group_id = :groupId)")
|
||||
suspend fun getAttachmentFilePathsForGroup(groupId: Long): List<String>
|
||||
|
||||
@Query("DELETE FROM attachments WHERE message_id IN (SELECT id FROM messages WHERE group_id = :groupId)")
|
||||
suspend fun deleteAttachmentsForGroup(groupId: Long)
|
||||
|
||||
@Query("DELETE FROM messages WHERE group_id = :groupId")
|
||||
suspend fun deleteMessagesForGroup(groupId: Long)
|
||||
|
||||
@Query("DELETE FROM `groups` WHERE group_id = :groupId")
|
||||
suspend fun deleteGroupById(groupId: Long)
|
||||
|
||||
@Query("SELECT * FROM messages WHERE group_id = :groupId ORDER BY id ASC")
|
||||
suspend fun getMessagesForGroup(groupId: Long): List<MessageEntity>
|
||||
|
||||
@Insert(onConflict = OnConflictStrategy.REPLACE)
|
||||
suspend fun insertAttachment(attachment: AttachmentEntity)
|
||||
|
||||
@Query("SELECT * FROM attachments WHERE message_id = :messageId ORDER BY id ASC")
|
||||
suspend fun getAttachmentsForMessage(messageId: Long): List<AttachmentEntity>
|
||||
|
||||
@Query("DELETE FROM attachments WHERE message_id = :messageId")
|
||||
suspend fun deleteAttachmentsForMessage(messageId: Long)
|
||||
}
|
||||
16
app/src/main/java/com/app/assistant/db/Converters.kt
Normal file
16
app/src/main/java/com/app/assistant/db/Converters.kt
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
package com.app.assistant.db
|
||||
|
||||
import androidx.room.TypeConverter
|
||||
import java.net.URI
|
||||
|
||||
class Converters {
|
||||
@TypeConverter
|
||||
fun fromString(value: String?): URI? {
|
||||
return value?.let { URI(it) }
|
||||
}
|
||||
|
||||
@TypeConverter
|
||||
fun uriToString(uri: URI?): String? {
|
||||
return uri?.toString()
|
||||
}
|
||||
}
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
package com.app.assistant.db
|
||||
|
||||
import android.content.Context
|
||||
import android.database.sqlite.SQLiteDatabase
|
||||
import android.database.sqlite.SQLiteOpenHelper
|
||||
import com.app.assistant.model.Conversation
|
||||
import kotlin.reflect.full.memberProperties
|
||||
|
||||
|
||||
class DynamicConversationDbHelper(context: Context) : SQLiteOpenHelper(context, DATABASE_NAME, null, DATABASE_VERSION) {
|
||||
|
||||
companion object {
|
||||
private const val DATABASE_NAME = "conversation.db"
|
||||
private const val DATABASE_VERSION = 1
|
||||
|
||||
private const val COLUMN_IV = "iv"
|
||||
|
||||
// Groups Table
|
||||
private const val TABLE_GROUPS = "groups"
|
||||
private const val COLUMN_GROUP_ID = "group_id"
|
||||
private const val COLUMN_TITLE = "title"
|
||||
private const val COLUMN_CREATED_AT = "created_at"
|
||||
|
||||
// Messages Table
|
||||
private const val TABLE_MESSAGES = "messages"
|
||||
}
|
||||
|
||||
override fun onCreate(db: SQLiteDatabase?) {
|
||||
val createGroupsTableQuery = """
|
||||
CREATE TABLE $TABLE_GROUPS (
|
||||
$COLUMN_GROUP_ID INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
$COLUMN_TITLE TEXT,
|
||||
$COLUMN_CREATED_AT INTEGER
|
||||
)
|
||||
""".trimIndent()
|
||||
db?.execSQL(createGroupsTableQuery)
|
||||
|
||||
val createMessageTableQuery = buildCreateTableQuery()
|
||||
db?.execSQL(createMessageTableQuery)
|
||||
}
|
||||
|
||||
override fun onUpgrade(db: SQLiteDatabase?, oldVersion: Int, newVersion: Int) {
|
||||
db?.execSQL("DROP TABLE IF EXISTS $TABLE_MESSAGES")
|
||||
db?.execSQL("DROP TABLE IF EXISTS $TABLE_GROUPS")
|
||||
onCreate(db)
|
||||
}
|
||||
|
||||
private fun buildCreateTableQuery(): String {
|
||||
val fields = Conversation::class.memberProperties
|
||||
val columns = fields.filterNot { it.name == "isLoading" }.joinToString(", ") { field ->
|
||||
val fieldType = mapFieldTypeToSQLiteType(field.returnType.toString())
|
||||
if (field.name == "id") {
|
||||
"${field.name} INTEGER PRIMARY KEY"
|
||||
} else {
|
||||
"${field.name} $fieldType"
|
||||
}
|
||||
}
|
||||
|
||||
return """
|
||||
CREATE TABLE $TABLE_MESSAGES (
|
||||
$columns,
|
||||
iv TEXT,
|
||||
$COLUMN_GROUP_ID INTEGER,
|
||||
FOREIGN KEY ($COLUMN_GROUP_ID) REFERENCES $TABLE_GROUPS($COLUMN_GROUP_ID)
|
||||
)
|
||||
""".trimIndent()
|
||||
}
|
||||
|
||||
private fun mapFieldTypeToSQLiteType(kotlinType: String): String {
|
||||
return when (kotlinType) {
|
||||
"kotlin.String" -> "TEXT"
|
||||
"kotlin.Int", "kotlin.Long", "kotlin.Boolean" -> "INTEGER"
|
||||
"java.net.URI" -> "TEXT"
|
||||
else -> "TEXT"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,46 +1,32 @@
|
|||
package com.app.assistant.db
|
||||
|
||||
import android.content.ContentValues
|
||||
import android.content.Context
|
||||
import android.database.Cursor
|
||||
import android.text.format.DateFormat
|
||||
import android.util.Log
|
||||
import androidx.compose.runtime.snapshots.SnapshotStateList
|
||||
import com.app.assistant.model.Conversation
|
||||
import com.app.assistant.model.Group
|
||||
import com.app.assistant.model.Attachment
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.net.URI
|
||||
import kotlinx.coroutines.sync.Mutex
|
||||
import kotlinx.coroutines.sync.withLock
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.Date
|
||||
import java.util.Locale
|
||||
import kotlin.reflect.full.memberProperties
|
||||
import kotlin.reflect.full.primaryConstructor
|
||||
|
||||
class DynamicConversationRepository(private val context: Context) {
|
||||
|
||||
companion object {
|
||||
private const val TABLE_GROUPS = "groups"
|
||||
private const val COLUMN_GROUP_ID = "group_id"
|
||||
private const val COLUMN_TITLE = "title"
|
||||
private const val COLUMN_CREATED_AT = "created_at"
|
||||
private const val TABLE_MESSAGES = "messages"
|
||||
private const val COLUMN_MESSAGE_ID = "id"
|
||||
}
|
||||
|
||||
private val dbHelper = DynamicConversationDbHelper(context)
|
||||
class DynamicConversationRepository(
|
||||
context: Context,
|
||||
) {
|
||||
private val db = AppDatabase.getDatabase(context)
|
||||
private val dao = db.conversationDao()
|
||||
var currentGroupId: Long = -1L
|
||||
private val writeMutex = Mutex()
|
||||
|
||||
private suspend fun startNewChat(msg: String): Long {
|
||||
withContext(Dispatchers.IO) {
|
||||
val db = dbHelper.writableDatabase
|
||||
val contentValues = ContentValues().apply {
|
||||
put(COLUMN_TITLE, generateChatTitle(msg))
|
||||
put(COLUMN_CREATED_AT, System.nanoTime())
|
||||
}
|
||||
currentGroupId = db.insert(TABLE_GROUPS, null, contentValues)
|
||||
return withContext(Dispatchers.IO) {
|
||||
val title = generateChatTitle(msg)
|
||||
val group = GroupEntity(title = title, createdAt = System.nanoTime())
|
||||
currentGroupId = dao.insertGroup(group)
|
||||
currentGroupId
|
||||
}
|
||||
return currentGroupId
|
||||
}
|
||||
|
||||
private fun generateChatTitle(msg: String): String {
|
||||
|
|
@ -51,151 +37,181 @@ class DynamicConversationRepository(private val context: Context) {
|
|||
}
|
||||
|
||||
suspend fun addMessage(conversation: Conversation) {
|
||||
withContext(Dispatchers.IO) {
|
||||
if (currentGroupId == -1L) {
|
||||
currentGroupId = startNewChat(conversation.englishText.takeIf { it.isNotEmpty() } ?: conversation.translatedText)
|
||||
}
|
||||
writeMutex.withLock {
|
||||
withContext(Dispatchers.IO) {
|
||||
if (currentGroupId == -1L) {
|
||||
currentGroupId = startNewChat(conversation.text)
|
||||
}
|
||||
|
||||
val db = dbHelper.writableDatabase
|
||||
val (encryptedMap, iv) = encryptConversation(conversation)
|
||||
val values = ContentValues().apply {
|
||||
encryptedMap.forEach { (key, value) -> put(key, value) }
|
||||
put("iv", iv)
|
||||
put(COLUMN_GROUP_ID, currentGroupId)
|
||||
}
|
||||
try{
|
||||
db.insert(TABLE_MESSAGES, null, values)
|
||||
}catch (e: Exception){
|
||||
e.message?.let { Log.d("Test", it) }
|
||||
val iv = EncryptionUtil.generateIV()
|
||||
val (encryptedText, _) = EncryptionUtil.encrypt(conversation.text, iv)
|
||||
|
||||
val messageEntity = MessageEntity(
|
||||
id = conversation.id,
|
||||
text = encryptedText,
|
||||
isMe = conversation.isMe,
|
||||
category = conversation.category,
|
||||
contentURL = conversation.contentURL,
|
||||
navigationURI = conversation.navigationURI,
|
||||
iv = iv,
|
||||
groupId = currentGroupId
|
||||
)
|
||||
dao.insertMessage(messageEntity)
|
||||
|
||||
// Insert attachments
|
||||
conversation.attachments.forEach { attachment ->
|
||||
val attachmentEntity = AttachmentEntity(
|
||||
id = attachment.id,
|
||||
messageId = conversation.id,
|
||||
filePath = attachment.filePath,
|
||||
mimeType = attachment.mimeType,
|
||||
fileName = attachment.fileName,
|
||||
iv = attachment.iv
|
||||
)
|
||||
dao.insertAttachment(attachmentEntity)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun deleteMessage(id: Long) {
|
||||
withContext(Dispatchers.IO) {
|
||||
val db = dbHelper.writableDatabase
|
||||
db.delete(
|
||||
TABLE_MESSAGES,
|
||||
"id = ?",
|
||||
arrayOf(id.toString())
|
||||
)
|
||||
writeMutex.withLock {
|
||||
withContext(Dispatchers.IO) {
|
||||
val attachmentEntities = dao.getAttachmentsForMessage(id)
|
||||
attachmentEntities.forEach { att ->
|
||||
try {
|
||||
val file = java.io.File(att.filePath)
|
||||
if (file.exists()) {
|
||||
file.delete()
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
dao.deleteAttachmentsForMessage(id)
|
||||
dao.deleteMessageById(id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun updateMessage(oldConversation: Conversation, newConversation: Conversation) {
|
||||
withContext(Dispatchers.IO) {
|
||||
val db = dbHelper.writableDatabase
|
||||
val (encryptedMap, iv) = encryptConversation(newConversation)
|
||||
val values = ContentValues().apply {
|
||||
encryptedMap.forEach { (key, value) -> put(key, value) }
|
||||
put("iv", iv)
|
||||
suspend fun updateMessage(
|
||||
oldConversation: Conversation,
|
||||
newConversation: Conversation,
|
||||
) {
|
||||
writeMutex.withLock {
|
||||
withContext(Dispatchers.IO) {
|
||||
val iv = EncryptionUtil.generateIV()
|
||||
val (encryptedText, _) = EncryptionUtil.encrypt(newConversation.text, iv)
|
||||
|
||||
val messageEntity = MessageEntity(
|
||||
id = newConversation.id,
|
||||
text = encryptedText,
|
||||
isMe = newConversation.isMe,
|
||||
category = newConversation.category,
|
||||
contentURL = newConversation.contentURL,
|
||||
navigationURI = newConversation.navigationURI,
|
||||
iv = iv,
|
||||
groupId = currentGroupId
|
||||
)
|
||||
dao.updateMessage(messageEntity)
|
||||
|
||||
// Delete old attachments first and insert new ones
|
||||
dao.deleteAttachmentsForMessage(newConversation.id)
|
||||
newConversation.attachments.forEach { attachment ->
|
||||
val attachmentEntity = AttachmentEntity(
|
||||
id = attachment.id,
|
||||
messageId = newConversation.id,
|
||||
filePath = attachment.filePath,
|
||||
mimeType = attachment.mimeType,
|
||||
fileName = attachment.fileName,
|
||||
iv = attachment.iv
|
||||
)
|
||||
dao.insertAttachment(attachmentEntity)
|
||||
}
|
||||
}
|
||||
db.update(
|
||||
TABLE_MESSAGES,
|
||||
values,
|
||||
"id = ?",
|
||||
arrayOf(oldConversation.id.toString())
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun clearMessages(conversations: List<Conversation>) {
|
||||
withContext(Dispatchers.IO) {
|
||||
val db = dbHelper.writableDatabase
|
||||
val ids = conversations.joinToString(",") { it.id.toString() }
|
||||
db.execSQL("DELETE FROM $TABLE_MESSAGES WHERE id IN ($ids)")
|
||||
db.execSQL("DELETE FROM $TABLE_GROUPS WHERE $COLUMN_GROUP_ID == $currentGroupId")
|
||||
currentGroupId = -1L
|
||||
writeMutex.withLock {
|
||||
withContext(Dispatchers.IO) {
|
||||
conversations.forEach { conversation ->
|
||||
val attachmentEntities = dao.getAttachmentsForMessage(conversation.id)
|
||||
attachmentEntities.forEach { att ->
|
||||
try {
|
||||
val file = java.io.File(att.filePath)
|
||||
if (file.exists()) {
|
||||
file.delete()
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
dao.deleteAttachmentsForMessage(conversation.id)
|
||||
}
|
||||
val ids = conversations.map { it.id }
|
||||
dao.deleteMessagesByIds(ids)
|
||||
dao.deleteGroupById(currentGroupId)
|
||||
currentGroupId = -1L
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun deleteGroup(groupId: Long) {
|
||||
writeMutex.withLock {
|
||||
withContext(Dispatchers.IO) {
|
||||
val filePaths = dao.getAttachmentFilePathsForGroup(groupId)
|
||||
filePaths.forEach { filePath ->
|
||||
try {
|
||||
val file = java.io.File(filePath)
|
||||
if (file.exists()) {
|
||||
file.delete()
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
dao.deleteAttachmentsForGroup(groupId)
|
||||
dao.deleteMessagesForGroup(groupId)
|
||||
dao.deleteGroupById(groupId)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun loadAllGroups(): MutableList<Group> {
|
||||
val groupList = mutableListOf<Group>()
|
||||
withContext(Dispatchers.IO) {
|
||||
val db = dbHelper.readableDatabase
|
||||
val cursor: Cursor = db.query(
|
||||
TABLE_GROUPS,
|
||||
arrayOf(COLUMN_GROUP_ID, COLUMN_TITLE),
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
"$COLUMN_GROUP_ID DESC" // Sort by latest created
|
||||
)
|
||||
|
||||
while (cursor.moveToNext()) {
|
||||
val groupId = cursor.getLong(cursor.getColumnIndexOrThrow(COLUMN_GROUP_ID))
|
||||
val title = cursor.getString(cursor.getColumnIndexOrThrow(COLUMN_TITLE))
|
||||
groupList.add(Group(groupId, title))
|
||||
}
|
||||
cursor.close()
|
||||
return withContext(Dispatchers.IO) {
|
||||
dao.getAllGroups().map { entity ->
|
||||
Group(groupId = entity.groupId, title = entity.title)
|
||||
}.toMutableList()
|
||||
}
|
||||
return groupList
|
||||
}
|
||||
|
||||
|
||||
suspend fun loadMessagesForGroup(groupId: Long): MutableList<Conversation> {
|
||||
val tempList = mutableListOf<Conversation>()
|
||||
withContext(Dispatchers.IO) {
|
||||
val db = dbHelper.readableDatabase
|
||||
val cursor: Cursor = db.query(
|
||||
TABLE_MESSAGES,
|
||||
null,
|
||||
"$COLUMN_GROUP_ID = ?",
|
||||
arrayOf(groupId.toString()),
|
||||
null,
|
||||
null,
|
||||
COLUMN_MESSAGE_ID // Sort by latest created
|
||||
)
|
||||
return withContext(Dispatchers.IO) {
|
||||
dao.getMessagesForGroup(groupId).map { entity ->
|
||||
val decryptedText = EncryptionUtil.decrypt(entity.text, entity.iv)
|
||||
val attachmentEntities = dao.getAttachmentsForMessage(entity.id)
|
||||
val attachments = attachmentEntities.map { attEntity ->
|
||||
Attachment(
|
||||
id = attEntity.id,
|
||||
filePath = attEntity.filePath,
|
||||
mimeType = attEntity.mimeType,
|
||||
fileName = attEntity.fileName,
|
||||
iv = attEntity.iv
|
||||
)
|
||||
}
|
||||
|
||||
while (cursor.moveToNext()) {
|
||||
val conversation = cursorToConversation(cursor)
|
||||
tempList.add(conversation)
|
||||
}
|
||||
cursor.close()
|
||||
Conversation(
|
||||
id = entity.id,
|
||||
text = decryptedText,
|
||||
isMe = entity.isMe,
|
||||
isLoading = false,
|
||||
category = entity.category,
|
||||
contentURL = entity.contentURL,
|
||||
navigationURI = entity.navigationURI,
|
||||
attachments = attachments
|
||||
)
|
||||
}.toMutableList()
|
||||
}
|
||||
return tempList
|
||||
}
|
||||
|
||||
private fun cursorToConversation(cursor: Cursor): Conversation {
|
||||
val properties = Conversation::class.memberProperties
|
||||
val fieldMap = mutableMapOf<String, Any?>()
|
||||
val iv = cursor.getString(cursor.getColumnIndexOrThrow("iv"))
|
||||
|
||||
properties.filter { it.name != "isLoading" }.forEach { property ->
|
||||
val columnName = property.name
|
||||
val columnIndex = cursor.getColumnIndexOrThrow(columnName)
|
||||
val value = when (property.returnType.toString()) {
|
||||
"kotlin.String" -> cursor.getString(columnIndex)//?.let { EncryptionUtil.decrypt(it, iv) }
|
||||
"kotlin.Int" -> cursor.getInt(columnIndex)
|
||||
"kotlin.Long" -> cursor.getLong(columnIndex)
|
||||
"kotlin.Boolean" -> cursor.getString(columnIndex).toBooleanStrictOrNull() ?: false
|
||||
"java.net.URI" -> URI(cursor.getString(columnIndex))
|
||||
else -> null
|
||||
}
|
||||
fieldMap[columnName] = value
|
||||
}
|
||||
|
||||
return Conversation::class.constructors.first().callBy(
|
||||
Conversation::class.primaryConstructor!!.parameters
|
||||
.filter { it.name != "isLoading" } // Ignore "isLoading"
|
||||
.associateWith { parameter -> fieldMap[parameter.name] }
|
||||
)
|
||||
}
|
||||
|
||||
private fun encryptConversation(conversation: Conversation): Pair<Map<String, String>, String> {
|
||||
val properties = Conversation::class.memberProperties
|
||||
val encryptedMap = mutableMapOf<String, String>()
|
||||
val iv = EncryptionUtil.generateIV()
|
||||
|
||||
properties.forEach { property ->
|
||||
val key = property.name
|
||||
val value = property.getter.call(conversation)?.toString()
|
||||
if (value != null && key!="isLoading") {
|
||||
//encryptedMap[key] = EncryptionUtil.encrypt(value, iv).first
|
||||
encryptedMap[key] = value
|
||||
}
|
||||
}
|
||||
return Pair(encryptedMap, iv)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
package com.app.assistant.db
|
||||
|
||||
import android.security.keystore.KeyGenParameterSpec
|
||||
import android.security.keystore.KeyProperties
|
||||
import android.util.Base64
|
||||
import java.security.KeyStore
|
||||
import java.security.SecureRandom
|
||||
import javax.crypto.Cipher
|
||||
import javax.crypto.KeyGenerator
|
||||
|
|
@ -14,56 +17,142 @@ object EncryptionUtil {
|
|||
private const val ALGORITHM = "AES"
|
||||
private const val TRANSFORMATION = "AES/GCM/NoPadding"
|
||||
private const val TAG_LENGTH_BIT = 128
|
||||
private const val KEY_ALIAS = "com.app.assistant.database_key"
|
||||
private const val ANDROID_KEY_STORE = "AndroidKeyStore"
|
||||
|
||||
// Generate a random encryption key
|
||||
private val secretKey: SecretKey = generateSecretKey()
|
||||
// Retrieve or generate a secure key persisted in Android Keystore
|
||||
private val secretKey: SecretKey by lazy {
|
||||
getOrCreateSecretKey()
|
||||
}
|
||||
|
||||
// Generates a new random initialization vector (IV)
|
||||
fun generateIV(): String {
|
||||
val secureRandom = SecureRandom()
|
||||
val iv = ByteArray(IV_SIZE)
|
||||
secureRandom.nextBytes(iv)
|
||||
return Base64.encodeToString(iv, Base64.DEFAULT)
|
||||
return Base64.encodeToString(iv, Base64.DEFAULT).trim()
|
||||
}
|
||||
|
||||
// Encrypt a string and return encrypted text and IV
|
||||
fun encrypt(input: String, ivString: String): Pair<String, String> {
|
||||
val iv = Base64.decode(ivString, Base64.DEFAULT)
|
||||
val cipher = Cipher.getInstance(TRANSFORMATION)
|
||||
val spec = GCMParameterSpec(TAG_LENGTH_BIT, iv)
|
||||
cipher.init(Cipher.ENCRYPT_MODE, secretKey, spec)
|
||||
fun encrypt(
|
||||
input: String,
|
||||
ivString: String,
|
||||
): Pair<String, String> {
|
||||
if (input.isEmpty()) return Pair("", ivString)
|
||||
return try {
|
||||
val iv = Base64.decode(ivString, Base64.DEFAULT)
|
||||
val cipher = Cipher.getInstance(TRANSFORMATION)
|
||||
val spec = GCMParameterSpec(TAG_LENGTH_BIT, iv)
|
||||
cipher.init(Cipher.ENCRYPT_MODE, secretKey, spec)
|
||||
|
||||
val encryptedBytes = cipher.doFinal(input.toByteArray())
|
||||
val encryptedText = Base64.encodeToString(encryptedBytes, Base64.DEFAULT)
|
||||
return Pair(encryptedText, ivString)
|
||||
val encryptedBytes = cipher.doFinal(input.toByteArray(Charsets.UTF_8))
|
||||
val encryptedText = Base64.encodeToString(encryptedBytes, Base64.DEFAULT).trim()
|
||||
Pair(encryptedText, ivString)
|
||||
} catch (e: Exception) {
|
||||
// Safe fallback to original text if encryption fails (to ensure no data loss)
|
||||
Pair(input, ivString)
|
||||
}
|
||||
}
|
||||
|
||||
// Decrypt an encrypted string using IV
|
||||
fun decrypt(encryptedInput: String, ivString: String): String {
|
||||
val iv = Base64.decode(ivString, Base64.DEFAULT)
|
||||
val cipher = Cipher.getInstance(TRANSFORMATION)
|
||||
val spec = GCMParameterSpec(TAG_LENGTH_BIT, iv)
|
||||
cipher.init(Cipher.DECRYPT_MODE, secretKey, spec)
|
||||
fun decrypt(
|
||||
encryptedInput: String,
|
||||
ivString: String,
|
||||
): String {
|
||||
if (encryptedInput.isEmpty() || ivString.isEmpty()) return encryptedInput
|
||||
return try {
|
||||
val iv = Base64.decode(ivString, Base64.DEFAULT)
|
||||
val cipher = Cipher.getInstance(TRANSFORMATION)
|
||||
val spec = GCMParameterSpec(TAG_LENGTH_BIT, iv)
|
||||
cipher.init(Cipher.DECRYPT_MODE, secretKey, spec)
|
||||
|
||||
val encryptedBytes = Base64.decode(encryptedInput, Base64.DEFAULT)
|
||||
val decryptedBytes = cipher.doFinal(encryptedBytes)
|
||||
return String(decryptedBytes)
|
||||
val encryptedBytes = Base64.decode(encryptedInput, Base64.DEFAULT)
|
||||
val decryptedBytes = cipher.doFinal(encryptedBytes)
|
||||
String(decryptedBytes, Charsets.UTF_8)
|
||||
} catch (e: Exception) {
|
||||
// Safe fallback to raw database value if decryption fails (e.g. for existing plaintext entries)
|
||||
encryptedInput
|
||||
}
|
||||
}
|
||||
|
||||
// Generate a new secret key
|
||||
private fun generateSecretKey(): SecretKey {
|
||||
// Encrypt a file byte array using IV
|
||||
fun encryptFile(inputBytes: ByteArray, ivString: String): ByteArray {
|
||||
if (inputBytes.isEmpty()) return ByteArray(0)
|
||||
return try {
|
||||
val iv = Base64.decode(ivString, Base64.DEFAULT)
|
||||
val cipher = Cipher.getInstance(TRANSFORMATION)
|
||||
val spec = GCMParameterSpec(TAG_LENGTH_BIT, iv)
|
||||
cipher.init(Cipher.ENCRYPT_MODE, secretKey, spec)
|
||||
cipher.doFinal(inputBytes)
|
||||
} catch (e: Exception) {
|
||||
inputBytes
|
||||
}
|
||||
}
|
||||
|
||||
// Decrypt a file byte array using IV
|
||||
fun decryptFile(encryptedBytes: ByteArray, ivString: String): ByteArray {
|
||||
if (encryptedBytes.isEmpty() || ivString.isEmpty()) return encryptedBytes
|
||||
return try {
|
||||
val iv = Base64.decode(ivString, Base64.DEFAULT)
|
||||
val cipher = Cipher.getInstance(TRANSFORMATION)
|
||||
val spec = GCMParameterSpec(TAG_LENGTH_BIT, iv)
|
||||
cipher.init(Cipher.DECRYPT_MODE, secretKey, spec)
|
||||
cipher.doFinal(encryptedBytes)
|
||||
} catch (e: Exception) {
|
||||
encryptedBytes
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Retrieve or create a persistent AES key in Android KeyStore
|
||||
private fun getOrCreateSecretKey(): SecretKey {
|
||||
return try {
|
||||
val keyStore = KeyStore.getInstance(ANDROID_KEY_STORE).apply { load(null) }
|
||||
if (keyStore.containsAlias(KEY_ALIAS)) {
|
||||
val entry = keyStore.getEntry(KEY_ALIAS, null) as? KeyStore.SecretKeyEntry
|
||||
if (entry != null) {
|
||||
return entry.secretKey
|
||||
}
|
||||
}
|
||||
|
||||
val keyGenerator = KeyGenerator.getInstance(
|
||||
KeyProperties.KEY_ALGORITHM_AES,
|
||||
ANDROID_KEY_STORE
|
||||
)
|
||||
val spec = KeyGenParameterSpec.Builder(
|
||||
KEY_ALIAS,
|
||||
KeyProperties.PURPOSE_ENCRYPT or KeyProperties.PURPOSE_DECRYPT
|
||||
)
|
||||
.setBlockModes(KeyProperties.BLOCK_MODE_GCM)
|
||||
.setEncryptionPaddings(KeyProperties.ENCRYPTION_PADDING_NONE)
|
||||
.setKeySize(KEY_SIZE)
|
||||
.build()
|
||||
keyGenerator.init(spec)
|
||||
keyGenerator.generateKey()
|
||||
} catch (e: Exception) {
|
||||
// In-memory key fallback if Android KeyStore is unavailable (e.g., local unit tests)
|
||||
generateFallbackSecretKey()
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback key generator
|
||||
private fun generateFallbackSecretKey(): SecretKey {
|
||||
val keyGenerator = KeyGenerator.getInstance(ALGORITHM)
|
||||
keyGenerator.init(KEY_SIZE, SecureRandom())
|
||||
return keyGenerator.generateKey()
|
||||
}
|
||||
|
||||
// Save and restore secret key as Base64 (for persistence)
|
||||
// Save and restore secret key as Base64 (kept for helper/debugging purposes)
|
||||
fun getSecretKeyAsString(): String {
|
||||
return Base64.encodeToString(secretKey.encoded, Base64.DEFAULT)
|
||||
return try {
|
||||
Base64.encodeToString(secretKey.encoded, Base64.DEFAULT).trim()
|
||||
} catch (e: Exception) {
|
||||
""
|
||||
}
|
||||
}
|
||||
|
||||
fun restoreSecretKeyFromString(keyString: String): SecretKey {
|
||||
val decodedKey = Base64.decode(keyString, Base64.DEFAULT)
|
||||
return SecretKeySpec(decodedKey, 0, decodedKey.size, ALGORITHM)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
18
app/src/main/java/com/app/assistant/db/GroupEntity.kt
Normal file
18
app/src/main/java/com/app/assistant/db/GroupEntity.kt
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
package com.app.assistant.db
|
||||
|
||||
import androidx.room.ColumnInfo
|
||||
import androidx.room.Entity
|
||||
import androidx.room.PrimaryKey
|
||||
|
||||
@Entity(tableName = "groups")
|
||||
data class GroupEntity(
|
||||
@PrimaryKey(autoGenerate = true)
|
||||
@ColumnInfo(name = "group_id")
|
||||
val groupId: Long = 0,
|
||||
|
||||
@ColumnInfo(name = "title")
|
||||
val title: String,
|
||||
|
||||
@ColumnInfo(name = "created_at")
|
||||
val createdAt: Long
|
||||
)
|
||||
46
app/src/main/java/com/app/assistant/db/MessageEntity.kt
Normal file
46
app/src/main/java/com/app/assistant/db/MessageEntity.kt
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
package com.app.assistant.db
|
||||
|
||||
import androidx.room.ColumnInfo
|
||||
import androidx.room.Entity
|
||||
import androidx.room.ForeignKey
|
||||
import androidx.room.Index
|
||||
import androidx.room.PrimaryKey
|
||||
import java.net.URI
|
||||
|
||||
@Entity(
|
||||
tableName = "messages",
|
||||
foreignKeys = [
|
||||
ForeignKey(
|
||||
entity = GroupEntity::class,
|
||||
parentColumns = ["group_id"],
|
||||
childColumns = ["group_id"],
|
||||
onDelete = ForeignKey.CASCADE
|
||||
)
|
||||
],
|
||||
indices = [Index(value = ["group_id"])]
|
||||
)
|
||||
data class MessageEntity(
|
||||
@PrimaryKey
|
||||
val id: Long,
|
||||
|
||||
@ColumnInfo(name = "text")
|
||||
val text: String,
|
||||
|
||||
@ColumnInfo(name = "isMe")
|
||||
val isMe: Boolean,
|
||||
|
||||
@ColumnInfo(name = "category")
|
||||
val category: String,
|
||||
|
||||
@ColumnInfo(name = "contentURL")
|
||||
val contentURL: String,
|
||||
|
||||
@ColumnInfo(name = "navigationURI")
|
||||
val navigationURI: URI,
|
||||
|
||||
@ColumnInfo(name = "iv")
|
||||
val iv: String,
|
||||
|
||||
@ColumnInfo(name = "group_id")
|
||||
val groupId: Long
|
||||
)
|
||||
|
|
@ -10,13 +10,13 @@ import kotlinx.coroutines.launch
|
|||
|
||||
class SyncStateList(
|
||||
private val repository: DynamicConversationRepository, // Repository to handle DB operations
|
||||
private val stateList: SnapshotStateList<Conversation> = mutableStateListOf()
|
||||
private val scope: CoroutineScope,
|
||||
private val stateList: SnapshotStateList<Conversation> = mutableStateListOf(),
|
||||
) : MutableList<Conversation> by stateList {
|
||||
|
||||
override fun add(element: Conversation): Boolean {
|
||||
val result = stateList.add(element)
|
||||
if (result && !element.isLoading) {
|
||||
CoroutineScope(Dispatchers.IO).launch {
|
||||
if (result && !element.isLoading && !element.isStreaming) {
|
||||
scope.launch(Dispatchers.IO) {
|
||||
repository.addMessage(element)
|
||||
}
|
||||
}
|
||||
|
|
@ -25,17 +25,26 @@ class SyncStateList(
|
|||
|
||||
override fun removeAt(index: Int): Conversation {
|
||||
val result = stateList.removeAt(index)
|
||||
CoroutineScope(Dispatchers.IO).launch {
|
||||
scope.launch(Dispatchers.IO) {
|
||||
repository.deleteMessage(result.id)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
override fun set(index: Int, element: Conversation): Conversation {
|
||||
override fun set(
|
||||
index: Int,
|
||||
element: Conversation,
|
||||
): Conversation {
|
||||
val oldElement = stateList[index]
|
||||
stateList[index] = element
|
||||
CoroutineScope(Dispatchers.IO).launch {
|
||||
repository.updateMessage(oldElement, element)
|
||||
if (!element.isLoading && !element.isStreaming) {
|
||||
scope.launch(Dispatchers.IO) {
|
||||
if (oldElement.isLoading || oldElement.isStreaming) {
|
||||
repository.addMessage(element)
|
||||
} else {
|
||||
repository.updateMessage(oldElement, element)
|
||||
}
|
||||
}
|
||||
}
|
||||
return oldElement
|
||||
}
|
||||
|
|
@ -43,7 +52,7 @@ class SyncStateList(
|
|||
fun clearAll(): Job {
|
||||
val oldList = stateList.toList()
|
||||
stateList.clear()
|
||||
return CoroutineScope(Dispatchers.IO).launch {
|
||||
return scope.launch(Dispatchers.IO) {
|
||||
repository.clearMessages(oldList) // Suspends until DB operations finish
|
||||
}
|
||||
}
|
||||
|
|
|
|||
353
app/src/main/java/com/app/assistant/llm/FlexibleLlmAdapter.kt
Normal file
353
app/src/main/java/com/app/assistant/llm/FlexibleLlmAdapter.kt
Normal file
|
|
@ -0,0 +1,353 @@
|
|||
package com.app.assistant.llm
|
||||
|
||||
import android.util.Log
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.flow
|
||||
import kotlinx.coroutines.flow.flowOn
|
||||
import kotlinx.serialization.json.Json
|
||||
import okhttp3.MediaType.Companion.toMediaTypeOrNull
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import okhttp3.RequestBody.Companion.toRequestBody
|
||||
import java.io.IOException
|
||||
import java.net.UnknownHostException
|
||||
|
||||
class FlexibleLlmAdapter(
|
||||
private val client: OkHttpClient,
|
||||
private val apiKey: String,
|
||||
private val model: String,
|
||||
private val config: LlmConfig,
|
||||
private val provider: LlmProvider
|
||||
) : LlmAdapter {
|
||||
private val json = Json { ignoreUnknownKeys = true }
|
||||
|
||||
override suspend fun generateResponse(
|
||||
systemContext: String,
|
||||
messages: List<LlmMessage>
|
||||
): String? {
|
||||
val url = config.url
|
||||
.replace("{{API_KEY}}", apiKey)
|
||||
.replace("{{MODEL}}", model)
|
||||
|
||||
val requestBuilder = Request.Builder().url(url)
|
||||
for ((key, value) in config.headers) {
|
||||
val resolvedValue = value
|
||||
.replace("{{API_KEY}}", apiKey)
|
||||
.replace("{{MODEL}}", model)
|
||||
requestBuilder.addHeader(key, resolvedValue)
|
||||
}
|
||||
|
||||
val requestBodyString = buildRequestBodyString(systemContext, messages, isStream = false)
|
||||
val mediaType = config.headers["Content-Type"]?.toMediaTypeOrNull() ?: "application/json".toMediaTypeOrNull()
|
||||
val requestBody = requestBodyString.toRequestBody(mediaType)
|
||||
|
||||
requestBuilder.post(requestBody)
|
||||
val request = requestBuilder.build()
|
||||
|
||||
val rawResponse = withContext(Dispatchers.IO) {
|
||||
try {
|
||||
client.newCall(request).execute().use { response ->
|
||||
if (!response.isSuccessful) {
|
||||
val errorBody = response.body?.string() ?: ""
|
||||
Log.e("FlexibleLlmAdapter", "API request failed with code ${response.code}: $errorBody")
|
||||
if (response.code == 401) {
|
||||
return@withContext """{"error_code": 401, "msg": "API key is missing or invalid."}"""
|
||||
}
|
||||
val errMsg = extractErrorMessage(errorBody) ?: "API error (HTTP ${response.code})."
|
||||
return@withContext """{"error_code": ${response.code}, "msg": "$errMsg"}"""
|
||||
}
|
||||
response.body!!.string()
|
||||
}
|
||||
} catch (e: UnknownHostException) {
|
||||
"""{"error_code": 503, "msg": "Seems this device is offline. Maybe try checking data connection."}"""
|
||||
} catch (e: Exception) {
|
||||
Log.e("FlexibleLlmAdapter", "Error during LLM API call", e)
|
||||
"""{"error_code": 500, "msg": "Seems some issue in LLM server. Please try again."}"""
|
||||
}
|
||||
}
|
||||
|
||||
return extractResponseContent(rawResponse, config.responsePath)
|
||||
}
|
||||
|
||||
override fun generateResponseStream(
|
||||
systemContext: String,
|
||||
messages: List<LlmMessage>
|
||||
): Flow<String> = flow {
|
||||
var url = config.url
|
||||
.replace("{{API_KEY}}", apiKey)
|
||||
.replace("{{MODEL}}", model)
|
||||
|
||||
if (url.contains(":generateContent")) {
|
||||
url = url.replace(":generateContent", ":streamGenerateContent")
|
||||
}
|
||||
|
||||
val requestBuilder = Request.Builder().url(url)
|
||||
for ((key, value) in config.headers) {
|
||||
val resolvedValue = value
|
||||
.replace("{{API_KEY}}", apiKey)
|
||||
.replace("{{MODEL}}", model)
|
||||
requestBuilder.addHeader(key, resolvedValue)
|
||||
}
|
||||
|
||||
val requestBodyString = buildRequestBodyString(systemContext, messages, isStream = true)
|
||||
val mediaType = config.headers["Content-Type"]?.toMediaTypeOrNull() ?: "application/json".toMediaTypeOrNull()
|
||||
val requestBody = requestBodyString.toRequestBody(mediaType)
|
||||
|
||||
requestBuilder.post(requestBody)
|
||||
val request = requestBuilder.build()
|
||||
|
||||
val response = try {
|
||||
client.newCall(request).execute()
|
||||
} catch (e: UnknownHostException) {
|
||||
null
|
||||
} catch (e: Exception) {
|
||||
null
|
||||
}
|
||||
|
||||
if (response == null) {
|
||||
emit("Seems this device is offline or LLM server is unreachable.")
|
||||
return@flow
|
||||
}
|
||||
|
||||
if (!response.isSuccessful) {
|
||||
val errorBody = response.body?.string() ?: ""
|
||||
Log.e("FlexibleLlmAdapter", "API streaming request failed with code ${response.code}: $errorBody")
|
||||
val errMsg = extractErrorMessage(errorBody) ?: "API error (HTTP ${response.code})."
|
||||
emit("Error: $errMsg")
|
||||
response.close()
|
||||
return@flow
|
||||
}
|
||||
|
||||
val responseBody = response.body
|
||||
if (responseBody == null) {
|
||||
emit("Error: Empty response body")
|
||||
response.close()
|
||||
return@flow
|
||||
}
|
||||
|
||||
responseBody.byteStream().bufferedReader().use { reader ->
|
||||
var line: String?
|
||||
while (reader.readLine().also { line = it } != null) {
|
||||
val currentLine = line ?: break
|
||||
if (currentLine.startsWith("data: ")) {
|
||||
val dataContent = currentLine.substring(6).trim()
|
||||
if (dataContent == "[DONE]") {
|
||||
break
|
||||
}
|
||||
if (dataContent.isNotEmpty()) {
|
||||
val chunkText = extractChunkContent(dataContent, config.responsePath)
|
||||
if (chunkText != null) {
|
||||
emit(chunkText)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
var cleanedLine = currentLine.trim()
|
||||
if (cleanedLine.endsWith(",")) {
|
||||
cleanedLine = cleanedLine.substring(0, cleanedLine.length - 1).trim()
|
||||
}
|
||||
if (cleanedLine.startsWith("{") && cleanedLine.endsWith("}")) {
|
||||
val chunkText = extractChunkContent(cleanedLine, config.responsePath)
|
||||
if (chunkText != null) {
|
||||
emit(chunkText)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}.flowOn(Dispatchers.IO)
|
||||
|
||||
private fun buildRequestBodyString(systemContext: String, messages: List<LlmMessage>, isStream: Boolean): String {
|
||||
val hasSystemContextPlaceholder = config.requestTemplate.contains("{{SYSTEM_CONTEXT}}")
|
||||
val messagesToSerialize = if (hasSystemContextPlaceholder) {
|
||||
messages.filter { it.role != "system" }
|
||||
} else {
|
||||
messages
|
||||
}
|
||||
|
||||
val serializedMessagesBuilder = StringBuilder()
|
||||
serializedMessagesBuilder.append("[")
|
||||
messagesToSerialize.forEachIndexed { index, msg ->
|
||||
val roleValue = when (msg.role) {
|
||||
"system" -> config.systemRole ?: "system"
|
||||
"user" -> config.userRole
|
||||
"assistant" -> config.assistantRole
|
||||
else -> msg.role
|
||||
}
|
||||
val msgJson = if (msg.attachments.isNullOrEmpty()) {
|
||||
val escapedContent = escapeJsonString(msg.content)
|
||||
config.messageFormat
|
||||
.replace("{{ROLE}}", roleValue)
|
||||
.replace("{{CONTENT}}", escapedContent)
|
||||
} else {
|
||||
buildMultimodalMessageJson(roleValue, msg.content, msg.attachments, provider)
|
||||
}
|
||||
|
||||
serializedMessagesBuilder.append(msgJson)
|
||||
if (index < messagesToSerialize.size - 1) {
|
||||
serializedMessagesBuilder.append(",")
|
||||
}
|
||||
}
|
||||
serializedMessagesBuilder.append("]")
|
||||
val messagesJsonArray = serializedMessagesBuilder.toString()
|
||||
|
||||
var requestBodyString = config.requestTemplate
|
||||
.replace("{{MODEL}}", model)
|
||||
.replace("{{MESSAGES}}", messagesJsonArray)
|
||||
|
||||
if (hasSystemContextPlaceholder) {
|
||||
requestBodyString = requestBodyString.replace("{{SYSTEM_CONTEXT}}", escapeJsonString(systemContext))
|
||||
}
|
||||
|
||||
if (isStream) {
|
||||
if (requestBodyString.contains("\"stream\": false")) {
|
||||
requestBodyString = requestBodyString.replace("\"stream\": false", "\"stream\": true")
|
||||
} else if (!requestBodyString.contains("\"stream\"")) {
|
||||
requestBodyString = requestBodyString.replaceFirst("{", "{\n \"stream\": true,\n")
|
||||
}
|
||||
}
|
||||
return requestBodyString
|
||||
}
|
||||
|
||||
private fun buildMultimodalMessageJson(
|
||||
role: String,
|
||||
content: String,
|
||||
attachments: List<LlmAttachment>,
|
||||
provider: LlmProvider
|
||||
): String {
|
||||
return when (provider) {
|
||||
LlmProvider.GEMINI -> {
|
||||
val parts = mutableListOf<String>()
|
||||
parts.add("{\"text\": \"${escapeJsonString(content)}\"}")
|
||||
attachments.forEach { att ->
|
||||
parts.add("{\"inlineData\": {\"mimeType\": \"${att.mimeType}\", \"data\": \"${att.base64Data}\"}}")
|
||||
}
|
||||
"{\"role\": \"$role\", \"parts\": [${parts.joinToString(",")}]}"
|
||||
}
|
||||
LlmProvider.ANTHROPIC -> {
|
||||
val contentArray = mutableListOf<String>()
|
||||
contentArray.add("{\"type\": \"text\", \"text\": \"${escapeJsonString(content)}\"}")
|
||||
attachments.forEach { att ->
|
||||
if (att.mimeType.startsWith("image/")) {
|
||||
contentArray.add("{\"type\": \"image\", \"source\": {\"type\": \"base64\", \"media_type\": \"${att.mimeType}\", \"data\": \"${att.base64Data}\"}}")
|
||||
} else if (att.mimeType == "application/pdf") {
|
||||
contentArray.add("{\"type\": \"document\", \"source\": {\"type\": \"base64\", \"media_type\": \"${att.mimeType}\", \"data\": \"${att.base64Data}\"}}")
|
||||
}
|
||||
}
|
||||
"{\"role\": \"$role\", \"content\": [${contentArray.joinToString(",")}]}"
|
||||
}
|
||||
else -> { // OpenAI, Groq, Ollama, DeepSeek, OpenRouter, Custom
|
||||
val contentArray = mutableListOf<String>()
|
||||
contentArray.add("{\"type\": \"text\", \"text\": \"${escapeJsonString(content)}\"}")
|
||||
attachments.forEach { att ->
|
||||
if (att.mimeType.startsWith("image/")) {
|
||||
contentArray.add("{\"type\": \"image_url\", \"image_url\": {\"url\": \"data:${att.mimeType};base64,${att.base64Data}\"}}")
|
||||
}
|
||||
}
|
||||
"{\"role\": \"$role\", \"content\": [${contentArray.joinToString(",")}]}"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun extractChunkContent(jsonStr: String, path: String): String? {
|
||||
var extracted = extractValueFromJson(jsonStr, path)
|
||||
if (extracted != null) return extracted
|
||||
|
||||
if (path == "choices[0].message.content") {
|
||||
extracted = extractValueFromJson(jsonStr, "choices[0].delta.content")
|
||||
if (extracted != null) return extracted
|
||||
}
|
||||
|
||||
for (fallbackPath in listOf("choices[0].delta.content", "message.content", "candidates[0].content.parts[0].text")) {
|
||||
extracted = extractValueFromJson(jsonStr, fallbackPath)
|
||||
if (extracted != null) return extracted
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private fun extractErrorMessage(errorBody: String): String? {
|
||||
return try {
|
||||
val element = json.parseToJsonElement(errorBody)
|
||||
if (element is kotlinx.serialization.json.JsonObject) {
|
||||
val errorObj = element["error"]
|
||||
if (errorObj is kotlinx.serialization.json.JsonObject) {
|
||||
val msg = errorObj["message"]
|
||||
if (msg is kotlinx.serialization.json.JsonPrimitive) return msg.content
|
||||
}
|
||||
val msg = element["message"] ?: element["error_description"] ?: element["msg"]
|
||||
if (msg is kotlinx.serialization.json.JsonPrimitive) return msg.content
|
||||
}
|
||||
null
|
||||
} catch (e: Exception) {
|
||||
errorBody.take(150)
|
||||
}
|
||||
}
|
||||
|
||||
private fun extractResponseContent(rawResponse: String, path: String): String? {
|
||||
try {
|
||||
if (rawResponse.contains("error_code")) {
|
||||
val errExtract = extractValueFromJson(rawResponse, "msg")
|
||||
if (errExtract != null) return errExtract
|
||||
}
|
||||
|
||||
val extracted = extractValueFromJson(rawResponse, path) ?: return null
|
||||
return extracted.trim()
|
||||
} catch (e: Exception) {
|
||||
Log.e("FlexibleLlmAdapter", "Error extracting LLM response", e)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
private fun extractValueFromJson(jsonStr: String, path: String): String? {
|
||||
return try {
|
||||
val jsonElement = json.parseToJsonElement(jsonStr)
|
||||
val tokens = path.split(Regex("[\\.\\[\\]]+")).filter { it.isNotEmpty() }
|
||||
var currentElement = jsonElement
|
||||
for (token in tokens) {
|
||||
currentElement = when (currentElement) {
|
||||
is kotlinx.serialization.json.JsonObject -> {
|
||||
currentElement[token] ?: return null
|
||||
}
|
||||
is kotlinx.serialization.json.JsonArray -> {
|
||||
val index = token.toIntOrNull() ?: return null
|
||||
if (index in 0 until currentElement.size) {
|
||||
currentElement[index]
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
}
|
||||
else -> return null
|
||||
}
|
||||
}
|
||||
if (currentElement is kotlinx.serialization.json.JsonPrimitive) {
|
||||
currentElement.content
|
||||
} else {
|
||||
currentElement.toString()
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
private fun escapeJsonString(input: String): String {
|
||||
val builder = StringBuilder()
|
||||
for (ch in input) {
|
||||
when (ch) {
|
||||
'"' -> builder.append("\\\"")
|
||||
'\\' -> builder.append("\\\\")
|
||||
'\n' -> builder.append("\\n")
|
||||
'\r' -> builder.append("\\r")
|
||||
'\t' -> builder.append("\\t")
|
||||
else -> {
|
||||
if (ch.code < 0x20) {
|
||||
builder.append(String.format("\\u%04x", ch.code))
|
||||
} else {
|
||||
builder.append(ch)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return builder.toString()
|
||||
}
|
||||
}
|
||||
15
app/src/main/java/com/app/assistant/llm/LlmAdapter.kt
Normal file
15
app/src/main/java/com/app/assistant/llm/LlmAdapter.kt
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
package com.app.assistant.llm
|
||||
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
|
||||
interface LlmAdapter {
|
||||
suspend fun generateResponse(
|
||||
systemContext: String,
|
||||
messages: List<LlmMessage>
|
||||
): String?
|
||||
|
||||
fun generateResponseStream(
|
||||
systemContext: String,
|
||||
messages: List<LlmMessage>
|
||||
): Flow<String>
|
||||
}
|
||||
13
app/src/main/java/com/app/assistant/llm/LlmConfig.kt
Normal file
13
app/src/main/java/com/app/assistant/llm/LlmConfig.kt
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
package com.app.assistant.llm
|
||||
|
||||
data class LlmConfig(
|
||||
val url: String,
|
||||
val method: String = "POST",
|
||||
val headers: Map<String, String> = emptyMap(),
|
||||
val responsePath: String,
|
||||
val messageFormat: String,
|
||||
val systemRole: String? = null,
|
||||
val userRole: String = "user",
|
||||
val assistantRole: String = "assistant",
|
||||
val requestTemplate: String
|
||||
)
|
||||
17
app/src/main/java/com/app/assistant/llm/LlmMessage.kt
Normal file
17
app/src/main/java/com/app/assistant/llm/LlmMessage.kt
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
package com.app.assistant.llm
|
||||
|
||||
import kotlinx.serialization.Serializable
|
||||
|
||||
@Serializable
|
||||
data class LlmAttachment(
|
||||
val base64Data: String,
|
||||
val mimeType: String,
|
||||
val fileName: String
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class LlmMessage(
|
||||
val role: String,
|
||||
val content: String,
|
||||
val attachments: List<LlmAttachment>? = null
|
||||
)
|
||||
250
app/src/main/java/com/app/assistant/llm/LlmProvider.kt
Normal file
250
app/src/main/java/com/app/assistant/llm/LlmProvider.kt
Normal file
|
|
@ -0,0 +1,250 @@
|
|||
package com.app.assistant.llm
|
||||
|
||||
enum class LlmProvider(
|
||||
val displayName: String,
|
||||
val defaultUrl: String,
|
||||
val defaultModel: String,
|
||||
val suggestedModels: List<String>,
|
||||
val config: LlmConfig
|
||||
) {
|
||||
GROQ(
|
||||
displayName = "Groq",
|
||||
defaultUrl = "https://api.groq.com/openai/v1/chat/completions",
|
||||
defaultModel = "llama-3.3-70b-versatile",
|
||||
suggestedModels = listOf(
|
||||
"llama-3.3-70b-versatile",
|
||||
"llama-3.1-8b-instant",
|
||||
"mixtral-8x7b-32768",
|
||||
"gemma2-9b-it"
|
||||
),
|
||||
config = LlmConfig(
|
||||
url = "https://api.groq.com/openai/v1/chat/completions",
|
||||
headers = mapOf(
|
||||
"Authorization" to "Bearer {{API_KEY}}",
|
||||
"Content-Type" to "application/json"
|
||||
),
|
||||
responsePath = "choices[0].message.content",
|
||||
messageFormat = "{\"role\": \"{{ROLE}}\", \"content\": \"{{CONTENT}}\"}",
|
||||
systemRole = "system",
|
||||
userRole = "user",
|
||||
assistantRole = "assistant",
|
||||
requestTemplate = """
|
||||
{
|
||||
"model": "{{MODEL}}",
|
||||
"messages": {{MESSAGES}},
|
||||
"temperature": 1.0,
|
||||
"top_p": 1.0
|
||||
}
|
||||
""".trimIndent()
|
||||
)
|
||||
),
|
||||
OPENAI(
|
||||
displayName = "OpenAI",
|
||||
defaultUrl = "https://api.openai.com/v1/chat/completions",
|
||||
defaultModel = "gpt-4o",
|
||||
suggestedModels = listOf(
|
||||
"gpt-4o",
|
||||
"gpt-4o-mini",
|
||||
"gpt-4-turbo",
|
||||
"gpt-3.5-turbo"
|
||||
),
|
||||
config = LlmConfig(
|
||||
url = "https://api.openai.com/v1/chat/completions",
|
||||
headers = mapOf(
|
||||
"Authorization" to "Bearer {{API_KEY}}",
|
||||
"Content-Type" to "application/json"
|
||||
),
|
||||
responsePath = "choices[0].message.content",
|
||||
messageFormat = "{\"role\": \"{{ROLE}}\", \"content\": \"{{CONTENT}}\"}",
|
||||
systemRole = "system",
|
||||
userRole = "user",
|
||||
assistantRole = "assistant",
|
||||
requestTemplate = """
|
||||
{
|
||||
"model": "{{MODEL}}",
|
||||
"messages": {{MESSAGES}},
|
||||
"temperature": 0.7
|
||||
}
|
||||
""".trimIndent()
|
||||
)
|
||||
),
|
||||
OPEN_ROUTER(
|
||||
displayName = "OpenRouter",
|
||||
defaultUrl = "https://openrouter.ai/api/v1/chat/completions",
|
||||
defaultModel = "meta-llama/llama-3-8b-instruct:free",
|
||||
suggestedModels = listOf(
|
||||
"meta-llama/llama-3-8b-instruct:free",
|
||||
"google/gemini-2.0-flash-exp:free",
|
||||
"deepseek/deepseek-chat",
|
||||
"mistralai/mistral-7b-instruct:free"
|
||||
),
|
||||
config = LlmConfig(
|
||||
url = "https://openrouter.ai/api/v1/chat/completions",
|
||||
headers = mapOf(
|
||||
"Authorization" to "Bearer {{API_KEY}}",
|
||||
"Content-Type" to "application/json"
|
||||
),
|
||||
responsePath = "choices[0].message.content",
|
||||
messageFormat = "{\"role\": \"{{ROLE}}\", \"content\": \"{{CONTENT}}\"}",
|
||||
systemRole = "system",
|
||||
userRole = "user",
|
||||
assistantRole = "assistant",
|
||||
requestTemplate = """
|
||||
{
|
||||
"model": "{{MODEL}}",
|
||||
"messages": {{MESSAGES}}
|
||||
}
|
||||
""".trimIndent()
|
||||
)
|
||||
),
|
||||
DEEPSEEK(
|
||||
displayName = "DeepSeek",
|
||||
defaultUrl = "https://api.deepseek.com/chat/completions",
|
||||
defaultModel = "deepseek-chat",
|
||||
suggestedModels = listOf(
|
||||
"deepseek-chat",
|
||||
"deepseek-coder"
|
||||
),
|
||||
config = LlmConfig(
|
||||
url = "https://api.deepseek.com/chat/completions",
|
||||
headers = mapOf(
|
||||
"Authorization" to "Bearer {{API_KEY}}",
|
||||
"Content-Type" to "application/json"
|
||||
),
|
||||
responsePath = "choices[0].message.content",
|
||||
messageFormat = "{\"role\": \"{{ROLE}}\", \"content\": \"{{CONTENT}}\"}",
|
||||
systemRole = "system",
|
||||
userRole = "user",
|
||||
assistantRole = "assistant",
|
||||
requestTemplate = """
|
||||
{
|
||||
"model": "{{MODEL}}",
|
||||
"messages": {{MESSAGES}},
|
||||
"temperature": 1.0
|
||||
}
|
||||
""".trimIndent()
|
||||
)
|
||||
),
|
||||
OLLAMA(
|
||||
displayName = "Ollama (Local)",
|
||||
defaultUrl = "http://10.0.2.2:11434/api/chat",
|
||||
defaultModel = "llama3",
|
||||
suggestedModels = listOf(
|
||||
"llama3",
|
||||
"llama3.1",
|
||||
"mistral",
|
||||
"gemma2",
|
||||
"phi3"
|
||||
),
|
||||
config = LlmConfig(
|
||||
url = "http://10.0.2.2:11434/api/chat",
|
||||
headers = mapOf(
|
||||
"Content-Type" to "application/json"
|
||||
),
|
||||
responsePath = "message.content",
|
||||
messageFormat = "{\"role\": \"{{ROLE}}\", \"content\": \"{{CONTENT}}\"}",
|
||||
systemRole = "system",
|
||||
userRole = "user",
|
||||
assistantRole = "assistant",
|
||||
requestTemplate = """
|
||||
{
|
||||
"model": "{{MODEL}}",
|
||||
"messages": {{MESSAGES}},
|
||||
"stream": false
|
||||
}
|
||||
""".trimIndent()
|
||||
)
|
||||
),
|
||||
GEMINI(
|
||||
displayName = "Google Gemini",
|
||||
defaultUrl = "https://generativelanguage.googleapis.com/v1beta/models/gemini-1.5-flash:generateContent",
|
||||
defaultModel = "gemini-1.5-flash",
|
||||
suggestedModels = listOf(
|
||||
"gemini-2.5-flash",
|
||||
"gemini-2.5-pro",
|
||||
"gemini-1.5-flash",
|
||||
"gemini-1.5-pro",
|
||||
"gemini-1.0-pro"
|
||||
),
|
||||
config = LlmConfig(
|
||||
url = "https://generativelanguage.googleapis.com/v1beta/models/{{MODEL}}:generateContent?key={{API_KEY}}",
|
||||
headers = mapOf(
|
||||
"Content-Type" to "application/json"
|
||||
),
|
||||
responsePath = "candidates[0].content.parts[0].text",
|
||||
messageFormat = "{\"role\": \"{{ROLE}}\", \"parts\": [{\"text\": \"{{CONTENT}}\"}]}",
|
||||
systemRole = null,
|
||||
userRole = "user",
|
||||
assistantRole = "model",
|
||||
requestTemplate = """
|
||||
{
|
||||
"contents": {{MESSAGES}},
|
||||
"systemInstruction": {
|
||||
"parts": [
|
||||
{
|
||||
"text": "{{SYSTEM_CONTEXT}}"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
""".trimIndent()
|
||||
)
|
||||
),
|
||||
ANTHROPIC(
|
||||
displayName = "Anthropic Claude",
|
||||
defaultUrl = "https://api.anthropic.com/v1/messages",
|
||||
defaultModel = "claude-3-5-sonnet-20240620",
|
||||
suggestedModels = listOf(
|
||||
"claude-3-5-sonnet-latest",
|
||||
"claude-3-5-haiku-latest",
|
||||
"claude-3-opus-latest",
|
||||
"claude-3-5-sonnet-20240620",
|
||||
"claude-3-5-haiku-20241022"
|
||||
),
|
||||
config = LlmConfig(
|
||||
url = "https://api.anthropic.com/v1/messages",
|
||||
headers = mapOf(
|
||||
"x-api-key" to "{{API_KEY}}",
|
||||
"anthropic-version" to "2023-06-01",
|
||||
"content-type" to "application/json"
|
||||
),
|
||||
responsePath = "content[0].text",
|
||||
messageFormat = "{\"role\": \"{{ROLE}}\", \"content\": \"{{CONTENT}}\"}",
|
||||
systemRole = null,
|
||||
userRole = "user",
|
||||
assistantRole = "assistant",
|
||||
requestTemplate = """
|
||||
{
|
||||
"model": "{{MODEL}}",
|
||||
"max_tokens": 1024,
|
||||
"system": "{{SYSTEM_CONTEXT}}",
|
||||
"messages": {{MESSAGES}}
|
||||
}
|
||||
""".trimIndent()
|
||||
)
|
||||
),
|
||||
CUSTOM(
|
||||
displayName = "Custom Config",
|
||||
defaultUrl = "https://api.example.com/v1/chat/completions",
|
||||
defaultModel = "custom-model",
|
||||
suggestedModels = emptyList(),
|
||||
config = LlmConfig(
|
||||
url = "https://api.example.com/v1/chat/completions",
|
||||
headers = mapOf(
|
||||
"Authorization" to "Bearer {{API_KEY}}",
|
||||
"Content-Type" to "application/json"
|
||||
),
|
||||
responsePath = "choices[0].message.content",
|
||||
messageFormat = "{\"role\": \"{{ROLE}}\", \"content\": \"{{CONTENT}}\"}",
|
||||
systemRole = "system",
|
||||
userRole = "user",
|
||||
assistantRole = "assistant",
|
||||
requestTemplate = """
|
||||
{
|
||||
"model": "{{MODEL}}",
|
||||
"messages": {{MESSAGES}}
|
||||
}
|
||||
""".trimIndent()
|
||||
)
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
package com.app.assistant.llm
|
||||
|
||||
data class ModelCapabilities(
|
||||
val hasImageInput: Boolean,
|
||||
val hasAudioInput: Boolean,
|
||||
val hasVideoInput: Boolean,
|
||||
val hasDocumentInput: Boolean
|
||||
)
|
||||
355
app/src/main/java/com/app/assistant/llm/ModelCapabilityProber.kt
Normal file
355
app/src/main/java/com/app/assistant/llm/ModelCapabilityProber.kt
Normal file
|
|
@ -0,0 +1,355 @@
|
|||
package com.app.assistant.llm
|
||||
|
||||
import android.util.Log
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import okhttp3.MediaType.Companion.toMediaTypeOrNull
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import okhttp3.RequestBody.Companion.toRequestBody
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.jsonObject
|
||||
import kotlinx.serialization.json.jsonArray
|
||||
import kotlinx.serialization.json.jsonPrimitive
|
||||
|
||||
|
||||
object ModelCapabilityProber {
|
||||
|
||||
fun resolveCapabilitiesOffline(provider: LlmProvider, modelName: String): ModelCapabilities {
|
||||
val model = modelName.lowercase()
|
||||
return when (provider) {
|
||||
LlmProvider.GEMINI -> {
|
||||
val isMultimodal = model.contains("gemini-1.5") || model.contains("gemini-2.0") || model.contains("gemini-pro") || model.contains("gemini-2.5")
|
||||
ModelCapabilities(
|
||||
hasImageInput = isMultimodal || model.contains("vision"),
|
||||
hasAudioInput = isMultimodal,
|
||||
hasVideoInput = isMultimodal,
|
||||
hasDocumentInput = isMultimodal
|
||||
)
|
||||
}
|
||||
LlmProvider.OPENAI -> {
|
||||
val hasVision = model.contains("gpt-4o") || model.contains("vision") || model.contains("gpt-4-turbo")
|
||||
val hasAudio = model.contains("gpt-4o-audio")
|
||||
ModelCapabilities(
|
||||
hasImageInput = hasVision,
|
||||
hasAudioInput = hasAudio,
|
||||
hasVideoInput = false,
|
||||
hasDocumentInput = false
|
||||
)
|
||||
}
|
||||
LlmProvider.ANTHROPIC -> {
|
||||
val supportsVision = model.contains("claude-3")
|
||||
ModelCapabilities(
|
||||
hasImageInput = supportsVision,
|
||||
hasAudioInput = false,
|
||||
hasVideoInput = false,
|
||||
hasDocumentInput = supportsVision
|
||||
)
|
||||
}
|
||||
LlmProvider.GROQ -> {
|
||||
ModelCapabilities(
|
||||
hasImageInput = model.contains("vision") || model.contains("scout"),
|
||||
hasAudioInput = false,
|
||||
hasVideoInput = false,
|
||||
hasDocumentInput = false
|
||||
)
|
||||
}
|
||||
LlmProvider.OLLAMA -> {
|
||||
val isMultimodal = model.contains("llava") || model.contains("bakllava") || model.contains("minicpm") || model.contains("vision")
|
||||
ModelCapabilities(
|
||||
hasImageInput = isMultimodal,
|
||||
hasAudioInput = false,
|
||||
hasVideoInput = false,
|
||||
hasDocumentInput = false
|
||||
)
|
||||
}
|
||||
LlmProvider.DEEPSEEK -> {
|
||||
ModelCapabilities(
|
||||
hasImageInput = model.contains("vl"),
|
||||
hasAudioInput = false,
|
||||
hasVideoInput = false,
|
||||
hasDocumentInput = false
|
||||
)
|
||||
}
|
||||
LlmProvider.OPEN_ROUTER -> {
|
||||
val hasVision = model.contains("vision") || model.contains("llava") || model.contains("gpt-4o") || model.contains("claude-3") || model.contains("gemini")
|
||||
val hasAudio = model.contains("gemini") || model.contains("audio")
|
||||
val hasVideo = model.contains("gemini")
|
||||
val hasDoc = model.contains("gemini") || model.contains("claude-3")
|
||||
ModelCapabilities(
|
||||
hasImageInput = hasVision,
|
||||
hasAudioInput = hasAudio,
|
||||
hasVideoInput = hasVideo,
|
||||
hasDocumentInput = hasDoc
|
||||
)
|
||||
}
|
||||
LlmProvider.CUSTOM -> {
|
||||
ModelCapabilities(
|
||||
hasImageInput = false,
|
||||
hasAudioInput = false,
|
||||
hasVideoInput = false,
|
||||
hasDocumentInput = false
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun probeTextConnection(
|
||||
client: OkHttpClient,
|
||||
apiKey: String,
|
||||
model: String,
|
||||
config: LlmConfig
|
||||
): Boolean = withContext(Dispatchers.IO) {
|
||||
try {
|
||||
val url = config.url
|
||||
.replace("{{API_KEY}}", apiKey)
|
||||
.replace("{{MODEL}}", model)
|
||||
|
||||
val requestBuilder = Request.Builder().url(url)
|
||||
for ((key, value) in config.headers) {
|
||||
val resolvedValue = value
|
||||
.replace("{{API_KEY}}", apiKey)
|
||||
.replace("{{MODEL}}", model)
|
||||
requestBuilder.addHeader(key, resolvedValue)
|
||||
}
|
||||
|
||||
val hasSystemContextPlaceholder = config.requestTemplate.contains("{{SYSTEM_CONTEXT}}")
|
||||
|
||||
val escapedContent = "say ok"
|
||||
val msgJson = config.messageFormat
|
||||
.replace("{{ROLE}}", config.userRole)
|
||||
.replace("{{CONTENT}}", escapedContent)
|
||||
|
||||
val messagesJsonArray = "[$msgJson]"
|
||||
var requestBodyString = config.requestTemplate
|
||||
.replace("{{MODEL}}", model)
|
||||
.replace("{{MESSAGES}}", messagesJsonArray)
|
||||
|
||||
if (hasSystemContextPlaceholder) {
|
||||
requestBodyString = requestBodyString.replace("{{SYSTEM_CONTEXT}}", "test connection")
|
||||
}
|
||||
|
||||
val mediaType = config.headers["Content-Type"]?.toMediaTypeOrNull() ?: "application/json".toMediaTypeOrNull()
|
||||
val requestBody = requestBodyString.toRequestBody(mediaType)
|
||||
requestBuilder.post(requestBody)
|
||||
|
||||
val request = requestBuilder.build()
|
||||
client.newCall(request).execute().use { response ->
|
||||
Log.d("ModelCapabilityProber", "Text test response code: ${response.code}")
|
||||
response.isSuccessful
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("ModelCapabilityProber", "Text test failed", e)
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun probeVisionConnection(
|
||||
client: OkHttpClient,
|
||||
provider: LlmProvider,
|
||||
apiKey: String,
|
||||
model: String,
|
||||
config: LlmConfig
|
||||
): Boolean = withContext(Dispatchers.IO) {
|
||||
try {
|
||||
val url = config.url
|
||||
.replace("{{API_KEY}}", apiKey)
|
||||
.replace("{{MODEL}}", model)
|
||||
|
||||
val requestBuilder = Request.Builder().url(url)
|
||||
for ((key, value) in config.headers) {
|
||||
val resolvedValue = value
|
||||
.replace("{{API_KEY}}", apiKey)
|
||||
.replace("{{MODEL}}", model)
|
||||
requestBuilder.addHeader(key, resolvedValue)
|
||||
}
|
||||
|
||||
// 2x2 transparent PNG base64
|
||||
val base64Image = "iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAC0lEQVR4nGNgQAcAABIAAXfx+gAAAAAASUVORK5CYII="
|
||||
|
||||
val requestBodyString = when (provider) {
|
||||
LlmProvider.GEMINI -> {
|
||||
"""
|
||||
{
|
||||
"contents": [{
|
||||
"role": "user",
|
||||
"parts": [
|
||||
{ "text": "is this image ok?" },
|
||||
{ "inlineData": { "mimeType": "image/png", "data": "$base64Image" } }
|
||||
]
|
||||
}],
|
||||
"generationConfig": { "maxOutputTokens": 1 }
|
||||
}
|
||||
""".trimIndent()
|
||||
}
|
||||
LlmProvider.ANTHROPIC -> {
|
||||
"""
|
||||
{
|
||||
"model": "$model",
|
||||
"max_tokens": 10,
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{ "type": "text", "text": "is this image ok?" },
|
||||
{ "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "$base64Image" } }
|
||||
]
|
||||
}]
|
||||
}
|
||||
""".trimIndent()
|
||||
}
|
||||
else -> { // OpenAI-compatible schema (OpenAI, Groq, OpenRouter, Custom, Ollama, DeepSeek)
|
||||
"""
|
||||
{
|
||||
"model": "$model",
|
||||
"max_tokens": 10,
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{ "type": "text", "text": "is this image ok?" },
|
||||
{ "type": "image_url", "image_url": { "url": "data:image/png;base64,$base64Image" } }
|
||||
]
|
||||
}]
|
||||
}
|
||||
""".trimIndent()
|
||||
}
|
||||
}
|
||||
|
||||
val mediaType = "application/json".toMediaTypeOrNull()
|
||||
val requestBody = requestBodyString.toRequestBody(mediaType)
|
||||
requestBuilder.post(requestBody)
|
||||
|
||||
val request = requestBuilder.build()
|
||||
client.newCall(request).execute().use { response ->
|
||||
Log.d("ModelCapabilityProber", "Vision test response code: ${response.code}")
|
||||
response.isSuccessful
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("ModelCapabilityProber", "Vision test failed", e)
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun fetchAvailableModels(
|
||||
client: OkHttpClient,
|
||||
provider: LlmProvider,
|
||||
apiKey: String,
|
||||
customUrl: String = "",
|
||||
customHeaders: String = ""
|
||||
): List<String> = withContext(Dispatchers.IO) {
|
||||
try {
|
||||
val url = when (provider) {
|
||||
LlmProvider.GROQ -> "https://api.groq.com/openai/v1/models"
|
||||
LlmProvider.OPENAI -> "https://api.openai.com/v1/models"
|
||||
LlmProvider.OPEN_ROUTER -> "https://openrouter.ai/api/v1/models"
|
||||
LlmProvider.DEEPSEEK -> "https://api.deepseek.com/v1/models"
|
||||
LlmProvider.OLLAMA -> {
|
||||
val configUrl = provider.config.url
|
||||
if (configUrl.endsWith("/api/chat")) {
|
||||
configUrl.substringBeforeLast("/api/chat") + "/api/tags"
|
||||
} else {
|
||||
"http://10.0.2.2:11434/api/tags"
|
||||
}
|
||||
}
|
||||
LlmProvider.GEMINI -> "https://generativelanguage.googleapis.com/v1beta/models?key=$apiKey"
|
||||
LlmProvider.ANTHROPIC -> "https://api.anthropic.com/v1/models"
|
||||
LlmProvider.CUSTOM -> {
|
||||
if (customUrl.isBlank()) return@withContext emptyList<String>()
|
||||
if (customUrl.endsWith("/chat/completions")) {
|
||||
customUrl.substringBeforeLast("/chat/completions") + "/models"
|
||||
} else if (customUrl.endsWith("/v1/chat/completions")) {
|
||||
customUrl.substringBeforeLast("/v1/chat/completions") + "/v1/models"
|
||||
} else {
|
||||
null
|
||||
}
|
||||
}
|
||||
} ?: return@withContext emptyList<String>()
|
||||
|
||||
val requestBuilder = Request.Builder().url(url).get()
|
||||
|
||||
when (provider) {
|
||||
LlmProvider.GROQ, LlmProvider.OPENAI, LlmProvider.OPEN_ROUTER, LlmProvider.DEEPSEEK -> {
|
||||
requestBuilder.addHeader("Authorization", "Bearer $apiKey")
|
||||
requestBuilder.addHeader("Content-Type", "application/json")
|
||||
}
|
||||
LlmProvider.ANTHROPIC -> {
|
||||
requestBuilder.addHeader("x-api-key", apiKey)
|
||||
requestBuilder.addHeader("anthropic-version", "2023-06-01")
|
||||
requestBuilder.addHeader("content-type", "application/json")
|
||||
}
|
||||
LlmProvider.CUSTOM -> {
|
||||
if (customHeaders.isNotBlank()) {
|
||||
try {
|
||||
val parsed = parseHeadersJson(customHeaders)
|
||||
for ((k, v) in parsed) {
|
||||
requestBuilder.addHeader(k, v.replace("{{API_KEY}}", apiKey))
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("ModelCapabilityProber", "Failed to parse custom headers", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
else -> {}
|
||||
}
|
||||
|
||||
val request = requestBuilder.build()
|
||||
client.newCall(request).execute().use { response ->
|
||||
if (!response.isSuccessful) {
|
||||
Log.e("ModelCapabilityProber", "Model fetch response not successful: ${response.code}")
|
||||
return@withContext emptyList<String>()
|
||||
}
|
||||
val bodyString = response.body?.string() ?: return@withContext emptyList<String>()
|
||||
|
||||
val element = Json.parseToJsonElement(bodyString)
|
||||
val modelsList = mutableListOf<String>()
|
||||
|
||||
when (provider) {
|
||||
LlmProvider.OLLAMA -> {
|
||||
val modelsArray = element.jsonObject["models"]?.jsonArray
|
||||
modelsArray?.forEach { modelObj ->
|
||||
modelObj.jsonObject["name"]?.jsonPrimitive?.content?.let { name ->
|
||||
modelsList.add(name)
|
||||
}
|
||||
}
|
||||
}
|
||||
LlmProvider.GEMINI -> {
|
||||
val modelsArray = element.jsonObject["models"]?.jsonArray
|
||||
modelsArray?.forEach { modelObj ->
|
||||
modelObj.jsonObject["name"]?.jsonPrimitive?.content?.let { name ->
|
||||
val cleanedName = if (name.startsWith("models/")) name.substringAfter("models/") else name
|
||||
modelsList.add(cleanedName)
|
||||
}
|
||||
}
|
||||
}
|
||||
else -> {
|
||||
val dataArray = element.jsonObject["data"]?.jsonArray
|
||||
dataArray?.forEach { modelObj ->
|
||||
modelObj.jsonObject["id"]?.jsonPrimitive?.content?.let { id ->
|
||||
modelsList.add(id)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
modelsList.distinct().sorted()
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("ModelCapabilityProber", "Failed to fetch models", e)
|
||||
emptyList()
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseHeadersJson(jsonStr: String): Map<String, String> {
|
||||
return try {
|
||||
val element = Json.parseToJsonElement(jsonStr)
|
||||
if (element is kotlinx.serialization.json.JsonObject) {
|
||||
element.mapValues { (_, value) ->
|
||||
if (value is kotlinx.serialization.json.JsonPrimitive) value.content else value.toString()
|
||||
}
|
||||
} else {
|
||||
emptyMap()
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
emptyMap()
|
||||
}
|
||||
}
|
||||
}
|
||||
12
app/src/main/java/com/app/assistant/model/Attachment.kt
Normal file
12
app/src/main/java/com/app/assistant/model/Attachment.kt
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
package com.app.assistant.model
|
||||
|
||||
import com.app.assistant.util.IdGenerator
|
||||
import java.io.Serializable
|
||||
|
||||
data class Attachment(
|
||||
val id: Long = IdGenerator.nextId(),
|
||||
val filePath: String,
|
||||
val mimeType: String,
|
||||
val fileName: String,
|
||||
val iv: String
|
||||
) : Serializable
|
||||
|
|
@ -1,3 +1,7 @@
|
|||
package com.app.assistant.model
|
||||
|
||||
data class Contact(val id: String ,val name: String, val phoneNumber: String)
|
||||
data class Contact(
|
||||
val id: String,
|
||||
val name: String,
|
||||
val phoneNumber: String,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,43 @@
|
|||
package com.app.assistant.model
|
||||
import com.app.assistant.util.Category
|
||||
import com.app.assistant.util.IdGenerator
|
||||
import java.net.URI
|
||||
|
||||
data class Conversation(var id: Long = System.currentTimeMillis() * 1_000_000 + (System.nanoTime() % 1_000_000), var englishText: String, var translatedText: String, val isMe: Boolean, val isLoading: Boolean = false, var category: String = Category.OTHER.name, val contentURL: String ="", val navigationURI: URI=URI("") )
|
||||
data class Conversation(
|
||||
var id: Long = IdGenerator.nextId(),
|
||||
var text: String,
|
||||
val isMe: Boolean,
|
||||
val isLoading: Boolean = false,
|
||||
var category: String = Category.OTHER.name,
|
||||
val contentURL: String = "",
|
||||
val navigationURI: URI = URI(""),
|
||||
val isStreaming: Boolean = false,
|
||||
val attachments: List<Attachment> = emptyList(),
|
||||
) {
|
||||
fun getThinkingProcess(): String? {
|
||||
val start = text.indexOf("<think>")
|
||||
val end = text.indexOf("</think>")
|
||||
if (start == -1) return null
|
||||
return if (end == -1) {
|
||||
text.substring(start + 7)
|
||||
} else {
|
||||
text.substring(start + 7, end)
|
||||
}
|
||||
}
|
||||
|
||||
fun getActualAnswer(): String {
|
||||
val end = text.indexOf("</think>")
|
||||
val mainText = if (end == -1) {
|
||||
if (text.contains("<think>")) "" else text
|
||||
} else {
|
||||
text.substring(end + 8)
|
||||
}
|
||||
if (isMe) {
|
||||
val fileIndex = mainText.indexOf("\n\n[Attached File:")
|
||||
if (fileIndex != -1) {
|
||||
return mainText.substring(0, fileIndex)
|
||||
}
|
||||
}
|
||||
return mainText
|
||||
}
|
||||
}
|
||||
|
|
|
|||
15
app/src/main/java/com/app/assistant/model/DeviceAction.kt
Normal file
15
app/src/main/java/com/app/assistant/model/DeviceAction.kt
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
package com.app.assistant.model
|
||||
|
||||
sealed class DeviceAction {
|
||||
data class MakeCall(val phoneNumber: String) : DeviceAction()
|
||||
data class NavigateTo(val location: String) : DeviceAction()
|
||||
data class PlaySong(val videoId: String) : DeviceAction()
|
||||
data class SearchSong(val query: String) : DeviceAction()
|
||||
data class SetAlarm(
|
||||
val message: String,
|
||||
val hour: Int,
|
||||
val minutes: Int,
|
||||
val repeatDays: List<Int>,
|
||||
val isReminder: Boolean
|
||||
) : DeviceAction()
|
||||
}
|
||||
|
|
@ -1,3 +1,6 @@
|
|||
package com.app.assistant.model
|
||||
|
||||
data class Group(val groupId: Long, val title: String)
|
||||
data class Group(
|
||||
val groupId: Long,
|
||||
val title: String,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,113 @@
|
|||
package com.app.assistant.repository
|
||||
|
||||
import android.annotation.SuppressLint
|
||||
import android.content.Context
|
||||
import android.provider.ContactsContract
|
||||
import com.app.assistant.model.Contact
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlin.math.max
|
||||
|
||||
class ContactsRepository(
|
||||
private val context: Context,
|
||||
) {
|
||||
@SuppressLint("Range")
|
||||
suspend fun searchContact(prompt: String): Contact? {
|
||||
val contentResolver = context.contentResolver
|
||||
val contactList = mutableListOf<Contact>()
|
||||
|
||||
withContext(Dispatchers.IO) {
|
||||
val cursor =
|
||||
contentResolver.query(
|
||||
ContactsContract.Data.CONTENT_URI,
|
||||
arrayOf(
|
||||
ContactsContract.Data.CONTACT_ID,
|
||||
ContactsContract.Data.DISPLAY_NAME,
|
||||
ContactsContract.CommonDataKinds.Phone.NUMBER,
|
||||
),
|
||||
"${ContactsContract.Data.MIMETYPE} =? AND ${ContactsContract.Data.CONTACT_ID} IS NOT NULL",
|
||||
arrayOf(ContactsContract.CommonDataKinds.Phone.CONTENT_ITEM_TYPE),
|
||||
null,
|
||||
)
|
||||
|
||||
cursor?.use { contactCursor ->
|
||||
while (contactCursor.moveToNext()) {
|
||||
val contactId = contactCursor.getString(contactCursor.getColumnIndex(ContactsContract.Data.CONTACT_ID))
|
||||
val name = contactCursor.getString(contactCursor.getColumnIndex(ContactsContract.Data.DISPLAY_NAME))
|
||||
val phoneNumber = contactCursor.getString(contactCursor.getColumnIndex(ContactsContract.CommonDataKinds.Phone.NUMBER))
|
||||
contactList.add(Contact(contactId, name, phoneNumber))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val contactName = cleanAndExtractName(prompt)
|
||||
return findBestMatch(contactName, contactList)
|
||||
}
|
||||
|
||||
private fun cleanAndExtractName(command: String): String {
|
||||
val stopWords = setOf("can", "you", "make", "call", "dial", "number", "to", "please", "the", "a", "on")
|
||||
val words = command.lowercase().trim().split(" ")
|
||||
return words.filter { !stopWords.contains(it) }.joinToString(" ")
|
||||
}
|
||||
|
||||
private fun findBestMatch(
|
||||
inputString: String,
|
||||
nameList: List<Contact>,
|
||||
similarityThreshold: Double = 0.7,
|
||||
): Contact? {
|
||||
if (inputString.isEmpty()) {
|
||||
return null
|
||||
}
|
||||
val inputLower = inputString.lowercase().trim()
|
||||
var exactMatch: Contact? = null
|
||||
var bestSubstringMatch: Contact? = null
|
||||
var bestFuzzyMatch: Contact? = null
|
||||
var bestFuzzyScore = 0.0
|
||||
|
||||
for (contact in nameList) {
|
||||
val nameLower = contact.name.lowercase().trim()
|
||||
|
||||
// Check for exact match
|
||||
if (inputLower == nameLower) {
|
||||
return contact
|
||||
}
|
||||
|
||||
// Check for exact match within tokens
|
||||
val tokens = nameLower.split(" ")
|
||||
for (token in tokens) {
|
||||
when {
|
||||
inputLower == token -> {
|
||||
exactMatch = contact
|
||||
}
|
||||
|
||||
token.contains(inputLower) -> {
|
||||
bestSubstringMatch = contact
|
||||
}
|
||||
|
||||
else -> {
|
||||
val similarityScore = stringSimilarity(inputLower, token)
|
||||
if (similarityScore > bestFuzzyScore) {
|
||||
bestFuzzyScore = similarityScore
|
||||
bestFuzzyMatch = contact
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return when {
|
||||
exactMatch != null -> exactMatch
|
||||
bestSubstringMatch != null -> bestSubstringMatch
|
||||
bestFuzzyScore >= similarityThreshold -> bestFuzzyMatch
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
|
||||
private fun stringSimilarity(
|
||||
a: String,
|
||||
b: String,
|
||||
): Double {
|
||||
val matcher = a.commonPrefixWith(b).length.toDouble() / max(a.length, b.length).toDouble()
|
||||
return matcher
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,184 @@
|
|||
package com.app.assistant.repository
|
||||
|
||||
import android.content.Context
|
||||
import android.content.SharedPreferences
|
||||
import androidx.security.crypto.EncryptedSharedPreferences
|
||||
import androidx.security.crypto.MasterKeys
|
||||
|
||||
class SettingsRepository(
|
||||
val context: Context,
|
||||
) {
|
||||
private val sharedPreferences: SharedPreferences =
|
||||
context.getSharedPreferences("app_prefs", Context.MODE_PRIVATE)
|
||||
|
||||
private val securedPreferences: SharedPreferences =
|
||||
EncryptedSharedPreferences.create(
|
||||
"secure_prefs",
|
||||
MasterKeys.getOrCreate(MasterKeys.AES256_GCM_SPEC),
|
||||
context.applicationContext,
|
||||
EncryptedSharedPreferences.PrefKeyEncryptionScheme.AES256_SIV,
|
||||
EncryptedSharedPreferences.PrefValueEncryptionScheme.AES256_GCM,
|
||||
)
|
||||
|
||||
fun getIsTranslationEnabled(): Boolean = sharedPreferences.getBoolean("is_translation_enabled", false)
|
||||
|
||||
fun setTranslationEnabled(enabled: Boolean) {
|
||||
sharedPreferences.edit().putBoolean("is_translation_enabled", enabled).apply()
|
||||
}
|
||||
|
||||
fun getActiveLanguageCode(): String = sharedPreferences.getString("active_language_code", "en") ?: "en"
|
||||
|
||||
fun setActiveLanguageCode(languageCode: String) {
|
||||
sharedPreferences.edit().putString("active_language_code", languageCode).apply()
|
||||
}
|
||||
|
||||
fun saveKeys(
|
||||
youtubeApiKey: String,
|
||||
chatApiKey: String,
|
||||
edgeTtsSubscriptionKey: String = "",
|
||||
) {
|
||||
with(securedPreferences.edit()) {
|
||||
if (youtubeApiKey.isNotBlank()) {
|
||||
putString("youtube_api_key", youtubeApiKey)
|
||||
}
|
||||
if (chatApiKey.isNotBlank()) {
|
||||
putString("chat_api_key", chatApiKey)
|
||||
}
|
||||
if (edgeTtsSubscriptionKey.isNotBlank()) {
|
||||
putString("edge_tts_subscription_key", edgeTtsSubscriptionKey)
|
||||
}
|
||||
apply()
|
||||
}
|
||||
}
|
||||
|
||||
fun getYoutubeApiKey(): String? = securedPreferences.getString("youtube_api_key", null)
|
||||
|
||||
fun getChatApiKey(): String? = securedPreferences.getString("chat_api_key", null)
|
||||
|
||||
fun getEdgeTtsSubscriptionKey(): String? = securedPreferences.getString("edge_tts_subscription_key", null)
|
||||
|
||||
fun getLlmProvider(): String = sharedPreferences.getString("llm_provider", "GROQ") ?: "GROQ"
|
||||
|
||||
fun setLlmProvider(provider: String) {
|
||||
sharedPreferences.edit().putString("llm_provider", provider).apply()
|
||||
}
|
||||
|
||||
fun getLlmModel(): String = sharedPreferences.getString("llm_model", "llama-3.3-70b-versatile") ?: "llama-3.3-70b-versatile"
|
||||
|
||||
fun setLlmModel(model: String) {
|
||||
sharedPreferences.edit().putString("llm_model", model).apply()
|
||||
}
|
||||
|
||||
fun getLlmCustomUrl(): String = sharedPreferences.getString("llm_custom_url", "https://api.example.com/v1/chat/completions") ?: "https://api.example.com/v1/chat/completions"
|
||||
|
||||
fun setLlmCustomUrl(url: String) {
|
||||
sharedPreferences.edit().putString("llm_custom_url", url).apply()
|
||||
}
|
||||
|
||||
fun getLlmCustomHeaders(): String = sharedPreferences.getString("llm_custom_headers", "{\"Authorization\": \"Bearer {{API_KEY}}\", \"Content-Type\": \"application/json\"}") ?: "{\"Authorization\": \"Bearer {{API_KEY}}\", \"Content-Type\": \"application/json\"}"
|
||||
|
||||
fun setLlmCustomHeaders(headers: String) {
|
||||
sharedPreferences.edit().putString("llm_custom_headers", headers).apply()
|
||||
}
|
||||
|
||||
fun getLlmCustomResponsePath(): String = sharedPreferences.getString("llm_custom_response_path", "choices[0].message.content") ?: "choices[0].message.content"
|
||||
|
||||
fun setLlmCustomResponsePath(path: String) {
|
||||
sharedPreferences.edit().putString("llm_custom_response_path", path).apply()
|
||||
}
|
||||
|
||||
fun getLlmCustomRequestTemplate(): String = sharedPreferences.getString("llm_custom_request_template", "{\n \"model\": \"{{MODEL}}\",\n \"messages\": {{MESSAGES}}\n}") ?: "{\n \"model\": \"{{MODEL}}\",\n \"messages\": {{MESSAGES}}\n}"
|
||||
|
||||
fun setLlmCustomRequestTemplate(template: String) {
|
||||
sharedPreferences.edit().putString("llm_custom_request_template", template).apply()
|
||||
}
|
||||
|
||||
fun getLlmCustomMessageFormat(): String = sharedPreferences.getString("llm_custom_message_format", "{\"role\": \"{{ROLE}}\", \"content\": \"{{CONTENT}}\"}") ?: "{\"role\": \"{{ROLE}}\", \"content\": \"{{CONTENT}}\"}"
|
||||
|
||||
fun setLlmCustomMessageFormat(format: String) {
|
||||
sharedPreferences.edit().putString("llm_custom_message_format", format).apply()
|
||||
}
|
||||
|
||||
fun getLlmCustomSystemRole(): String = sharedPreferences.getString("llm_custom_system_role", "system") ?: "system"
|
||||
|
||||
fun setLlmCustomSystemRole(role: String) {
|
||||
sharedPreferences.edit().putString("llm_custom_system_role", role).apply()
|
||||
}
|
||||
|
||||
fun getLlmCustomUserRole(): String = sharedPreferences.getString("llm_custom_user_role", "user") ?: "user"
|
||||
|
||||
fun setLlmCustomUserRole(role: String) {
|
||||
sharedPreferences.edit().putString("llm_custom_user_role", role).apply()
|
||||
}
|
||||
|
||||
fun getLlmCustomAssistantRole(): String = sharedPreferences.getString("llm_custom_assistant_role", "assistant") ?: "assistant"
|
||||
|
||||
fun setLlmCustomAssistantRole(role: String) {
|
||||
sharedPreferences.edit().putString("llm_custom_assistant_role", role).apply()
|
||||
}
|
||||
|
||||
fun getIsImageSupported(): Boolean = sharedPreferences.getBoolean("is_image_supported", false)
|
||||
fun setIsImageSupported(supported: Boolean) {
|
||||
sharedPreferences.edit().putBoolean("is_image_supported", supported).apply()
|
||||
}
|
||||
|
||||
fun getIsAudioSupported(): Boolean = sharedPreferences.getBoolean("is_audio_supported", false)
|
||||
fun setIsAudioSupported(supported: Boolean) {
|
||||
sharedPreferences.edit().putBoolean("is_audio_supported", supported).apply()
|
||||
}
|
||||
|
||||
fun getIsVideoSupported(): Boolean = sharedPreferences.getBoolean("is_video_supported", false)
|
||||
fun setIsVideoSupported(supported: Boolean) {
|
||||
sharedPreferences.edit().putBoolean("is_video_supported", supported).apply()
|
||||
}
|
||||
|
||||
fun getIsDocumentSupported(): Boolean = sharedPreferences.getBoolean("is_document_supported", false)
|
||||
fun setIsDocumentSupported(supported: Boolean) {
|
||||
sharedPreferences.edit().putBoolean("is_document_supported", supported).apply()
|
||||
}
|
||||
|
||||
fun getIsModelVerified(): Boolean = sharedPreferences.getBoolean("is_model_verified", false)
|
||||
fun setIsModelVerified(verified: Boolean) {
|
||||
sharedPreferences.edit().putBoolean("is_model_verified", verified).apply()
|
||||
}
|
||||
|
||||
fun getSttMode(): com.app.assistant.speech.SttMode {
|
||||
val modeStr = sharedPreferences.getString("stt_mode", null)
|
||||
if (modeStr != null) {
|
||||
return try {
|
||||
com.app.assistant.speech.SttMode.valueOf(modeStr)
|
||||
} catch (e: Exception) {
|
||||
com.app.assistant.speech.SttMode.NATIVE
|
||||
}
|
||||
}
|
||||
return if (getUseLocalWhisper()) {
|
||||
com.app.assistant.speech.SttMode.PARAKEET
|
||||
} else {
|
||||
com.app.assistant.speech.SttMode.NATIVE
|
||||
}
|
||||
}
|
||||
|
||||
fun setSttMode(mode: com.app.assistant.speech.SttMode) {
|
||||
sharedPreferences.edit().putString("stt_mode", mode.name).apply()
|
||||
setUseLocalWhisper(mode == com.app.assistant.speech.SttMode.PARAKEET)
|
||||
}
|
||||
|
||||
fun getUseLocalWhisper(): Boolean = sharedPreferences.getBoolean("use_local_whisper", false)
|
||||
fun setUseLocalWhisper(enabled: Boolean) {
|
||||
sharedPreferences.edit().putBoolean("use_local_whisper", enabled).apply()
|
||||
}
|
||||
|
||||
fun getTtsMode(): com.app.assistant.tts.TtsMode {
|
||||
val modeStr = sharedPreferences.getString("tts_mode", com.app.assistant.tts.TtsMode.NATIVE.name)
|
||||
return try {
|
||||
com.app.assistant.tts.TtsMode.valueOf(modeStr ?: com.app.assistant.tts.TtsMode.NATIVE.name)
|
||||
} catch (e: Exception) {
|
||||
com.app.assistant.tts.TtsMode.NATIVE
|
||||
}
|
||||
}
|
||||
|
||||
fun setTtsMode(mode: com.app.assistant.tts.TtsMode) {
|
||||
sharedPreferences.edit().putString("tts_mode", mode.name).apply()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,196 @@
|
|||
package com.app.assistant.repository
|
||||
|
||||
import android.content.Context
|
||||
import android.location.Geocoder
|
||||
import android.util.Log
|
||||
import kotlinx.coroutines.CompletableDeferred
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.json.Json
|
||||
import okhttp3.Call
|
||||
import okhttp3.Callback
|
||||
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import okhttp3.Response
|
||||
import java.io.IOException
|
||||
import java.util.Locale
|
||||
|
||||
@Serializable
|
||||
data class GeocodingResult(
|
||||
val name: String,
|
||||
val latitude: Double,
|
||||
val longitude: Double
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class GeocodingResponse(
|
||||
val results: List<GeocodingResult>? = null
|
||||
)
|
||||
|
||||
class WeatherRepository(
|
||||
private val context: Context,
|
||||
private val client: OkHttpClient
|
||||
) {
|
||||
private val json = Json { ignoreUnknownKeys = true }
|
||||
|
||||
fun extractPlaceName(inputText: String): String {
|
||||
// Pattern to match place names after "in" or "for", excluding trailing words like "today", "tomorrow", etc.
|
||||
val pattern =
|
||||
Regex(
|
||||
"\\b(?:in|for)\\s+([A-Za-z]+(?:\\s+[A-Za-z]+)*)" +
|
||||
"(?=\\b(?:\\s+(today|now|tomorrow|report|forecast|currently|evening|morning|afternoon)\\b|\\s*$))",
|
||||
)
|
||||
|
||||
// Search for a match in the input text
|
||||
val match = pattern.find(inputText)
|
||||
|
||||
// Check if the captured group does not match excluded words (e.g., "tomorrow") and return it
|
||||
val placeName =
|
||||
match
|
||||
?.groups
|
||||
?.get(1)
|
||||
?.value
|
||||
?.trim()
|
||||
return if (placeName != null &&
|
||||
!placeName.lowercase().matches(Regex("today|now|tomorrow|report|forecast|currently|evening|morning|afternoon"))
|
||||
) {
|
||||
placeName
|
||||
} else {
|
||||
""
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun getCoordinatesFromCity(cityName: String): Pair<Double, Double>? {
|
||||
val deferredResult = CompletableDeferred<Pair<Double, Double>?>()
|
||||
|
||||
val urlBuilder =
|
||||
"https://geocoding-api.open-meteo.com/v1/search"
|
||||
.toHttpUrlOrNull()
|
||||
?.newBuilder()
|
||||
?.addQueryParameter("name", cityName)
|
||||
?.build()
|
||||
|
||||
val request =
|
||||
Request
|
||||
.Builder()
|
||||
.url(urlBuilder.toString())
|
||||
.build()
|
||||
|
||||
client.newCall(request).enqueue(
|
||||
object : Callback {
|
||||
override fun onFailure(
|
||||
call: Call,
|
||||
e: IOException,
|
||||
) {
|
||||
Log.e("WeatherRepository", "Request failed: $e")
|
||||
deferredResult.completeExceptionally(e)
|
||||
}
|
||||
|
||||
override fun onResponse(
|
||||
call: Call,
|
||||
response: Response,
|
||||
) {
|
||||
if (response.isSuccessful) {
|
||||
val body = response.body?.string() ?: ""
|
||||
deferredResult.complete(extractCoordinates(body, cityName))
|
||||
} else {
|
||||
Log.e("WeatherRepository", "Request unsuccessful: ${response.code}")
|
||||
deferredResult.cancel()
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
return deferredResult.await()
|
||||
}
|
||||
|
||||
private fun extractCoordinates(
|
||||
body: String,
|
||||
cityName: String,
|
||||
): Pair<Double, Double>? {
|
||||
if (body.isEmpty()) return null
|
||||
try {
|
||||
val response = json.decodeFromString<GeocodingResponse>(body)
|
||||
val results = response.results ?: return null
|
||||
if (results.isEmpty()) return null
|
||||
|
||||
val matchedResult = results.find { it.name.equals(cityName, ignoreCase = true) } ?: results[0]
|
||||
return Pair(matchedResult.latitude, matchedResult.longitude)
|
||||
} catch (e: Exception) {
|
||||
Log.d("extractCoordinates Exception", e.message ?: "")
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
suspend fun getCityNameFromLocation(
|
||||
latitude: Double,
|
||||
longitude: Double,
|
||||
): String? = withContext(Dispatchers.IO) {
|
||||
val geocoder = Geocoder(context, Locale.getDefault())
|
||||
return@withContext try {
|
||||
val addresses = geocoder.getFromLocation(latitude, longitude, 1)
|
||||
if (addresses?.isNotEmpty() == true) {
|
||||
addresses[0].locality // This gives the city name
|
||||
} else {
|
||||
null
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun getWeatherData(
|
||||
latitude: Double,
|
||||
longitude: Double,
|
||||
): String? {
|
||||
val deferredResult = CompletableDeferred<String?>()
|
||||
|
||||
val urlBuilder =
|
||||
"https://api.open-meteo.com/v1/forecast"
|
||||
.toHttpUrlOrNull()
|
||||
?.newBuilder()
|
||||
?.addQueryParameter("latitude", latitude.toString())
|
||||
?.addQueryParameter("longitude", longitude.toString())
|
||||
?.addQueryParameter("daily", "temperature_2m_max,temperature_2m_min,precipitation_sum,windspeed_10m_max")
|
||||
?.addQueryParameter("hourly", "temperature_2m,precipitation,windspeed_10m")
|
||||
?.addQueryParameter("timezone", "auto")
|
||||
?.addQueryParameter("forecast_days", "10")
|
||||
?.build()
|
||||
|
||||
val request =
|
||||
Request
|
||||
.Builder()
|
||||
.url(urlBuilder.toString())
|
||||
.build()
|
||||
|
||||
client.newCall(request).enqueue(
|
||||
object : Callback {
|
||||
override fun onFailure(
|
||||
call: Call,
|
||||
e: IOException,
|
||||
) {
|
||||
Log.e("WeatherRepository", "Weather data request failed: $e")
|
||||
deferredResult.complete(null)
|
||||
}
|
||||
|
||||
override fun onResponse(
|
||||
call: Call,
|
||||
response: Response,
|
||||
) {
|
||||
if (response.isSuccessful) {
|
||||
val body = response.body?.string() ?: ""
|
||||
deferredResult.complete(body)
|
||||
} else {
|
||||
Log.e("WeatherRepository", "Weather data request unsuccessful: ${response.code}")
|
||||
deferredResult.complete(null)
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
return deferredResult.await()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,208 @@
|
|||
package com.app.assistant.speech
|
||||
|
||||
import android.Manifest
|
||||
import android.content.Context
|
||||
import android.content.pm.PackageManager
|
||||
import android.media.AudioFormat
|
||||
import android.media.AudioManager
|
||||
import android.media.AudioRecord
|
||||
import android.media.MediaRecorder
|
||||
import android.media.audiofx.AcousticEchoCanceler
|
||||
import android.media.audiofx.AutomaticGainControl
|
||||
import android.media.audiofx.NoiseSuppressor
|
||||
import android.util.Log
|
||||
import androidx.core.app.ActivityCompat
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.launch
|
||||
|
||||
class AudioHygieneProcessor(
|
||||
private val context: Context,
|
||||
private val scope: CoroutineScope,
|
||||
private val isHandsFree: Boolean,
|
||||
private val onSamplesReady: (FloatArray) -> Unit
|
||||
) {
|
||||
private val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
|
||||
private var audioRecord: AudioRecord? = null
|
||||
private var recordingJob: Job? = null
|
||||
|
||||
private var echoCanceler: AcousticEchoCanceler? = null
|
||||
private var noiseSuppressor: NoiseSuppressor? = null
|
||||
private var gainControl: AutomaticGainControl? = null
|
||||
|
||||
@Volatile
|
||||
private var isRecording = false
|
||||
|
||||
fun start(): Boolean {
|
||||
if (ActivityCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) {
|
||||
Log.e("AudioHygieneProcessor", "RECORD_AUDIO permission not granted")
|
||||
return false
|
||||
}
|
||||
|
||||
try {
|
||||
setupAudioRouting()
|
||||
|
||||
val sampleRate = 16000
|
||||
val channelConfig = AudioFormat.CHANNEL_IN_MONO
|
||||
val audioFormat = AudioFormat.ENCODING_PCM_16BIT
|
||||
val bufferSize = AudioRecord.getMinBufferSize(sampleRate, channelConfig, audioFormat)
|
||||
|
||||
val record = AudioRecord(
|
||||
MediaRecorder.AudioSource.VOICE_COMMUNICATION,
|
||||
sampleRate,
|
||||
channelConfig,
|
||||
audioFormat,
|
||||
bufferSize
|
||||
)
|
||||
|
||||
if (record.state != AudioRecord.STATE_INITIALIZED) {
|
||||
Log.e("AudioHygieneProcessor", "Failed to initialize AudioRecord")
|
||||
stopRouting()
|
||||
return false
|
||||
}
|
||||
|
||||
audioRecord = record
|
||||
|
||||
// Explicitly enable system audio effects on the audio session
|
||||
val sessionId = record.audioSessionId
|
||||
enableSystemAudioEffects(sessionId)
|
||||
|
||||
record.startRecording()
|
||||
isRecording = true
|
||||
|
||||
recordingJob = scope.launch(Dispatchers.IO) {
|
||||
val buffer = ShortArray(512)
|
||||
while (isRecording) {
|
||||
val read = record.read(buffer, 0, buffer.size)
|
||||
if (read > 0) {
|
||||
// Convert Short samples to Float samples in range [-1.0, 1.0]
|
||||
val floatSamples = FloatArray(read) { buffer[it] / 32768.0f }
|
||||
onSamplesReady(floatSamples)
|
||||
}
|
||||
//delay(10)
|
||||
}
|
||||
}
|
||||
|
||||
Log.i("AudioHygieneProcessor", "Audio recording started successfully.")
|
||||
return true
|
||||
|
||||
} catch (e: Exception) {
|
||||
Log.e("AudioHygieneProcessor", "Error starting audio recording", e)
|
||||
cleanResources()
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
fun stop() {
|
||||
Log.i("AudioHygieneProcessor", "Stopping audio recording.")
|
||||
isRecording = false
|
||||
recordingJob?.cancel()
|
||||
recordingJob = null
|
||||
cleanResources()
|
||||
}
|
||||
|
||||
private fun setupAudioRouting() {
|
||||
try {
|
||||
if (isHandsFree) {
|
||||
audioManager.mode = AudioManager.MODE_IN_COMMUNICATION
|
||||
audioManager.isSpeakerphoneOn = true
|
||||
Log.d("AudioHygieneProcessor", "Audio routing set to Speakerphone (Hands-Free)")
|
||||
} else {
|
||||
audioManager.mode = AudioManager.MODE_IN_CALL
|
||||
audioManager.isBluetoothScoOn = true
|
||||
audioManager.startBluetoothSco()
|
||||
Log.d("AudioHygieneProcessor", "Audio routing set to Bluetooth SCO")
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("AudioHygieneProcessor", "Error setting up audio routing", e)
|
||||
}
|
||||
}
|
||||
|
||||
private fun stopRouting() {
|
||||
try {
|
||||
audioManager.stopBluetoothSco()
|
||||
audioManager.isBluetoothScoOn = false
|
||||
audioManager.isSpeakerphoneOn = false
|
||||
if (audioManager.mode == AudioManager.MODE_IN_CALL || audioManager.mode == AudioManager.MODE_IN_COMMUNICATION) {
|
||||
audioManager.mode = AudioManager.MODE_NORMAL
|
||||
}
|
||||
Log.d("AudioHygieneProcessor", "Audio routing reset to normal")
|
||||
} catch (e: Exception) {
|
||||
Log.e("AudioHygieneProcessor", "Error stopping audio routing", e)
|
||||
}
|
||||
}
|
||||
|
||||
private fun enableSystemAudioEffects(audioSessionId: Int) {
|
||||
// Acoustic Echo Cancellation
|
||||
if (AcousticEchoCanceler.isAvailable()) {
|
||||
try {
|
||||
echoCanceler = AcousticEchoCanceler.create(audioSessionId)?.apply {
|
||||
enabled = true
|
||||
Log.d("AudioHygieneProcessor", "System AcousticEchoCanceler enabled on session $audioSessionId")
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("AudioHygieneProcessor", "Failed to create AcousticEchoCanceler", e)
|
||||
}
|
||||
} else {
|
||||
Log.w("AudioHygieneProcessor", "System AcousticEchoCanceler is not available on this device")
|
||||
}
|
||||
|
||||
// Noise Suppression
|
||||
if (NoiseSuppressor.isAvailable()) {
|
||||
try {
|
||||
noiseSuppressor = NoiseSuppressor.create(audioSessionId)?.apply {
|
||||
enabled = true
|
||||
Log.d("AudioHygieneProcessor", "System NoiseSuppressor enabled on session $audioSessionId")
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("AudioHygieneProcessor", "Failed to create NoiseSuppressor", e)
|
||||
}
|
||||
} else {
|
||||
Log.w("AudioHygieneProcessor", "System NoiseSuppressor is not available on this device")
|
||||
}
|
||||
|
||||
// Automatic Gain Control
|
||||
if (AutomaticGainControl.isAvailable()) {
|
||||
try {
|
||||
gainControl = AutomaticGainControl.create(audioSessionId)?.apply {
|
||||
enabled = true
|
||||
Log.d("AudioHygieneProcessor", "System AutomaticGainControl enabled on session $audioSessionId")
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("AudioHygieneProcessor", "Failed to create AutomaticGainControl", e)
|
||||
}
|
||||
} else {
|
||||
Log.w("AudioHygieneProcessor", "System AutomaticGainControl is not available on this device")
|
||||
}
|
||||
}
|
||||
|
||||
private fun cleanResources() {
|
||||
try {
|
||||
audioRecord?.apply {
|
||||
if (state == AudioRecord.STATE_INITIALIZED) {
|
||||
try {
|
||||
stop()
|
||||
} catch (e: Exception) {
|
||||
Log.e("AudioHygieneProcessor", "Error stopping AudioRecord", e)
|
||||
}
|
||||
}
|
||||
release()
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("AudioHygieneProcessor", "Error releasing AudioRecord", e)
|
||||
} finally {
|
||||
audioRecord = null
|
||||
}
|
||||
|
||||
echoCanceler?.release()
|
||||
echoCanceler = null
|
||||
noiseSuppressor?.release()
|
||||
noiseSuppressor = null
|
||||
gainControl?.release()
|
||||
gainControl = null
|
||||
|
||||
stopRouting()
|
||||
}
|
||||
}
|
||||
197
app/src/main/java/com/app/assistant/speech/SpeechModelManager.kt
Normal file
197
app/src/main/java/com/app/assistant/speech/SpeechModelManager.kt
Normal file
|
|
@ -0,0 +1,197 @@
|
|||
package com.app.assistant.speech
|
||||
|
||||
import android.content.Context
|
||||
import android.util.Log
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import java.io.File
|
||||
import java.io.FileOutputStream
|
||||
import java.io.IOException
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
|
||||
class SpeechModelManager(private val context: Context) {
|
||||
|
||||
private val modelDir = File(context.filesDir, "sherpa-onnx")
|
||||
|
||||
data class ModelFileDesc(
|
||||
val url: String,
|
||||
val localName: String,
|
||||
val sizeBytes: Long
|
||||
)
|
||||
|
||||
private val modelFiles = listOf(
|
||||
ModelFileDesc(
|
||||
url = "https://huggingface.co/csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8/resolve/main/encoder.int8.onnx",
|
||||
localName = "parakeet-encoder.onnx",
|
||||
sizeBytes = 652184296L
|
||||
),
|
||||
ModelFileDesc(
|
||||
url = "https://huggingface.co/csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8/resolve/main/decoder.int8.onnx",
|
||||
localName = "parakeet-decoder.onnx",
|
||||
sizeBytes = 7257753L
|
||||
),
|
||||
ModelFileDesc(
|
||||
url = "https://huggingface.co/csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8/resolve/main/joiner.int8.onnx",
|
||||
localName = "parakeet-joiner.onnx",
|
||||
sizeBytes = 1739080L
|
||||
),
|
||||
ModelFileDesc(
|
||||
url = "https://huggingface.co/csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8/resolve/main/tokens.txt",
|
||||
localName = "parakeet-tokens.txt",
|
||||
sizeBytes = 9384L
|
||||
),
|
||||
ModelFileDesc(
|
||||
url = "https://huggingface.co/csukuangfj/vad/resolve/main/silero_vad.onnx",
|
||||
localName = "silero_vad.onnx",
|
||||
sizeBytes = 1807522L
|
||||
)
|
||||
)
|
||||
|
||||
val totalSizeBytes: Long = modelFiles.sumOf { it.sizeBytes }
|
||||
|
||||
private var currentCall: okhttp3.Call? = null
|
||||
private val isCancelled = AtomicBoolean(false)
|
||||
|
||||
fun getModelDirectory(): File = modelDir
|
||||
|
||||
fun isModelDownloaded(): Boolean {
|
||||
if (!modelDir.exists()) return false
|
||||
return modelFiles.all { fileDesc ->
|
||||
val file = File(modelDir, fileDesc.localName)
|
||||
file.exists() && file.length() == fileDesc.sizeBytes
|
||||
}
|
||||
}
|
||||
|
||||
fun deleteLocalModel(): Boolean {
|
||||
if (!modelDir.exists()) return true
|
||||
var allDeleted = true
|
||||
modelFiles.forEach { fileDesc ->
|
||||
val file = File(modelDir, fileDesc.localName)
|
||||
if (file.exists()) {
|
||||
val deleted = file.delete()
|
||||
if (!deleted) allDeleted = false
|
||||
}
|
||||
// delete any tmp files too
|
||||
val tmpFile = File(modelDir, "${fileDesc.localName}.tmp")
|
||||
if (tmpFile.exists()) {
|
||||
tmpFile.delete()
|
||||
}
|
||||
}
|
||||
return allDeleted
|
||||
}
|
||||
|
||||
fun getEncoderPath(): String = File(modelDir, "parakeet-encoder.onnx").absolutePath
|
||||
fun getDecoderPath(): String = File(modelDir, "parakeet-decoder.onnx").absolutePath
|
||||
fun getJoinerPath(): String = File(modelDir, "parakeet-joiner.onnx").absolutePath
|
||||
fun getTokensPath(): String = File(modelDir, "parakeet-tokens.txt").absolutePath
|
||||
fun getVadPath(): String = File(modelDir, "silero_vad.onnx").absolutePath
|
||||
|
||||
suspend fun downloadModel(
|
||||
client: OkHttpClient,
|
||||
onProgress: (downloadedBytes: Long, totalBytes: Long) -> Unit,
|
||||
onComplete: (success: Boolean, errorMessage: String?) -> Unit
|
||||
) {
|
||||
isCancelled.set(false)
|
||||
withContext(Dispatchers.IO) {
|
||||
try {
|
||||
if (!modelDir.exists()) {
|
||||
modelDir.mkdirs()
|
||||
}
|
||||
|
||||
var cumulativeDownloaded = 0L
|
||||
|
||||
for (fileDesc in modelFiles) {
|
||||
val targetFile = File(modelDir, fileDesc.localName)
|
||||
|
||||
// If file is already fully and correctly downloaded, we skip it
|
||||
if (targetFile.exists() && targetFile.length() == fileDesc.sizeBytes) {
|
||||
cumulativeDownloaded += fileDesc.sizeBytes
|
||||
onProgress(cumulativeDownloaded, totalSizeBytes)
|
||||
continue
|
||||
}
|
||||
|
||||
// Otherwise, download it
|
||||
val tmpFile = File(modelDir, "${fileDesc.localName}.tmp")
|
||||
if (tmpFile.exists()) {
|
||||
tmpFile.delete()
|
||||
}
|
||||
|
||||
val request = Request.Builder()
|
||||
.url(fileDesc.url)
|
||||
.build()
|
||||
|
||||
val call = client.newCall(request)
|
||||
currentCall = call
|
||||
|
||||
if (isCancelled.get()) {
|
||||
throw IOException("Download cancelled by user")
|
||||
}
|
||||
|
||||
val response = call.execute()
|
||||
if (!response.isSuccessful) {
|
||||
throw IOException("Failed to download ${fileDesc.localName}: HTTP ${response.code}")
|
||||
}
|
||||
|
||||
val responseBody = response.body ?: throw IOException("Empty response body for ${fileDesc.localName}")
|
||||
|
||||
responseBody.byteStream().use { inputStream ->
|
||||
FileOutputStream(tmpFile).use { outputStream ->
|
||||
val buffer = ByteArray(1024 * 64) // 64kb buffer
|
||||
var bytesRead: Int
|
||||
var fileDownloaded = 0L
|
||||
|
||||
while (inputStream.read(buffer).also { bytesRead = it } != -1) {
|
||||
if (isCancelled.get()) {
|
||||
throw IOException("Download cancelled by user")
|
||||
}
|
||||
outputStream.write(buffer, 0, bytesRead)
|
||||
fileDownloaded += bytesRead
|
||||
onProgress(cumulativeDownloaded + fileDownloaded, totalSizeBytes)
|
||||
}
|
||||
outputStream.flush()
|
||||
}
|
||||
}
|
||||
response.close()
|
||||
|
||||
if (isCancelled.get()) {
|
||||
throw IOException("Download cancelled by user")
|
||||
}
|
||||
|
||||
// Rename tmp to actual
|
||||
if (tmpFile.renameTo(targetFile)) {
|
||||
cumulativeDownloaded += fileDesc.sizeBytes
|
||||
onProgress(cumulativeDownloaded, totalSizeBytes)
|
||||
} else {
|
||||
throw IOException("Failed to rename temporary file for ${fileDesc.localName}")
|
||||
}
|
||||
}
|
||||
|
||||
currentCall = null
|
||||
onComplete(true, null)
|
||||
|
||||
} catch (e: java.lang.Exception) {
|
||||
currentCall = null
|
||||
Log.e("SpeechModelManager", "Error downloading speech models", e)
|
||||
|
||||
// Clean up any remaining .tmp files
|
||||
modelFiles.forEach { fileDesc ->
|
||||
val tmpFile = File(modelDir, "${fileDesc.localName}.tmp")
|
||||
if (tmpFile.exists()) {
|
||||
tmpFile.delete()
|
||||
}
|
||||
}
|
||||
|
||||
val msg = if (isCancelled.get()) "Download cancelled." else e.message ?: "Unknown error"
|
||||
onComplete(false, msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun cancelDownload() {
|
||||
isCancelled.set(true)
|
||||
currentCall?.cancel()
|
||||
currentCall = null
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,689 @@
|
|||
package com.app.assistant.speech
|
||||
|
||||
import android.Manifest
|
||||
import android.content.ComponentCallbacks2
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.content.pm.PackageManager
|
||||
import android.content.res.Configuration
|
||||
import android.media.AudioFormat
|
||||
import android.media.AudioManager
|
||||
import android.os.Bundle
|
||||
import android.speech.RecognitionListener
|
||||
import android.speech.RecognizerIntent
|
||||
import android.speech.SpeechRecognizer
|
||||
import android.util.Log
|
||||
import androidx.core.app.ActivityCompat
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.util.Locale
|
||||
import okhttp3.MultipartBody
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import okhttp3.RequestBody.Companion.toRequestBody
|
||||
import okhttp3.MediaType.Companion.toMediaType
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.jsonObject
|
||||
import kotlinx.serialization.json.jsonPrimitive
|
||||
import java.io.ByteArrayOutputStream
|
||||
import com.app.assistant.config.SpeechConfig
|
||||
|
||||
class SpeechRecognizerManager(
|
||||
private val context: Context,
|
||||
private val scope: CoroutineScope,
|
||||
private val okHttpClient: okhttp3.OkHttpClient = com.app.assistant.viewmodel.MainViewModelFactory.okHttpClient,
|
||||
) {
|
||||
private val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
|
||||
private val settingsRepository = com.app.assistant.repository.SettingsRepository(context)
|
||||
private var speechRecognizer: android.speech.SpeechRecognizer? = null
|
||||
private var isListening = false
|
||||
|
||||
var shouldIgnoreUiHidden: () -> Boolean = { false }
|
||||
|
||||
fun isListeningActive(): Boolean = isListening
|
||||
|
||||
private var vad: com.k2fsa.sherpa.onnx.Vad? = null
|
||||
private var offlineRecognizer: com.k2fsa.sherpa.onnx.OfflineRecognizer? = null
|
||||
|
||||
private val modelManager = SpeechModelManager(context)
|
||||
private var isInHybridTransition = false
|
||||
private var hybridPrefixText = ""
|
||||
|
||||
private var activeListener: SpeechListener? = null
|
||||
private var isHandsFreeMode = false
|
||||
@Volatile
|
||||
private var isMicMuted = false
|
||||
|
||||
@Volatile
|
||||
var speechStartTimestamp: Long = 0L
|
||||
|
||||
private var audioHygieneProcessor: AudioHygieneProcessor? = null
|
||||
private var vadIntelligenceProcessor: VadIntelligenceProcessor? = null
|
||||
private var voiceStateMachine: VoiceStateMachine? = null
|
||||
|
||||
@Volatile
|
||||
var isTtsSpeaking = false
|
||||
set(value) {
|
||||
field = value
|
||||
voiceStateMachine?.onTtsStateChanged(value)
|
||||
}
|
||||
|
||||
private val componentCallbacks = object : ComponentCallbacks2 {
|
||||
override fun onTrimMemory(level: Int) {
|
||||
if (level == ComponentCallbacks2.TRIM_MEMORY_UI_HIDDEN) {
|
||||
if (!shouldIgnoreUiHidden()) {
|
||||
stop()
|
||||
}
|
||||
}
|
||||
}
|
||||
override fun onConfigurationChanged(newConfig: Configuration) {}
|
||||
override fun onLowMemory() {}
|
||||
}
|
||||
|
||||
init {
|
||||
context.registerComponentCallbacks(componentCallbacks)
|
||||
}
|
||||
|
||||
interface SpeechListener {
|
||||
fun onReadyForSpeech()
|
||||
fun onBeginningOfSpeech()
|
||||
fun onEndOfSpeech()
|
||||
fun onError(errorCode: Int)
|
||||
fun onResults(recognizedText: String)
|
||||
fun onPartialResults(recognizedText: String)
|
||||
fun onAudioLevelChanged(level: Float) {}
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
private fun initVad() {
|
||||
if (vad != null) return
|
||||
try {
|
||||
if (!modelManager.isModelDownloaded()) {
|
||||
Log.w("SpeechRecognizerManager", "Cannot init VAD: Model files not downloaded.")
|
||||
return
|
||||
}
|
||||
|
||||
val sileroConfig = com.k2fsa.sherpa.onnx.SileroVadModelConfig(
|
||||
model = modelManager.getVadPath(),
|
||||
threshold = 0.5f,
|
||||
minSilenceDuration = 0.5f,
|
||||
minSpeechDuration = 0.1f,
|
||||
windowSize = 512,
|
||||
maxSpeechDuration = 10.0f
|
||||
)
|
||||
val vadConfig = com.k2fsa.sherpa.onnx.VadModelConfig(
|
||||
sileroVadModelConfig = sileroConfig,
|
||||
sampleRate = 16000,
|
||||
numThreads = 4,
|
||||
provider = "cpu"
|
||||
)
|
||||
vad = com.k2fsa.sherpa.onnx.Vad(
|
||||
assetManager = null,
|
||||
config = vadConfig
|
||||
)
|
||||
Log.i("SpeechRecognizerManager", "[Silero VAD] Initialized successfully")
|
||||
} catch (e: Exception) {
|
||||
Log.e("SpeechRecognizerManager", "Failed to initialize VAD from disk", e)
|
||||
}
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
private fun initOfflineRecognizer() {
|
||||
if (offlineRecognizer != null) return
|
||||
try {
|
||||
if (!modelManager.isModelDownloaded()) {
|
||||
Log.w("SpeechRecognizerManager", "Cannot init Parakeet: Model files not downloaded.")
|
||||
return
|
||||
}
|
||||
|
||||
val transducerConfig = com.k2fsa.sherpa.onnx.OfflineTransducerModelConfig(
|
||||
encoder = modelManager.getEncoderPath(),
|
||||
decoder = modelManager.getDecoderPath(),
|
||||
joiner = modelManager.getJoinerPath()
|
||||
)
|
||||
val modelConfig = com.k2fsa.sherpa.onnx.OfflineModelConfig(
|
||||
transducer = transducerConfig,
|
||||
tokens = modelManager.getTokensPath(),
|
||||
modelType = "nemo_transducer",
|
||||
numThreads = 2,
|
||||
provider = "cpu"
|
||||
)
|
||||
val recognizerConfig = com.k2fsa.sherpa.onnx.OfflineRecognizerConfig(
|
||||
modelConfig = modelConfig,
|
||||
featConfig = com.k2fsa.sherpa.onnx.FeatureConfig(
|
||||
sampleRate = 16000,
|
||||
featureDim = 128
|
||||
),
|
||||
decodingMethod = "greedy_search"
|
||||
)
|
||||
offlineRecognizer = com.k2fsa.sherpa.onnx.OfflineRecognizer(
|
||||
assetManager = null,
|
||||
config = recognizerConfig
|
||||
)
|
||||
Log.i("SpeechRecognizerManager", "[Parakeet Recognizer] Initialized successfully")
|
||||
} catch (e: Exception) {
|
||||
Log.e("SpeechRecognizerManager", "Failed to initialize Parakeet from disk", e)
|
||||
}
|
||||
}
|
||||
|
||||
fun preLoadModelAsync() {
|
||||
val mode = settingsRepository.getSttMode()
|
||||
if (mode == SttMode.PARAKEET && modelManager.isModelDownloaded()) {
|
||||
scope.launch(Dispatchers.IO) {
|
||||
initVad()
|
||||
initOfflineRecognizer()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun startListening(isHandsFree: Boolean = false, listener: SpeechListener) {
|
||||
val mode = settingsRepository.getSttMode()
|
||||
Log.i("SpeechRecognizerManager", "[Listening] Start Listening requested. Hands-free: $isHandsFree, STT Mode: $mode")
|
||||
|
||||
isInHybridTransition = false
|
||||
hybridPrefixText = ""
|
||||
activeListener = listener
|
||||
isHandsFreeMode = isHandsFree
|
||||
|
||||
if (isHandsFree) {
|
||||
if (modelManager.isModelDownloaded()) {
|
||||
startThreeLayerPipeline(isHandsFree = true, listener)
|
||||
} else {
|
||||
Log.w("SpeechRecognizerManager", "Hands-free requested but model not downloaded. Falling back to Native STT.")
|
||||
startNativeListening(listener)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
when (mode) {
|
||||
SttMode.NATIVE -> {
|
||||
startNativeListening(listener)
|
||||
}
|
||||
SttMode.PARAKEET -> {
|
||||
if (modelManager.isModelDownloaded()) {
|
||||
startThreeLayerPipeline(isHandsFree = false, listener)
|
||||
} else {
|
||||
Log.w("SpeechRecognizerManager", "Parakeet selected but not downloaded. Falling back to Native STT.")
|
||||
startNativeListening(listener)
|
||||
}
|
||||
}
|
||||
SttMode.HYBRID -> {
|
||||
if (modelManager.isModelDownloaded()) {
|
||||
if (offlineRecognizer != null && vad != null) {
|
||||
startThreeLayerPipeline(isHandsFree = false, listener)
|
||||
} else {
|
||||
startHybridTransitionListening(listener)
|
||||
}
|
||||
} else {
|
||||
Log.w("SpeechRecognizerManager", "Hybrid selected but model not downloaded. Using Native STT.")
|
||||
startNativeListening(listener)
|
||||
}
|
||||
}
|
||||
SttMode.API -> {
|
||||
if (modelManager.isModelDownloaded()) {
|
||||
startThreeLayerPipeline(isHandsFree = isHandsFree, listener)
|
||||
} else {
|
||||
Log.w("SpeechRecognizerManager", "STT API selected but VAD model not downloaded. Falling back to Native STT.")
|
||||
startNativeListening(listener)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private val stateMachineCallback = object : VoiceStateMachine.StateCallback {
|
||||
override fun onTransitionToListening() {
|
||||
scope.launch(Dispatchers.Main) {
|
||||
isListening = true
|
||||
activeListener?.onBeginningOfSpeech()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onTransitionToProcessing(speechSamples: FloatArray) {
|
||||
scope.launch(Dispatchers.Main) {
|
||||
isListening = false
|
||||
activeListener?.onEndOfSpeech()
|
||||
}
|
||||
scope.launch(Dispatchers.IO) {
|
||||
transcribeAndDeliver(speechSamples)
|
||||
}
|
||||
}
|
||||
|
||||
override fun onTransitionToBotSpeaking() {
|
||||
isListening = false
|
||||
vadIntelligenceProcessor?.clear()
|
||||
}
|
||||
|
||||
override fun onTransitionToIdle() {
|
||||
isListening = false
|
||||
stopPipeline()
|
||||
}
|
||||
|
||||
override fun stopTtsPlayback() {
|
||||
// This propagates TTS state change
|
||||
isTtsSpeaking = false
|
||||
}
|
||||
|
||||
override fun onMicReady(ready: Boolean) {
|
||||
scope.launch(Dispatchers.Main) {
|
||||
if (ready) {
|
||||
activeListener?.onReadyForSpeech()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun startThreeLayerPipeline(isHandsFree: Boolean, listener: SpeechListener) {
|
||||
scope.launch {
|
||||
try {
|
||||
cleanupSpeechRecognizer()
|
||||
withContext(Dispatchers.IO) {
|
||||
initVad()
|
||||
val mode = settingsRepository.getSttMode()
|
||||
if (mode == SttMode.PARAKEET || mode == SttMode.HYBRID || mode == SttMode.NATIVE) {
|
||||
initOfflineRecognizer()
|
||||
}
|
||||
|
||||
val activeVad = vad
|
||||
if (activeVad == null) {
|
||||
Log.e("SpeechRecognizerManager", "VAD is null. Cannot listen.")
|
||||
withContext(Dispatchers.Main) {
|
||||
listener.onError(-1)
|
||||
}
|
||||
return@withContext
|
||||
}
|
||||
|
||||
// Stop any existing pipeline before starting a new one
|
||||
stopPipeline()
|
||||
|
||||
// Initialize the Orchestrator (Layer 3)
|
||||
val stateMachine = VoiceStateMachine(isHandsFree, stateMachineCallback)
|
||||
voiceStateMachine = stateMachine
|
||||
|
||||
// Initialize the Intelligence Layer (Layer 2)
|
||||
val vadProcessor = VadIntelligenceProcessor(activeVad, object : VadIntelligenceProcessor.VadListener {
|
||||
override fun onSpeechStart() {
|
||||
speechStartTimestamp = System.currentTimeMillis()
|
||||
stateMachine.onSpeechStartDetected()
|
||||
}
|
||||
|
||||
override fun onSpeechEnd(samples: FloatArray) {
|
||||
stateMachine.onSpeechEndDetected(samples)
|
||||
}
|
||||
})
|
||||
vadIntelligenceProcessor = vadProcessor
|
||||
|
||||
// Initialize the Audio Hygiene Layer (Layer 1)
|
||||
val audioProcessor = AudioHygieneProcessor(context, scope, isHandsFree) { floatSamples ->
|
||||
val rms = calculateRMS(floatSamples)
|
||||
val normalized = (rms / 0.15f).coerceIn(0f, 1f)
|
||||
scope.launch(Dispatchers.Main) {
|
||||
listener.onAudioLevelChanged(normalized)
|
||||
}
|
||||
// Pass samples directly up to Layer 2
|
||||
vadProcessor.acceptSamples(floatSamples)
|
||||
}
|
||||
audioHygieneProcessor = audioProcessor
|
||||
|
||||
vadProcessor.clear()
|
||||
if (isMicMuted) {
|
||||
Log.i("SpeechRecognizerManager", "Pipeline initialized but mic is muted. Not starting AudioRecord.")
|
||||
stateMachine.onUserStartedListening()
|
||||
} else {
|
||||
if (audioProcessor.start()) {
|
||||
stateMachine.onUserStartedListening()
|
||||
} else {
|
||||
withContext(Dispatchers.Main) {
|
||||
listener.onError(-1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("SpeechRecognizerManager", "Error starting 3-layer pipeline", e)
|
||||
withContext(Dispatchers.Main) {
|
||||
listener.onError(-1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun transcribeAndDeliver(samples: FloatArray) {
|
||||
val currentListener = activeListener ?: return
|
||||
val mode = settingsRepository.getSttMode()
|
||||
var text = ""
|
||||
|
||||
try {
|
||||
if (mode == SttMode.PARAKEET || mode == SttMode.HYBRID || mode == SttMode.NATIVE) {
|
||||
val recognizer = offlineRecognizer
|
||||
if (recognizer != null) {
|
||||
val stream = recognizer.createStream()
|
||||
stream.acceptWaveform(samples, 16000)
|
||||
recognizer.decode(stream)
|
||||
text = recognizer.getResult(stream).text.trim()
|
||||
stream.release()
|
||||
} else {
|
||||
Log.e("SpeechRecognizerManager", "Offline recognizer not initialized")
|
||||
}
|
||||
} else if (mode == SttMode.API) {
|
||||
// Convert float samples back to short samples for WAV encoding
|
||||
val shortSamples = ShortArray(samples.size) {
|
||||
(samples[it] * 32767.0f).toInt().coerceIn(-32768, 32767).toShort()
|
||||
}
|
||||
val wavBytes = getWavBytes(shortSamples, 16000)
|
||||
text = transcribeWithGroq(wavBytes)
|
||||
}
|
||||
|
||||
withContext(Dispatchers.Main) {
|
||||
if (text.isNotEmpty()) {
|
||||
currentListener.onResults(text)
|
||||
} else {
|
||||
currentListener.onError(SpeechRecognizer.ERROR_NO_MATCH)
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("SpeechRecognizerManager", "Transcription failed", e)
|
||||
withContext(Dispatchers.Main) {
|
||||
currentListener.onError(-1)
|
||||
}
|
||||
} finally {
|
||||
if (!isHandsFreeMode) {
|
||||
stop()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun startHybridTransitionListening(listener: SpeechListener) {
|
||||
isInHybridTransition = false
|
||||
hybridPrefixText = ""
|
||||
startNativeListening(listener)
|
||||
|
||||
scope.launch(Dispatchers.IO) {
|
||||
initVad()
|
||||
initOfflineRecognizer()
|
||||
}
|
||||
}
|
||||
|
||||
private fun startNativeListening(listener: SpeechListener) {
|
||||
try {
|
||||
cleanupSpeechRecognizer()
|
||||
if (isMicMuted) {
|
||||
Log.i("SpeechRecognizerManager", "startNativeListening: mic is muted, not starting SpeechRecognizer")
|
||||
return
|
||||
}
|
||||
|
||||
audioManager.mode = AudioManager.MODE_IN_CALL
|
||||
audioManager.isBluetoothScoOn = true
|
||||
audioManager.startBluetoothSco()
|
||||
|
||||
val recognizer = SpeechRecognizer.createSpeechRecognizer(context)
|
||||
speechRecognizer = recognizer
|
||||
|
||||
val speechRecognizerIntent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
|
||||
putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM)
|
||||
putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true)
|
||||
putExtra(RecognizerIntent.EXTRA_LANGUAGE, Locale.getDefault())
|
||||
}
|
||||
|
||||
recognizer.setRecognitionListener(object : RecognitionListener {
|
||||
override fun onReadyForSpeech(bundle: Bundle?) {
|
||||
listener.onReadyForSpeech()
|
||||
}
|
||||
|
||||
override fun onBeginningOfSpeech() {
|
||||
speechStartTimestamp = System.currentTimeMillis()
|
||||
isListening = true
|
||||
listener.onBeginningOfSpeech()
|
||||
}
|
||||
|
||||
override fun onRmsChanged(v: Float) {
|
||||
val normalized = ((v + 2f) / 12f).coerceIn(0f, 1f)
|
||||
listener.onAudioLevelChanged(normalized)
|
||||
}
|
||||
override fun onBufferReceived(bytes: ByteArray?) {}
|
||||
|
||||
override fun onEndOfSpeech() {
|
||||
isListening = false
|
||||
listener.onEndOfSpeech()
|
||||
}
|
||||
|
||||
override fun onError(errorCode: Int) {
|
||||
isListening = false
|
||||
listener.onError(errorCode)
|
||||
stopBluetoothSco()
|
||||
}
|
||||
|
||||
override fun onResults(bundle: Bundle?) {
|
||||
isListening = false
|
||||
val recognizedText = bundle?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)?.get(0) ?: ""
|
||||
listener.onResults(recognizedText)
|
||||
stopBluetoothSco()
|
||||
}
|
||||
|
||||
override fun onPartialResults(bundle: Bundle) {
|
||||
val recognizedText = bundle.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)?.get(0) ?: ""
|
||||
listener.onPartialResults(recognizedText)
|
||||
}
|
||||
|
||||
override fun onEvent(i: Int, bundle: Bundle?) {}
|
||||
})
|
||||
|
||||
recognizer.startListening(speechRecognizerIntent)
|
||||
} catch (e: Exception) {
|
||||
Log.e("SpeechRecognizerManager", "Error starting native speech recognition", e)
|
||||
stopBluetoothSco()
|
||||
listener.onError(-1)
|
||||
}
|
||||
}
|
||||
|
||||
private fun stopBluetoothSco() {
|
||||
try {
|
||||
audioManager.stopBluetoothSco()
|
||||
audioManager.isBluetoothScoOn = false
|
||||
audioManager.isSpeakerphoneOn = false
|
||||
if (audioManager.mode == AudioManager.MODE_IN_CALL || audioManager.mode == AudioManager.MODE_IN_COMMUNICATION) {
|
||||
audioManager.mode = AudioManager.MODE_NORMAL
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("SpeechRecognizerManager", "Error stopping Bluetooth SCO", e)
|
||||
}
|
||||
}
|
||||
|
||||
private fun stopPipeline() {
|
||||
audioHygieneProcessor?.stop()
|
||||
audioHygieneProcessor = null
|
||||
|
||||
vadIntelligenceProcessor?.clear()
|
||||
vadIntelligenceProcessor = null
|
||||
|
||||
voiceStateMachine = null
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun setMicMuted(muted: Boolean) {
|
||||
if (isMicMuted == muted) return
|
||||
isMicMuted = muted
|
||||
Log.i("SpeechRecognizerManager", "setMicMuted changed to $muted")
|
||||
|
||||
if (muted) {
|
||||
stopMicAccess()
|
||||
} else {
|
||||
resumeMicAccess()
|
||||
}
|
||||
}
|
||||
|
||||
private fun stopMicAccess() {
|
||||
Log.i("SpeechRecognizerManager", "stopMicAccess: making microphone unavailable")
|
||||
audioHygieneProcessor?.stop()
|
||||
|
||||
speechRecognizer?.let {
|
||||
it.stopListening()
|
||||
cleanupSpeechRecognizer()
|
||||
stopBluetoothSco()
|
||||
}
|
||||
}
|
||||
|
||||
private fun resumeMicAccess() {
|
||||
Log.i("SpeechRecognizerManager", "resumeMicAccess: resuming microphone access")
|
||||
val listener = activeListener ?: return
|
||||
|
||||
if (isHandsFreeMode) {
|
||||
val audioProcessor = audioHygieneProcessor
|
||||
val stateMachine = voiceStateMachine
|
||||
if (audioProcessor != null && stateMachine != null) {
|
||||
scope.launch(Dispatchers.IO) {
|
||||
if (audioProcessor.start()) {
|
||||
stateMachine.onUserStartedListening()
|
||||
} else {
|
||||
scope.launch(Dispatchers.Main) {
|
||||
listener.onError(-1)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (modelManager.isModelDownloaded()) {
|
||||
startThreeLayerPipeline(isHandsFree = true, listener)
|
||||
} else {
|
||||
startNativeListening(listener)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun stop() {
|
||||
Log.i("SpeechRecognizerManager", "[Listening] Stop Listening requested manually")
|
||||
voiceStateMachine?.onUserStoppedListening()
|
||||
stopPipeline()
|
||||
cleanupSpeechRecognizer()
|
||||
stopBluetoothSco()
|
||||
}
|
||||
|
||||
private fun cleanupSpeechRecognizer() {
|
||||
speechRecognizer?.let {
|
||||
it.destroy()
|
||||
speechRecognizer = null
|
||||
}
|
||||
}
|
||||
|
||||
fun destroy() {
|
||||
Log.i("SpeechRecognizerManager", "[Listening] destroy() requested")
|
||||
context.unregisterComponentCallbacks(componentCallbacks)
|
||||
stop()
|
||||
|
||||
vad?.let {
|
||||
it.release()
|
||||
vad = null
|
||||
Log.i("SpeechRecognizerManager", "[Silero VAD] Released/Stopped")
|
||||
}
|
||||
|
||||
offlineRecognizer?.let {
|
||||
it.release()
|
||||
offlineRecognizer = null
|
||||
Log.i("SpeechRecognizerManager", "[Parakeet Recognizer] Released/Stopped")
|
||||
}
|
||||
}
|
||||
|
||||
private fun getWavBytes(pcmData: ShortArray, sampleRate: Int): ByteArray {
|
||||
val totalAudioLen = pcmData.size * 2
|
||||
val totalDataLen = totalAudioLen + 36
|
||||
val longSampleRate = sampleRate.toLong()
|
||||
val channels = 1
|
||||
val byteRate = sampleRate * 2
|
||||
|
||||
val header = ByteArray(44)
|
||||
header[0] = 'R'.code.toByte()
|
||||
header[1] = 'I'.code.toByte()
|
||||
header[2] = 'F'.code.toByte()
|
||||
header[3] = 'F'.code.toByte()
|
||||
header[4] = (totalDataLen and 0xff).toByte()
|
||||
header[5] = ((totalDataLen shr 8) and 0xff).toByte()
|
||||
header[6] = ((totalDataLen shr 16) and 0xff).toByte()
|
||||
header[7] = ((totalDataLen shr 24) and 0xff).toByte()
|
||||
header[8] = 'W'.code.toByte()
|
||||
header[9] = 'A'.code.toByte()
|
||||
header[10] = 'V'.code.toByte()
|
||||
header[11] = 'E'.code.toByte()
|
||||
header[12] = 'f'.code.toByte()
|
||||
header[13] = 'm'.code.toByte()
|
||||
header[14] = 't'.code.toByte()
|
||||
header[15] = ' '.code.toByte()
|
||||
header[16] = 16
|
||||
header[17] = 0
|
||||
header[18] = 0
|
||||
header[19] = 0
|
||||
header[20] = 1
|
||||
header[21] = 0
|
||||
header[22] = channels.toByte()
|
||||
header[23] = 0
|
||||
header[24] = (longSampleRate and 0xff).toByte()
|
||||
header[25] = ((longSampleRate shr 8) and 0xff).toByte()
|
||||
header[26] = ((longSampleRate shr 16) and 0xff).toByte()
|
||||
header[27] = ((longSampleRate shr 24) and 0xff).toByte()
|
||||
header[28] = (byteRate and 0xff).toByte()
|
||||
header[29] = ((byteRate shr 8) and 0xff).toByte()
|
||||
header[30] = ((byteRate shr 16) and 0xff).toByte()
|
||||
header[31] = ((byteRate shr 24) and 0xff).toByte()
|
||||
header[32] = 2
|
||||
header[33] = 0
|
||||
header[34] = 16
|
||||
header[35] = 0
|
||||
header[36] = 'd'.code.toByte()
|
||||
header[37] = 'a'.code.toByte()
|
||||
header[38] = 't'.code.toByte()
|
||||
header[39] = 'a'.code.toByte()
|
||||
header[40] = (totalAudioLen and 0xff).toByte()
|
||||
header[41] = ((totalAudioLen shr 8) and 0xff).toByte()
|
||||
header[42] = ((totalAudioLen shr 16) and 0xff).toByte()
|
||||
header[43] = ((totalAudioLen shr 24) and 0xff).toByte()
|
||||
|
||||
val wavBytes = ByteArray(44 + totalAudioLen)
|
||||
System.arraycopy(header, 0, wavBytes, 0, 44)
|
||||
|
||||
var offset = 44
|
||||
for (sample in pcmData) {
|
||||
wavBytes[offset++] = (sample.toInt() and 0xff).toByte()
|
||||
wavBytes[offset++] = ((sample.toInt() shr 8) and 0xff).toByte()
|
||||
}
|
||||
return wavBytes
|
||||
}
|
||||
|
||||
private suspend fun transcribeWithGroq(wavBytes: ByteArray): String = withContext(Dispatchers.IO) {
|
||||
val requestBody = okhttp3.MultipartBody.Builder()
|
||||
.setType(okhttp3.MultipartBody.FORM)
|
||||
.addFormDataPart("model", com.app.assistant.config.SpeechConfig.GroqWhisper.MODEL)
|
||||
.addFormDataPart(
|
||||
"file",
|
||||
"speech.wav",
|
||||
wavBytes.toRequestBody("audio/wav".toMediaType())
|
||||
)
|
||||
.build()
|
||||
|
||||
val apiKey = com.app.assistant.config.SpeechConfig.GroqWhisper.getApiKey(settingsRepository)
|
||||
val request = okhttp3.Request.Builder()
|
||||
.url(com.app.assistant.config.SpeechConfig.GroqWhisper.BASE_URL)
|
||||
.addHeader("Authorization", "Bearer $apiKey")
|
||||
.post(requestBody)
|
||||
.build()
|
||||
|
||||
okHttpClient.newCall(request).execute().use { response ->
|
||||
if (!response.isSuccessful) {
|
||||
throw Exception("Unsuccessful response from Groq: ${response.code} - ${response.message}")
|
||||
}
|
||||
val bodyStr = response.body?.string() ?: ""
|
||||
val jsonElement = kotlinx.serialization.json.Json.parseToJsonElement(bodyStr)
|
||||
jsonElement.jsonObject["text"]?.jsonPrimitive?.content ?: ""
|
||||
}
|
||||
}
|
||||
|
||||
private fun calculateRMS(samples: FloatArray): Float {
|
||||
if (samples.isEmpty()) return 0f
|
||||
var sum = 0.0f
|
||||
for (sample in samples) {
|
||||
sum += sample * sample
|
||||
}
|
||||
return kotlin.math.sqrt(sum / samples.size)
|
||||
}
|
||||
}
|
||||
8
app/src/main/java/com/app/assistant/speech/SttMode.kt
Normal file
8
app/src/main/java/com/app/assistant/speech/SttMode.kt
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
package com.app.assistant.speech
|
||||
|
||||
enum class SttMode {
|
||||
NATIVE,
|
||||
PARAKEET,
|
||||
HYBRID,
|
||||
API
|
||||
}
|
||||
|
|
@ -0,0 +1,98 @@
|
|||
package com.app.assistant.speech
|
||||
|
||||
import android.util.Log
|
||||
import com.k2fsa.sherpa.onnx.Vad
|
||||
import kotlin.math.max
|
||||
import kotlin.math.sqrt
|
||||
|
||||
class VadIntelligenceProcessor(
|
||||
private val vad: Vad,
|
||||
private val listener: VadListener
|
||||
) {
|
||||
interface VadListener {
|
||||
fun onSpeechStart()
|
||||
fun onSpeechEnd(samples: FloatArray)
|
||||
}
|
||||
|
||||
private var isSpeechActive = false
|
||||
|
||||
private var ambientNoiseFloor = 0.002f
|
||||
|
||||
companion object {
|
||||
private const val SAMPLE_RATE = 16000
|
||||
private const val MIN_SPEECH_DURATION_SEC = 0.4f
|
||||
private const val MIN_SAMPLES_REQUIRED = (SAMPLE_RATE * MIN_SPEECH_DURATION_SEC).toInt()
|
||||
|
||||
private const val ABSOLUTE_MIN_THRESHOLD = 0.0015f
|
||||
|
||||
private const val SNR_MULTIPLIER = 1.8f
|
||||
|
||||
private const val EMA_ALPHA = 0.05f
|
||||
}
|
||||
|
||||
fun acceptSamples(samples: FloatArray) {
|
||||
try {
|
||||
vad.acceptWaveform(samples)
|
||||
|
||||
val currentSpeechDetected = vad.isSpeechDetected()
|
||||
|
||||
if (!currentSpeechDetected && samples.isNotEmpty()) {
|
||||
val chunkRms = calculateRMS(samples)
|
||||
ambientNoiseFloor = (ambientNoiseFloor * (1f - EMA_ALPHA)) + (chunkRms * EMA_ALPHA)
|
||||
}
|
||||
|
||||
if (currentSpeechDetected && !isSpeechActive) {
|
||||
isSpeechActive = true
|
||||
Log.d("VadIntelligenceProcessor", "VAD Speech start transition detected!")
|
||||
listener.onSpeechStart()
|
||||
} else if (!currentSpeechDetected && isSpeechActive) {
|
||||
isSpeechActive = false
|
||||
Log.d("VadIntelligenceProcessor", "VAD Speech end transition detected!")
|
||||
}
|
||||
|
||||
while (!vad.empty()) {
|
||||
val segment = vad.front()
|
||||
vad.pop()
|
||||
|
||||
val audioSamples = segment.samples
|
||||
|
||||
if (audioSamples.size < MIN_SAMPLES_REQUIRED) {
|
||||
Log.d("VadIntelligenceProcessor", "Rejected segment: Too short (${audioSamples.size} samples)")
|
||||
continue
|
||||
}
|
||||
|
||||
val rms = calculateRMS(audioSamples)
|
||||
val dynamicThreshold = max(ABSOLUTE_MIN_THRESHOLD, ambientNoiseFloor * SNR_MULTIPLIER)
|
||||
|
||||
if (rms < dynamicThreshold) {
|
||||
Log.d("VadIntelligenceProcessor", "Rejected segment: Low energy (RMS: $rms, Dynamic Threshold: $dynamicThreshold)")
|
||||
continue
|
||||
}
|
||||
|
||||
Log.i("VadIntelligenceProcessor", "Valid speech verified. Samples: ${audioSamples.size}, RMS: $rms, Threshold: $dynamicThreshold")
|
||||
listener.onSpeechEnd(audioSamples)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("VadIntelligenceProcessor", "Error processing VAD acceptWaveform", e)
|
||||
}
|
||||
}
|
||||
|
||||
private fun calculateRMS(samples: FloatArray): Float {
|
||||
if (samples.isEmpty()) return 0f
|
||||
var sum = 0.0f
|
||||
for (sample in samples) {
|
||||
sum += sample * sample
|
||||
}
|
||||
return sqrt(sum / samples.size)
|
||||
}
|
||||
|
||||
fun clear() {
|
||||
try {
|
||||
vad.clear()
|
||||
isSpeechActive = false
|
||||
Log.d("VadIntelligenceProcessor", "VAD state cleared.")
|
||||
} catch (e: Exception) {
|
||||
Log.e("VadIntelligenceProcessor", "Error clearing VAD", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
8
app/src/main/java/com/app/assistant/speech/VoiceState.kt
Normal file
8
app/src/main/java/com/app/assistant/speech/VoiceState.kt
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
package com.app.assistant.speech
|
||||
|
||||
enum class VoiceState {
|
||||
IDLE,
|
||||
LISTENING,
|
||||
PROCESSING,
|
||||
BOT_SPEAKING
|
||||
}
|
||||
125
app/src/main/java/com/app/assistant/speech/VoiceStateMachine.kt
Normal file
125
app/src/main/java/com/app/assistant/speech/VoiceStateMachine.kt
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
package com.app.assistant.speech
|
||||
|
||||
class VoiceStateMachine(
|
||||
private val isHandsFree: Boolean,
|
||||
private val callback: StateCallback
|
||||
) {
|
||||
interface StateCallback {
|
||||
fun onTransitionToListening()
|
||||
fun onTransitionToProcessing(speechSamples: FloatArray)
|
||||
fun onTransitionToBotSpeaking()
|
||||
fun onTransitionToIdle()
|
||||
fun stopTtsPlayback()
|
||||
fun onMicReady(ready: Boolean)
|
||||
}
|
||||
|
||||
private var state = VoiceState.IDLE
|
||||
private var lastTtsStartTime: Long = 0L
|
||||
|
||||
@Synchronized
|
||||
fun getCurrentState(): VoiceState = state
|
||||
|
||||
@Synchronized
|
||||
fun onUserStartedListening() {
|
||||
if (state == VoiceState.IDLE) {
|
||||
state = VoiceState.LISTENING
|
||||
logD("State transition: IDLE -> LISTENING")
|
||||
callback.onMicReady(true)
|
||||
callback.onTransitionToListening()
|
||||
}
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun onUserStoppedListening() {
|
||||
if (state != VoiceState.IDLE) {
|
||||
logD("State transition: $state -> IDLE (stopped manually)")
|
||||
state = VoiceState.IDLE
|
||||
callback.onMicReady(false)
|
||||
callback.onTransitionToIdle()
|
||||
}
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun onSpeechStartDetected() {
|
||||
when (state) {
|
||||
VoiceState.BOT_SPEAKING -> {
|
||||
val elapsedSinceTtsStart = System.currentTimeMillis() - lastTtsStartTime
|
||||
if (elapsedSinceTtsStart < 300) {
|
||||
logD("Ignoring speech start detected during echo bleed window ($elapsedSinceTtsStart ms < 300 ms).")
|
||||
return
|
||||
}
|
||||
logI("[Interruption] Speech detected during TTS. Stopping TTS and listening.")
|
||||
callback.stopTtsPlayback()
|
||||
state = VoiceState.LISTENING
|
||||
callback.onMicReady(true)
|
||||
callback.onTransitionToListening()
|
||||
}
|
||||
VoiceState.LISTENING -> {
|
||||
logD("VAD Speech start detected while listening.")
|
||||
callback.onTransitionToListening()
|
||||
}
|
||||
else -> {
|
||||
logD("VAD Speech start detected during state: $state. Ignoring.")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun onSpeechEndDetected(samples: FloatArray) {
|
||||
if (state == VoiceState.LISTENING || state == VoiceState.BOT_SPEAKING) {
|
||||
logD("State transition: $state -> PROCESSING")
|
||||
state = VoiceState.PROCESSING
|
||||
callback.onMicReady(false)
|
||||
callback.onTransitionToProcessing(samples)
|
||||
} else {
|
||||
logD("VAD Speech end detected during state: $state. Ignoring.")
|
||||
}
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun onTtsStateChanged(isSpeaking: Boolean) {
|
||||
logD("TTS state changed: isSpeaking = $isSpeaking, current state = $state")
|
||||
when {
|
||||
isSpeaking -> {
|
||||
lastTtsStartTime = System.currentTimeMillis()
|
||||
if (state == VoiceState.PROCESSING || state == VoiceState.LISTENING) {
|
||||
logD("State transition: $state -> BOT_SPEAKING")
|
||||
state = VoiceState.BOT_SPEAKING
|
||||
callback.onTransitionToBotSpeaking()
|
||||
}
|
||||
}
|
||||
else -> {
|
||||
// TTS stopped
|
||||
if (state == VoiceState.BOT_SPEAKING) {
|
||||
if (isHandsFree) {
|
||||
logD("State transition: BOT_SPEAKING -> LISTENING (Hands-Free auto-loop)")
|
||||
state = VoiceState.LISTENING
|
||||
callback.onMicReady(true)
|
||||
callback.onTransitionToListening()
|
||||
} else {
|
||||
logD("State transition: BOT_SPEAKING -> IDLE")
|
||||
state = VoiceState.IDLE
|
||||
callback.onMicReady(false)
|
||||
callback.onTransitionToIdle()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun logD(msg: String) {
|
||||
try {
|
||||
android.util.Log.d("VoiceStateMachine", msg)
|
||||
} catch (e: Throwable) {
|
||||
println("[VoiceStateMachine] D: $msg")
|
||||
}
|
||||
}
|
||||
|
||||
private fun logI(msg: String) {
|
||||
try {
|
||||
android.util.Log.i("VoiceStateMachine", msg)
|
||||
} catch (e: Throwable) {
|
||||
println("[VoiceStateMachine] I: $msg")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -10,51 +10,61 @@ import com.google.mlkit.nl.translate.Translator
|
|||
import com.google.mlkit.nl.translate.TranslatorOptions
|
||||
|
||||
class TranslatorManager {
|
||||
|
||||
private var toEnglishTranslator: Translator? = null
|
||||
private var fromEnglishTranslator: Translator? = null
|
||||
private val modelManager = RemoteModelManager.getInstance()
|
||||
|
||||
fun setupTranslators(selectedLanguageCode: String, onResult: (Boolean) -> Unit) {
|
||||
modelManager.getDownloadedModels(TranslateRemoteModel::class.java)
|
||||
fun setupTranslators(
|
||||
selectedLanguageCode: String,
|
||||
onResult: (Boolean) -> Unit,
|
||||
) {
|
||||
modelManager
|
||||
.getDownloadedModels(TranslateRemoteModel::class.java)
|
||||
.addOnSuccessListener { downloadedModels ->
|
||||
val downloadedLanguages = downloadedModels.map { it.language }.toSet()
|
||||
|
||||
val isSelectedModelDownloaded = selectedLanguageCode in downloadedLanguages
|
||||
val isEnglishModelDownloaded = TranslateLanguage.ENGLISH in downloadedLanguages
|
||||
|
||||
// Download selected language model if not already downloaded
|
||||
if (!isSelectedModelDownloaded) {
|
||||
downloadLanguageModel(selectedLanguageCode) { success ->
|
||||
if (!success) {
|
||||
onResult(false)
|
||||
return@downloadLanguageModel
|
||||
}else{
|
||||
// Download English model if not already downloaded
|
||||
} else {
|
||||
if (!isEnglishModelDownloaded) {
|
||||
downloadLanguageModel(TranslateLanguage.ENGLISH) { successEnglish ->
|
||||
if (!successEnglish) {
|
||||
onResult(false)
|
||||
return@downloadLanguageModel
|
||||
}else{
|
||||
} else {
|
||||
initializeTranslators(selectedLanguageCode)
|
||||
onResult(true)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Log.d("Translator", "English language model already exists. Not downloading.")
|
||||
initializeTranslators(selectedLanguageCode)
|
||||
onResult(true)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Log.d("Translator", "Selected language model ($selectedLanguageCode) already exists. Not downloading.")
|
||||
onResult(true)
|
||||
if (!isEnglishModelDownloaded) {
|
||||
Log.d("Translator", "Selected model exists, downloading English model.")
|
||||
downloadLanguageModel(TranslateLanguage.ENGLISH) { successEnglish ->
|
||||
if (!successEnglish) {
|
||||
onResult(false)
|
||||
} else {
|
||||
initializeTranslators(selectedLanguageCode)
|
||||
onResult(true)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Log.d("Translator", "Both models already exist. Not downloading.")
|
||||
initializeTranslators(selectedLanguageCode)
|
||||
onResult(true)
|
||||
}
|
||||
}
|
||||
|
||||
// Set up the translators for the selected language and English
|
||||
initializeTranslators(selectedLanguageCode)
|
||||
}
|
||||
.addOnFailureListener { e ->
|
||||
}.addOnFailureListener { e ->
|
||||
Log.e("Translator", "Failed to get downloaded models: ${e.message}")
|
||||
onResult(false)
|
||||
}
|
||||
|
|
@ -63,57 +73,69 @@ class TranslatorManager {
|
|||
// Helper function to initialize translators for the selected language
|
||||
private fun initializeTranslators(selectedLanguageCode: String) {
|
||||
// Set up translator for selected language -> English
|
||||
val toEnglishOptions = TranslatorOptions.Builder()
|
||||
.setSourceLanguage(selectedLanguageCode)
|
||||
.setTargetLanguage(TranslateLanguage.ENGLISH)
|
||||
.build()
|
||||
val toEnglishOptions =
|
||||
TranslatorOptions
|
||||
.Builder()
|
||||
.setSourceLanguage(selectedLanguageCode)
|
||||
.setTargetLanguage(TranslateLanguage.ENGLISH)
|
||||
.build()
|
||||
|
||||
toEnglishTranslator = Translation.getClient(toEnglishOptions)
|
||||
|
||||
// Set up translator for English -> selected language
|
||||
val fromEnglishOptions = TranslatorOptions.Builder()
|
||||
.setSourceLanguage(TranslateLanguage.ENGLISH)
|
||||
.setTargetLanguage(selectedLanguageCode)
|
||||
.build()
|
||||
val fromEnglishOptions =
|
||||
TranslatorOptions
|
||||
.Builder()
|
||||
.setSourceLanguage(TranslateLanguage.ENGLISH)
|
||||
.setTargetLanguage(selectedLanguageCode)
|
||||
.build()
|
||||
|
||||
fromEnglishTranslator = Translation.getClient(fromEnglishOptions)
|
||||
}
|
||||
|
||||
|
||||
// Use this method to translate from selected language to English
|
||||
fun translateToEnglish(text: String, callback: (translatedText: String?) -> Unit) {
|
||||
toEnglishTranslator?.translate(text)
|
||||
fun translateToEnglish(
|
||||
text: String,
|
||||
callback: (translatedText: String?) -> Unit,
|
||||
) {
|
||||
toEnglishTranslator
|
||||
?.translate(text)
|
||||
?.addOnSuccessListener { translatedText ->
|
||||
callback(translatedText)
|
||||
}
|
||||
?.addOnFailureListener { e ->
|
||||
}?.addOnFailureListener { e ->
|
||||
Log.e("Translator", "Translation to English failed: ${e.message}")
|
||||
callback(null)
|
||||
}
|
||||
}
|
||||
|
||||
// Use this method to translate from English to the selected language
|
||||
fun translateFromEnglish(text: String, callback: (translatedText: String?) -> Unit) {
|
||||
fromEnglishTranslator?.translate(text)
|
||||
fun translateFromEnglish(
|
||||
text: String,
|
||||
callback: (translatedText: String?) -> Unit,
|
||||
) {
|
||||
fromEnglishTranslator
|
||||
?.translate(text)
|
||||
?.addOnSuccessListener { translatedText ->
|
||||
callback(translatedText)
|
||||
}
|
||||
?.addOnFailureListener { e ->
|
||||
}?.addOnFailureListener { e ->
|
||||
Log.e("Translator", "Translation from English failed: ${e.message}")
|
||||
callback(null)
|
||||
}
|
||||
}
|
||||
|
||||
private fun downloadLanguageModel(languageCode: String, onDownloadComplete: (Boolean) -> Unit) {
|
||||
private fun downloadLanguageModel(
|
||||
languageCode: String,
|
||||
onDownloadComplete: (Boolean) -> Unit,
|
||||
) {
|
||||
val languageModel = TranslateRemoteModel.Builder(languageCode).build()
|
||||
val conditions = DownloadConditions.Builder().build()
|
||||
|
||||
modelManager.download(languageModel, conditions)
|
||||
modelManager
|
||||
.download(languageModel, conditions)
|
||||
.addOnSuccessListener {
|
||||
Log.d("Translator", "Language model for $languageCode downloaded successfully.")
|
||||
onDownloadComplete(true)
|
||||
}
|
||||
.addOnFailureListener { e ->
|
||||
}.addOnFailureListener { e ->
|
||||
Log.e("Translator", "Failed to download language model for $languageCode: ${e.message}")
|
||||
onDownloadComplete(false)
|
||||
}
|
||||
|
|
@ -122,17 +144,17 @@ class TranslatorManager {
|
|||
// Delete the previously downloaded language model
|
||||
private fun deletePreviousLanguageModel(languageCode: String) {
|
||||
val languageModel = TranslateRemoteModel.Builder(languageCode).build()
|
||||
modelManager.deleteDownloadedModel(languageModel)
|
||||
modelManager
|
||||
.deleteDownloadedModel(languageModel)
|
||||
.addOnSuccessListener {
|
||||
Log.d("Translator", "Language model for $languageCode deleted successfully.")
|
||||
}
|
||||
.addOnFailureListener { e ->
|
||||
}.addOnFailureListener { e ->
|
||||
Log.e("Translator", "Failed to delete language model for $languageCode: ${e.message}")
|
||||
}
|
||||
}
|
||||
|
||||
fun closeTranslator(){
|
||||
fun closeTranslator() {
|
||||
toEnglishTranslator?.close()
|
||||
fromEnglishTranslator?.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
76
app/src/main/java/com/app/assistant/tts/ApiTtsManager.kt
Normal file
76
app/src/main/java/com/app/assistant/tts/ApiTtsManager.kt
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
package com.app.assistant.tts
|
||||
|
||||
import android.content.Context
|
||||
import com.app.assistant.config.SpeechConfig
|
||||
import com.app.assistant.config.TtsApiProvider
|
||||
import com.app.assistant.repository.SettingsRepository
|
||||
import okhttp3.OkHttpClient
|
||||
|
||||
class ApiTtsManager(
|
||||
private val context: Context,
|
||||
private val settingsRepository: SettingsRepository,
|
||||
private val okHttpClient: OkHttpClient,
|
||||
private val onSpeakingStateChanged: (isSpeaking: Boolean) -> Unit
|
||||
) : TtsManager {
|
||||
|
||||
private var edgeTtsApiManager: EdgeTtsApiManager? = null
|
||||
private var googleTtsApiManager: GoogleTtsApiManager? = null
|
||||
|
||||
private fun getEdgeTtsApiManager(): EdgeTtsApiManager {
|
||||
return edgeTtsApiManager ?: EdgeTtsApiManager(
|
||||
context,
|
||||
settingsRepository,
|
||||
okHttpClient,
|
||||
::handleSpeakingStateChanged
|
||||
).also { edgeTtsApiManager = it }
|
||||
}
|
||||
|
||||
private fun getGoogleTtsApiManager(): GoogleTtsApiManager {
|
||||
return googleTtsApiManager ?: GoogleTtsApiManager(
|
||||
context,
|
||||
settingsRepository,
|
||||
okHttpClient,
|
||||
::handleSpeakingStateChanged
|
||||
).also { googleTtsApiManager = it }
|
||||
}
|
||||
|
||||
private val activeProvider: TtsApiProvider
|
||||
get() = SpeechConfig.ACTIVE_TTS_PROVIDER
|
||||
|
||||
private val currentManager: TtsManager
|
||||
get() = when (activeProvider) {
|
||||
TtsApiProvider.EDGE_TTS -> getEdgeTtsApiManager()
|
||||
TtsApiProvider.GOOGLE_TTS -> getGoogleTtsApiManager()
|
||||
}
|
||||
|
||||
private fun handleSpeakingStateChanged(isSpeaking: Boolean) {
|
||||
onSpeakingStateChanged(isSpeaking)
|
||||
}
|
||||
|
||||
override fun speak(text: String, queueMode: Int) {
|
||||
// Stop the other manager to prevent overlapped audio if provider was changed dynamically
|
||||
when (activeProvider) {
|
||||
TtsApiProvider.EDGE_TTS -> googleTtsApiManager?.stop()
|
||||
TtsApiProvider.GOOGLE_TTS -> edgeTtsApiManager?.stop()
|
||||
}
|
||||
currentManager.speak(text, queueMode)
|
||||
}
|
||||
|
||||
override fun stop() {
|
||||
edgeTtsApiManager?.stop()
|
||||
googleTtsApiManager?.stop()
|
||||
}
|
||||
|
||||
override fun isSpeaking(): Boolean {
|
||||
val manager = when (activeProvider) {
|
||||
TtsApiProvider.EDGE_TTS -> edgeTtsApiManager
|
||||
TtsApiProvider.GOOGLE_TTS -> googleTtsApiManager
|
||||
}
|
||||
return manager?.isSpeaking() ?: false
|
||||
}
|
||||
|
||||
override fun shutdown() {
|
||||
edgeTtsApiManager?.shutdown()
|
||||
googleTtsApiManager?.shutdown()
|
||||
}
|
||||
}
|
||||
656
app/src/main/java/com/app/assistant/tts/EdgeTtsApiManager.kt
Normal file
656
app/src/main/java/com/app/assistant/tts/EdgeTtsApiManager.kt
Normal file
|
|
@ -0,0 +1,656 @@
|
|||
package com.app.assistant.tts
|
||||
|
||||
import android.content.Context
|
||||
import android.media.AudioAttributes
|
||||
import android.media.AudioFormat
|
||||
import android.media.AudioManager
|
||||
import android.media.AudioTrack
|
||||
import android.media.MediaCodec
|
||||
import android.media.MediaFormat
|
||||
import android.util.Log
|
||||
import com.app.assistant.config.SpeechConfig
|
||||
import com.app.assistant.repository.SettingsRepository
|
||||
import kotlinx.coroutines.*
|
||||
import kotlinx.coroutines.channels.Channel
|
||||
import okhttp3.*
|
||||
import okio.ByteString
|
||||
import java.io.ByteArrayOutputStream
|
||||
import java.security.MessageDigest
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.*
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
|
||||
@OptIn(ExperimentalCoroutinesApi::class, ObsoleteCoroutinesApi::class, DelicateCoroutinesApi::class)
|
||||
class EdgeTtsApiManager(
|
||||
private val context: Context,
|
||||
private val settingsRepository: SettingsRepository,
|
||||
okHttpClient: OkHttpClient,
|
||||
private val onSpeakingStateChanged: (isSpeaking: Boolean) -> Unit
|
||||
) : TtsManager {
|
||||
|
||||
private val ttsDispatcher = newSingleThreadContext("EdgeTtsApiThread")
|
||||
private val job = SupervisorJob()
|
||||
private val scope = CoroutineScope(ttsDispatcher + job)
|
||||
|
||||
private val speechQueue = Channel<String>(Channel.UNLIMITED)
|
||||
private val streamChannel = Channel<SentenceStream>(Channel.UNLIMITED)
|
||||
|
||||
private val activeStreams = ConcurrentHashMap<String, SentenceStream>()
|
||||
|
||||
private var webSocket: WebSocket? = null
|
||||
private var activeWebSocketCall: Call? = null
|
||||
|
||||
private var generatorJob: Job? = null
|
||||
private var playbackJob: Job? = null
|
||||
private var idleTimeoutJob: Job? = null
|
||||
|
||||
private val client = okHttpClient.newBuilder()
|
||||
.connectTimeout(5, java.util.concurrent.TimeUnit.SECONDS)
|
||||
.readTimeout(5, java.util.concurrent.TimeUnit.SECONDS)
|
||||
.writeTimeout(5, java.util.concurrent.TimeUnit.SECONDS)
|
||||
.build()
|
||||
|
||||
@Volatile
|
||||
private var isSpeaking = false
|
||||
@Volatile
|
||||
private var isGenerating = false
|
||||
@Volatile
|
||||
private var isPlaying = false
|
||||
|
||||
private class SentenceStream(
|
||||
val requestId: String,
|
||||
val rawDataChannel: Channel<ByteArray> = Channel(Channel.UNLIMITED),
|
||||
val frameChannel: Channel<ByteArray> = Channel(Channel.UNLIMITED),
|
||||
val completionDeferred: CompletableDeferred<Unit> = CompletableDeferred()
|
||||
)
|
||||
|
||||
init {
|
||||
Log.i("EdgeTtsApiManager", "[EdgeTTS] Initialized")
|
||||
startProcessing()
|
||||
}
|
||||
|
||||
private fun startProcessing() {
|
||||
generatorJob?.cancel()
|
||||
playbackJob?.cancel()
|
||||
|
||||
generatorJob = scope.launch {
|
||||
for (text in speechQueue) {
|
||||
isGenerating = true
|
||||
updateSpeakingState()
|
||||
|
||||
try {
|
||||
ensureWebSocketConnected()
|
||||
} catch (e: Exception) {
|
||||
Log.e("EdgeTtsApiManager", "Could not connect to WebSocket, skipping text: $text", e)
|
||||
isGenerating = false
|
||||
updateSpeakingState()
|
||||
continue
|
||||
}
|
||||
|
||||
idleTimeoutJob?.cancel()
|
||||
|
||||
val sentences = splitIntoSentences(text)
|
||||
for (sentence in sentences) {
|
||||
val cleaned = cleanTextForTts(sentence)
|
||||
if (cleaned.isEmpty()) continue
|
||||
|
||||
val requestId = UUID.randomUUID().toString().replace("-", "").lowercase()
|
||||
val stream = SentenceStream(requestId)
|
||||
activeStreams[requestId] = stream
|
||||
|
||||
launchParser(stream)
|
||||
|
||||
try {
|
||||
sendSsmlRequest(requestId, cleaned)
|
||||
streamChannel.send(stream)
|
||||
stream.completionDeferred.await()
|
||||
} catch (e: Exception) {
|
||||
Log.e("EdgeTtsApiManager", "Error in sequential streaming for requestId: $requestId", e)
|
||||
stream.rawDataChannel.close()
|
||||
stream.completionDeferred.complete(Unit)
|
||||
}
|
||||
|
||||
activeStreams.remove(requestId)
|
||||
}
|
||||
|
||||
if (speechQueue.isEmpty) {
|
||||
isGenerating = false
|
||||
updateSpeakingState()
|
||||
resetIdleTimeout()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
playbackJob = scope.launch {
|
||||
for (stream in streamChannel) {
|
||||
isPlaying = true
|
||||
updateSpeakingState()
|
||||
|
||||
playStream(stream)
|
||||
|
||||
if (streamChannel.isEmpty && !isGenerating && speechQueue.isEmpty) {
|
||||
isPlaying = false
|
||||
updateSpeakingState()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun playStream(stream: SentenceStream) {
|
||||
withContext(Dispatchers.Default) {
|
||||
val sampleRate = 24000
|
||||
val channelConfig = AudioFormat.CHANNEL_OUT_MONO
|
||||
val audioFormat = AudioFormat.ENCODING_PCM_16BIT
|
||||
val minBufferSize = AudioTrack.getMinBufferSize(sampleRate, channelConfig, audioFormat)
|
||||
|
||||
val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
|
||||
val useVoiceCall = audioManager.isBluetoothScoOn ||
|
||||
audioManager.mode == AudioManager.MODE_IN_CALL ||
|
||||
audioManager.mode == AudioManager.MODE_IN_COMMUNICATION
|
||||
|
||||
val audioTrack = AudioTrack.Builder()
|
||||
.setAudioAttributes(
|
||||
AudioAttributes.Builder()
|
||||
.setUsage(if (useVoiceCall) AudioAttributes.USAGE_VOICE_COMMUNICATION else AudioAttributes.USAGE_MEDIA)
|
||||
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
|
||||
.build()
|
||||
)
|
||||
.setAudioFormat(
|
||||
AudioFormat.Builder()
|
||||
.setEncoding(audioFormat)
|
||||
.setSampleRate(sampleRate)
|
||||
.setChannelMask(channelConfig)
|
||||
.build()
|
||||
)
|
||||
.setBufferSizeInBytes(Math.max(minBufferSize, 1024 * 16))
|
||||
.setTransferMode(AudioTrack.MODE_STREAM)
|
||||
.build()
|
||||
|
||||
var codec: MediaCodec? = null
|
||||
try {
|
||||
audioTrack.play()
|
||||
|
||||
val mediaFormat = MediaFormat.createAudioFormat(MediaFormat.MIMETYPE_AUDIO_MPEG, sampleRate, 1)
|
||||
codec = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_AUDIO_MPEG).apply {
|
||||
configure(mediaFormat, null, null, 0)
|
||||
start()
|
||||
}
|
||||
|
||||
var decodingActive = true
|
||||
var hasMoreInput = true
|
||||
var presentationTimeUs = 0L
|
||||
val frameIterator = stream.frameChannel.iterator()
|
||||
|
||||
while (decodingActive && coroutineContext.isActive) {
|
||||
var didWork = false
|
||||
|
||||
if (hasMoreInput) {
|
||||
val inputIndex = codec.dequeueInputBuffer(0)
|
||||
if (inputIndex >= 0) {
|
||||
val inputBuffer = codec.getInputBuffer(inputIndex)
|
||||
if (inputBuffer != null) {
|
||||
inputBuffer.clear()
|
||||
if (frameIterator.hasNext()) {
|
||||
val frame = frameIterator.next()
|
||||
inputBuffer.put(frame)
|
||||
codec.queueInputBuffer(inputIndex, 0, frame.size, presentationTimeUs, 0)
|
||||
presentationTimeUs += (frame.size * 8000000L) / 48000L
|
||||
didWork = true
|
||||
} else {
|
||||
if (stream.frameChannel.isClosedForReceive) {
|
||||
codec.queueInputBuffer(inputIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
|
||||
hasMoreInput = false
|
||||
didWork = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val bufferInfo = MediaCodec.BufferInfo()
|
||||
val outputIndex = codec.dequeueOutputBuffer(bufferInfo, 0)
|
||||
if (outputIndex >= 0) {
|
||||
val outputBuffer = codec.getOutputBuffer(outputIndex)
|
||||
if (outputBuffer != null && bufferInfo.size > 0) {
|
||||
val pcmData = ByteArray(bufferInfo.size)
|
||||
outputBuffer.get(pcmData)
|
||||
outputBuffer.clear()
|
||||
audioTrack.write(pcmData, 0, pcmData.size)
|
||||
}
|
||||
codec.releaseOutputBuffer(outputIndex, false)
|
||||
|
||||
if ((bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
|
||||
decodingActive = false
|
||||
}
|
||||
didWork = true
|
||||
}
|
||||
|
||||
if (!didWork) {
|
||||
delay(5)
|
||||
}
|
||||
}
|
||||
|
||||
if (coroutineContext.isActive) {
|
||||
delay(150)
|
||||
}
|
||||
|
||||
} catch (e: Exception) {
|
||||
Log.e("EdgeTtsApiManager", "Error decoding/playing stream", e)
|
||||
} finally {
|
||||
try {
|
||||
audioTrack.stop()
|
||||
audioTrack.flush()
|
||||
audioTrack.release()
|
||||
} catch (e: Exception) {}
|
||||
try {
|
||||
codec?.stop()
|
||||
codec?.release()
|
||||
} catch (e: Exception) {}
|
||||
}
|
||||
Unit
|
||||
}
|
||||
}
|
||||
|
||||
private fun launchParser(stream: SentenceStream): Job {
|
||||
return scope.launch(Dispatchers.Default) {
|
||||
val parserBuffer = ByteArrayOutputStream()
|
||||
try {
|
||||
for (bytes in stream.rawDataChannel) {
|
||||
parserBuffer.write(bytes)
|
||||
var bufferBytes = parserBuffer.toByteArray()
|
||||
var offset = 0
|
||||
while (offset + 4 <= bufferBytes.size) {
|
||||
val b0 = bufferBytes[offset].toInt() and 0xFF
|
||||
val b1 = bufferBytes[offset + 1].toInt() and 0xFF
|
||||
if (b0 == 0xFF && (b1 and 0xE0) == 0xE0) {
|
||||
val version = (b1 shr 3) and 0x03
|
||||
val layer = (b1 shr 1) and 0x03
|
||||
val bitrateIndex = (bufferBytes[offset + 2].toInt() and 0xFF) shr 4
|
||||
val srIndex = (bufferBytes[offset + 2].toInt() and 0xFF shr 2) and 0x03
|
||||
val padding = (bufferBytes[offset + 2].toInt() and 0xFF shr 1) and 0x01
|
||||
|
||||
val mpeg1Bitrates = intArrayOf(0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 0)
|
||||
val mpeg2Bitrates = intArrayOf(0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 0)
|
||||
val sampleRates = arrayOf(
|
||||
intArrayOf(11025, 12000, 8000),
|
||||
intArrayOf(0, 0, 0),
|
||||
intArrayOf(22050, 24000, 16000),
|
||||
intArrayOf(44100, 48000, 32000)
|
||||
)
|
||||
|
||||
if (version < sampleRates.size && srIndex < sampleRates[version].size) {
|
||||
val bitrate = (if (version == 3) mpeg1Bitrates[bitrateIndex] else mpeg2Bitrates[bitrateIndex]) * 1000
|
||||
val sampleRate = sampleRates[version][srIndex]
|
||||
|
||||
if (bitrate > 0 && sampleRate > 0 && layer == 1) {
|
||||
val coeff = if (version == 3) 144 else 72
|
||||
val frameSize = (coeff * bitrate) / sampleRate + padding
|
||||
|
||||
if (offset + frameSize <= bufferBytes.size) {
|
||||
val frame = bufferBytes.copyOfRange(offset, offset + frameSize)
|
||||
stream.frameChannel.send(frame)
|
||||
offset += frameSize
|
||||
continue
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
offset++
|
||||
}
|
||||
|
||||
val temp = parserBuffer.toByteArray()
|
||||
parserBuffer.reset()
|
||||
if (offset < temp.size) {
|
||||
parserBuffer.write(temp, offset, temp.size - offset)
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("EdgeTtsApiManager", "Error parsing MP3 frames", e)
|
||||
} finally {
|
||||
stream.frameChannel.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun ensureWebSocketConnected() {
|
||||
if (webSocket != null) return
|
||||
|
||||
val connectionId = UUID.randomUUID().toString().replace("-", "").uppercase()
|
||||
val secMsGec = generateSecMsGec()
|
||||
val url = "${SpeechConfig.EdgeTts.WSS_URL}" +
|
||||
"?Ocp-Apim-Subscription-Key=${SpeechConfig.EdgeTts.getSubscriptionKey(settingsRepository)}" +
|
||||
"&ConnectionId=$connectionId" +
|
||||
"&Sec-MS-GEC=$secMsGec" +
|
||||
"&Sec-MS-GEC-Version=1-130.0.2849.68"
|
||||
|
||||
val request = Request.Builder()
|
||||
.url(url)
|
||||
.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0")
|
||||
.addHeader("Origin", "chrome-extension://jdiccldimpdaibmpdkjnbmckianbfold")
|
||||
.addHeader("Pragma", "no-cache")
|
||||
.addHeader("Cache-Control", "no-cache")
|
||||
.build()
|
||||
|
||||
val connectDeferred = CompletableDeferred<WebSocket>()
|
||||
val listener = object : WebSocketListener() {
|
||||
override fun onOpen(ws: WebSocket, response: Response) {
|
||||
Log.d("EdgeTtsApiManager", "WebSocket connected successfully")
|
||||
connectDeferred.complete(ws)
|
||||
}
|
||||
|
||||
override fun onMessage(ws: WebSocket, text: String) {
|
||||
handleTextMessage(text)
|
||||
}
|
||||
|
||||
override fun onMessage(ws: WebSocket, bytes: ByteString) {
|
||||
handleBinaryMessage(bytes.toByteArray())
|
||||
}
|
||||
|
||||
override fun onClosing(ws: WebSocket, code: Int, reason: String) {
|
||||
ws.close(1000, null)
|
||||
}
|
||||
|
||||
override fun onClosed(ws: WebSocket, code: Int, reason: String) {
|
||||
Log.d("EdgeTtsApiManager", "WebSocket closed")
|
||||
if (webSocket == ws) {
|
||||
webSocket = null
|
||||
}
|
||||
}
|
||||
|
||||
override fun onFailure(ws: WebSocket, t: Throwable, response: Response?) {
|
||||
Log.e("EdgeTtsApiManager", "WebSocket failure: ${t.message}", t)
|
||||
if (connectDeferred.isActive) {
|
||||
connectDeferred.completeExceptionally(t)
|
||||
}
|
||||
if (webSocket == ws) {
|
||||
webSocket = null
|
||||
}
|
||||
for (stream in activeStreams.values) {
|
||||
stream.rawDataChannel.close()
|
||||
stream.completionDeferred.complete(Unit)
|
||||
}
|
||||
activeStreams.clear()
|
||||
}
|
||||
}
|
||||
|
||||
activeWebSocketCall?.cancel()
|
||||
|
||||
try {
|
||||
webSocket = client.newWebSocket(request, listener)
|
||||
withTimeout(5000L) {
|
||||
connectDeferred.await()
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("EdgeTtsApiManager", "Failed to connect WebSocket", e)
|
||||
webSocket = null
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
private fun handleTextMessage(text: String) {
|
||||
val lines = text.lines()
|
||||
var reqId: String? = null
|
||||
var isTurnEnd = false
|
||||
for (line in lines) {
|
||||
val trimmed = line.trim()
|
||||
if (trimmed.startsWith("X-RequestId:", ignoreCase = true)) {
|
||||
reqId = trimmed.substringAfter(":").trim()
|
||||
}
|
||||
if (trimmed.startsWith("Path:", ignoreCase = true)) {
|
||||
val path = trimmed.substringAfter(":").trim()
|
||||
if (path.equals("turn.end", ignoreCase = true)) {
|
||||
isTurnEnd = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (reqId != null && isTurnEnd) {
|
||||
val stream = activeStreams[reqId]
|
||||
if (stream != null) {
|
||||
stream.rawDataChannel.close()
|
||||
stream.completionDeferred.complete(Unit)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun handleBinaryMessage(bytesArray: ByteArray) {
|
||||
if (bytesArray.size < 2) return
|
||||
val headerLength = ((bytesArray[0].toInt() and 0xFF) shl 8) or (bytesArray[1].toInt() and 0xFF)
|
||||
if (headerLength + 2 <= bytesArray.size) {
|
||||
val headerStr = String(bytesArray, 2, headerLength, Charsets.UTF_8)
|
||||
val lines = headerStr.lines()
|
||||
var reqId: String? = null
|
||||
var isAudio = false
|
||||
for (line in lines) {
|
||||
val trimmed = line.trim()
|
||||
if (trimmed.startsWith("X-RequestId:", ignoreCase = true)) {
|
||||
reqId = trimmed.substringAfter(":").trim()
|
||||
}
|
||||
if (trimmed.startsWith("Path:", ignoreCase = true)) {
|
||||
val path = trimmed.substringAfter(":").trim()
|
||||
if (path.equals("audio", ignoreCase = true)) {
|
||||
isAudio = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (reqId != null && isAudio) {
|
||||
val stream = activeStreams[reqId]
|
||||
if (stream != null) {
|
||||
val audioOffset = 2 + headerLength
|
||||
val audioData = bytesArray.copyOfRange(audioOffset, bytesArray.size)
|
||||
if (audioData.isNotEmpty()) {
|
||||
stream.rawDataChannel.trySend(audioData)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun sendSsmlRequest(requestId: String, text: String) {
|
||||
val ws = webSocket ?: return
|
||||
val dateStr = getFormattedDate()
|
||||
|
||||
val configMsg = "X-Timestamp:$dateStr\r\n" +
|
||||
"Content-Type:application/json; charset=utf-8\r\n" +
|
||||
"Path:speech.config\r\n\r\n" +
|
||||
"{\"context\":{\"synthesis\":{\"audio\":{\"metadataoptions\":{\"sentenceBoundaryEnabled\":\"true\",\"wordBoundaryEnabled\":\"true\"},\"outputFormat\":\"${SpeechConfig.EdgeTts.OUTPUT_FORMAT}\"}}}}"
|
||||
ws.send(configMsg)
|
||||
|
||||
val escapedText = escapeXml(text)
|
||||
val ssmlMsg = "X-RequestId:$requestId\r\n" +
|
||||
"Content-Type:application/ssml+xml\r\n" +
|
||||
"X-Timestamp:$dateStr\r\n" +
|
||||
"Path:ssml\r\n\r\n" +
|
||||
"<speak version='1.0' xmlns='http://www.w3.org/2001/10/synthesis' xml:lang='en-US'>" +
|
||||
"<voice name='${SpeechConfig.EdgeTts.VOICE}'>" +
|
||||
"<prosody pitch='+0Hz' rate='+0%' volume='+0%'>" +
|
||||
escapedText +
|
||||
"</prosody></voice></speak>"
|
||||
|
||||
ws.send(ssmlMsg)
|
||||
}
|
||||
|
||||
private fun getFormattedDate(): String {
|
||||
val sdf = SimpleDateFormat("EEE MMM dd yyyy HH:mm:ss 'GMT+0000 (Coordinated Universal Time)'", Locale.US)
|
||||
sdf.timeZone = TimeZone.getTimeZone("UTC")
|
||||
return sdf.format(Date())
|
||||
}
|
||||
|
||||
private fun escapeXml(text: String): String {
|
||||
return text
|
||||
.replace("&", "&")
|
||||
.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
.replace("\"", """)
|
||||
.replace("'", "'")
|
||||
}
|
||||
|
||||
private fun generateSecMsGec(): String {
|
||||
val unixTime = System.currentTimeMillis() / 1000
|
||||
var ticks = unixTime
|
||||
ticks -= ticks % 300
|
||||
val winTicks = (ticks + 11644473600L) * 10000000L
|
||||
val strToHash = "${winTicks}${SpeechConfig.EdgeTts.getSubscriptionKey(settingsRepository)}"
|
||||
val md = MessageDigest.getInstance("SHA-256")
|
||||
val digest = md.digest(strToHash.toByteArray(Charsets.UTF_8))
|
||||
return digest.joinToString("") { "%02X".format(it) }
|
||||
}
|
||||
|
||||
private fun cleanTextForTts(text: String): String {
|
||||
return text
|
||||
.replace("—", " - ")
|
||||
.replace("–", " - ")
|
||||
.replace("“", "")
|
||||
.replace("”", "")
|
||||
.replace("\"", "")
|
||||
.replace("‘", "'")
|
||||
.replace("’", "'")
|
||||
.trim()
|
||||
}
|
||||
|
||||
private fun splitIntoSentences(text: String): List<String> {
|
||||
val sentences = mutableListOf<String>()
|
||||
val currentSentence = StringBuilder()
|
||||
val len = text.length
|
||||
var i = 0
|
||||
while (i < len) {
|
||||
val c = text[i]
|
||||
currentSentence.append(c)
|
||||
|
||||
if (c == '\n' || c == '\r') {
|
||||
val sentenceStr = currentSentence.toString().trim()
|
||||
if (sentenceStr.isNotEmpty()) {
|
||||
sentences.add(sentenceStr)
|
||||
}
|
||||
currentSentence.clear()
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
if (c == '.' || c == '?' || c == '!' || c == ':' || c == ';') {
|
||||
var isBoundary = false
|
||||
if (i + 1 == len) {
|
||||
isBoundary = true
|
||||
} else {
|
||||
val nextChar = text[i + 1]
|
||||
if (nextChar.isWhitespace()) {
|
||||
isBoundary = true
|
||||
}
|
||||
}
|
||||
|
||||
if (isBoundary) {
|
||||
if (c == '.' && isAbbreviationOrDecimal(text, i)) {
|
||||
// abbreviation
|
||||
} else {
|
||||
val sentenceStr = currentSentence.toString().trim()
|
||||
if (sentenceStr.isNotEmpty()) {
|
||||
sentences.add(sentenceStr)
|
||||
}
|
||||
currentSentence.clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
i++
|
||||
}
|
||||
|
||||
val remaining = currentSentence.toString().trim()
|
||||
if (remaining.isNotEmpty()) {
|
||||
sentences.add(remaining)
|
||||
}
|
||||
|
||||
return sentences
|
||||
}
|
||||
|
||||
private fun isAbbreviationOrDecimal(text: String, dotIndex: Int): Boolean {
|
||||
if (dotIndex > 0 && dotIndex + 1 < text.length) {
|
||||
if (text[dotIndex - 1].isDigit() && text[dotIndex + 1].isDigit()) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
var start = dotIndex - 1
|
||||
while (start >= 0 && text[start].isLetter()) {
|
||||
start--
|
||||
}
|
||||
val word = text.substring(start + 1, dotIndex).lowercase()
|
||||
val abbreviations = setOf(
|
||||
"mr", "mrs", "ms", "dr", "prof", "sr", "jr", "eg", "ie", "vs", "etc", "st", "co",
|
||||
"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p",
|
||||
"q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
|
||||
)
|
||||
return word in abbreviations
|
||||
}
|
||||
|
||||
private fun resetIdleTimeout() {
|
||||
idleTimeoutJob?.cancel()
|
||||
idleTimeoutJob = scope.launch {
|
||||
delay(10000L)
|
||||
closeWebSocketConnection()
|
||||
}
|
||||
}
|
||||
|
||||
private fun closeWebSocketConnection() {
|
||||
webSocket?.close(1000, "Idle timeout")
|
||||
webSocket = null
|
||||
}
|
||||
|
||||
private fun updateSpeakingState() {
|
||||
val shouldBeSpeaking = isGenerating || isPlaying || !speechQueue.isEmpty || !streamChannel.isEmpty
|
||||
if (shouldBeSpeaking != isSpeaking) {
|
||||
isSpeaking = shouldBeSpeaking
|
||||
scope.launch(Dispatchers.Main) {
|
||||
onSpeakingStateChanged(shouldBeSpeaking)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun speak(text: String, queueMode: Int) {
|
||||
scope.launch {
|
||||
if (queueMode == TtsManager.QUEUE_FLUSH) {
|
||||
stopPlaybackOnly()
|
||||
}
|
||||
speechQueue.send(text)
|
||||
updateSpeakingState()
|
||||
}
|
||||
}
|
||||
|
||||
private fun stopPlaybackOnly() {
|
||||
Log.i("EdgeTtsApiManager", "[EdgeTTS] Playback stopped/cleared")
|
||||
while (!speechQueue.isEmpty) {
|
||||
speechQueue.tryReceive()
|
||||
}
|
||||
while (!streamChannel.isEmpty) {
|
||||
val stream = streamChannel.tryReceive().getOrNull()
|
||||
stream?.rawDataChannel?.close()
|
||||
stream?.completionDeferred?.complete(Unit)
|
||||
}
|
||||
for (stream in activeStreams.values) {
|
||||
stream.rawDataChannel.close()
|
||||
stream.completionDeferred.complete(Unit)
|
||||
}
|
||||
activeStreams.clear()
|
||||
|
||||
startProcessing()
|
||||
|
||||
isGenerating = false
|
||||
isPlaying = false
|
||||
updateSpeakingState()
|
||||
}
|
||||
|
||||
override fun stop() {
|
||||
stopPlaybackOnly()
|
||||
}
|
||||
|
||||
override fun isSpeaking(): Boolean {
|
||||
return isSpeaking
|
||||
}
|
||||
|
||||
override fun shutdown() {
|
||||
Log.i("EdgeTtsApiManager", "[EdgeTTS] Shutdown")
|
||||
job.cancel()
|
||||
closeWebSocketConnection()
|
||||
ttsDispatcher.close()
|
||||
}
|
||||
}
|
||||
488
app/src/main/java/com/app/assistant/tts/GoogleTtsApiManager.kt
Normal file
488
app/src/main/java/com/app/assistant/tts/GoogleTtsApiManager.kt
Normal file
|
|
@ -0,0 +1,488 @@
|
|||
package com.app.assistant.tts
|
||||
|
||||
import android.content.Context
|
||||
import android.media.AudioAttributes
|
||||
import android.media.AudioFormat
|
||||
import android.media.AudioManager
|
||||
import android.media.AudioTrack
|
||||
import android.media.MediaCodec
|
||||
import android.media.MediaFormat
|
||||
import android.util.Log
|
||||
import com.app.assistant.config.SpeechConfig
|
||||
import com.app.assistant.repository.SettingsRepository
|
||||
import kotlinx.coroutines.*
|
||||
import kotlinx.coroutines.channels.Channel
|
||||
import okhttp3.*
|
||||
import okhttp3.MediaType.Companion.toMediaType
|
||||
import okhttp3.RequestBody.Companion.toRequestBody
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.buildJsonObject
|
||||
import kotlinx.serialization.json.put
|
||||
import kotlinx.serialization.json.putJsonObject
|
||||
import kotlinx.serialization.json.jsonObject
|
||||
import kotlinx.serialization.json.jsonPrimitive
|
||||
import java.io.ByteArrayOutputStream
|
||||
import java.util.*
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
|
||||
@OptIn(ExperimentalCoroutinesApi::class, ObsoleteCoroutinesApi::class, DelicateCoroutinesApi::class)
|
||||
class GoogleTtsApiManager(
|
||||
private val context: Context,
|
||||
private val settingsRepository: SettingsRepository,
|
||||
okHttpClient: OkHttpClient,
|
||||
private val onSpeakingStateChanged: (isSpeaking: Boolean) -> Unit
|
||||
) : TtsManager {
|
||||
|
||||
private val ttsDispatcher = newSingleThreadContext("GoogleTtsApiThread")
|
||||
private val job = SupervisorJob()
|
||||
private val scope = CoroutineScope(ttsDispatcher + job)
|
||||
|
||||
private val speechQueue = Channel<String>(Channel.UNLIMITED)
|
||||
private val streamChannel = Channel<SentenceStream>(Channel.UNLIMITED)
|
||||
|
||||
private val activeStreams = ConcurrentHashMap<String, SentenceStream>()
|
||||
|
||||
private var generatorJob: Job? = null
|
||||
private var playbackJob: Job? = null
|
||||
|
||||
private val client = okHttpClient.newBuilder()
|
||||
.connectTimeout(10, java.util.concurrent.TimeUnit.SECONDS)
|
||||
.readTimeout(10, java.util.concurrent.TimeUnit.SECONDS)
|
||||
.writeTimeout(10, java.util.concurrent.TimeUnit.SECONDS)
|
||||
.build()
|
||||
|
||||
@Volatile
|
||||
private var isSpeaking = false
|
||||
@Volatile
|
||||
private var isGenerating = false
|
||||
@Volatile
|
||||
private var isPlaying = false
|
||||
|
||||
private class SentenceStream(
|
||||
val requestId: String,
|
||||
val rawDataChannel: Channel<ByteArray> = Channel(Channel.UNLIMITED),
|
||||
val frameChannel: Channel<ByteArray> = Channel(Channel.UNLIMITED)
|
||||
)
|
||||
|
||||
init {
|
||||
Log.i("GoogleTtsApiManager", "[GoogleTTS] Initialized")
|
||||
startProcessing()
|
||||
}
|
||||
|
||||
private fun startProcessing() {
|
||||
generatorJob?.cancel()
|
||||
playbackJob?.cancel()
|
||||
|
||||
generatorJob = scope.launch {
|
||||
for (text in speechQueue) {
|
||||
isGenerating = true
|
||||
updateSpeakingState()
|
||||
|
||||
val sentences = splitIntoSentences(text)
|
||||
for (sentence in sentences) {
|
||||
val cleaned = cleanTextForTts(sentence)
|
||||
if (cleaned.isEmpty()) continue
|
||||
|
||||
val requestId = UUID.randomUUID().toString().replace("-", "").lowercase()
|
||||
val stream = SentenceStream(requestId)
|
||||
activeStreams[requestId] = stream
|
||||
|
||||
launchParser(stream)
|
||||
|
||||
try {
|
||||
val audioBytes = fetchGoogleTts(cleaned)
|
||||
stream.rawDataChannel.send(audioBytes)
|
||||
stream.rawDataChannel.close()
|
||||
streamChannel.send(stream)
|
||||
} catch (e: Exception) {
|
||||
Log.e("GoogleTtsApiManager", "Error fetching Google TTS for sentence: $cleaned", e)
|
||||
stream.rawDataChannel.close()
|
||||
}
|
||||
|
||||
activeStreams.remove(requestId)
|
||||
}
|
||||
|
||||
if (speechQueue.isEmpty) {
|
||||
isGenerating = false
|
||||
updateSpeakingState()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
playbackJob = scope.launch {
|
||||
for (stream in streamChannel) {
|
||||
isPlaying = true
|
||||
updateSpeakingState()
|
||||
|
||||
playStream(stream)
|
||||
|
||||
if (streamChannel.isEmpty && !isGenerating && speechQueue.isEmpty) {
|
||||
isPlaying = false
|
||||
updateSpeakingState()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun fetchGoogleTts(text: String): ByteArray = withContext(Dispatchers.IO) {
|
||||
val apiKey = SpeechConfig.GoogleTts.getApiKey(settingsRepository)
|
||||
if (apiKey.isEmpty()) {
|
||||
throw Exception("Google Cloud TTS API key is empty.")
|
||||
}
|
||||
val url = "${SpeechConfig.GoogleTts.BASE_URL}?key=$apiKey"
|
||||
|
||||
val voiceName = SpeechConfig.GoogleTts.VOICE
|
||||
val parts = voiceName.split("-")
|
||||
val languageCode = if (parts.size >= 2) "${parts[0]}-${parts[1]}" else "en-US"
|
||||
|
||||
val jsonBodyElement = buildJsonObject {
|
||||
putJsonObject("input") {
|
||||
put("text", text)
|
||||
}
|
||||
putJsonObject("voice") {
|
||||
put("languageCode", languageCode)
|
||||
put("name", voiceName)
|
||||
}
|
||||
putJsonObject("audioConfig") {
|
||||
put("audioEncoding", SpeechConfig.GoogleTts.AUDIO_ENCODING)
|
||||
put("sampleRateHertz", 24000)
|
||||
}
|
||||
}
|
||||
val jsonBody = jsonBodyElement.toString()
|
||||
|
||||
val requestBody = jsonBody.toRequestBody("application/json; charset=utf-8".toMediaType())
|
||||
val request = Request.Builder()
|
||||
.url(url)
|
||||
.post(requestBody)
|
||||
.build()
|
||||
|
||||
client.newCall(request).execute().use { response ->
|
||||
if (!response.isSuccessful) {
|
||||
val errBody = response.body?.string() ?: ""
|
||||
throw Exception("Google Cloud TTS API call failed: ${response.code} - ${response.message}. Body: $errBody")
|
||||
}
|
||||
val bodyStr = response.body?.string() ?: throw Exception("Empty response body from Google TTS")
|
||||
val jsonElement = Json.parseToJsonElement(bodyStr)
|
||||
val audioContentBase64 = jsonElement.jsonObject["audioContent"]?.jsonPrimitive?.content
|
||||
?: throw Exception("No audioContent field in response JSON")
|
||||
android.util.Base64.decode(audioContentBase64, android.util.Base64.DEFAULT)
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun playStream(stream: SentenceStream) {
|
||||
withContext(Dispatchers.Default) {
|
||||
val sampleRate = 24000
|
||||
val channelConfig = AudioFormat.CHANNEL_OUT_MONO
|
||||
val audioFormat = AudioFormat.ENCODING_PCM_16BIT
|
||||
val minBufferSize = AudioTrack.getMinBufferSize(sampleRate, channelConfig, audioFormat)
|
||||
|
||||
val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
|
||||
val useVoiceCall = audioManager.isBluetoothScoOn ||
|
||||
audioManager.mode == AudioManager.MODE_IN_CALL ||
|
||||
audioManager.mode == AudioManager.MODE_IN_COMMUNICATION
|
||||
|
||||
val audioTrack = AudioTrack.Builder()
|
||||
.setAudioAttributes(
|
||||
AudioAttributes.Builder()
|
||||
.setUsage(if (useVoiceCall) AudioAttributes.USAGE_VOICE_COMMUNICATION else AudioAttributes.USAGE_MEDIA)
|
||||
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
|
||||
.build()
|
||||
)
|
||||
.setAudioFormat(
|
||||
AudioFormat.Builder()
|
||||
.setEncoding(audioFormat)
|
||||
.setSampleRate(sampleRate)
|
||||
.setChannelMask(channelConfig)
|
||||
.build()
|
||||
)
|
||||
.setBufferSizeInBytes(Math.max(minBufferSize, 1024 * 16))
|
||||
.setTransferMode(AudioTrack.MODE_STREAM)
|
||||
.build()
|
||||
|
||||
var codec: MediaCodec? = null
|
||||
try {
|
||||
audioTrack.play()
|
||||
|
||||
val mediaFormat = MediaFormat.createAudioFormat(MediaFormat.MIMETYPE_AUDIO_MPEG, sampleRate, 1)
|
||||
codec = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_AUDIO_MPEG).apply {
|
||||
configure(mediaFormat, null, null, 0)
|
||||
start()
|
||||
}
|
||||
|
||||
var decodingActive = true
|
||||
var hasMoreInput = true
|
||||
var presentationTimeUs = 0L
|
||||
val frameIterator = stream.frameChannel.iterator()
|
||||
|
||||
while (decodingActive && coroutineContext.isActive) {
|
||||
var didWork = false
|
||||
|
||||
if (hasMoreInput) {
|
||||
val inputIndex = codec.dequeueInputBuffer(0)
|
||||
if (inputIndex >= 0) {
|
||||
val inputBuffer = codec.getInputBuffer(inputIndex)
|
||||
if (inputBuffer != null) {
|
||||
inputBuffer.clear()
|
||||
if (frameIterator.hasNext()) {
|
||||
val frame = frameIterator.next()
|
||||
inputBuffer.put(frame)
|
||||
codec.queueInputBuffer(inputIndex, 0, frame.size, presentationTimeUs, 0)
|
||||
presentationTimeUs += (frame.size * 8000000L) / 48000L
|
||||
didWork = true
|
||||
} else {
|
||||
if (stream.frameChannel.isClosedForReceive) {
|
||||
codec.queueInputBuffer(inputIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
|
||||
hasMoreInput = false
|
||||
didWork = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val bufferInfo = MediaCodec.BufferInfo()
|
||||
val outputIndex = codec.dequeueOutputBuffer(bufferInfo, 0)
|
||||
if (outputIndex >= 0) {
|
||||
val outputBuffer = codec.getOutputBuffer(outputIndex)
|
||||
if (outputBuffer != null && bufferInfo.size > 0) {
|
||||
val pcmData = ByteArray(bufferInfo.size)
|
||||
outputBuffer.get(pcmData)
|
||||
outputBuffer.clear()
|
||||
audioTrack.write(pcmData, 0, pcmData.size)
|
||||
}
|
||||
codec.releaseOutputBuffer(outputIndex, false)
|
||||
|
||||
if ((bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
|
||||
decodingActive = false
|
||||
}
|
||||
didWork = true
|
||||
}
|
||||
|
||||
if (!didWork) {
|
||||
delay(5)
|
||||
}
|
||||
}
|
||||
|
||||
if (coroutineContext.isActive) {
|
||||
delay(150)
|
||||
}
|
||||
|
||||
} catch (e: Exception) {
|
||||
Log.e("GoogleTtsApiManager", "Error decoding/playing stream", e)
|
||||
} finally {
|
||||
try {
|
||||
audioTrack.stop()
|
||||
audioTrack.flush()
|
||||
audioTrack.release()
|
||||
} catch (e: Exception) {}
|
||||
try {
|
||||
codec?.stop()
|
||||
codec?.release()
|
||||
} catch (e: Exception) {}
|
||||
}
|
||||
Unit
|
||||
}
|
||||
}
|
||||
|
||||
private fun launchParser(stream: SentenceStream): Job {
|
||||
return scope.launch(Dispatchers.Default) {
|
||||
val parserBuffer = ByteArrayOutputStream()
|
||||
try {
|
||||
for (bytes in stream.rawDataChannel) {
|
||||
parserBuffer.write(bytes)
|
||||
var bufferBytes = parserBuffer.toByteArray()
|
||||
var offset = 0
|
||||
while (offset + 4 <= bufferBytes.size) {
|
||||
val b0 = bufferBytes[offset].toInt() and 0xFF
|
||||
val b1 = bufferBytes[offset + 1].toInt() and 0xFF
|
||||
if (b0 == 0xFF && (b1 and 0xE0) == 0xE0) {
|
||||
val version = (b1 shr 3) and 0x03
|
||||
val layer = (b1 shr 1) and 0x03
|
||||
val bitrateIndex = (bufferBytes[offset + 2].toInt() and 0xFF) shr 4
|
||||
val srIndex = (bufferBytes[offset + 2].toInt() and 0xFF shr 2) and 0x03
|
||||
val padding = (bufferBytes[offset + 2].toInt() and 0xFF shr 1) and 0x01
|
||||
|
||||
val mpeg1Bitrates = intArrayOf(0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 0)
|
||||
val mpeg2Bitrates = intArrayOf(0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 0)
|
||||
val sampleRates = arrayOf(
|
||||
intArrayOf(11025, 12000, 8000),
|
||||
intArrayOf(0, 0, 0),
|
||||
intArrayOf(22050, 24000, 16000),
|
||||
intArrayOf(44100, 48000, 32000)
|
||||
)
|
||||
|
||||
if (version < sampleRates.size && srIndex < sampleRates[version].size) {
|
||||
val bitrate = (if (version == 3) mpeg1Bitrates[bitrateIndex] else mpeg2Bitrates[bitrateIndex]) * 1000
|
||||
val sampleRate = sampleRates[version][srIndex]
|
||||
|
||||
if (bitrate > 0 && sampleRate > 0 && layer == 1) {
|
||||
val coeff = if (version == 3) 144 else 72
|
||||
val frameSize = (coeff * bitrate) / sampleRate + padding
|
||||
|
||||
if (offset + frameSize <= bufferBytes.size) {
|
||||
val frame = bufferBytes.copyOfRange(offset, offset + frameSize)
|
||||
stream.frameChannel.send(frame)
|
||||
offset += frameSize
|
||||
continue
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
offset++
|
||||
}
|
||||
|
||||
val temp = parserBuffer.toByteArray()
|
||||
parserBuffer.reset()
|
||||
if (offset < temp.size) {
|
||||
parserBuffer.write(temp, offset, temp.size - offset)
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("GoogleTtsApiManager", "Error parsing MP3 frames", e)
|
||||
} finally {
|
||||
stream.frameChannel.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun cleanTextForTts(text: String): String {
|
||||
return text
|
||||
.replace("—", " - ")
|
||||
.replace("–", " - ")
|
||||
.replace("“", "")
|
||||
.replace("”", "")
|
||||
.replace("\"", "")
|
||||
.replace("‘", "'")
|
||||
.replace("’", "'")
|
||||
.trim()
|
||||
}
|
||||
|
||||
private fun splitIntoSentences(text: String): List<String> {
|
||||
val sentences = mutableListOf<String>()
|
||||
val currentSentence = StringBuilder()
|
||||
val len = text.length
|
||||
var i = 0
|
||||
while (i < len) {
|
||||
val c = text[i]
|
||||
currentSentence.append(c)
|
||||
|
||||
if (c == '\n' || c == '\r') {
|
||||
val sentenceStr = currentSentence.toString().trim()
|
||||
if (sentenceStr.isNotEmpty()) {
|
||||
sentences.add(sentenceStr)
|
||||
}
|
||||
currentSentence.clear()
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
if (c == '.' || c == '?' || c == '!' || c == ':' || c == ';') {
|
||||
var isBoundary = false
|
||||
if (i + 1 == len) {
|
||||
isBoundary = true
|
||||
} else {
|
||||
val nextChar = text[i + 1]
|
||||
if (nextChar.isWhitespace()) {
|
||||
isBoundary = true
|
||||
}
|
||||
}
|
||||
|
||||
if (isBoundary) {
|
||||
if (c == '.' && isAbbreviationOrDecimal(text, i)) {
|
||||
// abbreviation
|
||||
} else {
|
||||
val sentenceStr = currentSentence.toString().trim()
|
||||
if (sentenceStr.isNotEmpty()) {
|
||||
sentences.add(sentenceStr)
|
||||
}
|
||||
currentSentence.clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
i++
|
||||
}
|
||||
|
||||
val remaining = currentSentence.toString().trim()
|
||||
if (remaining.isNotEmpty()) {
|
||||
sentences.add(remaining)
|
||||
}
|
||||
|
||||
return sentences
|
||||
}
|
||||
|
||||
private fun isAbbreviationOrDecimal(text: String, dotIndex: Int): Boolean {
|
||||
if (dotIndex > 0 && dotIndex + 1 < text.length) {
|
||||
if (text[dotIndex - 1].isDigit() && text[dotIndex + 1].isDigit()) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
var start = dotIndex - 1
|
||||
while (start >= 0 && text[start].isLetter()) {
|
||||
start--
|
||||
}
|
||||
val word = text.substring(start + 1, dotIndex).lowercase()
|
||||
val abbreviations = setOf(
|
||||
"mr", "mrs", "ms", "dr", "prof", "sr", "jr", "eg", "ie", "vs", "etc", "st", "co",
|
||||
"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p",
|
||||
"q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
|
||||
)
|
||||
return word in abbreviations
|
||||
}
|
||||
|
||||
private fun updateSpeakingState() {
|
||||
val shouldBeSpeaking = isGenerating || isPlaying || !speechQueue.isEmpty || !streamChannel.isEmpty
|
||||
if (shouldBeSpeaking != isSpeaking) {
|
||||
isSpeaking = shouldBeSpeaking
|
||||
scope.launch(Dispatchers.Main) {
|
||||
onSpeakingStateChanged(shouldBeSpeaking)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun speak(text: String, queueMode: Int) {
|
||||
scope.launch {
|
||||
if (queueMode == TtsManager.QUEUE_FLUSH) {
|
||||
stopPlaybackOnly()
|
||||
}
|
||||
speechQueue.send(text)
|
||||
updateSpeakingState()
|
||||
}
|
||||
}
|
||||
|
||||
private fun stopPlaybackOnly() {
|
||||
Log.i("GoogleTtsApiManager", "[GoogleTTS] Playback stopped/cleared")
|
||||
while (!speechQueue.isEmpty) {
|
||||
speechQueue.tryReceive()
|
||||
}
|
||||
while (!streamChannel.isEmpty) {
|
||||
val stream = streamChannel.tryReceive().getOrNull()
|
||||
stream?.rawDataChannel?.close()
|
||||
}
|
||||
for (stream in activeStreams.values) {
|
||||
stream.rawDataChannel.close()
|
||||
}
|
||||
activeStreams.clear()
|
||||
|
||||
startProcessing()
|
||||
|
||||
isGenerating = false
|
||||
isPlaying = false
|
||||
updateSpeakingState()
|
||||
}
|
||||
|
||||
override fun stop() {
|
||||
stopPlaybackOnly()
|
||||
}
|
||||
|
||||
override fun isSpeaking(): Boolean {
|
||||
return isSpeaking
|
||||
}
|
||||
|
||||
override fun shutdown() {
|
||||
Log.i("GoogleTtsApiManager", "[GoogleTTS] Shutdown")
|
||||
job.cancel()
|
||||
ttsDispatcher.close()
|
||||
}
|
||||
}
|
||||
163
app/src/main/java/com/app/assistant/tts/NativeTtsManager.kt
Normal file
163
app/src/main/java/com/app/assistant/tts/NativeTtsManager.kt
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
package com.app.assistant.tts
|
||||
|
||||
import android.content.Context
|
||||
import android.media.AudioAttributes
|
||||
import android.media.AudioManager
|
||||
import android.speech.tts.TextToSpeech
|
||||
import android.speech.tts.UtteranceProgressListener
|
||||
import android.util.Log
|
||||
import java.util.Collections
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
|
||||
class NativeTtsManager(
|
||||
private val context: Context,
|
||||
private val onSpeakingStateChanged: (isSpeaking: Boolean) -> Unit
|
||||
) : TtsManager {
|
||||
private var textToSpeech: TextToSpeech? = null
|
||||
private var isInitialized = false
|
||||
|
||||
// Track active/queued utterances to prevent premature transition to LISTENING state
|
||||
private val activeUtterances = Collections.newSetFromMap(ConcurrentHashMap<String, Boolean>())
|
||||
|
||||
// Buffer incoming speak requests while the platform TTS engine is initializing
|
||||
private val pendingUtterances = Collections.synchronizedList(mutableListOf<Pair<String, Int>>())
|
||||
|
||||
init {
|
||||
textToSpeech = TextToSpeech(context.applicationContext) { status ->
|
||||
if (status == TextToSpeech.SUCCESS) {
|
||||
isInitialized = true
|
||||
Log.d("NativeTtsManager", "Initialization Success")
|
||||
textToSpeech?.setOnUtteranceProgressListener(object : UtteranceProgressListener() {
|
||||
override fun onStart(utteranceId: String?) {
|
||||
onSpeakingStateChanged(true)
|
||||
}
|
||||
|
||||
override fun onDone(utteranceId: String?) {
|
||||
utteranceId?.let { activeUtterances.remove(it) }
|
||||
if (activeUtterances.isEmpty()) {
|
||||
onSpeakingStateChanged(false)
|
||||
}
|
||||
}
|
||||
|
||||
@Deprecated("Deprecated in Java")
|
||||
override fun onError(utteranceId: String?) {
|
||||
utteranceId?.let { activeUtterances.remove(it) }
|
||||
if (activeUtterances.isEmpty()) {
|
||||
onSpeakingStateChanged(false)
|
||||
}
|
||||
}
|
||||
|
||||
override fun onStop(utteranceId: String?, interrupted: Boolean) {
|
||||
utteranceId?.let { activeUtterances.remove(it) }
|
||||
if (activeUtterances.isEmpty()) {
|
||||
onSpeakingStateChanged(false)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// Flush any buffered utterances now that the TTS service is bound
|
||||
synchronized(pendingUtterances) {
|
||||
for (pair in pendingUtterances) {
|
||||
speakInternal(pair.first, pair.second)
|
||||
}
|
||||
pendingUtterances.clear()
|
||||
}
|
||||
} else {
|
||||
Log.e("NativeTtsManager", "Initialization Failed")
|
||||
synchronized(pendingUtterances) {
|
||||
pendingUtterances.clear()
|
||||
}
|
||||
onSpeakingStateChanged(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun speak(text: String, queueMode: Int) {
|
||||
val tts = textToSpeech
|
||||
if (tts == null) {
|
||||
Log.w("NativeTtsManager", "TTS is null")
|
||||
return
|
||||
}
|
||||
|
||||
if (!isInitialized) {
|
||||
Log.i("NativeTtsManager", "TTS not initialized yet. Queueing utterance: $text")
|
||||
synchronized(pendingUtterances) {
|
||||
if (queueMode == TtsManager.QUEUE_FLUSH) {
|
||||
pendingUtterances.clear()
|
||||
}
|
||||
pendingUtterances.add(Pair(text, queueMode))
|
||||
}
|
||||
onSpeakingStateChanged(true)
|
||||
return
|
||||
}
|
||||
|
||||
speakInternal(text, queueMode)
|
||||
}
|
||||
|
||||
private fun speakInternal(text: String, queueMode: Int) {
|
||||
val tts = textToSpeech ?: return
|
||||
|
||||
val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
|
||||
val useVoiceCall = audioManager.isBluetoothScoOn ||
|
||||
audioManager.mode == AudioManager.MODE_IN_CALL ||
|
||||
audioManager.mode == AudioManager.MODE_IN_COMMUNICATION
|
||||
|
||||
val audioAttributes = AudioAttributes.Builder()
|
||||
.setUsage(if (useVoiceCall) AudioAttributes.USAGE_VOICE_COMMUNICATION else AudioAttributes.USAGE_MEDIA)
|
||||
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
|
||||
.build()
|
||||
tts.setAudioAttributes(audioAttributes)
|
||||
|
||||
val mode = if (queueMode == TtsManager.QUEUE_ADD) {
|
||||
TextToSpeech.QUEUE_ADD
|
||||
} else {
|
||||
activeUtterances.clear()
|
||||
TextToSpeech.QUEUE_FLUSH
|
||||
}
|
||||
|
||||
val utteranceId = "tts_utterance_id_${System.currentTimeMillis()}"
|
||||
activeUtterances.add(utteranceId)
|
||||
|
||||
val result = tts.speak(text, mode, null, utteranceId)
|
||||
if (result != TextToSpeech.SUCCESS) {
|
||||
activeUtterances.remove(utteranceId)
|
||||
if (activeUtterances.isEmpty()) {
|
||||
onSpeakingStateChanged(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun stop() {
|
||||
synchronized(pendingUtterances) {
|
||||
pendingUtterances.clear()
|
||||
}
|
||||
if (isInitialized) {
|
||||
activeUtterances.clear()
|
||||
val wasSpeaking = isSpeaking()
|
||||
textToSpeech?.stop()
|
||||
if (wasSpeaking) {
|
||||
onSpeakingStateChanged(false)
|
||||
}
|
||||
} else {
|
||||
onSpeakingStateChanged(false)
|
||||
}
|
||||
}
|
||||
|
||||
override fun isSpeaking(): Boolean {
|
||||
return (isInitialized && (textToSpeech?.isSpeaking == true || activeUtterances.isNotEmpty())) ||
|
||||
pendingUtterances.isNotEmpty()
|
||||
}
|
||||
|
||||
override fun shutdown() {
|
||||
synchronized(pendingUtterances) {
|
||||
pendingUtterances.clear()
|
||||
}
|
||||
activeUtterances.clear()
|
||||
textToSpeech?.let {
|
||||
it.stop()
|
||||
it.shutdown()
|
||||
}
|
||||
textToSpeech = null
|
||||
isInitialized = false
|
||||
}
|
||||
}
|
||||
397
app/src/main/java/com/app/assistant/tts/OfflineTtsManager.kt
Normal file
397
app/src/main/java/com/app/assistant/tts/OfflineTtsManager.kt
Normal file
|
|
@ -0,0 +1,397 @@
|
|||
package com.app.assistant.tts
|
||||
|
||||
import android.content.Context
|
||||
import android.media.AudioAttributes
|
||||
import android.media.AudioFormat
|
||||
import android.media.AudioManager
|
||||
import android.media.AudioTrack
|
||||
import android.util.Log
|
||||
import com.app.assistant.repository.SettingsRepository
|
||||
import kotlinx.coroutines.*
|
||||
import kotlinx.coroutines.channels.Channel
|
||||
import java.io.File
|
||||
import com.k2fsa.sherpa.onnx.*
|
||||
import kotlin.coroutines.coroutineContext
|
||||
|
||||
@OptIn(ExperimentalCoroutinesApi::class, ObsoleteCoroutinesApi::class, DelicateCoroutinesApi::class)
|
||||
class OfflineTtsManager(
|
||||
private val context: Context,
|
||||
private val settingsRepository: SettingsRepository,
|
||||
private val onSpeakingStateChanged: (isSpeaking: Boolean) -> Unit
|
||||
) : TtsManager {
|
||||
|
||||
private val ttsDispatcher = newSingleThreadContext("OfflineTtsThread")
|
||||
private val job = SupervisorJob()
|
||||
private val scope = CoroutineScope(ttsDispatcher + job)
|
||||
|
||||
private val speechQueue = Channel<String>(Channel.UNLIMITED)
|
||||
private val audioQueue = Channel<GeneratedAudio>(Channel.UNLIMITED)
|
||||
|
||||
private var offlineTts: OfflineTts? = null
|
||||
private var audioTrack: AudioTrack? = null
|
||||
|
||||
private var generatorJob: Job? = null
|
||||
private var playbackJob: Job? = null
|
||||
|
||||
@Volatile
|
||||
private var isSpeaking = false
|
||||
@Volatile
|
||||
private var isGenerating = false
|
||||
@Volatile
|
||||
private var isPlaying = false
|
||||
private var isInitialized = false
|
||||
|
||||
private class GeneratedAudio(val samples: ShortArray, val sampleRate: Int)
|
||||
|
||||
init {
|
||||
startQueueProcessing()
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
private fun initTtsEngine(): Boolean {
|
||||
if (isInitialized && offlineTts != null) return true
|
||||
|
||||
try {
|
||||
val modelConfig: OfflineTtsModelConfig
|
||||
|
||||
val supertonicDir = File(context.filesDir, "sherpa-onnx-tts/sherpa-onnx-supertonic-3-tts-int8-2026-05-11")
|
||||
val durationPredictor = File(supertonicDir, "duration_predictor.int8.onnx")
|
||||
val textEncoder = File(supertonicDir, "text_encoder.int8.onnx")
|
||||
val vectorEstimator = File(supertonicDir, "vector_estimator.int8.onnx")
|
||||
val vocoder = File(supertonicDir, "vocoder.int8.onnx")
|
||||
val ttsJson = File(supertonicDir, "tts.json")
|
||||
val unicodeIndexer = File(supertonicDir, "unicode_indexer.bin")
|
||||
val voiceStyle = File(supertonicDir, "voice.bin")
|
||||
|
||||
if (!durationPredictor.exists() || !textEncoder.exists() || !vectorEstimator.exists() ||
|
||||
!vocoder.exists() || !ttsJson.exists() || !unicodeIndexer.exists() || !voiceStyle.exists()) {
|
||||
Log.w("OfflineTtsManager", "Cannot init OfflineTts (SUPERTONIC): Missing files. durationPredictor: ${durationPredictor.exists()}, textEncoder: ${textEncoder.exists()}, vectorEstimator: ${vectorEstimator.exists()}, vocoder: ${vocoder.exists()}, ttsJson: ${ttsJson.exists()}, unicodeIndexer: ${unicodeIndexer.exists()}, voiceStyle: ${voiceStyle.exists()}")
|
||||
return false
|
||||
}
|
||||
|
||||
val supertonicConfig = OfflineTtsSupertonicModelConfig(
|
||||
durationPredictor = durationPredictor.absolutePath,
|
||||
textEncoder = textEncoder.absolutePath,
|
||||
vectorEstimator = vectorEstimator.absolutePath,
|
||||
vocoder = vocoder.absolutePath,
|
||||
ttsJson = ttsJson.absolutePath,
|
||||
unicodeIndexer = unicodeIndexer.absolutePath,
|
||||
voiceStyle = voiceStyle.absolutePath
|
||||
)
|
||||
modelConfig = OfflineTtsModelConfig(
|
||||
supertonic = supertonicConfig,
|
||||
numThreads = 2,
|
||||
debug = true
|
||||
)
|
||||
|
||||
val config = OfflineTtsConfig(
|
||||
model = modelConfig,
|
||||
maxNumSentences = 1
|
||||
)
|
||||
|
||||
// JNI initialization requires AssetManager to be null when loading absolute paths
|
||||
offlineTts = OfflineTts(null, config)
|
||||
isInitialized = true
|
||||
Log.i("OfflineTtsManager", "Engine initialized successfully (type: SUPERTONIC)")
|
||||
return true
|
||||
} catch (e: Exception) {
|
||||
Log.e("OfflineTtsManager", "Failed to initialize OfflineTts engine", e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
private fun cleanTextForTts(text: String): String {
|
||||
return text
|
||||
.replace("—", " - ")
|
||||
.replace("–", " - ")
|
||||
.replace("“", "")
|
||||
.replace("”", "")
|
||||
.replace("\"", "")
|
||||
.replace("‘", "'")
|
||||
.replace("’", "'")
|
||||
.trim()
|
||||
}
|
||||
|
||||
private fun startQueueProcessing() {
|
||||
generatorJob?.cancel()
|
||||
playbackJob?.cancel()
|
||||
|
||||
while (!audioQueue.isEmpty) {
|
||||
audioQueue.tryReceive()
|
||||
}
|
||||
|
||||
generatorJob = scope.launch {
|
||||
for (text in speechQueue) {
|
||||
if (!initTtsEngine()) {
|
||||
Log.w("OfflineTtsManager", "Engine not initialized. Skipping text: $text")
|
||||
continue
|
||||
}
|
||||
|
||||
val tts = offlineTts ?: continue
|
||||
val cleanedText = cleanTextForTts(text)
|
||||
if (cleanedText.isEmpty()) continue
|
||||
|
||||
isGenerating = true
|
||||
updateSpeakingState()
|
||||
|
||||
try {
|
||||
val audio = withContext(Dispatchers.Default) {
|
||||
tts.generate(cleanedText, 0, 1.0f)
|
||||
}
|
||||
val floatSamples = audio.samples
|
||||
val sampleRate = audio.sampleRate
|
||||
|
||||
if (floatSamples.isNotEmpty()) {
|
||||
val shortSamples = ShortArray(floatSamples.size)
|
||||
for (i in floatSamples.indices) {
|
||||
val sample = floatSamples[i]
|
||||
val clamped = Math.max(-1.0f, Math.min(1.0f, sample))
|
||||
shortSamples[i] = (clamped * 32767.0f).toInt().toShort()
|
||||
}
|
||||
audioQueue.send(GeneratedAudio(shortSamples, sampleRate))
|
||||
}
|
||||
} catch (e: CancellationException) {
|
||||
break
|
||||
} catch (e: Exception) {
|
||||
Log.e("OfflineTtsManager", "Error generating speech", e)
|
||||
}
|
||||
|
||||
if (speechQueue.isEmpty) {
|
||||
isGenerating = false
|
||||
updateSpeakingState()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
playbackJob = scope.launch {
|
||||
var activeTrack: AudioTrack? = null
|
||||
var currentSampleRate = 0
|
||||
|
||||
try {
|
||||
while (coroutineContext.isActive) {
|
||||
val audio = audioQueue.receive()
|
||||
isPlaying = true
|
||||
updateSpeakingState()
|
||||
|
||||
if (activeTrack == null || currentSampleRate != audio.sampleRate) {
|
||||
activeTrack?.let {
|
||||
try { it.stop(); it.release() } catch (e: Exception) {}
|
||||
}
|
||||
currentSampleRate = audio.sampleRate
|
||||
val minBufferSize = AudioTrack.getMinBufferSize(
|
||||
currentSampleRate,
|
||||
AudioFormat.CHANNEL_OUT_MONO,
|
||||
AudioFormat.ENCODING_PCM_16BIT
|
||||
)
|
||||
val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
|
||||
val useVoiceCall = audioManager.isBluetoothScoOn ||
|
||||
audioManager.mode == AudioManager.MODE_IN_CALL ||
|
||||
audioManager.mode == AudioManager.MODE_IN_COMMUNICATION
|
||||
|
||||
activeTrack = AudioTrack.Builder()
|
||||
.setAudioAttributes(
|
||||
AudioAttributes.Builder()
|
||||
.setUsage(if (useVoiceCall) AudioAttributes.USAGE_VOICE_COMMUNICATION else AudioAttributes.USAGE_MEDIA)
|
||||
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
|
||||
.build()
|
||||
)
|
||||
.setAudioFormat(
|
||||
AudioFormat.Builder()
|
||||
.setEncoding(AudioFormat.ENCODING_PCM_16BIT)
|
||||
.setSampleRate(currentSampleRate)
|
||||
.setChannelMask(AudioFormat.CHANNEL_OUT_MONO)
|
||||
.build()
|
||||
)
|
||||
.setBufferSizeInBytes(Math.max(minBufferSize, audio.samples.size * 2))
|
||||
.setTransferMode(AudioTrack.MODE_STREAM)
|
||||
.build()
|
||||
activeTrack.play()
|
||||
audioTrack = activeTrack
|
||||
}
|
||||
|
||||
// Write short samples to stream track
|
||||
activeTrack.write(audio.samples, 0, audio.samples.size, AudioTrack.WRITE_BLOCKING)
|
||||
|
||||
// If there is no more audio in the queue, and we are not generating more,
|
||||
// we wait for the current audio block to finish playing before releasing the track.
|
||||
if (audioQueue.isEmpty && !isGenerating && speechQueue.isEmpty) {
|
||||
val durationMs = (audio.samples.size.toDouble() / currentSampleRate * 1000).toLong()
|
||||
delay(durationMs)
|
||||
|
||||
// Check again to ensure no new audio arrived during the delay
|
||||
if (audioQueue.isEmpty && !isGenerating && speechQueue.isEmpty) {
|
||||
try {
|
||||
activeTrack.stop()
|
||||
activeTrack.release()
|
||||
} catch (e: Exception) {}
|
||||
activeTrack = null
|
||||
audioTrack = null
|
||||
isPlaying = false
|
||||
updateSpeakingState()
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e: CancellationException) {
|
||||
// normal cancellation
|
||||
} catch (e: Exception) {
|
||||
Log.e("OfflineTtsManager", "Error playing audio stream", e)
|
||||
} finally {
|
||||
activeTrack?.let {
|
||||
try { it.stop(); it.release() } catch (e: Exception) {}
|
||||
}
|
||||
audioTrack = null
|
||||
isPlaying = false
|
||||
updateSpeakingState()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun updateSpeakingState() {
|
||||
val shouldBeSpeaking = isGenerating || isPlaying || !speechQueue.isEmpty || !audioQueue.isEmpty
|
||||
if (shouldBeSpeaking != isSpeaking) {
|
||||
isSpeaking = shouldBeSpeaking
|
||||
scope.launch(Dispatchers.Main) {
|
||||
onSpeakingStateChanged(shouldBeSpeaking)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun speak(text: String, queueMode: Int) {
|
||||
scope.launch {
|
||||
if (queueMode == TtsManager.QUEUE_FLUSH) {
|
||||
stopPlaybackOnly()
|
||||
}
|
||||
val sentences = splitIntoSentences(text)
|
||||
for (sentence in sentences) {
|
||||
val cleanedText = cleanTextForTts(sentence)
|
||||
if (cleanedText.isNotEmpty()) {
|
||||
speechQueue.send(cleanedText)
|
||||
}
|
||||
}
|
||||
updateSpeakingState()
|
||||
}
|
||||
}
|
||||
|
||||
private fun splitIntoSentences(text: String): List<String> {
|
||||
val sentences = mutableListOf<String>()
|
||||
val currentSentence = StringBuilder()
|
||||
val len = text.length
|
||||
var i = 0
|
||||
while (i < len) {
|
||||
val c = text[i]
|
||||
currentSentence.append(c)
|
||||
|
||||
// Split on newline or carriage return
|
||||
if (c == '\n' || c == '\r') {
|
||||
val sentenceStr = currentSentence.toString().trim()
|
||||
if (sentenceStr.isNotEmpty()) {
|
||||
sentences.add(sentenceStr)
|
||||
}
|
||||
currentSentence.clear()
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
// Split on sentence-ending punctuation followed by whitespace or end of text
|
||||
if (c == '.' || c == '?' || c == '!' || c == ':' || c == ';') {
|
||||
var isBoundary = false
|
||||
if (i + 1 == len) {
|
||||
isBoundary = true
|
||||
} else {
|
||||
val nextChar = text[i + 1]
|
||||
if (nextChar.isWhitespace()) {
|
||||
isBoundary = true
|
||||
}
|
||||
}
|
||||
|
||||
if (isBoundary) {
|
||||
// Check if it's an abbreviation or decimal number (only for period)
|
||||
if (c == '.' && isAbbreviationOrDecimal(text, i)) {
|
||||
// Not a boundary
|
||||
} else {
|
||||
val sentenceStr = currentSentence.toString().trim()
|
||||
if (sentenceStr.isNotEmpty()) {
|
||||
sentences.add(sentenceStr)
|
||||
}
|
||||
currentSentence.clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
i++
|
||||
}
|
||||
|
||||
// Add any remaining text
|
||||
val remaining = currentSentence.toString().trim()
|
||||
if (remaining.isNotEmpty()) {
|
||||
sentences.add(remaining)
|
||||
}
|
||||
|
||||
return sentences
|
||||
}
|
||||
|
||||
private fun isAbbreviationOrDecimal(text: String, dotIndex: Int): Boolean {
|
||||
if (dotIndex > 0 && dotIndex + 1 < text.length) {
|
||||
if (text[dotIndex - 1].isDigit() && text[dotIndex + 1].isDigit()) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
var start = dotIndex - 1
|
||||
while (start >= 0 && text[start].isLetter()) {
|
||||
start--
|
||||
}
|
||||
val word = text.substring(start + 1, dotIndex).lowercase()
|
||||
val abbreviations = setOf(
|
||||
"mr", "mrs", "ms", "dr", "prof", "sr", "jr", "eg", "ie", "vs", "etc", "st", "co",
|
||||
"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p",
|
||||
"q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
|
||||
)
|
||||
return word in abbreviations
|
||||
}
|
||||
|
||||
private fun stopPlaybackOnly() {
|
||||
while (!speechQueue.isEmpty) {
|
||||
speechQueue.tryReceive()
|
||||
}
|
||||
|
||||
try {
|
||||
audioTrack?.let {
|
||||
if (it.playState == AudioTrack.PLAYSTATE_PLAYING) {
|
||||
it.stop()
|
||||
}
|
||||
it.release()
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("OfflineTtsManager", "Error stopping AudioTrack", e)
|
||||
}
|
||||
audioTrack = null
|
||||
|
||||
startQueueProcessing()
|
||||
|
||||
isGenerating = false
|
||||
isPlaying = false
|
||||
updateSpeakingState()
|
||||
}
|
||||
|
||||
override fun stop() {
|
||||
stopPlaybackOnly()
|
||||
}
|
||||
|
||||
override fun isSpeaking(): Boolean {
|
||||
return isSpeaking
|
||||
}
|
||||
|
||||
override fun shutdown() {
|
||||
job.cancel()
|
||||
stopPlaybackOnly()
|
||||
try {
|
||||
runBlocking(ttsDispatcher) {
|
||||
offlineTts?.release()
|
||||
offlineTts = null
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("OfflineTtsManager", "Error releasing offline TTS", e)
|
||||
}
|
||||
isInitialized = false
|
||||
ttsDispatcher.close()
|
||||
}
|
||||
}
|
||||
99
app/src/main/java/com/app/assistant/tts/TtsEngineSelector.kt
Normal file
99
app/src/main/java/com/app/assistant/tts/TtsEngineSelector.kt
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
package com.app.assistant.tts
|
||||
|
||||
import android.content.Context
|
||||
import com.app.assistant.repository.SettingsRepository
|
||||
import java.io.File
|
||||
|
||||
class TtsEngineSelector(
|
||||
private val context: Context,
|
||||
private val settingsRepository: SettingsRepository,
|
||||
private val okHttpClient: okhttp3.OkHttpClient = com.app.assistant.viewmodel.MainViewModelFactory.okHttpClient,
|
||||
private val onSpeakingStateChanged: (isSpeaking: Boolean) -> Unit
|
||||
) : TtsManager {
|
||||
|
||||
private var nativeTtsManager: NativeTtsManager? = null
|
||||
private var offlineTtsManager: OfflineTtsManager? = null
|
||||
private var apiTtsManager: ApiTtsManager? = null
|
||||
|
||||
private fun getNativeTtsManager(): NativeTtsManager {
|
||||
return nativeTtsManager ?: NativeTtsManager(context, onSpeakingStateChanged).also { nativeTtsManager = it }
|
||||
}
|
||||
|
||||
private fun getOfflineTtsManager(): OfflineTtsManager {
|
||||
return offlineTtsManager ?: OfflineTtsManager(context, settingsRepository, onSpeakingStateChanged).also { offlineTtsManager = it }
|
||||
}
|
||||
|
||||
private fun getApiTtsManager(): ApiTtsManager {
|
||||
return apiTtsManager ?: ApiTtsManager(context, settingsRepository, okHttpClient, onSpeakingStateChanged).also { apiTtsManager = it }
|
||||
}
|
||||
|
||||
override fun speak(text: String, queueMode: Int) {
|
||||
val mode = settingsRepository.getTtsMode()
|
||||
|
||||
if (queueMode == TtsManager.QUEUE_FLUSH) {
|
||||
nativeTtsManager?.stop()
|
||||
offlineTtsManager?.stop()
|
||||
apiTtsManager?.stop()
|
||||
}
|
||||
|
||||
when (mode) {
|
||||
TtsMode.OFFLINE -> {
|
||||
if (isOfflineModelInstalled()) {
|
||||
getOfflineTtsManager().speak(text, queueMode)
|
||||
} else {
|
||||
getNativeTtsManager().speak(text, queueMode)
|
||||
}
|
||||
}
|
||||
TtsMode.API -> {
|
||||
getApiTtsManager().speak(text, queueMode)
|
||||
}
|
||||
else -> {
|
||||
getNativeTtsManager().speak(text, queueMode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun stop() {
|
||||
nativeTtsManager?.stop()
|
||||
offlineTtsManager?.stop()
|
||||
apiTtsManager?.stop()
|
||||
}
|
||||
|
||||
override fun isSpeaking(): Boolean {
|
||||
val mode = settingsRepository.getTtsMode()
|
||||
return when (mode) {
|
||||
TtsMode.OFFLINE -> {
|
||||
if (isOfflineModelInstalled()) {
|
||||
offlineTtsManager?.isSpeaking() ?: false
|
||||
} else {
|
||||
nativeTtsManager?.isSpeaking() ?: false
|
||||
}
|
||||
}
|
||||
TtsMode.API -> {
|
||||
apiTtsManager?.isSpeaking() ?: false
|
||||
}
|
||||
else -> {
|
||||
nativeTtsManager?.isSpeaking() ?: false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun shutdown() {
|
||||
nativeTtsManager?.shutdown()
|
||||
offlineTtsManager?.shutdown()
|
||||
apiTtsManager?.shutdown()
|
||||
}
|
||||
|
||||
private fun isOfflineModelInstalled(): Boolean {
|
||||
val supertonicDir = File(context.filesDir, "sherpa-onnx-tts/sherpa-onnx-supertonic-3-tts-int8-2026-05-11")
|
||||
val durationPredictor = File(supertonicDir, "duration_predictor.int8.onnx")
|
||||
val textEncoder = File(supertonicDir, "text_encoder.int8.onnx")
|
||||
val vectorEstimator = File(supertonicDir, "vector_estimator.int8.onnx")
|
||||
val vocoder = File(supertonicDir, "vocoder.int8.onnx")
|
||||
val ttsJson = File(supertonicDir, "tts.json")
|
||||
val unicodeIndexer = File(supertonicDir, "unicode_indexer.bin")
|
||||
val voiceStyle = File(supertonicDir, "voice.bin")
|
||||
return durationPredictor.exists() && textEncoder.exists() && vectorEstimator.exists() &&
|
||||
vocoder.exists() && ttsJson.exists() && unicodeIndexer.exists() && voiceStyle.exists()
|
||||
}
|
||||
}
|
||||
19
app/src/main/java/com/app/assistant/tts/TtsManager.kt
Normal file
19
app/src/main/java/com/app/assistant/tts/TtsManager.kt
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
package com.app.assistant.tts
|
||||
|
||||
interface TtsManager {
|
||||
fun speak(text: String, queueMode: Int = QUEUE_FLUSH)
|
||||
fun stop()
|
||||
fun isSpeaking(): Boolean
|
||||
fun shutdown()
|
||||
|
||||
companion object {
|
||||
const val QUEUE_FLUSH = 0
|
||||
const val QUEUE_ADD = 1
|
||||
}
|
||||
}
|
||||
|
||||
enum class TtsMode {
|
||||
NATIVE,
|
||||
OFFLINE,
|
||||
API
|
||||
}
|
||||
178
app/src/main/java/com/app/assistant/tts/TtsModelManager.kt
Normal file
178
app/src/main/java/com/app/assistant/tts/TtsModelManager.kt
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
package com.app.assistant.tts
|
||||
|
||||
import android.content.Context
|
||||
import android.os.Build
|
||||
import android.util.Log
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import java.io.File
|
||||
import java.io.FileOutputStream
|
||||
import java.io.IOException
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
|
||||
class TtsModelManager(private val context: Context) {
|
||||
|
||||
private val baseDir = File(context.filesDir, "sherpa-onnx-tts")
|
||||
private val modelDir = File(baseDir, "sherpa-onnx-supertonic-3-tts-int8-2026-05-11")
|
||||
|
||||
private val downloadUrl = "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-supertonic-3-tts-int8-2026-05-11.tar.bz2"
|
||||
|
||||
private var currentCall: okhttp3.Call? = null
|
||||
private val isCancelled = AtomicBoolean(false)
|
||||
|
||||
fun isModelDownloaded(): Boolean {
|
||||
val durationPredictor = File(modelDir, "duration_predictor.int8.onnx")
|
||||
val textEncoder = File(modelDir, "text_encoder.int8.onnx")
|
||||
val vectorEstimator = File(modelDir, "vector_estimator.int8.onnx")
|
||||
val vocoder = File(modelDir, "vocoder.int8.onnx")
|
||||
val ttsJson = File(modelDir, "tts.json")
|
||||
val unicodeIndexer = File(modelDir, "unicode_indexer.bin")
|
||||
val voiceStyle = File(modelDir, "voice.bin")
|
||||
return durationPredictor.exists() && durationPredictor.length() > 0 &&
|
||||
textEncoder.exists() && textEncoder.length() > 0 &&
|
||||
vectorEstimator.exists() && vectorEstimator.length() > 0 &&
|
||||
vocoder.exists() && vocoder.length() > 0 &&
|
||||
ttsJson.exists() && ttsJson.length() > 0 &&
|
||||
unicodeIndexer.exists() && unicodeIndexer.length() > 0 &&
|
||||
voiceStyle.exists() && voiceStyle.length() > 0
|
||||
}
|
||||
|
||||
fun deleteLocalModel(): Boolean {
|
||||
return deleteDirectory(modelDir)
|
||||
}
|
||||
|
||||
private fun deleteDirectory(dir: File): Boolean {
|
||||
if (dir.isDirectory) {
|
||||
val children = dir.listFiles()
|
||||
if (children != null) {
|
||||
for (child in children) {
|
||||
deleteDirectory(child)
|
||||
}
|
||||
}
|
||||
}
|
||||
return dir.delete()
|
||||
}
|
||||
|
||||
suspend fun downloadModel(
|
||||
client: OkHttpClient,
|
||||
onProgress: (downloadedBytes: Long, totalBytes: Long) -> Unit,
|
||||
onComplete: (success: Boolean, errorMessage: String?) -> Unit
|
||||
) {
|
||||
isCancelled.set(false)
|
||||
withContext(Dispatchers.IO) {
|
||||
val tempArchive = File(context.cacheDir, "sherpa-onnx-supertonic-3-tts-int8-2026-05-11.tar.bz2")
|
||||
try {
|
||||
if (tempArchive.exists()) {
|
||||
tempArchive.delete()
|
||||
}
|
||||
|
||||
if (!baseDir.exists()) {
|
||||
baseDir.mkdirs()
|
||||
}
|
||||
|
||||
val request = Request.Builder()
|
||||
.url(downloadUrl)
|
||||
.build()
|
||||
|
||||
val call = client.newCall(request)
|
||||
currentCall = call
|
||||
|
||||
if (isCancelled.get()) {
|
||||
throw IOException("Download cancelled by user")
|
||||
}
|
||||
|
||||
val response = call.execute()
|
||||
if (!response.isSuccessful) {
|
||||
throw IOException("HTTP error code: ${response.code}")
|
||||
}
|
||||
|
||||
val responseBody = response.body ?: throw IOException("Empty response body")
|
||||
val totalBytes = responseBody.contentLength()
|
||||
|
||||
responseBody.byteStream().use { inputStream ->
|
||||
FileOutputStream(tempArchive).use { outputStream ->
|
||||
val buffer = ByteArray(1024 * 64)
|
||||
var bytesRead: Int
|
||||
var downloadedBytes = 0L
|
||||
|
||||
while (inputStream.read(buffer).also { bytesRead = it } != -1) {
|
||||
if (isCancelled.get()) {
|
||||
throw IOException("Download cancelled by user")
|
||||
}
|
||||
outputStream.write(buffer, 0, bytesRead)
|
||||
downloadedBytes += bytesRead
|
||||
onProgress(downloadedBytes, totalBytes)
|
||||
}
|
||||
outputStream.flush()
|
||||
}
|
||||
}
|
||||
response.close()
|
||||
|
||||
if (isCancelled.get()) {
|
||||
throw IOException("Download cancelled by user")
|
||||
}
|
||||
|
||||
// Extract archive
|
||||
onProgress(totalBytes, totalBytes) // Mark download as 100% before extraction
|
||||
val extractionSuccess = extractTarBz2(tempArchive, baseDir)
|
||||
|
||||
if (extractionSuccess && isModelDownloaded()) {
|
||||
onComplete(true, null)
|
||||
} else {
|
||||
onComplete(false, "Extraction failed. Ensure your device has enough free storage.")
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("TtsModelManager", "Error downloading/extracting TTS model", e)
|
||||
val msg = if (isCancelled.get()) "Download cancelled." else e.message ?: "Unknown error"
|
||||
onComplete(false, msg)
|
||||
} finally {
|
||||
currentCall = null
|
||||
if (tempArchive.exists()) {
|
||||
tempArchive.delete()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun cancelDownload() {
|
||||
isCancelled.set(true)
|
||||
currentCall?.cancel()
|
||||
currentCall = null
|
||||
}
|
||||
|
||||
private fun extractTarBz2(archiveFile: File, targetDir: File): Boolean {
|
||||
targetDir.mkdirs()
|
||||
try {
|
||||
Log.i("TtsModelManager", "Starting native tar extraction of ${archiveFile.name} to ${targetDir.name}...")
|
||||
val process = ProcessBuilder()
|
||||
.command("tar", "-xjf", archiveFile.absolutePath, "-C", targetDir.absolutePath)
|
||||
.redirectErrorStream(true)
|
||||
.start()
|
||||
|
||||
val exitCode = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
|
||||
if (process.waitFor(90, java.util.concurrent.TimeUnit.SECONDS)) { // 90 secs timeout for slower devices
|
||||
process.exitValue()
|
||||
} else {
|
||||
process.destroyForcibly()
|
||||
-1
|
||||
}
|
||||
} else {
|
||||
process.waitFor()
|
||||
}
|
||||
|
||||
if (exitCode == 0) {
|
||||
Log.i("TtsModelManager", "Native tar extraction completed successfully.")
|
||||
return true
|
||||
}
|
||||
|
||||
val reader = process.inputStream.bufferedReader()
|
||||
val output = reader.readText()
|
||||
Log.e("TtsModelManager", "Native tar failed with exit code $exitCode. Output: $output")
|
||||
} catch (e: Exception) {
|
||||
Log.e("TtsModelManager", "Failed to run native tar command", e)
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,225 @@
|
|||
package com.app.assistant.ui.screen
|
||||
|
||||
import android.util.Log
|
||||
import android.util.Size
|
||||
import androidx.camera.core.CameraSelector
|
||||
import androidx.camera.core.FocusMeteringAction
|
||||
import androidx.camera.core.ImageAnalysis
|
||||
import androidx.camera.core.Preview
|
||||
import androidx.camera.core.resolutionselector.ResolutionSelector
|
||||
import androidx.camera.core.resolutionselector.ResolutionStrategy
|
||||
import androidx.camera.lifecycle.ProcessCameraProvider
|
||||
import androidx.camera.view.PreviewView
|
||||
import androidx.compose.animation.core.Animatable
|
||||
import androidx.compose.animation.core.tween
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.foundation.border
|
||||
import androidx.compose.foundation.gestures.detectTapGestures
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.DisposableEffect
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.rememberCoroutineScope
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.geometry.Offset
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.graphics.graphicsLayer
|
||||
import androidx.compose.ui.input.pointer.pointerInput
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.platform.LocalLifecycleOwner
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.compose.ui.viewinterop.AndroidView
|
||||
import androidx.core.content.ContextCompat
|
||||
import com.app.assistant.camera.VisualBufferManager
|
||||
import com.app.assistant.camera.imageProxyToRotatedJpeg
|
||||
import kotlinx.coroutines.launch
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
|
||||
@Composable
|
||||
fun CameraPreviewContainer(
|
||||
modifier: Modifier = Modifier,
|
||||
lensFacing: Int = CameraSelector.LENS_FACING_BACK
|
||||
) {
|
||||
val context = LocalContext.current
|
||||
val lifecycleOwner = LocalLifecycleOwner.current
|
||||
val coroutineScope = rememberCoroutineScope()
|
||||
|
||||
var activeCamera by remember { mutableStateOf<androidx.camera.core.Camera?>(null) }
|
||||
var tapPoint by remember { mutableStateOf<Offset?>(null) }
|
||||
|
||||
val focusRingAlpha = remember { Animatable(0f) }
|
||||
val focusRingScale = remember { Animatable(1f) }
|
||||
|
||||
val previewView = remember {
|
||||
PreviewView(context).apply {
|
||||
scaleType = PreviewView.ScaleType.FILL_CENTER
|
||||
}
|
||||
}
|
||||
|
||||
DisposableEffect(lifecycleOwner, lensFacing) {
|
||||
var cameraExecutor: java.util.concurrent.ExecutorService? = null
|
||||
val cameraProviderFuture = ProcessCameraProvider.getInstance(context)
|
||||
|
||||
val observer = androidx.lifecycle.LifecycleEventObserver { _, event ->
|
||||
when (event) {
|
||||
androidx.lifecycle.Lifecycle.Event.ON_RESUME -> {
|
||||
val executor = Executors.newSingleThreadExecutor()
|
||||
cameraExecutor = executor
|
||||
|
||||
cameraProviderFuture.addListener({
|
||||
try {
|
||||
val cameraProvider = cameraProviderFuture.get()
|
||||
|
||||
val preview = Preview.Builder().build().apply {
|
||||
setSurfaceProvider(previewView.surfaceProvider)
|
||||
}
|
||||
|
||||
val resolutionSelector = ResolutionSelector.Builder()
|
||||
.setResolutionStrategy(
|
||||
ResolutionStrategy(
|
||||
Size(640, 480),
|
||||
ResolutionStrategy.FALLBACK_RULE_CLOSEST_HIGHER_THEN_LOWER
|
||||
)
|
||||
)
|
||||
.build()
|
||||
|
||||
val imageAnalysis = ImageAnalysis.Builder()
|
||||
.setResolutionSelector(resolutionSelector)
|
||||
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
|
||||
.build()
|
||||
|
||||
var lastFrameTimestamp = 0L
|
||||
|
||||
imageAnalysis.setAnalyzer(executor) { imageProxy ->
|
||||
val now = System.currentTimeMillis()
|
||||
if (now - lastFrameTimestamp < 150) {
|
||||
imageProxy.close()
|
||||
return@setAnalyzer
|
||||
}
|
||||
lastFrameTimestamp = now
|
||||
|
||||
val jpegBytes = imageProxyToRotatedJpeg(imageProxy)
|
||||
imageProxy.close()
|
||||
|
||||
if (jpegBytes != null) {
|
||||
val cachedFrame = com.app.assistant.camera.CachedFrame(jpegBytes, now)
|
||||
VisualBufferManager.addFrame(cachedFrame)
|
||||
}
|
||||
}
|
||||
|
||||
val cameraSelector = if (lensFacing == CameraSelector.LENS_FACING_BACK) {
|
||||
CameraSelector.DEFAULT_BACK_CAMERA
|
||||
} else {
|
||||
CameraSelector.DEFAULT_FRONT_CAMERA
|
||||
}
|
||||
|
||||
cameraProvider.unbindAll()
|
||||
val camera = cameraProvider.bindToLifecycle(
|
||||
lifecycleOwner,
|
||||
cameraSelector,
|
||||
preview,
|
||||
imageAnalysis
|
||||
)
|
||||
activeCamera = camera
|
||||
} catch (e: Exception) {
|
||||
Log.e("CameraPreview", "Use case binding failed", e)
|
||||
}
|
||||
}, ContextCompat.getMainExecutor(context))
|
||||
}
|
||||
androidx.lifecycle.Lifecycle.Event.ON_PAUSE -> {
|
||||
try {
|
||||
if (cameraProviderFuture.isDone) {
|
||||
val cameraProvider = cameraProviderFuture.get()
|
||||
cameraProvider.unbindAll()
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("CameraPreview", "Error unbinding on ON_PAUSE", e)
|
||||
}
|
||||
cameraExecutor?.shutdown()
|
||||
cameraExecutor = null
|
||||
activeCamera = null
|
||||
VisualBufferManager.clear()
|
||||
}
|
||||
else -> {}
|
||||
}
|
||||
}
|
||||
|
||||
lifecycleOwner.lifecycle.addObserver(observer)
|
||||
|
||||
onDispose {
|
||||
lifecycleOwner.lifecycle.removeObserver(observer)
|
||||
try {
|
||||
if (cameraProviderFuture.isDone) {
|
||||
val cameraProvider = cameraProviderFuture.get()
|
||||
cameraProvider.unbindAll()
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("CameraPreview", "Error unbinding on dispose", e)
|
||||
}
|
||||
cameraExecutor?.shutdown()
|
||||
activeCamera = null
|
||||
VisualBufferManager.clear()
|
||||
}
|
||||
}
|
||||
|
||||
Box(modifier = modifier) {
|
||||
AndroidView(
|
||||
factory = { previewView },
|
||||
modifier = Modifier
|
||||
.fillMaxSize()
|
||||
.pointerInput(activeCamera) {
|
||||
detectTapGestures { offset ->
|
||||
val camera = activeCamera ?: return@detectTapGestures
|
||||
tapPoint = offset
|
||||
|
||||
coroutineScope.launch {
|
||||
focusRingAlpha.snapTo(1f)
|
||||
focusRingScale.snapTo(1.5f)
|
||||
|
||||
focusRingScale.animateTo(1f, tween(300))
|
||||
focusRingAlpha.animateTo(0f, tween(400, delayMillis = 100))
|
||||
}
|
||||
|
||||
val factory = previewView.meteringPointFactory
|
||||
val point = factory.createPoint(offset.x, offset.y)
|
||||
val action = FocusMeteringAction.Builder(point, FocusMeteringAction.FLAG_AF or FocusMeteringAction.FLAG_AE)
|
||||
.setAutoCancelDuration(5, TimeUnit.SECONDS)
|
||||
.build()
|
||||
camera.cameraControl.startFocusAndMetering(action)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
tapPoint?.let { point ->
|
||||
val scale = focusRingScale.value
|
||||
val alpha = focusRingAlpha.value
|
||||
if (alpha > 0f) {
|
||||
Box(
|
||||
modifier = Modifier
|
||||
.size(64.dp)
|
||||
.graphicsLayer {
|
||||
translationX = point.x - size.width / 2f
|
||||
translationY = point.y - size.height / 2f
|
||||
scaleX = scale
|
||||
scaleY = scale
|
||||
this.alpha = alpha
|
||||
}
|
||||
.border(
|
||||
width = 2.dp,
|
||||
color = MaterialTheme.colorScheme.primary,
|
||||
shape = androidx.compose.foundation.shape.CircleShape
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
446
app/src/main/java/com/app/assistant/ui/screen/ChatLayout.kt
Normal file
446
app/src/main/java/com/app/assistant/ui/screen/ChatLayout.kt
Normal file
|
|
@ -0,0 +1,446 @@
|
|||
package com.app.assistant.ui.screen
|
||||
|
||||
import android.os.Build
|
||||
import android.os.VibrationEffect
|
||||
import android.os.Vibrator
|
||||
import android.os.VibratorManager
|
||||
import android.widget.Toast
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.height
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.lazy.LazyColumn
|
||||
import androidx.compose.foundation.lazy.itemsIndexed
|
||||
import androidx.compose.foundation.lazy.rememberLazyListState
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.LaunchedEffect
|
||||
import kotlinx.coroutines.delay
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.platform.LocalFocusManager
|
||||
import androidx.compose.ui.platform.LocalSoftwareKeyboardController
|
||||
import androidx.compose.ui.unit.dp
|
||||
import com.app.assistant.model.Conversation
|
||||
import com.app.assistant.R
|
||||
import androidx.camera.core.CameraSelector
|
||||
import androidx.compose.ui.tooling.preview.Preview
|
||||
import com.app.assistant.ui.theme.AssistantTheme
|
||||
import androidx.compose.ui.focus.FocusManager
|
||||
import androidx.compose.ui.platform.SoftwareKeyboardController
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.foundation.shape.CircleShape
|
||||
import androidx.compose.foundation.shape.RoundedCornerShape
|
||||
import androidx.compose.material.icons.Icons
|
||||
import androidx.compose.material.icons.filled.Warning
|
||||
import androidx.compose.material.icons.filled.Close
|
||||
import androidx.compose.ui.res.painterResource
|
||||
import androidx.compose.material3.Icon
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.IconButton
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.draw.clip
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.animation.AnimatedContent
|
||||
import androidx.compose.animation.slideInVertically
|
||||
import androidx.compose.animation.slideOutVertically
|
||||
import androidx.compose.animation.fadeIn
|
||||
import androidx.compose.animation.fadeOut
|
||||
import androidx.compose.animation.togetherWith
|
||||
import androidx.compose.animation.core.tween
|
||||
|
||||
@Composable
|
||||
@Suppress("ktlint:standard:function-naming")
|
||||
fun ChatLayout(
|
||||
modifier: Modifier = Modifier,
|
||||
onShowCopyIconChange: (Boolean) -> Unit,
|
||||
selectedItemIndex: Int?,
|
||||
onSelectedItemChange: (Int) -> Unit,
|
||||
isCustomUI: Boolean,
|
||||
chatList: List<Conversation>,
|
||||
isSpeaking: Boolean,
|
||||
isListening: Boolean,
|
||||
audioAmplitude: Float = 0f,
|
||||
question: String,
|
||||
onQuestionChange: (String) -> Unit,
|
||||
onStopSpeaking: () -> Unit,
|
||||
onStartListening: () -> Unit,
|
||||
onProcessQuestion: (FocusManager, SoftwareKeyboardController, Boolean) -> Unit,
|
||||
isTranslateEnabled: Boolean,
|
||||
selectedAttachments: List<com.app.assistant.model.Attachment> = emptyList(),
|
||||
onRemoveAttachment: (com.app.assistant.model.Attachment) -> Unit = {},
|
||||
onAttachClick: (String) -> Unit = {},
|
||||
isImageSupported: Boolean = false,
|
||||
isAudioSupported: Boolean = false,
|
||||
isVideoSupported: Boolean = false,
|
||||
isDocumentSupported: Boolean = false,
|
||||
isHandsFree: Boolean = false,
|
||||
onToggleHandsFree: () -> Unit = {},
|
||||
onExitHandsFree: () -> Unit = {},
|
||||
isVoiceProcessing: Boolean = false,
|
||||
isMicReady: Boolean = false,
|
||||
isMicMuted: Boolean = false,
|
||||
onToggleMicMute: () -> Unit = {},
|
||||
isVisionModeActive: Boolean = false,
|
||||
onToggleVisionMode: () -> Unit = {},
|
||||
isScreenModeActive: Boolean = false,
|
||||
onToggleScreenMode: () -> Unit = {},
|
||||
) {
|
||||
val keyboardController = LocalSoftwareKeyboardController.current
|
||||
val focusManager = LocalFocusManager.current
|
||||
val scrollState = rememberLazyListState()
|
||||
|
||||
var isCameraExpanded by remember(isVisionModeActive) { mutableStateOf(true) }
|
||||
var lensFacing by remember { mutableStateOf(CameraSelector.LENS_FACING_BACK) }
|
||||
|
||||
val context = LocalContext.current
|
||||
val vibrator =
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
|
||||
context.getSystemService(VibratorManager::class.java)?.defaultVibrator
|
||||
} else {
|
||||
@Suppress("DEPRECATION")
|
||||
context.getSystemService(Vibrator::class.java)
|
||||
}
|
||||
|
||||
LaunchedEffect(selectedItemIndex) {
|
||||
onShowCopyIconChange(selectedItemIndex != null)
|
||||
if (selectedItemIndex != null) {
|
||||
if (vibrator?.hasVibrator() == true) {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
|
||||
vibrator.vibrate(VibrationEffect.createOneShot(50, VibrationEffect.DEFAULT_AMPLITUDE))
|
||||
} else {
|
||||
@Suppress("DEPRECATION")
|
||||
vibrator.vibrate(50)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Box(modifier = modifier.fillMaxSize()) {
|
||||
if (isVisionModeActive && isCameraExpanded) {
|
||||
CameraPreviewContainer(
|
||||
modifier = Modifier.fillMaxSize(),
|
||||
lensFacing = lensFacing
|
||||
)
|
||||
}
|
||||
|
||||
Column(modifier = Modifier.fillMaxSize()) {
|
||||
Box(
|
||||
modifier = Modifier
|
||||
.weight(1f)
|
||||
.fillMaxWidth()
|
||||
) {
|
||||
if (!(isVisionModeActive && isCameraExpanded)) {
|
||||
Column(modifier = Modifier.fillMaxSize()) {
|
||||
LazyColumn(
|
||||
modifier =
|
||||
Modifier
|
||||
.weight(1f)
|
||||
.padding(0.dp, 0.dp, 0.dp, 0.dp),
|
||||
verticalArrangement =
|
||||
if (isCustomUI) {
|
||||
Arrangement.spacedBy(8.dp, Alignment.Bottom)
|
||||
} else {
|
||||
Arrangement.spacedBy(8.dp)
|
||||
},
|
||||
state = scrollState,
|
||||
) {
|
||||
itemsIndexed(chatList) { index, conversation ->
|
||||
ConversationItem(
|
||||
conversation = conversation,
|
||||
index = index,
|
||||
isSelected = index == selectedItemIndex,
|
||||
onLongClick = { newIndex -> onSelectedItemChange(newIndex) },
|
||||
isTranslateEnabled = isTranslateEnabled,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
var isDismissed by remember(chatList.size, isImageSupported, isAudioSupported, isVideoSupported, isDocumentSupported) {
|
||||
mutableStateOf(false)
|
||||
}
|
||||
|
||||
val showWarning = remember(chatList.size, isImageSupported, isAudioSupported, isVideoSupported, isDocumentSupported) {
|
||||
chatList.any { conversation ->
|
||||
conversation.attachments.any { att ->
|
||||
val isTextDoc = att.mimeType.startsWith("text/") || att.mimeType == "application/json"
|
||||
if (isTextDoc) {
|
||||
false
|
||||
} else {
|
||||
when {
|
||||
att.mimeType.startsWith("image/") -> !isImageSupported
|
||||
att.mimeType.startsWith("audio/") -> !isAudioSupported
|
||||
att.mimeType.startsWith("video/") -> !isVideoSupported
|
||||
att.mimeType.startsWith("application/pdf") -> !isDocumentSupported
|
||||
else -> true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (showWarning && !isDismissed) {
|
||||
Row(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.padding(horizontal = 16.dp, vertical = 4.dp)
|
||||
.background(
|
||||
color = MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.6f),
|
||||
shape = RoundedCornerShape(12.dp)
|
||||
)
|
||||
.padding(12.dp),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.spacedBy(8.dp)
|
||||
) {
|
||||
Icon(
|
||||
imageVector = Icons.Default.Warning,
|
||||
contentDescription = "Warning",
|
||||
tint = MaterialTheme.colorScheme.primary,
|
||||
modifier = Modifier.size(20.dp)
|
||||
)
|
||||
Text(
|
||||
text = "Current model does not support some media in this chat. Incompatible past attachments will be ignored by the LLM.",
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
modifier = Modifier.weight(1f)
|
||||
)
|
||||
IconButton(
|
||||
onClick = { isDismissed = true },
|
||||
modifier = Modifier.size(20.dp)
|
||||
) {
|
||||
Icon(
|
||||
imageVector = Icons.Default.Close,
|
||||
contentDescription = "Dismiss warning",
|
||||
tint = MaterialTheme.colorScheme.onSurfaceVariant.copy(alpha = 0.7f),
|
||||
modifier = Modifier.size(16.dp)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (isVisionModeActive) {
|
||||
Box(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.height(200.dp)
|
||||
.padding(horizontal = 16.dp, vertical = 8.dp)
|
||||
.clip(RoundedCornerShape(16.dp))
|
||||
.background(Color.Black)
|
||||
) {
|
||||
CameraPreviewContainer(
|
||||
modifier = Modifier.fillMaxSize(),
|
||||
lensFacing = lensFacing
|
||||
)
|
||||
Column(
|
||||
modifier = Modifier
|
||||
.align(Alignment.BottomEnd)
|
||||
.padding(8.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(8.dp),
|
||||
horizontalAlignment = Alignment.CenterHorizontally
|
||||
) {
|
||||
IconButton(
|
||||
onClick = {
|
||||
lensFacing = if (lensFacing == CameraSelector.LENS_FACING_BACK) {
|
||||
CameraSelector.LENS_FACING_FRONT
|
||||
} else {
|
||||
CameraSelector.LENS_FACING_BACK
|
||||
}
|
||||
},
|
||||
modifier = Modifier
|
||||
.background(Color.Black.copy(alpha = 0.5f), CircleShape)
|
||||
) {
|
||||
Icon(
|
||||
painter = painterResource(id = R.drawable.ic_switch_camera),
|
||||
contentDescription = "Switch Camera",
|
||||
tint = Color.White
|
||||
)
|
||||
}
|
||||
IconButton(
|
||||
onClick = { isCameraExpanded = true },
|
||||
modifier = Modifier
|
||||
.background(Color.Black.copy(alpha = 0.5f), CircleShape)
|
||||
) {
|
||||
Icon(
|
||||
painter = painterResource(id = R.drawable.ic_fullscreen),
|
||||
contentDescription = "Expand Camera Preview",
|
||||
tint = Color.White
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Expanded mode overlay buttons stacked on bottom right
|
||||
Column(
|
||||
modifier = Modifier
|
||||
.align(Alignment.BottomEnd)
|
||||
.padding(16.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(8.dp),
|
||||
horizontalAlignment = Alignment.CenterHorizontally
|
||||
) {
|
||||
IconButton(
|
||||
onClick = {
|
||||
lensFacing = if (lensFacing == CameraSelector.LENS_FACING_BACK) {
|
||||
CameraSelector.LENS_FACING_FRONT
|
||||
} else {
|
||||
CameraSelector.LENS_FACING_BACK
|
||||
}
|
||||
},
|
||||
modifier = Modifier
|
||||
.background(Color.Black.copy(alpha = 0.5f), CircleShape)
|
||||
) {
|
||||
Icon(
|
||||
painter = painterResource(id = R.drawable.ic_switch_camera),
|
||||
contentDescription = "Switch Camera",
|
||||
tint = Color.White
|
||||
)
|
||||
}
|
||||
IconButton(
|
||||
onClick = { isCameraExpanded = false },
|
||||
modifier = Modifier
|
||||
.background(Color.Black.copy(alpha = 0.5f), CircleShape)
|
||||
) {
|
||||
Icon(
|
||||
painter = painterResource(id = R.drawable.ic_fullscreen_exit),
|
||||
contentDescription = "Collapse Camera Preview",
|
||||
tint = Color.White
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
AnimatedContent(
|
||||
targetState = isHandsFree,
|
||||
transitionSpec = {
|
||||
(slideInVertically(animationSpec = tween(300)) { height -> height } + fadeIn(animationSpec = tween(300)))
|
||||
.togetherWith(
|
||||
slideOutVertically(animationSpec = tween(300)) { height -> height } + fadeOut(animationSpec = tween(300))
|
||||
)
|
||||
},
|
||||
label = "InputFieldTransition",
|
||||
modifier = Modifier.fillMaxWidth()
|
||||
) { handsFreeActive ->
|
||||
if (handsFreeActive) {
|
||||
HandsFreeBar(
|
||||
isSpeaking = isSpeaking,
|
||||
isListening = isListening,
|
||||
audioAmplitude = audioAmplitude,
|
||||
isThinking = isVoiceProcessing || chatList.any { it.isLoading || it.isStreaming },
|
||||
isMicReady = isMicReady,
|
||||
onExitHandsFree = onExitHandsFree,
|
||||
isMuted = isMicMuted,
|
||||
onToggleMute = onToggleMicMute,
|
||||
isVisionModeActive = isVisionModeActive,
|
||||
onToggleVisionMode = onToggleVisionMode,
|
||||
isScreenModeActive = isScreenModeActive,
|
||||
onToggleScreenMode = onToggleScreenMode,
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.padding(8.dp)
|
||||
)
|
||||
} else {
|
||||
if (keyboardController != null) {
|
||||
UserInputField(
|
||||
focusManager = focusManager,
|
||||
keyboardController = keyboardController,
|
||||
isSpeaking = isSpeaking,
|
||||
isListening = isListening,
|
||||
question = question,
|
||||
onQuestionChange = onQuestionChange,
|
||||
onStopSpeaking = onStopSpeaking,
|
||||
onStartListening = onStartListening,
|
||||
onProcessQuestion = onProcessQuestion,
|
||||
selectedAttachments = selectedAttachments,
|
||||
onRemoveAttachment = onRemoveAttachment,
|
||||
onAttachClick = onAttachClick,
|
||||
isImageSupported = isImageSupported,
|
||||
isAudioSupported = isAudioSupported,
|
||||
isVideoSupported = isVideoSupported,
|
||||
isDocumentSupported = isDocumentSupported,
|
||||
isHandsFree = isHandsFree,
|
||||
onToggleHandsFree = onToggleHandsFree,
|
||||
hasMessages = chatList.isNotEmpty(),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val lastItem = chatList.lastOrNull()
|
||||
LaunchedEffect(chatList.size, lastItem?.text, lastItem?.isStreaming, lastItem?.isLoading) {
|
||||
if (chatList.isNotEmpty()) {
|
||||
val lastIndex = chatList.size - 1
|
||||
|
||||
var lastVisibleItem = scrollState.layoutInfo.visibleItemsInfo.firstOrNull { it.index == lastIndex }
|
||||
if (lastVisibleItem == null) {
|
||||
if (lastItem?.isStreaming == true) {
|
||||
scrollState.scrollToItem(lastIndex, 0)
|
||||
} else {
|
||||
scrollState.animateScrollToItem(lastIndex, 0)
|
||||
}
|
||||
delay(50)
|
||||
lastVisibleItem = scrollState.layoutInfo.visibleItemsInfo.firstOrNull { it.index == lastIndex }
|
||||
}
|
||||
|
||||
val itemHeight = lastVisibleItem?.size ?: 0
|
||||
val viewportHeight = scrollState.layoutInfo.viewportEndOffset - scrollState.layoutInfo.viewportStartOffset
|
||||
if (viewportHeight > 0 && itemHeight > 0) {
|
||||
val scrollOffset = viewportHeight - itemHeight
|
||||
if (lastItem?.isStreaming == true) {
|
||||
scrollState.scrollToItem(lastIndex, scrollOffset)
|
||||
} else {
|
||||
scrollState.animateScrollToItem(lastIndex, scrollOffset)
|
||||
}
|
||||
} else {
|
||||
if (lastItem?.isStreaming == true) {
|
||||
scrollState.scrollToItem(lastIndex, 0)
|
||||
} else {
|
||||
scrollState.animateScrollToItem(lastIndex, 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Preview(showBackground = true)
|
||||
@Composable
|
||||
fun ChatLayoutPreview() {
|
||||
AssistantTheme {
|
||||
ChatLayout(
|
||||
onShowCopyIconChange = {},
|
||||
selectedItemIndex = null,
|
||||
onSelectedItemChange = {},
|
||||
isCustomUI = false,
|
||||
chatList = listOf(
|
||||
Conversation(
|
||||
text = "What is the weather today?",
|
||||
isMe = true
|
||||
),
|
||||
Conversation(
|
||||
text = "The weather is sunny at 22°C.",
|
||||
isMe = false
|
||||
)
|
||||
),
|
||||
isSpeaking = false,
|
||||
isListening = false,
|
||||
question = "",
|
||||
onQuestionChange = {},
|
||||
onStopSpeaking = {},
|
||||
onStartListening = {},
|
||||
onProcessQuestion = { _, _, _ -> },
|
||||
isTranslateEnabled = false
|
||||
)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,387 @@
|
|||
package com.app.assistant.ui.screen
|
||||
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.foundation.clickable
|
||||
import androidx.compose.foundation.gestures.detectDragGestures
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.height
|
||||
import androidx.compose.foundation.layout.imePadding
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.width
|
||||
import androidx.compose.foundation.lazy.LazyColumn
|
||||
import androidx.compose.foundation.lazy.items
|
||||
import androidx.compose.foundation.rememberScrollState
|
||||
import androidx.compose.foundation.shape.RoundedCornerShape
|
||||
import androidx.compose.foundation.verticalScroll
|
||||
import androidx.compose.material3.AlertDialog
|
||||
import androidx.compose.material3.ExperimentalMaterial3Api
|
||||
import androidx.compose.material3.HorizontalDivider
|
||||
import androidx.compose.material3.Icon
|
||||
import androidx.compose.material3.IconButton
|
||||
import androidx.compose.foundation.ExperimentalFoundationApi
|
||||
import androidx.compose.foundation.combinedClickable
|
||||
import androidx.compose.material3.DropdownMenu
|
||||
import androidx.compose.material3.DropdownMenuItem
|
||||
import androidx.compose.material3.LinearProgressIndicator
|
||||
import androidx.compose.ui.draw.clip
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.ModalBottomSheet
|
||||
import androidx.compose.material3.ModalDrawerSheet
|
||||
import androidx.compose.material3.NavigationDrawerItem
|
||||
import androidx.compose.material3.SheetState
|
||||
import androidx.compose.material3.Switch
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.material3.TextButton
|
||||
import androidx.compose.material3.TextField
|
||||
import androidx.compose.material3.TextFieldDefaults
|
||||
import androidx.compose.material3.TopAppBar
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.input.pointer.pointerInput
|
||||
import androidx.compose.ui.res.painterResource
|
||||
import androidx.compose.ui.res.stringResource
|
||||
import androidx.compose.ui.unit.dp
|
||||
import com.app.assistant.R
|
||||
import com.app.assistant.model.Group
|
||||
import androidx.compose.ui.text.style.TextOverflow
|
||||
import androidx.compose.ui.tooling.preview.Preview
|
||||
import com.app.assistant.ui.theme.AssistantTheme
|
||||
|
||||
@OptIn(ExperimentalFoundationApi::class)
|
||||
@Composable
|
||||
fun ChatDrawerItem(
|
||||
group: Group,
|
||||
isDeletingChat: Boolean,
|
||||
onGroupClick: (Long) -> Unit,
|
||||
onDeleteGroupClick: (Long) -> Unit,
|
||||
) {
|
||||
var showMenu by remember { mutableStateOf(false) }
|
||||
var showConfirmDialog by remember { mutableStateOf(false) }
|
||||
|
||||
Box(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.padding(vertical = 4.dp)
|
||||
.clip(RoundedCornerShape(28.dp))
|
||||
.combinedClickable(
|
||||
enabled = !isDeletingChat,
|
||||
onClick = { onGroupClick(group.groupId) },
|
||||
onLongClick = { showMenu = true }
|
||||
)
|
||||
.padding(horizontal = 16.dp, vertical = 14.dp)
|
||||
) {
|
||||
Text(
|
||||
text = group.title,
|
||||
maxLines = 1,
|
||||
overflow = TextOverflow.Ellipsis,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant
|
||||
)
|
||||
|
||||
DropdownMenu(
|
||||
expanded = showMenu,
|
||||
onDismissRequest = { showMenu = false }
|
||||
) {
|
||||
DropdownMenuItem(
|
||||
text = { Text(stringResource(id = R.string.delete)) },
|
||||
onClick = {
|
||||
showMenu = false
|
||||
showConfirmDialog = true
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if (showConfirmDialog) {
|
||||
AlertDialog(
|
||||
onDismissRequest = { showConfirmDialog = false },
|
||||
text = { Text(stringResource(id = R.string.clear_chat_history_confirm)) },
|
||||
confirmButton = {
|
||||
TextButton(
|
||||
onClick = {
|
||||
showConfirmDialog = false
|
||||
onDeleteGroupClick(group.groupId)
|
||||
}
|
||||
) {
|
||||
Text(stringResource(id = R.string.yes))
|
||||
}
|
||||
},
|
||||
dismissButton = {
|
||||
TextButton(onClick = { showConfirmDialog = false }) {
|
||||
Text(stringResource(id = R.string.no))
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@OptIn(ExperimentalFoundationApi::class)
|
||||
@Composable
|
||||
fun ChatDrawerContent(
|
||||
groupList: List<Group>,
|
||||
isDeletingChat: Boolean,
|
||||
onGroupClick: (Long) -> Unit,
|
||||
onSettingsClick: () -> Unit,
|
||||
onNewChatClick: () -> Unit,
|
||||
onDeleteGroupClick: (Long) -> Unit,
|
||||
) {
|
||||
ModalDrawerSheet {
|
||||
Row(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.padding(16.dp),
|
||||
verticalAlignment = Alignment.CenterVertically
|
||||
) {
|
||||
Text(stringResource(id = R.string.chats))
|
||||
}
|
||||
|
||||
if (isDeletingChat) {
|
||||
LinearProgressIndicator(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.padding(horizontal = 16.dp, vertical = 8.dp)
|
||||
)
|
||||
}
|
||||
|
||||
Column(
|
||||
modifier =
|
||||
Modifier
|
||||
.padding(horizontal = 8.dp)
|
||||
.weight(1f)
|
||||
.verticalScroll(rememberScrollState()),
|
||||
) {
|
||||
groupList.forEach { group ->
|
||||
ChatDrawerItem(
|
||||
group = group,
|
||||
isDeletingChat = isDeletingChat,
|
||||
onGroupClick = onGroupClick,
|
||||
onDeleteGroupClick = onDeleteGroupClick
|
||||
)
|
||||
}
|
||||
}
|
||||
HorizontalDivider(modifier = Modifier.padding(vertical = 8.dp))
|
||||
NavigationDrawerItem(
|
||||
label = { Text(text = stringResource(id = R.string.settings_title)) },
|
||||
icon = {
|
||||
Icon(
|
||||
painter = painterResource(id = R.drawable.ic_settings),
|
||||
contentDescription = stringResource(id = R.string.settings_title),
|
||||
)
|
||||
},
|
||||
selected = false,
|
||||
onClick = {
|
||||
if (!isDeletingChat) {
|
||||
onSettingsClick()
|
||||
}
|
||||
},
|
||||
)
|
||||
NavigationDrawerItem(
|
||||
label = { Text(text = stringResource(id = R.string.start_new_chat)) },
|
||||
icon = {
|
||||
Icon(
|
||||
painter = painterResource(id = R.drawable.ic_add_new_chat),
|
||||
contentDescription = stringResource(id = R.string.start_new_chat),
|
||||
)
|
||||
},
|
||||
selected = false,
|
||||
onClick = {
|
||||
if (!isDeletingChat) {
|
||||
onNewChatClick()
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@OptIn(ExperimentalMaterial3Api::class)
|
||||
@Composable
|
||||
fun ChatTopAppBar(
|
||||
showCopyIcon: Boolean,
|
||||
onCopyClick: () -> Unit,
|
||||
onDeleteClick: () -> Unit,
|
||||
chatListNotEmpty: Boolean,
|
||||
onClearChatClick: () -> Unit,
|
||||
onMenuClick: () -> Unit,
|
||||
) {
|
||||
TopAppBar(
|
||||
navigationIcon = {
|
||||
IconButton(onClick = onMenuClick) {
|
||||
Icon(
|
||||
painter = painterResource(id = R.drawable.ic_menu),
|
||||
contentDescription = stringResource(id = R.string.menu),
|
||||
)
|
||||
}
|
||||
},
|
||||
title = { Text("") },
|
||||
actions = {
|
||||
if (showCopyIcon) {
|
||||
IconButton(onClick = onCopyClick) {
|
||||
Icon(
|
||||
painter = painterResource(id = R.drawable.ic_content_copy),
|
||||
contentDescription = stringResource(id = R.string.copy),
|
||||
)
|
||||
}
|
||||
IconButton(onClick = onDeleteClick) {
|
||||
Icon(
|
||||
painter = painterResource(id = R.drawable.ic_delete),
|
||||
contentDescription = stringResource(id = R.string.delete),
|
||||
)
|
||||
}
|
||||
}
|
||||
if (chatListNotEmpty) {
|
||||
IconButton(onClick = onClearChatClick) {
|
||||
Icon(
|
||||
painter = painterResource(id = R.drawable.ic_restart),
|
||||
contentDescription = stringResource(id = R.string.restart_chat),
|
||||
)
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun CustomUiDragHandle(
|
||||
onDragEnd: () -> Unit,
|
||||
) {
|
||||
Box(
|
||||
modifier =
|
||||
Modifier
|
||||
.fillMaxWidth()
|
||||
.height(24.dp)
|
||||
.then(
|
||||
Modifier.pointerInput(Unit) {
|
||||
detectDragGestures(
|
||||
onDragStart = { /* Optional: trigger on drag start */ },
|
||||
onDragEnd = onDragEnd,
|
||||
onDrag = { _, _ ->
|
||||
// track dragAmount if needed
|
||||
},
|
||||
)
|
||||
},
|
||||
)
|
||||
.clickable { onDragEnd() },
|
||||
contentAlignment = Alignment.Center,
|
||||
) {
|
||||
Box(
|
||||
modifier =
|
||||
Modifier
|
||||
.width(36.dp)
|
||||
.height(4.dp)
|
||||
.background(
|
||||
color = Color.Gray,
|
||||
shape = RoundedCornerShape(2.dp),
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// LanguageSelectionBottomSheet removed (moved to Settings)
|
||||
|
||||
@Composable
|
||||
fun ClearChatDialog(
|
||||
onConfirm: () -> Unit,
|
||||
onDismiss: () -> Unit,
|
||||
) {
|
||||
AlertDialog(
|
||||
onDismissRequest = onDismiss,
|
||||
text = { Text(stringResource(id = R.string.clear_chat_history_confirm)) },
|
||||
confirmButton = {
|
||||
TextButton(onClick = onConfirm) {
|
||||
Text(stringResource(id = R.string.yes))
|
||||
}
|
||||
},
|
||||
dismissButton = {
|
||||
TextButton(onClick = onDismiss) {
|
||||
Text(stringResource(id = R.string.no))
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun DeleteMessageDialog(
|
||||
onConfirm: () -> Unit,
|
||||
onDismiss: () -> Unit,
|
||||
) {
|
||||
AlertDialog(
|
||||
onDismissRequest = onDismiss,
|
||||
text = { Text(stringResource(id = R.string.delete_message_confirm)) },
|
||||
confirmButton = {
|
||||
TextButton(onClick = onConfirm) {
|
||||
Text(stringResource(id = R.string.yes))
|
||||
}
|
||||
},
|
||||
dismissButton = {
|
||||
TextButton(onClick = onDismiss) {
|
||||
Text(stringResource(id = R.string.no))
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
@Preview(showBackground = true)
|
||||
@Composable
|
||||
fun ChatDrawerContentPreview() {
|
||||
AssistantTheme {
|
||||
ChatDrawerContent(
|
||||
groupList = listOf(
|
||||
Group(1L, "Recent Conversation 1"),
|
||||
Group(2L, "Recent Conversation 2")
|
||||
),
|
||||
isDeletingChat = false,
|
||||
onGroupClick = {},
|
||||
onSettingsClick = {},
|
||||
onNewChatClick = {},
|
||||
onDeleteGroupClick = {}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Preview(showBackground = true)
|
||||
@Composable
|
||||
fun ChatTopAppBarPreview() {
|
||||
AssistantTheme {
|
||||
ChatTopAppBar(
|
||||
showCopyIcon = true,
|
||||
onCopyClick = {},
|
||||
onDeleteClick = {},
|
||||
chatListNotEmpty = true,
|
||||
onClearChatClick = {},
|
||||
onMenuClick = {}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Preview(showBackground = true)
|
||||
@Composable
|
||||
fun CustomUiDragHandlePreview() {
|
||||
AssistantTheme {
|
||||
CustomUiDragHandle(onDragEnd = {})
|
||||
}
|
||||
}
|
||||
|
||||
@Preview(showBackground = true)
|
||||
@Composable
|
||||
fun ClearChatDialogPreview() {
|
||||
AssistantTheme {
|
||||
ClearChatDialog(onConfirm = {}, onDismiss = {})
|
||||
}
|
||||
}
|
||||
|
||||
@Preview(showBackground = true)
|
||||
@Composable
|
||||
fun DeleteMessageDialogPreview() {
|
||||
AssistantTheme {
|
||||
DeleteMessageDialog(onConfirm = {}, onDismiss = {})
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,895 @@
|
|||
package com.app.assistant.ui.screen
|
||||
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.foundation.clickable
|
||||
import androidx.compose.foundation.gestures.detectTapGestures
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.Spacer
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.width
|
||||
import androidx.compose.foundation.layout.widthIn
|
||||
import androidx.compose.foundation.layout.wrapContentWidth
|
||||
import androidx.compose.foundation.shape.CircleShape
|
||||
import androidx.compose.foundation.shape.RoundedCornerShape
|
||||
import androidx.compose.material.icons.Icons
|
||||
import androidx.compose.material.icons.filled.Call
|
||||
import androidx.compose.material.icons.filled.LocationOn
|
||||
import androidx.compose.material3.Card
|
||||
import androidx.compose.material3.CardDefaults
|
||||
import androidx.compose.material3.Icon
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.input.pointer.pointerInput
|
||||
import androidx.compose.ui.platform.LocalConfiguration
|
||||
import androidx.compose.ui.platform.LocalUriHandler
|
||||
import androidx.compose.ui.platform.UriHandler
|
||||
import androidx.compose.ui.text.style.TextAlign
|
||||
import androidx.compose.ui.text.style.TextOverflow
|
||||
import androidx.compose.ui.unit.Dp
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.compose.ui.res.stringResource
|
||||
import coil3.compose.AsyncImage
|
||||
import com.app.assistant.R
|
||||
import com.app.assistant.model.Conversation
|
||||
import com.app.assistant.model.Attachment
|
||||
import com.app.assistant.util.Category
|
||||
import androidx.compose.ui.tooling.preview.Preview
|
||||
import androidx.compose.ui.unit.sp
|
||||
import com.app.assistant.ui.theme.AssistantTheme
|
||||
import androidx.compose.animation.AnimatedVisibility
|
||||
import androidx.compose.animation.core.animateFloatAsState
|
||||
import androidx.compose.animation.core.tween
|
||||
import androidx.compose.foundation.layout.height
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.material.icons.filled.KeyboardArrowDown
|
||||
import androidx.compose.material.icons.filled.KeyboardArrowUp
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.ui.text.font.FontWeight
|
||||
import androidx.compose.foundation.Image
|
||||
import androidx.compose.ui.graphics.asImageBitmap
|
||||
import androidx.compose.ui.layout.ContentScale
|
||||
import androidx.compose.runtime.rememberCoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.delay
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.material3.CircularProgressIndicator
|
||||
import androidx.compose.runtime.LaunchedEffect
|
||||
import androidx.compose.ui.draw.clip
|
||||
import androidx.compose.material3.IconButton
|
||||
import androidx.compose.ui.res.painterResource
|
||||
|
||||
@Composable
|
||||
@Suppress("ktlint:standard:function-naming")
|
||||
fun ConversationItem(
|
||||
conversation: Conversation,
|
||||
index: Int,
|
||||
isSelected: Boolean,
|
||||
onLongClick: (Int) -> Unit,
|
||||
isTranslateEnabled: Boolean,
|
||||
backgroundColor: Color =
|
||||
if (conversation.isMe) {
|
||||
MaterialTheme.colorScheme.secondary
|
||||
} else {
|
||||
MaterialTheme.colorScheme.surfaceContainerHighest
|
||||
},
|
||||
cornerRadius: Dp = 20.dp,
|
||||
) {
|
||||
val uriHandler = LocalUriHandler.current
|
||||
Column(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
) {
|
||||
Row(
|
||||
modifier =
|
||||
Modifier
|
||||
.fillMaxWidth()
|
||||
.padding(horizontal = 8.dp),
|
||||
horizontalArrangement = if (conversation.isMe) Arrangement.Start else Arrangement.End,
|
||||
) {
|
||||
if (conversation.isMe) {
|
||||
Spacer(modifier = Modifier.weight(1f))
|
||||
}
|
||||
Card(
|
||||
Modifier
|
||||
.wrapContentWidth()
|
||||
.widthIn(max = LocalConfiguration.current.screenWidthDp.dp * 0.85f),
|
||||
shape = RoundedCornerShape(cornerRadius),
|
||||
colors =
|
||||
CardDefaults.cardColors(
|
||||
containerColor =
|
||||
if (isSelected) {
|
||||
MaterialTheme.colorScheme.secondaryContainer
|
||||
} else {
|
||||
backgroundColor
|
||||
},
|
||||
),
|
||||
) {
|
||||
if (conversation.isLoading) {
|
||||
LoadingDots() // Show loading animation when isLoading is true
|
||||
} else {
|
||||
if (conversation.isMe) {
|
||||
OtherCard(conversation, onLongClick, index, cornerRadius)
|
||||
} else {
|
||||
when (conversation.category) {
|
||||
Category.CALL.name -> {
|
||||
MakeCall(uriHandler, conversation)
|
||||
}
|
||||
|
||||
Category.OTHER.name -> {
|
||||
OtherCard(conversation, onLongClick, index, cornerRadius)
|
||||
}
|
||||
|
||||
Category.SETTINGS.name -> {
|
||||
// TODO
|
||||
OtherCard(conversation, onLongClick, index, cornerRadius)
|
||||
}
|
||||
|
||||
Category.SONGS.name -> {
|
||||
PlaySong(uriHandler, conversation)
|
||||
}
|
||||
|
||||
Category.NAVIGATION.name -> {
|
||||
StartNavigation(uriHandler, conversation)
|
||||
}
|
||||
|
||||
Category.WEATHER.name -> {
|
||||
ShowWeather(uriHandler, conversation)
|
||||
}
|
||||
|
||||
Category.REMINDER.name -> {
|
||||
OtherCard(conversation, onLongClick, index, cornerRadius)
|
||||
}
|
||||
|
||||
Category.ALARM.name -> {
|
||||
OtherCard(conversation, onLongClick, index, cornerRadius)
|
||||
}
|
||||
|
||||
else -> {
|
||||
OtherCard(conversation, onLongClick, index, cornerRadius)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!conversation.isMe) {
|
||||
Spacer(modifier = Modifier.weight(1f))
|
||||
}
|
||||
}
|
||||
|
||||
if (conversation.isMe) {
|
||||
Text(
|
||||
text = conversation.category,
|
||||
modifier =
|
||||
Modifier
|
||||
.padding(0.dp, 0.dp, 20.dp, 0.dp)
|
||||
.fillMaxWidth(),
|
||||
maxLines = 1,
|
||||
overflow = TextOverflow.Ellipsis,
|
||||
color = MaterialTheme.colorScheme.secondary,
|
||||
textAlign = TextAlign.End,
|
||||
fontSize = 10.sp
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun StartNavigation(
|
||||
uriHandler: UriHandler,
|
||||
conversation: Conversation,
|
||||
) {
|
||||
Row(
|
||||
modifier =
|
||||
Modifier
|
||||
.padding(8.dp)
|
||||
.clickable {
|
||||
uriHandler.openUri(conversation.navigationURI.toString())
|
||||
},
|
||||
horizontalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
Box(
|
||||
modifier =
|
||||
Modifier
|
||||
.background(
|
||||
shape = CircleShape,
|
||||
color = MaterialTheme.colorScheme.secondaryContainer,
|
||||
),
|
||||
) {
|
||||
Icon(
|
||||
modifier = Modifier.padding(8.dp),
|
||||
imageVector = Icons.Default.LocationOn,
|
||||
contentDescription = stringResource(id = R.string.navigation_desc),
|
||||
)
|
||||
}
|
||||
|
||||
MarkdownText(
|
||||
modifier =
|
||||
Modifier
|
||||
.align(alignment = Alignment.CenterVertically),
|
||||
markdown = conversation.text,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun PlaySong(
|
||||
uriHandler: UriHandler,
|
||||
conversation: Conversation,
|
||||
) {
|
||||
AsyncImage(
|
||||
modifier =
|
||||
Modifier
|
||||
.clickable {
|
||||
uriHandler.openUri(conversation.navigationURI.toString())
|
||||
},
|
||||
model = conversation.contentURL,
|
||||
contentDescription = conversation.text,
|
||||
)
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun MakeCall(
|
||||
uriHandler: UriHandler,
|
||||
conversation: Conversation,
|
||||
) {
|
||||
Row(
|
||||
modifier =
|
||||
Modifier
|
||||
.padding(8.dp)
|
||||
.clickable {
|
||||
uriHandler.openUri(conversation.navigationURI.toString())
|
||||
},
|
||||
horizontalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
Box(
|
||||
modifier =
|
||||
Modifier
|
||||
.background(
|
||||
shape = CircleShape,
|
||||
color = MaterialTheme.colorScheme.secondaryContainer,
|
||||
),
|
||||
) {
|
||||
Icon(
|
||||
modifier = Modifier.padding(8.dp),
|
||||
imageVector = Icons.Default.Call,
|
||||
contentDescription = stringResource(id = R.string.call_desc),
|
||||
)
|
||||
}
|
||||
|
||||
MarkdownText(
|
||||
modifier =
|
||||
Modifier
|
||||
.align(alignment = Alignment.CenterVertically),
|
||||
markdown = conversation.text,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun ShowWeather(
|
||||
uriHandler: UriHandler,
|
||||
conversation: Conversation,
|
||||
) {
|
||||
MarkdownText(
|
||||
modifier =
|
||||
Modifier
|
||||
.padding(16.dp)
|
||||
.clickable { uriHandler.openUri(conversation.navigationURI.toString()) },
|
||||
markdown = conversation.text,
|
||||
)
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun OtherCard(
|
||||
conversation: Conversation,
|
||||
onLongClick: (Int) -> Unit,
|
||||
index: Int,
|
||||
cornerRadius: Dp = 20.dp,
|
||||
) {
|
||||
val thinkingText = conversation.getThinkingProcess()
|
||||
val answerText = conversation.getActualAnswer()
|
||||
val hasText = answerText.isNotEmpty() || conversation.isStreaming || thinkingText != null
|
||||
|
||||
Column(
|
||||
modifier = Modifier
|
||||
.pointerInput(Unit) {
|
||||
detectTapGestures(
|
||||
onLongPress = {
|
||||
onLongClick(index)
|
||||
},
|
||||
)
|
||||
}
|
||||
) {
|
||||
if (conversation.attachments.isNotEmpty()) {
|
||||
ConversationAttachments(
|
||||
attachments = conversation.attachments,
|
||||
hasTextBelow = hasText,
|
||||
cornerRadius = cornerRadius,
|
||||
modifier = Modifier
|
||||
)
|
||||
}
|
||||
|
||||
if (thinkingText != null || answerText.isNotEmpty() || conversation.isStreaming) {
|
||||
Column(
|
||||
modifier = Modifier.padding(
|
||||
start = 16.dp,
|
||||
end = 16.dp,
|
||||
top = if (conversation.attachments.isNotEmpty()) 8.dp else 8.dp,
|
||||
bottom = 8.dp
|
||||
)
|
||||
) {
|
||||
if (thinkingText != null) {
|
||||
var isExpanded by remember { mutableStateOf(true) }
|
||||
|
||||
Card(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.padding(bottom = 8.dp),
|
||||
shape = RoundedCornerShape(12.dp),
|
||||
colors = CardDefaults.cardColors(
|
||||
containerColor = MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.5f)
|
||||
)
|
||||
) {
|
||||
Column(modifier = Modifier.padding(12.dp)) {
|
||||
Row(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.clickable { isExpanded = !isExpanded },
|
||||
verticalAlignment = Alignment.CenterVertically
|
||||
) {
|
||||
Text(
|
||||
text = "🧠",
|
||||
fontSize = 16.sp,
|
||||
modifier = Modifier.padding(end = 8.dp)
|
||||
)
|
||||
Text(
|
||||
text = "Thinking Process",
|
||||
style = MaterialTheme.typography.labelLarge.copy(
|
||||
fontWeight = FontWeight.Bold,
|
||||
color = MaterialTheme.colorScheme.primary
|
||||
),
|
||||
modifier = Modifier.weight(1f)
|
||||
)
|
||||
Icon(
|
||||
imageVector = if (isExpanded) {
|
||||
Icons.Default.KeyboardArrowUp
|
||||
} else {
|
||||
Icons.Default.KeyboardArrowDown
|
||||
},
|
||||
contentDescription = if (isExpanded) "Collapse" else "Expand",
|
||||
tint = MaterialTheme.colorScheme.primary,
|
||||
modifier = Modifier.size(20.dp)
|
||||
)
|
||||
}
|
||||
|
||||
AnimatedVisibility(visible = isExpanded) {
|
||||
Column {
|
||||
Spacer(modifier = Modifier.height(8.dp))
|
||||
Text(
|
||||
text = thinkingText,
|
||||
style = MaterialTheme.typography.bodyMedium.copy(
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant.copy(alpha = 0.8f)
|
||||
),
|
||||
modifier = Modifier.fillMaxWidth()
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (answerText.isNotEmpty() || conversation.isStreaming) {
|
||||
val displayAnswer = if (conversation.isStreaming) {
|
||||
"$answerText ▮"
|
||||
} else {
|
||||
answerText
|
||||
}
|
||||
|
||||
MarkdownText(
|
||||
markdown = displayAnswer,
|
||||
modifier = Modifier
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun ConversationAttachments(
|
||||
attachments: List<Attachment>,
|
||||
hasTextBelow: Boolean,
|
||||
cornerRadius: Dp,
|
||||
modifier: Modifier = Modifier
|
||||
) {
|
||||
Column(
|
||||
modifier = modifier.fillMaxWidth(),
|
||||
verticalArrangement = Arrangement.spacedBy(8.dp)
|
||||
) {
|
||||
attachments.forEachIndexed { index, attachment ->
|
||||
val isFirst = index == 0
|
||||
val isLast = index == attachments.size - 1
|
||||
when {
|
||||
attachment.mimeType.startsWith("image/") -> {
|
||||
ImageAttachmentBubble(
|
||||
attachment = attachment,
|
||||
isFirst = isFirst,
|
||||
isLast = isLast,
|
||||
hasTextBelow = hasTextBelow,
|
||||
cornerRadius = cornerRadius
|
||||
)
|
||||
}
|
||||
attachment.mimeType.startsWith("audio/") -> {
|
||||
Box(modifier = Modifier.padding(horizontal = 16.dp, vertical = 4.dp)) {
|
||||
AudioAttachmentBubble(attachment)
|
||||
}
|
||||
}
|
||||
attachment.mimeType.startsWith("video/") -> {
|
||||
Box(modifier = Modifier.padding(horizontal = 16.dp, vertical = 4.dp)) {
|
||||
VideoAttachmentBubble(attachment)
|
||||
}
|
||||
}
|
||||
else -> {
|
||||
Box(modifier = Modifier.padding(horizontal = 16.dp, vertical = 4.dp)) {
|
||||
DocumentAttachmentBubble(attachment)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun ImageAttachmentBubble(
|
||||
attachment: Attachment,
|
||||
isFirst: Boolean,
|
||||
isLast: Boolean,
|
||||
hasTextBelow: Boolean,
|
||||
cornerRadius: Dp
|
||||
) {
|
||||
val context = LocalContext.current
|
||||
val coroutineScope = rememberCoroutineScope()
|
||||
var bitmap by remember { mutableStateOf<android.graphics.Bitmap?>(null) }
|
||||
var isLoading by remember { mutableStateOf(true) }
|
||||
var imageLoaded by remember { mutableStateOf(false) }
|
||||
|
||||
LaunchedEffect(attachment) {
|
||||
isLoading = true
|
||||
imageLoaded = false
|
||||
withContext(Dispatchers.IO) {
|
||||
try {
|
||||
val file = java.io.File(attachment.filePath)
|
||||
if (file.exists()) {
|
||||
val fileBytes = file.readBytes()
|
||||
val decrypted = com.app.assistant.db.EncryptionUtil.decryptFile(fileBytes, attachment.iv)
|
||||
val decoded = android.graphics.BitmapFactory.decodeByteArray(decrypted, 0, decrypted.size)
|
||||
bitmap = decoded
|
||||
if (decoded != null) {
|
||||
imageLoaded = true
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
isLoading = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val topRounding = if (isFirst) cornerRadius else 0.dp
|
||||
val bottomRounding = if (isLast && !hasTextBelow) cornerRadius else 0.dp
|
||||
val imageShape = RoundedCornerShape(
|
||||
topStart = topRounding,
|
||||
topEnd = topRounding,
|
||||
bottomStart = bottomRounding,
|
||||
bottomEnd = bottomRounding
|
||||
)
|
||||
|
||||
val imageAlpha by animateFloatAsState(
|
||||
targetValue = if (imageLoaded) 1f else 0f,
|
||||
animationSpec = tween(durationMillis = 400),
|
||||
label = "imageFadeIn"
|
||||
)
|
||||
|
||||
Box(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.clip(imageShape)
|
||||
.background(MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.3f))
|
||||
.clickable {
|
||||
coroutineScope.launch(Dispatchers.IO) {
|
||||
try {
|
||||
val fileBytes = java.io.File(attachment.filePath).readBytes()
|
||||
val decrypted = com.app.assistant.db.EncryptionUtil.decryptFile(fileBytes, attachment.iv)
|
||||
val ext = attachment.fileName.substringAfterLast('.', "jpg")
|
||||
val temp = java.io.File(context.cacheDir, "view_${attachment.id}.$ext")
|
||||
temp.writeBytes(decrypted)
|
||||
|
||||
val uri = androidx.core.content.FileProvider.getUriForFile(
|
||||
context,
|
||||
"com.app.assistant.fileprovider",
|
||||
temp
|
||||
)
|
||||
val intent = android.content.Intent(android.content.Intent.ACTION_VIEW).apply {
|
||||
setDataAndType(uri, "image/*")
|
||||
addFlags(android.content.Intent.FLAG_GRANT_READ_URI_PERMISSION)
|
||||
}
|
||||
context.startActivity(intent)
|
||||
} catch (e: Exception) {
|
||||
withContext(Dispatchers.Main) {
|
||||
android.widget.Toast.makeText(context, "Cannot open image: ${e.message}", android.widget.Toast.LENGTH_SHORT).show()
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
contentAlignment = Alignment.Center
|
||||
) {
|
||||
if (isLoading) {
|
||||
Box(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.height(240.dp),
|
||||
contentAlignment = Alignment.Center
|
||||
) {
|
||||
CircularProgressIndicator(
|
||||
modifier = Modifier.size(36.dp),
|
||||
color = MaterialTheme.colorScheme.primary,
|
||||
strokeWidth = 3.dp
|
||||
)
|
||||
}
|
||||
} else if (bitmap != null) {
|
||||
Image(
|
||||
bitmap = bitmap!!.asImageBitmap(),
|
||||
contentDescription = null,
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.height(240.dp),
|
||||
contentScale = ContentScale.Crop,
|
||||
alpha = imageAlpha
|
||||
)
|
||||
} else {
|
||||
Box(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.height(100.dp),
|
||||
contentAlignment = Alignment.Center
|
||||
) {
|
||||
Text(
|
||||
text = "Failed to load image",
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = MaterialTheme.colorScheme.error
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun AudioAttachmentBubble(attachment: Attachment) {
|
||||
val context = LocalContext.current
|
||||
var isPlaying by remember { mutableStateOf(false) }
|
||||
var progress by remember { mutableStateOf(0f) }
|
||||
var duration by remember { mutableStateOf(0) }
|
||||
var currentPosition by remember { mutableStateOf(0) }
|
||||
val mediaPlayer = remember { android.media.MediaPlayer() }
|
||||
var tempFile by remember { mutableStateOf<java.io.File?>(null) }
|
||||
val coroutineScope = rememberCoroutineScope()
|
||||
|
||||
androidx.compose.runtime.DisposableEffect(Unit) {
|
||||
onDispose {
|
||||
try {
|
||||
if (mediaPlayer.isPlaying) {
|
||||
mediaPlayer.stop()
|
||||
}
|
||||
mediaPlayer.release()
|
||||
} catch (e: Exception) {}
|
||||
tempFile?.delete()
|
||||
}
|
||||
}
|
||||
|
||||
LaunchedEffect(isPlaying) {
|
||||
if (isPlaying) {
|
||||
while (isPlaying) {
|
||||
try {
|
||||
currentPosition = mediaPlayer.currentPosition
|
||||
progress = if (duration > 0) currentPosition.toFloat() / duration else 0f
|
||||
} catch (e: Exception) {}
|
||||
kotlinx.coroutines.delay(200)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Card(
|
||||
colors = CardDefaults.cardColors(containerColor = MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.5f)),
|
||||
shape = RoundedCornerShape(12.dp),
|
||||
modifier = Modifier.fillMaxWidth().padding(vertical = 4.dp)
|
||||
) {
|
||||
Row(
|
||||
modifier = Modifier.padding(8.dp),
|
||||
verticalAlignment = Alignment.CenterVertically
|
||||
) {
|
||||
IconButton(onClick = {
|
||||
if (isPlaying) {
|
||||
mediaPlayer.pause()
|
||||
isPlaying = false
|
||||
} else {
|
||||
if (tempFile == null) {
|
||||
coroutineScope.launch(Dispatchers.IO) {
|
||||
try {
|
||||
val fileBytes = java.io.File(attachment.filePath).readBytes()
|
||||
val decrypted = com.app.assistant.db.EncryptionUtil.decryptFile(fileBytes, attachment.iv)
|
||||
val temp = java.io.File(context.cacheDir, "play_${attachment.id}.mp3")
|
||||
temp.writeBytes(decrypted)
|
||||
tempFile = temp
|
||||
|
||||
withContext(Dispatchers.Main) {
|
||||
mediaPlayer.reset()
|
||||
mediaPlayer.setDataSource(temp.absolutePath)
|
||||
mediaPlayer.prepare()
|
||||
duration = mediaPlayer.duration
|
||||
mediaPlayer.start()
|
||||
isPlaying = true
|
||||
mediaPlayer.setOnCompletionListener {
|
||||
isPlaying = false
|
||||
progress = 0f
|
||||
currentPosition = 0
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
mediaPlayer.start()
|
||||
isPlaying = true
|
||||
}
|
||||
}
|
||||
}) {
|
||||
Icon(
|
||||
painter = painterResource(id = if (isPlaying) R.drawable.ic_stop else R.drawable.ic_mic),
|
||||
contentDescription = if (isPlaying) "Pause" else "Play",
|
||||
tint = MaterialTheme.colorScheme.primary
|
||||
)
|
||||
}
|
||||
Spacer(modifier = Modifier.width(8.dp))
|
||||
Column(modifier = Modifier.weight(1f)) {
|
||||
Text(
|
||||
text = attachment.fileName,
|
||||
style = MaterialTheme.typography.labelMedium,
|
||||
maxLines = 1,
|
||||
overflow = TextOverflow.Ellipsis
|
||||
)
|
||||
Spacer(modifier = Modifier.height(4.dp))
|
||||
androidx.compose.material3.LinearProgressIndicator(
|
||||
progress = { progress },
|
||||
modifier = Modifier.fillMaxWidth().height(4.dp),
|
||||
color = MaterialTheme.colorScheme.primary,
|
||||
trackColor = MaterialTheme.colorScheme.onSurfaceVariant.copy(alpha = 0.2f)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun VideoAttachmentBubble(attachment: Attachment) {
|
||||
val context = LocalContext.current
|
||||
val coroutineScope = rememberCoroutineScope()
|
||||
Card(
|
||||
colors = CardDefaults.cardColors(containerColor = MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.5f)),
|
||||
shape = RoundedCornerShape(12.dp),
|
||||
modifier = Modifier.fillMaxWidth().padding(vertical = 4.dp).clickable {
|
||||
coroutineScope.launch(Dispatchers.IO) {
|
||||
try {
|
||||
val fileBytes = java.io.File(attachment.filePath).readBytes()
|
||||
val decrypted = com.app.assistant.db.EncryptionUtil.decryptFile(fileBytes, attachment.iv)
|
||||
val ext = attachment.fileName.substringAfterLast('.', "mp4")
|
||||
val temp = java.io.File(context.cacheDir, "play_${attachment.id}.$ext")
|
||||
temp.writeBytes(decrypted)
|
||||
|
||||
val uri = androidx.core.content.FileProvider.getUriForFile(
|
||||
context,
|
||||
"com.app.assistant.fileprovider",
|
||||
temp
|
||||
)
|
||||
val intent = android.content.Intent(android.content.Intent.ACTION_VIEW).apply {
|
||||
setDataAndType(uri, "video/*")
|
||||
addFlags(android.content.Intent.FLAG_GRANT_READ_URI_PERMISSION)
|
||||
}
|
||||
context.startActivity(intent)
|
||||
} catch (e: Exception) {
|
||||
withContext(Dispatchers.Main) {
|
||||
android.widget.Toast.makeText(context, "Cannot play video: ${e.message}", android.widget.Toast.LENGTH_SHORT).show()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
) {
|
||||
Row(
|
||||
modifier = Modifier.padding(12.dp),
|
||||
verticalAlignment = Alignment.CenterVertically
|
||||
) {
|
||||
Icon(
|
||||
painter = painterResource(id = R.drawable.ic_stop),
|
||||
contentDescription = "Play Video",
|
||||
tint = MaterialTheme.colorScheme.primary,
|
||||
modifier = Modifier.size(32.dp)
|
||||
)
|
||||
Spacer(modifier = Modifier.width(12.dp))
|
||||
Column {
|
||||
Text(
|
||||
text = attachment.fileName,
|
||||
style = MaterialTheme.typography.bodyMedium,
|
||||
fontWeight = FontWeight.Bold,
|
||||
maxLines = 1,
|
||||
overflow = TextOverflow.Ellipsis
|
||||
)
|
||||
Text(
|
||||
text = "Tap to play video",
|
||||
style = MaterialTheme.typography.labelSmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant.copy(alpha = 0.7f)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun DocumentAttachmentBubble(attachment: Attachment) {
|
||||
val context = LocalContext.current
|
||||
val coroutineScope = rememberCoroutineScope()
|
||||
Card(
|
||||
colors = CardDefaults.cardColors(containerColor = MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.5f)),
|
||||
shape = RoundedCornerShape(12.dp),
|
||||
modifier = Modifier.fillMaxWidth().padding(vertical = 4.dp).clickable {
|
||||
coroutineScope.launch(Dispatchers.IO) {
|
||||
try {
|
||||
val fileBytes = java.io.File(attachment.filePath).readBytes()
|
||||
val decrypted = com.app.assistant.db.EncryptionUtil.decryptFile(fileBytes, attachment.iv)
|
||||
val ext = attachment.fileName.substringAfterLast('.', "pdf")
|
||||
val temp = java.io.File(context.cacheDir, "doc_${attachment.id}.$ext")
|
||||
temp.writeBytes(decrypted)
|
||||
|
||||
val uri = androidx.core.content.FileProvider.getUriForFile(
|
||||
context,
|
||||
"com.app.assistant.fileprovider",
|
||||
temp
|
||||
)
|
||||
val intent = android.content.Intent(android.content.Intent.ACTION_VIEW).apply {
|
||||
setDataAndType(uri, attachment.mimeType)
|
||||
addFlags(android.content.Intent.FLAG_GRANT_READ_URI_PERMISSION)
|
||||
}
|
||||
context.startActivity(intent)
|
||||
} catch (e: Exception) {
|
||||
withContext(Dispatchers.Main) {
|
||||
android.widget.Toast.makeText(context, "Cannot open document: ${e.message}", android.widget.Toast.LENGTH_SHORT).show()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
) {
|
||||
Row(
|
||||
modifier = Modifier.padding(12.dp),
|
||||
verticalAlignment = Alignment.CenterVertically
|
||||
) {
|
||||
Icon(
|
||||
painter = painterResource(id = R.drawable.ic_translate),
|
||||
contentDescription = "Open Document",
|
||||
tint = MaterialTheme.colorScheme.primary,
|
||||
modifier = Modifier.size(32.dp)
|
||||
)
|
||||
Spacer(modifier = Modifier.width(12.dp))
|
||||
Column {
|
||||
Text(
|
||||
text = attachment.fileName,
|
||||
style = MaterialTheme.typography.bodyMedium,
|
||||
fontWeight = FontWeight.Bold,
|
||||
maxLines = 1,
|
||||
overflow = TextOverflow.Ellipsis
|
||||
)
|
||||
val subText = if (attachment.mimeType.startsWith("text/") || attachment.mimeType == "application/json") {
|
||||
"Text appended inline (Tap to open)"
|
||||
} else {
|
||||
"Tap to open document"
|
||||
}
|
||||
Text(
|
||||
text = subText,
|
||||
style = MaterialTheme.typography.labelSmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant.copy(alpha = 0.7f)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Preview(showBackground = true)
|
||||
@Composable
|
||||
fun UserConversationItemPreview() {
|
||||
AssistantTheme {
|
||||
Column(modifier = Modifier.padding(16.dp)) {
|
||||
ConversationItem(
|
||||
conversation = Conversation(
|
||||
text = "Hello! Can you help me?",
|
||||
isMe = true,
|
||||
category = Category.OTHER.name
|
||||
),
|
||||
index = 0,
|
||||
isSelected = false,
|
||||
onLongClick = {},
|
||||
isTranslateEnabled = false
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Preview(showBackground = true)
|
||||
@Composable
|
||||
fun AiConversationItemPreview() {
|
||||
AssistantTheme {
|
||||
Column(modifier = Modifier.padding(16.dp)) {
|
||||
ConversationItem(
|
||||
conversation = Conversation(
|
||||
text = "Sure! I can help you set an alarm or make a call.",
|
||||
isMe = false,
|
||||
category = Category.OTHER.name
|
||||
),
|
||||
index = 1,
|
||||
isSelected = false,
|
||||
onLongClick = {},
|
||||
isTranslateEnabled = true
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Preview(showBackground = true)
|
||||
@Composable
|
||||
fun LoadingConversationItemPreview() {
|
||||
AssistantTheme {
|
||||
Column(modifier = Modifier.padding(16.dp)) {
|
||||
ConversationItem(
|
||||
conversation = Conversation(
|
||||
text = "",
|
||||
isMe = false,
|
||||
isLoading = true
|
||||
),
|
||||
index = 2,
|
||||
isSelected = false,
|
||||
onLongClick = {},
|
||||
isTranslateEnabled = false
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Preview(showBackground = true)
|
||||
@Composable
|
||||
fun WeatherConversationItemPreview() {
|
||||
AssistantTheme {
|
||||
Column(modifier = Modifier.padding(16.dp)) {
|
||||
ConversationItem(
|
||||
conversation = Conversation(
|
||||
text = "Today's weather is sunny, 25°C.",
|
||||
isMe = false,
|
||||
category = Category.WEATHER.name
|
||||
),
|
||||
index = 3,
|
||||
isSelected = false,
|
||||
onLongClick = {},
|
||||
isTranslateEnabled = false
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
563
app/src/main/java/com/app/assistant/ui/screen/HandsFreeBar.kt
Normal file
563
app/src/main/java/com/app/assistant/ui/screen/HandsFreeBar.kt
Normal file
|
|
@ -0,0 +1,563 @@
|
|||
package com.app.assistant.ui.screen
|
||||
|
||||
import androidx.compose.animation.animateColorAsState
|
||||
import androidx.compose.animation.core.animateDpAsState
|
||||
import androidx.compose.animation.core.animateFloat
|
||||
import androidx.compose.animation.core.animateFloatAsState
|
||||
import androidx.compose.animation.core.infiniteRepeatable
|
||||
import androidx.compose.animation.core.rememberInfiniteTransition
|
||||
import androidx.compose.animation.core.FastOutSlowInEasing
|
||||
import androidx.compose.animation.core.LinearEasing
|
||||
import androidx.compose.animation.core.RepeatMode
|
||||
import androidx.compose.animation.core.tween
|
||||
import androidx.compose.animation.core.spring
|
||||
import androidx.compose.animation.core.Spring
|
||||
import androidx.compose.foundation.Canvas
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.height
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.foundation.layout.width
|
||||
import androidx.compose.foundation.shape.CircleShape
|
||||
import androidx.compose.foundation.shape.RoundedCornerShape
|
||||
import androidx.compose.material.icons.Icons
|
||||
import androidx.compose.material.icons.filled.Close
|
||||
import androidx.compose.material3.CircularProgressIndicator
|
||||
import androidx.compose.material3.Icon
|
||||
import androidx.compose.material3.IconButton
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.graphics.Brush
|
||||
import androidx.compose.ui.graphics.Path
|
||||
import androidx.compose.ui.graphics.StrokeCap
|
||||
import androidx.compose.ui.graphics.drawscope.Stroke
|
||||
import androidx.compose.ui.graphics.drawscope.rotate
|
||||
import androidx.compose.ui.geometry.Offset
|
||||
import androidx.compose.ui.geometry.CornerRadius
|
||||
import androidx.compose.ui.geometry.Size
|
||||
import androidx.compose.ui.graphics.graphicsLayer
|
||||
import androidx.compose.ui.res.painterResource
|
||||
import androidx.compose.ui.unit.dp
|
||||
import com.app.assistant.R
|
||||
import androidx.compose.ui.tooling.preview.Preview
|
||||
import com.app.assistant.ui.theme.AssistantTheme
|
||||
|
||||
enum class HandsFreeState {
|
||||
STARTING,
|
||||
LISTENING,
|
||||
THINKING,
|
||||
SPEAKING
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun HandsFreeBar(
|
||||
isSpeaking: Boolean,
|
||||
isListening: Boolean,
|
||||
isThinking: Boolean,
|
||||
isMicReady: Boolean,
|
||||
onExitHandsFree: () -> Unit,
|
||||
isMuted: Boolean = false,
|
||||
onToggleMute: () -> Unit = {},
|
||||
isVisionModeActive: Boolean = false,
|
||||
onToggleVisionMode: () -> Unit = {},
|
||||
isScreenModeActive: Boolean = false,
|
||||
onToggleScreenMode: () -> Unit = {},
|
||||
audioAmplitude: Float = 0f,
|
||||
modifier: Modifier = Modifier
|
||||
) {
|
||||
val currentState = remember(isSpeaking, isListening, isThinking, isMicReady) {
|
||||
when {
|
||||
isThinking -> HandsFreeState.THINKING
|
||||
isSpeaking -> HandsFreeState.SPEAKING
|
||||
isListening -> HandsFreeState.LISTENING
|
||||
isMicReady -> HandsFreeState.LISTENING
|
||||
else -> HandsFreeState.STARTING
|
||||
}
|
||||
}
|
||||
|
||||
// Dynamic state transition parameters
|
||||
val startingAlpha by animateFloatAsState(
|
||||
targetValue = if (currentState == HandsFreeState.STARTING) 1f else 0f,
|
||||
animationSpec = tween(durationMillis = 350),
|
||||
label = "startingAlpha"
|
||||
)
|
||||
|
||||
val listeningProgress by animateFloatAsState(
|
||||
targetValue = if (currentState == HandsFreeState.LISTENING) 1f else 0f,
|
||||
animationSpec = tween(durationMillis = 350),
|
||||
label = "listeningProgress"
|
||||
)
|
||||
|
||||
val animatedAmplitude by animateFloatAsState(
|
||||
targetValue = if (currentState == HandsFreeState.LISTENING) audioAmplitude else 0f,
|
||||
animationSpec = spring(
|
||||
dampingRatio = Spring.DampingRatioLowBouncy,
|
||||
stiffness = Spring.StiffnessMedium
|
||||
),
|
||||
label = "animatedAmplitude"
|
||||
)
|
||||
|
||||
val thinkingProgress by animateFloatAsState(
|
||||
targetValue = if (currentState == HandsFreeState.THINKING) 1f else 0f,
|
||||
animationSpec = tween(durationMillis = 350),
|
||||
label = "thinkingProgress"
|
||||
)
|
||||
|
||||
val speakingProgress by animateFloatAsState(
|
||||
targetValue = if (currentState == HandsFreeState.SPEAKING) 1f else 0f,
|
||||
animationSpec = tween(durationMillis = 350),
|
||||
label = "speakingProgress"
|
||||
)
|
||||
|
||||
val dotsSpreadProgress by animateFloatAsState(
|
||||
targetValue = if (currentState == HandsFreeState.THINKING || currentState == HandsFreeState.SPEAKING) 1f else 0f,
|
||||
animationSpec = tween(durationMillis = 350),
|
||||
label = "dotsSpreadProgress"
|
||||
)
|
||||
|
||||
// Keep the orb background color and content color consistent across all states
|
||||
val orbBackgroundColor = MaterialTheme.colorScheme.primaryContainer.copy(alpha = 0.5f)
|
||||
val orbContentColor = MaterialTheme.colorScheme.primary
|
||||
|
||||
val orbWidth by animateDpAsState(
|
||||
targetValue = when (currentState) {
|
||||
HandsFreeState.STARTING -> 48.dp
|
||||
HandsFreeState.LISTENING -> 72.dp
|
||||
HandsFreeState.THINKING -> 72.dp
|
||||
HandsFreeState.SPEAKING -> 88.dp
|
||||
},
|
||||
animationSpec = tween(durationMillis = 350),
|
||||
label = "orbWidth"
|
||||
)
|
||||
|
||||
val orbHeight = 48.dp
|
||||
|
||||
val infiniteTransition = rememberInfiniteTransition(label = "pulseTransition")
|
||||
|
||||
// Pulse animation for thinking outer glow ring
|
||||
val thinkingPulseScale by infiniteTransition.animateFloat(
|
||||
initialValue = 1.0f,
|
||||
targetValue = 1.5f,
|
||||
animationSpec = infiniteRepeatable(
|
||||
animation = tween(1200, easing = FastOutSlowInEasing),
|
||||
repeatMode = RepeatMode.Restart
|
||||
),
|
||||
label = "thinkingPulseScale"
|
||||
)
|
||||
val thinkingPulseAlpha by infiniteTransition.animateFloat(
|
||||
initialValue = 0.5f,
|
||||
targetValue = 0.0f,
|
||||
animationSpec = infiniteRepeatable(
|
||||
animation = tween(1200, easing = FastOutSlowInEasing),
|
||||
repeatMode = RepeatMode.Restart
|
||||
),
|
||||
label = "thinkingPulseAlpha"
|
||||
)
|
||||
|
||||
// Wave phase animation for Listening state sine wave
|
||||
val wavePhase by infiniteTransition.animateFloat(
|
||||
initialValue = 0f,
|
||||
targetValue = (2 * Math.PI).toFloat(),
|
||||
animationSpec = infiniteRepeatable(
|
||||
animation = tween(2200, easing = LinearEasing),
|
||||
repeatMode = RepeatMode.Restart
|
||||
),
|
||||
label = "wavePhase"
|
||||
)
|
||||
|
||||
// Breathing phase animation for organic pulsing
|
||||
val breathingPhase by infiniteTransition.animateFloat(
|
||||
initialValue = 0f,
|
||||
targetValue = (2 * Math.PI).toFloat(),
|
||||
animationSpec = infiniteRepeatable(
|
||||
animation = tween(4000, easing = LinearEasing),
|
||||
repeatMode = RepeatMode.Restart
|
||||
),
|
||||
label = "breathingPhase"
|
||||
)
|
||||
|
||||
// Starting state animations
|
||||
val startingRotation by infiniteTransition.animateFloat(
|
||||
initialValue = 0f,
|
||||
targetValue = 360f,
|
||||
animationSpec = infiniteRepeatable(
|
||||
animation = tween(1500, easing = LinearEasing),
|
||||
repeatMode = RepeatMode.Restart
|
||||
),
|
||||
label = "startingRotation"
|
||||
)
|
||||
|
||||
val coreScale by infiniteTransition.animateFloat(
|
||||
initialValue = 0.7f,
|
||||
targetValue = 1.0f,
|
||||
animationSpec = infiniteRepeatable(
|
||||
animation = tween(800, easing = FastOutSlowInEasing),
|
||||
repeatMode = RepeatMode.Reverse
|
||||
),
|
||||
label = "startingCore"
|
||||
)
|
||||
|
||||
// Speaking state voice equalizer animations
|
||||
val speakTransition = rememberInfiniteTransition(label = "speakingEqualizer")
|
||||
val s1 by speakTransition.animateFloat(
|
||||
initialValue = 0.15f,
|
||||
targetValue = 0.9f,
|
||||
animationSpec = infiniteRepeatable(
|
||||
animation = tween(350, easing = FastOutSlowInEasing),
|
||||
repeatMode = RepeatMode.Reverse
|
||||
),
|
||||
label = "speakBar1"
|
||||
)
|
||||
val s2 by speakTransition.animateFloat(
|
||||
initialValue = 0.25f,
|
||||
targetValue = 0.75f,
|
||||
animationSpec = infiniteRepeatable(
|
||||
animation = tween(280, easing = FastOutSlowInEasing),
|
||||
repeatMode = RepeatMode.Reverse
|
||||
),
|
||||
label = "speakBar2"
|
||||
)
|
||||
val s3 by speakTransition.animateFloat(
|
||||
initialValue = 0.1f,
|
||||
targetValue = 1.0f,
|
||||
animationSpec = infiniteRepeatable(
|
||||
animation = tween(420, easing = FastOutSlowInEasing),
|
||||
repeatMode = RepeatMode.Reverse
|
||||
),
|
||||
label = "speakBar3"
|
||||
)
|
||||
val s4 by speakTransition.animateFloat(
|
||||
initialValue = 0.2f,
|
||||
targetValue = 0.8f,
|
||||
animationSpec = infiniteRepeatable(
|
||||
animation = tween(310, easing = FastOutSlowInEasing),
|
||||
repeatMode = RepeatMode.Reverse
|
||||
),
|
||||
label = "speakBar4"
|
||||
)
|
||||
val s5 by speakTransition.animateFloat(
|
||||
initialValue = 0.15f,
|
||||
targetValue = 0.6f,
|
||||
animationSpec = infiniteRepeatable(
|
||||
animation = tween(460, easing = FastOutSlowInEasing),
|
||||
repeatMode = RepeatMode.Reverse
|
||||
),
|
||||
label = "speakBar5"
|
||||
)
|
||||
|
||||
Row(
|
||||
modifier = modifier
|
||||
.fillMaxWidth()
|
||||
.background(
|
||||
color = MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.85f),
|
||||
shape = RoundedCornerShape(32.dp)
|
||||
)
|
||||
.padding(horizontal = 16.dp, vertical = 8.dp),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.SpaceBetween
|
||||
) {
|
||||
// Left actions: Camera and Share Screen
|
||||
Row(
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.spacedBy(16.dp)
|
||||
) {
|
||||
IconButton(
|
||||
onClick = onToggleVisionMode
|
||||
) {
|
||||
Icon(
|
||||
painter = painterResource(id = R.drawable.ic_camera),
|
||||
contentDescription = "Camera (Hands-free)",
|
||||
tint = if (isVisionModeActive) MaterialTheme.colorScheme.primary else MaterialTheme.colorScheme.onSurfaceVariant
|
||||
)
|
||||
}
|
||||
|
||||
IconButton(
|
||||
onClick = onToggleScreenMode
|
||||
) {
|
||||
Icon(
|
||||
painter = painterResource(id = R.drawable.ic_screen_share),
|
||||
contentDescription = "Share Screen (Hands-free)",
|
||||
tint = if (isScreenModeActive) MaterialTheme.colorScheme.primary else MaterialTheme.colorScheme.onSurfaceVariant
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Center State Orb
|
||||
Box(
|
||||
modifier = Modifier.weight(1f),
|
||||
contentAlignment = Alignment.Center
|
||||
) {
|
||||
// Outer pulsing ring (drawn behind using graphicsLayer scale so it does not affect parent height)
|
||||
if (thinkingProgress > 0f) {
|
||||
Box(
|
||||
modifier = Modifier
|
||||
.size(orbWidth, orbHeight)
|
||||
.graphicsLayer {
|
||||
scaleX = thinkingPulseScale
|
||||
scaleY = thinkingPulseScale
|
||||
alpha = thinkingPulseAlpha * thinkingProgress
|
||||
}
|
||||
.background(
|
||||
color = orbBackgroundColor,
|
||||
shape = RoundedCornerShape(24.dp)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
Box(
|
||||
modifier = Modifier
|
||||
.size(orbWidth, orbHeight)
|
||||
.background(color = orbBackgroundColor, shape = RoundedCornerShape(24.dp)),
|
||||
contentAlignment = Alignment.Center
|
||||
) {
|
||||
val colorPrimary = MaterialTheme.colorScheme.primary
|
||||
val colorSecondary = MaterialTheme.colorScheme.secondary
|
||||
val colorTertiary = MaterialTheme.colorScheme.tertiary
|
||||
|
||||
Canvas(modifier = Modifier.fillMaxSize()) {
|
||||
val width = size.width
|
||||
val height = size.height
|
||||
val centerX = width / 2
|
||||
val centerY = height / 2
|
||||
|
||||
// Helper to lerp floats
|
||||
fun lerp(start: Float, stop: Float, fraction: Float): Float {
|
||||
return start + fraction * (stop - start)
|
||||
}
|
||||
|
||||
// A. STARTING outer ring rotating arc segment
|
||||
if (startingAlpha > 0f) {
|
||||
val arcSize = 32.dp.toPx()
|
||||
rotate(degrees = startingRotation, pivot = Offset(centerX, centerY)) {
|
||||
drawArc(
|
||||
color = orbContentColor,
|
||||
startAngle = 0f,
|
||||
sweepAngle = 270f,
|
||||
useCenter = false,
|
||||
topLeft = Offset(centerX - arcSize / 2, centerY - arcSize / 2),
|
||||
size = Size(arcSize, arcSize),
|
||||
style = Stroke(width = 3.dp.toPx(), cap = StrokeCap.Round),
|
||||
alpha = startingAlpha
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// B. STARTING core dot (solid color)
|
||||
if (startingAlpha > 0f) {
|
||||
val startDotRadius = 6.dp.toPx() * coreScale
|
||||
drawCircle(
|
||||
color = orbContentColor,
|
||||
radius = startDotRadius,
|
||||
center = Offset(centerX, centerY),
|
||||
alpha = startingAlpha
|
||||
)
|
||||
}
|
||||
|
||||
// C. LISTENING state morphing blob (gradient with white outline)
|
||||
if (listeningProgress > 0f) {
|
||||
// 1. Soft glowing background aura inside the capsule
|
||||
val auraRadius = (height / 2) * (0.85f + 0.4f * animatedAmplitude)
|
||||
val auraAlpha = (0.25f + 0.25f * animatedAmplitude).coerceIn(0f, 1f)
|
||||
drawCircle(
|
||||
brush = Brush.radialGradient(
|
||||
colors = listOf(
|
||||
colorTertiary.copy(alpha = auraAlpha * listeningProgress),
|
||||
colorPrimary.copy(alpha = (auraAlpha * 0.3f) * listeningProgress),
|
||||
Color.Transparent
|
||||
),
|
||||
center = Offset(centerX, centerY),
|
||||
radius = auraRadius
|
||||
),
|
||||
radius = auraRadius,
|
||||
center = Offset(centerX, centerY)
|
||||
)
|
||||
|
||||
// 2. Slow breathing animation
|
||||
val breathe = kotlin.math.sin(breathingPhase.toDouble()).toFloat() * 1.5f.dp.toPx() * (1f - animatedAmplitude)
|
||||
|
||||
// 3. Dynamic morphing blob path calculation (Single organic blob)
|
||||
val baseRadius = lerp(6.dp.toPx(), height * 0.32f, listeningProgress) + (height * 0.08f * animatedAmplitude)
|
||||
|
||||
// Modest ripple amplitudes to keep the shape smooth
|
||||
val amp1 = lerp(0.8f.dp.toPx(), 2.2f.dp.toPx(), listeningProgress) + (5.dp.toPx() * animatedAmplitude)
|
||||
val amp2 = lerp(0.6f.dp.toPx(), 1.6f.dp.toPx(), listeningProgress) + (4.dp.toPx() * animatedAmplitude)
|
||||
val amp3 = 3.dp.toPx() * animatedAmplitude // only when speaking to represent absorbing voice
|
||||
|
||||
val blobPath = Path()
|
||||
val steps = 72
|
||||
for (i in 0..steps) {
|
||||
val angle = (i * 2 * Math.PI / steps).toFloat()
|
||||
val r = baseRadius + breathe +
|
||||
amp1 * kotlin.math.sin(2f * angle + wavePhase) +
|
||||
amp2 * kotlin.math.cos(3f * angle - wavePhase) +
|
||||
amp3 * kotlin.math.sin(4f * angle + wavePhase * 1.3f)
|
||||
val x = centerX + r * kotlin.math.cos(angle)
|
||||
val y = centerY + r * kotlin.math.sin(angle)
|
||||
if (i == 0) {
|
||||
blobPath.moveTo(x, y)
|
||||
} else {
|
||||
blobPath.lineTo(x, y)
|
||||
}
|
||||
}
|
||||
blobPath.close()
|
||||
|
||||
val blobGradient = Brush.linearGradient(
|
||||
colors = listOf(colorPrimary, colorTertiary, colorSecondary),
|
||||
start = Offset(centerX - baseRadius, centerY - baseRadius),
|
||||
end = Offset(centerX + baseRadius, centerY + baseRadius)
|
||||
)
|
||||
|
||||
// Fill the blob
|
||||
drawPath(path = blobPath, brush = blobGradient, alpha = listeningProgress)
|
||||
|
||||
// Thin white outline
|
||||
drawPath(
|
||||
path = blobPath,
|
||||
color = Color.White.copy(alpha = (0.35f + 0.15f * animatedAmplitude) * listeningProgress),
|
||||
style = Stroke(width = (1f + 0.5f * animatedAmplitude).dp.toPx())
|
||||
)
|
||||
}
|
||||
|
||||
// D. THINKING and SPEAKING states: 5 orbiting dots / equalizer bars
|
||||
if (dotsSpreadProgress > 0f) {
|
||||
val spacing = 8.dp.toPx()
|
||||
val dotWidth = 5.dp.toPx()
|
||||
val dotHeight = 5.dp.toPx()
|
||||
|
||||
val horizontalAmplitude = 5.dp.toPx()
|
||||
val verticalAmplitude = 1.5.dp.toPx()
|
||||
|
||||
for (i in 0 until 5) {
|
||||
// Equalizer height for this bar
|
||||
val eqHeight = when (i) {
|
||||
0 -> 8.dp.toPx() + 16.dp.toPx() * s1
|
||||
1 -> 12.dp.toPx() + 14.dp.toPx() * s2
|
||||
2 -> 6.dp.toPx() + 22.dp.toPx() * s3
|
||||
3 -> 10.dp.toPx() + 18.dp.toPx() * s4
|
||||
else -> 8.dp.toPx() + 12.dp.toPx() * s5
|
||||
}
|
||||
|
||||
// Interpolate height between circle (dotHeight) and equalizer bar height
|
||||
val currentHeight = lerp(dotHeight, eqHeight, speakingProgress)
|
||||
|
||||
// 3D-like orbital offsets (only active in THINKING)
|
||||
val dotPhase = wavePhase - i * 0.6f
|
||||
val dx = horizontalAmplitude * kotlin.math.cos(dotPhase.toDouble()).toFloat() * thinkingProgress
|
||||
val dy = verticalAmplitude * kotlin.math.sin(dotPhase.toDouble()).toFloat() * thinkingProgress
|
||||
|
||||
// Interpolate home horizontal position based on spread progress
|
||||
val homeX = centerX + (i - 2) * spacing * dotsSpreadProgress
|
||||
val x = homeX + dx
|
||||
val y = centerY + dy
|
||||
|
||||
val topLeftX = x - dotWidth / 2
|
||||
val topLeftY = y - currentHeight / 2
|
||||
|
||||
drawRoundRect(
|
||||
color = orbContentColor,
|
||||
topLeft = Offset(topLeftX, topLeftY),
|
||||
size = Size(dotWidth, currentHeight),
|
||||
cornerRadius = CornerRadius(dotWidth / 2, dotWidth / 2),
|
||||
alpha = dotsSpreadProgress
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Right actions: Mic and Close
|
||||
Row(
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.spacedBy(16.dp)
|
||||
) {
|
||||
IconButton(
|
||||
onClick = onToggleMute
|
||||
) {
|
||||
Icon(
|
||||
painter = painterResource(
|
||||
id = if (isMuted) R.drawable.ic_mic_off else R.drawable.ic_mic
|
||||
),
|
||||
contentDescription = if (isMuted) "Unmute Mic" else "Mute Mic",
|
||||
tint = MaterialTheme.colorScheme.onSurfaceVariant
|
||||
)
|
||||
}
|
||||
|
||||
IconButton(
|
||||
onClick = onExitHandsFree
|
||||
) {
|
||||
Icon(
|
||||
imageVector = Icons.Default.Close,
|
||||
contentDescription = "Exit Hands-free",
|
||||
tint = MaterialTheme.colorScheme.onSurfaceVariant
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Preview(showBackground = true)
|
||||
@Composable
|
||||
fun HandsFreeBarStartingPreview() {
|
||||
AssistantTheme {
|
||||
HandsFreeBar(
|
||||
isSpeaking = false,
|
||||
isListening = false,
|
||||
isThinking = false,
|
||||
isMicReady = false,
|
||||
onExitHandsFree = {}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Preview(showBackground = true)
|
||||
@Composable
|
||||
fun HandsFreeBarListeningPreview() {
|
||||
AssistantTheme {
|
||||
HandsFreeBar(
|
||||
isSpeaking = false,
|
||||
isListening = true,
|
||||
isMicReady = true,
|
||||
isThinking = false,
|
||||
onExitHandsFree = {}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Preview(showBackground = true)
|
||||
@Composable
|
||||
fun HandsFreeBarThinkingPreview() {
|
||||
AssistantTheme {
|
||||
HandsFreeBar(
|
||||
isSpeaking = false,
|
||||
isListening = false,
|
||||
isMicReady = true,
|
||||
isThinking = true,
|
||||
onExitHandsFree = {}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Preview(showBackground = true)
|
||||
@Composable
|
||||
fun HandsFreeBarSpeakingPreview() {
|
||||
AssistantTheme {
|
||||
HandsFreeBar(
|
||||
isSpeaking = true,
|
||||
isListening = false,
|
||||
isMicReady = true,
|
||||
isThinking = false,
|
||||
onExitHandsFree = {}
|
||||
)
|
||||
}
|
||||
}
|
||||
101
app/src/main/java/com/app/assistant/ui/screen/LoadingDots.kt
Normal file
101
app/src/main/java/com/app/assistant/ui/screen/LoadingDots.kt
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
package com.app.assistant.ui.screen
|
||||
|
||||
import androidx.compose.animation.core.LinearEasing
|
||||
import androidx.compose.animation.core.RepeatMode
|
||||
import androidx.compose.animation.core.StartOffset
|
||||
import androidx.compose.animation.core.animateFloat
|
||||
import androidx.compose.animation.core.infiniteRepeatable
|
||||
import androidx.compose.animation.core.rememberInfiniteTransition
|
||||
import androidx.compose.animation.core.tween
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.Spacer
|
||||
import androidx.compose.foundation.layout.offset
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.width
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.unit.dp
|
||||
|
||||
import androidx.compose.ui.tooling.preview.Preview
|
||||
import com.app.assistant.ui.theme.AssistantTheme
|
||||
|
||||
@Composable
|
||||
fun LoadingDots() {
|
||||
val infiniteTransition = rememberInfiniteTransition()
|
||||
|
||||
// Animating each dot separately
|
||||
val dot1Offset by infiniteTransition.animateFloat(
|
||||
initialValue = 0f,
|
||||
targetValue = -10f,
|
||||
animationSpec =
|
||||
infiniteRepeatable(
|
||||
animation = tween(durationMillis = 400, easing = LinearEasing),
|
||||
repeatMode = RepeatMode.Reverse,
|
||||
initialStartOffset = StartOffset(0), // First dot starts immediately
|
||||
),
|
||||
label = "",
|
||||
)
|
||||
|
||||
val dot2Offset by infiniteTransition.animateFloat(
|
||||
initialValue = 0f,
|
||||
targetValue = -10f,
|
||||
animationSpec =
|
||||
infiniteRepeatable(
|
||||
animation = tween(durationMillis = 400, easing = LinearEasing),
|
||||
repeatMode = RepeatMode.Reverse,
|
||||
initialStartOffset = StartOffset(200), // Second dot starts with a delay
|
||||
),
|
||||
label = "",
|
||||
)
|
||||
|
||||
val dot3Offset by infiniteTransition.animateFloat(
|
||||
initialValue = 0f,
|
||||
targetValue = -10f,
|
||||
animationSpec =
|
||||
infiniteRepeatable(
|
||||
animation = tween(durationMillis = 400, easing = LinearEasing),
|
||||
repeatMode = RepeatMode.Reverse,
|
||||
initialStartOffset = StartOffset(400), // Third dot starts with a larger delay
|
||||
),
|
||||
label = "",
|
||||
)
|
||||
|
||||
// Unicode character for big dot (⏺ or ● for larger circles)
|
||||
val dotChar = "\u25CF"
|
||||
|
||||
Row(modifier = Modifier.padding(16.dp)) {
|
||||
// First dot
|
||||
Text(
|
||||
text = dotChar,
|
||||
modifier = Modifier.offset(y = dot1Offset.dp),
|
||||
style = MaterialTheme.typography.bodyLarge,
|
||||
)
|
||||
Spacer(modifier = Modifier.width(4.dp))
|
||||
|
||||
// Second dot
|
||||
Text(
|
||||
text = dotChar,
|
||||
modifier = Modifier.offset(y = dot2Offset.dp),
|
||||
style = MaterialTheme.typography.bodyLarge,
|
||||
)
|
||||
Spacer(modifier = Modifier.width(4.dp))
|
||||
|
||||
// Third dot
|
||||
Text(
|
||||
text = dotChar,
|
||||
modifier = Modifier.offset(y = dot3Offset.dp),
|
||||
style = MaterialTheme.typography.bodyLarge,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Preview(showBackground = true)
|
||||
@Composable
|
||||
fun LoadingDotsPreview() {
|
||||
AssistantTheme {
|
||||
LoadingDots()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
package com.app.assistant.ui.screen
|
||||
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.text.AnnotatedString
|
||||
import androidx.compose.ui.text.SpanStyle
|
||||
import androidx.compose.ui.text.font.FontFamily
|
||||
import androidx.compose.ui.text.font.FontStyle
|
||||
import androidx.compose.ui.text.font.FontWeight
|
||||
import androidx.compose.ui.text.style.TextDecoration
|
||||
import androidx.compose.ui.unit.sp
|
||||
import org.commonmark.node.BlockQuote
|
||||
import org.commonmark.node.BulletList
|
||||
import org.commonmark.node.Code
|
||||
import org.commonmark.node.Emphasis
|
||||
import org.commonmark.node.FencedCodeBlock
|
||||
import org.commonmark.node.HardLineBreak
|
||||
import org.commonmark.node.Heading
|
||||
import org.commonmark.node.IndentedCodeBlock
|
||||
import org.commonmark.node.Link
|
||||
import org.commonmark.node.ListItem
|
||||
import org.commonmark.node.Node
|
||||
import org.commonmark.node.OrderedList
|
||||
import org.commonmark.node.Paragraph
|
||||
import org.commonmark.node.SoftLineBreak
|
||||
import org.commonmark.node.StrongEmphasis
|
||||
import org.commonmark.node.ThematicBreak
|
||||
import org.commonmark.parser.Parser
|
||||
import org.commonmark.node.Text as CText
|
||||
|
||||
import androidx.compose.ui.tooling.preview.Preview
|
||||
import com.app.assistant.ui.theme.AssistantTheme
|
||||
|
||||
@Composable
|
||||
fun MarkdownText(
|
||||
modifier: Modifier = Modifier,
|
||||
markdown: String,
|
||||
linkColor: Color = androidx.compose.material3.MaterialTheme.colorScheme.primary,
|
||||
) {
|
||||
val annotatedText =
|
||||
remember(markdown, linkColor) {
|
||||
com.app.assistant.util.MarkdownUtils.parseMarkdownToAnnotatedString(markdown, linkColor)
|
||||
}
|
||||
Text(
|
||||
text = annotatedText,
|
||||
modifier = modifier,
|
||||
)
|
||||
}
|
||||
|
||||
@Preview(showBackground = true)
|
||||
@Composable
|
||||
fun MarkdownTextPreview() {
|
||||
AssistantTheme {
|
||||
MarkdownText(
|
||||
modifier = Modifier.padding(16.dp),
|
||||
markdown = """
|
||||
# Markdown Preview
|
||||
This is a **bold** text and *italic* text.
|
||||
- List item 1
|
||||
- List item 2
|
||||
|
||||
[Link to Google](https://google.com)
|
||||
""".trimIndent()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
418
app/src/main/java/com/app/assistant/ui/screen/QrScannerDialog.kt
Normal file
418
app/src/main/java/com/app/assistant/ui/screen/QrScannerDialog.kt
Normal file
|
|
@ -0,0 +1,418 @@
|
|||
package com.app.assistant.ui.screen
|
||||
|
||||
import android.Manifest
|
||||
import android.content.pm.PackageManager
|
||||
import android.util.Log
|
||||
import android.util.Size
|
||||
import androidx.activity.compose.rememberLauncherForActivityResult
|
||||
import androidx.activity.result.contract.ActivityResultContracts
|
||||
import androidx.camera.core.CameraSelector
|
||||
import androidx.camera.core.ExperimentalGetImage
|
||||
import androidx.camera.core.ImageAnalysis
|
||||
import androidx.camera.core.ImageProxy
|
||||
import androidx.camera.core.Preview
|
||||
import androidx.camera.lifecycle.ProcessCameraProvider
|
||||
import androidx.camera.view.PreviewView
|
||||
import androidx.compose.animation.core.*
|
||||
import androidx.compose.foundation.Canvas
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.foundation.clickable
|
||||
import androidx.compose.foundation.layout.*
|
||||
import androidx.compose.foundation.shape.RoundedCornerShape
|
||||
import androidx.compose.material.icons.Icons
|
||||
import androidx.compose.material.icons.filled.Close
|
||||
import androidx.compose.material.icons.filled.Warning
|
||||
import androidx.compose.material3.*
|
||||
import androidx.compose.runtime.*
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.geometry.CornerRadius
|
||||
import androidx.compose.ui.geometry.Offset
|
||||
import androidx.compose.ui.geometry.Rect
|
||||
import androidx.compose.ui.geometry.RoundRect
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.graphics.Path
|
||||
import androidx.compose.ui.graphics.PathOperation
|
||||
import androidx.compose.ui.graphics.drawscope.Stroke
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.platform.LocalLifecycleOwner
|
||||
import androidx.compose.ui.platform.LocalUriHandler
|
||||
import androidx.compose.ui.text.font.FontWeight
|
||||
import androidx.compose.ui.text.style.TextAlign
|
||||
import androidx.compose.ui.text.style.TextDecoration
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.compose.ui.unit.sp
|
||||
import androidx.compose.ui.viewinterop.AndroidView
|
||||
import androidx.compose.ui.window.Dialog
|
||||
import androidx.compose.ui.window.DialogProperties
|
||||
import androidx.core.content.ContextCompat
|
||||
import com.google.mlkit.vision.barcode.BarcodeScannerOptions
|
||||
import com.google.mlkit.vision.barcode.BarcodeScanning
|
||||
import com.google.mlkit.vision.barcode.common.Barcode
|
||||
import com.google.mlkit.vision.common.InputImage
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
// The URL where users can securely import and export their keys as a QR code
|
||||
const val EXPORT_KEYS_WEBSITE_URL = "https://souravanand001.github.io/ai-assistant-android/"
|
||||
|
||||
@Composable
|
||||
fun QrScannerDialog(
|
||||
onDismissRequest: () -> Unit,
|
||||
onCodeScanned: (String) -> Unit
|
||||
) {
|
||||
val context = LocalContext.current
|
||||
var hasCameraPermission by remember {
|
||||
mutableStateOf(
|
||||
ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED
|
||||
)
|
||||
}
|
||||
|
||||
val permissionLauncher = rememberLauncherForActivityResult(
|
||||
contract = ActivityResultContracts.RequestPermission(),
|
||||
onResult = { granted ->
|
||||
hasCameraPermission = granted
|
||||
}
|
||||
)
|
||||
|
||||
LaunchedEffect(Unit) {
|
||||
if (!hasCameraPermission) {
|
||||
permissionLauncher.launch(Manifest.permission.CAMERA)
|
||||
}
|
||||
}
|
||||
|
||||
Dialog(
|
||||
onDismissRequest = onDismissRequest,
|
||||
properties = DialogProperties(
|
||||
usePlatformDefaultWidth = false,
|
||||
dismissOnBackPress = true,
|
||||
dismissOnClickOutside = false
|
||||
)
|
||||
) {
|
||||
Box(
|
||||
modifier = Modifier
|
||||
.fillMaxSize()
|
||||
.background(Color.Black)
|
||||
) {
|
||||
if (hasCameraPermission) {
|
||||
ScannerCameraPreview(
|
||||
onCodeScanned = onCodeScanned,
|
||||
onDismissRequest = onDismissRequest
|
||||
)
|
||||
} else {
|
||||
PermissionDeniedContent(
|
||||
onRequestPermission = { permissionLauncher.launch(Manifest.permission.CAMERA) },
|
||||
onClose = onDismissRequest
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun ScannerCameraPreview(
|
||||
onCodeScanned: (String) -> Unit,
|
||||
onDismissRequest: () -> Unit
|
||||
) {
|
||||
val context = LocalContext.current
|
||||
val lifecycleOwner = LocalLifecycleOwner.current
|
||||
val uriHandler = LocalUriHandler.current
|
||||
var isScanned by remember { mutableStateOf(false) }
|
||||
|
||||
val previewView = remember {
|
||||
PreviewView(context).apply {
|
||||
scaleType = PreviewView.ScaleType.FILL_CENTER
|
||||
}
|
||||
}
|
||||
|
||||
// Camera Lifecycle & Setup
|
||||
DisposableEffect(lifecycleOwner) {
|
||||
val cameraProviderFuture = ProcessCameraProvider.getInstance(context)
|
||||
val executor = Executors.newSingleThreadExecutor()
|
||||
|
||||
cameraProviderFuture.addListener({
|
||||
try {
|
||||
val cameraProvider = cameraProviderFuture.get()
|
||||
val preview = Preview.Builder().build().apply {
|
||||
setSurfaceProvider(previewView.surfaceProvider)
|
||||
}
|
||||
|
||||
// Analyze frames with ML Kit Barcode Scanner
|
||||
val imageAnalysis = ImageAnalysis.Builder()
|
||||
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
|
||||
.build()
|
||||
|
||||
imageAnalysis.setAnalyzer(
|
||||
executor,
|
||||
QrCodeAnalyzer { result ->
|
||||
if (!isScanned) {
|
||||
isScanned = true
|
||||
// Trigger callback on main thread
|
||||
ContextCompat.getMainExecutor(context).execute {
|
||||
onCodeScanned(result)
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
val cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA
|
||||
cameraProvider.unbindAll()
|
||||
cameraProvider.bindToLifecycle(
|
||||
lifecycleOwner,
|
||||
cameraSelector,
|
||||
preview,
|
||||
imageAnalysis
|
||||
)
|
||||
|
||||
} catch (e: Exception) {
|
||||
Log.e("QrScannerDialog", "CameraX UseCase binding failed", e)
|
||||
}
|
||||
}, ContextCompat.getMainExecutor(context))
|
||||
|
||||
onDispose {
|
||||
try {
|
||||
if (cameraProviderFuture.isDone) {
|
||||
val cameraProvider = cameraProviderFuture.get()
|
||||
cameraProvider.unbindAll()
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("QrScannerDialog", "Error unbinding CameraX", e)
|
||||
}
|
||||
executor.shutdown()
|
||||
}
|
||||
}
|
||||
|
||||
// Camera Feed Underlay
|
||||
AndroidView(
|
||||
factory = { previewView },
|
||||
modifier = Modifier.fillMaxSize()
|
||||
)
|
||||
|
||||
// Scanner overlay (Canvas with square cutout + neon green frame + scanning line)
|
||||
val infiniteTransition = rememberInfiniteTransition(label = "laser")
|
||||
val laserPosition by infiniteTransition.animateFloat(
|
||||
initialValue = 0f,
|
||||
targetValue = 1f,
|
||||
animationSpec = infiniteRepeatable(
|
||||
animation = tween(durationMillis = 2500, easing = LinearEasing),
|
||||
repeatMode = RepeatMode.Reverse
|
||||
),
|
||||
label = "laserPosition"
|
||||
)
|
||||
|
||||
Canvas(modifier = Modifier.fillMaxSize()) {
|
||||
val canvasWidth = size.width
|
||||
val canvasHeight = size.height
|
||||
val boxWidth = canvasWidth * 0.7f
|
||||
val boxHeight = boxWidth // Square viewfinder
|
||||
|
||||
val left = (canvasWidth - boxWidth) / 2
|
||||
val top = (canvasHeight - boxHeight) / 2
|
||||
|
||||
// Draw transparent black mask with cutout
|
||||
val path = Path().apply {
|
||||
addRect(Rect(0f, 0f, canvasWidth, canvasHeight))
|
||||
}
|
||||
val cutoutPath = Path().apply {
|
||||
addRoundRect(
|
||||
RoundRect(
|
||||
left = left,
|
||||
top = top,
|
||||
right = left + boxWidth,
|
||||
bottom = top + boxHeight,
|
||||
cornerRadius = CornerRadius(24.dp.toPx(), 24.dp.toPx())
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
val finalPath = Path.combine(PathOperation.Difference, path, cutoutPath)
|
||||
drawPath(finalPath, color = Color.Black.copy(alpha = 0.65f))
|
||||
|
||||
// Viewfinder Frame Outline
|
||||
drawRoundRect(
|
||||
color = Color(0xFF00E676), // Neon Green
|
||||
topLeft = Offset(left, top),
|
||||
size = androidx.compose.ui.geometry.Size(boxWidth, boxHeight),
|
||||
cornerRadius = CornerRadius(24.dp.toPx(), 24.dp.toPx()),
|
||||
style = Stroke(width = 3.dp.toPx())
|
||||
)
|
||||
|
||||
// Viewfinder Target Breathing Line (Red Laser)
|
||||
val laserY = top + (boxHeight * laserPosition)
|
||||
drawLine(
|
||||
color = Color(0xFFFF1744), // Bright Red
|
||||
start = Offset(left + 15.dp.toPx(), laserY),
|
||||
end = Offset(left + boxWidth - 15.dp.toPx(), laserY),
|
||||
strokeWidth = 2.dp.toPx()
|
||||
)
|
||||
}
|
||||
|
||||
// UI Controls Overlay
|
||||
Box(
|
||||
modifier = Modifier
|
||||
.fillMaxSize()
|
||||
.padding(16.dp)
|
||||
) {
|
||||
// Close Button
|
||||
IconButton(
|
||||
onClick = onDismissRequest,
|
||||
modifier = Modifier
|
||||
.align(Alignment.TopStart)
|
||||
.statusBarsPadding()
|
||||
.background(Color.Black.copy(alpha = 0.5f), RoundedCornerShape(50))
|
||||
) {
|
||||
Icon(
|
||||
imageVector = Icons.Default.Close,
|
||||
contentDescription = "Close Scanner",
|
||||
tint = Color.White
|
||||
)
|
||||
}
|
||||
|
||||
// Instructions Card
|
||||
Card(
|
||||
modifier = Modifier
|
||||
.align(Alignment.TopCenter)
|
||||
.padding(top = 70.dp)
|
||||
.statusBarsPadding()
|
||||
.fillMaxWidth(0.9f),
|
||||
colors = CardDefaults.cardColors(
|
||||
containerColor = Color(0xFF1E1E2C).copy(alpha = 0.9f)
|
||||
),
|
||||
shape = RoundedCornerShape(16.dp),
|
||||
elevation = CardDefaults.cardElevation(defaultElevation = 8.dp)
|
||||
) {
|
||||
Column(
|
||||
modifier = Modifier.padding(16.dp),
|
||||
horizontalAlignment = Alignment.CenterHorizontally
|
||||
) {
|
||||
Text(
|
||||
text = "Import API Keys Safely",
|
||||
color = Color.White,
|
||||
fontWeight = FontWeight.Bold,
|
||||
fontSize = 16.sp,
|
||||
textAlign = TextAlign.Center
|
||||
)
|
||||
Spacer(modifier = Modifier.height(6.dp))
|
||||
Text(
|
||||
text = "To import your API keys securely, visit this URL on your desktop/other device:",
|
||||
color = Color.LightGray,
|
||||
fontSize = 12.sp,
|
||||
textAlign = TextAlign.Center
|
||||
)
|
||||
Spacer(modifier = Modifier.height(8.dp))
|
||||
Text(
|
||||
text = EXPORT_KEYS_WEBSITE_URL,
|
||||
color = Color(0xFF29B6F6), // Sky Blue Link Color
|
||||
fontWeight = FontWeight.SemiBold,
|
||||
fontSize = 13.sp,
|
||||
textDecoration = TextDecoration.Underline,
|
||||
textAlign = TextAlign.Center,
|
||||
modifier = Modifier
|
||||
.clickable {
|
||||
try {
|
||||
uriHandler.openUri(EXPORT_KEYS_WEBSITE_URL)
|
||||
} catch (e: Exception) {
|
||||
Log.e("QrScannerDialog", "Failed to open URI", e)
|
||||
}
|
||||
}
|
||||
.padding(4.dp)
|
||||
)
|
||||
Spacer(modifier = Modifier.height(4.dp))
|
||||
Text(
|
||||
text = "Align the QR code on the screen within the frame to scan.",
|
||||
color = Color.Gray,
|
||||
fontSize = 11.sp,
|
||||
textAlign = TextAlign.Center
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun PermissionDeniedContent(
|
||||
onRequestPermission: () -> Unit,
|
||||
onClose: () -> Unit
|
||||
) {
|
||||
Column(
|
||||
modifier = Modifier
|
||||
.fillMaxSize()
|
||||
.padding(24.dp),
|
||||
horizontalAlignment = Alignment.CenterHorizontally,
|
||||
verticalArrangement = Arrangement.Center
|
||||
) {
|
||||
Icon(
|
||||
imageVector = Icons.Default.Warning,
|
||||
contentDescription = "Warning",
|
||||
tint = Color(0xFFFFB300),
|
||||
modifier = Modifier.size(64.dp)
|
||||
)
|
||||
Spacer(modifier = Modifier.height(16.dp))
|
||||
Text(
|
||||
text = "Camera Permission Required",
|
||||
color = Color.White,
|
||||
fontWeight = FontWeight.Bold,
|
||||
fontSize = 20.sp,
|
||||
textAlign = TextAlign.Center
|
||||
)
|
||||
Spacer(modifier = Modifier.height(8.dp))
|
||||
Text(
|
||||
text = "We need access to your camera to scan QR codes and safely import API keys.",
|
||||
color = Color.LightGray,
|
||||
fontSize = 14.sp,
|
||||
textAlign = TextAlign.Center
|
||||
)
|
||||
Spacer(modifier = Modifier.height(24.dp))
|
||||
Button(
|
||||
onClick = onRequestPermission,
|
||||
colors = ButtonDefaults.buttonColors(
|
||||
containerColor = Color(0xFF00E676),
|
||||
contentColor = Color.Black
|
||||
),
|
||||
shape = RoundedCornerShape(12.dp),
|
||||
modifier = Modifier.fillMaxWidth()
|
||||
) {
|
||||
Text("Grant Permission", fontWeight = FontWeight.Bold)
|
||||
}
|
||||
Spacer(modifier = Modifier.height(12.dp))
|
||||
TextButton(onClick = onClose) {
|
||||
Text("Cancel", color = Color.White)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class QrCodeAnalyzer(
|
||||
private val onQrCodeScanned: (String) -> Unit
|
||||
) : ImageAnalysis.Analyzer {
|
||||
|
||||
private val options = BarcodeScannerOptions.Builder()
|
||||
.setBarcodeFormats(Barcode.FORMAT_QR_CODE)
|
||||
.build()
|
||||
|
||||
private val scanner = BarcodeScanning.getClient(options)
|
||||
|
||||
@ExperimentalGetImage
|
||||
override fun analyze(imageProxy: ImageProxy) {
|
||||
val mediaImage = imageProxy.image
|
||||
if (mediaImage != null) {
|
||||
val image = InputImage.fromMediaImage(mediaImage, imageProxy.imageInfo.rotationDegrees)
|
||||
scanner.process(image)
|
||||
.addOnSuccessListener { barcodes ->
|
||||
for (barcode in barcodes) {
|
||||
val rawValue = barcode.rawValue
|
||||
if (rawValue != null && rawValue.isNotBlank()) {
|
||||
onQrCodeScanned(rawValue)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
.addOnFailureListener { e ->
|
||||
Log.e("QrCodeAnalyzer", "Barcode scanning failed", e)
|
||||
}
|
||||
.addOnCompleteListener {
|
||||
imageProxy.close()
|
||||
}
|
||||
} else {
|
||||
imageProxy.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
2244
app/src/main/java/com/app/assistant/ui/screen/SettingsScreen.kt
Normal file
2244
app/src/main/java/com/app/assistant/ui/screen/SettingsScreen.kt
Normal file
File diff suppressed because it is too large
Load diff
463
app/src/main/java/com/app/assistant/ui/screen/UserInputField.kt
Normal file
463
app/src/main/java/com/app/assistant/ui/screen/UserInputField.kt
Normal file
|
|
@ -0,0 +1,463 @@
|
|||
package com.app.assistant.ui.screen
|
||||
|
||||
import androidx.compose.animation.core.Animatable
|
||||
import androidx.compose.animation.core.LinearEasing
|
||||
import androidx.compose.animation.core.tween
|
||||
import androidx.compose.animation.AnimatedVisibility
|
||||
import androidx.compose.animation.fadeIn
|
||||
import androidx.compose.animation.fadeOut
|
||||
import androidx.compose.animation.expandHorizontally
|
||||
import androidx.compose.animation.shrinkHorizontally
|
||||
import androidx.compose.foundation.Canvas
|
||||
import androidx.compose.foundation.Image
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.ui.draw.clip
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.Spacer
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.height
|
||||
import androidx.compose.foundation.layout.imePadding
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.foundation.lazy.LazyRow
|
||||
import androidx.compose.foundation.lazy.items
|
||||
import androidx.compose.foundation.shape.CircleShape
|
||||
import androidx.compose.foundation.shape.RoundedCornerShape
|
||||
import androidx.compose.material.icons.Icons
|
||||
import androidx.compose.material.icons.automirrored.filled.Send
|
||||
import androidx.compose.material.icons.filled.Close
|
||||
import androidx.compose.material3.CircularProgressIndicator
|
||||
import androidx.compose.material3.DropdownMenu
|
||||
import androidx.compose.material3.DropdownMenuItem
|
||||
import androidx.compose.material3.Icon
|
||||
import androidx.compose.material3.IconButton
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.material3.TextField
|
||||
import androidx.compose.material3.TextFieldDefaults
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.LaunchedEffect
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.graphics.asImageBitmap
|
||||
import androidx.compose.ui.graphics.luminance
|
||||
import androidx.compose.ui.layout.ContentScale
|
||||
import androidx.compose.ui.layout.onGloballyPositioned
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.platform.SoftwareKeyboardController
|
||||
import androidx.compose.ui.res.painterResource
|
||||
import androidx.compose.ui.res.stringResource
|
||||
import androidx.compose.ui.text.style.TextAlign
|
||||
import androidx.compose.ui.text.style.TextOverflow
|
||||
import androidx.compose.ui.tooling.preview.Preview
|
||||
import androidx.compose.ui.unit.IntSize
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.compose.ui.unit.sp
|
||||
import com.app.assistant.R
|
||||
import com.app.assistant.model.Attachment
|
||||
import com.app.assistant.ui.theme.AssistantTheme
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.currentCoroutineContext
|
||||
import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlin.math.hypot
|
||||
|
||||
@Composable
|
||||
@Suppress("ktlint:standard:function-naming")
|
||||
fun UserInputField(
|
||||
focusManager: androidx.compose.ui.focus.FocusManager,
|
||||
keyboardController: SoftwareKeyboardController,
|
||||
isSpeaking: Boolean,
|
||||
isListening: Boolean,
|
||||
question: String,
|
||||
onQuestionChange: (String) -> Unit,
|
||||
onStopSpeaking: () -> Unit,
|
||||
onStartListening: () -> Unit,
|
||||
onProcessQuestion: (androidx.compose.ui.focus.FocusManager, SoftwareKeyboardController, Boolean) -> Unit,
|
||||
modifier: Modifier = Modifier,
|
||||
selectedAttachments: List<Attachment> = emptyList(),
|
||||
onRemoveAttachment: (Attachment) -> Unit = {},
|
||||
onAttachClick: (String) -> Unit = {},
|
||||
isImageSupported: Boolean = false,
|
||||
isAudioSupported: Boolean = false,
|
||||
isVideoSupported: Boolean = false,
|
||||
isDocumentSupported: Boolean = false,
|
||||
isHandsFree: Boolean = false,
|
||||
onToggleHandsFree: () -> Unit = {},
|
||||
hasMessages: Boolean = false,
|
||||
) {
|
||||
val containerColor = TextFieldDefaults.colors().unfocusedContainerColor
|
||||
val rippleColor = adjustContrast(containerColor)
|
||||
|
||||
// State for current radius + alpha
|
||||
val rippleRadius = remember { Animatable(0f) }
|
||||
val rippleAlpha = remember { Animatable(0f) }
|
||||
|
||||
// Size of TextField (for max ripple radius)
|
||||
var textFieldSize by remember { mutableStateOf(IntSize.Zero) }
|
||||
|
||||
LaunchedEffect(isListening, textFieldSize) {
|
||||
if (isListening && textFieldSize != IntSize.Zero) {
|
||||
val maxRadius =
|
||||
hypot(
|
||||
textFieldSize.width / 2f,
|
||||
textFieldSize.height / 2f,
|
||||
)
|
||||
while (currentCoroutineContext().isActive) { // keep running until LaunchedEffect cancels
|
||||
rippleRadius.snapTo(0f)
|
||||
rippleAlpha.snapTo(0.5f)
|
||||
|
||||
// animate simultaneously
|
||||
launch {
|
||||
rippleRadius.animateTo(
|
||||
targetValue = maxRadius,
|
||||
animationSpec = tween(1000, easing = LinearEasing),
|
||||
)
|
||||
}
|
||||
rippleAlpha.animateTo(
|
||||
targetValue = 0f,
|
||||
animationSpec = tween(1000, easing = LinearEasing),
|
||||
)
|
||||
}
|
||||
} else {
|
||||
rippleRadius.snapTo(0f)
|
||||
rippleAlpha.snapTo(0f)
|
||||
}
|
||||
}
|
||||
|
||||
Column(
|
||||
modifier = modifier
|
||||
.fillMaxWidth()
|
||||
.imePadding()
|
||||
.padding(8.dp)
|
||||
) {
|
||||
if (selectedAttachments.isNotEmpty()) {
|
||||
AttachmentPreviewList(
|
||||
attachments = selectedAttachments,
|
||||
onRemove = onRemoveAttachment,
|
||||
modifier = Modifier.padding(bottom = 8.dp)
|
||||
)
|
||||
}
|
||||
|
||||
Box(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.clip(RoundedCornerShape(20)) // clip ripple inside
|
||||
.onGloballyPositioned { coords -> textFieldSize = coords.size },
|
||||
) {
|
||||
TextField(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
shape = RoundedCornerShape(20),
|
||||
value = question,
|
||||
placeholder = { Text(stringResource(id = R.string.type_here_placeholder)) },
|
||||
maxLines = 4,
|
||||
onValueChange = onQuestionChange,
|
||||
colors = TextFieldDefaults.colors(
|
||||
focusedIndicatorColor = Color.Transparent,
|
||||
unfocusedIndicatorColor = Color.Transparent,
|
||||
disabledIndicatorColor = Color.Transparent,
|
||||
),
|
||||
leadingIcon = if ((isImageSupported || isAudioSupported || isVideoSupported || isDocumentSupported) && !isSpeaking) {
|
||||
{
|
||||
var menuExpanded by remember { mutableStateOf(false) }
|
||||
Box {
|
||||
IconButton(onClick = { menuExpanded = true }) {
|
||||
Icon(
|
||||
painter = painterResource(id = R.drawable.ic_attachment),
|
||||
contentDescription = "Attach file",
|
||||
tint = MaterialTheme.colorScheme.onSurfaceVariant
|
||||
)
|
||||
}
|
||||
DropdownMenu(
|
||||
expanded = menuExpanded,
|
||||
onDismissRequest = { menuExpanded = false }
|
||||
) {
|
||||
if (isImageSupported) {
|
||||
DropdownMenuItem(
|
||||
text = { Text("Image") },
|
||||
onClick = {
|
||||
menuExpanded = false
|
||||
onAttachClick("image/*")
|
||||
}
|
||||
)
|
||||
DropdownMenuItem(
|
||||
text = { Text("Take Photo (Camera)") },
|
||||
onClick = {
|
||||
menuExpanded = false
|
||||
onAttachClick("camera")
|
||||
}
|
||||
)
|
||||
}
|
||||
if (isAudioSupported) {
|
||||
DropdownMenuItem(
|
||||
text = { Text("Audio") },
|
||||
onClick = {
|
||||
menuExpanded = false
|
||||
onAttachClick("audio/*")
|
||||
}
|
||||
)
|
||||
}
|
||||
if (isVideoSupported) {
|
||||
DropdownMenuItem(
|
||||
text = { Text("Video") },
|
||||
onClick = {
|
||||
menuExpanded = false
|
||||
onAttachClick("video/*")
|
||||
}
|
||||
)
|
||||
}
|
||||
if (isDocumentSupported) {
|
||||
DropdownMenuItem(
|
||||
text = { Text("Document (PDF)") },
|
||||
onClick = {
|
||||
menuExpanded = false
|
||||
onAttachClick("application/pdf")
|
||||
}
|
||||
)
|
||||
}
|
||||
// Always support text files as they are decrypted and appended inline as text
|
||||
DropdownMenuItem(
|
||||
text = { Text("Text Document (.txt/.csv/.json)") },
|
||||
onClick = {
|
||||
menuExpanded = false
|
||||
onAttachClick("text/*")
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else null,
|
||||
trailingIcon = {
|
||||
val hasInput = question.isNotEmpty() || selectedAttachments.isNotEmpty()
|
||||
val showSendButton = hasInput || isListening || hasMessages
|
||||
Row(
|
||||
horizontalArrangement = Arrangement.End,
|
||||
verticalAlignment = Alignment.CenterVertically
|
||||
) {
|
||||
if (isSpeaking) {
|
||||
IconButton(onClick = {
|
||||
if (isSpeaking) {
|
||||
onStopSpeaking()
|
||||
}
|
||||
}) {
|
||||
Icon(
|
||||
painter = painterResource(id = R.drawable.ic_stop),
|
||||
contentDescription = stringResource(id = R.string.stop_desc),
|
||||
)
|
||||
}
|
||||
} else {
|
||||
IconButton(onClick = {
|
||||
onStartListening()
|
||||
}) {
|
||||
Icon(
|
||||
painter = painterResource(id = R.drawable.ic_mic),
|
||||
contentDescription = stringResource(id = R.string.mic_desc),
|
||||
)
|
||||
}
|
||||
}
|
||||
AnimatedVisibility(
|
||||
visible = !showSendButton,
|
||||
enter = fadeIn() + expandHorizontally(),
|
||||
exit = fadeOut() + shrinkHorizontally()
|
||||
) {
|
||||
IconButton(onClick = onToggleHandsFree) {
|
||||
Icon(
|
||||
painter = painterResource(id = R.drawable.ic_live_modes),
|
||||
contentDescription = "Toggle Hands-free Mode",
|
||||
tint = if (isHandsFree) MaterialTheme.colorScheme.primary else MaterialTheme.colorScheme.onSurfaceVariant
|
||||
)
|
||||
}
|
||||
}
|
||||
AnimatedVisibility(
|
||||
visible = showSendButton,
|
||||
enter = fadeIn() + expandHorizontally(),
|
||||
exit = fadeOut() + shrinkHorizontally()
|
||||
) {
|
||||
IconButton(onClick = {
|
||||
if (question.isNotEmpty() || selectedAttachments.isNotEmpty()) {
|
||||
onProcessQuestion(focusManager, keyboardController, false)
|
||||
}
|
||||
}) {
|
||||
Icon(
|
||||
imageVector = Icons.AutoMirrored.Filled.Send,
|
||||
contentDescription = stringResource(id = R.string.send_desc),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
Canvas(
|
||||
modifier = Modifier
|
||||
.matchParentSize()
|
||||
.clip(RoundedCornerShape(20)),
|
||||
) {
|
||||
if (rippleAlpha.value > 0f) {
|
||||
drawCircle(
|
||||
color = rippleColor.copy(alpha = rippleAlpha.value),
|
||||
radius = rippleRadius.value,
|
||||
center = center,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun AttachmentPreviewList(
|
||||
attachments: List<Attachment>,
|
||||
onRemove: (Attachment) -> Unit,
|
||||
modifier: Modifier = Modifier
|
||||
) {
|
||||
LazyRow(
|
||||
modifier = modifier.fillMaxWidth(),
|
||||
horizontalArrangement = Arrangement.spacedBy(8.dp)
|
||||
) {
|
||||
items(attachments) { attachment ->
|
||||
AttachmentPreviewCard(attachment = attachment, onRemove = { onRemove(attachment) })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun AttachmentPreviewCard(
|
||||
attachment: Attachment,
|
||||
onRemove: () -> Unit
|
||||
) {
|
||||
Box(
|
||||
modifier = Modifier
|
||||
.size(72.dp)
|
||||
.clip(RoundedCornerShape(8.dp))
|
||||
.background(MaterialTheme.colorScheme.surfaceVariant)
|
||||
) {
|
||||
if (attachment.mimeType.startsWith("image/")) {
|
||||
var bitmap by remember { mutableStateOf<android.graphics.Bitmap?>(null) }
|
||||
LaunchedEffect(attachment) {
|
||||
withContext(Dispatchers.IO) {
|
||||
try {
|
||||
val fileBytes = java.io.File(attachment.filePath).readBytes()
|
||||
val decrypted = com.app.assistant.db.EncryptionUtil.decryptFile(fileBytes, attachment.iv)
|
||||
bitmap = android.graphics.BitmapFactory.decodeByteArray(decrypted, 0, decrypted.size)
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
if (bitmap != null) {
|
||||
Image(
|
||||
bitmap = bitmap!!.asImageBitmap(),
|
||||
contentDescription = null,
|
||||
modifier = Modifier.fillMaxSize(),
|
||||
contentScale = ContentScale.Crop
|
||||
)
|
||||
} else {
|
||||
Box(modifier = Modifier.fillMaxSize(), contentAlignment = Alignment.Center) {
|
||||
CircularProgressIndicator(modifier = Modifier.size(16.dp), strokeWidth = 2.dp)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Column(
|
||||
modifier = Modifier
|
||||
.fillMaxSize()
|
||||
.padding(4.dp),
|
||||
horizontalAlignment = Alignment.CenterHorizontally,
|
||||
verticalArrangement = Arrangement.Center
|
||||
) {
|
||||
val iconRes = when {
|
||||
attachment.mimeType.startsWith("audio/") -> R.drawable.ic_mic
|
||||
attachment.mimeType.startsWith("video/") -> R.drawable.ic_stop
|
||||
else -> R.drawable.ic_translate
|
||||
}
|
||||
Icon(
|
||||
painter = painterResource(id = iconRes),
|
||||
contentDescription = null,
|
||||
modifier = Modifier.size(24.dp),
|
||||
tint = MaterialTheme.colorScheme.primary
|
||||
)
|
||||
Spacer(modifier = Modifier.height(2.dp))
|
||||
Text(
|
||||
text = attachment.fileName,
|
||||
fontSize = 8.sp,
|
||||
maxLines = 1,
|
||||
overflow = TextOverflow.Ellipsis,
|
||||
textAlign = TextAlign.Center,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
IconButton(
|
||||
onClick = onRemove,
|
||||
modifier = Modifier
|
||||
.align(Alignment.TopEnd)
|
||||
.size(18.dp)
|
||||
.background(Color.Black.copy(alpha = 0.6f), shape = CircleShape)
|
||||
.padding(2.dp)
|
||||
) {
|
||||
Icon(
|
||||
imageVector = Icons.Default.Close,
|
||||
contentDescription = "Remove",
|
||||
tint = Color.White,
|
||||
modifier = Modifier.size(12.dp)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun adjustContrast(
|
||||
color: Color,
|
||||
factor: Float = 0.15f,
|
||||
): Color {
|
||||
val isDark = color.luminance() < 0.5
|
||||
return if (isDark) {
|
||||
Color(
|
||||
red = (color.red + factor).coerceAtMost(1f),
|
||||
green = (color.green + factor).coerceAtMost(1f),
|
||||
blue = (color.blue + factor).coerceAtMost(1f),
|
||||
alpha = color.alpha,
|
||||
)
|
||||
} else {
|
||||
Color(
|
||||
red = (color.red - factor).coerceAtLeast(0f),
|
||||
green = (color.green - factor).coerceAtLeast(0f),
|
||||
blue = (color.blue - factor).coerceAtLeast(0f),
|
||||
alpha = color.alpha,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Preview(showBackground = true)
|
||||
@Composable
|
||||
fun UserInputFieldIdlePreview() {
|
||||
val focusManager = androidx.compose.ui.platform.LocalFocusManager.current
|
||||
val keyboardController = androidx.compose.ui.platform.LocalSoftwareKeyboardController.current
|
||||
AssistantTheme {
|
||||
if (keyboardController != null) {
|
||||
UserInputField(
|
||||
focusManager = focusManager,
|
||||
keyboardController = keyboardController,
|
||||
isSpeaking = false,
|
||||
isListening = false,
|
||||
question = "",
|
||||
onQuestionChange = {},
|
||||
onStopSpeaking = {},
|
||||
onStartListening = {},
|
||||
onProcessQuestion = { _, _, _ -> },
|
||||
isImageSupported = true,
|
||||
hasMessages = true
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -8,4 +8,4 @@ val Pink80 = Color(0xFFEFB8C8)
|
|||
|
||||
val Purple40 = Color(0xFF6650a4)
|
||||
val PurpleGrey40 = Color(0xFF625b71)
|
||||
val Pink40 = Color(0xFF7D5260)
|
||||
val Pink40 = Color(0xFF7D5260)
|
||||
|
|
|
|||
|
|
@ -11,17 +11,18 @@ import androidx.compose.material3.lightColorScheme
|
|||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
|
||||
private val DarkColorScheme = darkColorScheme(
|
||||
primary = Purple80,
|
||||
secondary = PurpleGrey80,
|
||||
tertiary = Pink80
|
||||
)
|
||||
|
||||
private val LightColorScheme = lightColorScheme(
|
||||
primary = Purple40,
|
||||
secondary = PurpleGrey40,
|
||||
tertiary = Pink40
|
||||
private val DarkColorScheme =
|
||||
darkColorScheme(
|
||||
primary = Purple80,
|
||||
secondary = PurpleGrey80,
|
||||
tertiary = Pink80,
|
||||
)
|
||||
|
||||
private val LightColorScheme =
|
||||
lightColorScheme(
|
||||
primary = Purple40,
|
||||
secondary = PurpleGrey40,
|
||||
tertiary = Pink40,
|
||||
/* Other default colors to override
|
||||
background = Color(0xFFFFFBFE),
|
||||
surface = Color(0xFFFFFBFE),
|
||||
|
|
@ -30,28 +31,35 @@ private val LightColorScheme = lightColorScheme(
|
|||
onTertiary = Color.White,
|
||||
onBackground = Color(0xFF1C1B1F),
|
||||
onSurface = Color(0xFF1C1B1F),
|
||||
*/
|
||||
)
|
||||
*/
|
||||
)
|
||||
|
||||
@Composable
|
||||
fun AssistantTheme(
|
||||
darkTheme: Boolean = isSystemInDarkTheme(),
|
||||
// Dynamic color is available on Android 12+
|
||||
dynamicColor: Boolean = true,
|
||||
content: @Composable () -> Unit
|
||||
content: @Composable () -> Unit,
|
||||
) {
|
||||
val colorScheme = when {
|
||||
dynamicColor && Build.VERSION.SDK_INT >= Build.VERSION_CODES.S -> {
|
||||
val context = LocalContext.current
|
||||
if (darkTheme) dynamicDarkColorScheme(context) else dynamicLightColorScheme(context)
|
||||
}
|
||||
darkTheme -> DarkColorScheme
|
||||
else -> LightColorScheme
|
||||
}
|
||||
val colorScheme =
|
||||
when {
|
||||
dynamicColor && Build.VERSION.SDK_INT >= Build.VERSION_CODES.S -> {
|
||||
val context = LocalContext.current
|
||||
if (darkTheme) dynamicDarkColorScheme(context) else dynamicLightColorScheme(context)
|
||||
}
|
||||
|
||||
darkTheme -> {
|
||||
DarkColorScheme
|
||||
}
|
||||
|
||||
else -> {
|
||||
LightColorScheme
|
||||
}
|
||||
}
|
||||
|
||||
MaterialTheme(
|
||||
colorScheme = colorScheme,
|
||||
typography = Typography,
|
||||
content = content
|
||||
colorScheme = colorScheme,
|
||||
typography = Typography,
|
||||
content = content,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,14 +7,16 @@ import androidx.compose.ui.text.font.FontWeight
|
|||
import androidx.compose.ui.unit.sp
|
||||
|
||||
// Set of Material typography styles to start with
|
||||
val Typography = Typography(
|
||||
bodyLarge = TextStyle(
|
||||
fontFamily = FontFamily.Default,
|
||||
fontWeight = FontWeight.Normal,
|
||||
fontSize = 16.sp,
|
||||
lineHeight = 24.sp,
|
||||
letterSpacing = 0.5.sp
|
||||
)
|
||||
val Typography =
|
||||
Typography(
|
||||
bodyLarge =
|
||||
TextStyle(
|
||||
fontFamily = FontFamily.Default,
|
||||
fontWeight = FontWeight.Normal,
|
||||
fontSize = 16.sp,
|
||||
lineHeight = 24.sp,
|
||||
letterSpacing = 0.5.sp,
|
||||
),
|
||||
/* Other default text styles to override
|
||||
titleLarge = TextStyle(
|
||||
fontFamily = FontFamily.Default,
|
||||
|
|
@ -30,5 +32,5 @@ val Typography = Typography(
|
|||
lineHeight = 16.sp,
|
||||
letterSpacing = 0.5.sp
|
||||
)
|
||||
*/
|
||||
)
|
||||
*/
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
package com.app.assistant.usecase
|
||||
|
||||
import com.app.assistant.model.DeviceAction
|
||||
import com.app.assistant.repository.ContactsRepository
|
||||
import java.net.URI
|
||||
|
||||
class CallContactUseCase(
|
||||
private val permissionChecker: PermissionChecker,
|
||||
private val contactsRepository: ContactsRepository
|
||||
) {
|
||||
suspend fun execute(
|
||||
prompt: String,
|
||||
onPermissionRequest: suspend (Array<String>) -> Unit,
|
||||
onIntentTriggered: suspend (DeviceAction) -> Unit,
|
||||
onSuccess: suspend (name: String, dialUri: URI) -> Unit,
|
||||
onFailure: suspend (errorMsg: String) -> Unit
|
||||
) {
|
||||
val requiredPermissions = arrayOf(
|
||||
"android.permission.READ_CONTACTS",
|
||||
"android.permission.CALL_PHONE"
|
||||
).filter {
|
||||
!permissionChecker.hasPermission(it)
|
||||
}
|
||||
|
||||
if (requiredPermissions.isNotEmpty()) {
|
||||
onPermissionRequest(requiredPermissions.toTypedArray())
|
||||
return
|
||||
}
|
||||
|
||||
val bestMatch = contactsRepository.searchContact(prompt)
|
||||
if (bestMatch != null) {
|
||||
try {
|
||||
val callAction = DeviceAction.MakeCall(bestMatch.phoneNumber)
|
||||
onIntentTriggered(callAction)
|
||||
onSuccess(bestMatch.name, URI("tel:${bestMatch.phoneNumber.replace(" ", "")}"))
|
||||
} catch (e: Exception) {
|
||||
System.err.println("Error making call: ${e.message}")
|
||||
onFailure("Sorry, failed to make call. Please try again.")
|
||||
}
|
||||
} else {
|
||||
onFailure("I cannot find such contact, please try again.")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,118 @@
|
|||
package com.app.assistant.usecase
|
||||
|
||||
import com.app.assistant.llm.FlexibleLlmAdapter
|
||||
import com.app.assistant.llm.LlmConfig
|
||||
import com.app.assistant.llm.LlmMessage
|
||||
import com.app.assistant.llm.LlmProvider
|
||||
import com.app.assistant.repository.SettingsRepository
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import okhttp3.OkHttpClient
|
||||
|
||||
class GetAiResponseUseCase(
|
||||
private val settingsRepository: SettingsRepository,
|
||||
private val client: OkHttpClient
|
||||
) {
|
||||
private val json = Json { ignoreUnknownKeys = true }
|
||||
|
||||
suspend fun execute(messages: List<LlmMessage>): String? {
|
||||
val providerName = settingsRepository.getLlmProvider()
|
||||
val provider = try {
|
||||
LlmProvider.valueOf(providerName)
|
||||
} catch (e: Exception) {
|
||||
LlmProvider.GROQ
|
||||
}
|
||||
|
||||
var apiKey = settingsRepository.getChatApiKey()
|
||||
if (apiKey.isNullOrBlank() && provider == LlmProvider.GROQ) {
|
||||
apiKey = com.app.assistant.BuildConfig.GROQ_API_KEY
|
||||
}
|
||||
|
||||
val model = settingsRepository.getLlmModel()
|
||||
|
||||
val config = if (provider == LlmProvider.CUSTOM) {
|
||||
LlmConfig(
|
||||
url = settingsRepository.getLlmCustomUrl(),
|
||||
headers = parseHeadersJson(settingsRepository.getLlmCustomHeaders()),
|
||||
responsePath = settingsRepository.getLlmCustomResponsePath(),
|
||||
requestTemplate = settingsRepository.getLlmCustomRequestTemplate(),
|
||||
messageFormat = settingsRepository.getLlmCustomMessageFormat(),
|
||||
systemRole = settingsRepository.getLlmCustomSystemRole().takeIf { it.isNotBlank() },
|
||||
userRole = settingsRepository.getLlmCustomUserRole(),
|
||||
assistantRole = settingsRepository.getLlmCustomAssistantRole()
|
||||
)
|
||||
} else {
|
||||
provider.config
|
||||
}
|
||||
|
||||
val adapter = FlexibleLlmAdapter(
|
||||
client = client,
|
||||
apiKey = apiKey ?: "",
|
||||
model = model,
|
||||
config = config,
|
||||
provider = provider
|
||||
)
|
||||
|
||||
val systemContext = messages.find { it.role == "system" }?.content ?: ""
|
||||
return adapter.generateResponse(systemContext, messages)
|
||||
}
|
||||
|
||||
fun executeStream(messages: List<LlmMessage>): Flow<String> {
|
||||
val providerName = settingsRepository.getLlmProvider()
|
||||
val provider = try {
|
||||
LlmProvider.valueOf(providerName)
|
||||
} catch (e: Exception) {
|
||||
LlmProvider.GROQ
|
||||
}
|
||||
|
||||
var apiKey = settingsRepository.getChatApiKey()
|
||||
if (apiKey.isNullOrBlank() && provider == LlmProvider.GROQ) {
|
||||
apiKey = com.app.assistant.BuildConfig.GROQ_API_KEY
|
||||
}
|
||||
|
||||
val model = settingsRepository.getLlmModel()
|
||||
|
||||
val config = if (provider == LlmProvider.CUSTOM) {
|
||||
LlmConfig(
|
||||
url = settingsRepository.getLlmCustomUrl(),
|
||||
headers = parseHeadersJson(settingsRepository.getLlmCustomHeaders()),
|
||||
responsePath = settingsRepository.getLlmCustomResponsePath(),
|
||||
requestTemplate = settingsRepository.getLlmCustomRequestTemplate(),
|
||||
messageFormat = settingsRepository.getLlmCustomMessageFormat(),
|
||||
systemRole = settingsRepository.getLlmCustomSystemRole().takeIf { it.isNotBlank() },
|
||||
userRole = settingsRepository.getLlmCustomUserRole(),
|
||||
assistantRole = settingsRepository.getLlmCustomAssistantRole()
|
||||
)
|
||||
} else {
|
||||
provider.config
|
||||
}
|
||||
|
||||
val adapter = FlexibleLlmAdapter(
|
||||
client = client,
|
||||
apiKey = apiKey ?: "",
|
||||
model = model,
|
||||
config = config,
|
||||
provider = provider
|
||||
)
|
||||
|
||||
val systemContext = messages.find { it.role == "system" }?.content ?: ""
|
||||
return adapter.generateResponseStream(systemContext, messages)
|
||||
}
|
||||
|
||||
private fun parseHeadersJson(jsonStr: String): Map<String, String> {
|
||||
return try {
|
||||
val element = json.parseToJsonElement(jsonStr)
|
||||
if (element is JsonObject) {
|
||||
element.mapValues { (_, value) ->
|
||||
if (value is JsonPrimitive) value.content else value.toString()
|
||||
}
|
||||
} else {
|
||||
emptyMap()
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
emptyMap()
|
||||
}
|
||||
}
|
||||
}
|
||||
103
app/src/main/java/com/app/assistant/usecase/GetWeatherUseCase.kt
Normal file
103
app/src/main/java/com/app/assistant/usecase/GetWeatherUseCase.kt
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
package com.app.assistant.usecase
|
||||
|
||||
import com.app.assistant.llm.LlmMessage
|
||||
import com.app.assistant.repository.WeatherRepository
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.jsonObject
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.Calendar
|
||||
import java.util.Locale
|
||||
|
||||
class GetWeatherUseCase(
|
||||
private val permissionChecker: PermissionChecker,
|
||||
private val weatherRepository: WeatherRepository,
|
||||
private val getAiResponseUseCase: GetAiResponseUseCase
|
||||
) {
|
||||
suspend fun execute(
|
||||
prompt: String,
|
||||
onPermissionRequest: suspend (Array<String>) -> Unit,
|
||||
onLocationRequest: suspend () -> Unit,
|
||||
onSuccess: suspend (response: String, city: String) -> Unit,
|
||||
onFailure: suspend (errorMsg: String) -> Unit
|
||||
) {
|
||||
try {
|
||||
val location = weatherRepository.extractPlaceName(prompt)
|
||||
if (location.isNotEmpty()) {
|
||||
val coordinates = weatherRepository.getCoordinatesFromCity(location)
|
||||
if (coordinates != null) {
|
||||
val (lat, long) = coordinates
|
||||
val weatherData = weatherRepository.getWeatherData(lat, long)
|
||||
if (weatherData != null) {
|
||||
val weatherJson = Json.parseToJsonElement(weatherData).jsonObject
|
||||
val filteredMap = weatherJson.filterKeys { it != "latitude" && it != "longitude" }
|
||||
val filteredJson = JsonObject(filteredMap)
|
||||
|
||||
val aiResponse = queryWeatherAI(prompt, filteredJson.toString())
|
||||
if (!aiResponse.isNullOrEmpty()) {
|
||||
onSuccess(aiResponse, location)
|
||||
} else {
|
||||
onFailure("Seems weather report is not available, please try again.")
|
||||
}
|
||||
} else {
|
||||
onFailure("Seems weather report is not available, please try again.")
|
||||
}
|
||||
} else {
|
||||
onFailure("I can not find such location, please try again.")
|
||||
}
|
||||
} else {
|
||||
val isPermissionGranted = permissionChecker.hasPermission("android.permission.ACCESS_COARSE_LOCATION")
|
||||
|
||||
if (!isPermissionGranted) {
|
||||
onPermissionRequest(arrayOf("android.permission.ACCESS_COARSE_LOCATION"))
|
||||
} else {
|
||||
onLocationRequest()
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
System.err.println("Error fetching weather: ${e.message}")
|
||||
onFailure("Something went wrong, please try again.")
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun processLocationWeather(
|
||||
lat: Double,
|
||||
long: Double,
|
||||
prompt: String,
|
||||
onSuccess: suspend (response: String, city: String) -> Unit,
|
||||
onFailure: suspend (errorMsg: String) -> Unit
|
||||
) {
|
||||
val city = weatherRepository.getCityNameFromLocation(lat, long) ?: "your location"
|
||||
val weatherData = weatherRepository.getWeatherData(lat, long)
|
||||
if (weatherData != null) {
|
||||
val weatherJson = Json.parseToJsonElement(weatherData).jsonObject
|
||||
val filteredMap = weatherJson.filterKeys { it != "latitude" && it != "longitude" }
|
||||
val filteredJson = JsonObject(filteredMap)
|
||||
|
||||
val aiResponse = queryWeatherAI(prompt, filteredJson.toString())
|
||||
if (!aiResponse.isNullOrEmpty()) {
|
||||
onSuccess(aiResponse, city)
|
||||
} else {
|
||||
onFailure("Seems weather report is not available, please try again.")
|
||||
}
|
||||
} else {
|
||||
onFailure("Seems weather report is not available, please try again.")
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun queryWeatherAI(question: String, weatherData: String): String? {
|
||||
val date = SimpleDateFormat("yyyy-MM-dd", Locale.getDefault(Locale.Category.FORMAT)).format(Calendar.getInstance().time)
|
||||
val time = SimpleDateFormat("HH:mm:ss", Locale.getDefault(Locale.Category.FORMAT)).format(Calendar.getInstance().time)
|
||||
val systemContext =
|
||||
"You are a smart weather assistant, up-to-date with the current date and time, " +
|
||||
"which is $date at $time. Please respond with an answer to the user's question based on " +
|
||||
"the latest weather data provided. No need to mention data or time; just answer naturally in 2 to 3 lines."
|
||||
|
||||
val messages = listOf(
|
||||
LlmMessage(role = "system", content = systemContext),
|
||||
LlmMessage(role = "user", content = question + System.lineSeparator() + weatherData)
|
||||
)
|
||||
|
||||
return getAiResponseUseCase.execute(messages)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
package com.app.assistant.usecase
|
||||
|
||||
import com.app.assistant.model.DeviceAction
|
||||
import java.net.URI
|
||||
|
||||
class NavigateUseCase(private val resourceProvider: ResourceProvider) {
|
||||
suspend fun execute(
|
||||
prompt: String,
|
||||
onIntentTriggered: suspend (DeviceAction) -> Unit,
|
||||
onSuccess: suspend (location: String, navigationUri: URI) -> Unit,
|
||||
onFailure: suspend (errorMsg: String) -> Unit
|
||||
) {
|
||||
try {
|
||||
val sanitizedPrompt = prompt.replace("\\p{Punct}+".toRegex(), "")
|
||||
val location = extractLocation(sanitizedPrompt)
|
||||
if (location != null) {
|
||||
val encodedURI = java.net.URLEncoder.encode(
|
||||
location,
|
||||
java.nio.charset.StandardCharsets.UTF_8.toString()
|
||||
)
|
||||
onIntentTriggered(DeviceAction.NavigateTo(location))
|
||||
onSuccess(location, URI("google.navigation:q=$encodedURI"))
|
||||
} else {
|
||||
onFailure(resourceProvider.getString("location_not_found"))
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
System.err.println("Error navigating: ${e.message}")
|
||||
onFailure(resourceProvider.getString("generic_error"))
|
||||
}
|
||||
}
|
||||
|
||||
private fun extractLocation(command: String): String? {
|
||||
val regex = Regex(
|
||||
"(?<=to |find |show me |give me directions to |navigate me to |" +
|
||||
"how do I get to |I'm on my way to |start a navigation to |" +
|
||||
"where is |help me find |traffic like on the way to |way to )" +
|
||||
"([A-Za-z0-9\\s&]+)",
|
||||
RegexOption.IGNORE_CASE
|
||||
)
|
||||
|
||||
val matchResult = regex.find(command)
|
||||
return matchResult?.value?.trim()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
package com.app.assistant.usecase
|
||||
|
||||
interface PermissionChecker {
|
||||
fun hasPermission(permission: String): Boolean
|
||||
}
|
||||
164
app/src/main/java/com/app/assistant/usecase/PlaySongUseCase.kt
Normal file
164
app/src/main/java/com/app/assistant/usecase/PlaySongUseCase.kt
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
package com.app.assistant.usecase
|
||||
|
||||
import com.app.assistant.BuildConfig
|
||||
import com.app.assistant.model.DeviceAction
|
||||
import com.app.assistant.repository.SettingsRepository
|
||||
import kotlinx.coroutines.CompletableDeferred
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.json.Json
|
||||
import okhttp3.Call
|
||||
import okhttp3.Callback
|
||||
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import okhttp3.Response
|
||||
import java.io.IOException
|
||||
import java.net.URI
|
||||
|
||||
@Serializable
|
||||
data class YouTubeItemId(
|
||||
val kind: String? = null,
|
||||
val videoId: String? = null
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class YouTubeThumbnailDetail(
|
||||
val url: String? = null
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class YouTubeThumbnails(
|
||||
val high: YouTubeThumbnailDetail? = null
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class YouTubeSnippet(
|
||||
val thumbnails: YouTubeThumbnails? = null
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class YouTubeSearchItem(
|
||||
val id: YouTubeItemId? = null,
|
||||
val snippet: YouTubeSnippet? = null
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class YouTubeSearchResponse(
|
||||
val items: List<YouTubeSearchItem>? = null
|
||||
)
|
||||
|
||||
class PlaySongUseCase(
|
||||
private val resourceProvider: ResourceProvider,
|
||||
private val settingsRepository: SettingsRepository,
|
||||
private val client: OkHttpClient
|
||||
) {
|
||||
private val json = Json { ignoreUnknownKeys = true }
|
||||
|
||||
suspend fun execute(
|
||||
prompt: String,
|
||||
onIntentTriggered: suspend (DeviceAction) -> Unit,
|
||||
onSuccess: suspend (songName: String, videoId: String, thumbnailUrl: String, videoUri: URI) -> Unit,
|
||||
onMissingApiKey: suspend (searchQuery: String) -> Unit,
|
||||
onFailure: suspend (errorMsg: String) -> Unit
|
||||
) {
|
||||
try {
|
||||
val sanitizedPrompt = prompt.replace("\\p{Punct}+".toRegex(), "")
|
||||
if (!sanitizedPrompt.lowercase().contains("play ")) {
|
||||
onFailure(resourceProvider.getString("song_not_found"))
|
||||
return
|
||||
}
|
||||
val searchQuery = sanitizedPrompt.lowercase().substringAfter("play").trim()
|
||||
|
||||
val (videoId, thumbnailUrl) = youtubeApiCall(searchQuery)
|
||||
|
||||
if (videoId.isEmpty()) {
|
||||
onFailure(resourceProvider.getString("song_not_found"))
|
||||
} else if (videoId == "Missing API Key") {
|
||||
onIntentTriggered(DeviceAction.SearchSong(searchQuery))
|
||||
onMissingApiKey(searchQuery)
|
||||
} else {
|
||||
onIntentTriggered(DeviceAction.PlaySong(videoId))
|
||||
onSuccess(
|
||||
searchQuery,
|
||||
videoId,
|
||||
thumbnailUrl,
|
||||
URI("https://www.youtube.com/watch?v=$videoId")
|
||||
)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
System.err.println("Error playing song: ${e.message}")
|
||||
onFailure(resourceProvider.getString("generic_error"))
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun youtubeApiCall(query: String): Pair<String, String> {
|
||||
val deferredResult = CompletableDeferred<Pair<String, String>>()
|
||||
|
||||
val urlBuilder = "https://www.googleapis.com/youtube/v3/search"
|
||||
.toHttpUrlOrNull()
|
||||
?.newBuilder()
|
||||
?.addQueryParameter("key", loadYoutubeKey())
|
||||
?.addQueryParameter("q", query)
|
||||
?.addQueryParameter("type", "video")
|
||||
?.addQueryParameter("part", "snippet")
|
||||
?.addQueryParameter("maxResults", "1")
|
||||
?.build()
|
||||
|
||||
val request = Request.Builder()
|
||||
.url(urlBuilder.toString())
|
||||
.build()
|
||||
|
||||
client.newCall(request).enqueue(object : Callback {
|
||||
override fun onFailure(call: Call, e: IOException) {
|
||||
deferredResult.completeExceptionally(e)
|
||||
}
|
||||
|
||||
override fun onResponse(call: Call, response: Response) {
|
||||
if (response.isSuccessful) {
|
||||
val body = response.body?.string()
|
||||
if (body != null) {
|
||||
deferredResult.complete(extractVideoIdAndThumbnail(body))
|
||||
} else {
|
||||
deferredResult.complete(Pair("", ""))
|
||||
}
|
||||
} else {
|
||||
if (response.code == 403) {
|
||||
deferredResult.complete(Pair("Missing API Key", ""))
|
||||
} else {
|
||||
deferredResult.complete(Pair("", ""))
|
||||
}
|
||||
}
|
||||
response.close()
|
||||
}
|
||||
})
|
||||
|
||||
return deferredResult.await()
|
||||
}
|
||||
|
||||
private fun extractVideoIdAndThumbnail(jsonString: String): Pair<String, String> {
|
||||
try {
|
||||
val response = json.decodeFromString<YouTubeSearchResponse>(jsonString)
|
||||
val items = response.items ?: return Pair("", "")
|
||||
if (items.isNotEmpty()) {
|
||||
val firstItem = items[0]
|
||||
val idObject = firstItem.id ?: return Pair("", "")
|
||||
if (idObject.kind == "youtube#video") {
|
||||
val videoId = idObject.videoId ?: ""
|
||||
val thumbnailUrl = firstItem.snippet?.thumbnails?.high?.url ?: ""
|
||||
return Pair(videoId, thumbnailUrl)
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
System.err.println("Error parsing YouTube JSON: ${e.message}")
|
||||
}
|
||||
return Pair("", "")
|
||||
}
|
||||
|
||||
private fun loadYoutubeKey(): String? {
|
||||
var youtubeKey = settingsRepository.getYoutubeApiKey()
|
||||
if (youtubeKey.isNullOrBlank()) {
|
||||
youtubeKey = BuildConfig.YOUTUBE_API_KEY
|
||||
}
|
||||
return youtubeKey
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,193 @@
|
|||
package com.app.assistant.usecase
|
||||
|
||||
import com.app.assistant.llm.LlmMessage
|
||||
import com.app.assistant.model.Conversation
|
||||
import com.app.assistant.repository.SettingsRepository
|
||||
import com.app.assistant.util.Category
|
||||
import com.google.mediapipe.tasks.text.textclassifier.TextClassifierResult
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.coroutines.flow.flow
|
||||
import kotlinx.coroutines.flow.flowOn
|
||||
import kotlinx.coroutines.flow.collect
|
||||
import java.util.Locale
|
||||
|
||||
class ProcessChatCommandUseCase(
|
||||
private val getAiResponseUseCase: GetAiResponseUseCase,
|
||||
private val settingsRepository: SettingsRepository
|
||||
) {
|
||||
fun cleanAndPunctuate(input: String): String {
|
||||
val trimmedInput = input.trim()
|
||||
if (trimmedInput.isEmpty()) return ""
|
||||
|
||||
val lastChar = trimmedInput.last()
|
||||
return if (lastChar == '.' || lastChar == '?') {
|
||||
trimmedInput
|
||||
} else {
|
||||
val questionWords = listOf("wh", "how", "can", "do", "is", "are", "does", "did", "will", "could", "should", "would")
|
||||
if (questionWords.any { trimmedInput.lowercase().startsWith(it) }) {
|
||||
"$trimmedInput?"
|
||||
} else {
|
||||
trimmedInput
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun isNegativeOrNotRequired(phrase: String): Boolean {
|
||||
return negativePatterns.any { it.containsMatchIn(phrase) }
|
||||
}
|
||||
|
||||
fun resolveCategory(results: TextClassifierResult, inputText: String): Category {
|
||||
val categories = results
|
||||
.classificationResult()
|
||||
.classifications()
|
||||
.first()
|
||||
.categories()
|
||||
|
||||
|
||||
|
||||
val highestCategory = categories.maxByOrNull { it.score() }
|
||||
val highestCategoryEnum = highestCategory?.categoryName()?.uppercase()?.let { Category.valueOf(it) }
|
||||
|
||||
val wordsToMatch = categoryWords[highestCategoryEnum] ?: emptyList()
|
||||
val containsWord = wordsToMatch.any { word -> word in inputText.lowercase() }
|
||||
|
||||
val finalCategory = if (wordsToMatch.isNotEmpty() && !containsWord) {
|
||||
categories.firstOrNull { it.categoryName().equals(Category.OTHER.name, ignoreCase = true) }
|
||||
} else {
|
||||
highestCategory
|
||||
}
|
||||
|
||||
return getCategoryFromString(finalCategory?.categoryName()?.uppercase() ?: "")
|
||||
}
|
||||
|
||||
fun getCategoryFromString(categoryName: String): Category =
|
||||
Category.entries.find { it.name == categoryName.uppercase() } ?: Category.OTHER
|
||||
|
||||
suspend fun getAiChatResponse(
|
||||
systemContext: String,
|
||||
chatHistory: List<Conversation>,
|
||||
isHandsFreeActive: Boolean = false,
|
||||
isVisionActive: Boolean = false,
|
||||
isScreenActive: Boolean = false
|
||||
): String? = withContext(Dispatchers.IO) {
|
||||
val messages = mapConversationsToLlmMessages(systemContext, chatHistory, isHandsFreeActive, isVisionActive, isScreenActive)
|
||||
getAiResponseUseCase.execute(messages)
|
||||
}
|
||||
|
||||
fun getAiChatResponseStream(
|
||||
systemContext: String,
|
||||
chatHistory: List<Conversation>,
|
||||
isHandsFreeActive: Boolean = false,
|
||||
isVisionActive: Boolean = false,
|
||||
isScreenActive: Boolean = false
|
||||
): kotlinx.coroutines.flow.Flow<String> = flow {
|
||||
val messages = withContext(Dispatchers.IO) {
|
||||
mapConversationsToLlmMessages(systemContext, chatHistory, isHandsFreeActive, isVisionActive, isScreenActive)
|
||||
}
|
||||
getAiResponseUseCase.executeStream(messages).collect {
|
||||
emit(it)
|
||||
}
|
||||
}.flowOn(Dispatchers.IO)
|
||||
|
||||
private fun mapConversationsToLlmMessages(
|
||||
systemContext: String,
|
||||
chatHistory: List<Conversation>,
|
||||
isHandsFreeActive: Boolean = false,
|
||||
isVisionActive: Boolean = false,
|
||||
isScreenActive: Boolean = false
|
||||
): List<LlmMessage> {
|
||||
val messages = mutableListOf<LlmMessage>()
|
||||
messages.add(LlmMessage(role = "system", content = systemContext))
|
||||
|
||||
val latestImageAttachmentId = if (isHandsFreeActive && (isVisionActive || isScreenActive)) {
|
||||
chatHistory.flatMap { it.attachments }
|
||||
.lastOrNull { it.mimeType.startsWith("image/") }
|
||||
?.id
|
||||
} else {
|
||||
null
|
||||
}
|
||||
|
||||
for (item in chatHistory) {
|
||||
if (item.isLoading) continue
|
||||
val role = if (item.isMe) "user" else "assistant"
|
||||
val llmAttachments = if (item.attachments.isNotEmpty()) {
|
||||
val filtered = item.attachments.filter { att ->
|
||||
if (att.mimeType.startsWith("text/") || att.mimeType == "application/json") {
|
||||
false
|
||||
} else {
|
||||
when {
|
||||
att.mimeType.startsWith("image/") -> {
|
||||
val isSupported = settingsRepository.getIsImageSupported() || att.fileName.startsWith("vision_frame")
|
||||
val isLatest = latestImageAttachmentId == null || att.id == latestImageAttachmentId
|
||||
isSupported && isLatest
|
||||
}
|
||||
att.mimeType.startsWith("audio/") -> settingsRepository.getIsAudioSupported()
|
||||
att.mimeType.startsWith("video/") -> settingsRepository.getIsVideoSupported()
|
||||
att.mimeType.startsWith("application/pdf") -> settingsRepository.getIsDocumentSupported()
|
||||
else -> false
|
||||
}
|
||||
}
|
||||
}
|
||||
if (filtered.isNotEmpty()) {
|
||||
filtered.map { att ->
|
||||
val file = java.io.File(att.filePath)
|
||||
val base64 = if (file.exists()) {
|
||||
val encryptedBytes = file.readBytes()
|
||||
val decryptedBytes = com.app.assistant.db.EncryptionUtil.decryptFile(encryptedBytes, att.iv)
|
||||
android.util.Base64.encodeToString(decryptedBytes, android.util.Base64.NO_WRAP)
|
||||
} else {
|
||||
""
|
||||
}
|
||||
com.app.assistant.llm.LlmAttachment(
|
||||
base64Data = base64,
|
||||
mimeType = att.mimeType,
|
||||
fileName = att.fileName
|
||||
)
|
||||
}
|
||||
} else null
|
||||
} else null
|
||||
messages.add(LlmMessage(role = role, content = item.text, attachments = llmAttachments))
|
||||
}
|
||||
return messages
|
||||
}
|
||||
|
||||
companion object {
|
||||
private val negativePatterns = listOf(
|
||||
Regex("\\bno\\b", RegexOption.IGNORE_CASE),
|
||||
Regex("\\bnah\\b", RegexOption.IGNORE_CASE),
|
||||
Regex("\\bnot\\b.*", RegexOption.IGNORE_CASE),
|
||||
Regex("\\bnever\\b.*", RegexOption.IGNORE_CASE),
|
||||
Regex("\\bforget\\b.*", RegexOption.IGNORE_CASE),
|
||||
Regex("\\bleave\\b.*", RegexOption.IGNORE_CASE),
|
||||
Regex("\\bdrop\\b.*", RegexOption.IGNORE_CASE),
|
||||
Regex("\\bcancel\\b", RegexOption.IGNORE_CASE),
|
||||
Regex("\\babort\\b", RegexOption.IGNORE_CASE),
|
||||
Regex("\\bstop\\b", RegexOption.IGNORE_CASE),
|
||||
Regex("\\bquit\\b", RegexOption.IGNORE_CASE),
|
||||
Regex("\\bdisregard\\b", RegexOption.IGNORE_CASE),
|
||||
Regex("changed my mind", RegexOption.IGNORE_CASE),
|
||||
Regex("don't bother", RegexOption.IGNORE_CASE),
|
||||
Regex("let it go", RegexOption.IGNORE_CASE),
|
||||
Regex("scratch that", RegexOption.IGNORE_CASE),
|
||||
Regex("just kidding", RegexOption.IGNORE_CASE)
|
||||
)
|
||||
|
||||
private val categoryWords = mapOf(
|
||||
Category.CALL to listOf("call", "phone", "ring", "connect", "need", "get", "dial"),
|
||||
Category.SONGS to listOf("music", "song", "play", "tune", "listen"),
|
||||
Category.ALARM to listOf("alarm", "wake", "set", "remind", "morning"),
|
||||
Category.REMINDER to listOf("remind", "notify", "alert", "remember"),
|
||||
Category.NAVIGATION to listOf(
|
||||
"navigate", "find", "show", "directions", "take me to",
|
||||
"where is", "get to", "go to", "navigation to", "way"
|
||||
),
|
||||
Category.WEATHER to listOf(
|
||||
"sunny", "cloudy", "rain", "temperature", "umbrella", "weather",
|
||||
"outside", "report", "forecast", "carry", "going to", "how is", "will it be"
|
||||
),
|
||||
Category.SETTINGS to emptyList(),
|
||||
Category.OTHER to emptyList()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
package com.app.assistant.usecase
|
||||
|
||||
interface ResourceProvider {
|
||||
fun getString(key: String): String
|
||||
}
|
||||
148
app/src/main/java/com/app/assistant/usecase/SetAlarmUseCase.kt
Normal file
148
app/src/main/java/com/app/assistant/usecase/SetAlarmUseCase.kt
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
package com.app.assistant.usecase
|
||||
|
||||
import com.app.assistant.model.DeviceAction
|
||||
import java.util.Calendar
|
||||
import java.util.Locale
|
||||
|
||||
class SetAlarmUseCase(private val resourceProvider: ResourceProvider) {
|
||||
suspend fun execute(
|
||||
prompt: String,
|
||||
dayOverride: String? = null,
|
||||
onPromptForTime: suspend (dayMatch: String?) -> Unit,
|
||||
onSuccess: suspend (action: DeviceAction) -> Unit,
|
||||
onFailure: suspend (errorMsg: String) -> Unit
|
||||
) {
|
||||
try {
|
||||
val sanitizedPrompt = prompt.replace(PunctuationRegex, "")
|
||||
|
||||
val dayMatch = dayOverride ?: DayRegex.find(sanitizedPrompt)?.value?.lowercase(Locale.ROOT)
|
||||
val timeMatch = TimeRegex.find(sanitizedPrompt)
|
||||
val relativeTimeMatch = RelativeTimeRegex.find(sanitizedPrompt)
|
||||
|
||||
if (timeMatch == null && relativeTimeMatch == null) {
|
||||
onPromptForTime(dayMatch)
|
||||
} else {
|
||||
val action = calculateAlarmAction(dayMatch, timeMatch, relativeTimeMatch, resourceProvider.getString("new_alarm_message"))
|
||||
onSuccess(action)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
System.err.println("Error setting alarm: ${e.message}")
|
||||
onFailure(resourceProvider.getString("generic_error"))
|
||||
}
|
||||
}
|
||||
|
||||
private fun calculateAlarmAction(
|
||||
dayMatch: String?,
|
||||
timeMatch: MatchResult?,
|
||||
relativeTimeMatch: MatchResult?,
|
||||
message: String
|
||||
): DeviceAction {
|
||||
val calendar = Calendar.getInstance()
|
||||
|
||||
if (relativeTimeMatch != null) {
|
||||
val firstValue = relativeTimeMatch.groupValues[1].takeIf { it.isNotEmpty() }?.toIntOrNull() ?: 0
|
||||
val halfValue = if (relativeTimeMatch.groupValues[2] == "half") 0.5 else 0.0
|
||||
val firstUnit = relativeTimeMatch.groupValues[3].lowercase(Locale.ROOT)
|
||||
|
||||
var secondValue = 0
|
||||
var secondUnit = ""
|
||||
|
||||
if (relativeTimeMatch.groupValues[4].isNotEmpty()) {
|
||||
secondValue = relativeTimeMatch.groupValues[4].toInt()
|
||||
secondUnit = relativeTimeMatch.groupValues[5].lowercase(Locale.ROOT)
|
||||
}
|
||||
|
||||
when {
|
||||
firstUnit.contains("hour") || firstUnit.contains("hrs") -> {
|
||||
calendar.add(Calendar.HOUR_OF_DAY, firstValue)
|
||||
if (halfValue > 0) {
|
||||
calendar.add(Calendar.MINUTE, 30)
|
||||
}
|
||||
}
|
||||
firstUnit.contains("min") -> {
|
||||
calendar.add(Calendar.MINUTE, firstValue)
|
||||
}
|
||||
}
|
||||
|
||||
when {
|
||||
secondUnit.contains("hour") || secondUnit.contains("hrs") -> calendar.add(Calendar.HOUR_OF_DAY, secondValue)
|
||||
secondUnit.contains("minute") || secondUnit.contains("minutes") || secondUnit.contains("min") -> calendar.add(Calendar.MINUTE, secondValue)
|
||||
}
|
||||
} else if (timeMatch != null) {
|
||||
val hour = timeMatch.groupValues[1].toInt()
|
||||
val minutes = timeMatch.groupValues[3].toIntOrNull() ?: 0
|
||||
val amPm = timeMatch.groupValues[4].uppercase(Locale.ROOT)
|
||||
|
||||
val hour24 = when {
|
||||
amPm == "PM" && hour != 12 -> hour + 12
|
||||
amPm == "AM" && hour == 12 -> 0
|
||||
else -> hour
|
||||
}
|
||||
|
||||
calendar.set(Calendar.HOUR_OF_DAY, hour24)
|
||||
calendar.set(Calendar.MINUTE, minutes)
|
||||
}
|
||||
|
||||
val repeatDays = mutableListOf<Int>()
|
||||
|
||||
when {
|
||||
dayMatch == "today" -> { /* No change needed */ }
|
||||
dayMatch == "tomorrow" -> {
|
||||
calendar.add(Calendar.DAY_OF_MONTH, 1)
|
||||
}
|
||||
dayMatch == "next week" -> {
|
||||
calendar.add(Calendar.DAY_OF_MONTH, 7)
|
||||
}
|
||||
dayMatch == "next weekend" -> {
|
||||
val currentDayOfWeek = calendar.get(Calendar.DAY_OF_WEEK)
|
||||
val daysUntilSaturday = (Calendar.SATURDAY - currentDayOfWeek + 7) % 7
|
||||
calendar.add(Calendar.DAY_OF_MONTH, daysUntilSaturday)
|
||||
}
|
||||
dayMatch != null -> {
|
||||
val dayMap = mapOf(
|
||||
"sunday" to Calendar.SUNDAY,
|
||||
"monday" to Calendar.MONDAY,
|
||||
"tuesday" to Calendar.TUESDAY,
|
||||
"wednesday" to Calendar.WEDNESDAY,
|
||||
"thursday" to Calendar.THURSDAY,
|
||||
"friday" to Calendar.FRIDAY,
|
||||
"saturday" to Calendar.SATURDAY
|
||||
)
|
||||
|
||||
dayMap.entries.forEach { (dayName, dayConstant) ->
|
||||
if (dayMatch.contains(dayName)) {
|
||||
repeatDays.add(dayConstant)
|
||||
}
|
||||
}
|
||||
|
||||
if (repeatDays.isEmpty()) {
|
||||
val targetDay = dayMap.entries.firstOrNull { dayMatch.contains(it.key) }?.value
|
||||
if (targetDay != null) {
|
||||
val currentDayOfWeek = calendar.get(Calendar.DAY_OF_WEEK)
|
||||
val daysToAdd = (targetDay - currentDayOfWeek + 7) % 7
|
||||
calendar.add(Calendar.DAY_OF_MONTH, if (daysToAdd == 0) 7 else daysToAdd)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return DeviceAction.SetAlarm(
|
||||
message = message,
|
||||
hour = calendar.get(Calendar.HOUR_OF_DAY),
|
||||
minutes = calendar.get(Calendar.MINUTE),
|
||||
repeatDays = repeatDays,
|
||||
isReminder = false
|
||||
)
|
||||
}
|
||||
|
||||
companion object {
|
||||
private val PunctuationRegex = "[.?!]+".toRegex()
|
||||
private val DayRegex = ("(?i)\\b(today|tomorrow|next week|next weekend|" +
|
||||
"((Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday)" +
|
||||
"( morning| evening| night| afternoon)?))\\b").toRegex(RegexOption.IGNORE_CASE)
|
||||
private val TimeRegex = "(?i)\\b(\\d{1,2})(:(\\d{2}))?\\s*(AM|PM)?\\b".toRegex(RegexOption.IGNORE_CASE)
|
||||
private val RelativeTimeRegex = ("(?i)\\b(\\d+)?\\s*(?:and\\s*(half))?\\s*" +
|
||||
"(hours?|hrs?|mins?|minutes?|seconds?|secs?)\\s*(?:and\\s*(\\d+))?\\s*" +
|
||||
"(mins?|minutes?|secs?|seconds?)?\\s*(from now|later|next)?\\b").toRegex(RegexOption.IGNORE_CASE)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,153 @@
|
|||
package com.app.assistant.usecase
|
||||
|
||||
import com.app.assistant.model.DeviceAction
|
||||
import java.util.Calendar
|
||||
import java.util.Locale
|
||||
|
||||
class SetReminderUseCase(private val resourceProvider: ResourceProvider) {
|
||||
suspend fun execute(
|
||||
prompt: String,
|
||||
dayOverride: String? = null,
|
||||
contextOverride: String? = null,
|
||||
onPromptForTime: suspend (dayMatch: String?, context: String) -> Unit,
|
||||
onSuccess: suspend (action: DeviceAction) -> Unit,
|
||||
onFailure: suspend (errorMsg: String) -> Unit
|
||||
) {
|
||||
try {
|
||||
val sanitizedPrompt = prompt.replace(PunctuationRegex, "")
|
||||
|
||||
val contextMatch = ContextRegex.find(sanitizedPrompt)
|
||||
val context = contextOverride ?: contextMatch?.groupValues?.get(1)?.trim() ?: resourceProvider.getString("reminder_default_context")
|
||||
|
||||
val timeMatch = TimeRegex.find(sanitizedPrompt)
|
||||
val relativeTimeMatch = RelativeTimeRegex.find(sanitizedPrompt)
|
||||
val dayMatch = dayOverride ?: DayRegex.find(sanitizedPrompt)?.value?.lowercase(Locale.ROOT)
|
||||
|
||||
if (timeMatch == null && relativeTimeMatch == null) {
|
||||
onPromptForTime(dayMatch, context)
|
||||
} else {
|
||||
val action = calculateReminderAction(dayMatch, timeMatch, relativeTimeMatch, context)
|
||||
onSuccess(action)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
System.err.println("Error setting reminder: ${e.message}")
|
||||
onFailure(resourceProvider.getString("generic_error"))
|
||||
}
|
||||
}
|
||||
|
||||
private fun calculateReminderAction(
|
||||
dayMatch: String?,
|
||||
timeMatch: MatchResult?,
|
||||
relativeTimeMatch: MatchResult?,
|
||||
context: String
|
||||
): DeviceAction {
|
||||
val calendar = Calendar.getInstance()
|
||||
|
||||
if (relativeTimeMatch != null) {
|
||||
val firstValue = relativeTimeMatch.groupValues[1].takeIf { it.isNotEmpty() }?.toIntOrNull() ?: 0
|
||||
val halfValue = if (relativeTimeMatch.groupValues[2] == "half") 0.5 else 0.0
|
||||
val firstUnit = relativeTimeMatch.groupValues[3].lowercase(Locale.ROOT)
|
||||
|
||||
var secondValue = 0
|
||||
var secondUnit = ""
|
||||
|
||||
if (relativeTimeMatch.groupValues[4].isNotEmpty()) {
|
||||
secondValue = relativeTimeMatch.groupValues[4].toInt()
|
||||
secondUnit = relativeTimeMatch.groupValues[5].lowercase(Locale.ROOT)
|
||||
}
|
||||
|
||||
when {
|
||||
firstUnit.contains("hour") || firstUnit.contains("hrs") -> {
|
||||
calendar.add(Calendar.HOUR_OF_DAY, firstValue)
|
||||
if (halfValue > 0) {
|
||||
calendar.add(Calendar.MINUTE, 30)
|
||||
}
|
||||
}
|
||||
firstUnit.contains("min") -> {
|
||||
calendar.add(Calendar.MINUTE, firstValue)
|
||||
}
|
||||
}
|
||||
|
||||
when {
|
||||
secondUnit.contains("hour") || secondUnit.contains("hrs") -> calendar.add(Calendar.HOUR_OF_DAY, secondValue)
|
||||
secondUnit.contains("minute") || secondUnit.contains("minutes") || secondUnit.contains("min") -> calendar.add(Calendar.MINUTE, secondValue)
|
||||
}
|
||||
} else if (timeMatch != null) {
|
||||
val hour = timeMatch.groupValues[1].toInt()
|
||||
val minutes = timeMatch.groupValues[3].toIntOrNull() ?: 0
|
||||
val amPm = timeMatch.groupValues[4].uppercase(Locale.ROOT)
|
||||
|
||||
val hour24 = when {
|
||||
amPm == "PM" && hour != 12 -> hour + 12
|
||||
amPm == "AM" && hour == 12 -> 0
|
||||
else -> hour
|
||||
}
|
||||
|
||||
calendar.set(Calendar.HOUR_OF_DAY, hour24)
|
||||
calendar.set(Calendar.MINUTE, minutes)
|
||||
}
|
||||
|
||||
val repeatDays = mutableListOf<Int>()
|
||||
|
||||
when {
|
||||
dayMatch == "today" -> { /* No change needed */ }
|
||||
dayMatch == "tomorrow" -> {
|
||||
calendar.add(Calendar.DAY_OF_MONTH, 1)
|
||||
}
|
||||
dayMatch == "next week" -> {
|
||||
calendar.add(Calendar.DAY_OF_MONTH, 7)
|
||||
}
|
||||
dayMatch == "next weekend" -> {
|
||||
val currentDayOfWeek = calendar.get(Calendar.DAY_OF_WEEK)
|
||||
val daysUntilSaturday = (Calendar.SATURDAY - currentDayOfWeek + 7) % 7
|
||||
calendar.add(Calendar.DAY_OF_MONTH, daysUntilSaturday)
|
||||
}
|
||||
dayMatch != null -> {
|
||||
val dayMap = mapOf(
|
||||
"sunday" to Calendar.SUNDAY,
|
||||
"monday" to Calendar.MONDAY,
|
||||
"tuesday" to Calendar.TUESDAY,
|
||||
"wednesday" to Calendar.WEDNESDAY,
|
||||
"thursday" to Calendar.THURSDAY,
|
||||
"friday" to Calendar.FRIDAY,
|
||||
"saturday" to Calendar.SATURDAY
|
||||
)
|
||||
|
||||
dayMap.entries.forEach { (dayName, dayConstant) ->
|
||||
if (dayMatch.contains(dayName)) {
|
||||
repeatDays.add(dayConstant)
|
||||
}
|
||||
}
|
||||
|
||||
if (repeatDays.isEmpty()) {
|
||||
val targetDay = dayMap.entries.firstOrNull { dayMatch.contains(it.key) }?.value
|
||||
if (targetDay != null) {
|
||||
val currentDayOfWeek = calendar.get(Calendar.DAY_OF_WEEK)
|
||||
val daysToAdd = (targetDay - currentDayOfWeek + 7) % 7
|
||||
calendar.add(Calendar.DAY_OF_MONTH, if (daysToAdd == 0) 7 else daysToAdd)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return DeviceAction.SetAlarm(
|
||||
message = context,
|
||||
hour = calendar.get(Calendar.HOUR_OF_DAY),
|
||||
minutes = calendar.get(Calendar.MINUTE),
|
||||
repeatDays = repeatDays,
|
||||
isReminder = true
|
||||
)
|
||||
}
|
||||
|
||||
companion object {
|
||||
private val PunctuationRegex = "[.?!]+".toRegex()
|
||||
private val DayRegex = ("(?i)\\b(today|tomorrow|next week|next weekend|" +
|
||||
"((Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday)" +
|
||||
"( morning| evening| night| afternoon)?))\\b").toRegex(RegexOption.IGNORE_CASE)
|
||||
private val TimeRegex = "(?i)\\b(\\d{1,2})(:(\\d{2}))?\\s*(AM|PM)?\\b".toRegex(RegexOption.IGNORE_CASE)
|
||||
private val RelativeTimeRegex = ("(?i)\\b(\\d+)?\\s*(?:and\\s*(half))?\\s*" +
|
||||
"(hours?|hrs?|mins?|minutes?|seconds?|secs?)\\s*(?:and\\s*(\\d+))?\\s*" +
|
||||
"(mins?|minutes?|secs?|seconds?)?\\s*(from now|later|next)?\\b").toRegex(RegexOption.IGNORE_CASE)
|
||||
private val ContextRegex = "remind me(?: to| for| of| about| to create)?\\s+(.*?)(?:\\s+(at|on|in)\\s+.*)?$".toRegex(RegexOption.IGNORE_CASE)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,12 @@
|
|||
package com.app.assistant.util
|
||||
|
||||
enum class Category {
|
||||
SONGS, CALL, ALARM, REMINDER, OTHER, NAVIGATION, SETTINGS, WEATHER
|
||||
}
|
||||
SONGS,
|
||||
CALL,
|
||||
ALARM,
|
||||
REMINDER,
|
||||
OTHER,
|
||||
NAVIGATION,
|
||||
SETTINGS,
|
||||
WEATHER,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,12 @@
|
|||
package com.app.assistant.util
|
||||
|
||||
object Constants {
|
||||
const val MAIN_CONTEXT = "You are my smart assistant who answers just like normal human response, not too long. Don't refer yourself as an AI."
|
||||
val CATEGORY_CONTEXT = "You are a assistant who can classify commands based on previous conversation, from these categories. "+ enumValues<Category>().joinToString(",") { it.name } +". Just say the category name."
|
||||
}
|
||||
const val MAIN_CONTEXT =
|
||||
"You are my smart assistant who answers just like normal human response, not too long. Don't refer yourself as an AI."
|
||||
const val HANDS_FREE_MODE_IMAGE_CONTEXT =
|
||||
"You are my smart assistant who answers just like normal human response, not too long. Don't refer yourself as an AI and don't mention words like image, photo, picture, screen in your response. "
|
||||
val CATEGORY_CONTEXT =
|
||||
"You are a assistant who can classify commands based on previous conversation, from these categories. " +
|
||||
enumValues<Category>().joinToString(",") { it.name } +
|
||||
". Just say the category name."
|
||||
}
|
||||
|
|
|
|||
18
app/src/main/java/com/app/assistant/util/IdGenerator.kt
Normal file
18
app/src/main/java/com/app/assistant/util/IdGenerator.kt
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
package com.app.assistant.util
|
||||
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
object IdGenerator {
|
||||
private val lastId = AtomicLong(0L)
|
||||
|
||||
fun nextId(): Long {
|
||||
while (true) {
|
||||
val now = System.currentTimeMillis()
|
||||
val last = lastId.get()
|
||||
val next = if (now > last) now else last + 1
|
||||
if (lastId.compareAndSet(last, next)) {
|
||||
return next
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
24
app/src/main/java/com/app/assistant/util/LocaleUtils.kt
Normal file
24
app/src/main/java/com/app/assistant/util/LocaleUtils.kt
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
package com.app.assistant.util
|
||||
|
||||
import java.util.Locale
|
||||
|
||||
object LocaleUtils {
|
||||
fun getLocaleCode(languageCode: String): String {
|
||||
val languageCountryMapping = mapOf(
|
||||
"af" to "ZA", "sq" to "AL", "ar" to "SA", "be" to "BY", "bg" to "BG",
|
||||
"bn" to "BD", "ca" to "ES", "zh" to "CN", "hr" to "HR", "cs" to "CZ",
|
||||
"da" to "DK", "nl" to "NL", "en" to "US", "eo" to "EO", "et" to "EE",
|
||||
"fi" to "FI", "fr" to "FR", "gl" to "ES", "ka" to "GE", "de" to "DE",
|
||||
"el" to "GR", "gu" to "IN", "ht" to "HT", "he" to "IL", "hi" to "IN",
|
||||
"hu" to "HU", "is" to "IS", "id" to "ID", "ga" to "IE", "it" to "IT",
|
||||
"ja" to "JP", "kn" to "IN", "ko" to "KR", "lt" to "LT", "lv" to "LV",
|
||||
"mk" to "MK", "mr" to "IN", "ms" to "MY", "mt" to "MT", "no" to "NO",
|
||||
"fa" to "IR", "pl" to "PL", "pt" to "BR", "ro" to "RO", "ru" to "RU",
|
||||
"sk" to "SK", "sl" to "SI", "es" to "ES", "sv" to "SE", "sw" to "KE",
|
||||
"tl" to "PH", "ta" to "IN", "te" to "IN", "th" to "TH", "tr" to "TR",
|
||||
"uk" to "UA", "ur" to "PK", "vi" to "VN", "cy" to "GB"
|
||||
)
|
||||
val countryCode = languageCountryMapping[languageCode] ?: Locale.getDefault().country
|
||||
return "$languageCode-$countryCode"
|
||||
}
|
||||
}
|
||||
|
|
@ -4,14 +4,19 @@ sealed class LockState {
|
|||
object None : LockState() // Default state when no task is locked
|
||||
|
||||
data class LockAlarm(
|
||||
var day: String? = null, // Already processed day (e.g., "tomorrow")
|
||||
var time: String? = null // Time to be provided
|
||||
// Already processed day (e.g., "tomorrow")
|
||||
var day: String? = null,
|
||||
// Time to be provided
|
||||
var time: String? = null,
|
||||
) : LockState()
|
||||
|
||||
data class LockReminder(
|
||||
var day: String? = null, // Already processed day (e.g., "tomorrow")
|
||||
var time: String? = null, // Time to be provided
|
||||
var context: String? = null // Message for reminder
|
||||
// Already processed day (e.g., "tomorrow")
|
||||
var day: String? = null,
|
||||
// Time to be provided
|
||||
var time: String? = null,
|
||||
// Message for reminder
|
||||
var context: String? = null,
|
||||
) : LockState()
|
||||
|
||||
object LockNavigation : LockState() // No additional data needed
|
||||
|
|
|
|||
250
app/src/main/java/com/app/assistant/util/MarkdownUtils.kt
Normal file
250
app/src/main/java/com/app/assistant/util/MarkdownUtils.kt
Normal file
|
|
@ -0,0 +1,250 @@
|
|||
package com.app.assistant.util
|
||||
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.text.AnnotatedString
|
||||
import androidx.compose.ui.text.SpanStyle
|
||||
import androidx.compose.ui.text.font.FontFamily
|
||||
import androidx.compose.ui.text.font.FontStyle
|
||||
import androidx.compose.ui.text.font.FontWeight
|
||||
import androidx.compose.ui.text.style.TextDecoration
|
||||
import androidx.compose.ui.unit.sp
|
||||
import org.commonmark.node.BlockQuote
|
||||
import org.commonmark.node.BulletList
|
||||
import org.commonmark.node.Code
|
||||
import org.commonmark.node.Emphasis
|
||||
import org.commonmark.node.FencedCodeBlock
|
||||
import org.commonmark.node.HardLineBreak
|
||||
import org.commonmark.node.Heading
|
||||
import org.commonmark.node.IndentedCodeBlock
|
||||
import org.commonmark.node.Link
|
||||
import org.commonmark.node.ListItem
|
||||
import org.commonmark.node.Node
|
||||
import org.commonmark.node.OrderedList
|
||||
import org.commonmark.node.Paragraph
|
||||
import org.commonmark.node.SoftLineBreak
|
||||
import org.commonmark.node.StrongEmphasis
|
||||
import org.commonmark.node.Text
|
||||
import org.commonmark.node.ThematicBreak
|
||||
import org.commonmark.parser.Parser
|
||||
|
||||
object MarkdownUtils {
|
||||
private val HeaderRegex = Regex("#+ ")
|
||||
private val BoldRegex = Regex("\\*\\*(.*?)\\*\\*")
|
||||
private val ItalicRegex = Regex("\\*(.*?)\\*")
|
||||
private val ImageRegex = Regex("!\\[.*?\\]\\(.*?\\)")
|
||||
private val LinkRegex = Regex("\\[.*?\\]\\(.*?\\)")
|
||||
private val CodeRegex = Regex("`{1,3}.*?`{1,3}")
|
||||
private val NewlinesRegex = Regex("\\n\\n")
|
||||
|
||||
private val parser = Parser.builder().build()
|
||||
|
||||
fun markdownToPlainText(markdown: String): String {
|
||||
var plainText = markdown
|
||||
plainText = plainText.replace(HeaderRegex, "") // Remove headers
|
||||
plainText = plainText.replace(BoldRegex, "$1") // Bold
|
||||
plainText = plainText.replace(ItalicRegex, "$1") // Italic
|
||||
plainText = plainText.replace(ImageRegex, "") // Images
|
||||
plainText = plainText.replace(LinkRegex, "") // Links
|
||||
plainText = plainText.replace(CodeRegex, "") // Inline code
|
||||
plainText = plainText.replace(NewlinesRegex, "\n") // Simplify newlines
|
||||
return plainText.trim()
|
||||
}
|
||||
|
||||
fun parseMarkdownToAnnotatedString(
|
||||
markdown: String,
|
||||
linkColor: Color = Color.Blue
|
||||
): AnnotatedString {
|
||||
val document = parser.parse(markdown)
|
||||
val builder = AnnotatedString.Builder()
|
||||
processNodes(document, builder, linkColor)
|
||||
return builder.toAnnotatedString()
|
||||
}
|
||||
|
||||
private fun processNodes(
|
||||
node: Node,
|
||||
builder: AnnotatedString.Builder,
|
||||
linkColor: Color,
|
||||
) {
|
||||
var child = node.firstChild
|
||||
while (child != null) {
|
||||
val nextSibling = child.next
|
||||
when (child) {
|
||||
is Text -> {
|
||||
builder.append(child.literal)
|
||||
}
|
||||
|
||||
is Emphasis -> {
|
||||
val start = builder.length
|
||||
processNodes(child, builder, linkColor)
|
||||
val end = builder.length
|
||||
builder.addStyle(SpanStyle(fontStyle = FontStyle.Italic), start, end)
|
||||
}
|
||||
|
||||
is StrongEmphasis -> {
|
||||
val start = builder.length
|
||||
processNodes(child, builder, linkColor)
|
||||
val end = builder.length
|
||||
builder.addStyle(SpanStyle(fontWeight = FontWeight.Bold), start, end)
|
||||
}
|
||||
|
||||
is Code -> {
|
||||
// Inline code (single backticks)
|
||||
val start = builder.length
|
||||
builder.append(child.literal)
|
||||
val end = builder.length
|
||||
builder.addStyle(
|
||||
SpanStyle(fontFamily = FontFamily.Monospace),
|
||||
start,
|
||||
end,
|
||||
)
|
||||
}
|
||||
|
||||
is Link -> {
|
||||
val start = builder.length
|
||||
processNodes(child, builder, linkColor)
|
||||
val end = builder.length
|
||||
builder.addStringAnnotation(
|
||||
tag = "URL",
|
||||
annotation = child.destination,
|
||||
start = start,
|
||||
end = end,
|
||||
)
|
||||
builder.addStyle(
|
||||
SpanStyle(color = linkColor, textDecoration = TextDecoration.Underline),
|
||||
start,
|
||||
end,
|
||||
)
|
||||
}
|
||||
|
||||
is Paragraph -> {
|
||||
processNodes(child, builder, linkColor)
|
||||
if (nextSibling != null) {
|
||||
builder.append("\n")
|
||||
}
|
||||
}
|
||||
|
||||
is Heading -> {
|
||||
if (builder.isNotEmpty() && !builder.toString().endsWith("\n")) {
|
||||
builder.append("\n")
|
||||
}
|
||||
val start = builder.length
|
||||
processNodes(child, builder, linkColor)
|
||||
val end = builder.length
|
||||
val headingStyle =
|
||||
when (child.level) {
|
||||
1 -> SpanStyle(fontWeight = FontWeight.Bold, fontSize = 28.sp)
|
||||
2 -> SpanStyle(fontWeight = FontWeight.Bold, fontSize = 24.sp)
|
||||
3 -> SpanStyle(fontWeight = FontWeight.Bold, fontSize = 20.sp)
|
||||
4 -> SpanStyle(fontWeight = FontWeight.Bold, fontSize = 18.sp)
|
||||
5 -> SpanStyle(fontWeight = FontWeight.Bold, fontSize = 16.sp)
|
||||
else -> SpanStyle(fontWeight = FontWeight.Bold, fontSize = 14.sp)
|
||||
}
|
||||
builder.addStyle(headingStyle, start, end)
|
||||
if (nextSibling != null) {
|
||||
builder.append("\n")
|
||||
}
|
||||
}
|
||||
|
||||
is BlockQuote -> {
|
||||
val start = builder.length
|
||||
processNodes(child, builder, linkColor)
|
||||
val end = builder.length
|
||||
builder.addStyle(
|
||||
SpanStyle(color = Color.Gray, fontStyle = FontStyle.Italic),
|
||||
start,
|
||||
end,
|
||||
)
|
||||
if (nextSibling != null) {
|
||||
builder.append("\n")
|
||||
}
|
||||
}
|
||||
|
||||
is BulletList -> {
|
||||
var listItem = child.firstChild
|
||||
while (listItem != null) {
|
||||
if (listItem is ListItem) {
|
||||
builder.append("• ")
|
||||
processNodes(listItem, builder, linkColor)
|
||||
builder.append("\n")
|
||||
}
|
||||
listItem = listItem.next
|
||||
}
|
||||
if (nextSibling != null) {
|
||||
builder.append("\n")
|
||||
}
|
||||
}
|
||||
|
||||
is OrderedList -> {
|
||||
var index = child.startNumber
|
||||
var listItem = child.firstChild
|
||||
while (listItem != null) {
|
||||
if (listItem is ListItem) {
|
||||
builder.append("$index. ")
|
||||
processNodes(listItem, builder, linkColor)
|
||||
builder.append("\n")
|
||||
index++
|
||||
}
|
||||
listItem = listItem.next
|
||||
}
|
||||
if (nextSibling != null) {
|
||||
builder.append("\n")
|
||||
}
|
||||
}
|
||||
|
||||
is FencedCodeBlock -> {
|
||||
val start = builder.length
|
||||
builder.append(child.literal)
|
||||
val end = builder.length
|
||||
builder.addStyle(
|
||||
SpanStyle(
|
||||
fontFamily = FontFamily.Monospace,
|
||||
),
|
||||
start,
|
||||
end,
|
||||
)
|
||||
if (nextSibling != null) {
|
||||
builder.append("\n")
|
||||
}
|
||||
}
|
||||
|
||||
is IndentedCodeBlock -> {
|
||||
val start = builder.length
|
||||
builder.append(child.literal)
|
||||
val end = builder.length
|
||||
builder.addStyle(
|
||||
SpanStyle(
|
||||
fontFamily = FontFamily.Monospace,
|
||||
),
|
||||
start,
|
||||
end,
|
||||
)
|
||||
if (nextSibling != null) {
|
||||
builder.append("\n")
|
||||
}
|
||||
}
|
||||
|
||||
is ThematicBreak -> {
|
||||
builder.append("\n----------------\n")
|
||||
if (nextSibling != null) {
|
||||
builder.append("\n")
|
||||
}
|
||||
}
|
||||
|
||||
is SoftLineBreak -> {
|
||||
builder.append(" ")
|
||||
}
|
||||
|
||||
is HardLineBreak -> {
|
||||
builder.append("\n")
|
||||
}
|
||||
|
||||
else -> {
|
||||
processNodes(child, builder, linkColor)
|
||||
}
|
||||
}
|
||||
child = nextSibling
|
||||
}
|
||||
}
|
||||
|
||||
private fun AnnotatedString.Builder.isNotEmpty(): Boolean = this.length > 0
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
package com.app.assistant.viewmodel
|
||||
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.content.pm.PackageManager
|
||||
import android.net.Uri
|
||||
import android.provider.AlarmClock
|
||||
import androidx.core.content.ContextCompat
|
||||
import com.app.assistant.R
|
||||
import com.app.assistant.model.DeviceAction
|
||||
import com.app.assistant.usecase.PermissionChecker
|
||||
import com.app.assistant.usecase.ResourceProvider
|
||||
|
||||
class AndroidPermissionChecker(private val context: Context) : PermissionChecker {
|
||||
override fun hasPermission(permission: String): Boolean {
|
||||
return ContextCompat.checkSelfPermission(context, permission) == PackageManager.PERMISSION_GRANTED
|
||||
}
|
||||
}
|
||||
|
||||
class AndroidResourceProvider(private val context: Context) : ResourceProvider {
|
||||
override fun getString(key: String): String {
|
||||
return when (key) {
|
||||
"song_not_found" -> context.getString(R.string.song_not_found_fallback)
|
||||
"generic_error" -> context.getString(R.string.generic_error_fallback)
|
||||
"location_not_found" -> context.getString(R.string.location_not_found_fallback)
|
||||
"new_alarm_message" -> context.getString(R.string.new_alarm_message)
|
||||
"reminder_default_context" -> context.getString(R.string.reminder_default_context)
|
||||
else -> ""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun DeviceAction.toIntent(): Intent {
|
||||
return when (this) {
|
||||
is DeviceAction.MakeCall -> Intent(Intent.ACTION_CALL).apply {
|
||||
addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
|
||||
data = Uri.parse("tel:$phoneNumber")
|
||||
}
|
||||
is DeviceAction.NavigateTo -> {
|
||||
val gmmIntentUri = Uri.parse("google.navigation:q=$location")
|
||||
Intent(Intent.ACTION_VIEW, gmmIntentUri).apply {
|
||||
addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
|
||||
setPackage("com.google.android.apps.maps")
|
||||
}
|
||||
}
|
||||
is DeviceAction.PlaySong -> Intent(Intent.ACTION_VIEW, Uri.parse("https://www.youtube.com/watch?v=$videoId")).apply {
|
||||
addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
|
||||
}
|
||||
is DeviceAction.SearchSong -> Intent(Intent.ACTION_SEARCH).apply {
|
||||
setPackage("com.google.android.youtube")
|
||||
putExtra("query", query)
|
||||
addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
|
||||
}
|
||||
is DeviceAction.SetAlarm -> Intent(AlarmClock.ACTION_SET_ALARM).apply {
|
||||
putExtra(AlarmClock.EXTRA_MESSAGE, message)
|
||||
putExtra(AlarmClock.EXTRA_HOUR, hour)
|
||||
putExtra(AlarmClock.EXTRA_MINUTES, minutes)
|
||||
putExtra(AlarmClock.EXTRA_SKIP_UI, true)
|
||||
if (isReminder) {
|
||||
putExtra(AlarmClock.EXTRA_RINGTONE, AlarmClock.VALUE_RINGTONE_SILENT)
|
||||
}
|
||||
if (repeatDays.isNotEmpty()) {
|
||||
putExtra(AlarmClock.EXTRA_DAYS, ArrayList(repeatDays))
|
||||
}
|
||||
addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,323 @@
|
|||
package com.app.assistant.viewmodel
|
||||
|
||||
import android.content.Intent
|
||||
import android.app.Application
|
||||
import androidx.lifecycle.viewModelScope
|
||||
import com.app.assistant.model.Conversation
|
||||
import com.app.assistant.util.Category
|
||||
import com.app.assistant.util.LockState
|
||||
import com.app.assistant.util.Constants.MAIN_CONTEXT
|
||||
import kotlinx.coroutines.launch
|
||||
import java.net.URI
|
||||
|
||||
// Call Contact UseCase Delegate
|
||||
fun MainViewModel.callContact(
|
||||
itemId: Long,
|
||||
loadingItemId: Long,
|
||||
speak: Boolean,
|
||||
category: Category,
|
||||
) {
|
||||
if (chatList.isNotEmpty()) {
|
||||
val prompt = chatList.find { it.id == itemId }?.text ?: return
|
||||
viewModelScope.launch {
|
||||
callContactUseCase.execute(
|
||||
prompt = prompt,
|
||||
onPermissionRequest = { permissions ->
|
||||
_uiEvent.send(UIEvent.RequestPermissions(permissions, 102))
|
||||
processResponse(getRandomResponse(ResponseStrings.permissionContactsCall(getApplication())), loadingItemId, speak, Category.OTHER)
|
||||
},
|
||||
onIntentTriggered = { action ->
|
||||
_uiEvent.send(UIEvent.StartIntent(action.toIntent()))
|
||||
},
|
||||
onSuccess = { name, dialUri ->
|
||||
processResponse(
|
||||
name,
|
||||
loadingItemId,
|
||||
false,
|
||||
category,
|
||||
navigationURI = dialUri
|
||||
)
|
||||
},
|
||||
onFailure = { errorMsg ->
|
||||
processResponse(errorMsg, loadingItemId, speak, Category.OTHER)
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Play Song UseCase Delegate
|
||||
fun MainViewModel.playSong(
|
||||
itemId: Long,
|
||||
loadingItemId: Long,
|
||||
speak: Boolean,
|
||||
category: Category,
|
||||
) {
|
||||
val prompt = chatList.find { it.id == itemId }?.text ?: return
|
||||
viewModelScope.launch {
|
||||
playSongUseCase.execute(
|
||||
prompt = prompt,
|
||||
onIntentTriggered = { action ->
|
||||
_uiEvent.send(UIEvent.StartIntent(action.toIntent()))
|
||||
},
|
||||
onSuccess = { songName, videoId, thumbnailUrl, videoUri ->
|
||||
processResponse(
|
||||
getApplication<Application>().getString(com.app.assistant.R.string.playing_song, songName),
|
||||
loadingItemId,
|
||||
speak,
|
||||
category,
|
||||
thumbnailUrl,
|
||||
videoUri
|
||||
)
|
||||
},
|
||||
onMissingApiKey = { searchQuery ->
|
||||
processResponse(getApplication<Application>().getString(com.app.assistant.R.string.youtube_key_missing), loadingItemId, speak, Category.OTHER)
|
||||
},
|
||||
onFailure = { errorMsg ->
|
||||
processResponse(errorMsg, loadingItemId, speak, Category.OTHER)
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Navigate UseCase Delegate
|
||||
fun MainViewModel.navigate(
|
||||
itemId: Long,
|
||||
loadingItemId: Long,
|
||||
speak: Boolean,
|
||||
category: Category,
|
||||
) {
|
||||
val prompt = chatList.find { it.id == itemId }?.text ?: return
|
||||
viewModelScope.launch {
|
||||
navigateUseCase.execute(
|
||||
prompt = prompt,
|
||||
onIntentTriggered = { action ->
|
||||
_uiEvent.send(UIEvent.StartIntent(action.toIntent()))
|
||||
},
|
||||
onSuccess = { location, navigationUri ->
|
||||
processResponse(
|
||||
getApplication<Application>().getString(com.app.assistant.R.string.navigating_to, location),
|
||||
loadingItemId,
|
||||
speak,
|
||||
category,
|
||||
navigationURI = navigationUri
|
||||
)
|
||||
},
|
||||
onFailure = { errorMsg ->
|
||||
processResponse(errorMsg, loadingItemId, speak, Category.OTHER)
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Weather UseCase Delegate
|
||||
fun MainViewModel.fetchWeather(
|
||||
itemId: Long,
|
||||
loadingItemId: Long,
|
||||
speak: Boolean,
|
||||
category: Category,
|
||||
) {
|
||||
val prompt = chatList.find { it.id == itemId }?.text ?: return
|
||||
viewModelScope.launch {
|
||||
getWeatherUseCase.execute(
|
||||
prompt = prompt,
|
||||
onPermissionRequest = { permissions ->
|
||||
_uiEvent.send(UIEvent.RequestPermissions(permissions, 103))
|
||||
processResponse(getRandomResponse(ResponseStrings.permissionLocation(getApplication())), loadingItemId, speak, Category.OTHER)
|
||||
},
|
||||
onLocationRequest = {
|
||||
_uiEvent.send(
|
||||
UIEvent.GetLocationForWeather(
|
||||
itemId,
|
||||
loadingItemId,
|
||||
speak,
|
||||
category.name,
|
||||
prompt
|
||||
)
|
||||
)
|
||||
},
|
||||
onSuccess = { response, location ->
|
||||
processResponse(
|
||||
response,
|
||||
loadingItemId,
|
||||
speak,
|
||||
category,
|
||||
navigationURI = URI("https://www.google.com/search?q=weather+$location")
|
||||
)
|
||||
},
|
||||
onFailure = { errorMsg ->
|
||||
processResponse(errorMsg, loadingItemId, speak, Category.OTHER)
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fun MainViewModel.onLocationReceived(
|
||||
lat: Double,
|
||||
long: Double,
|
||||
itemId: Long,
|
||||
loadingItemId: Long,
|
||||
speak: Boolean,
|
||||
categoryName: String,
|
||||
prompt: String,
|
||||
) {
|
||||
viewModelScope.launch {
|
||||
getWeatherUseCase.processLocationWeather(
|
||||
lat = lat,
|
||||
long = long,
|
||||
prompt = prompt,
|
||||
onSuccess = { response, city ->
|
||||
processResponse(
|
||||
response,
|
||||
loadingItemId,
|
||||
speak,
|
||||
Category.valueOf(categoryName),
|
||||
navigationURI = URI("https://www.google.com/search?q=weather+${
|
||||
java.net.URLEncoder.encode(city, java.nio.charset.StandardCharsets.UTF_8.toString())
|
||||
}")
|
||||
)
|
||||
},
|
||||
onFailure = { errorMsg ->
|
||||
processResponse(errorMsg, loadingItemId, speak, Category.OTHER)
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fun MainViewModel.onLocationFailed(
|
||||
loadingItemId: Long,
|
||||
speak: Boolean,
|
||||
errorType: String,
|
||||
) {
|
||||
viewModelScope.launch {
|
||||
val responseText = when (errorType) {
|
||||
"GPS_OFF" -> getRandomResponse(ResponseStrings.locationServiceOff(getApplication()))
|
||||
"UNAVAILABLE" -> getRandomResponse(ResponseStrings.weatherReportUnavailable(getApplication()))
|
||||
else -> getRandomResponse(ResponseStrings.locationUnknownSuggestCity(getApplication()))
|
||||
}
|
||||
processResponse(responseText, loadingItemId, speak, Category.OTHER)
|
||||
}
|
||||
}
|
||||
|
||||
// Set Alarm UseCase Delegate
|
||||
fun MainViewModel.setAlarm(
|
||||
itemId: Long,
|
||||
loadingItemId: Long,
|
||||
speak: Boolean,
|
||||
category: Category,
|
||||
) {
|
||||
val prompt = chatList.find { it.id == itemId }?.text ?: return
|
||||
viewModelScope.launch {
|
||||
setAlarmUseCase.execute(
|
||||
prompt = prompt,
|
||||
onPromptForTime = { dayMatch ->
|
||||
lockState = LockState.LockAlarm(day = dayMatch)
|
||||
processResponse(
|
||||
getRandomResponse(ResponseStrings.promptForTime(getApplication())),
|
||||
loadingItemId,
|
||||
speak,
|
||||
Category.OTHER
|
||||
)
|
||||
},
|
||||
onSuccess = { action ->
|
||||
_uiEvent.send(UIEvent.StartIntent(action.toIntent()))
|
||||
processResponse(
|
||||
getRandomResponse(ResponseStrings.alarmSetSuccess(getApplication())),
|
||||
loadingItemId,
|
||||
speak,
|
||||
category
|
||||
)
|
||||
},
|
||||
onFailure = { errorMsg ->
|
||||
processResponse(errorMsg, loadingItemId, speak, Category.OTHER)
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fun MainViewModel.handleAlarmLockState(
|
||||
itemId: Long,
|
||||
loadingItemId: Long,
|
||||
speak: Boolean,
|
||||
state: LockState.LockAlarm,
|
||||
) {
|
||||
val prompt = chatList.find { it.id == itemId }?.text ?: return
|
||||
viewModelScope.launch {
|
||||
setAlarmUseCase.execute(
|
||||
prompt = prompt,
|
||||
dayOverride = state.day,
|
||||
onPromptForTime = {
|
||||
processResponse(getRandomResponse(ResponseStrings.invalidTime(getApplication())), loadingItemId, speak, Category.OTHER)
|
||||
},
|
||||
onSuccess = { action ->
|
||||
lockState = LockState.None
|
||||
_uiEvent.send(UIEvent.StartIntent(action.toIntent()))
|
||||
processResponse(getRandomResponse(ResponseStrings.alarmSetSuccess(getApplication())), loadingItemId, speak, Category.ALARM)
|
||||
},
|
||||
onFailure = { errorMsg ->
|
||||
processResponse(errorMsg, loadingItemId, speak, Category.OTHER)
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Set Reminder UseCase Delegate
|
||||
fun MainViewModel.setReminder(
|
||||
itemId: Long,
|
||||
loadingItemId: Long,
|
||||
speak: Boolean,
|
||||
category: Category,
|
||||
) {
|
||||
val prompt = chatList.find { it.id == itemId }?.text ?: return
|
||||
viewModelScope.launch {
|
||||
setReminderUseCase.execute(
|
||||
prompt = prompt,
|
||||
onPromptForTime = { dayMatch, context ->
|
||||
lockState = LockState.LockReminder(day = dayMatch, context = context)
|
||||
processResponse(getRandomResponse(ResponseStrings.promptForTime(getApplication())), loadingItemId, speak, Category.OTHER)
|
||||
},
|
||||
onSuccess = { action ->
|
||||
_uiEvent.send(UIEvent.StartIntent(action.toIntent()))
|
||||
processResponse(getRandomResponse(ResponseStrings.reminderSetSuccess(getApplication())), loadingItemId, speak, category)
|
||||
},
|
||||
onFailure = { errorMsg ->
|
||||
processResponse(errorMsg, loadingItemId, speak, Category.OTHER)
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fun MainViewModel.handleReminderLockState(
|
||||
itemId: Long,
|
||||
loadingItemId: Long,
|
||||
speak: Boolean,
|
||||
state: LockState.LockReminder,
|
||||
) {
|
||||
val prompt = chatList.find { it.id == itemId }?.text ?: return
|
||||
viewModelScope.launch {
|
||||
setReminderUseCase.execute(
|
||||
prompt = prompt,
|
||||
dayOverride = state.day,
|
||||
contextOverride = state.context,
|
||||
onPromptForTime = { _, _ ->
|
||||
processResponse(getRandomResponse(ResponseStrings.invalidTime(getApplication())), loadingItemId, speak, Category.OTHER)
|
||||
},
|
||||
onSuccess = { action ->
|
||||
lockState = LockState.None
|
||||
_uiEvent.send(UIEvent.StartIntent(action.toIntent()))
|
||||
processResponse(getRandomResponse(ResponseStrings.reminderSetSuccess(getApplication())), loadingItemId, speak, Category.ALARM)
|
||||
},
|
||||
onFailure = { errorMsg ->
|
||||
processResponse(errorMsg, loadingItemId, speak, Category.OTHER)
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fun MainViewModel.callAI(
|
||||
loadingItemId: Long,
|
||||
speak: Boolean,
|
||||
category: Category,
|
||||
) {
|
||||
this.callAI(loadingItemId, speak, category)
|
||||
}
|
||||
|
|
@ -3,17 +3,72 @@ package com.app.assistant.viewmodel
|
|||
import android.app.Application
|
||||
import androidx.lifecycle.ViewModel
|
||||
import androidx.lifecycle.ViewModelProvider
|
||||
import com.app.assistant.db.DynamicConversationRepository
|
||||
import com.app.assistant.repository.ContactsRepository
|
||||
import com.app.assistant.repository.SettingsRepository
|
||||
import com.app.assistant.repository.WeatherRepository
|
||||
import com.app.assistant.usecase.CallContactUseCase
|
||||
import com.app.assistant.usecase.GetAiResponseUseCase
|
||||
import com.app.assistant.usecase.GetWeatherUseCase
|
||||
import com.app.assistant.usecase.NavigateUseCase
|
||||
import com.app.assistant.usecase.PlaySongUseCase
|
||||
import com.app.assistant.usecase.ProcessChatCommandUseCase
|
||||
import com.app.assistant.usecase.SetAlarmUseCase
|
||||
import com.app.assistant.usecase.SetReminderUseCase
|
||||
import okhttp3.OkHttpClient
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
class MainViewModelFactory(
|
||||
private val application: Application,
|
||||
private val speak: Boolean
|
||||
private val speak: Boolean,
|
||||
) : ViewModelProvider.Factory {
|
||||
|
||||
override fun <T : ViewModel> create(modelClass: Class<T>): T {
|
||||
if (modelClass.isAssignableFrom(MainViewModel::class.java)) {
|
||||
val settingsRepo = SettingsRepository(application)
|
||||
val contactsRepo = ContactsRepository(application)
|
||||
val dbRepo = DynamicConversationRepository(application)
|
||||
|
||||
val okHttpClient = okHttpClient
|
||||
|
||||
val weatherRepo = WeatherRepository(application, okHttpClient)
|
||||
|
||||
val permissionChecker = AndroidPermissionChecker(application)
|
||||
val resourceProvider = AndroidResourceProvider(application)
|
||||
|
||||
val getAiResponseUseCase = GetAiResponseUseCase(settingsRepo, okHttpClient)
|
||||
val callContactUseCase = CallContactUseCase(permissionChecker, contactsRepo)
|
||||
val playSongUseCase = PlaySongUseCase(resourceProvider, settingsRepo, okHttpClient)
|
||||
val navigateUseCase = NavigateUseCase(resourceProvider)
|
||||
val getWeatherUseCase = GetWeatherUseCase(permissionChecker, weatherRepo, getAiResponseUseCase)
|
||||
val setAlarmUseCase = SetAlarmUseCase(resourceProvider)
|
||||
val setReminderUseCase = SetReminderUseCase(resourceProvider)
|
||||
val processChatCommandUseCase = ProcessChatCommandUseCase(getAiResponseUseCase, settingsRepo)
|
||||
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
return MainViewModel(application, speak) as T
|
||||
return MainViewModel(
|
||||
application,
|
||||
speak,
|
||||
settingsRepo,
|
||||
dbRepo,
|
||||
callContactUseCase,
|
||||
playSongUseCase,
|
||||
navigateUseCase,
|
||||
getWeatherUseCase,
|
||||
setAlarmUseCase,
|
||||
setReminderUseCase,
|
||||
processChatCommandUseCase
|
||||
) as T
|
||||
}
|
||||
throw IllegalArgumentException("Unknown ViewModel class")
|
||||
}
|
||||
|
||||
companion object {
|
||||
val okHttpClient: OkHttpClient by lazy {
|
||||
OkHttpClient.Builder()
|
||||
.connectTimeout(60, TimeUnit.SECONDS)
|
||||
.readTimeout(60, TimeUnit.SECONDS)
|
||||
.writeTimeout(60, TimeUnit.SECONDS)
|
||||
.build()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,51 @@
|
|||
package com.app.assistant.viewmodel
|
||||
|
||||
import android.content.Context
|
||||
import com.app.assistant.R
|
||||
|
||||
fun getRandomResponse(responses: List<String>): String =
|
||||
responses.randomOrNull() ?: responses.firstOrNull() ?: "An unexpected error occurred."
|
||||
|
||||
object ResponseStrings {
|
||||
fun permissionContactsCall(context: Context): List<String> =
|
||||
context.resources.getStringArray(R.array.permission_contacts_call).toList()
|
||||
|
||||
fun permissionLocation(context: Context): List<String> =
|
||||
context.resources.getStringArray(R.array.permission_location).toList()
|
||||
|
||||
fun locationServiceOff(context: Context): List<String> =
|
||||
context.resources.getStringArray(R.array.location_service_off).toList()
|
||||
|
||||
fun callFailed(context: Context): List<String> =
|
||||
context.resources.getStringArray(R.array.call_failed).toList()
|
||||
|
||||
fun contactNotFound(context: Context): List<String> =
|
||||
context.resources.getStringArray(R.array.contact_not_found).toList()
|
||||
|
||||
fun songNotFound(context: Context): List<String> =
|
||||
context.resources.getStringArray(R.array.song_not_found).toList()
|
||||
|
||||
fun locationNotFound(context: Context): List<String> =
|
||||
context.resources.getStringArray(R.array.location_not_found).toList()
|
||||
|
||||
fun weatherReportUnavailable(context: Context): List<String> =
|
||||
context.resources.getStringArray(R.array.weather_report_unavailable).toList()
|
||||
|
||||
fun locationUnknownSuggestCity(context: Context): List<String> =
|
||||
context.resources.getStringArray(R.array.location_unknown_suggest_city).toList()
|
||||
|
||||
fun invalidTime(context: Context): List<String> =
|
||||
context.resources.getStringArray(R.array.invalid_time).toList()
|
||||
|
||||
fun alarmSetSuccess(context: Context): List<String> =
|
||||
context.resources.getStringArray(R.array.alarm_set_success).toList()
|
||||
|
||||
fun reminderSetSuccess(context: Context): List<String> =
|
||||
context.resources.getStringArray(R.array.reminder_set_success).toList()
|
||||
|
||||
fun promptForTime(context: Context): List<String> =
|
||||
context.resources.getStringArray(R.array.prompt_for_time).toList()
|
||||
|
||||
fun genericError(context: Context): List<String> =
|
||||
context.resources.getStringArray(R.array.generic_error).toList()
|
||||
}
|
||||
|
|
@ -0,0 +1,559 @@
|
|||
package com.app.assistant.viewmodel
|
||||
|
||||
import androidx.lifecycle.ViewModel
|
||||
import androidx.lifecycle.viewModelScope
|
||||
import com.app.assistant.BuildConfig
|
||||
import com.app.assistant.llm.LlmConfig
|
||||
import com.app.assistant.llm.LlmProvider
|
||||
import com.app.assistant.llm.ModelCapabilities
|
||||
import com.app.assistant.llm.ModelCapabilityProber
|
||||
import com.app.assistant.repository.SettingsRepository
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.flow.asStateFlow
|
||||
import kotlinx.coroutines.launch
|
||||
import okhttp3.OkHttpClient
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import java.io.File
|
||||
import android.util.Log
|
||||
import java.io.IOException
|
||||
import com.google.mlkit.common.model.DownloadConditions
|
||||
import com.google.mlkit.common.model.RemoteModelManager
|
||||
import com.google.mlkit.nl.translate.TranslateLanguage
|
||||
import com.google.mlkit.nl.translate.TranslateRemoteModel
|
||||
import java.lang.reflect.Modifier
|
||||
import android.content.Context
|
||||
import android.app.role.RoleManager
|
||||
import android.content.ComponentName
|
||||
import android.provider.Settings
|
||||
import android.os.Build
|
||||
|
||||
sealed class DownloadState {
|
||||
object Idle : DownloadState()
|
||||
data class Downloading(val progress: Float, val currentBytes: Long, val totalBytes: Long) : DownloadState()
|
||||
object Completed : DownloadState()
|
||||
data class Error(val message: String) : DownloadState()
|
||||
}
|
||||
|
||||
sealed class VerificationState {
|
||||
object Idle : VerificationState()
|
||||
object Verifying : VerificationState()
|
||||
data class Success(val capabilities: ModelCapabilities) : VerificationState()
|
||||
data class Error(val message: String) : VerificationState()
|
||||
}
|
||||
|
||||
class SettingsViewModel(
|
||||
private val settingsRepository: SettingsRepository,
|
||||
private val okHttpClient: OkHttpClient
|
||||
) : ViewModel() {
|
||||
|
||||
private val modelManager = com.app.assistant.speech.SpeechModelManager(settingsRepository.context)
|
||||
private val ttsModelManager = com.app.assistant.tts.TtsModelManager(settingsRepository.context)
|
||||
|
||||
private val _modelDownloadState = MutableStateFlow<DownloadState>(DownloadState.Idle)
|
||||
val modelDownloadState: StateFlow<DownloadState> = _modelDownloadState.asStateFlow()
|
||||
|
||||
private val _isModelInstalled = MutableStateFlow(modelManager.isModelDownloaded())
|
||||
|
||||
private val _isTranslationEnabled = MutableStateFlow(settingsRepository.getIsTranslationEnabled())
|
||||
val isTranslationEnabled: StateFlow<Boolean> = _isTranslationEnabled.asStateFlow()
|
||||
|
||||
private val _activeLanguageCode = MutableStateFlow(settingsRepository.getActiveLanguageCode())
|
||||
val activeLanguageCode: StateFlow<String> = _activeLanguageCode.asStateFlow()
|
||||
|
||||
private val _downloadedTranslationLanguages = MutableStateFlow<Set<String>>(emptySet())
|
||||
val downloadedTranslationLanguages: StateFlow<Set<String>> = _downloadedTranslationLanguages.asStateFlow()
|
||||
|
||||
private val _downloadingTranslationLanguages = MutableStateFlow<Set<String>>(emptySet())
|
||||
val downloadingTranslationLanguages: StateFlow<Set<String>> = _downloadingTranslationLanguages.asStateFlow()
|
||||
|
||||
var translationLanguages: List<Pair<String, String>> = emptyList()
|
||||
private set
|
||||
|
||||
private val _isDefaultAssistant = MutableStateFlow(false)
|
||||
val isDefaultAssistant: StateFlow<Boolean> = _isDefaultAssistant.asStateFlow()
|
||||
|
||||
init {
|
||||
translationLanguages = getPublicStaticFinalStringsWithNames(TranslateLanguage::class.java)
|
||||
refreshDownloadedTranslationModels()
|
||||
checkDefaultAssistantStatus()
|
||||
}
|
||||
|
||||
fun checkDefaultAssistantStatus() {
|
||||
val context = settingsRepository.context
|
||||
val isDefault = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
|
||||
val roleManager = context.getSystemService(Context.ROLE_SERVICE) as? RoleManager
|
||||
roleManager?.isRoleHeld(RoleManager.ROLE_ASSISTANT) == true
|
||||
} else {
|
||||
val assistantSetting = Settings.Secure.getString(context.contentResolver, "assistant")
|
||||
if (assistantSetting.isNullOrEmpty()) {
|
||||
false
|
||||
} else {
|
||||
val currentAssistant = ComponentName.unflattenFromString(assistantSetting)
|
||||
val myAssistant = ComponentName(context, com.app.assistant.AssistActivity::class.java)
|
||||
currentAssistant == myAssistant
|
||||
}
|
||||
}
|
||||
_isDefaultAssistant.value = isDefault
|
||||
}
|
||||
|
||||
val isModelInstalled: StateFlow<Boolean> = _isModelInstalled.asStateFlow()
|
||||
|
||||
private val _ttsModelDownloadState = MutableStateFlow<DownloadState>(DownloadState.Idle)
|
||||
val ttsModelDownloadState: StateFlow<DownloadState> = _ttsModelDownloadState.asStateFlow()
|
||||
|
||||
private val _isTtsModelInstalled = MutableStateFlow(ttsModelManager.isModelDownloaded())
|
||||
val isTtsModelInstalled: StateFlow<Boolean> = _isTtsModelInstalled.asStateFlow()
|
||||
|
||||
private val _verificationState = MutableStateFlow<VerificationState>(VerificationState.Idle)
|
||||
val verificationState: StateFlow<VerificationState> = _verificationState.asStateFlow()
|
||||
|
||||
private val _fetchedModels = MutableStateFlow<Map<LlmProvider, List<String>>>(emptyMap())
|
||||
val fetchedModels: StateFlow<Map<LlmProvider, List<String>>> = _fetchedModels.asStateFlow()
|
||||
|
||||
private val _isFetchingModels = MutableStateFlow(false)
|
||||
val isFetchingModels: StateFlow<Boolean> = _isFetchingModels.asStateFlow()
|
||||
|
||||
private val _modelFetchError = MutableStateFlow<String?>(null)
|
||||
val modelFetchError: StateFlow<String?> = _modelFetchError.asStateFlow()
|
||||
|
||||
fun resetVerificationState() {
|
||||
_verificationState.value = VerificationState.Idle
|
||||
}
|
||||
|
||||
fun loadYoutubeKey(): String {
|
||||
val key = settingsRepository.getYoutubeApiKey()
|
||||
return if (key.isNullOrBlank()) BuildConfig.YOUTUBE_API_KEY else key
|
||||
}
|
||||
|
||||
fun loadChatKey(): String {
|
||||
val key = settingsRepository.getChatApiKey()
|
||||
return if (key.isNullOrBlank()) BuildConfig.GROQ_API_KEY else key
|
||||
}
|
||||
|
||||
fun loadEdgeTtsSubscriptionKey(): String {
|
||||
val key = settingsRepository.getEdgeTtsSubscriptionKey()
|
||||
return if (key.isNullOrBlank()) BuildConfig.EDGE_TTS_SUBSCRIPTION_KEY else key
|
||||
}
|
||||
|
||||
fun loadLlmProvider(): LlmProvider {
|
||||
val name = settingsRepository.getLlmProvider()
|
||||
return try {
|
||||
LlmProvider.valueOf(name)
|
||||
} catch (e: Exception) {
|
||||
LlmProvider.GROQ
|
||||
}
|
||||
}
|
||||
|
||||
fun loadLlmModel(): String = settingsRepository.getLlmModel()
|
||||
|
||||
fun loadLlmCustomUrl(): String = settingsRepository.getLlmCustomUrl()
|
||||
fun loadLlmCustomHeaders(): String = settingsRepository.getLlmCustomHeaders()
|
||||
fun loadLlmCustomResponsePath(): String = settingsRepository.getLlmCustomResponsePath()
|
||||
fun loadLlmCustomRequestTemplate(): String = settingsRepository.getLlmCustomRequestTemplate()
|
||||
fun loadLlmCustomMessageFormat(): String = settingsRepository.getLlmCustomMessageFormat()
|
||||
fun loadLlmCustomSystemRole(): String = settingsRepository.getLlmCustomSystemRole()
|
||||
fun loadLlmCustomUserRole(): String = settingsRepository.getLlmCustomUserRole()
|
||||
fun loadLlmCustomAssistantRole(): String = settingsRepository.getLlmCustomAssistantRole()
|
||||
fun loadUseLocalWhisper(): Boolean = settingsRepository.getUseLocalWhisper()
|
||||
fun loadSttMode(): com.app.assistant.speech.SttMode = settingsRepository.getSttMode()
|
||||
|
||||
fun loadTtsMode(): com.app.assistant.tts.TtsMode = settingsRepository.getTtsMode()
|
||||
|
||||
|
||||
fun deleteModel() {
|
||||
viewModelScope.launch(Dispatchers.IO) {
|
||||
modelManager.deleteLocalModel()
|
||||
_isModelInstalled.value = false
|
||||
val currentMode = settingsRepository.getSttMode()
|
||||
if (currentMode != com.app.assistant.speech.SttMode.NATIVE) {
|
||||
settingsRepository.setSttMode(com.app.assistant.speech.SttMode.NATIVE)
|
||||
}
|
||||
_modelDownloadState.value = DownloadState.Idle
|
||||
}
|
||||
}
|
||||
|
||||
fun cancelModelDownload() {
|
||||
modelManager.cancelDownload()
|
||||
_modelDownloadState.value = DownloadState.Idle
|
||||
}
|
||||
|
||||
fun startModelDownload() {
|
||||
viewModelScope.launch {
|
||||
_modelDownloadState.value = DownloadState.Downloading(0f, 0L, modelManager.totalSizeBytes)
|
||||
modelManager.downloadModel(
|
||||
client = okHttpClient,
|
||||
onProgress = { downloadedBytes, totalBytes ->
|
||||
val progress = if (totalBytes > 0) downloadedBytes.toFloat() / totalBytes else 0f
|
||||
_modelDownloadState.value = DownloadState.Downloading(progress, downloadedBytes, totalBytes)
|
||||
},
|
||||
onComplete = { success, errorMsg ->
|
||||
viewModelScope.launch {
|
||||
if (success) {
|
||||
_modelDownloadState.value = DownloadState.Completed
|
||||
_isModelInstalled.value = true
|
||||
} else {
|
||||
_modelDownloadState.value = DownloadState.Error(errorMsg ?: "Unknown download error")
|
||||
_isModelInstalled.value = modelManager.isModelDownloaded()
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fun startTtsModelDownload() {
|
||||
viewModelScope.launch {
|
||||
_ttsModelDownloadState.value = DownloadState.Downloading(0f, 0L, 0L)
|
||||
ttsModelManager.downloadModel(
|
||||
client = okHttpClient,
|
||||
onProgress = { downloadedBytes, totalBytes ->
|
||||
val progress = if (totalBytes > 0) downloadedBytes.toFloat() / totalBytes else 0f
|
||||
_ttsModelDownloadState.value = DownloadState.Downloading(progress, downloadedBytes, totalBytes)
|
||||
},
|
||||
onComplete = { success, errorMsg ->
|
||||
viewModelScope.launch {
|
||||
if (success) {
|
||||
_ttsModelDownloadState.value = DownloadState.Completed
|
||||
_isTtsModelInstalled.value = true
|
||||
} else {
|
||||
_ttsModelDownloadState.value = DownloadState.Error(errorMsg ?: "Unknown download error")
|
||||
_isTtsModelInstalled.value = ttsModelManager.isModelDownloaded()
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fun cancelTtsModelDownload() {
|
||||
ttsModelManager.cancelDownload()
|
||||
_ttsModelDownloadState.value = DownloadState.Idle
|
||||
}
|
||||
|
||||
fun deleteTtsModel() {
|
||||
viewModelScope.launch(Dispatchers.IO) {
|
||||
ttsModelManager.deleteLocalModel()
|
||||
_isTtsModelInstalled.value = false
|
||||
_ttsModelDownloadState.value = DownloadState.Idle
|
||||
val currentMode = settingsRepository.getTtsMode()
|
||||
if (currentMode != com.app.assistant.tts.TtsMode.NATIVE) {
|
||||
settingsRepository.setTtsMode(com.app.assistant.tts.TtsMode.NATIVE)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun saveSettings(
|
||||
youtubeApiKey: String,
|
||||
chatApiKey: String,
|
||||
edgeTtsSubscriptionKey: String,
|
||||
provider: LlmProvider,
|
||||
model: String,
|
||||
customUrl: String,
|
||||
customHeaders: String,
|
||||
customResponsePath: String,
|
||||
customRequestTemplate: String,
|
||||
customMessageFormat: String,
|
||||
customSystemRole: String,
|
||||
customUserRole: String,
|
||||
customAssistantRole: String,
|
||||
sttMode: com.app.assistant.speech.SttMode,
|
||||
ttsMode: com.app.assistant.tts.TtsMode
|
||||
) {
|
||||
settingsRepository.saveKeys(youtubeApiKey, chatApiKey, edgeTtsSubscriptionKey)
|
||||
settingsRepository.setLlmProvider(provider.name)
|
||||
settingsRepository.setLlmModel(model)
|
||||
settingsRepository.setLlmCustomUrl(customUrl)
|
||||
settingsRepository.setLlmCustomHeaders(customHeaders)
|
||||
settingsRepository.setLlmCustomResponsePath(customResponsePath)
|
||||
settingsRepository.setLlmCustomRequestTemplate(customRequestTemplate)
|
||||
settingsRepository.setLlmCustomMessageFormat(customMessageFormat)
|
||||
settingsRepository.setLlmCustomSystemRole(customSystemRole)
|
||||
settingsRepository.setLlmCustomUserRole(customUserRole)
|
||||
settingsRepository.setLlmCustomAssistantRole(customAssistantRole)
|
||||
settingsRepository.setSttMode(sttMode)
|
||||
settingsRepository.setTtsMode(ttsMode)
|
||||
}
|
||||
|
||||
fun updateYoutubeApiKey(key: String) {
|
||||
settingsRepository.saveKeys(
|
||||
youtubeApiKey = key,
|
||||
chatApiKey = settingsRepository.getChatApiKey() ?: "",
|
||||
edgeTtsSubscriptionKey = settingsRepository.getEdgeTtsSubscriptionKey() ?: ""
|
||||
)
|
||||
}
|
||||
|
||||
fun updateChatApiKey(key: String) {
|
||||
settingsRepository.saveKeys(
|
||||
youtubeApiKey = settingsRepository.getYoutubeApiKey() ?: "",
|
||||
chatApiKey = key,
|
||||
edgeTtsSubscriptionKey = settingsRepository.getEdgeTtsSubscriptionKey() ?: ""
|
||||
)
|
||||
}
|
||||
|
||||
fun updateEdgeTtsSubscriptionKey(key: String) {
|
||||
settingsRepository.saveKeys(
|
||||
youtubeApiKey = settingsRepository.getYoutubeApiKey() ?: "",
|
||||
chatApiKey = settingsRepository.getChatApiKey() ?: "",
|
||||
edgeTtsSubscriptionKey = key
|
||||
)
|
||||
}
|
||||
|
||||
fun updateLlmProvider(provider: LlmProvider) {
|
||||
settingsRepository.setLlmProvider(provider.name)
|
||||
}
|
||||
|
||||
fun updateLlmModel(model: String) {
|
||||
settingsRepository.setLlmModel(model)
|
||||
}
|
||||
|
||||
fun updateLlmCustomUrl(url: String) {
|
||||
settingsRepository.setLlmCustomUrl(url)
|
||||
}
|
||||
|
||||
fun updateLlmCustomHeaders(headers: String) {
|
||||
settingsRepository.setLlmCustomHeaders(headers)
|
||||
}
|
||||
|
||||
fun updateLlmCustomResponsePath(path: String) {
|
||||
settingsRepository.setLlmCustomResponsePath(path)
|
||||
}
|
||||
|
||||
fun updateLlmCustomRequestTemplate(template: String) {
|
||||
settingsRepository.setLlmCustomRequestTemplate(template)
|
||||
}
|
||||
|
||||
fun updateLlmCustomMessageFormat(format: String) {
|
||||
settingsRepository.setLlmCustomMessageFormat(format)
|
||||
}
|
||||
|
||||
fun updateLlmCustomSystemRole(role: String) {
|
||||
settingsRepository.setLlmCustomSystemRole(role)
|
||||
}
|
||||
|
||||
fun updateLlmCustomUserRole(role: String) {
|
||||
settingsRepository.setLlmCustomUserRole(role)
|
||||
}
|
||||
|
||||
fun updateLlmCustomAssistantRole(role: String) {
|
||||
settingsRepository.setLlmCustomAssistantRole(role)
|
||||
}
|
||||
|
||||
fun updateSttMode(mode: com.app.assistant.speech.SttMode) {
|
||||
settingsRepository.setSttMode(mode)
|
||||
}
|
||||
|
||||
fun updateTtsMode(mode: com.app.assistant.tts.TtsMode) {
|
||||
settingsRepository.setTtsMode(mode)
|
||||
}
|
||||
|
||||
|
||||
|
||||
fun verifyModelAndSaveCapabilities(
|
||||
provider: LlmProvider,
|
||||
model: String,
|
||||
apiKey: String,
|
||||
customUrl: String,
|
||||
customHeaders: String,
|
||||
customResponsePath: String,
|
||||
customRequestTemplate: String,
|
||||
customMessageFormat: String,
|
||||
customSystemRole: String,
|
||||
customUserRole: String,
|
||||
customAssistantRole: String
|
||||
) {
|
||||
viewModelScope.launch {
|
||||
_verificationState.value = VerificationState.Verifying
|
||||
|
||||
val config = if (provider == LlmProvider.CUSTOM) {
|
||||
LlmConfig(
|
||||
url = customUrl,
|
||||
headers = parseHeadersJson(customHeaders),
|
||||
responsePath = customResponsePath,
|
||||
requestTemplate = customRequestTemplate,
|
||||
messageFormat = customMessageFormat,
|
||||
systemRole = customSystemRole.takeIf { it.isNotBlank() },
|
||||
userRole = customUserRole,
|
||||
assistantRole = customAssistantRole
|
||||
)
|
||||
} else {
|
||||
provider.config
|
||||
}
|
||||
|
||||
// 1. Test Text connection
|
||||
val isTextOk = ModelCapabilityProber.probeTextConnection(
|
||||
client = okHttpClient,
|
||||
apiKey = apiKey,
|
||||
model = model,
|
||||
config = config
|
||||
)
|
||||
|
||||
if (!isTextOk) {
|
||||
_verificationState.value = VerificationState.Error("Connection check failed. Please verify API key, URL, or internet connection.")
|
||||
settingsRepository.setIsModelVerified(false)
|
||||
return@launch
|
||||
}
|
||||
|
||||
// 2. Test Vision (Image) connection
|
||||
val isVisionOk = ModelCapabilityProber.probeVisionConnection(
|
||||
client = okHttpClient,
|
||||
provider = provider,
|
||||
apiKey = apiKey,
|
||||
model = model,
|
||||
config = config
|
||||
)
|
||||
|
||||
// 3. Audio/Video checks (currently Gemini 1.5/2.0 is the main provider that supports audio/video input natively in the API)
|
||||
val hasAudio = provider == LlmProvider.GEMINI && (model.contains("1.5") || model.contains("2.0") || model.contains("2.5"))
|
||||
val hasVideo = provider == LlmProvider.GEMINI && (model.contains("1.5") || model.contains("2.0") || model.contains("2.5"))
|
||||
val hasDoc = (provider == LlmProvider.GEMINI && (model.contains("1.5") || model.contains("2.0") || model.contains("2.5"))) || (provider == LlmProvider.ANTHROPIC && model.contains("claude-3"))
|
||||
|
||||
val capabilities = ModelCapabilities(
|
||||
hasImageInput = isVisionOk,
|
||||
hasAudioInput = hasAudio,
|
||||
hasVideoInput = hasVideo,
|
||||
hasDocumentInput = hasDoc
|
||||
)
|
||||
|
||||
// Save detected capabilities in repository
|
||||
settingsRepository.setIsImageSupported(isVisionOk)
|
||||
settingsRepository.setIsAudioSupported(hasAudio)
|
||||
settingsRepository.setIsVideoSupported(hasVideo)
|
||||
settingsRepository.setIsDocumentSupported(hasDoc)
|
||||
settingsRepository.setIsModelVerified(true)
|
||||
|
||||
_verificationState.value = VerificationState.Success(capabilities)
|
||||
|
||||
// Auto-fetch models on successful connection verification
|
||||
fetchModelsForProvider(
|
||||
provider = provider,
|
||||
apiKey = apiKey,
|
||||
customUrl = customUrl,
|
||||
customHeaders = customHeaders
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fun fetchModelsForProvider(
|
||||
provider: LlmProvider,
|
||||
apiKey: String,
|
||||
customUrl: String = "",
|
||||
customHeaders: String = ""
|
||||
) {
|
||||
viewModelScope.launch {
|
||||
if (apiKey.isBlank() && provider != LlmProvider.OLLAMA && provider != LlmProvider.OPEN_ROUTER) {
|
||||
_modelFetchError.value = "API Key is required to fetch models."
|
||||
return@launch
|
||||
}
|
||||
_isFetchingModels.value = true
|
||||
_modelFetchError.value = null
|
||||
try {
|
||||
val models = ModelCapabilityProber.fetchAvailableModels(
|
||||
client = okHttpClient,
|
||||
provider = provider,
|
||||
apiKey = apiKey,
|
||||
customUrl = customUrl,
|
||||
customHeaders = customHeaders
|
||||
)
|
||||
if (models.isNotEmpty()) {
|
||||
_fetchedModels.value = _fetchedModels.value + (provider to models)
|
||||
} else {
|
||||
_modelFetchError.value = "No models returned from API."
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
_modelFetchError.value = "Failed to fetch models: ${e.message}"
|
||||
} finally {
|
||||
_isFetchingModels.value = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private fun parseHeadersJson(jsonStr: String): Map<String, String> {
|
||||
return try {
|
||||
val element = kotlinx.serialization.json.Json.parseToJsonElement(jsonStr)
|
||||
if (element is kotlinx.serialization.json.JsonObject) {
|
||||
element.mapValues { (_, value) ->
|
||||
if (value is kotlinx.serialization.json.JsonPrimitive) value.content else value.toString()
|
||||
}
|
||||
} else {
|
||||
emptyMap()
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
emptyMap()
|
||||
}
|
||||
}
|
||||
|
||||
fun updateTranslationEnabled(enabled: Boolean) {
|
||||
_isTranslationEnabled.value = enabled
|
||||
settingsRepository.setTranslationEnabled(enabled)
|
||||
}
|
||||
|
||||
fun updateActiveLanguageCode(languageCode: String) {
|
||||
_activeLanguageCode.value = languageCode
|
||||
settingsRepository.setActiveLanguageCode(languageCode)
|
||||
}
|
||||
|
||||
fun refreshDownloadedTranslationModels() {
|
||||
RemoteModelManager.getInstance()
|
||||
.getDownloadedModels(TranslateRemoteModel::class.java)
|
||||
.addOnSuccessListener { downloadedModels ->
|
||||
val langs = downloadedModels.map { it.language }.toSet()
|
||||
_downloadedTranslationLanguages.value = langs
|
||||
}
|
||||
.addOnFailureListener { e ->
|
||||
Log.e("SettingsViewModel", "Failed to fetch downloaded translation models: ${e.message}")
|
||||
}
|
||||
}
|
||||
|
||||
fun downloadTranslationModel(languageCode: String, onResult: (Boolean) -> Unit = {}) {
|
||||
_downloadingTranslationLanguages.value = _downloadingTranslationLanguages.value + languageCode
|
||||
val modelManager = RemoteModelManager.getInstance()
|
||||
val languageModel = TranslateRemoteModel.Builder(languageCode).build()
|
||||
val conditions = DownloadConditions.Builder().build()
|
||||
|
||||
modelManager.download(languageModel, conditions)
|
||||
.addOnSuccessListener {
|
||||
_downloadingTranslationLanguages.value = _downloadingTranslationLanguages.value - languageCode
|
||||
refreshDownloadedTranslationModels()
|
||||
onResult(true)
|
||||
}
|
||||
.addOnFailureListener { e ->
|
||||
Log.e("SettingsViewModel", "Failed to download model for $languageCode: ${e.message}")
|
||||
_downloadingTranslationLanguages.value = _downloadingTranslationLanguages.value - languageCode
|
||||
onResult(false)
|
||||
}
|
||||
}
|
||||
|
||||
fun deleteTranslationModel(languageCode: String) {
|
||||
val modelManager = RemoteModelManager.getInstance()
|
||||
val languageModel = TranslateRemoteModel.Builder(languageCode).build()
|
||||
modelManager.deleteDownloadedModel(languageModel)
|
||||
.addOnSuccessListener {
|
||||
refreshDownloadedTranslationModels()
|
||||
}
|
||||
.addOnFailureListener { e ->
|
||||
Log.e("SettingsViewModel", "Failed to delete model for $languageCode: ${e.message}")
|
||||
}
|
||||
}
|
||||
|
||||
private fun getPublicStaticFinalStringsWithNames(clazz: Class<*>): List<Pair<String, String>> {
|
||||
val publicStaticFinalStringsWithNames = mutableListOf<Pair<String, String>>()
|
||||
val fields = clazz.declaredFields
|
||||
for (field in fields) {
|
||||
val modifiers = field.modifiers
|
||||
if (Modifier.isPublic(modifiers) && Modifier.isStatic(modifiers) && Modifier.isFinal(modifiers)) {
|
||||
if (field.type == String::class.java) {
|
||||
try {
|
||||
val name = field.name
|
||||
val value = field.get(null) as String
|
||||
publicStaticFinalStringsWithNames.add(Pair(name, value))
|
||||
} catch (e: IllegalAccessException) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return publicStaticFinalStringsWithNames
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package com.app.assistant.viewmodel
|
||||
|
||||
import androidx.lifecycle.ViewModel
|
||||
import androidx.lifecycle.ViewModelProvider
|
||||
import com.app.assistant.repository.SettingsRepository
|
||||
import okhttp3.OkHttpClient
|
||||
|
||||
class SettingsViewModelFactory(
|
||||
private val settingsRepository: SettingsRepository,
|
||||
private val okHttpClient: OkHttpClient
|
||||
) : ViewModelProvider.Factory {
|
||||
override fun <T : ViewModel> create(modelClass: Class<T>): T {
|
||||
if (modelClass.isAssignableFrom(SettingsViewModel::class.java)) {
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
return SettingsViewModel(settingsRepository, okHttpClient) as T
|
||||
}
|
||||
throw IllegalArgumentException("Unknown ViewModel class")
|
||||
}
|
||||
}
|
||||
42
app/src/main/java/com/app/assistant/viewmodel/UIEvent.kt
Normal file
42
app/src/main/java/com/app/assistant/viewmodel/UIEvent.kt
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
package com.app.assistant.viewmodel
|
||||
|
||||
import android.content.Intent
|
||||
import com.google.android.gms.common.api.ResolvableApiException
|
||||
|
||||
sealed class UIEvent {
|
||||
data class RequestPermissions(
|
||||
val permissions: Array<String>,
|
||||
val requestCode: Int,
|
||||
) : UIEvent()
|
||||
|
||||
data class StartIntent(
|
||||
val intent: Intent,
|
||||
) : UIEvent()
|
||||
|
||||
data class ShowToast(
|
||||
val message: String,
|
||||
) : UIEvent()
|
||||
|
||||
data class SpeakText(
|
||||
val text: String,
|
||||
val queueMode: Int = 0,
|
||||
) : UIEvent()
|
||||
|
||||
object StopSpeaking : UIEvent()
|
||||
|
||||
object StartSpeechRecognition : UIEvent()
|
||||
|
||||
object StopSpeechRecognition : UIEvent()
|
||||
|
||||
data class GetLocationForWeather(
|
||||
val itemId: Long,
|
||||
val loadingItemId: Long,
|
||||
val speak: Boolean,
|
||||
val categoryName: String,
|
||||
val prompt: String,
|
||||
) : UIEvent()
|
||||
|
||||
data class ResolveLocationSettings(
|
||||
val exception: ResolvableApiException,
|
||||
) : UIEvent()
|
||||
}
|
||||
6
app/src/main/res/drawable/bg_notification_button.xml
Normal file
6
app/src/main/res/drawable/bg_notification_button.xml
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<shape xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:shape="rectangle">
|
||||
<corners android:radius="20dp" />
|
||||
<solid android:color="@android:color/white" />
|
||||
</shape>
|
||||
9
app/src/main/res/drawable/ic_arrow_drop_down.xml
Normal file
9
app/src/main/res/drawable/ic_arrow_drop_down.xml
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
<vector xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:width="24dp"
|
||||
android:height="24dp"
|
||||
android:viewportWidth="24"
|
||||
android:viewportHeight="24">
|
||||
<path
|
||||
android:fillColor="@android:color/black"
|
||||
android:pathData="M7,10l5,5 5,-5z" />
|
||||
</vector>
|
||||
10
app/src/main/res/drawable/ic_attachment.xml
Normal file
10
app/src/main/res/drawable/ic_attachment.xml
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
<vector xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:width="24dp"
|
||||
android:height="24dp"
|
||||
android:viewportWidth="24"
|
||||
android:viewportHeight="24"
|
||||
android:tint="?attr/colorControlNormal">
|
||||
<path
|
||||
android:fillColor="@android:color/black"
|
||||
android:pathData="M19,13h-6v6h-2v-6H5v-2h6V5h2v6h6v2z"/>
|
||||
</vector>
|
||||
9
app/src/main/res/drawable/ic_back.xml
Normal file
9
app/src/main/res/drawable/ic_back.xml
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
<vector xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:width="24dp"
|
||||
android:height="24dp"
|
||||
android:viewportWidth="24"
|
||||
android:viewportHeight="24">
|
||||
<path
|
||||
android:fillColor="@android:color/black"
|
||||
android:pathData="M20,11H7.83l5.59,-5.59L12,4l-8,8 8,8 1.41,-1.41L7.83,13H20v-2z" />
|
||||
</vector>
|
||||
20
app/src/main/res/drawable/ic_camera.xml
Normal file
20
app/src/main/res/drawable/ic_camera.xml
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
<!--
|
||||
~ Copyright (C) 2026 The Android Open Source Project
|
||||
~
|
||||
~ Licensed under the Apache License, Version 2.0 (the "License");
|
||||
~ you may not use this file except in compliance with the License.
|
||||
~ You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<vector xmlns:android="http://schemas.android.com/apk/res/android" android:height="24dp" android:tint="#000000" android:viewportHeight="24" android:viewportWidth="24" android:width="24dp">
|
||||
|
||||
<path android:fillColor="@android:color/white" android:pathData="M20,4h-3.17L15,2L9,2L7.17,4L4,4c-1.1,0 -2,0.9 -2,2v12c0,1.1 0.9,2 2,2h16c1.1,0 2,-0.9 2,-2L22,6c0,-1.1 -0.9,-2 -2,-2zM20,18L4,18L4,6h4.05l1.83,-2h4.24l1.83,2L20,6v12zM12,7c-2.76,0 -5,2.24 -5,5s2.24,5 5,5 5,-2.24 5,-5 -2.24,-5 -5,-5zM12,15c-1.65,0 -3,-1.35 -3,-3s1.35,-3 3,-3 3,1.35 3,3 -1.35,3 -3,3z"/>
|
||||
|
||||
</vector>
|
||||
9
app/src/main/res/drawable/ic_download.xml
Normal file
9
app/src/main/res/drawable/ic_download.xml
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
<vector xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:width="24dp"
|
||||
android:height="24dp"
|
||||
android:viewportWidth="24"
|
||||
android:viewportHeight="24">
|
||||
<path
|
||||
android:fillColor="#FF000000"
|
||||
android:pathData="M19,9h-4V3H9v6H5l7,7 7,-7zM5,18v2h14v-2H5z"/>
|
||||
</vector>
|
||||
9
app/src/main/res/drawable/ic_fullscreen.xml
Normal file
9
app/src/main/res/drawable/ic_fullscreen.xml
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
<vector xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:width="24dp"
|
||||
android:height="24dp"
|
||||
android:viewportWidth="24"
|
||||
android:viewportHeight="24">
|
||||
<path
|
||||
android:fillColor="#FFFFFFFF"
|
||||
android:pathData="M7,14H5v5h5v-2H7v-3zm-2,-4h2V7h3V5H5v5zm12,7h-3v2h5v-5h-2v3zM14,5v2h3v3h2V5h-5z"/>
|
||||
</vector>
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue