feat: 添加唤醒功能

2025-09-28 18:42:56 +08:00
parent c6765ab29c
commit 5be50b4dbb
18 changed files with 746 additions and 28 deletions
--- a/packages/flutter_vosk_wakeword/android/models/src/main/assets/model-cn/uuid
+++ b/packages/flutter_vosk_wakeword/android/models/src/main/assets/model-cn/uuid
@@ -0,0 +1 @@
+91a26295-76d2-4da7-a414-4b2ebf9ddeb0
--- a/packages/flutter_vosk_wakeword/android/models/src/main/assets/model-en-us/uuid
+++ b/packages/flutter_vosk_wakeword/android/models/src/main/assets/model-en-us/uuid
@@ -0,0 +1 @@
+2d4432c8-7421-492b-93c2-ad728fd1947d
--- a/packages/flutter_vosk_wakeword/android/src/main/AndroidManifest.xml
+++ b/packages/flutter_vosk_wakeword/android/src/main/AndroidManifest.xml
@@ -1,3 +1,9 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
  package="com.tsystems.flutter_vosk_wakeword">
+    <permission
+        android:name="com.tsystems.flutter_vosk_wakeword.RECORD_AUDIO"
+        android:protectionLevel="signature" />
+    <uses-permission android:name="android.permission.RECORD_AUDIO" />
+    <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
+    <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
 </manifest>
--- a/packages/flutter_vosk_wakeword/android/src/main/kotlin/com/tsystems/flutter_vosk_wakeword/AudioRecorder.kt
+++ b/packages/flutter_vosk_wakeword/android/src/main/kotlin/com/tsystems/flutter_vosk_wakeword/AudioRecorder.kt
@@ -0,0 +1,81 @@
+package com.tsystems.flutter_vosk_wakeword
+
+import android.Manifest
+import android.media.AudioFormat
+import android.media.AudioRecord
+import android.media.MediaRecorder
+import android.os.Process
+import android.util.Log
+import androidx.annotation.RequiresPermission
+import org.vosk.Recognizer
+import org.vosk.android.RecognitionListener
+import java.util.concurrent.Executors
+
+class AudioRecorder(private val recognizer: Recognizer,
+                    private val listener: RecognitionListener) {
+    companion object {
+        private const val TAG = "AudioRecorder"
+        private const val SAMPLE_RATE = 16000
+        private const val BUFFER_SIZE_FACTOR = 2
+    }
+
+    private var audioRecord: AudioRecord? = null
+    private val executor = Executors.newSingleThreadExecutor()
+
+    fun isRecording(): Boolean = audioRecord?.recordingState == AudioRecord.RECORDSTATE_RECORDING
+
+    @RequiresPermission(Manifest.permission.RECORD_AUDIO)
+    fun start() {
+        if (isRecording()) return
+
+        val bufferSize = AudioRecord.getMinBufferSize(
+            SAMPLE_RATE,
+            AudioFormat.CHANNEL_IN_MONO,
+            AudioFormat.ENCODING_PCM_16BIT
+        ) * BUFFER_SIZE_FACTOR
+
+        // 权限检查应该在调用此方法之前在App层完成
+        audioRecord = AudioRecord(
+            MediaRecorder.AudioSource.VOICE_RECOGNITION,
+            SAMPLE_RATE,
+            AudioFormat.CHANNEL_IN_MONO,
+            AudioFormat.ENCODING_PCM_16BIT,
+            bufferSize
+        )
+
+        if (audioRecord?.state == AudioRecord.STATE_INITIALIZED) {
+            audioRecord?.startRecording()
+            executor.execute {
+                Process.setThreadPriority(Process.THREAD_PRIORITY_URGENT_AUDIO)
+                val buffer = ByteArray(bufferSize)
+                while (isRecording()) {
+                    val nread = audioRecord?.read(buffer, 0, buffer.size) ?: 0
+                    if (nread > 0) {
+                        Log.i(TAG, "Read $nread bytes from AudioRecord")
+                        val ac = recognizer.acceptWaveForm(buffer, nread)
+                        Log.i(TAG, "Recognizer acceptWaveForm returned: $ac")
+                        if (ac) {
+                            listener.onResult(recognizer.result)
+                        } else {
+                            listener.onPartialResult(recognizer.partialResult)
+                        }
+                    }
+                }
+                listener.onFinalResult(recognizer.finalResult)
+            }
+            Log.i(TAG, "Audio recording started.")
+        } else {
+            Log.e(TAG, "AudioRecord initialization failed.")
+            listener.onError(Exception("AudioRecord initialization failed."))
+        }
+    }
+
+    fun stop() {
+        if (isRecording()) {
+            audioRecord?.stop()
+            audioRecord?.release()
+            audioRecord = null
+            Log.i(TAG, "Audio recording stopped.")
+        }
+    }
+}
--- a/packages/flutter_vosk_wakeword/android/src/main/kotlin/com/tsystems/flutter_vosk_wakeword/EventReceiver.kt
+++ b/packages/flutter_vosk_wakeword/android/src/main/kotlin/com/tsystems/flutter_vosk_wakeword/EventReceiver.kt
@@ -0,0 +1,54 @@
+package com.tsystems.flutter_vosk_wakeword
+
+import android.content.BroadcastReceiver
+import android.content.Context
+import android.content.Intent
+import android.content.IntentFilter
+import androidx.core.content.ContextCompat
+
+/**
+ * Service 和 Plugin 之间安全地传递事件
+ */
+class EventReceiver(private val onEvent: (type: String, text: String?) -> Unit) : BroadcastReceiver() {
+
+    companion object {
+        private const val ACTION_WAKEWORD_EVENT = "com.tsystems.flutter_vosk_wakeword.WAKEWORD_EVENT"
+        private const val EXTRA_EVENT_TYPE = "EXTRA_EVENT_TYPE"
+        private const val EXTRA_EVENT_TEXT = "EXTRA_EVENT_TEXT"
+
+        fun broadcastPartial(context: Context, text: String) = broadcast(context, "partial", text)
+        fun broadcastFinal(context: Context, text: String) = broadcast(context, "finalResult", text)
+        fun broadcastWakeWord(context: Context, text: String) = broadcast(context, "wakeword", text)
+        fun broadcastError(context: Context, text: String) = broadcast(context, "error", text)
+
+        private fun broadcast(context: Context, type: String, text: String?) {
+            val intent = Intent(ACTION_WAKEWORD_EVENT).apply {
+                setPackage(context.packageName)
+                putExtra(EXTRA_EVENT_TYPE, type)
+                putExtra(EXTRA_EVENT_TEXT, text)
+            }
+            context.sendBroadcast(intent)
+        }
+    }
+
+    override fun onReceive(context: Context?, intent: Intent?) {
+        val type = intent?.getStringExtra(EXTRA_EVENT_TYPE)
+        val text = intent?.getStringExtra(EXTRA_EVENT_TEXT)
+        if (type != null) {
+            onEvent(type, text)
+        }
+    }
+
+    fun register(context: Context) {
+        ContextCompat.registerReceiver(
+            context,
+            this,
+            IntentFilter(ACTION_WAKEWORD_EVENT),
+            ContextCompat.RECEIVER_NOT_EXPORTED
+        )
+    }
+
+    fun unregister(context: Context) {
+        context.unregisterReceiver(this)
+    }
+}
--- a/packages/flutter_vosk_wakeword/android/src/main/kotlin/com/tsystems/flutter_vosk_wakeword/FlutterVoskWakewordPlugin.kt
+++ b/packages/flutter_vosk_wakeword/android/src/main/kotlin/com/tsystems/flutter_vosk_wakeword/FlutterVoskWakewordPlugin.kt
@@ -1,38 +1,88 @@
 package com.tsystems.flutter_vosk_wakeword

+import android.content.Intent
 import androidx.annotation.NonNull

 import io.flutter.embedding.engine.plugins.FlutterPlugin
+import io.flutter.plugin.common.EventChannel
 import io.flutter.plugin.common.MethodCall
 import io.flutter.plugin.common.MethodChannel
 import io.flutter.plugin.common.MethodChannel.MethodCallHandler
 import io.flutter.plugin.common.MethodChannel.Result

-// # vosk
+// vosk
 import org.vosk.LibVosk

 /** FlutterVoskWakewordPlugin */
-class FlutterVoskWakewordPlugin: FlutterPlugin, MethodCallHandler {
-  /// The MethodChannel that will the communication between Flutter and native Android
-  ///
-  /// This local reference serves to register the plugin with the Flutter Engine and unregister it
-  /// when the Flutter Engine is detached from the Activity
-  private lateinit var channel : MethodChannel
+class FlutterVoskWakewordPlugin: FlutterPlugin, MethodCallHandler, EventChannel.StreamHandler {

-  override fun onAttachedToEngine(flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) {
-    channel = MethodChannel(flutterPluginBinding.binaryMessenger, "flutter_vosk_wakeword")
-    channel.setMethodCallHandler(this)
-  }
+    private lateinit var methodChannel : MethodChannel
+    private lateinit var eventChannel: EventChannel
+    private var eventSink: EventChannel.EventSink? = null
+    private lateinit var context: android.content.Context
+    private var eventReceiver: EventReceiver? = null

-  override fun onMethodCall(call: MethodCall, result: Result) {
-    if (call.method == "getPlatformVersion") {
-      result.success("Android ${android.os.Build.VERSION.RELEASE}")
-    } else {
-      result.notImplemented()
+    override fun onAttachedToEngine(flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) {
+        context = flutterPluginBinding.applicationContext
+        methodChannel = MethodChannel(flutterPluginBinding.binaryMessenger, "flutter_vosk_wakeword/methods")
+        methodChannel.setMethodCallHandler(this)
+
+        eventChannel = EventChannel(flutterPluginBinding.binaryMessenger, "flutter_vosk_wakeword/events")
+        eventChannel.setStreamHandler(this)
+    }
+
+    override fun onMethodCall(call: MethodCall, result: Result) {
+        val callName = call.method
+        if (callName == "initialize") {
+//            val modelPath = call.argument<String>("modelPath")
+            val wakeWords = call.argument<ArrayList<String>>("wakeWords")
+            if (wakeWords == null) {
+                result.error("INVALID_ARGUMENTS", "wakeWords is null", null)
+                return
+            }
+            val intent = Intent(context, WakewordService::class.java).apply {
+                action = WakewordService.ACTION_INITIALIZE
+                putExtra(WakewordService.EXTRA_WAKE_WORDS, wakeWords)
+            }
+            context.startService(intent)
+        } else if (callName == "start") {
+            val intent = Intent(context, WakewordService::class.java).apply {
+                action = WakewordService.ACTION_START
+            }
+            context.startService(intent)
+            result.success(null)
+        } else if (callName == "stop") {
+            val intent = Intent(context, WakewordService::class.java).apply {
+                action = WakewordService.ACTION_STOP
+            }
+            context.startService(intent)
+            result.success(null)
+        } else {
+            result.notImplemented()
+        }
+    }
+
+    override fun onDetachedFromEngine(binding: FlutterPlugin.FlutterPluginBinding) {
+        methodChannel.setMethodCallHandler(null)
+        eventChannel.setStreamHandler(null)
+        context.stopService(Intent(context, WakewordService::class.java)) }
+
+
+    // --- EventChannel.StreamHandler Callbacks ---
+    override fun onListen(arguments: Any?, events: EventChannel.EventSink?) {
+        eventSink = events
+        // 注册广播接收器以从 Service 接收事件
+         eventReceiver = EventReceiver { eventType, text ->
+             // 将事件发送回 Dart
+             eventSink?.success(mapOf("type" to eventType, "text" to text))
+         }
+         eventReceiver?.register(context)
+    }
+
+    override fun onCancel(arguments: Any?) {
+        eventSink = null
+        eventReceiver?.unregister(context)
+        eventReceiver = null
    }
-  }

-  override fun onDetachedFromEngine(binding: FlutterPlugin.FlutterPluginBinding) {
-    channel.setMethodCallHandler(null)
-  }
 }
--- a/packages/flutter_vosk_wakeword/android/src/main/kotlin/com/tsystems/flutter_vosk_wakeword/VoskRecognizer.kt
+++ b/packages/flutter_vosk_wakeword/android/src/main/kotlin/com/tsystems/flutter_vosk_wakeword/VoskRecognizer.kt
@@ -0,0 +1,21 @@
+package com.tsystems.flutter_vosk_wakeword
+
+import org.json.JSONObject
+
+object VoskRecognizer {
+    fun parsePartialResult(json: String): String {
+        return try {
+            JSONObject(json).getString("partial")
+        } catch (e: Exception) {
+            ""
+        }
+    }
+
+    fun parseFinalResult(json: String): String {
+        return try {
+            JSONObject(json).getString("text")
+        } catch (e: Exception) {
+            ""
+        }
+    }
+}
--- a/packages/flutter_vosk_wakeword/android/src/main/kotlin/com/tsystems/flutter_vosk_wakeword/WakewordDetector.kt
+++ b/packages/flutter_vosk_wakeword/android/src/main/kotlin/com/tsystems/flutter_vosk_wakeword/WakewordDetector.kt
@@ -0,0 +1,27 @@
+package com.tsystems.flutter_vosk_wakeword
+
+import org.json.JSONArray
+
+class WakewordDetector(private val wakeWords: ArrayList<String>) {
+
+    /**
+     * 检查部分识别文本中是否包含任何一个唤醒词
+     */
+    fun checkForWakeWord(partialText: String): Boolean {
+        if (partialText.isBlank()) return false
+        // 简单包含匹配，可以根据需求改为更复杂的匹配逻辑
+        return wakeWords.any { wakeWord -> partialText.contains(wakeWord, ignoreCase = true) }
+    }
+
+    /**
+     * 为Vosk生成语法字符串，这可以提高唤醒词的识别准确率
+     * 例如，对于 ["你好小帅", "你好小美"]，生成 "[\"你好小帅\", \"你好小美\", \"[unk]\"]"
+     */
+    fun getVoskGrammar(): String {
+        val jsonArray = JSONArray()
+        wakeWords.forEach { jsonArray.put(it) }
+        // 添加 [unk] 允许识别唤醒词之外的其他词语
+//        jsonArray.put("[unk]")
+        return jsonArray.toString()
+    }
+}
--- a/packages/flutter_vosk_wakeword/android/src/main/kotlin/com/tsystems/flutter_vosk_wakeword/WakewordService.kt
+++ b/packages/flutter_vosk_wakeword/android/src/main/kotlin/com/tsystems/flutter_vosk_wakeword/WakewordService.kt
@@ -0,0 +1,269 @@
+package com.tsystems.flutter_vosk_wakeword
+
+import android.Manifest
+import android.app.Service
+import android.content.Intent
+import android.os.Handler
+import android.os.IBinder
+import android.os.Looper
+import android.os.Process
+import android.util.Log
+import androidx.annotation.RequiresPermission
+
+
+import org.vosk.Model
+import org.vosk.Recognizer
+import org.vosk.android.RecognitionListener
+import org.vosk.android.StorageService
+import java.io.IOException
+import kotlin.collections.contentToString
+
+class WakewordService: Service(), RecognitionListener {
+
+    companion object {
+        const val ACTION_INITIALIZE = "ACTION_INITIALIZE"
+        const val ACTION_START = "ACTION_START"
+        const val ACTION_STOP = "ACTION_STOP"
+        const val EXTRA_WAKE_WORDS = "EXTRA_WAKE_WORDS"
+        private const val TAG = "WakewordService"
+        private const val ASSET_MODEL_DIR = "model-cn"
+        private const val LOCAL_MODEL_DIR = "model"
+
+        // 冷却时间: 同一唤醒词触发后在此毫秒内不再重复广播
+        private const val DETECTION_COOLDOWN_MS = 2500L
+    }
+
+    private val mainHandle = Handler(Looper.getMainLooper())
+
+    private var model: Model? = null
+    private var recognizer: Recognizer? = null
+    private var audioRecorder: AudioRecorder? = null
+    private var wakeWordDetector: WakewordDetector? = null
+
+    private var pendingStart = false
+    private var modelLoading = false
+
+    // 去重/状态控制
+    private var lastWakeWord: String? = null
+    private var lastDetectTs: Long = 0L
+    private var suppressUntilFinal = false
+
+    override fun onCreate() {
+        super.onCreate()
+        Log.w(TAG, "onCreate pid=${Process.myPid()} thread=${Thread.currentThread().name}")
+        dumpAssets()
+    }
+
+    private fun dumpAssets() {
+        try {
+            val root = assets.list("")?.joinToString()
+            Log.w(TAG, "assets root=$root")
+            val cn = assets.list("model-cn")?.joinToString()
+            Log.w(TAG, "assets/model-cn=$cn")
+        } catch (e: Exception) {
+            Log.e(TAG, "dumpAssets error", e)
+        }
+    }
+
+    @RequiresPermission(Manifest.permission.RECORD_AUDIO)
+    override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
+        val action = intent?.action
+        when (action) {
+            ACTION_INITIALIZE -> {
+                val wakeWords = intent.getStringArrayListExtra(EXTRA_WAKE_WORDS)
+                Log.i(TAG, "ACTION_INITIALIZE wakeWords=${wakeWords.toString()}")
+                if (wakeWords != null && wakeWords.isNotEmpty()) {
+                    initialize(wakeWords)
+                } else {
+                    Log.e(TAG, "No valid wake words found in Intent extras.")
+                }
+            }
+            ACTION_START -> {
+                pendingStart = true
+                Log.i(TAG, "ACTION_START received (modelReady=${model != null}, loading=$modelLoading)")
+                startRecognition()
+            }
+            ACTION_STOP -> {
+                Log.i(TAG, "ACTION_STOP received")
+                stopRecognition()
+            }
+        }
+
+        return START_STICKY
+    }
+
+    private fun initialize(wakeWords: ArrayList<String>) {
+        if (model != null) {
+            Log.i(TAG, "Model already loaded; skip initialize.")
+            return
+        }
+        if (modelLoading) {
+            Log.i(TAG, "Model already loading; skip duplicate initialize.")
+            return
+        }
+        Log.i(TAG, "initialize() begin")
+
+        try {
+            stopRecognition()
+            wakeWordDetector = WakewordDetector(wakeWords)
+            modelLoading = true
+            Log.i(TAG, "Submitting StorageService.unpack source=model-cn target=model")
+            StorageService.unpack(this,
+                ASSET_MODEL_DIR,
+                LOCAL_MODEL_DIR,
+                { loadedModel ->
+                    Log.i(TAG, "Vosk model loaded successfully")
+                    this.model = loadedModel
+                    modelLoading = false
+                    if (pendingStart) {
+                        mainHandle.post {
+                            startRecognition()
+                        }
+                    }
+                },
+                { exception ->
+                    modelLoading = false
+                    val errorMsg = "Failed to unpack the model: ${exception.message}"
+                    Log.e(TAG, errorMsg, exception)
+                    EventReceiver.broadcastError(this, errorMsg)
+                })
+        } catch (e: IOException) {
+            modelLoading = false
+            Log.e(TAG, "Failed to load Vosk model", e)
+            EventReceiver.broadcastError(this, "Failed to load Vosk model: ${e.message}")
+        }
+    }
+
+    @RequiresPermission(Manifest.permission.RECORD_AUDIO)
+    private fun startRecognition() {
+        val modelReady = model != null
+        Log.i(TAG, "startRecognition(modelReady=$modelReady pendingStart=$pendingStart loading=$modelLoading)")
+        if (!pendingStart) {
+            Log.i(TAG, "No pending start request; abort.")
+            return
+        }
+        if (!modelReady) {
+            Log.i(TAG, "Model not ready yet; start deferred.")
+            return
+        }
+        if (audioRecorder?.isRecording() == true) {
+            Log.w(TAG, "Already recording.")
+            return
+        }
+        try {
+            val grammar = wakeWordDetector?.getVoskGrammar()
+            val currentRecognizer = Recognizer(model, 16000.0f, grammar)
+            recognizer = currentRecognizer
+
+            audioRecorder = AudioRecorder(currentRecognizer, this)
+            audioRecorder?.start()
+            Log.i(TAG, "Recognition started.")
+            resetDetectionState()
+        } catch (e: Exception) {
+            val errorMsg = "Failed to start recognition: ${e.message}"
+            Log.e(TAG, "Failed to start recognition", e)
+            EventReceiver.broadcastError(this, errorMsg)
+        }
+    }
+
+    private fun stopRecognition() {
+        Log.i(TAG, "stopRecognition()")
+        audioRecorder?.stop()
+        audioRecorder = null
+        recognizer?.close()
+        recognizer = null
+        Log.i(TAG, "Recognition stopped.")
+        resetDetectionState()
+    }
+
+    private fun resetDetectionState() {
+        suppressUntilFinal = false
+        lastWakeWord = null
+        lastDetectTs = 0L
+    }
+
+    // --- RecognitionListener Callbacks ---
+    override fun onPartialResult(hypothesis: String?) {
+        hypothesis ?: return
+        val partialText = VoskRecognizer.parsePartialResult(hypothesis)
+        if (partialText.isBlank()) return
+        Log.d(TAG, "partial=$partialText suppress=$suppressUntilFinal")
+
+        if (suppressUntilFinal) return
+
+        if (wakeWordDetector?.checkForWakeWord(partialText) == true) {
+            val now = System.currentTimeMillis()
+            if (shouldBroadcast(partialText, now)) {
+                broadcastWake(partialText, now, source = "partial")
+                // 抑制后续重复直到 finalResult 或超时
+                suppressUntilFinal = true
+            }
+        }
+    }
+
+    override fun onResult(hypothesis: String?) {
+        hypothesis ?: return
+        val finalText = VoskRecognizer.parseFinalResult(hypothesis)
+        if (finalText.isBlank()) return
+        Log.d(TAG, "result=$finalText suppress=$suppressUntilFinal")
+
+        if (wakeWordDetector?.checkForWakeWord(finalText) == true) {
+            val now = System.currentTimeMillis()
+            if (shouldBroadcast(finalText, now)) {
+                broadcastWake(finalText, now, source = "final")
+            }
+        }
+        // final / result 结束一轮，允许下一次
+        suppressUntilFinal = false
+    }
+
+    override fun onFinalResult(hypothesis: String?) {
+        // 已在 onResult 处理; 这里可以忽略或调用 onResult
+        onResult(hypothesis)
+    }
+
+    private fun shouldBroadcast(word: String, now: Long): Boolean {
+        val cooldownPassed = (now - lastDetectTs) > DETECTION_COOLDOWN_MS
+        val differentWord = word != lastWakeWord
+        return cooldownPassed || differentWord
+    }
+
+    private fun broadcastWake(word: String, now: Long, source: String) {
+        lastWakeWord = word
+        lastDetectTs = now
+        Log.i(TAG, "Wake word detected($source): $word")
+        EventReceiver.broadcastWakeWord(this, word)
+        // 重置识别器状态，为下一次唤醒做准备
+        recognizer?.reset()
+        // 可选：重建 recognizer 减少同一发音尾部抖动:
+        // try {
+        //     val grammar = wakeWordDetector?.getVoskGrammar()
+        //     recognizer?.close()
+        //     recognizer = Recognizer(model, 16000.0f, grammar)
+        // } catch (e: Exception) {
+        //     Log.w(TAG, "Recognizer reset failed: ${e.message}")
+        // }
+    }
+
+    override fun onError(e: Exception?) {
+        Log.e(TAG, "Recognition error", e)
+        EventReceiver.broadcastError(this, e?.message ?: "Unknown recognition error")
+        suppressUntilFinal = false
+    }
+
+    override fun onTimeout() {
+        Log.w(TAG, "Recognition timeout.")
+        // 超时视为一轮结束
+        suppressUntilFinal = false
+    }
+
+    override fun onBind(intent: Intent?): IBinder? = null
+
+    override fun onDestroy() {
+        stopRecognition()
+        model?.close()
+        model = null
+        super.onDestroy()
+        Log.i(TAG, "WakewordService destroyed.")
+    }
+}