feat:绘本语音评测逻辑

吴启风
1 parent 08a0f5a8
Showing 14 changed files with 279 additions and 69 deletions
android/app/src/main/assets/vad.0.1.bin
android/app/src/main/kotlin/com/kouyuxingqiu/wow_english/methodChannels/SingSoungMethodChannel.kt
android/app/src/main/kotlin/com/kouyuxingqiu/wow_english/singsound/SingEngineHelper.kt
android/app/src/main/kotlin/com/kouyuxingqiu/wow_english/singsound/bean/BaseResultModel.kt
android/app/src/main/kotlin/com/kouyuxingqiu/wow_english/util/GlobalHandler.kt
ios/Runner/XSMessageMehtodChannel.swift
lib/models/course_process_entity.dart
lib/pages/home/home_page.dart
lib/pages/practice/bloc/topic_picture_bloc.dart
lib/pages/reading/bloc/reading_bloc.dart
lib/pages/reading/bloc/reading_event.dart
lib/pages/reading/bloc/reading_state.dart
lib/pages/reading/reading_page.dart
lib/route/route.dart
@@ -2,9 +2,13 @@ package com.kouyuxingqiu.wow_english.methodChannels
 import android.util.Log
 import com.kouyuxingqiu.wow_english.singsound.SingEngineHelper
+import com.kouyuxingqiu.wow_english.singsound.SingEngineHelper.init
+import com.kouyuxingqiu.wow_english.singsound.SingEngineLifecycles
+import com.kouyuxingqiu.wow_english.util.GlobalHandler
 import io.flutter.embedding.android.FlutterActivity
 import io.flutter.embedding.engine.FlutterEngine
 import io.flutter.plugin.common.MethodChannel
+import org.json.JSONObject
 import java.lang.ref.WeakReference
 /**
@@ -12,13 +16,13 @@ import java.lang.ref.WeakReference
  * @date:          2023/6/27 00:32
  * @description:
  */
-class SingSoungMethodChannel(val activity: FlutterActivity, val flutterEngine: FlutterEngine) {
+class SingSoungMethodChannel(activity: FlutterActivity, flutterEngine: FlutterEngine): SingEngineLifecycles.OnSingEngineAdapter() {
     private var methodChannel: MethodChannel? = null
     companion object {
         var channel: WeakReference<SingSoungMethodChannel>? = null
-        fun invokeMethod(method: String, arguments: Any) {
+        fun invokeMethod(method: String, arguments: Any?) {
             channel?.get()?.methodChannel?.invokeMethod(method, arguments)
         }
     }
@@ -30,10 +34,11 @@ class SingSoungMethodChannel(val activity: FlutterActivity, val flutterEngine: F
                 flutterEngine.dartExecutor.binaryMessenger,
                 "wow_english/sing_sound_method_channel"
             )
+        init(activity)
         methodChannel?.setMethodCallHandler { call, result ->
             when (call.method) {
                 "initVoiceSdk" -> {
-                    SingEngineHelper.init(activity)
+
                 }
                 "startVoice" -> {
                     val paramMap = call.arguments as HashMap<String, String>
@@ -43,6 +48,7 @@ class SingSoungMethodChannel(val activity: FlutterActivity, val flutterEngine: F
                 }
                 "stopVoice" -> {
                     Log.d("WQF", "SingSoungMethodChannel stopVoice")
+                    SingEngineHelper.stopRecord()
                 }
                 else -> {
                     result.notImplemented()
@@ -51,5 +57,26 @@ class SingSoungMethodChannel(val activity: FlutterActivity, val flutterEngine: F
         }
         channel = WeakReference(this)
+
+        SingEngineHelper.addOnResultListener(this)
+    }
+
+    override fun onResult(jsonObject: JSONObject, evalType: Int?) {
+        //先声回调在子线程，需要切换到主线程
+        GlobalHandler.runOnMainThread {
+            invokeMethod("voiceResult", jsonObject.toString())
+        }
+    }
+
+    override fun onRecordBegin() {
+        GlobalHandler.runOnMainThread {
+            invokeMethod("voiceStart", null)
+        }
+    }
+
+    override fun onRecordStop() {
+        GlobalHandler.runOnMainThread {
+            invokeMethod("voiceEnd", null)
+        }
     }
 }
 \ No newline at end of file
@@ -73,7 +73,7 @@ object SingEngineHelper :
                         // 设置录音音频路径
                         wavPath = AiUtil.getFilesDir(context).path + "/userdata/sound_record/"
                         // 设置是否开启 VAD 功能
-                        setOpenVad(true, "vad.0.1.bin")
+//                        setOpenVad(true, "vad.0.1.bin")
                         //setOpenVad(false, null);
                         // 设置 VAD 前置超时时间
                         setFrontVadTime(3000)
@@ -383,6 +383,7 @@ object SingEngineHelper :
     }
     fun addOnResultListener(listener: SingEngineLifecycles) {
+        Log.i(TAG, "addOnResultListener")
         if (mListeners?.contains(listener) == true) {
             return
         }
@@ -390,6 +391,7 @@ object SingEngineHelper :
     }
     fun removeOnResultListener(listener: SingEngineLifecycles) {
+        Log.i(TAG, "removeOnResultListener")
         if (mListeners?.contains(listener) == true) {
             mListeners?.remove(listener)
         }
@@ -19,7 +19,9 @@ class BaseResultModel(
     constructor(parcel: Parcel) : this(
         parcel.readString(),
         parcel.readDouble(),
-        TODO("scores"),
+        mutableListOf<SingleResultModel>().apply {
+            parcel.readTypedList(this, SingleResultModel.CREATOR)
+        },
         parcel.readDouble(),
         parcel.readDouble()
     ) {
@@ -59,6 +61,7 @@ class BaseResultModel(
     override fun writeToParcel(parcel: Parcel, flags: Int) {
         parcel.writeString(originText)
         parcel.writeDouble(score)
+        parcel.writeTypedList(scores)
         parcel.writeDouble(pronounce)
         parcel.writeDouble(fluency)
     }
+package com.kouyuxingqiu.wow_english.util
+
+import android.os.Handler
+import android.os.Looper
+
+/**
+ * @author: stay
+ * @date: 2023/7/1 16:55
+ * @description: 全局Handler, 用于切线程等操作
+ */
+object GlobalHandler {
+    private var handler: Handler? = null
+
+    init {
+        handler = Handler(Looper.getMainLooper())
+    }
+
+    fun runOnMainThread(runnable: Runnable?) {
+        runnable?.let {
+            handler?.post(it)
+        }
+    }
+}
 \ No newline at end of file
@@ -13,7 +13,7 @@ class XSMessageMehtodChannel: NSObject,SSOralEvaluatingManagerDelegate {
     init(message:FlutterBinaryMessenger) {
         super.init()
         resultData = Dictionary()
-        messageChannel = FlutterMethodChannel.init(name: "wow_english/sing_sound_method_channely",  binaryMessenger: message)
+        messageChannel = FlutterMethodChannel.init(name: "wow_english/sing_sound_method_channel",  binaryMessenger: message)
         messageChannel!.setMethodCallHandler { call, result in
             self.handle(call, result)
         }
@@ -34,6 +34,8 @@ class CourseProcessReadings {
 	String? picUrl;
 	int? sortOrder;
 	String? word;
+	String? recordUrl;
+	String? recordScore;
 	CourseProcessReadings();
@@ -116,6 +116,7 @@ class _HomePageView extends StatelessWidget {
                                 return;
                               }
                               if (data.courseType == 4) {//绘本
+                                Navigator.of(context).pushNamed(AppRouteName.reading, arguments: {'courseLessonId':data.id!});
                                 return;
                               }
@@ -90,7 +90,7 @@ class TopicPictureBloc extends Bloc&lt;TopicPictureEvent, TopicPictureState&gt; {
         add(VoicePlayStateChangeEvent());
       });
-      methodChannel = const MethodChannel('wow_english/sing_sound_method_channely');
+      methodChannel = const MethodChannel('wow_english/sing_sound_method_channel');
       methodChannel.setMethodCallHandler((call) async {
         if (call.method == 'voiceResult') {//评测结果
           add(XSVoiceResultEvent(call.arguments));
@@ -181,7 +181,7 @@ class TopicPictureBloc extends Bloc&lt;TopicPictureEvent, TopicPictureState&gt; {
   ///先声测试
   void _voiceXsTest(XSVoiceTestEvent event,Emitter<TopicPictureState> emitter) async {
-    audioPlayer.stop();
+    await audioPlayer.stop();
     methodChannel.invokeMethod(
         'startVoice',
         {'word':event.testWord,'type':event.type,'userId':event.userId.toString()}
@@ -10,13 +10,30 @@ import &#39;../../../common/request/dao/listen_dao.dart&#39;;
 import '../../../common/request/exception.dart';
 import '../../../models/course_process_entity.dart';
 import '../../../utils/loading.dart';
+import 'dart:convert';
 part 'reading_event.dart';
 part 'reading_state.dart';
+enum VoicePlayState {
+  ///未知
+  unKnow,
+
+  ///播放中
+  playing,
+
+  ///播放完成
+  completed,
+
+  ///播放终止
+  stop
+}
+
 class ReadingPageBloc extends Bloc<ReadingPageEvent, ReadingPageState> {
   final PageController pageController;
+  final String courseLessonId;
+
   ///当前页索引
   int _currentPage = 0;
@@ -36,40 +53,98 @@ class ReadingPageBloc extends Bloc&lt;ReadingPageEvent, ReadingPageState&gt; {
   bool get isRecording => _isRecording;
+  ///原始音频是否正在播放
+  bool _isOriginAudioPlaying = false;
+
+  bool get isOriginAudioPlaying => _isOriginAudioPlaying;
+
+  ///录音音频是否正在播放
+  bool _isRecordAudioPlaying = false;
+
+  bool get isRecordAudioPlaying => _isRecordAudioPlaying;
+
+  ///正在播放音频状态
+  VoicePlayState _voicePlayState = VoicePlayState.unKnow;
+
+  VoicePlayState get voicePlayState => _voicePlayState;
+
   late MethodChannel methodChannel;
   late AudioPlayer audioPlayer;
-  ReadingPageBloc(this.pageController) : super(ReadingPageInitial()) {
+  ReadingPageBloc(this.pageController, this.courseLessonId)
+      : super(ReadingPageInitial()) {
     on<CurrentPageIndexChangeEvent>(_pageControllerChange);
-    on<CurrentModeChangeEvent>(_selectItemLoad);
+    on<CurrentModeChangeEvent>(_playModeChange);
     // pageController.addListener(() {
     //   _currentPage = pageController.page!.round();
     // });
     on<RequestDataEvent>(_requestData);
-    on<ReadingPageEvent>((event, emit) {
+    on<InitBlocEvent>((event, emit) {
       //音频播放器
       audioPlayer = AudioPlayer();
-      audioPlayer.onPlayerStateChanged.listen((event) {
+      audioPlayer.onPlayerStateChanged.listen((event) async {
+        debugPrint('播放状态变化');
         if (event == PlayerState.completed) {
-          if (kDebugMode) {
-            print('绘本播放完成');
+          debugPrint('播放完成');
+          _voicePlayState = VoicePlayState.completed;
+        }
+        if (event == PlayerState.stopped) {
+          debugPrint('播放结束');
+          _voicePlayState = VoicePlayState.stop;
+        }
-          }
+        if (event == PlayerState.playing) {
+          debugPrint('正在播放中');
+          _voicePlayState = VoicePlayState.playing;
+        }
+        if (isClosed) {
+          return;
         }
+        add(VoicePlayStateChangeEvent());
       });
-      methodChannel = const MethodChannel('sing_sound_method_channel');
-      methodChannel.invokeMethod('initVoiceSdk',{});
+      methodChannel =
+          const MethodChannel('wow_english/sing_sound_method_channel');
+      methodChannel.invokeMethod('initVoiceSdk', {}); //初始化评测
       methodChannel.setMethodCallHandler((call) async {
-        if (call.method == 'voiceResult') {//评测结束
-          // add(XSVoiceResultEvent(call.arguments));
+        if (call.method == 'voiceResult') {
+          //评测结果
+          add(XSVoiceResultEvent(call.arguments));
+          return;
+        }
+
+        if (call.method == 'voiceStart') {
+          //评测开始
+          if (kDebugMode) {
+            print('评测开始');
+          }
+          return;
+        }
+
+        if (call.method == 'voiceEnd') {
+          //评测结束
+          if (kDebugMode) {
+            print('评测结束');
+          }
+          return;
+        }
+
+        if (call.method == 'voiceFail') {
+          //评测失败
+          EasyLoading.showToast('评测失败');
+          return;
         }
       });
     });
+    on<VoicePlayStateChangeEvent>(_voicePlayStateChange);
     on<PlayOriginalAudioEvent>(_playOriginalAudio);
-    on<XSVoiceTestEvent>(_voiceXsTest);
+    on<XSVoiceInitEvent>(_initVoiceSdk);
+    on<XSVoiceStartEvent>(_voiceXsStart);
+    on<XSVoiceStopEvent>(_voiceXsStop);
     on<XSVoiceResultEvent>(_voiceXsResult);
+
+    on<PlayRecordAudioEvent>(_playRecordAudio);
   }
   @override
@@ -83,11 +158,11 @@ class ReadingPageBloc extends Bloc&lt;ReadingPageEvent, ReadingPageState&gt; {
   void _pageControllerChange(CurrentPageIndexChangeEvent event,
       Emitter<ReadingPageState> emitter) async {
     _currentPage = event.pageIndex;
-    _playOriginVoice(null);
+    _playOriginalAudioInner(null);
     emitter(CurrentPageIndexState());
   }
-  void _selectItemLoad(
+  void _playModeChange(
       CurrentModeChangeEvent event, Emitter<ReadingPageState> emitter) async {
     if (_currentMode == ReadingModeType.auto) {
       _currentMode = ReadingModeType.manual;
@@ -102,7 +177,7 @@ class ReadingPageBloc extends Bloc&lt;ReadingPageEvent, ReadingPageState&gt; {
       RequestDataEvent event, Emitter<ReadingPageState> emitter) async {
     try {
       await loading(() async {
-        _entity = await ListenDao.process('1');
+        _entity = await ListenDao.process(courseLessonId);
         print("reading page entity: ${_entity!.toJson()}");
         emitter(RequestDataState());
       });
@@ -113,20 +188,36 @@ class ReadingPageBloc extends Bloc&lt;ReadingPageEvent, ReadingPageState&gt; {
     }
   }
-  void _playOriginalAudio(PlayOriginalAudioEvent event, Emitter<ReadingPageState> emitter) async {
-    print("_playOriginalAudio");
-    _playOriginVoice(event.url);
+  /// 播放绘本原音
+  void _playOriginalAudio(
+      PlayOriginalAudioEvent event, Emitter<ReadingPageState> emitter) async {
+    _playOriginalAudioInner(event.url);
+  }
+
+  void _playOriginalAudioInner(String? audioUrl) {
+    print("_playOriginalAudio url=$audioUrl");
+    audioUrl ??= currentPageData()?.audioUrl ?? '';
+    _playAudio(audioUrl);
+    _isOriginAudioPlaying = true;
   }
-  /// 播放绘本原音
-  void _playOriginVoice(String? audioUrl) async {
-    audioPlayer.stop();
-    final readingData = currentPageData();
-    if (readingData?.audioUrl != null) {
-      final urlStr = audioUrl ?? readingData?.audioUrl ?? '';
-      if (urlStr.isNotEmpty) {
-         audioPlayer.play(UrlSource(urlStr));
-      }
+  /// 播放录音
+  void _playRecordAudio(
+      PlayRecordAudioEvent event, Emitter<ReadingPageState> emitter) async {
+    _playRecordAudioInner();
+  }
+
+  void _playRecordAudioInner() {
+    final recordAudioUrl = currentPageData()?.recordUrl;
+    print("_playRecordAudioInner url=${currentPageData()?.recordUrl}");
+    _playAudio(recordAudioUrl);
+    _isRecordAudioPlaying = true;
+  }
+
+  void _playAudio(String? audioUrl) async {
+    await audioPlayer.stop();
+    if (audioUrl!.isNotEmpty) {
+      await audioPlayer.play(UrlSource(audioUrl));
     }
   }
@@ -139,8 +230,16 @@ class ReadingPageBloc extends Bloc&lt;ReadingPageEvent, ReadingPageState&gt; {
     return _entity?.readings?[_currentPage];
   }
+  ///初始化SDK
+  _initVoiceSdk(
+      XSVoiceInitEvent event, Emitter<ReadingPageState> emitter) async {
+    methodChannel.invokeMethod('initVoiceSdk', event.data);
+  }
+
   ///先声测试
-  void _voiceXsTest(XSVoiceTestEvent event, Emitter<ReadingPageState> emitter) async {
+  void _voiceXsStart(
+      XSVoiceStartEvent event, Emitter<ReadingPageState> emitter) async {
+    _stopAudio();
     startRecord(event.content);
     emitter(XSVoiceTestState());
   }
@@ -149,24 +248,44 @@ class ReadingPageBloc extends Bloc&lt;ReadingPageEvent, ReadingPageState&gt; {
     if (_isRecording == true) {
       return;
     }
-    EasyLoading.show(status: '录音中....');
     methodChannel.invokeMethod(
-        'startVoice',
-        {'word':'how old are you','type':'0','userId':'1'}
-    );
+        'startVoice', {'word': 'how old are you', 'type': '0', 'userId': '1'});
     _isRecording = true;
   }
-  void _voiceXsResult(XSVoiceResultEvent event,Emitter<ReadingPageState> emitter) async {
-    final Map args = event.message as Map;
-    final result = args['result'] as String;
-    if (result == '1') {
-      final overall = args['overall'].toString();
-      EasyLoading.showToast('测评成功,分数是$overall',duration: const Duration(seconds: 10));
+  void _voiceXsResult(
+      XSVoiceResultEvent event, Emitter<ReadingPageState> emitter) async {
+    final Map args = json.decode(event.message);
+    final result = args['result'];
+    print("_voiceXsResult result=${result}");
+    if (result != null) {
+      final overall = result['overall'].toString();
+      EasyLoading.showToast('测评成功,分数是$overall',
+          duration: const Duration(seconds: 10));
+      currentPageData()?.recordScore = overall;
+      currentPageData()?.recordUrl = args['audioUrl'] + '.mp3';
+      _playRecordAudioInner();
     } else {
-      EasyLoading.showToast('测评失败',duration: const Duration(seconds: 10));
+      EasyLoading.showToast('测评失败', duration: const Duration(seconds: 10));
     }
     _isRecording = false;
     emitter(XSVoiceTestState());
   }
+
+  ///终止评测
+  void _voiceXsStop(
+      XSVoiceStopEvent event, Emitter<ReadingPageState> emitter) async {
+    methodChannel.invokeMethod('stopVoice');
+  }
+
+  void _voicePlayStateChange(VoicePlayStateChangeEvent event,
+      Emitter<ReadingPageState> emitter) async {
+    emitter(VoicePlayStateChange());
+  }
+
+  void _stopAudio() async {
+    await audioPlayer.stop();
+    _isOriginAudioPlaying = false;
+    _isRecordAudioPlaying = false;
+  }
 }
@@ -3,6 +3,9 @@ part of &#39;reading_bloc.dart&#39;;
 @immutable
 abstract class ReadingPageEvent {}
+///页面初始化
+class InitBlocEvent extends ReadingPageEvent {}
+
 class CurrentPageIndexChangeEvent extends ReadingPageEvent {
   final int pageIndex;
   CurrentPageIndexChangeEvent(this.pageIndex);
@@ -32,9 +35,18 @@ class XSVoiceResultEvent extends ReadingPageEvent {
 }
 ///先声测试
-class XSVoiceTestEvent extends ReadingPageEvent {
+class XSVoiceStartEvent extends ReadingPageEvent {
   final String content;
   final String type;
   final String userId;
-  XSVoiceTestEvent(this.content,this.type,this.userId);
-}
 \ No newline at end of file
+  XSVoiceStartEvent(this.content,this.type,this.userId);
+}
+
+///先声评测停止
+class XSVoiceStopEvent extends ReadingPageEvent {}
+
+///音频播放状态
+class VoicePlayStateChangeEvent extends ReadingPageEvent {}
+
+///录音播放
+class PlayRecordAudioEvent extends ReadingPageEvent {}
 \ No newline at end of file
@@ -13,3 +13,5 @@ class CurrentModeState extends ReadingPageState {}
 class RequestDataState extends ReadingPageState {}
 class XSVoiceTestState extends ReadingPageState {}
+
+class VoicePlayStateChange extends ReadingPageState {}
@@ -9,12 +9,16 @@ import &#39;../../models/course_process_entity.dart&#39;;
 import 'bloc/reading_bloc.dart';
 class ReadingPage extends StatelessWidget {
-  const ReadingPage({super.key});
+  const ReadingPage({super.key, this.courseLessonId});
+
+  final String? courseLessonId;
   @override
   Widget build(BuildContext context) {
     return BlocProvider(
-      create: (_) => ReadingPageBloc(PageController())..add(RequestDataEvent()),
+      create: (_) => ReadingPageBloc(PageController(), courseLessonId ?? '')
+        ..add(InitBlocEvent())
+        ..add(RequestDataEvent()),
       child: _ReadingPage(),
     );
   }
@@ -135,11 +139,13 @@ class _ReadingPage extends StatelessWidget {
                           if (bloc.isRecording) {
                             return;
                           }
-                          print("voice tap");
                           bloc.add(PlayOriginalAudioEvent(null));
                         },
                         child: Image.asset(
-                          'voice'.assetPng,
+                          bloc.voicePlayState == VoicePlayState.playing &&
+                                  bloc.isOriginAudioPlaying
+                              ? 'reade_answer'.assetGif
+                              : 'voice'.assetPng,
                           height: 40.h,
                           width: 45.w,
                         ),
@@ -161,9 +167,13 @@ class _ReadingPage extends StatelessWidget {
                       GestureDetector(
                           onTap: () {
                             if (bloc.isRecording) {
-                              return;
+                              bloc.add(XSVoiceStopEvent());
+                            } else {
+                              bloc.add(XSVoiceStartEvent(
+                                  bloc.currentPageData()?.word ?? '',
+                                  '0',
+                                  UserUtil.getUser()!.id.toString()));
                             }
-                            bloc.add(XSVoiceTestEvent(bloc.currentPageData()?.word??'', '0',UserUtil.getUser()!.id.toString()));
                           },
                           child: Image.asset(
                             'micro_phone'.assetPng,
@@ -173,18 +183,23 @@ class _ReadingPage extends StatelessWidget {
                       SizedBox(
                         width: 10.w,
                       ),
-                      Visibility(
-                        visible: false,
-
-                        ///todo 依据是否录过音
-                        child: Image.asset(
-                          'record_pause'.assetWebp,
-
-                          ///todo 根据播放状态切换图片
-                          height: 33.h,
-                          width: 33.w,
-                        ),
-                      )
+                      GestureDetector(
+                          onTap: () {
+                            if (bloc.isRecording) {
+                              return;
+                            }
+                            bloc.add(PlayRecordAudioEvent());
+                          },
+                          child: Visibility(
+                            visible: bloc.currentPageData()?.recordUrl != null,
+                            child: Image.asset(
+                              bloc.isRecordAudioPlaying
+                                  ? 'record_pause'.assetWebp
+                                  : 'record_play'.assetWebp,
+                              height: 33.h,
+                              width: 33.w,
+                            ),
+                          )),
                     ],
                   ),
                 ),
@@ -139,7 +139,11 @@ class AppRouter {
             pageBuilder: (_, __, ___) => const TabPage(),
             transitionsBuilder: (_, __, ___, child) => child);
       case AppRouteName.reading:
-        return CupertinoPageRoute(builder: (_) => const ReadingPage());
+        var courseLessonId = '';
+        if (settings.arguments != null) {
+          courseLessonId = (settings.arguments as Map)['courseLessonId'] as String??'';
+        }
+        return CupertinoPageRoute(builder: (_) => ReadingPage(courseLessonId: courseLessonId));
       default:
         return CupertinoPageRoute(
             builder: (_) => Scaffold(body: Center(child: Text('No route defined for ${settings.name}'))));