diff --git a/android/app/src/main/assets/vad.0.1.bin b/android/app/src/main/assets/vad.0.1.bin old mode 100755 new mode 100644 index 7777ef6..7256990 --- a/android/app/src/main/assets/vad.0.1.bin +++ b/android/app/src/main/assets/vad.0.1.bin diff --git a/android/app/src/main/kotlin/com/kouyuxingqiu/wow_english/methodChannels/SingSoungMethodChannel.kt b/android/app/src/main/kotlin/com/kouyuxingqiu/wow_english/methodChannels/SingSoungMethodChannel.kt index d68998c..a832bce 100644 --- a/android/app/src/main/kotlin/com/kouyuxingqiu/wow_english/methodChannels/SingSoungMethodChannel.kt +++ b/android/app/src/main/kotlin/com/kouyuxingqiu/wow_english/methodChannels/SingSoungMethodChannel.kt @@ -2,9 +2,13 @@ package com.kouyuxingqiu.wow_english.methodChannels import android.util.Log import com.kouyuxingqiu.wow_english.singsound.SingEngineHelper +import com.kouyuxingqiu.wow_english.singsound.SingEngineHelper.init +import com.kouyuxingqiu.wow_english.singsound.SingEngineLifecycles +import com.kouyuxingqiu.wow_english.util.GlobalHandler import io.flutter.embedding.android.FlutterActivity import io.flutter.embedding.engine.FlutterEngine import io.flutter.plugin.common.MethodChannel +import org.json.JSONObject import java.lang.ref.WeakReference /** @@ -12,13 +16,13 @@ import java.lang.ref.WeakReference * @date: 2023/6/27 00:32 * @description: */ -class SingSoungMethodChannel(val activity: FlutterActivity, val flutterEngine: FlutterEngine) { +class SingSoungMethodChannel(activity: FlutterActivity, flutterEngine: FlutterEngine): SingEngineLifecycles.OnSingEngineAdapter() { private var methodChannel: MethodChannel? = null companion object { var channel: WeakReference? = null - fun invokeMethod(method: String, arguments: Any) { + fun invokeMethod(method: String, arguments: Any?) { channel?.get()?.methodChannel?.invokeMethod(method, arguments) } } @@ -30,10 +34,11 @@ class SingSoungMethodChannel(val activity: FlutterActivity, val flutterEngine: F flutterEngine.dartExecutor.binaryMessenger, "wow_english/sing_sound_method_channel" ) + init(activity) methodChannel?.setMethodCallHandler { call, result -> when (call.method) { "initVoiceSdk" -> { - SingEngineHelper.init(activity) + } "startVoice" -> { val paramMap = call.arguments as HashMap @@ -43,6 +48,7 @@ class SingSoungMethodChannel(val activity: FlutterActivity, val flutterEngine: F } "stopVoice" -> { Log.d("WQF", "SingSoungMethodChannel stopVoice") + SingEngineHelper.stopRecord() } else -> { result.notImplemented() @@ -51,5 +57,26 @@ class SingSoungMethodChannel(val activity: FlutterActivity, val flutterEngine: F } channel = WeakReference(this) + + SingEngineHelper.addOnResultListener(this) + } + + override fun onResult(jsonObject: JSONObject, evalType: Int?) { + //先声回调在子线程,需要切换到主线程 + GlobalHandler.runOnMainThread { + invokeMethod("voiceResult", jsonObject.toString()) + } + } + + override fun onRecordBegin() { + GlobalHandler.runOnMainThread { + invokeMethod("voiceStart", null) + } + } + + override fun onRecordStop() { + GlobalHandler.runOnMainThread { + invokeMethod("voiceEnd", null) + } } } \ No newline at end of file diff --git a/android/app/src/main/kotlin/com/kouyuxingqiu/wow_english/singsound/SingEngineHelper.kt b/android/app/src/main/kotlin/com/kouyuxingqiu/wow_english/singsound/SingEngineHelper.kt index 6b59ccb..465f04d 100644 --- a/android/app/src/main/kotlin/com/kouyuxingqiu/wow_english/singsound/SingEngineHelper.kt +++ b/android/app/src/main/kotlin/com/kouyuxingqiu/wow_english/singsound/SingEngineHelper.kt @@ -73,7 +73,7 @@ object SingEngineHelper : // 设置录音音频路径 wavPath = AiUtil.getFilesDir(context).path + "/userdata/sound_record/" // 设置是否开启 VAD 功能 - setOpenVad(true, "vad.0.1.bin") +// setOpenVad(true, "vad.0.1.bin") //setOpenVad(false, null); // 设置 VAD 前置超时时间 setFrontVadTime(3000) @@ -383,6 +383,7 @@ object SingEngineHelper : } fun addOnResultListener(listener: SingEngineLifecycles) { + Log.i(TAG, "addOnResultListener") if (mListeners?.contains(listener) == true) { return } @@ -390,6 +391,7 @@ object SingEngineHelper : } fun removeOnResultListener(listener: SingEngineLifecycles) { + Log.i(TAG, "removeOnResultListener") if (mListeners?.contains(listener) == true) { mListeners?.remove(listener) } diff --git a/android/app/src/main/kotlin/com/kouyuxingqiu/wow_english/singsound/bean/BaseResultModel.kt b/android/app/src/main/kotlin/com/kouyuxingqiu/wow_english/singsound/bean/BaseResultModel.kt index 7420463..2005dd6 100644 --- a/android/app/src/main/kotlin/com/kouyuxingqiu/wow_english/singsound/bean/BaseResultModel.kt +++ b/android/app/src/main/kotlin/com/kouyuxingqiu/wow_english/singsound/bean/BaseResultModel.kt @@ -19,7 +19,9 @@ class BaseResultModel( constructor(parcel: Parcel) : this( parcel.readString(), parcel.readDouble(), - TODO("scores"), + mutableListOf().apply { + parcel.readTypedList(this, SingleResultModel.CREATOR) + }, parcel.readDouble(), parcel.readDouble() ) { @@ -59,6 +61,7 @@ class BaseResultModel( override fun writeToParcel(parcel: Parcel, flags: Int) { parcel.writeString(originText) parcel.writeDouble(score) + parcel.writeTypedList(scores) parcel.writeDouble(pronounce) parcel.writeDouble(fluency) } diff --git a/android/app/src/main/kotlin/com/kouyuxingqiu/wow_english/util/GlobalHandler.kt b/android/app/src/main/kotlin/com/kouyuxingqiu/wow_english/util/GlobalHandler.kt new file mode 100644 index 0000000..ea7cefd --- /dev/null +++ b/android/app/src/main/kotlin/com/kouyuxingqiu/wow_english/util/GlobalHandler.kt @@ -0,0 +1,23 @@ +package com.kouyuxingqiu.wow_english.util + +import android.os.Handler +import android.os.Looper + +/** + * @author: stay + * @date: 2023/7/1 16:55 + * @description: 全局Handler, 用于切线程等操作 + */ +object GlobalHandler { + private var handler: Handler? = null + + init { + handler = Handler(Looper.getMainLooper()) + } + + fun runOnMainThread(runnable: Runnable?) { + runnable?.let { + handler?.post(it) + } + } +} \ No newline at end of file diff --git a/ios/Runner/XSMessageMehtodChannel.swift b/ios/Runner/XSMessageMehtodChannel.swift index a4a5c57..430e093 100644 --- a/ios/Runner/XSMessageMehtodChannel.swift +++ b/ios/Runner/XSMessageMehtodChannel.swift @@ -13,7 +13,7 @@ class XSMessageMehtodChannel: NSObject,SSOralEvaluatingManagerDelegate { init(message:FlutterBinaryMessenger) { super.init() resultData = Dictionary() - messageChannel = FlutterMethodChannel.init(name: "wow_english/sing_sound_method_channely", binaryMessenger: message) + messageChannel = FlutterMethodChannel.init(name: "wow_english/sing_sound_method_channel", binaryMessenger: message) messageChannel!.setMethodCallHandler { call, result in self.handle(call, result) } diff --git a/lib/models/course_process_entity.dart b/lib/models/course_process_entity.dart index dacf237..80e1cf1 100644 --- a/lib/models/course_process_entity.dart +++ b/lib/models/course_process_entity.dart @@ -34,6 +34,8 @@ class CourseProcessReadings { String? picUrl; int? sortOrder; String? word; + String? recordUrl; + String? recordScore; CourseProcessReadings(); diff --git a/lib/pages/home/home_page.dart b/lib/pages/home/home_page.dart index 17588a2..03c9989 100644 --- a/lib/pages/home/home_page.dart +++ b/lib/pages/home/home_page.dart @@ -116,6 +116,7 @@ class _HomePageView extends StatelessWidget { return; } if (data.courseType == 4) {//绘本 + Navigator.of(context).pushNamed(AppRouteName.reading, arguments: {'courseLessonId':data.id!}); return; } diff --git a/lib/pages/practice/bloc/topic_picture_bloc.dart b/lib/pages/practice/bloc/topic_picture_bloc.dart index 252d86b..d86a450 100644 --- a/lib/pages/practice/bloc/topic_picture_bloc.dart +++ b/lib/pages/practice/bloc/topic_picture_bloc.dart @@ -90,7 +90,7 @@ class TopicPictureBloc extends Bloc { add(VoicePlayStateChangeEvent()); }); - methodChannel = const MethodChannel('wow_english/sing_sound_method_channely'); + methodChannel = const MethodChannel('wow_english/sing_sound_method_channel'); methodChannel.setMethodCallHandler((call) async { if (call.method == 'voiceResult') {//评测结果 add(XSVoiceResultEvent(call.arguments)); @@ -181,7 +181,7 @@ class TopicPictureBloc extends Bloc { ///先声测试 void _voiceXsTest(XSVoiceTestEvent event,Emitter emitter) async { - audioPlayer.stop(); + await audioPlayer.stop(); methodChannel.invokeMethod( 'startVoice', {'word':event.testWord,'type':event.type,'userId':event.userId.toString()} diff --git a/lib/pages/reading/bloc/reading_bloc.dart b/lib/pages/reading/bloc/reading_bloc.dart index c36aac6..50056d6 100644 --- a/lib/pages/reading/bloc/reading_bloc.dart +++ b/lib/pages/reading/bloc/reading_bloc.dart @@ -10,13 +10,30 @@ import '../../../common/request/dao/listen_dao.dart'; import '../../../common/request/exception.dart'; import '../../../models/course_process_entity.dart'; import '../../../utils/loading.dart'; +import 'dart:convert'; part 'reading_event.dart'; part 'reading_state.dart'; +enum VoicePlayState { + ///未知 + unKnow, + + ///播放中 + playing, + + ///播放完成 + completed, + + ///播放终止 + stop +} + class ReadingPageBloc extends Bloc { final PageController pageController; + final String courseLessonId; + ///当前页索引 int _currentPage = 0; @@ -36,40 +53,98 @@ class ReadingPageBloc extends Bloc { bool get isRecording => _isRecording; + ///原始音频是否正在播放 + bool _isOriginAudioPlaying = false; + + bool get isOriginAudioPlaying => _isOriginAudioPlaying; + + ///录音音频是否正在播放 + bool _isRecordAudioPlaying = false; + + bool get isRecordAudioPlaying => _isRecordAudioPlaying; + + ///正在播放音频状态 + VoicePlayState _voicePlayState = VoicePlayState.unKnow; + + VoicePlayState get voicePlayState => _voicePlayState; + late MethodChannel methodChannel; late AudioPlayer audioPlayer; - ReadingPageBloc(this.pageController) : super(ReadingPageInitial()) { + ReadingPageBloc(this.pageController, this.courseLessonId) + : super(ReadingPageInitial()) { on(_pageControllerChange); - on(_selectItemLoad); + on(_playModeChange); // pageController.addListener(() { // _currentPage = pageController.page!.round(); // }); on(_requestData); - on((event, emit) { + on((event, emit) { //音频播放器 audioPlayer = AudioPlayer(); - audioPlayer.onPlayerStateChanged.listen((event) { + audioPlayer.onPlayerStateChanged.listen((event) async { + debugPrint('播放状态变化'); if (event == PlayerState.completed) { - if (kDebugMode) { - print('绘本播放完成'); + debugPrint('播放完成'); + _voicePlayState = VoicePlayState.completed; + } + if (event == PlayerState.stopped) { + debugPrint('播放结束'); + _voicePlayState = VoicePlayState.stop; + } - } + if (event == PlayerState.playing) { + debugPrint('正在播放中'); + _voicePlayState = VoicePlayState.playing; + } + if (isClosed) { + return; } + add(VoicePlayStateChangeEvent()); }); - methodChannel = const MethodChannel('sing_sound_method_channel'); - methodChannel.invokeMethod('initVoiceSdk',{}); + methodChannel = + const MethodChannel('wow_english/sing_sound_method_channel'); + methodChannel.invokeMethod('initVoiceSdk', {}); //初始化评测 methodChannel.setMethodCallHandler((call) async { - if (call.method == 'voiceResult') {//评测结束 - // add(XSVoiceResultEvent(call.arguments)); + if (call.method == 'voiceResult') { + //评测结果 + add(XSVoiceResultEvent(call.arguments)); + return; + } + + if (call.method == 'voiceStart') { + //评测开始 + if (kDebugMode) { + print('评测开始'); + } + return; + } + + if (call.method == 'voiceEnd') { + //评测结束 + if (kDebugMode) { + print('评测结束'); + } + return; + } + + if (call.method == 'voiceFail') { + //评测失败 + EasyLoading.showToast('评测失败'); + return; } }); }); + on(_voicePlayStateChange); on(_playOriginalAudio); - on(_voiceXsTest); + on(_initVoiceSdk); + on(_voiceXsStart); + on(_voiceXsStop); on(_voiceXsResult); + + on(_playRecordAudio); } @override @@ -83,11 +158,11 @@ class ReadingPageBloc extends Bloc { void _pageControllerChange(CurrentPageIndexChangeEvent event, Emitter emitter) async { _currentPage = event.pageIndex; - _playOriginVoice(null); + _playOriginalAudioInner(null); emitter(CurrentPageIndexState()); } - void _selectItemLoad( + void _playModeChange( CurrentModeChangeEvent event, Emitter emitter) async { if (_currentMode == ReadingModeType.auto) { _currentMode = ReadingModeType.manual; @@ -102,7 +177,7 @@ class ReadingPageBloc extends Bloc { RequestDataEvent event, Emitter emitter) async { try { await loading(() async { - _entity = await ListenDao.process('1'); + _entity = await ListenDao.process(courseLessonId); print("reading page entity: ${_entity!.toJson()}"); emitter(RequestDataState()); }); @@ -113,20 +188,36 @@ class ReadingPageBloc extends Bloc { } } - void _playOriginalAudio(PlayOriginalAudioEvent event, Emitter emitter) async { - print("_playOriginalAudio"); - _playOriginVoice(event.url); + /// 播放绘本原音 + void _playOriginalAudio( + PlayOriginalAudioEvent event, Emitter emitter) async { + _playOriginalAudioInner(event.url); + } + + void _playOriginalAudioInner(String? audioUrl) { + print("_playOriginalAudio url=$audioUrl"); + audioUrl ??= currentPageData()?.audioUrl ?? ''; + _playAudio(audioUrl); + _isOriginAudioPlaying = true; } - /// 播放绘本原音 - void _playOriginVoice(String? audioUrl) async { - audioPlayer.stop(); - final readingData = currentPageData(); - if (readingData?.audioUrl != null) { - final urlStr = audioUrl ?? readingData?.audioUrl ?? ''; - if (urlStr.isNotEmpty) { - audioPlayer.play(UrlSource(urlStr)); - } + /// 播放录音 + void _playRecordAudio( + PlayRecordAudioEvent event, Emitter emitter) async { + _playRecordAudioInner(); + } + + void _playRecordAudioInner() { + final recordAudioUrl = currentPageData()?.recordUrl; + print("_playRecordAudioInner url=${currentPageData()?.recordUrl}"); + _playAudio(recordAudioUrl); + _isRecordAudioPlaying = true; + } + + void _playAudio(String? audioUrl) async { + await audioPlayer.stop(); + if (audioUrl!.isNotEmpty) { + await audioPlayer.play(UrlSource(audioUrl)); } } @@ -139,8 +230,16 @@ class ReadingPageBloc extends Bloc { return _entity?.readings?[_currentPage]; } + ///初始化SDK + _initVoiceSdk( + XSVoiceInitEvent event, Emitter emitter) async { + methodChannel.invokeMethod('initVoiceSdk', event.data); + } + ///先声测试 - void _voiceXsTest(XSVoiceTestEvent event, Emitter emitter) async { + void _voiceXsStart( + XSVoiceStartEvent event, Emitter emitter) async { + _stopAudio(); startRecord(event.content); emitter(XSVoiceTestState()); } @@ -149,24 +248,44 @@ class ReadingPageBloc extends Bloc { if (_isRecording == true) { return; } - EasyLoading.show(status: '录音中....'); methodChannel.invokeMethod( - 'startVoice', - {'word':'how old are you','type':'0','userId':'1'} - ); + 'startVoice', {'word': 'how old are you', 'type': '0', 'userId': '1'}); _isRecording = true; } - void _voiceXsResult(XSVoiceResultEvent event,Emitter emitter) async { - final Map args = event.message as Map; - final result = args['result'] as String; - if (result == '1') { - final overall = args['overall'].toString(); - EasyLoading.showToast('测评成功,分数是$overall',duration: const Duration(seconds: 10)); + void _voiceXsResult( + XSVoiceResultEvent event, Emitter emitter) async { + final Map args = json.decode(event.message); + final result = args['result']; + print("_voiceXsResult result=${result}"); + if (result != null) { + final overall = result['overall'].toString(); + EasyLoading.showToast('测评成功,分数是$overall', + duration: const Duration(seconds: 10)); + currentPageData()?.recordScore = overall; + currentPageData()?.recordUrl = args['audioUrl'] + '.mp3'; + _playRecordAudioInner(); } else { - EasyLoading.showToast('测评失败',duration: const Duration(seconds: 10)); + EasyLoading.showToast('测评失败', duration: const Duration(seconds: 10)); } _isRecording = false; emitter(XSVoiceTestState()); } + + ///终止评测 + void _voiceXsStop( + XSVoiceStopEvent event, Emitter emitter) async { + methodChannel.invokeMethod('stopVoice'); + } + + void _voicePlayStateChange(VoicePlayStateChangeEvent event, + Emitter emitter) async { + emitter(VoicePlayStateChange()); + } + + void _stopAudio() async { + await audioPlayer.stop(); + _isOriginAudioPlaying = false; + _isRecordAudioPlaying = false; + } } diff --git a/lib/pages/reading/bloc/reading_event.dart b/lib/pages/reading/bloc/reading_event.dart index cbcd3ea..0373230 100644 --- a/lib/pages/reading/bloc/reading_event.dart +++ b/lib/pages/reading/bloc/reading_event.dart @@ -3,6 +3,9 @@ part of 'reading_bloc.dart'; @immutable abstract class ReadingPageEvent {} +///页面初始化 +class InitBlocEvent extends ReadingPageEvent {} + class CurrentPageIndexChangeEvent extends ReadingPageEvent { final int pageIndex; CurrentPageIndexChangeEvent(this.pageIndex); @@ -32,9 +35,18 @@ class XSVoiceResultEvent extends ReadingPageEvent { } ///先声测试 -class XSVoiceTestEvent extends ReadingPageEvent { +class XSVoiceStartEvent extends ReadingPageEvent { final String content; final String type; final String userId; - XSVoiceTestEvent(this.content,this.type,this.userId); -} \ No newline at end of file + XSVoiceStartEvent(this.content,this.type,this.userId); +} + +///先声评测停止 +class XSVoiceStopEvent extends ReadingPageEvent {} + +///音频播放状态 +class VoicePlayStateChangeEvent extends ReadingPageEvent {} + +///录音播放 +class PlayRecordAudioEvent extends ReadingPageEvent {} \ No newline at end of file diff --git a/lib/pages/reading/bloc/reading_state.dart b/lib/pages/reading/bloc/reading_state.dart index cb76e63..6b0d212 100644 --- a/lib/pages/reading/bloc/reading_state.dart +++ b/lib/pages/reading/bloc/reading_state.dart @@ -13,3 +13,5 @@ class CurrentModeState extends ReadingPageState {} class RequestDataState extends ReadingPageState {} class XSVoiceTestState extends ReadingPageState {} + +class VoicePlayStateChange extends ReadingPageState {} diff --git a/lib/pages/reading/reading_page.dart b/lib/pages/reading/reading_page.dart index 1993f56..01858e8 100644 --- a/lib/pages/reading/reading_page.dart +++ b/lib/pages/reading/reading_page.dart @@ -9,12 +9,16 @@ import '../../models/course_process_entity.dart'; import 'bloc/reading_bloc.dart'; class ReadingPage extends StatelessWidget { - const ReadingPage({super.key}); + const ReadingPage({super.key, this.courseLessonId}); + + final String? courseLessonId; @override Widget build(BuildContext context) { return BlocProvider( - create: (_) => ReadingPageBloc(PageController())..add(RequestDataEvent()), + create: (_) => ReadingPageBloc(PageController(), courseLessonId ?? '') + ..add(InitBlocEvent()) + ..add(RequestDataEvent()), child: _ReadingPage(), ); } @@ -135,11 +139,13 @@ class _ReadingPage extends StatelessWidget { if (bloc.isRecording) { return; } - print("voice tap"); bloc.add(PlayOriginalAudioEvent(null)); }, child: Image.asset( - 'voice'.assetPng, + bloc.voicePlayState == VoicePlayState.playing && + bloc.isOriginAudioPlaying + ? 'reade_answer'.assetGif + : 'voice'.assetPng, height: 40.h, width: 45.w, ), @@ -161,9 +167,13 @@ class _ReadingPage extends StatelessWidget { GestureDetector( onTap: () { if (bloc.isRecording) { - return; + bloc.add(XSVoiceStopEvent()); + } else { + bloc.add(XSVoiceStartEvent( + bloc.currentPageData()?.word ?? '', + '0', + UserUtil.getUser()!.id.toString())); } - bloc.add(XSVoiceTestEvent(bloc.currentPageData()?.word??'', '0',UserUtil.getUser()!.id.toString())); }, child: Image.asset( 'micro_phone'.assetPng, @@ -173,18 +183,23 @@ class _ReadingPage extends StatelessWidget { SizedBox( width: 10.w, ), - Visibility( - visible: false, - - ///todo 依据是否录过音 - child: Image.asset( - 'record_pause'.assetWebp, - - ///todo 根据播放状态切换图片 - height: 33.h, - width: 33.w, - ), - ) + GestureDetector( + onTap: () { + if (bloc.isRecording) { + return; + } + bloc.add(PlayRecordAudioEvent()); + }, + child: Visibility( + visible: bloc.currentPageData()?.recordUrl != null, + child: Image.asset( + bloc.isRecordAudioPlaying + ? 'record_pause'.assetWebp + : 'record_play'.assetWebp, + height: 33.h, + width: 33.w, + ), + )), ], ), ), diff --git a/lib/route/route.dart b/lib/route/route.dart index eee4b75..c2071fd 100644 --- a/lib/route/route.dart +++ b/lib/route/route.dart @@ -139,7 +139,11 @@ class AppRouter { pageBuilder: (_, __, ___) => const TabPage(), transitionsBuilder: (_, __, ___, child) => child); case AppRouteName.reading: - return CupertinoPageRoute(builder: (_) => const ReadingPage()); + var courseLessonId = ''; + if (settings.arguments != null) { + courseLessonId = (settings.arguments as Map)['courseLessonId'] as String??''; + } + return CupertinoPageRoute(builder: (_) => ReadingPage(courseLessonId: courseLessonId)); default: return CupertinoPageRoute( builder: (_) => Scaffold(body: Center(child: Text('No route defined for ${settings.name}'))));