#ifdef SUPPORT_MS_EXTENSIONS #pragma bss_seg(".jl_kws_platform.data.bss") #pragma data_seg(".jl_kws_platform.data") #pragma const_seg(".jl_kws_platform.text.const") #pragma code_seg(".jl_kws_platform.text") #endif /***************************************************************** >file name : jl_asr.c >create time : Sat 16 Apr 2022 09:46:07 AM CST *****************************************************************/ #define LOG_TAG "[Smart-Voice]" #define LOG_INFO_ENABLE #define LOG_DEBUG_ENABLE #define LOG_DUMP_ENABLE #define LOG_ERROR_ENABLE #define LOG_WARN_ENABLE #include "smart_voice.h" #include "debug.h" #include "includes.h" #include "voice_mic_data.h" #include "kws_event.h" #include "nn_vad.h" #include "asr/jl_kws.h" #include "clock_manager/clock_manager.h" #if CONFIG_VAD_PLATFORM_SUPPORT_EN #include "vad_mic.h" #endif /*CONFIG_VAD_PLATFORM_SUPPORT_EN*/ #if ((defined TCFG_SMART_VOICE_ENABLE) && TCFG_SMART_VOICE_ENABLE && !TCFG_AUDIO_ASR_DEVELOP) #define SMART_VOICE_TEST_LISTEN_SOUND 0 #define SMART_VOICE_TEST_PRINT_PCM 0 #define SMART_VOICE_TEST_WRITE_FILE 0 #define SMART_VOICE_DEBUG_KWS_RESULT 0 #define AUDIO_NN_VAD_ENABLE 0 /* * Audio语音识别 */ struct smart_voice_context { u8 kws_model; void *kws; void *mic; void *nn_vad; #if SMART_VOICE_TEST_WRITE_FILE void *file; #endif #if SMART_VOICE_DEBUG_KWS_RESULT void *dump_hdl; #endif }; extern const int config_jl_audio_kws_enable; #define SMART_VOICE_SAMPLE_RATE 16000 #define SMART_VOICE_REC_MIC_SECS 8 #define SMART_VOICE_REC_DATA_LEN (SMART_VOICE_SAMPLE_RATE * SMART_VOICE_REC_MIC_SECS * 2) #define SMART_VOICE_KWS_FRAME_LEN (320) #if SMART_VOICE_TEST_WRITE_FILE #define VOICE_DATA_BUFFER_SIZE 16 * 1024 #elif SMART_VOICE_TEST_PRINT_PCM #define VOICE_DATA_BUFFER_SIZE SMART_VOICE_REC_DATA_LEN #else #define VOICE_DATA_BUFFER_SIZE 2 * 1024 #endif #if ((defined TCFG_AUDIO_DATA_EXPORT_ENABLE && TCFG_AUDIO_DATA_EXPORT_ENABLE) || SMART_VOICE_TEST_WRITE_FILE) #define CONFIG_VAD_KWS_DETECT_ENABLE 0 #else #define CONFIG_VAD_KWS_DETECT_ENABLE 1 #endif static struct smart_voice_context *this_sv = NULL; static u8 volatile smart_voice_wakeup = 0; #if SMART_VOICE_TEST_LISTEN_SOUND #include "audio_config.h" extern struct audio_dac_hdl dac_hdl; #endif static inline void smart_voice_data_listen_sound(void *data, int len) { #if SMART_VOICE_TEST_LISTEN_SOUND if (audio_dac_is_working(&dac_hdl)) { audio_dac_write(&dac_hdl, data, len); } #endif } static inline void smart_voice_data_write_file(struct smart_voice_context *sv, void *data, int len) { #if SMART_VOICE_TEST_WRITE_FILE if (sv->file) { fwrite(data, len, 1, sv->file); } #endif } static void voice_mic_data_debug_stop(struct smart_voice_context *sv) { #if SMART_VOICE_TEST_PRINT_PCM if (sv->mic) { voice_mic_data_dump(sv->mic); } #endif #if SMART_VOICE_TEST_WRITE_FILE if (sv->file) { fclose(sv->file); sv->file = NULL; } #endif } static void voice_mic_data_debug_start(struct smart_voice_context *sv) { #if SMART_VOICE_TEST_WRITE_FILE sv->file = fopen("storage/sd0/C/AudioVAD/vad***.raw", "w+"); if (!sv->file) { printf("Open file failed, can not test.\n"); } #endif } static void smart_voice_kws_close(struct smart_voice_context *sv) { #if AUDIO_NN_VAD_ENABLE if (sv->nn_vad) { audio_nn_vad_close(sv->nn_vad); sv->nn_vad = NULL; } #else if (sv->kws) { audio_kws_close(sv->kws); sv->kws = NULL; } #endif } static void smart_voice_kws_open(struct smart_voice_context *sv, u8 model) { #if AUDIO_NN_VAD_ENABLE if (sv->nn_vad) { return; } sv->nn_vad = audio_nn_vad_open(); #else if (sv->kws) { if (sv->kws_model == model) { return; } smart_voice_kws_close(sv); } sv->kws_model = model; sv->kws = audio_kws_open(model, NULL);//kws_model_files[model]); #endif } /* * 语音识别的KWS处理 */ static int smart_voice_data_handler(struct smart_voice_context *sv) { if (!config_jl_audio_kws_enable) { return 0; } #if SMART_VOICE_TEST_PRINT_PCM putchar('*'); return 1; #endif if (!sv->mic) { return -EINVAL; } s16 data[SMART_VOICE_KWS_FRAME_LEN / 2]; int rlen = voice_mic_data_read(sv->mic, data, SMART_VOICE_KWS_FRAME_LEN); if (rlen < SMART_VOICE_KWS_FRAME_LEN) { return -EINVAL; } if (sv->nn_vad) { #if AUDIO_NN_VAD_ENABLE audio_nn_vad_data_handler(sv->nn_vad, data, sizeof(data)); #endif } if (sv->kws) { /*putchar('*');*/ smart_voice_data_listen_sound(data, sizeof(data)); smart_voice_data_write_file(sv, data, sizeof(data)); #if (CONFIG_VAD_KWS_DETECT_ENABLE) int result = audio_kws_detect_handler(sv->kws, (void *)data, sizeof(data)); if (result > 1) { printf("result : %d\n", result); } smart_voice_kws_event_handler(sv->kws_model, result); #endif #if SMART_VOICE_DEBUG_KWS_RESULT smart_voice_kws_dump_result_add(sv->dump_hdl, sv->kws_model, result); #endif } return 0; } int smart_voice_core_handler(void *priv, int taskq_type, int *msg) { int err = ASR_CORE_STANDBY; struct smart_voice_context *sv = (struct smart_voice_context *)priv; if (taskq_type == OS_TASKQ) { switch (msg[0]) { case SMART_VOICE_MSG_MIC_OPEN: sv->mic = voice_mic_data_open(msg[1], msg[2], msg[3]); if (!sv->mic) { printf("VAD mic open failed.\n"); } smart_voice_kws_open(sv, msg[4]); break; case SMART_VOICE_MSG_SWITCH_SOURCE: if (sv->mic) { voice_mic_data_clear(sv->mic); voice_mic_data_switch_source(sv->mic, msg[1], msg[2], msg[3]); } smart_voice_kws_open(sv, msg[4]); break; case SMART_VOICE_MSG_MIC_CLOSE: smart_voice_kws_close(sv); voice_mic_data_close(sv->mic); sv->mic = NULL; os_sem_post((OS_SEM *)msg[1]); break; case SMART_VOICE_MSG_WAKE: err = ASR_CORE_WAKEUP; putchar('W'); voice_mic_data_debug_start(sv); smart_voice_wakeup = 1; break; case SMART_VOICE_MSG_STANDBY: smart_voice_wakeup = 0; voice_mic_data_debug_stop(sv); if (sv->mic) { voice_mic_data_clear(sv->mic); } break; case SMART_VOICE_MSG_DMA: err = ASR_CORE_WAKEUP; smart_voice_wakeup = 1; /* msg[0] = (int)audio_codec_clock_check; */ /* msg[1] = 0; */ /* os_taskq_post_type("app_core", Q_CALLBACK, 2, msg); */ /*audio_codec_clock_check();*/ break; } } if (smart_voice_wakeup) { err = smart_voice_data_handler(sv); err = err ? ASR_CORE_STANDBY : ASR_CORE_WAKEUP; } return err; } int audio_smart_voice_detect_create(u8 model, u8 mic, int buffer_size) { int err = 0; // printf("audio_smart_voice_detect_create"); if (!config_jl_audio_kws_enable) { return 0; } if (this_sv) { smart_voice_core_post_msg(5, SMART_VOICE_MSG_SWITCH_SOURCE, mic, buffer_size, SMART_VOICE_SAMPLE_RATE, model); return 0; } struct smart_voice_context *sv = (struct smart_voice_context *)zalloc(sizeof(struct smart_voice_context)); if (!sv) { goto __err; } smart_voice_wakeup = 0; err = smart_voice_core_create(sv); if (err) { goto __err; } clock_alloc("jl_kws", 90 * 1000000L); smart_voice_core_post_msg(5, SMART_VOICE_MSG_MIC_OPEN, mic, buffer_size, SMART_VOICE_SAMPLE_RATE, model); #if SMART_VOICE_DEBUG_KWS_RESULT if (!sv->dump_hdl) { sv->dump_hdl = smart_voice_kws_dump_open(2000); } #endif this_sv = sv; return 0; __err: if (sv) { free(sv); } return err; } void audio_smart_voice_detect_close(void) { // printf("audio_smart_voice_detect_close"); if (config_jl_audio_kws_enable && this_sv) { #if SMART_VOICE_DEBUG_KWS_RESULT smart_voice_kws_dump_close(this_sv->dump_hdl); #endif OS_SEM *sem = (OS_SEM *)malloc(sizeof(OS_SEM)); os_sem_create(sem, 0); smart_voice_core_post_msg(2, SMART_VOICE_MSG_MIC_CLOSE, (int)sem); os_sem_pend(sem, 0); free(sem); smart_voice_core_free(); free(this_sv); this_sv = NULL; clock_free("jl_kws"); } } static void __audio_smart_voice_detect_open(u8 mic, u8 model) { #if SMART_VOICE_TEST_WRITE_FILE extern void force_set_sd_online(char *sdx); force_set_sd_online("sd0"); void *mnt = mount("sd0", "storage/sd0", "fat", 3, NULL); if (!mnt) { printf("sd0 mount fat failed.\n"); } #endif audio_smart_voice_detect_create(model, mic, VOICE_DATA_BUFFER_SIZE); } void audio_smart_voice_detect_open(u8 model) { // printf("audio_smart_voice_detect_open "); #if CONFIG_VAD_PLATFORM_SUPPORT_EN return __audio_smart_voice_detect_open(model == JL_KWS_COMMAND_KEYWORD ? VOICE_VAD_MIC : VOICE_MCU_MIC, model); #else return __audio_smart_voice_detect_open(VOICE_MCU_MIC, model); #endif /*CONFIG_VAD_PLATFORM_SUPPORT_EN*/ } int audio_smart_voice_detect_init(void *vad_mic_data) { #if CONFIG_VAD_PLATFORM_SUPPORT_EN u8 mic_type = VOICE_VAD_MIC; lp_vad_mic_data_init((struct vad_mic_platform_data *)vad_mic_data); #else u8 mic_type = VOICE_MCU_MIC; #endif /*CONFIG_VAD_PLATFORM_SUPPORT_EN*/ __audio_smart_voice_detect_open(mic_type, JL_KWS_COMMAND_KEYWORD); return 0; } /* * 来电KWS关键词识别 */ int audio_phone_call_kws_start(void) { // printf("audio_phone_call_kws_start"); if (this_sv) { /*通话语音识别,由LP VAD的MIC切到系统主MIC进行识别*/ smart_voice_core_post_msg(5, SMART_VOICE_MSG_SWITCH_SOURCE, VOICE_MCU_MIC, VOICE_DATA_BUFFER_SIZE, SMART_VOICE_SAMPLE_RATE, JL_KWS_CALL_KEYWORD); return 0; } __audio_smart_voice_detect_open(VOICE_MCU_MIC, JL_KWS_CALL_KEYWORD); return 0; } /* * 来电KWS关闭(接通或拒接) */ int audio_phone_call_kws_close(void) { // printf("audio_phone_call_kws_close"); if (!this_sv) { return 0; } /*通话语音识别结束后,由系统的主MIC切换回LP VAD的MIC源*/ #if CONFIG_VAD_PLATFORM_SUPPORT_EN u8 mic_type = VOICE_VAD_MIC; #else u8 mic_type = VOICE_MCU_MIC; #endif /*CONFIG_VAD_PLATFORM_SUPPORT_EN*/ smart_voice_core_post_msg(5, SMART_VOICE_MSG_SWITCH_SOURCE, mic_type, VOICE_DATA_BUFFER_SIZE, SMART_VOICE_SAMPLE_RATE, JL_KWS_COMMAND_KEYWORD); return 0; } int audio_smart_voice_kws_get_model(void) { if (this_sv) { return this_sv->kws_model; } return -1; } #if !TCFG_VAD_LP_CLOSE static u8 smart_voice_idle_query(void) { return !smart_voice_wakeup; } static enum LOW_POWER_LEVEL smart_voice_level_query(void) { return LOW_POWER_MODE_DEEP_SLEEP; } REGISTER_LP_TARGET(smart_voice_lp_target) = { .name = "smart_voice", .level = smart_voice_level_query, .is_idle = smart_voice_idle_query, }; #endif void audio_vad_test(void) { #if SMART_VOICE_TEST_LISTEN_SOUND app_audio_state_switch(APP_AUDIO_STATE_MUSIC, get_max_sys_vol()); audio_dac_set_sample_rate(&dac_hdl, SMART_VOICE_SAMPLE_RATE); audio_dac_set_volume(&dac_hdl, 15); audio_dac_start(&dac_hdl); audio_vad_m2p_event_post(M2P_VAD_CMD_TEST); while (1) { os_time_dly(10); } #endif } #endif