111
This commit is contained in:
HP
parent
45f8aa2b58
commit
ad63d5640b
|
|
@ -14,8 +14,8 @@ add_executable(${TOM_GAME_TARGET}
|
|||
src/gameplay/TomHud.cpp
|
||||
src/gameplay/VoiceInteractionController.cpp
|
||||
src/recognition/KeywordRecognizer.cpp
|
||||
src/recognition/TinyKwsRecognizer.cpp
|
||||
src/recognition/TinyKwsModelData.cpp
|
||||
src/recognition/ResBnKwsRecognizer.cpp
|
||||
src/recognition/ResBnKwsModelData.cpp
|
||||
src/ui/TomSettingsPanel.cpp
|
||||
${TOM_ATLAS_HEADER}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
#include "Timer.h"
|
||||
#include "app/TomGameApp.h"
|
||||
#include "recognition/KeywordRecognizer.h"
|
||||
#include "recognition/TinyKwsRecognizer.h"
|
||||
#include "recognition/ResBnKwsRecognizer.h"
|
||||
|
||||
#ifdef TARGET_IMX
|
||||
#include "FBDisplay.h"
|
||||
|
|
@ -349,7 +349,7 @@ int main(int argc, char *argv[])
|
|||
Platform::DefaultButtonInput buttonInput;
|
||||
Platform::DefaultPointerInput pointerInput;
|
||||
|
||||
Game::TinyKwsRecognizer keywordRecognizer;
|
||||
Game::ResBnKwsRecognizer keywordRecognizer;
|
||||
keywordRecognizer.set_confidence_threshold(options.kws_threshold);
|
||||
keywordRecognizer.set_input_gain(options.kws_input_gain);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,239 +1,373 @@
|
|||
# Tom Game 语音识别与板端运行
|
||||
# TomGame、Desktop 与板端部署
|
||||
|
||||
本文说明 Tom Game 当前的关键词识别链路、Windows 验证方式，以及 IMX6ULL-ALPHA 板端构建/部署流程。
|
||||
本文说明 TomGame 当前的关键词识别方式，以及 Windows 验证、WSL
|
||||
交叉编译、Windows SCP 部署和 IMX6ULL 板端运行流程。
|
||||
|
||||
## 当前结论
|
||||
TomGame 使用内嵌的 `ResBnKwsRecognizer` INT8 模型，不需要 Python、
|
||||
TensorFlow、TensorFlow Lite 运行库或外部模型文件。
|
||||
|
||||
- 游戏内识别器是 `src/recognition/TinyKwsRecognizer.cpp`。
|
||||
- 模型权重已从 `kws_ref_model_float32.tflite` 导出并内嵌到 `src/recognition/TinyKwsModelData.cpp`。
|
||||
- Windows 和 IMX6ULL 板端现在都默认走同一套 C++ 内嵌推理，不再调用 Python 子进程，也不再链接 TensorFlow Lite C API。
|
||||
- 构建和部署不再需要 `libtensorflowlite_c.so`、TensorFlow Lite 头文件、`TOM_GAME_ENABLE_TFLITE_C_API`、`TOM_GAME_TFLITE_INCLUDE_DIR` 或 `TOM_GAME_TFLITE_LIBRARY`。
|
||||
- `--kws-model` 参数已废弃。模型权重已经编译进可执行文件，运行时不会加载外部 `.tflite` 文件。
|
||||
- 仍可使用 `--kws-threshold` 调整识别置信度阈值，默认是 `0.75`。
|
||||
- 仍可使用 `--kws-input-gain` 调整 KWS 归一化后的输入增益，默认是 `1.0`。
|
||||
- 12 个模型类别是 `Down, Go, Left, No, Off, On, Right, Stop, Up, Yes, Silence, Unknown`。
|
||||
- 当前游戏动作映射是 `Up/On -> Jump`，`Stop -> Idle`，其他类别暂不触发动作。
|
||||
|
||||
## 识别链路
|
||||
|
||||
当前板端完整链路如下：
|
||||
模型支持以下 12 个类别：
|
||||
|
||||
```text
|
||||
AudioInput PCM
|
||||
-> VoiceRecorder
|
||||
-> VoiceInteractionController
|
||||
-> TinyKwsRecognizer
|
||||
-> 16 kHz / mono / trim silence
|
||||
-> MFCC feature extraction
|
||||
-> embedded DS-CNN forward pass
|
||||
-> KeywordCommand
|
||||
-> KeywordCommandRouter
|
||||
-> TomGameState
|
||||
Down, Go, Left, No, Off, On, Right, Stop, Up, Yes, Silence, Unknown
|
||||
```
|
||||
|
||||
内嵌模型只实现当前 Tiny KWS 模型需要的固定算子：
|
||||
当前游戏动作映射：
|
||||
|
||||
```text
|
||||
Conv2D
|
||||
DepthwiseConv2D
|
||||
AveragePool2D
|
||||
Reshape
|
||||
FullyConnected
|
||||
Softmax
|
||||
Up / On -> Jump
|
||||
Stop -> Idle
|
||||
```
|
||||
|
||||
该实现不是通用 TFLite runtime，只服务于当前 `kws_ref_model_float32.tflite` 导出的固定网络结构。
|
||||
|
||||
## Windows 验证
|
||||
|
||||
Windows 游戏现在也默认使用 C++ 内嵌模型。构建并运行游戏后，点击左侧 `ui-hand` 按钮切到语音识别模式，再按录音按钮测试。
|
||||
Windows 版本和板端版本使用同一套 C++ 特征提取、INT8 模型推理和
|
||||
多窗口融合逻辑，不再需要 Python 对照脚本。
|
||||
|
||||
常用运行方式：
|
||||
在项目根目录构建 TomGame：
|
||||
|
||||
```powershell
|
||||
IMX6U-Game.exe --fps 30 --kws-threshold 0.75 --kws-input-gain 1.0
|
||||
cmake --build build --config Release --target IMX6U-Game
|
||||
```
|
||||
|
||||
旧参数 `--kws-model` 会被忽略：
|
||||
直接运行：
|
||||
|
||||
```powershell
|
||||
.\build\Release\IMX6U-Game.exe `
|
||||
--fps 30 `
|
||||
--kws-threshold 0.75 `
|
||||
--kws-input-gain 1.0
|
||||
```
|
||||
|
||||
程序支持以下关键词识别参数：
|
||||
|
||||
```text
|
||||
--kws-model is ignored: Tiny KWS weights are embedded in the executable.
|
||||
--kws-threshold 识别置信度阈值，默认 0.75
|
||||
--kws-input-gain MFCC 输入增益，范围 (0, 1]，默认 1.0
|
||||
```
|
||||
|
||||
Python 脚本仍可作为离线对照工具，用于比较 `.tflite` 原模型输出和 C++ 内嵌推理输出：
|
||||
|
||||
```powershell
|
||||
python src\Apps\Game\tools\kws_python_test.py --wav src\Apps\Game\model\mlcommons-tiny-kws\down_0c40e715_nohash_0.wav
|
||||
python src\Apps\Game\tools\kws_python_test.py --wav src\Apps\Game\model\mlcommons-tiny-kws\no_0cb74144_nohash_1.wav
|
||||
```
|
||||
|
||||
如果需要验证 Python 环境：
|
||||
|
||||
```powershell
|
||||
python -c "import tensorflow as tf; import numpy as np; print(tf.__version__); print(np.__version__)"
|
||||
```
|
||||
|
||||
已知对齐情况：
|
||||
识别成功或未达到阈值时，控制台会打印类似日志：
|
||||
|
||||
```text
|
||||
down_0c40e715_nohash_0.wav:
|
||||
Python TFLite: Down @ ~0.9388
|
||||
C++ embedded: Down @ ~0.9366
|
||||
|
||||
no_0cb74144_nohash_1.wav:
|
||||
Python TFLite: Go @ ~0.7522
|
||||
C++ embedded: Go @ ~0.7080, below default threshold
|
||||
[INFO] ResBN INT8 KWS result: command=Up, confidence=0.92, windows=...
|
||||
```
|
||||
|
||||
`no` 样例处在阈值边缘，C++ 自研 FFT/MFCC 与 Python/TFLite 的浮点细节差异会影响置信度。如果要让边界样例更容易通过，可临时降低阈值，例如：
|
||||
Windows Desktop 和全部游戏可以一起构建：
|
||||
|
||||
```powershell
|
||||
IMX6U-Game.exe --kws-threshold 0.70
|
||||
cmake --build build --config Release --target `
|
||||
IMX6U-Desktop `
|
||||
IMX6U-Game `
|
||||
IMX6U-LightGame `
|
||||
IMX6U-Demo
|
||||
```
|
||||
|
||||
如果日志长期显示 `class=Unknown`，可以先尝试提高输入增益：
|
||||
运行主页：
|
||||
|
||||
```powershell
|
||||
IMX6U-Game.exe --fps 30 --kws-threshold 0.70 --kws-input-gain 1.5
|
||||
.\build\Release\IMX6U-Desktop.exe
|
||||
```
|
||||
|
||||
## IMX6ULL-ALPHA 板端构建
|
||||
Desktop 会从自身所在目录寻找并启动以下程序：
|
||||
|
||||
在 WSL2 项目根目录下构建 framebuffer 板端版本：
|
||||
|
||||
```bash
|
||||
cd ~/IMX6U-Game
|
||||
|
||||
rm -rf build-arm-fb
|
||||
|
||||
cmake -B build-arm-fb \
|
||||
-DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-arm-linux-gnueabihf.cmake \
|
||||
-DTARGET_IMX=ON \
|
||||
.
|
||||
|
||||
cmake --build build-arm-fb --target IMX6U-Game -j$(nproc)
|
||||
```text
|
||||
IMX6U-Game.exe
|
||||
IMX6U-LightGame.exe
|
||||
IMX6U-Demo.exe
|
||||
```
|
||||
|
||||
`TARGET_IMX=ON` 必须能找到 ALSA 头文件和 ARMHF 版 `libasound`。在 WSL 中推荐直接安装 ARMHF 开发包：
|
||||
因此 Desktop 和各游戏的可执行文件需要位于同一个目录。
|
||||
|
||||
## WSL 交叉编译
|
||||
|
||||
### 1. 安装工具链
|
||||
|
||||
首次编译时安装 ARM 交叉编译器和 ALSA 开发文件：
|
||||
|
||||
```bash
|
||||
sudo dpkg --add-architecture armhf
|
||||
sudo apt update
|
||||
sudo apt install -y libasound2-dev:armhf
|
||||
sudo apt install -y \
|
||||
cmake \
|
||||
build-essential \
|
||||
gcc-arm-linux-gnueabihf \
|
||||
g++-arm-linux-gnueabihf \
|
||||
libasound2-dev:armhf
|
||||
```
|
||||
|
||||
由于 ATK-IMX6U 的系统较老，不要链接 WSL 自带的新版 ARMHF `libasound.so`。从板子复制匹配的 ALSA 运行库到项目内：
|
||||
检查交叉编译器：
|
||||
|
||||
```bash
|
||||
arm-linux-gnueabihf-g++ --version
|
||||
```
|
||||
|
||||
### 2. 确认生成资源
|
||||
|
||||
交叉编译不会在板端生成 PNG 图集。开始编译前确认以下文件存在：
|
||||
|
||||
```bash
|
||||
ls src/Apps/Game/generated/tom_atlas.h
|
||||
ls src/Apps/Desktop/generated/desktop_atlas.h
|
||||
```
|
||||
|
||||
如果文件不存在，先在 Windows 主机构建一次对应目标，生成图集后再同步到
|
||||
WSL 源码目录。
|
||||
|
||||
### 3. 配置并编译
|
||||
|
||||
在 WSL 项目根目录执行：
|
||||
|
||||
```bash
|
||||
rm -rf build-arm-fb
|
||||
|
||||
cmake -S . -B build-arm-fb \
|
||||
-DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-arm-linux-gnueabihf.cmake \
|
||||
-DTARGET_IMX=ON \
|
||||
-DCMAKE_BUILD_TYPE=Release
|
||||
|
||||
cmake --build build-arm-fb \
|
||||
--target \
|
||||
IMX6U-Desktop \
|
||||
IMX6U-Game \
|
||||
IMX6U-LightGame \
|
||||
IMX6U-Demo \
|
||||
-j"$(nproc)"
|
||||
```
|
||||
|
||||
查找编译产物：
|
||||
|
||||
```bash
|
||||
find build-arm-fb -type f \
|
||||
\( -name IMX6U-Desktop \
|
||||
-o -name IMX6U-Game \
|
||||
-o -name IMX6U-LightGame \
|
||||
-o -name IMX6U-Demo \)
|
||||
```
|
||||
|
||||
使用 `file` 确认产物是 ARM 程序：
|
||||
|
||||
```bash
|
||||
file "$(find build-arm-fb -type f -name IMX6U-Desktop | head -n 1)"
|
||||
file "$(find build-arm-fb -type f -name IMX6U-Game | head -n 1)"
|
||||
file "$(find build-arm-fb -type f -name IMX6U-LightGame | head -n 1)"
|
||||
file "$(find build-arm-fb -type f -name IMX6U-Demo | head -n 1)"
|
||||
```
|
||||
|
||||
正确结果应包含：
|
||||
|
||||
```text
|
||||
third_party/arm-linux-gnueabihf/alsa/lib/libasound.so
|
||||
third_party/arm-linux-gnueabihf/alsa/lib/libasound.so.2
|
||||
third_party/arm-linux-gnueabihf/alsa/lib/libasound.so.2.0.0
|
||||
ELF 32-bit LSB executable, ARM, EABI5
|
||||
```
|
||||
|
||||
CMake 会优先使用这个项目内库目录；ALSA 头文件可继续来自 WSL 的 `libasound2-dev:armhf`。
|
||||
如果显示 `x86-64`，说明没有使用 ARM 工具链。
|
||||
|
||||
不需要再传这些旧参数：
|
||||
## Windows SCP 部署
|
||||
|
||||
```bash
|
||||
-DTOM_GAME_ENABLE_TFLITE_C_API=ON
|
||||
-DTOM_GAME_TFLITE_INCLUDE_DIR=...
|
||||
-DTOM_GAME_TFLITE_LIBRARY=...
|
||||
```
|
||||
|
||||
构建产物位置可用以下命令确认：
|
||||
|
||||
```bash
|
||||
find build-arm-fb -name IMX6U-Game -type f
|
||||
```
|
||||
|
||||
## 板端部署与运行
|
||||
|
||||
假设板子地址是 `root@imx6u`：
|
||||
|
||||
```bash
|
||||
ssh root@imx6u 'mkdir -p /opt/imx6u-game'
|
||||
scp "$(find build-arm-fb -name IMX6U-Game -type f | head -n 1)" root@imx6u:/opt/imx6u-game/
|
||||
```
|
||||
|
||||
不需要再拷贝：
|
||||
将 WSL 编译出的四个程序复制到 Windows。以下示例假设文件位于：
|
||||
|
||||
```text
|
||||
libtensorflowlite_c.so
|
||||
kws_ref_model_float32.tflite
|
||||
E:\嵌入式实验\IMX6U-Desktop
|
||||
E:\嵌入式实验\IMX6U-Game
|
||||
E:\嵌入式实验\IMX6U-LightGame
|
||||
E:\嵌入式实验\IMX6U-Demo
|
||||
```
|
||||
|
||||
运行前先检查麦克风：
|
||||
板子地址示例：
|
||||
|
||||
```bash
|
||||
arecord -l
|
||||
aplay -l
|
||||
arecord -L
|
||||
aplay -L
|
||||
|
||||
arecord -D sysdefault:CARD=wm8960audio -f S16_LE -r 16000 -c 1 -d 2 /tmp/kws_mic_test.wav
|
||||
aplay -D sysdefault:CARD=wm8960audio /tmp/kws_mic_test.wav
|
||||
```text
|
||||
root@192.168.0.200
|
||||
```
|
||||
|
||||
ATK-IMX6U 板载 WM8960 通常显示为 `wm8960audio`。如果 `sysdefault:CARD=wm8960audio` 不可用，可尝试硬件设备：
|
||||
板端统一部署目录：
|
||||
|
||||
```bash
|
||||
arecord -D plughw:0,0 -f S16_LE -r 16000 -c 1 -d 2 /tmp/kws_mic_test.wav
|
||||
aplay -D plughw:0,0 /tmp/kws_mic_test.wav
|
||||
```text
|
||||
/home/root/opt
|
||||
```
|
||||
|
||||
程序默认仍使用 ALSA `"default"`，也可以通过启动参数显式指定输入/输出设备，避免依赖 `/etc/asound.conf` 或 `~/.asoundrc`。
|
||||
先创建目录：
|
||||
|
||||
启动游戏：
|
||||
```powershell
|
||||
ssh `
|
||||
-o HostKeyAlgorithms=+ssh-rsa `
|
||||
root@192.168.0.200 `
|
||||
"mkdir -p /home/root/opt"
|
||||
```
|
||||
|
||||
批量上传：
|
||||
|
||||
```powershell
|
||||
$files = @(
|
||||
"IMX6U-Desktop",
|
||||
"IMX6U-Game",
|
||||
"IMX6U-LightGame",
|
||||
"IMX6U-Demo"
|
||||
)
|
||||
|
||||
foreach ($file in $files) {
|
||||
scp -O `
|
||||
-o HostKeyAlgorithms=+ssh-rsa `
|
||||
"E:\嵌入式实验\$file" `
|
||||
"root@192.168.0.200:/home/root/opt/$file"
|
||||
}
|
||||
```
|
||||
|
||||
其中：
|
||||
|
||||
```text
|
||||
-O 强制使用旧版 SCP 协议
|
||||
HostKeyAlgorithms=+ssh-rsa 兼容板子上的旧 SSH 服务
|
||||
```
|
||||
|
||||
ResBN 模型已经编译进 `IMX6U-Game`，不需要上传 `.tflite` 或其他模型文件。
|
||||
|
||||
## 板端运行
|
||||
|
||||
### 1. 登录板子
|
||||
|
||||
```powershell
|
||||
ssh -o HostKeyAlgorithms=+ssh-rsa root@192.168.0.200
|
||||
```
|
||||
|
||||
### 2. 初始化音频
|
||||
|
||||
```bash
|
||||
cd /opt/imx6u-game
|
||||
cd /home/root/shell/audio
|
||||
./mic_in_config.sh
|
||||
amixer cset numid=41 1
|
||||
```
|
||||
|
||||
`amixer cset numid=41 1` 会将 WM8960 的左 ADC 同时输出到左右声道。
|
||||
板载 MIC 实测主要连接在 Left ADC，未设置时可能只有单声道有效。
|
||||
|
||||
### 3. 设置执行权限
|
||||
|
||||
```bash
|
||||
cd /home/root/opt
|
||||
chmod +x IMX6U-Desktop
|
||||
chmod +x IMX6U-Game
|
||||
chmod +x IMX6U-LightGame
|
||||
chmod +x IMX6U-Demo
|
||||
```
|
||||
|
||||
也可以统一执行：
|
||||
|
||||
```bash
|
||||
chmod +x IMX6U-*
|
||||
```
|
||||
|
||||
### 4. 启动 Desktop
|
||||
|
||||
```bash
|
||||
cd /home/root/opt
|
||||
./IMX6U-Desktop
|
||||
```
|
||||
|
||||
Desktop 操作方式：
|
||||
|
||||
```text
|
||||
点击屏幕左侧或右侧 切换游戏
|
||||
水平滑动 切换游戏
|
||||
点击中间游戏封面 启动当前游戏
|
||||
```
|
||||
|
||||
Desktop 启动游戏前会释放 framebuffer 和触摸设备。游戏退出后，
|
||||
Desktop 会重新初始化硬件并恢复主页。
|
||||
|
||||
Desktop 使用相对路径启动游戏，因此以下程序必须与 `IMX6U-Desktop`
|
||||
处于同一目录：
|
||||
|
||||
```text
|
||||
/home/root/opt/IMX6U-Desktop
|
||||
/home/root/opt/IMX6U-Game
|
||||
/home/root/opt/IMX6U-LightGame
|
||||
/home/root/opt/IMX6U-Demo
|
||||
```
|
||||
|
||||
### 5. 单独运行 TomGame
|
||||
|
||||
排查语音或音频问题时，可以绕过 Desktop 直接启动 TomGame：
|
||||
|
||||
```bash
|
||||
cd /home/root/opt
|
||||
|
||||
cd /opt/imx6u-game
|
||||
./IMX6U-Game \
|
||||
--fps 30 \
|
||||
--kws-threshold 0.70 \
|
||||
--kws-input-gain 1.5 \
|
||||
--kws-threshold 0.75 \
|
||||
--kws-input-gain 1.0 \
|
||||
--audio-input-device sysdefault:CARD=wm8960audio \
|
||||
--audio-output-device sysdefault:CARD=wm8960audio
|
||||
```
|
||||
|
||||
其中 `amixer cset numid=41 1` 将 WM8960 的左 ADC 同时输出到左右声道。IMX6ULL-ALPHA 板载 MIC 实测主要接在 Left ADC，如果不设置该项，录音可能只有左声道有效，播放或游戏回放时可能只听到噪声。
|
||||
|
||||
如果 `sysdefault:CARD=wm8960audio` 初始化失败，再换成：
|
||||
如果 `sysdefault:CARD=wm8960audio` 初始化失败，改用：
|
||||
|
||||
```bash
|
||||
cd /home/root/shell/audio
|
||||
./mic_in_config.sh
|
||||
amixer cset numid=41 1
|
||||
|
||||
cd /opt/imx6u-game
|
||||
./IMX6U-Game \
|
||||
--fps 30 \
|
||||
--kws-threshold 0.70 \
|
||||
--kws-input-gain 1.5 \
|
||||
--kws-threshold 0.75 \
|
||||
--kws-input-gain 1.0 \
|
||||
--audio-input-device plughw:0,0 \
|
||||
--audio-output-device plughw:0,0
|
||||
```
|
||||
|
||||
进入游戏后：
|
||||
Desktop 启动 TomGame 时不会附加命令行参数，因此 TomGame 会使用 ALSA
|
||||
的 `"default"` 输入和输出设备。如果直接运行 TomGame 正常，但从 Desktop
|
||||
启动后音频初始化失败，需要将板子的 ALSA default 配置映射到 WM8960。
|
||||
|
||||
1. 点击左侧 `ui-hand` 切到语音识别模式。
|
||||
2. 点击录音按钮。
|
||||
3. 说 `Up` / `On` / `Stop`。
|
||||
4. 观察 Tom 是否跳跃或回到 Idle。
|
||||
## 常见问题
|
||||
|
||||
## 排查顺序
|
||||
### SSH 提示不支持 ssh-rsa
|
||||
|
||||
1. `IMX6U-Game` 能启动并正常刷新画面。
|
||||
2. 程序启动后不再打印 `AlsaAudioInput backend is unavailable`；如果仍打印，说明当前二进制没有编进 ALSA，需要重新交叉编译并部署。
|
||||
3. 运行 `/home/root/shell/audio/mic_in_config.sh`，再执行 `amixer cset numid=41 1`，确保板载 MIC 的 Left ADC 映射到左右声道。
|
||||
4. `arecord -D sysdefault:CARD=wm8960audio` 或 `arecord -D plughw:0,0` 能录到 16-bit 音频，且说话时 `arecord -vv` 音量条有明显波动。
|
||||
5. `aplay -D sysdefault:CARD=wm8960audio` 或 `aplay -D plughw:0,0` 能从扬声器放出录音。
|
||||
6. 游戏中已切到左侧 `ui-hand` 语音识别模式。
|
||||
7. 试用较低阈值确认链路，例如 `--kws-threshold 0.70`。
|
||||
8. 如果模型日志长期显示 `class=Unknown`，尝试提高输入增益，例如 `--kws-input-gain 1.5`。
|
||||
9. 如果仍然没有识别结果，优先检查 WM8960 混音器开关、麦克风增益、采样率、ALSA 设备名和录音幅度。
|
||||
10. 如果边界词不稳定，再对齐 C++ MFCC 与 Python/TFLite 前处理细节。
|
||||
错误示例：
|
||||
|
||||
```text
|
||||
Unable to negotiate ... no matching host key type found.
|
||||
Their offer: ssh-rsa
|
||||
```
|
||||
|
||||
连接时增加：
|
||||
|
||||
```text
|
||||
-o HostKeyAlgorithms=+ssh-rsa
|
||||
```
|
||||
|
||||
SCP 还建议增加 `-O`，以兼容板子上的旧版 SSH/Dropbear。
|
||||
|
||||
### 找不到 libasound
|
||||
|
||||
项目会优先使用：
|
||||
|
||||
```text
|
||||
third_party/arm-linux-gnueabihf/alsa/lib/libasound.so
|
||||
```
|
||||
|
||||
如果该文件不存在，需要安装兼容的 ARMHF ALSA 包，或者从板子复制匹配的
|
||||
`libasound.so*` 到该目录。不要使用与板端系统版本不兼容的新版库。
|
||||
|
||||
### Desktop 可以启动，但点击游戏没有反应
|
||||
|
||||
检查四个程序是否位于同一个目录，并具有执行权限：
|
||||
|
||||
```bash
|
||||
cd /home/root/opt
|
||||
ls -l IMX6U-*
|
||||
```
|
||||
|
||||
Desktop 的游戏目录配置为：
|
||||
|
||||
```text
|
||||
TomGame -> IMX6U-Game
|
||||
LightGame -> IMX6U-LightGame
|
||||
Demo -> IMX6U-Demo
|
||||
```
|
||||
|
||||
### TomGame 没有识别结果
|
||||
|
||||
按以下顺序检查：
|
||||
|
||||
1. 执行 `mic_in_config.sh` 和 `amixer cset numid=41 1`。
|
||||
2. 使用 `arecord -l`、`arecord -L` 确认 WM8960 设备存在。
|
||||
3. 使用 `arecord -vv` 检查说话时是否有明显音量变化。
|
||||
4. 确认游戏已切换到关键词识别模式。
|
||||
5. 检查日志中是否出现 `ResBN INT8 KWS result`。
|
||||
6. 保持 `--kws-input-gain` 在 `(0, 1]` 范围。
|
||||
7. 必要时将 `--kws-threshold` 从 `0.75` 临时降低到 `0.70` 进行链路测试。
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,106 @@
|
|||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
|
||||
namespace Game
|
||||
{
|
||||
namespace ResBnKwsModelData
|
||||
{
|
||||
const size_t InputFrameCount = 49;
|
||||
const size_t InputMfccCount = 10;
|
||||
const size_t ChannelCount = 64;
|
||||
const size_t ClassCount = 12;
|
||||
const size_t BlockCount = 4;
|
||||
|
||||
struct Quantization
|
||||
{
|
||||
float scale;
|
||||
int32_t zeroPoint;
|
||||
};
|
||||
|
||||
struct ConvLayerData
|
||||
{
|
||||
const int8_t* weights;
|
||||
const int32_t* bias;
|
||||
const int32_t* multipliers;
|
||||
const int32_t* shifts;
|
||||
Quantization input;
|
||||
Quantization output;
|
||||
bool relu;
|
||||
};
|
||||
|
||||
struct AddLayerData
|
||||
{
|
||||
Quantization input0;
|
||||
Quantization input1;
|
||||
Quantization output;
|
||||
int32_t input0Multiplier;
|
||||
int32_t input0Shift;
|
||||
int32_t input1Multiplier;
|
||||
int32_t input1Shift;
|
||||
int32_t outputMultiplier;
|
||||
int32_t outputShift;
|
||||
int32_t leftShift;
|
||||
};
|
||||
|
||||
struct MeanLayerData
|
||||
{
|
||||
Quantization input;
|
||||
Quantization output;
|
||||
int32_t multiplier;
|
||||
int32_t shift;
|
||||
int32_t elementCount;
|
||||
};
|
||||
|
||||
extern const int8_t StemWeights[2560];
|
||||
extern const int32_t StemBias[64];
|
||||
extern const int32_t StemMultipliers[64];
|
||||
extern const int32_t StemShifts[64];
|
||||
extern const int8_t Dw0Weights[576];
|
||||
extern const int32_t Dw0Bias[64];
|
||||
extern const int32_t Dw0Multipliers[64];
|
||||
extern const int32_t Dw0Shifts[64];
|
||||
extern const int8_t Dw1Weights[576];
|
||||
extern const int32_t Dw1Bias[64];
|
||||
extern const int32_t Dw1Multipliers[64];
|
||||
extern const int32_t Dw1Shifts[64];
|
||||
extern const int8_t Dw2Weights[576];
|
||||
extern const int32_t Dw2Bias[64];
|
||||
extern const int32_t Dw2Multipliers[64];
|
||||
extern const int32_t Dw2Shifts[64];
|
||||
extern const int8_t Dw3Weights[576];
|
||||
extern const int32_t Dw3Bias[64];
|
||||
extern const int32_t Dw3Multipliers[64];
|
||||
extern const int32_t Dw3Shifts[64];
|
||||
extern const int8_t Pw0Weights[4096];
|
||||
extern const int32_t Pw0Bias[64];
|
||||
extern const int32_t Pw0Multipliers[64];
|
||||
extern const int32_t Pw0Shifts[64];
|
||||
extern const int8_t Pw1Weights[4096];
|
||||
extern const int32_t Pw1Bias[64];
|
||||
extern const int32_t Pw1Multipliers[64];
|
||||
extern const int32_t Pw1Shifts[64];
|
||||
extern const int8_t Pw2Weights[4096];
|
||||
extern const int32_t Pw2Bias[64];
|
||||
extern const int32_t Pw2Multipliers[64];
|
||||
extern const int32_t Pw2Shifts[64];
|
||||
extern const int8_t Pw3Weights[4096];
|
||||
extern const int32_t Pw3Bias[64];
|
||||
extern const int32_t Pw3Multipliers[64];
|
||||
extern const int32_t Pw3Shifts[64];
|
||||
extern const int8_t DenseWeights[768];
|
||||
extern const int32_t DenseBias[12];
|
||||
extern const int32_t DenseMultipliers[12];
|
||||
extern const int32_t DenseShifts[12];
|
||||
|
||||
extern const ConvLayerData Stem;
|
||||
extern const ConvLayerData Depthwise[BlockCount];
|
||||
extern const ConvLayerData Pointwise[BlockCount];
|
||||
extern const AddLayerData Adds[BlockCount];
|
||||
extern const MeanLayerData GlobalMean;
|
||||
extern const ConvLayerData Dense;
|
||||
extern const Quantization ModelInput;
|
||||
extern const Quantization ModelOutput;
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,50 @@
|
|||
#pragma once
|
||||
|
||||
#include "KeywordRecognizer.h"
|
||||
#include <cstddef>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace Game
|
||||
{
|
||||
class ResBnKwsRecognizer : public IKeywordRecognizer
|
||||
{
|
||||
private:
|
||||
float confidenceThreshold;
|
||||
float marginThreshold;
|
||||
float smoothingAlpha;
|
||||
float inputGain;
|
||||
size_t minConsecutiveHits;
|
||||
size_t featureWindowStepFrames;
|
||||
bool initialized;
|
||||
|
||||
bool run_embedded_model_logits(
|
||||
const std::vector<float>& features,
|
||||
std::vector<float>& logits) const;
|
||||
|
||||
public:
|
||||
ResBnKwsRecognizer();
|
||||
explicit ResBnKwsRecognizer(const std::string& unusedModelPath);
|
||||
~ResBnKwsRecognizer();
|
||||
|
||||
bool init();
|
||||
KeywordRecognitionResult recognize(
|
||||
const std::vector<int16_t>& samples,
|
||||
uint32_t sampleRate,
|
||||
uint32_t channels);
|
||||
|
||||
void set_confidence_threshold(float threshold);
|
||||
float get_confidence_threshold() const { return confidenceThreshold; }
|
||||
void set_margin_threshold(float threshold);
|
||||
float get_margin_threshold() const { return marginThreshold; }
|
||||
void set_smoothing_alpha(float alpha);
|
||||
float get_smoothing_alpha() const { return smoothingAlpha; }
|
||||
void set_min_consecutive_hits(size_t hits);
|
||||
size_t get_min_consecutive_hits() const { return minConsecutiveHits; }
|
||||
void set_feature_window_step_frames(size_t frames);
|
||||
size_t get_feature_window_step_frames() const { return featureWindowStepFrames; }
|
||||
void set_input_gain(float gain) override;
|
||||
float get_input_gain() const override { return inputGain; }
|
||||
bool is_initialized() const { return initialized; }
|
||||
};
|
||||
}
|
||||
Loading…
Reference in New Issue