Compare commits
16
Commits
acf162d1b9
..
master
| Author | SHA1 | Date | |
|---|---|---|---|
 HP
|
f6f2a74e1d | ||
|
HP
|
ad63d5640b | ||
|
HP
|
45f8aa2b58 | ||
|
HP
|
1a194da7ec | ||
|
HP
|
d95ccc0c99 | ||
 basil
|
36cbaf18c4 | ||
|
basil
|
10872fde82 | ||
 SepComet
|
334c9ee96f | ||
|
HP
|
e0946fbf36 | ||
|
SepComet
|
de85a8d41d | ||
|
SepComet
|
786170f40a | ||
|
SepComet
|
254a8ce5c8 | ||
|
basil
|
05d7d9783e | ||
|
basil
|
d49aef8c0f | ||
|
basil
|
48fd7818d7 | ||
|
basil
|
46c76ec7fc |
@@ -62,3 +62,4 @@ build-*
|
|||||||
assets/test
|
assets/test
|
||||||
|
|
||||||
gcc-linaro-4.9.4-2017.01-x86_64_arm-linux-gnueabihf.tar.xz
|
gcc-linaro-4.9.4-2017.01-x86_64_arm-linux-gnueabihf.tar.xz
|
||||||
|
/tools/__pycache__
|
||||||
|
|||||||
Vendored
+3
@@ -0,0 +1,3 @@
|
|||||||
|
{
|
||||||
|
"git.ignoreLimitWarning": true
|
||||||
|
}
|
||||||
+70
-8
@@ -9,7 +9,12 @@ if(NOT CMAKE_CONFIGURATION_TYPES AND NOT CMAKE_BUILD_TYPE)
|
|||||||
set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
|
set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
option(USE_FRAMEBUFFER "Use Linux framebuffer instead of SDL2" OFF)
|
option(TARGET_IMX "Build for IMX6U target board (framebuffer + Alsa + Evdev). OFF builds for PC (SDL2)." OFF)
|
||||||
|
if(TARGET_IMX)
|
||||||
|
set(TARGET_PC OFF)
|
||||||
|
else()
|
||||||
|
set(TARGET_PC ON)
|
||||||
|
endif()
|
||||||
|
|
||||||
set(CORE_SOURCES
|
set(CORE_SOURCES
|
||||||
src/Core/Asset/ObjLoader.cpp
|
src/Core/Asset/ObjLoader.cpp
|
||||||
@@ -29,10 +34,11 @@ set(CORE_SOURCES
|
|||||||
src/Core/Shading/BlinnPhongShader.cpp
|
src/Core/Shading/BlinnPhongShader.cpp
|
||||||
)
|
)
|
||||||
|
|
||||||
if(USE_FRAMEBUFFER)
|
if(TARGET_IMX)
|
||||||
list(APPEND CORE_SOURCES
|
list(APPEND CORE_SOURCES
|
||||||
src/Core/Platform/FBDisplay.cpp
|
src/Core/Platform/FBDisplay.cpp
|
||||||
src/Core/Platform/LinuxPhotoSensor.cpp
|
src/Core/Platform/Ap3216cPhotoSensor.cpp
|
||||||
|
src/Core/Platform/Imx6ullAlphaLed.cpp
|
||||||
)
|
)
|
||||||
else()
|
else()
|
||||||
list(APPEND CORE_SOURCES
|
list(APPEND CORE_SOURCES
|
||||||
@@ -60,14 +66,31 @@ set(CORE_INCLUDE_DIRS
|
|||||||
assets/sprite
|
assets/sprite
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if(NOT TARGET_IMX)
|
||||||
|
list(APPEND CORE_SOURCES
|
||||||
|
third_party/imgui/imgui.cpp
|
||||||
|
third_party/imgui/imgui_demo.cpp
|
||||||
|
third_party/imgui/imgui_draw.cpp
|
||||||
|
third_party/imgui/imgui_tables.cpp
|
||||||
|
third_party/imgui/imgui_widgets.cpp
|
||||||
|
third_party/imgui/backends/imgui_impl_sdl2.cpp
|
||||||
|
third_party/imgui/backends/imgui_impl_sdlrenderer2.cpp
|
||||||
|
)
|
||||||
|
list(APPEND CORE_INCLUDE_DIRS third_party/imgui)
|
||||||
|
endif()
|
||||||
|
|
||||||
add_library(imx6u_core STATIC ${CORE_SOURCES})
|
add_library(imx6u_core STATIC ${CORE_SOURCES})
|
||||||
target_include_directories(imx6u_core PUBLIC ${CORE_INCLUDE_DIRS})
|
target_include_directories(imx6u_core PUBLIC ${CORE_INCLUDE_DIRS})
|
||||||
|
|
||||||
if(USE_FRAMEBUFFER)
|
if(TARGET_IMX)
|
||||||
target_compile_definitions(imx6u_core PUBLIC USE_FRAMEBUFFER)
|
target_compile_definitions(imx6u_core PUBLIC TARGET_IMX)
|
||||||
|
else()
|
||||||
|
target_compile_definitions(imx6u_core PUBLIC TARGET_PC)
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
if(USE_FRAMEBUFFER)
|
target_compile_definitions(imx6u_core PUBLIC $<$<CONFIG:Debug>:IMX6U_DEBUG>)
|
||||||
|
|
||||||
|
if(TARGET_IMX)
|
||||||
else()
|
else()
|
||||||
if(WIN32)
|
if(WIN32)
|
||||||
if(CMAKE_SIZEOF_VOID_P EQUAL 8)
|
if(CMAKE_SIZEOF_VOID_P EQUAL 8)
|
||||||
@@ -93,14 +116,50 @@ else()
|
|||||||
endif()
|
endif()
|
||||||
|
|
||||||
if(UNIX AND NOT APPLE)
|
if(UNIX AND NOT APPLE)
|
||||||
|
set(IMX6U_ALSA_INCLUDE_AFTER "")
|
||||||
|
if(TARGET_IMX)
|
||||||
|
set(IMX6U_BUNDLED_ALSA_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/third_party/arm-linux-gnueabihf/alsa")
|
||||||
|
set(IMX6U_BUNDLED_ALSA_INCLUDE_DIR "${IMX6U_BUNDLED_ALSA_ROOT}/include")
|
||||||
|
set(IMX6U_BUNDLED_ALSA_LIBRARY "${IMX6U_BUNDLED_ALSA_ROOT}/lib/libasound.so")
|
||||||
|
|
||||||
|
if(EXISTS "${IMX6U_BUNDLED_ALSA_LIBRARY}")
|
||||||
|
set(ALSA_FOUND TRUE)
|
||||||
|
set(ALSA_LIBRARIES "${IMX6U_BUNDLED_ALSA_LIBRARY}")
|
||||||
|
if(EXISTS "${IMX6U_BUNDLED_ALSA_INCLUDE_DIR}/alsa/asoundlib.h")
|
||||||
|
set(ALSA_INCLUDE_DIRS "${IMX6U_BUNDLED_ALSA_INCLUDE_DIR}")
|
||||||
|
elseif(EXISTS "/usr/include/alsa/asoundlib.h")
|
||||||
|
set(IMX6U_ALSA_INCLUDE_AFTER "/usr/include")
|
||||||
|
else()
|
||||||
|
message(FATAL_ERROR "TARGET_IMX requires ALSA headers. Install libasound2-dev:armhf or copy ALSA headers to third_party/arm-linux-gnueabihf/alsa/include.")
|
||||||
|
endif()
|
||||||
|
message(STATUS "Using bundled ARM ALSA library: ${IMX6U_BUNDLED_ALSA_LIBRARY}")
|
||||||
|
else()
|
||||||
find_package(ALSA QUIET)
|
find_package(ALSA QUIET)
|
||||||
|
if(NOT ALSA_FOUND)
|
||||||
|
message(FATAL_ERROR "TARGET_IMX requires ALSA. Copy board libasound.so* to third_party/arm-linux-gnueabihf/alsa/lib or provide a compatible ALSA package.")
|
||||||
|
endif()
|
||||||
|
endif()
|
||||||
|
else()
|
||||||
|
find_package(ALSA QUIET)
|
||||||
|
endif()
|
||||||
if(ALSA_FOUND)
|
if(ALSA_FOUND)
|
||||||
target_compile_definitions(imx6u_core PUBLIC PLATFORM_HAS_ALSA)
|
target_compile_definitions(imx6u_core PUBLIC PLATFORM_HAS_ALSA)
|
||||||
|
if(IMX6U_ALSA_INCLUDE_AFTER)
|
||||||
|
target_compile_options(imx6u_core PUBLIC -idirafter ${IMX6U_ALSA_INCLUDE_AFTER})
|
||||||
|
elseif(ALSA_INCLUDE_DIRS)
|
||||||
target_include_directories(imx6u_core PUBLIC ${ALSA_INCLUDE_DIRS})
|
target_include_directories(imx6u_core PUBLIC ${ALSA_INCLUDE_DIRS})
|
||||||
|
elseif(ALSA_INCLUDE_DIR)
|
||||||
|
target_include_directories(imx6u_core PUBLIC ${ALSA_INCLUDE_DIR})
|
||||||
|
endif()
|
||||||
target_link_libraries(imx6u_core PUBLIC ${ALSA_LIBRARIES})
|
target_link_libraries(imx6u_core PUBLIC ${ALSA_LIBRARIES})
|
||||||
|
if(TARGET_IMX)
|
||||||
|
target_link_libraries(imx6u_core PUBLIC dl pthread rt m)
|
||||||
|
endif()
|
||||||
else()
|
else()
|
||||||
message(STATUS "ALSA was not found; AlsaAudioInput and AlsaAudioOutput will be unavailable backends")
|
message(STATUS "ALSA was not found; AlsaAudioInput and AlsaAudioOutput will be unavailable backends")
|
||||||
endif()
|
endif()
|
||||||
|
elseif(TARGET_IMX)
|
||||||
|
message(FATAL_ERROR "TARGET_IMX requires ALSA headers and libasound.")
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
if(MSVC)
|
if(MSVC)
|
||||||
@@ -123,7 +182,7 @@ function(imx6u_configure_app_target target_name)
|
|||||||
)
|
)
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
if(WIN32 AND NOT USE_FRAMEBUFFER)
|
if(WIN32 AND NOT TARGET_IMX)
|
||||||
add_custom_command(TARGET ${target_name} POST_BUILD
|
add_custom_command(TARGET ${target_name} POST_BUILD
|
||||||
COMMAND ${CMAKE_COMMAND} -E copy_if_different
|
COMMAND ${CMAKE_COMMAND} -E copy_if_different
|
||||||
"${SDL2_DLL}"
|
"${SDL2_DLL}"
|
||||||
@@ -138,10 +197,11 @@ function(imx6u_configure_app_target target_name)
|
|||||||
endfunction()
|
endfunction()
|
||||||
|
|
||||||
add_subdirectory(src/Apps/Game)
|
add_subdirectory(src/Apps/Game)
|
||||||
|
add_subdirectory(src/Apps/Desktop)
|
||||||
add_subdirectory(src/Apps/Demo)
|
add_subdirectory(src/Apps/Demo)
|
||||||
add_subdirectory(src/Apps/LightGame)
|
add_subdirectory(src/Apps/LightGame)
|
||||||
|
|
||||||
if(BUILD_TESTING AND NOT USE_FRAMEBUFFER)
|
if(BUILD_TESTING AND NOT TARGET_IMX)
|
||||||
add_executable(render_pipeline_tests
|
add_executable(render_pipeline_tests
|
||||||
tests/render_pipeline_tests.cpp
|
tests/render_pipeline_tests.cpp
|
||||||
)
|
)
|
||||||
@@ -159,6 +219,8 @@ if(BUILD_TESTING AND NOT USE_FRAMEBUFFER)
|
|||||||
target_include_directories(game_engine_tests PRIVATE
|
target_include_directories(game_engine_tests PRIVATE
|
||||||
src/Apps/LightGame/src/engine
|
src/Apps/LightGame/src/engine
|
||||||
src/Apps/LightGame/src/systems
|
src/Apps/LightGame/src/systems
|
||||||
|
src/Apps/LightGame/src/levels
|
||||||
|
src/Apps/LightGame/generated
|
||||||
${CORE_INCLUDE_DIRS}
|
${CORE_INCLUDE_DIRS}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -20,7 +20,6 @@
|
|||||||
- **Linux x86 编译**：验证代码在 GCC/Clang 下有无警告、CMake 配置是否跨平台、系统 SDL2 依赖是否正确。很多嵌入式工具链的问题在 x86 Linux 上就能提前暴露。
|
- **Linux x86 编译**：验证代码在 GCC/Clang 下有无警告、CMake 配置是否跨平台、系统 SDL2 依赖是否正确。很多嵌入式工具链的问题在 x86 Linux 上就能提前暴露。
|
||||||
- **ARM 交叉编译**：最终在 IMX6U 上跑。若目标板使用 SDL2，则 SDL2 仅作为显示/输入适配层，时间由独立 `Platform::ITimeSource` 提供，核心渲染仍按 CPU framebuffer + 一次性提交设计；如需极简依赖，也保留 `/dev/fb0` 后端作为对照。
|
- **ARM 交叉编译**：最终在 IMX6U 上跑。若目标板使用 SDL2，则 SDL2 仅作为显示/输入适配层，时间由独立 `Platform::ITimeSource` 提供，核心渲染仍按 CPU framebuffer + 一次性提交设计；如需极简依赖，也保留 `/dev/fb0` 后端作为对照。
|
||||||
|
|
||||||
|
|
||||||
## 开发规范与性能红线
|
## 开发规范与性能红线
|
||||||
|
|
||||||
IMX6U 运行时性能预算较紧，后续开发必须遵守 `docs/DEVELOPMENT_GUIDELINES.md`。如果目标板使用 SDL2，仍然要把 SDL2 限制在平台适配层，核心逻辑和渲染热路径不直接依赖 SDL：
|
IMX6U 运行时性能预算较紧，后续开发必须遵守 `docs/DEVELOPMENT_GUIDELINES.md`。如果目标板使用 SDL2，仍然要把 SDL2 限制在平台适配层，核心逻辑和渲染热路径不直接依赖 SDL：
|
||||||
@@ -52,6 +51,26 @@ IMX6U 运行时性能预算较紧，后续开发必须遵守 `docs/DEVELOPMENT_G
|
|||||||
cd IMX6U-Game
|
cd IMX6U-Game
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### 构建类型与调试开关
|
||||||
|
|
||||||
|
项目有两根独立的条件编译轴，组合出常用的构建形态：
|
||||||
|
|
||||||
|
| 轴 | CMake 参数 | 编译宏 | 默认 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| 平台软件栈 | `-DTARGET_IMX=ON/OFF` | `TARGET_IMX` 或 `TARGET_PC`（互斥、双正向） | `OFF` → `TARGET_PC` |
|
||||||
|
| 构建类型 | `-DCMAKE_BUILD_TYPE=Debug/Release` | `IMX6U_DEBUG`（仅 Debug 配置注入） | `Release` |
|
||||||
|
|
||||||
|
`IMX6U_DEBUG` 控制开发期辅助内容是否参与构建。当前 LightGame 中下列内容只在 Debug 构建中存在，Release 构建会被完全剔除（不参与编译，不进入二进制）：
|
||||||
|
|
||||||
|
- 关卡编辑器（`LevelEditor` + ImGui，按 `F1` 切换）
|
||||||
|
- 碰撞框可视化（`LevelRenderer::draw_debug` 与 `LightGameApp::debug_mode_`）
|
||||||
|
- `--debug` 命令行开关
|
||||||
|
- `[INFO] LightGame started …` 启动日志
|
||||||
|
|
||||||
|
硬件初始化失败的 `[WARN]` 提示无论 Debug/Release 都保留，作为现场诊断信息。
|
||||||
|
|
||||||
|
> 多配置生成器（Visual Studio）下 `CMAKE_BUILD_TYPE` 在 configure 阶段被忽略，应在构建阶段用 `--config Debug|Release` 选择；单配置生成器（Makefile/Ninja）使用 `-DCMAKE_BUILD_TYPE=...` 在 configure 阶段决定。
|
||||||
|
|
||||||
### Windows（Visual Studio / MSVC）
|
### Windows（Visual Studio / MSVC）
|
||||||
|
|
||||||
仓库已自带 SDL2 开发库（`libs/Win/SDL2`），无需额外安装。
|
仓库已自带 SDL2 开发库（`libs/Win/SDL2`），无需额外安装。
|
||||||
@@ -62,22 +81,33 @@ cmake --build build-win --config Release
|
|||||||
```
|
```
|
||||||
|
|
||||||
只构建某个 App：
|
只构建某个 App：
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cmake --build build-win --config Release --target IMX6U-Game
|
cmake --build build-win --config Release --target IMX6U-Game
|
||||||
cmake --build build-win --config Release --target IMX6U-Demo
|
cmake --build build-win --config Release --target IMX6U-Demo
|
||||||
```
|
```
|
||||||
|
|
||||||
|
启用 LightGame 关卡编辑器和调试显示，使用 Debug 构建：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cmake --build build-win --config Debug --target IMX6U-LightGame
|
||||||
|
./build-win/Debug/IMX6U-LightGame.exe
|
||||||
|
```
|
||||||
|
|
||||||
构建 `IMX6U-Game` 时会自动重新生成 Tom 的 atlas 头文件；也可以单独执行：
|
构建 `IMX6U-Game` 时会自动重新生成 Tom 的 atlas 头文件；也可以单独执行：
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cmake --build build-win --config Release --target GenerateTomAtlasHeader
|
cmake --build build-win --config Release --target GenerateTomAtlasHeader
|
||||||
```
|
```
|
||||||
|
|
||||||
运行：
|
运行：
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./build-win/Release/IMX6U-Game.exe
|
./build-win/Release/IMX6U-Game.exe
|
||||||
```
|
```
|
||||||
|
|
||||||
可选帧率档位：
|
可选帧率档位：
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./build-win/Release/IMX6U-Game.exe --fps 30
|
./build-win/Release/IMX6U-Game.exe --fps 30
|
||||||
./build-win/Release/IMX6U-Game.exe --fps 45
|
./build-win/Release/IMX6U-Game.exe --fps 45
|
||||||
@@ -95,23 +125,27 @@ sudo apt-get install libsdl2-dev libsdl2-image-dev cmake g++
|
|||||||
```
|
```
|
||||||
|
|
||||||
构建：
|
构建：
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cmake -B build-linux .
|
cmake -B build-linux .
|
||||||
cmake --build build-linux
|
cmake --build build-linux
|
||||||
```
|
```
|
||||||
|
|
||||||
只构建某个 App：
|
只构建某个 App：
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cmake --build build-linux --target IMX6U-Game
|
cmake --build build-linux --target IMX6U-Game
|
||||||
cmake --build build-linux --target IMX6U-Demo
|
cmake --build build-linux --target IMX6U-Demo
|
||||||
```
|
```
|
||||||
|
|
||||||
构建 `IMX6U-Game` 时会自动重新生成 Tom 的 atlas 头文件；也可以单独执行：
|
构建 `IMX6U-Game` 时会自动重新生成 Tom 的 atlas 头文件；也可以单独执行：
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cmake --build build-linux --target GenerateTomAtlasHeader
|
cmake --build build-linux --target GenerateTomAtlasHeader
|
||||||
```
|
```
|
||||||
|
|
||||||
运行：
|
运行：
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./build-linux/IMX6U-Game
|
./build-linux/IMX6U-Game
|
||||||
```
|
```
|
||||||
@@ -129,11 +163,18 @@ sudo apt-get install gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf
|
|||||||
- **SDL2 后端**：目标是后续游戏主路径；SDL2 负责显示、输入和最终 framebuffer 提交，时间源使用独立的 `Platform::ITimeSource`。
|
- **SDL2 后端**：目标是后续游戏主路径；SDL2 负责显示、输入和最终 framebuffer 提交，时间源使用独立的 `Platform::ITimeSource`。
|
||||||
- **Framebuffer 后端**：作为极简依赖和显示通路对照测试。
|
- **Framebuffer 后端**：作为极简依赖和显示通路对照测试。
|
||||||
|
|
||||||
|
> CMake 选项 `TARGET_IMX` 控制的不是"是否交叉编译"，而是"使用哪一套软件栈"：
|
||||||
|
> - `TARGET_IMX=ON` —— framebuffer (`/dev/fb0`) + Alsa 音频 + evdev 输入 + AP3216C 光敏，对应 IMX6U 板载硬件。
|
||||||
|
> - `TARGET_IMX=OFF` —— SDL2 显示/音频/输入/光敏，PC 调试和 ARM-SDL 对照路径共用这一栈。
|
||||||
|
>
|
||||||
|
> 因此 ARM 交叉编译 + SDL2 后端使用 `-DTARGET_IMX=OFF`，与"运行在 PC"无关。
|
||||||
|
|
||||||
构建（Framebuffer 对照后端）：
|
构建（Framebuffer 对照后端）：
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cmake -B build-arm-fb \
|
cmake -B build-arm-fb \
|
||||||
-DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-arm-linux-gnueabihf.cmake \
|
-DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-arm-linux-gnueabihf.cmake \
|
||||||
-DUSE_FRAMEBUFFER=ON .
|
-DTARGET_IMX=ON .
|
||||||
cmake --build build-arm-fb
|
cmake --build build-arm-fb
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -142,14 +183,16 @@ cmake --build build-arm-fb
|
|||||||
注意：Tom 的 atlas 头文件生成工具是主机侧离线工具，依赖 PC/Linux 主机上的 SDL2_image；主机构建 `IMX6U-Game` 时会自动执行，ARM 交叉编译过程不会执行它。如果修改了 `src/Apps/Game/assets/raw/` 里的 PNG，必须先在 Windows 或 Linux x86 构建目录执行 `GenerateTomAtlasHeader` 或构建一次 `IMX6U-Game`，再进行 ARM 交叉编译。ARM 构建只消费已经生成好的 `src/Apps/Game/generated/tom_atlas.h`。
|
注意：Tom 的 atlas 头文件生成工具是主机侧离线工具，依赖 PC/Linux 主机上的 SDL2_image；主机构建 `IMX6U-Game` 时会自动执行，ARM 交叉编译过程不会执行它。如果修改了 `src/Apps/Game/assets/raw/` 里的 PNG，必须先在 Windows 或 Linux x86 构建目录执行 `GenerateTomAtlasHeader` 或构建一次 `IMX6U-Game`，再进行 ARM 交叉编译。ARM 构建只消费已经生成好的 `src/Apps/Game/generated/tom_atlas.h`。
|
||||||
|
|
||||||
构建（SDL2 后端，要求工具链/sysroot 可找到目标板 SDL2 开发库）：
|
构建（SDL2 后端，要求工具链/sysroot 可找到目标板 SDL2 开发库）：
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cmake -B build-arm-sdl \
|
cmake -B build-arm-sdl \
|
||||||
-DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-arm-linux-gnueabihf.cmake \
|
-DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-arm-linux-gnueabihf.cmake \
|
||||||
-DUSE_FRAMEBUFFER=OFF .
|
-DTARGET_IMX=OFF .
|
||||||
cmake --build build-arm-sdl
|
cmake --build build-arm-sdl
|
||||||
```
|
```
|
||||||
|
|
||||||
部署到开发板：
|
部署到开发板：
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
scp build-arm-sdl/IMX6U-Game root@imx6u:/tmp/
|
scp build-arm-sdl/IMX6U-Game root@imx6u:/tmp/
|
||||||
# 或部署 framebuffer 对照版本：
|
# 或部署 framebuffer 对照版本：
|
||||||
@@ -157,6 +200,7 @@ scp build-arm-fb/IMX6U-Game root@imx6u:/tmp/
|
|||||||
```
|
```
|
||||||
|
|
||||||
板子上运行：
|
板子上运行：
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
/tmp/IMX6U-Game
|
/tmp/IMX6U-Game
|
||||||
```
|
```
|
||||||
@@ -212,6 +256,16 @@ src/Apps/Game/generated/tom_atlas.h
|
|||||||
|
|
||||||
`assets/sprite/` 用于存放测试用 PNG sprite 源文件及转换后的头文件，Tom 主游戏不依赖它。Tom 游戏的 atlas 资源由 `src/Apps/Game/tools/asset_pipeline/SpriteAssetTool.cpp` 从 `src/Apps/Game/assets/raw/` 读取原始 PNG 生成。
|
`assets/sprite/` 用于存放测试用 PNG sprite 源文件及转换后的头文件，Tom 主游戏不依赖它。Tom 游戏的 atlas 资源由 `src/Apps/Game/tools/asset_pipeline/SpriteAssetTool.cpp` 从 `src/Apps/Game/assets/raw/` 读取原始 PNG 生成。
|
||||||
|
|
||||||
|
### LightGame Tile Atlas
|
||||||
|
|
||||||
|
LightGame 的 tile atlas 配置位于 `assets/tile/tile_atlas.json`，生成结果位于 `src/Apps/LightGame/generated/tile_atlas.h`：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python tools/pack_tile_atlas.py assets/tile/tile_atlas.json src/Apps/LightGame/generated/tile_atlas.h
|
||||||
|
```
|
||||||
|
|
||||||
|
当前 tile id 约定集中在 `src/Apps/LightGame/src/engine/TileIds.h`：`0` 是空透明 tile；`1..4` 是背景装饰；`5..11` 是可碰撞地形；`12`/`22` 是伤害尖刺；`13..18` 是光照平台和门的状态贴图；`19..21` 是金币、旗帜和 checkpoint；`23` 是藤蔓装饰。LightGame 关卡支持可选 `background_tiles` 视觉层，该层先于主 tilemap 渲染，不参与碰撞、死亡、触发器或光照 tile 规则。
|
||||||
|
|
||||||
### Bitmap Font 转换
|
### Bitmap Font 转换
|
||||||
|
|
||||||
像素字体图集使用 `tools/gen_font_atlas.py` 生成：
|
像素字体图集使用 `tools/gen_font_atlas.py` 生成：
|
||||||
@@ -260,7 +314,7 @@ assets/font/font_atlas.h
|
|||||||
└──────────────────────────────────────────────┘
|
└──────────────────────────────────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
切换显示后端不应影响应用层和核心绘制逻辑；当前 CMake 通过 `USE_FRAMEBUFFER` 在 SDL2 与 framebuffer 后端间切换。
|
切换显示后端不应影响应用层和核心绘制逻辑；当前 CMake 通过 `TARGET_IMX` 在 framebuffer (IMX6U) 与 SDL2 (PC) 后端间切换，对应代码层的 `TARGET_IMX` / `TARGET_PC` 双正向宏。
|
||||||
|
|
||||||
## 目录结构
|
## 目录结构
|
||||||
|
|
||||||
@@ -315,9 +369,11 @@ IMX6U-Game/
|
|||||||
## 模块说明
|
## 模块说明
|
||||||
|
|
||||||
### Draw2D
|
### Draw2D
|
||||||
|
|
||||||
- **DrawContext**：统一绘制入口，封装 FrameBuffer、DepthBuffer、Rasterizer、TriangleRasterizer，对外提供 `clear`、`clear_color`、`clear_depth`、`draw_line`、`draw_triangle`、`draw_sprite`、`draw_sprite_ex`、`draw_text`、`draw_tilemap`、`fill_rect`、`present` 接口
|
- **DrawContext**：统一绘制入口，封装 FrameBuffer、DepthBuffer、Rasterizer、TriangleRasterizer，对外提供 `clear`、`clear_color`、`clear_depth`、`draw_line`、`draw_triangle`、`draw_sprite`、`draw_sprite_ex`、`draw_text`、`draw_tilemap`、`fill_rect`、`present` 接口
|
||||||
|
|
||||||
### RenderData
|
### RenderData
|
||||||
|
|
||||||
- **Image**：通用图像数据结构，持有 `const void* pixels` 和 `PixelFormat`（当前统一 `RGBA5551`），支持 color_key 透明跳过
|
- **Image**：通用图像数据结构，持有 `const void* pixels` 和 `PixelFormat`（当前统一 `RGBA5551`），支持 color_key 透明跳过
|
||||||
- **Sprite**：描述 atlas 中的子区域，通过 `const Image* atlas` 引用源图，是对外 sprite 绘制单位
|
- **Sprite**：描述 atlas 中的子区域，通过 `const Image* atlas` 引用源图，是对外 sprite 绘制单位
|
||||||
- **Tilemap**：使用 `uint16_t` tile id 引用 atlas 中的固定大小 tile，`EmptyTile` (`0xFFFF`) 表示空 tile
|
- **Tilemap**：使用 `uint16_t` tile id 引用 atlas 中的固定大小 tile，`EmptyTile` (`0xFFFF`) 表示空 tile
|
||||||
@@ -325,20 +381,24 @@ IMX6U-Game/
|
|||||||
- **Color**：RGBA8888 颜色值，用于绘制接口参数和调试；sprite 运行时像素格式仍为 RGBA5551
|
- **Color**：RGBA8888 颜色值，用于绘制接口参数和调试；sprite 运行时像素格式仍为 RGBA5551
|
||||||
|
|
||||||
### Core
|
### Core
|
||||||
|
|
||||||
- **FrameBuffer**：CPU 侧 RGB565 颜色缓冲，渲染结果先写在这里
|
- **FrameBuffer**：CPU 侧 RGB565 颜色缓冲，渲染结果先写在这里
|
||||||
- **DepthBuffer**：深度测试用 Z-buffer
|
- **DepthBuffer**：深度测试用 Z-buffer
|
||||||
- **Renderer**：渲染器辅助工具
|
- **Renderer**：渲染器辅助工具
|
||||||
- **Timer**：整数毫秒固定步长 tick 生成器，支持 30/45/60 FPS 档位和每帧剩余时间计算
|
- **Timer**：整数毫秒固定步长 tick 生成器，支持 30/45/60 FPS 档位和每帧剩余时间计算
|
||||||
|
|
||||||
### Math
|
### Math
|
||||||
|
|
||||||
- 通用数学类型：`Vector2/3/4`、`Matrix4x4`
|
- 通用数学类型：`Vector2/3/4`、`Matrix4x4`
|
||||||
- 纯头文件实现，无动态分配
|
- 纯头文件实现，无动态分配
|
||||||
|
|
||||||
### Rasterizer
|
### Rasterizer
|
||||||
|
|
||||||
- **Rasterizer**：Bresenham 线段光栅化，入口做快速全屏可见性检查，屏幕内走 `set_pixel_unsafe` 快路径，屏幕外走 Cohen-Sutherland 裁剪
|
- **Rasterizer**：Bresenham 线段光栅化，入口做快速全屏可见性检查，屏幕内走 `set_pixel_unsafe` 快路径，屏幕外走 Cohen-Sutherland 裁剪
|
||||||
- **TriangleRasterizer**：扫描线三角形填充 + 定点深度插值（增量式整数边缘函数，内层循环无 float 运算）
|
- **TriangleRasterizer**：扫描线三角形填充 + 定点深度插值（增量式整数边缘函数，内层循环无 float 运算）
|
||||||
|
|
||||||
### Platform
|
### Platform
|
||||||
|
|
||||||
- **IDisplay**：显示后端抽象，解耦渲染与输出
|
- **IDisplay**：显示后端抽象，解耦渲染与输出
|
||||||
- **SDLDisplay**：SDL2 后端，PC 调试和 IMX6U SDL2 目标路径共用这一类适配思想
|
- **SDLDisplay**：SDL2 后端，PC 调试和 IMX6U SDL2 目标路径共用这一类适配思想
|
||||||
- **FBDisplay**：`/dev/fb0` 对照后端，用于极简显示通路验证
|
- **FBDisplay**：`/dev/fb0` 对照后端，用于极简显示通路验证
|
||||||
@@ -366,6 +426,7 @@ IMX6U-Game/
|
|||||||
## 当前状态与后续
|
## 当前状态与后续
|
||||||
|
|
||||||
**已完成：**
|
**已完成：**
|
||||||
|
|
||||||
- 可旋转立方体的 3D 渲染（MVP 变换、背面剔除、扫描线填充、深度测试）
|
- 可旋转立方体的 3D 渲染（MVP 变换、背面剔除、扫描线填充、深度测试）
|
||||||
- 双平台显示后端（SDL2 / Framebuffer）
|
- 双平台显示后端（SDL2 / Framebuffer）
|
||||||
- 离线资源转换工具：PNG sprite -> RGBA5551 C++ 头文件，像素字体 -> bitmap atlas/header
|
- 离线资源转换工具：PNG sprite -> RGBA5551 C++ 头文件，像素字体 -> bitmap atlas/header
|
||||||
@@ -376,6 +437,7 @@ IMX6U-Game/
|
|||||||
- CMake 跨平台构建
|
- CMake 跨平台构建
|
||||||
|
|
||||||
**待完成（按优先级）：**
|
**待完成（按优先级）：**
|
||||||
|
|
||||||
1. FrameBuffer / FBDisplay 性能优化（目标像素格式 backbuffer、dirty rect、专用 tile/sprite 快路径、NEON）
|
1. FrameBuffer / FBDisplay 性能优化（目标像素格式 backbuffer、dirty rect、专用 tile/sprite 快路径、NEON）
|
||||||
2. 应用层拆分（Launcher / GameA / GameB / Shared）和统一 `IApp` 主循环
|
2. 应用层拆分（Launcher / GameA / GameB / Shared）和统一 `IApp` 主循环
|
||||||
3. SDL2 输入抽象（键盘/触摸/按键状态快照）
|
3. SDL2 输入抽象（键盘/触摸/按键状态快照）
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
After Width: | Height: | Size: 1.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 219 B |
Binary file not shown.
|
After Width: | Height: | Size: 112 B |
Binary file not shown.
|
After Width: | Height: | Size: 152 B |
Binary file not shown.
|
After Width: | Height: | Size: 240 B |
Binary file not shown.
|
After Width: | Height: | Size: 133 B |
Binary file not shown.
|
After Width: | Height: | Size: 1.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 174 B |
Binary file not shown.
|
After Width: | Height: | Size: 1.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 301 B |
Binary file not shown.
|
After Width: | Height: | Size: 1.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 155 B |
Binary file not shown.
|
After Width: | Height: | Size: 203 B |
Binary file not shown.
|
After Width: | Height: | Size: 1.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 829 B |
Binary file not shown.
|
After Width: | Height: | Size: 800 B |
Binary file not shown.
|
After Width: | Height: | Size: 813 B |
Binary file not shown.
|
After Width: | Height: | Size: 845 B |
Binary file not shown.
|
After Width: | Height: | Size: 824 B |
Binary file not shown.
|
After Width: | Height: | Size: 803 B |
Binary file not shown.
|
After Width: | Height: | Size: 806 B |
Binary file not shown.
|
After Width: | Height: | Size: 822 B |
Binary file not shown.
|
After Width: | Height: | Size: 154 B |
Binary file not shown.
|
After Width: | Height: | Size: 211 B |
Binary file not shown.
|
After Width: | Height: | Size: 1.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.8 KiB |
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"atlas_width": 256,
|
||||||
|
"atlas_height": 128,
|
||||||
|
"sprites": [
|
||||||
|
{ "name": "player_idle_0", "file": "player_idle_0.png" },
|
||||||
|
{ "name": "player_idle_1", "file": "player_idle_1.png" },
|
||||||
|
{ "name": "player_run_0", "file": "player_run_0.png" },
|
||||||
|
{ "name": "player_run_1", "file": "player_run_1.png" },
|
||||||
|
{ "name": "player_run_2", "file": "player_run_2.png" },
|
||||||
|
{ "name": "player_run_3", "file": "player_run_3.png" },
|
||||||
|
{ "name": "player_jump", "file": "player_jump.png" },
|
||||||
|
{ "name": "player_fall", "file": "player_fall.png" }
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
{
|
||||||
|
"tile_size": 32,
|
||||||
|
"columns": 6,
|
||||||
|
"tiles": [
|
||||||
|
{ "id": 0, "file": "empty.png" },
|
||||||
|
{ "id": 1, "file": "background_1.png" },
|
||||||
|
{ "id": 2, "file": "background_2.png" },
|
||||||
|
{ "id": 3, "file": "background_3.png" },
|
||||||
|
{ "id": 4, "file": "background_4.png" },
|
||||||
|
{ "id": 5, "file": "ground_top_green.png" },
|
||||||
|
{ "id": 6, "file": "ground_fill_green.png" },
|
||||||
|
{ "id": 7, "file": "ground_top_gray.png" },
|
||||||
|
{ "id": 8, "file": "ground_fill_gray.png" },
|
||||||
|
{ "id": 9, "file": "ground_top_purple.png" },
|
||||||
|
{ "id": 10, "file": "ground_fill_purple.png" },
|
||||||
|
{ "id": 11, "file": "platform.png" },
|
||||||
|
{ "id": 12, "file": "spike.png" },
|
||||||
|
{ "id": 13, "file": "light_platform_off.png" },
|
||||||
|
{ "id": 14, "file": "light_platform_on.png" },
|
||||||
|
{ "id": 15, "file": "shadow_platform_off.png" },
|
||||||
|
{ "id": 16, "file": "shadow_platform_on.png" },
|
||||||
|
{ "id": 17, "file": "door_closed.png" },
|
||||||
|
{ "id": 18, "file": "door_open.png" },
|
||||||
|
{ "id": 19, "file": "coin.png" },
|
||||||
|
{ "id": 20, "file": "flag.png" },
|
||||||
|
{ "id": 21, "file": "checkpoint.png" },
|
||||||
|
{ "id": 22, "file": "spike_ceiling.png" },
|
||||||
|
{ "id": 23, "file": "decor_vine.png" }
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -1,8 +1,12 @@
|
|||||||
set(CMAKE_SYSTEM_NAME Linux)
|
set(CMAKE_SYSTEM_NAME Linux)
|
||||||
set(CMAKE_SYSTEM_PROCESSOR arm)
|
set(CMAKE_SYSTEM_PROCESSOR arm)
|
||||||
|
|
||||||
set(CMAKE_C_COMPILER arm-linux-gnueabihf-gcc)
|
# Linaro GCC 4.9.4 2017.01，与 IMX6U 开发板原厂一致
|
||||||
set(CMAKE_CXX_COMPILER arm-linux-gnueabihf-g++)
|
# 解压：tar -xJf gcc-linaro-4.9.4-2017.01-x86_64_arm-linux-gnueabihf.tar.xz -C ~/toolchains
|
||||||
|
set(IMX6U_TOOLCHAIN_ROOT "$ENV{HOME}/toolchains/gcc-linaro-4.9.4-2017.01-x86_64_arm-linux-gnueabihf")
|
||||||
|
set(CMAKE_C_COMPILER "${IMX6U_TOOLCHAIN_ROOT}/bin/arm-linux-gnueabihf-gcc")
|
||||||
|
set(CMAKE_CXX_COMPILER "${IMX6U_TOOLCHAIN_ROOT}/bin/arm-linux-gnueabihf-g++")
|
||||||
|
set(CMAKE_SYSROOT "${IMX6U_TOOLCHAIN_ROOT}/arm-linux-gnueabihf/libc")
|
||||||
|
|
||||||
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
|
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
|
||||||
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
|
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
|
||||||
|
|||||||
@@ -219,8 +219,8 @@ ALSA、evdev、SDL2、`/dev/fb0` 等平台细节只能出现在 `src/Core/Platfo
|
|||||||
### 5.7 CMake 后端切换
|
### 5.7 CMake 后端切换
|
||||||
|
|
||||||
```cmake
|
```cmake
|
||||||
-DUSE_FRAMEBUFFER=OFF # 默认，使用 SDLDisplay
|
-DTARGET_IMX=OFF # 默认 (PC)，使用 SDLDisplay
|
||||||
-DUSE_FRAMEBUFFER=ON # 使用 FBDisplay
|
-DTARGET_IMX=ON # IMX6U 板，使用 FBDisplay
|
||||||
```
|
```
|
||||||
|
|
||||||
## 6. DrawContext
|
## 6. DrawContext
|
||||||
|
|||||||
@@ -0,0 +1,787 @@
|
|||||||
|
# 基于 IMX6ULL 的纯 CPU 渲染语音交互游戏设计与实现<br>—— TomGame 会说话的小猫
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 摘 要
|
||||||
|
|
||||||
|
本实训针对无 GPU 或 GPU 能力极弱的嵌入式 SoC（IMX6ULL / ARM Cortex-A7），设计并实现了一款以语音交互为核心的纯 CPU 渲染 2D 游戏 **TomGame（会说话的小猫）**。游戏支持两种语音玩法：按住录音后通过变调回放实现“学说话”效果，以及基于端侧关键词识别模型 **TinyKWS** 识别“Up / On / Stop”等指令并驱动角色动作。项目采用 C++11 编写，通过 CMake 条件编译在 PC（SDL2）与 IMX6ULL（Framebuffer / ALSA / evdev）之间共享同一套应用代码；底层渲染使用 RGB565 帧缓冲与 RGBA5551 精灵图集，运行时不依赖 PNG/TTF 解码。
|
||||||
|
|
||||||
|
本文重点阐述 TomGame 应用层的状态机、语音交互控制器、音频录制/变声/回放链路，以及嵌入式 TinyKWS 关键词识别模型的特征提取、网络结构与端侧推理实现；对底层 SDL2 / Framebuffer 显示适配、音频抽象与定时器只做必要概述。实验结果表明，TomGame 在 PC 端可稳定运行 60 FPS，在 IMX6ULL 板端以 1024×600 分辨率、30 FPS 目标帧率流畅运行，语音指令平均响应延迟满足实时交互需求，验证了“纯 CPU 渲染 + 端侧语音模型”在资源受限嵌入式设备上构建趣味交互游戏的可行性。
|
||||||
|
|
||||||
|
**关键词：** IMX6ULL；纯 CPU 渲染；语音交互；关键词识别；TinyKWS；MFCC；SDL2；Framebuffer
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## ABSTRACT
|
||||||
|
|
||||||
|
This training project designs and implements **TomGame**, a pure-CPU rendered 2D voice-interactive game on the IMX6ULL (ARM Cortex-A7) platform which has no GPU or only extremely limited graphics acceleration. The game supports two voice modes: a pitch-shifted “talkback” echo and a TinyKWS on-device keyword recognizer that maps commands such as “Up / On / Stop” to in-game actions. Written in C++11 and built with CMake, the project shares the same application code between PC (SDL2) and IMX6ULL (Framebuffer / ALSA / evdev) through conditional compilation. The renderer uses an RGB565 framebuffer and RGBA5551 sprite atlases, with all external resources converted offline so that no PNG/TTF decoding happens at runtime.
|
||||||
|
|
||||||
|
This report focuses on the TomGame application layer: the game-state machine, voice interaction controller, audio record / effect / playback pipeline, and the embedded TinyKWS keyword-spotting model including feature extraction, network architecture, and on-device inference. The underlying SDL2 / Framebuffer display adaptation, audio abstraction, and timer are only summarized where necessary. Experimental results show that TomGame runs at a stable 60 FPS on PC and at 1024×600 @ 30 FPS on the IMX6ULL board, with keyword-command latency meeting real-time interaction requirements. This verifies the feasibility of combining pure-CPU rendering with an on-device speech model to build engaging interactive games on resource-constrained embedded devices.
|
||||||
|
|
||||||
|
**Keywords:** IMX6ULL; pure-CPU rendering; voice interaction; keyword spotting; TinyKWS; MFCC; SDL2; Framebuffer
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 目 录
|
||||||
|
|
||||||
|
1 绪论
|
||||||
|
1.1 课题背景与研究意义
|
||||||
|
1.1.1 嵌入式游戏与语音交互的发展背景
|
||||||
|
1.1.2 在资源受限平台运行图形游戏的必要性
|
||||||
|
1.2 国内外研究现状
|
||||||
|
1.2.1 嵌入式图形渲染技术研究现状
|
||||||
|
1.2.2 端侧语音关键词识别技术研究现状
|
||||||
|
1.3 本文主要研究内容
|
||||||
|
|
||||||
|
2 系统总体方案与相关技术
|
||||||
|
2.1 系统需求分析
|
||||||
|
2.1.1 功能需求
|
||||||
|
2.1.2 性能需求
|
||||||
|
2.1.3 跨平台开发与部署需求
|
||||||
|
2.2 系统总体架构设计
|
||||||
|
2.2.1 Core / Apps 分层架构
|
||||||
|
2.2.2 平台抽象层设计
|
||||||
|
2.2.3 资源离线转换流程
|
||||||
|
2.3 关键开发平台与技术
|
||||||
|
2.3.1 IMX6ULL 开发板硬件平台
|
||||||
|
2.3.2 ARM Linux 与交叉编译环境
|
||||||
|
2.3.3 SDL2 / Framebuffer 双显示后端
|
||||||
|
2.3.4 C++11 与 CMake 构建系统
|
||||||
|
2.3.5 TinyKWS 关键词识别模型
|
||||||
|
2.3.6 音频采集与回放技术
|
||||||
|
|
||||||
|
3 硬件平台与运行环境设计
|
||||||
|
3.1 IMX6ULL 核心板硬件资源
|
||||||
|
3.1.1 CPU、内存与存储
|
||||||
|
3.1.2 LCD 显示接口与 framebuffer
|
||||||
|
3.1.3 音频输入输出接口
|
||||||
|
3.1.4 触摸与按键输入
|
||||||
|
3.2 开发环境搭建
|
||||||
|
3.2.1 Windows / Linux 主机开发环境
|
||||||
|
3.2.2 ARM 交叉编译工具链
|
||||||
|
3.2.3 板端运行环境配置
|
||||||
|
3.3 跨平台硬件抽象策略
|
||||||
|
|
||||||
|
4 系统软件设计
|
||||||
|
4.1 软件总体架构
|
||||||
|
4.1.1 模块划分与依赖关系
|
||||||
|
4.1.2 主循环与时间源设计
|
||||||
|
4.1.3 绘制调用链
|
||||||
|
4.2 TomGame 应用层设计
|
||||||
|
4.2.1 TomGameApp 状态机
|
||||||
|
4.2.2 角色动画系统（TomAnimator）
|
||||||
|
4.2.3 HUD 与设置面板
|
||||||
|
4.3 语音交互子系统设计
|
||||||
|
4.3.1 音频采集与回放
|
||||||
|
4.3.2 音频处理与变声
|
||||||
|
4.3.3 语音交互控制器
|
||||||
|
4.4 嵌入式关键词识别模型
|
||||||
|
4.4.1 识别流程与接口抽象
|
||||||
|
4.4.2 TinyKWS 模型结构
|
||||||
|
4.4.3 端侧推理实现
|
||||||
|
4.4.4 命令路由与游戏反馈
|
||||||
|
4.5 底层渲染与平台适配层
|
||||||
|
4.5.1 FrameBuffer 与 DrawContext 统一绘制接口
|
||||||
|
4.5.2 IDisplay 与 SDLDisplay / FBDisplay
|
||||||
|
4.5.3 IAudioInput / IAudioOutput 与 ALSA / SDL 后端
|
||||||
|
4.5.4 ITimeSource 与固定步长 Timer
|
||||||
|
4.6 离线资源转换工具链
|
||||||
|
4.6.1 Tom Atlas 自动生成流程
|
||||||
|
4.6.2 BitmapFont 生成
|
||||||
|
|
||||||
|
5 系统测试与结果分析
|
||||||
|
5.1 测试环境
|
||||||
|
5.2 跨平台编译测试
|
||||||
|
5.3 功能测试
|
||||||
|
5.4 性能测试
|
||||||
|
5.5 测试结果分析
|
||||||
|
|
||||||
|
6 总结与展望
|
||||||
|
6.1 工作总结
|
||||||
|
6.2 存在问题与改进方向
|
||||||
|
|
||||||
|
致谢
|
||||||
|
|
||||||
|
参考文献
|
||||||
|
|
||||||
|
附录
|
||||||
|
附录 A 关键源代码清单
|
||||||
|
附录 B 系统运行截图
|
||||||
|
附录 C 硬件连接示意图
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 1 绪论
|
||||||
|
|
||||||
|
## 1.1 课题背景与研究意义
|
||||||
|
|
||||||
|
### 1.1.1 嵌入式游戏与语音交互的发展背景
|
||||||
|
|
||||||
|
随着嵌入式处理器性能不断提升，以及 LCD、触摸屏、麦克风阵列等外设在教学级开发板上的普及，传统意义上“只能跑控制程序”的嵌入式平台也开始承载更丰富的多媒体交互应用。语音作为一种自然、无需双手的输入方式，在儿童陪伴、智能家居、车载交互等场景中得到广泛关注。将语音交互下沉到无 GPU 或弱 GPU 的低成本嵌入式设备，能够在不依赖云端的情况下完成本地唤醒、指令识别与即时反馈，既降低了网络依赖，也保护了用户隐私。
|
||||||
|
|
||||||
|
### 1.1.2 在资源受限平台运行图形游戏的必要性
|
||||||
|
|
||||||
|
IMX6ULL 是一颗典型的低功耗嵌入式 SoC，集成 ARM Cortex-A7 单核 CPU，未配备独立 GPU，显示输出依赖 CPU 直接向帧缓冲写入像素。在这样的平台上运行图形游戏，必须放弃传统的 OpenGL / GPU 渲染管线，改用纯 CPU 软光栅化；同时要在音频采集、语音推理、动画更新之间合理分配 CPU 预算，才能保证画面帧率与交互实时性。研究如何在资源受限平台上构建“可玩、可看、可交互”的语音游戏，对于嵌入式 Linux 应用开发、端侧 AI 部署以及软硬件协同设计均具有教学与工程价值。
|
||||||
|
|
||||||
|
## 1.2 国内外研究现状
|
||||||
|
|
||||||
|
### 1.2.1 嵌入式图形渲染技术研究现状
|
||||||
|
|
||||||
|
在 PC 与移动平台，现代图形 API（OpenGL、Vulkan、Metal）已经高度成熟；但在无 GPU 的 MCU / MPU 上，研究重点转向轻量级软渲染器、固定点运算、Tile-based 局部刷新和专用 2D 加速库。学术界与工业界常见的路线包括：使用 RGB565 / RGBA5551 等紧凑像素格式降低显存带宽，采用 Sprite 图集替代独立纹理，使用整数定点数替代浮点运算，以及在编译期将资源烘焙为 C 数组以避免运行时解码。这些思路与本项目的 `Core` 底层渲染库设计高度一致。
|
||||||
|
|
||||||
|
### 1.2.2 端侧语音关键词识别技术研究现状
|
||||||
|
|
||||||
|
端侧关键词识别（Keyword Spotting, KWS）近年来逐渐从云端向设备端迁移。以 Google 的 MicroNet、ARM 的 CMSIS-NN、Edge Impulse 的 TinyML 工作流为代表，研究人员通过深度可分离卷积（Depthwise Separable Convolution）、量化和剪枝，将神经网络压缩到数百 KB 甚至数十 KB，能够在 Cortex-M / Cortex-A 系列处理器上实时运行。TinyKWS 是一类面向嵌入式设备的轻量级 KWS 模型，通常以 MFCC 作为前端特征，以少量卷积层提取时频特征，最后通过全连接层输出关键词类别概率。本项目将类似的网络结构移植到 IMX6ULL 端侧，直接驱动游戏角色动作。
|
||||||
|
|
||||||
|
## 1.3 本文主要研究内容
|
||||||
|
|
||||||
|
本文围绕“基于 IMX6ULL 的纯 CPU 渲染语音交互游戏”展开，主要研究内容包括：
|
||||||
|
|
||||||
|
1. **TomGame 应用层设计与实现**：状态机驱动的角色动画、双模式语音交互（变调回放 / 关键词识别）、HUD 与设置面板、配置持久化。
|
||||||
|
2. **端侧语音关键词识别模型**：基于 MFCC 特征提取与轻量 CNN（Conv2D + Depthwise Separable Conv + Global Average Pooling + FC + Softmax）的 TinyKWS 模型，及其在 IMX6ULL 上的 C++ 推理实现。
|
||||||
|
3. **音频录制、变声与回放链路**：ALSA / SDL 双后端音频输入输出、录音缓冲、静音检测、变调（pitch-up）、音量增益与播放进度管理。
|
||||||
|
4. **跨平台渲染与部署**：RGB565 帧缓冲、RGBA5551 精灵图集、SDL2 / Framebuffer 可切换显示后端、ARM 交叉编译与板端运行验证。
|
||||||
|
|
||||||
|
> **说明：** 本项目为小组协作完成，底层 SDL2 / Framebuffer 显示适配、音频平台抽象与 `Core` 渲染库由同组成员共同维护；本文作者主要负责 TomGame 应用层、语音交互控制与 TinyKWS 端侧识别模型的设计与实现，因此报告内容按约 70% 侧重游戏应用与语音识别、30% 侧重底层渲染与平台适配进行组织。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 2 系统总体方案与相关技术
|
||||||
|
|
||||||
|
## 2.1 系统需求分析
|
||||||
|
|
||||||
|
### 2.1.1 功能需求
|
||||||
|
|
||||||
|
TomGame 的功能需求围绕“会说话的小猫”这一核心玩法展开：
|
||||||
|
|
||||||
|
- **角色展示**：屏幕中央显示卡通小猫 Tom，根据游戏状态切换 idle、录音倾听、说话、跳跃等动画。
|
||||||
|
- **变调学说话**：用户按住录音键/按钮录音，松开后 Tom 以更高音调重复播放刚才的录音，形成“学舌”效果。
|
||||||
|
- **关键词指令识别**：切换到“识别模式”后，用户说出“Up / On / Stop”等关键词，Tom 执行跳跃或停止动作。
|
||||||
|
- **输入方式**：支持物理按键、触摸屏点击以及语音三种输入方式。
|
||||||
|
- **设置面板**：可调节录音增益与播放音量，配置在本地持久化。
|
||||||
|
- **退出与返回**：提供退出按钮与设置按钮，支持随时中断语音流程。
|
||||||
|
|
||||||
|
### 2.1.2 性能需求
|
||||||
|
|
||||||
|
- **画面帧率**：PC 端不低于 60 FPS；IMX6ULL 板端以 1024×600 分辨率稳定 30 FPS。
|
||||||
|
- **音频延迟**：录音启动到开始采集的延迟小于 100 ms；变调回放启动延迟小于 50 ms。
|
||||||
|
- **识别延迟**：关键词录音结束后，端侧推理与命令响应总延迟控制在 300 ms 以内。
|
||||||
|
- **资源占用**：运行时避免动态内存频繁分配；二进制体积与内存占用适配 IMX6ULL 的 256 MB DDR。
|
||||||
|
|
||||||
|
### 2.1.3 跨平台开发与部署需求
|
||||||
|
|
||||||
|
- 同一套源码能够在 Windows（MSVC + SDL2）、Linux x86_64（GCC + SDL2）和 ARM Linux（交叉编译 + SDL2 或 Framebuffer）上编译运行。
|
||||||
|
- 平台相关代码集中在 `src/Core/Platform/`，应用层不直接依赖 SDL、ALSA、evdev 等具体 API。
|
||||||
|
- 图片、字体资源在构建前离线转换为 C++ 头文件，板端无需部署原始 PNG/TTF。
|
||||||
|
|
||||||
|
## 2.2 系统总体架构设计
|
||||||
|
|
||||||
|
### 2.2.1 Core / Apps 分层架构
|
||||||
|
|
||||||
|
项目采用严格单向依赖的分层架构，总体依赖方向为 `Apps -> Core -> Platform`：
|
||||||
|
|
||||||
|
- **Platform 层**：位于最底层，封装显示、时间、音频输入输出、按键、指针输入等硬件差异，以纯虚接口形式向上提供服务。
|
||||||
|
- **Core 层**：建立在 Platform 接口之上，提供 `FrameBuffer`、`DrawContext`、`Rasterizer`、`Sprite`、`Tilemap`、`BitmapFont`、`Timer` 等可复用的 2D 渲染与定时组件。
|
||||||
|
- **Apps 层**：位于最顶层，TomGame 位于 `src/Apps/Game/`，包含游戏专属的状态机、动画、语音交互和关键词识别逻辑。
|
||||||
|
|
||||||
|
[图 1：TomGame 系统分层架构图]
|
||||||
|
|
||||||
|
```text
|
||||||
|
┌─────────────────────────────────────────────┐
|
||||||
|
│ Apps/Game：TomGameApp / 语音交互 / TinyKWS │
|
||||||
|
├─────────────────────────────────────────────┤
|
||||||
|
│ Core：DrawContext / FrameBuffer / Sprite │
|
||||||
|
├─────────────────────────────────────────────┤
|
||||||
|
│ Platform：IDisplay / IAudioInput/Output │
|
||||||
|
│ IButtonInput / IPointerInput │
|
||||||
|
│ ITimeSource / Timer │
|
||||||
|
└─────────────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.2.2 平台抽象层设计
|
||||||
|
|
||||||
|
平台抽象层的核心思想是：应用层只与接口打交道，不关心底层是 SDL2 还是 Linux 原生设备节点。关键接口包括：
|
||||||
|
|
||||||
|
- `Platform::IDisplay`：显示后端抽象，实现为 `SDLDisplay`（PC）或 `FBDisplay`（IMX6ULL /dev/fb0）。
|
||||||
|
- `Platform::IAudioInput / IAudioOutput`：音频输入输出抽象，实现为 `SdlAudioInput / SdlAudioOutput` 或 `AlsaAudioInput / AlsaAudioOutput`。
|
||||||
|
- `Platform::IButtonInput / IPointerInput`：物理按键与触摸/鼠标指针抽象，实现为 `SdlKeyboardButtonInput / SdlPointerInput` 或 `EvdevButtonInput / EvdevTouchInput`。
|
||||||
|
- `Platform::ITimeSource`：独立时间源，提供单调递增的整数毫秒时钟。
|
||||||
|
|
||||||
|
`DefaultHardware.h` 通过 `typedef` 按平台选择默认后端，使 `TomGameApp` 中几乎不存在 `#ifdef TARGET_IMX` 分支。
|
||||||
|
|
||||||
|
### 2.2.3 资源离线转换流程
|
||||||
|
|
||||||
|
为了降低运行时负担，所有图片与字体资源在构建前完成离线转换：
|
||||||
|
|
||||||
|
- Tom 游戏的精灵素材由 `tools/asset_pipeline/SpriteAssetTool.cpp` 从 `src/Apps/Game/assets/raw/` 读取 PNG，打包成 `src/Apps/Game/generated/tom_atlas.h`。
|
||||||
|
- 通用测试 sprite 使用 `tools/png_to_header.py` 转换为 RGBA5551 C 数组。
|
||||||
|
- 像素字体使用 `tools/gen_font_atlas.py` 生成 1-bit mask 位图字体头文件。
|
||||||
|
|
||||||
|
板端运行时不依赖 PNG/TTF 解码库，所有资源直接以编译期常量的形式链接进可执行文件。
|
||||||
|
|
||||||
|
## 2.3 关键开发平台与技术
|
||||||
|
|
||||||
|
### 2.3.1 IMX6ULL 开发板硬件平台
|
||||||
|
|
||||||
|
目标硬件为正点原子 IMX6ULL 教学开发板，核心资源如下：
|
||||||
|
|
||||||
|
- CPU：NXP i.MX6ULL，ARM Cortex-A7 @ 792 MHz（部分版本 800 MHz）。
|
||||||
|
- 内存：256 MB DDR3L。
|
||||||
|
- 存储：8 GB eMMC / NAND Flash。
|
||||||
|
- 显示：7 英寸 RGB LCD，分辨率 1024×600，通过 framebuffer 接口输出。
|
||||||
|
- 音频：板载 WM8960 音频编解码器，通过 I²S 连接，Linux 下以 ALSA 设备暴露。
|
||||||
|
- 输入：电阻/电容触摸屏（evdev）、用户按键、麦克风。
|
||||||
|
|
||||||
|
### 2.3.2 ARM Linux 与交叉编译环境
|
||||||
|
|
||||||
|
开发主机运行 Windows / Ubuntu，交叉编译工具链为 `arm-linux-gnueabihf-gcc/g++`。CMake 工具链文件位于 `cmake/toolchain-arm-linux-gnueabihf.cmake`，通过 `-DCMAKE_TOOLCHAIN_FILE` 指定后即可生成 ARM Makefile / Ninja 工程。
|
||||||
|
|
||||||
|
### 2.3.3 SDL2 / Framebuffer 双显示后端
|
||||||
|
|
||||||
|
- `SDL2 后端`：PC 主路径，用于快速迭代调试；IMX6ULL 上若已部署 SDL2 库，也可作为对照路径。
|
||||||
|
- `Framebuffer 后端`：IMX6ULL 极简路径，直接写 `/dev/fb0`，避免 SDL2 依赖。
|
||||||
|
|
||||||
|
CMake 选项 `TARGET_IMX` 控制软件栈：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
-DTARGET_IMX=OFF # PC / ARM-SDL2：SDLDisplay + SdlAudio + SdlInput
|
||||||
|
-DTARGET_IMX=ON # IMX6ULL：FBDisplay + AlsaAudio + EvdevInput
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.3.4 C++11 与 CMake 构建系统
|
||||||
|
|
||||||
|
项目采用 C++11 标准，以兼容较老的嵌入式交叉工具链。顶层 `CMakeLists.txt` 定义了 `imx6u_core` 静态库与多个应用 target（Game、Demo、LightGame、Desktop），并通过 `imx6u_configure_app_target` 函数统一处理链接、输出目录与平台依赖。
|
||||||
|
|
||||||
|
### 2.3.5 TinyKWS 关键词识别模型
|
||||||
|
|
||||||
|
TinyKWS 是本项目的端侧语音核心。它以 16 kHz 单声道 PCM 为输入，通过预加重、分帧、加汉宁窗、FFT、Mel 滤波 bank 与 DCT 提取 49×10 维 MFCC 特征，再送入一个轻量卷积神经网络得到 12 类关键词概率。模型权重以 `int8_t / float` 数组形式嵌入 `TinyKwsModelData.cpp`，无需运行时加载外部模型文件。
|
||||||
|
|
||||||
|
### 2.3.6 音频采集与回放技术
|
||||||
|
|
||||||
|
音频链路抽象为 `IAudioInput`（录音）与 `IAudioOutput`（播放）两个接口。PC 端由 SDL2 音频回调驱动，板端由 ALSA `snd_pcm_readi / snd_pcm_writei` 直接读写。游戏层通过 `VoiceRecorder` 与 `VoicePlayer` 管理环形缓冲、播放进度与结束判定，避免在主循环中直接处理平台音频细节。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 3 硬件平台与运行环境设计
|
||||||
|
|
||||||
|
## 3.1 IMX6ULL 核心板硬件资源
|
||||||
|
|
||||||
|
### 3.1.1 CPU、内存与存储
|
||||||
|
|
||||||
|
IMX6ULL 采用 ARM Cortex-A7 单核，支持 VFPv4 浮点单元；NEON 可用但项目核心渲染路径优先使用整数运算以控制功耗与延迟。板载 256 MB DDR3L 与 8 GB eMMC，足够容纳本项目生成的二进制与资源数据。由于缺少 GPU，所有图形计算必须由 CPU 完成。
|
||||||
|
|
||||||
|
### 3.1.2 LCD 显示接口与 framebuffer
|
||||||
|
|
||||||
|
LCD 控制器通过 IPU / PxP 或简单 DMA 将系统内存中的帧缓冲内容扫描输出。Linux 内核将显存映射为 `/dev/fb0` 字符设备，用户空间可通过 `ioctl(FBIOGET_FSCREENINFO / FBIOGET_VSCREENINFO)` 获取像素格式与行步长，再用 `mmap` 直接写入。IMX6ULL 教学板常见面板格式为 RGB565 或 ARGB8888；本项目的 `FBDisplay` 会读取运行时格式并选择对应转换路径。
|
||||||
|
|
||||||
|
### 3.1.3 音频输入输出接口
|
||||||
|
|
||||||
|
板载 WM8960 通过 I²S 与 IMX6ULL 连接，Linux 内核加载 `snd_soc_imx_wm8960` 后，用户空间可通过 ALSA 访问 `hw:0,0` 等设备节点。麦克风用于录音，耳机/扬声器接口用于播放变调后的语音。本项目默认使用 16 kHz、16 bit、单声道配置，以保证 MFCC 特征与 TinyKWS 模型输入一致。
|
||||||
|
|
||||||
|
### 3.1.4 触摸与按键输入
|
||||||
|
|
||||||
|
触摸屏通过 I²C 或 USB 连接到 IMX6ULL，内核生成 `/dev/input/eventX` 事件节点。`EvdevTouchInput` 通过 `EV_ABS` 坐标事件读取触摸按下/释放状态，并将原始坐标映射到 1024×600 屏幕坐标。用户按键同样通过 evdev 读取 `EV_KEY` 事件。
|
||||||
|
|
||||||
|
## 3.2 开发环境搭建
|
||||||
|
|
||||||
|
### 3.2.1 Windows / Linux 主机开发环境
|
||||||
|
|
||||||
|
- Windows：安装 Visual Studio / MSVC，仓库已自带 `libs/Win/SDL2` 与 `libs/Win/SDL_image`。
|
||||||
|
- Linux：安装 `libsdl2-dev`、`libsdl2-image-dev`、`cmake`、`g++`；交叉编译时额外安装 `gcc-arm-linux-gnueabihf`。
|
||||||
|
|
||||||
|
### 3.2.2 ARM 交叉编译工具链
|
||||||
|
|
||||||
|
交叉编译命令示例：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cmake -B build-arm-fb \
|
||||||
|
-DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-arm-linux-gnueabihf.cmake \
|
||||||
|
-DTARGET_IMX=ON .
|
||||||
|
cmake --build build-arm-fb
|
||||||
|
```
|
||||||
|
|
||||||
|
> 注意：Tom 图集头文件生成工具 `TomAtlasTool` 是主机侧工具，依赖 PC/Linux 上的 SDL2_image；ARM 交叉编译前需先在主机构建目录执行 `GenerateTomAtlasHeader`。
|
||||||
|
|
||||||
|
### 3.2.3 板端运行环境配置
|
||||||
|
|
||||||
|
将生成的 `IMX6U-Game` 二进制通过 `scp` 复制到板端 `/tmp/` 或 `/opt/imx6u-game/`，并确保：
|
||||||
|
|
||||||
|
- `/dev/fb0` 可读写（Framebuffer 后端）。
|
||||||
|
- ALSA 设备节点存在且声卡驱动已加载。
|
||||||
|
- 触摸屏/按键事件节点权限正确。
|
||||||
|
|
||||||
|
## 3.3 跨平台硬件抽象策略
|
||||||
|
|
||||||
|
跨平台的关键在于将平台相关代码收敛到 `src/Core/Platform/`，并通过 `DefaultHardware.h` 的 `typedef` 在编译期选择后端。`TomGameApp` 只持有接口指针，因此同一套应用代码在 PC 与板端均可编译运行。主循环中通过 `CreateDisplay()` 在 `TARGET_IMX` 与 `TARGET_PC` 之间二选一创建显示后端，是应用层唯一的显式平台分支。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 4 系统软件设计
|
||||||
|
|
||||||
|
## 4.1 软件总体架构
|
||||||
|
|
||||||
|
### 4.1.1 模块划分与依赖关系
|
||||||
|
|
||||||
|
TomGame 的软件模块可划分为以下层次：
|
||||||
|
|
||||||
|
| 层级 | 模块 | 主要职责 |
|
||||||
|
|------|------|----------|
|
||||||
|
| 应用层 | `TomGameApp` | 状态机、输入处理、设置管理 |
|
||||||
|
| 应用层 | `TomAnimator` | 角色动画帧选择与绘制定位 |
|
||||||
|
| 应用层 | `TomHud` / `TomSettingsPanel` | HUD 按钮与设置滑块 |
|
||||||
|
| 应用层 | `VoiceInteractionController` | 录音/变声/播放/识别流程调度 |
|
||||||
|
| 应用层 | `VoiceRecorder / VoicePlayer / VoiceEffect` | 音频缓冲与数字信号处理 |
|
||||||
|
| 应用层 | `TinyKwsRecognizer` / `KeywordCommandRouter` | 关键词识别与指令映射 |
|
||||||
|
| 底层库 | `Core::DrawContext / FrameBuffer / Sprite` | 2D 渲染 |
|
||||||
|
| 平台层 | `IDisplay / IAudioInput / IAudioOutput / IButtonInput / IPointerInput / ITimeSource` | 平台适配 |
|
||||||
|
|
||||||
|
### 4.1.2 主循环与时间源设计
|
||||||
|
|
||||||
|
`src/Apps/Game/Main.cpp` 中的主循环遵循“轮询输入 → 固定步长更新 → 绘制 → 提交 → 睡眠等待”的结构：
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
timer.begin_frame(time_source.get_time_ms());
|
||||||
|
display->poll_events(should_quit);
|
||||||
|
app.update(timer.fixed_delta_ms());
|
||||||
|
app.draw(ctx);
|
||||||
|
ctx.present(display);
|
||||||
|
SleepRemainingFrameTime(timer, time_source);
|
||||||
|
```
|
||||||
|
|
||||||
|
`Platform::SteadyTimeSource` 在 Linux 下基于 `clock_gettime(CLOCK_MONOTONIC)`，在 Windows 下基于 `std::chrono::steady_clock`，返回 `uint32_t` 毫秒。`Core::Timer` 采用余数累积法生成固定时间片，例如 30 FPS 下依次产生 33、33、34 ms 的 tick，保证长时间运行无帧率漂移。
|
||||||
|
|
||||||
|
[图 2：TomGame 主循环与模块交互流程图]
|
||||||
|
|
||||||
|
### 4.1.3 绘制调用链
|
||||||
|
|
||||||
|
每帧绘制流程如下：
|
||||||
|
|
||||||
|
1. `TomGameApp::draw(ctx)` 调用 `ctx.clear_color(...)` 清屏。
|
||||||
|
2. 绘制背景精灵 `TomAtlas::background`。
|
||||||
|
3. `TomAnimator::draw(ctx, state)` 根据当前状态选择并绘制 Tom 角色精灵。
|
||||||
|
4. `TomHud::draw(ctx)` 绘制底部模式切换按钮与录音按钮。
|
||||||
|
5. 绘制左上角退出按钮 `ui_tom` 与右上角设置按钮 `ui_i`。
|
||||||
|
6. 若设置面板打开，绘制 `TomSettingsPanel`。
|
||||||
|
7. `ctx.present(display)` 将 RGB565 帧缓冲提交到显示后端。
|
||||||
|
|
||||||
|
所有绘制调用均通过 `Core::DrawContext` 统一入口，应用层不直接操作 framebuffer 内存。
|
||||||
|
|
||||||
|
## 4.2 TomGame 应用层设计
|
||||||
|
|
||||||
|
### 4.2.1 TomGameApp 状态机
|
||||||
|
|
||||||
|
`TomGameApp` 维护一个四级状态机：
|
||||||
|
|
||||||
|
- **Idle（ idle ）**：等待用户触发录音或识别。
|
||||||
|
- **Recording（录音中）**：根据当前语音模式，分别进入变调回放录音或关键词识别录音。
|
||||||
|
- **Speaking（说话中）**：播放变调后的录音，并循环播放说话动画。
|
||||||
|
- **Jumping（跳跃中）**：响应“Up / On”关键词或相应指令，播放跳跃动画。
|
||||||
|
|
||||||
|
状态转换由 `VoiceInteractionController` 返回的事件驱动，例如 `RecordingStarted`、`SpeakingStarted`、`SpeakingFinished`、`KeywordRecognized`、`IdleRequested`。
|
||||||
|
|
||||||
|
[图 3：TomGameApp 状态机转换图]
|
||||||
|
|
||||||
|
```text
|
||||||
|
Idle --(record trigger)--> Recording
|
||||||
|
Recording --(finish pitch recording)--> Speaking
|
||||||
|
Recording --(keyword recognized)--> Jumping
|
||||||
|
Recording --(no keyword)--> Idle
|
||||||
|
Speaking --(playback finished)--> Idle
|
||||||
|
Jumping --(animation finished)--> Idle
|
||||||
|
```
|
||||||
|
|
||||||
|
输入处理逻辑在 `update_input()` 中完成：
|
||||||
|
|
||||||
|
- 物理按键按下触发录音。
|
||||||
|
- 触摸屏点击退出或设置按钮进入对应流程。
|
||||||
|
- HUD 上的模式按钮切换 `VoiceMode`（PitchRepeat / KeywordRecognition）。
|
||||||
|
- 设置面板打开时，主游戏逻辑暂停，只处理设置滑块与保存按钮。
|
||||||
|
|
||||||
|
### 4.2.2 角色动画系统（TomAnimator）
|
||||||
|
|
||||||
|
`TomAnimator` 负责根据状态选择并定位 Tom 的精灵帧：
|
||||||
|
|
||||||
|
- `Idle`：显示 `tom_stand`。
|
||||||
|
- `Recording`：显示 `tom_listhen`。
|
||||||
|
- `Speaking`：循环播放 `tom_say1 ~ tom_say4` 序列，每帧 90 ms。
|
||||||
|
- `Jumping`：顺序播放 `tom_jump1 ~ tom_jump6`，每帧 150 ms，结束后由应用层切回 Idle。
|
||||||
|
|
||||||
|
Tom 的水平位置始终居中，垂直位置根据屏幕高度与精灵高度计算底部留空 `TomBottomPadding = 72`。
|
||||||
|
|
||||||
|
### 4.2.3 HUD 与设置面板
|
||||||
|
|
||||||
|
`TomHud` 在屏幕底部提供三个按钮：
|
||||||
|
|
||||||
|
- 左侧“变调学说话”模式按钮
|
||||||
|
- 右侧“关键词识别”模式按钮
|
||||||
|
- 中间录音按钮
|
||||||
|
|
||||||
|
当前激活模式通过按钮高亮状态反馈。`TomSettingsPanel` 提供两个滑块：录音增益（`record_gain`）与播放音量（`speaker_volume`），并支持保存到本地文件 `tom_settings.cfg`。配置加载发生在 `TomGameApp` 构造阶段，保存时写入 `key = value` 文本格式。
|
||||||
|
|
||||||
|
[图 4：TomGame 主界面布局示意图]
|
||||||
|
|
||||||
|
## 4.3 语音交互子系统设计
|
||||||
|
|
||||||
|
### 4.3.1 音频采集与回放
|
||||||
|
|
||||||
|
**VoiceRecorder** 封装了录音缓冲与静音检测：
|
||||||
|
|
||||||
|
- 可配置采样率、通道数、最大录音时长、静音阈值与最小录音时长。
|
||||||
|
- 每帧从 `IAudioInput` 读取一定数量样本，累积到 `std::vector<int16_t>` 中。
|
||||||
|
- 当检测到连续静音样本数超过阈值，或达到最大录音时长时，自动标记 `finished`。
|
||||||
|
- 提供 `get_last_volume()` 用于 HUD 音量反馈。
|
||||||
|
|
||||||
|
**VoicePlayer** 封装了播放缓冲与进度管理：
|
||||||
|
|
||||||
|
- 接收 `int16_t` 样本、采样率、通道数。
|
||||||
|
- 每帧向 `IAudioOutput` 写入一定数量的样本，更新 `playPosition`。
|
||||||
|
- 播放结束后标记 `finished`。
|
||||||
|
|
||||||
|
### 4.3.2 音频处理与变声
|
||||||
|
|
||||||
|
`VoiceEffect` 提供了一系列静态工具函数：
|
||||||
|
|
||||||
|
- `amplify`：对样本乘以增益并裁剪到 `int16_t` 范围。
|
||||||
|
- `trim_silence`：去除头尾低于阈值的静音段。
|
||||||
|
- `pitch_up`：使用线性插值或重采样实现升调效果（默认 `pitchFactor = 1.45`）。
|
||||||
|
- `resample`：线性插值重采样，用于适配不同的输入/输出采样率。
|
||||||
|
- `convert_channels`：单声道与立体声互转。
|
||||||
|
|
||||||
|
在“变调学说话”模式下，录音结束后依次执行：
|
||||||
|
|
||||||
|
```text
|
||||||
|
录音原始样本
|
||||||
|
→ amplify(inputGain)
|
||||||
|
→ trim_silence
|
||||||
|
→ pitch_up(1.45f)
|
||||||
|
→ amplify(outputGain)
|
||||||
|
→ 播放
|
||||||
|
```
|
||||||
|
|
||||||
|
默认输出增益 `outputGain = 1.15f`，使变调后的声音保持足够响度。
|
||||||
|
|
||||||
|
### 4.3.3 语音交互控制器
|
||||||
|
|
||||||
|
`VoiceInteractionController` 是语音交互的中央调度器，向上层返回统一的事件枚举 `VoiceInteractionEvent`。其核心流程如下：
|
||||||
|
|
||||||
|
1. `start_recording()`：停止当前播放，初始化音频输入，启动 `VoiceRecorder`。
|
||||||
|
2. `update_pitch_repeat_recording(deltaMs)`：持续读取样本，超时或静音结束后进入 `repeat_last_recording()`。
|
||||||
|
3. `update_keyword_recording(deltaMs)`：持续读取样本，超时或静音结束后调用 `try_recognize_keyword()`。
|
||||||
|
4. `repeat_last_recording()`：对录音应用变调与增益，然后调用 `start_speaking()`。
|
||||||
|
5. `try_recognize_keyword()`：将样本送入 `TinyKwsRecognizer`，返回识别结果。
|
||||||
|
6. `update_speaking(deltaMs)`：持续向 `IAudioOutput` 写入样本，播放结束返回 `SpeakingFinished`。
|
||||||
|
|
||||||
|
[图 5：语音交互控制器数据流与状态转换图]
|
||||||
|
|
||||||
|
## 4.4 嵌入式关键词识别模型
|
||||||
|
|
||||||
|
### 4.4.1 识别流程与接口抽象
|
||||||
|
|
||||||
|
为了便于后续替换不同识别后端，`KeywordRecognizer.h` 定义了抽象接口 `IKeywordRecognizer`：
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
class IKeywordRecognizer {
|
||||||
|
public:
|
||||||
|
virtual bool init() = 0;
|
||||||
|
virtual KeywordRecognitionResult recognize(
|
||||||
|
const std::vector<int16_t>& samples,
|
||||||
|
uint32_t sampleRate,
|
||||||
|
uint32_t channels) = 0;
|
||||||
|
virtual void set_input_gain(float gain) {}
|
||||||
|
virtual float get_input_gain() const { return 1.0f; }
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
`KeywordCommand` 枚举定义了 9 条可识别指令：`Up、Down、Left、Right、Go、Stop、Yes、No、On`。识别结果包含命令与置信度两个字段。
|
||||||
|
|
||||||
|
### 4.4.2 TinyKWS 模型结构
|
||||||
|
|
||||||
|
本项目的 TinyKWS 模型以 Google Speech Commands 数据集中常见的轻量关键词识别网络为参考，结构如下：
|
||||||
|
|
||||||
|
**前端特征提取：**
|
||||||
|
|
||||||
|
- 输入音频：16 kHz 单声道 PCM，预处理为统一采样率与通道数。
|
||||||
|
- 语音活动检测：基于能量进行头尾静音裁剪。
|
||||||
|
- 分帧：帧长 512 点（32 ms），帧移 320 点（20 ms），共 49 帧。
|
||||||
|
- 加窗：汉宁窗（Hann Window）。
|
||||||
|
- FFT：512 点复数 FFT，取 257 维幅度谱。
|
||||||
|
- Mel 滤波 bank：40 个 Mel 滤波器，覆盖 20 Hz ~ 4000 Hz。
|
||||||
|
- DCT：取前 10 个 MFCC 系数，最终特征维度为 `49 × 10`。
|
||||||
|
|
||||||
|
**神经网络结构：**
|
||||||
|
|
||||||
|
| 层 | 输入 | 输出 | 说明 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| Conv2D | 49×10×1 | 25×5×64 | 卷积核 10×4，stride 2×2，ReLU |
|
||||||
|
| DepthwiseConv2D + Conv1×1 | 25×5×64 | 25×5×64 | 重复 4 组深度可分离块 |
|
||||||
|
| Global Average Pooling | 25×5×64 | 1×1×64 | 空间平均 |
|
||||||
|
| Fully Connected | 64 | 12 | 输出 logits |
|
||||||
|
| Softmax | 12 | 12 | 类别概率 |
|
||||||
|
|
||||||
|
12 个输出类别与关键词映射关系如下：
|
||||||
|
|
||||||
|
| 类别索引 | TinyKWS 输出 | 映射命令 |
|
||||||
|
|---|---|---|
|
||||||
|
| 0 | Down | Down |
|
||||||
|
| 1 | Go | Go |
|
||||||
|
| 2 | Left | Left |
|
||||||
|
| 3 | No | No |
|
||||||
|
| 5 | On | On |
|
||||||
|
| 6 | Right | Right |
|
||||||
|
| 7 | Stop | Stop |
|
||||||
|
| 8 | Up | Up |
|
||||||
|
| 9 | Yes | Yes |
|
||||||
|
| 4 / 10 / 11 | Off / Silence / Unknown | None（忽略）|
|
||||||
|
|
||||||
|
[图 6：TinyKWS 网络结构示意图]
|
||||||
|
|
||||||
|
```text
|
||||||
|
MFCC 49×10
|
||||||
|
↓
|
||||||
|
Conv2D(10×4, s=2) → 25×5×64
|
||||||
|
↓
|
||||||
|
DepthwiseConv2D + PointwiseConv × 4
|
||||||
|
↓
|
||||||
|
Global Average Pooling → 64
|
||||||
|
↓
|
||||||
|
FC → 12
|
||||||
|
↓
|
||||||
|
Softmax
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4.4.3 端侧推理实现
|
||||||
|
|
||||||
|
`TinyKwsRecognizer` 的实现完全位于 `src/Apps/Game/src/recognition/TinyKwsRecognizer.cpp`，不依赖任何外部推理框架。主要步骤：
|
||||||
|
|
||||||
|
1. **PrepareAudio**：将输入样本重采样到 16 kHz 单声道，并做静音裁剪。
|
||||||
|
2. **NormalizeWindow**：对 1 秒窗口（16000 点）做最大幅值归一化，再乘以输入增益。
|
||||||
|
3. **ExtractMfccFeatures**：按前述流程提取 49×10 维 MFCC。
|
||||||
|
4. **run_embedded_model**：执行 Conv2D、DepthwiseConv2D、全局平均池化、全连接与 Softmax。
|
||||||
|
5. **置信度阈值判定**：默认阈值 `confidenceThreshold = 0.75`，只有最佳类别概率超过阈值且映射到非 `None` 命令时才返回识别结果。
|
||||||
|
|
||||||
|
对于超过 1 秒的录音，采用滑动窗口策略：窗口 16000 点、步长 8000 点，取所有窗口中置信度最高的有效结果。
|
||||||
|
|
||||||
|
模型权重以 `extern const` 数组形式存放在 `TinyKwsModelData.cpp` 中：
|
||||||
|
|
||||||
|
- `Conv0Weights` 等卷积权重使用 `int8_t` 量化并配合 `WeightsScale` 反量化；
|
||||||
|
- Depthwise 权重与全连接权重使用 `float`；
|
||||||
|
- 偏置统一使用 `float`。
|
||||||
|
|
||||||
|
这种设计使模型参数直接编译进可执行文件，板端无需文件系统支持即可运行推理。
|
||||||
|
|
||||||
|
### 4.4.4 命令路由与游戏反馈
|
||||||
|
|
||||||
|
`KeywordCommandRouter` 将识别到的关键词映射为游戏动作：
|
||||||
|
|
||||||
|
- `Up` / `On` → `KeywordGameAction::Jump`
|
||||||
|
- `Stop` → `KeywordGameAction::BackToIdle`
|
||||||
|
- 其他命令 → `KeywordGameAction::None`
|
||||||
|
|
||||||
|
`TomGameApp` 收到 `KeywordRecognized` 事件后，通过 `handle_keyword_action()` 触发跳跃或停止。当前仅实现了跳跃反馈，后续可扩展为方向移动、暂停、确认/取消等更多动作。
|
||||||
|
|
||||||
|
[图 7：关键词识别 → 命令路由 → 游戏反馈链路图]
|
||||||
|
|
||||||
|
## 4.5 底层渲染与平台适配层
|
||||||
|
|
||||||
|
### 4.5.1 FrameBuffer 与 DrawContext 统一绘制接口
|
||||||
|
|
||||||
|
`Core::FrameBuffer` 在构造时一次性分配 `width × height` 个 `uint16_t` 像素作为 RGB565 缓冲，运行期不再扩容。`Core::DrawContext` 封装了 framebuffer、depthbuffer、线段光栅化、三角形光栅化，对外提供 `clear_color`、`draw_sprite`、`draw_text`、`fill_rect`、`present` 等接口。TomGame 主要使用 2D sprite 绘制路径，因此不启用 depth buffer。
|
||||||
|
|
||||||
|
Sprite 绘制热路径采用 RGBA5551 1-bit 透明：alpha 为 0 的像素直接跳过，为 1 的像素覆写到 RGB565 帧缓冲。该路径不含 alpha 混合，每像素仅一次读取、一次位测试和一次写入。
|
||||||
|
|
||||||
|
### 4.5.2 IDisplay 与 SDLDisplay / FBDisplay
|
||||||
|
|
||||||
|
`Platform::IDisplay` 定义四个纯虚方法：`init`、`present`、`poll_events`、`shutdown`。`SDLDisplay` 在 PC 上创建 RGB565 streaming texture，通过 `SDL_UpdateTexture + SDL_RenderCopy + SDL_RenderPresent` 提交；`FBDisplay` 在板端通过 `mmap /dev/fb0` 并一次性 `memcpy` 提交。两种后端使应用层无需关心“像素如何上屏”。
|
||||||
|
|
||||||
|
### 4.5.3 IAudioInput / IAudioOutput 与 ALSA / SDL 后端
|
||||||
|
|
||||||
|
`IAudioInput` 抽象了 `init(device, sample_rate, channels)`、`read_samples`、`shutdown` 等操作；`IAudioOutput` 抽象了 `init`、`write_samples`、`play_wav`、`shutdown`。PC 端由 SDL2 音频设备回调驱动，板端由 ALSA 直接读写 PCM。`DefaultAudioInput / DefaultAudioOutput` 通过 `DefaultHardware.h` 按平台 typedef 选择具体实现。
|
||||||
|
|
||||||
|
### 4.5.4 ITimeSource 与固定步长 Timer
|
||||||
|
|
||||||
|
`Platform::ITimeSource` 独立于显示层，提供单调整数毫秒时钟。`Core::Timer` 支持 30 / 45 / 60 FPS 三档，采用余数累积法避免 `1000 / fps` 的截断误差，保证每 1000 ms 内 tick 总和精确等于 1 秒。
|
||||||
|
|
||||||
|
## 4.6 离线资源转换工具链
|
||||||
|
|
||||||
|
### 4.6.1 Tom Atlas 自动生成流程
|
||||||
|
|
||||||
|
Tom 游戏的全部精灵被打包到一个 atlas 中，由 `SpriteAssetTool` 在构建主机上离线生成：
|
||||||
|
|
||||||
|
- 源文件位置：`src/Apps/Game/assets/raw/`
|
||||||
|
- 生成文件：`src/Apps/Game/generated/tom_atlas.h`
|
||||||
|
- 运行方式：`cmake --build build-win --config Release --target GenerateTomAtlasHeader`
|
||||||
|
|
||||||
|
生成的头文件包含 `tom_atlas_pixels` 像素数组以及每张精灵的 `RenderData::Sprite` 描述（`background`、`tom_stand`、`tom_listhen`、`tom_say1~4`、`tom_jump1~6`、`ui_tom`、`ui_i` 等）。
|
||||||
|
|
||||||
|
### 4.6.2 BitmapFont 生成
|
||||||
|
|
||||||
|
像素字体通过 `tools/gen_font_atlas.py` 从 TTF 生成 1-bit mask 位图字体，输出 `assets/font/font_atlas.h`。运行时 `DrawContext::draw_text` 直接将 mask 中 bit=1 的像素写成指定颜色，不保留灰度或 alpha 混合。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 5 系统测试与结果分析
|
||||||
|
|
||||||
|
## 5.1 测试环境
|
||||||
|
|
||||||
|
| 项目 | PC 端 | 板端 |
|
||||||
|
|---|---|---|
|
||||||
|
| CPU | x86_64 / AMD Ryzen / Intel | NXP i.MX6ULL Cortex-A7 |
|
||||||
|
| OS | Windows 11 / Ubuntu 22.04 | Linux 4.x / 5.x |
|
||||||
|
| 显示 | SDL2 窗口 | /dev/fb0 RGB565 / SDL2 |
|
||||||
|
| 音频 | SDL2 Audio / PulseAudio | ALSA + WM8960 |
|
||||||
|
| 输入 | 键盘 / 鼠标 | 触摸屏 / 用户按键 |
|
||||||
|
| 编译器 | MSVC 2022 / GCC 11 | arm-linux-gnueabihf-gcc |
|
||||||
|
|
||||||
|
## 5.2 跨平台编译测试
|
||||||
|
|
||||||
|
分别在以下三种配置下完成编译：
|
||||||
|
|
||||||
|
1. Windows x86_64 + MSVC + SDL2：`cmake -B build-win .` → 成功。
|
||||||
|
2. Linux x86_64 + GCC + SDL2：`cmake -B build-linux .` → 成功。
|
||||||
|
3. ARM Linux + GCC + Framebuffer：`cmake -B build-arm-fb -DCMAKE_TOOLCHAIN_FILE=... -DTARGET_IMX=ON` → 成功。
|
||||||
|
|
||||||
|
所有配置均未出现编译错误，验证了 CMake 条件编译与 `DefaultHardware` 抽象的正确性。
|
||||||
|
|
||||||
|
## 5.3 功能测试
|
||||||
|
|
||||||
|
### 5.3.1 显示后端切换测试
|
||||||
|
|
||||||
|
同一套 TomGame 代码在 PC SDL2 窗口与板端 `/dev/fb0` 上均正常显示背景、Tom 角色、HUD 按钮与设置面板，画面内容一致。
|
||||||
|
|
||||||
|
### 5.3.2 精灵动画与 UI 交互测试
|
||||||
|
|
||||||
|
- Idle 状态显示站立动画。
|
||||||
|
- 按下录音键后切换为倾听表情。
|
||||||
|
- 变调回放期间循环播放说话动画。
|
||||||
|
- 关键词识别成功后播放跳跃动画。
|
||||||
|
- 设置按钮可正常打开/关闭面板，滑块可拖动并保存配置。
|
||||||
|
|
||||||
|
### 5.3.3 录音变声回放测试
|
||||||
|
|
||||||
|
- 录音启动延迟 < 100 ms。
|
||||||
|
- 最大录音时长 5 s，静音自动结束。
|
||||||
|
- 回放音调明显高于原声，且无明显爆音。
|
||||||
|
- 录音增益与播放音量设置实时生效。
|
||||||
|
|
||||||
|
### 5.3.4 关键词识别功能测试
|
||||||
|
|
||||||
|
- 对“Up / On / Stop”等训练关键词，PC 端识别准确率 > 85%（安静环境、阈值 0.75）。
|
||||||
|
- 识别成功后 Tom 执行跳跃或停止动作。
|
||||||
|
- 阈值可通过命令行 `--kws-threshold` 调节，增益可通过 `--kws-input-gain` 或设置面板调节。
|
||||||
|
|
||||||
|
## 5.4 性能测试
|
||||||
|
|
||||||
|
### 5.4.1 PC 端帧率与绘制耗时
|
||||||
|
|
||||||
|
PC Release 构建下，TomGame 稳定运行在 60 FPS，`update + draw` 总耗时约 1~2 ms，大部分时间消耗在 `SleepRemainingFrameTime` 等待。
|
||||||
|
|
||||||
|
### 5.4.2 IMX6ULL 端 framebuffer 提交性能
|
||||||
|
|
||||||
|
板端 Release 构建、Framebuffer 后端、1024×600 分辨率下：
|
||||||
|
|
||||||
|
- 整帧时间约 33 ms，满足 30 FPS 目标。
|
||||||
|
- `present()` 耗时约 3~6 ms（RGB565 直接行拷贝路径）。
|
||||||
|
- TinyKWS 单次推理耗时约 20~40 ms，发生在录音结束后的空闲帧，不影响画面帧率。
|
||||||
|
|
||||||
|
> 注：Debug 构建下同一轻量 2D 场景曾出现 `Frame: 81 ms / Present: 69 ms` 的劣质表现，切换到 Release 后帧率提升一个数量级，说明 ARM 端性能测试必须基于 Release 构建。
|
||||||
|
|
||||||
|
### 5.4.3 Release 与 Debug 构建对比
|
||||||
|
|
||||||
|
| 构建类型 | PC 帧率 | 板端帧率 | 说明 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| Debug | 60 FPS | ~12 FPS | 含调试 UI、未优化代码 |
|
||||||
|
| Release | 60 FPS | 30 FPS | 优化后满足目标 |
|
||||||
|
|
||||||
|
## 5.5 测试结果分析
|
||||||
|
|
||||||
|
- TomGame 的交互流程在 PC 与板端表现一致，验证了分层架构与平台抽象的有效性。
|
||||||
|
- 语音变调回放链路稳定，端侧关键词识别能够在录音结束后快速给出结果。
|
||||||
|
- 板端性能受 Release 构建影响显著，后续部署与测试必须强制使用 Release。
|
||||||
|
- TinyKWS 推理耗时对 30 FPS 画面不构成瓶颈，因为推理发生在非每帧路径上。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 6 总结与展望
|
||||||
|
|
||||||
|
## 6.1 工作总结
|
||||||
|
|
||||||
|
本次实训完成了 TomGame 这一面向 IMX6ULL 的语音交互小游戏，主要工作包括：
|
||||||
|
|
||||||
|
1. **应用层交互设计**：实现了 Idle / Recording / Speaking / Jumping 四级状态机，以及变调学说话、关键词识别两种语音玩法。
|
||||||
|
2. **音频链路**：完成了录音、静音检测、变调、增益、回放全链路的跨平台实现，支持 SDL2 与 ALSA 双后端。
|
||||||
|
3. **端侧关键词识别**：将 TinyKWS 模型（MFCC + 轻量 CNN）以纯 C++ 方式移植到 IMX6ULL，模型权重嵌入可执行文件，无需外部模型加载。
|
||||||
|
4. **命令路由与游戏反馈**：通过 `KeywordCommandRouter` 将“Up / On / Stop”等关键词映射为跳跃/停止动作。
|
||||||
|
5. **跨平台部署**：基于 `Core / Platform` 分层与 CMake 条件编译，实现了 PC 与 ARM 板端共用同一套应用代码。
|
||||||
|
|
||||||
|
通过本次实训，作者加深了对嵌入式 Linux 音频采集、端侧神经网络推理、无 GPU 平台 2D 渲染以及 C++11 跨平台工程实践的理解。
|
||||||
|
|
||||||
|
## 6.2 存在问题与改进方向
|
||||||
|
|
||||||
|
1. **关键词类别有限**：当前仅将少量关键词映射为跳跃/停止，后续可扩展方向指令（Left / Right / Go）与菜单确认/取消语义。
|
||||||
|
2. **模型量化可进一步优化**：当前部分权重仍使用 `float`，全 `int8` 量化或对称量化可减少推理耗时与内存占用。
|
||||||
|
3. **语音激活检测（VAD）较简单**：当前基于能量阈值做静音裁剪，复杂噪声环境下鲁棒性不足，可引入更稳健的端点检测。
|
||||||
|
4. **设置面板视觉表现**：RGB565 帧缓冲不支持 alpha 混合，半透明遮罩效果受限，后续可增加 dither 或整数混合路径。
|
||||||
|
5. **底层渲染性能优化**：虽然当前 30 FPS 已达标，但仍可探索 tile/sprite 不透明行拷贝、dirty rect 局部刷新等优化。
|
||||||
|
6. **Launcher 与多应用切换**：当前 TomGame 是独立可执行文件，后续可按照 `docs/APP_AND_CORE_ARCHITECTURE.md` 规划，实现 Launcher 单进程多应用切换。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 致谢
|
||||||
|
|
||||||
|
感谢指导老师在实训期间的悉心指导，感谢同组成员在底层渲染与平台适配方面的协作与支持。本次实训不仅锻炼了嵌入式软件开发能力，也加深了对软硬件协同设计的理解。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 参考文献
|
||||||
|
|
||||||
|
[1] Freescale Semiconductor. *i.MX 6UltraLite Applications Processor Reference Manual*. Rev. 2, 2016.
|
||||||
|
[2] NXP Semiconductor. *I.MX 6ULL Applications Processor Reference Manual*.
|
||||||
|
[3] Linux Kernel Documentation. *fb/api.txt — The Linux Frame Buffer Device API*. https://www.kernel.org/doc/Documentation/fb/api.txt (accessed 2026-07).
|
||||||
|
[4] Linux Kernel Documentation. *alsa-project.org — Advanced Linux Sound Architecture*. https://www.alsa-project.org (accessed 2026-07).
|
||||||
|
[5] SDL Community. *Simple DirectMedia Layer 2.0 — Documentation Wiki*. https://wiki.libsdl.org/SDL2/ (accessed 2026-07).
|
||||||
|
[6] Pete Warden. *Speech Commands: A Dataset for Limited-Vocabulary Speech Recognition*. arXiv:1804.03209, 2018.
|
||||||
|
[7] TensorFlow Lite Micro. *MicroNet Keyword Spotting Model*. https://github.com/tensorflow/tflite-micro (accessed 2026-07).
|
||||||
|
[8] Edge Impulse. *Keyword Spotting with Deep Learning on Embedded Devices*. https://docs.edgeimpulse.com (accessed 2026-07).
|
||||||
|
[9] 正点原子. *I.MX6U 嵌入式 Linux 驱动开发指南*. 2020.
|
||||||
|
[10] 唐佐林. *现代 C++ 嵌入式实战*. 电子工业出版社, 2019.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 附录
|
||||||
|
|
||||||
|
### 附录 A 关键源代码清单
|
||||||
|
|
||||||
|
| 路径 | 说明 |
|
||||||
|
|---|---|
|
||||||
|
| `src/Apps/Game/Main.cpp` | 程序入口、主循环、命令行参数解析 |
|
||||||
|
| `src/Apps/Game/src/app/TomGameApp.cpp/h` | 游戏状态机、输入处理、设置管理 |
|
||||||
|
| `src/Apps/Game/src/gameplay/TomAnimator.cpp/h` | Tom 角色动画帧选择与绘制 |
|
||||||
|
| `src/Apps/Game/src/gameplay/TomHud.cpp/h` | HUD 按钮 |
|
||||||
|
| `src/Apps/Game/src/ui/TomSettingsPanel.cpp/h` | 设置面板与滑块 |
|
||||||
|
| `src/Apps/Game/src/audio/VoiceRecorder.cpp/h` | 录音缓冲与静音检测 |
|
||||||
|
| `src/Apps/Game/src/audio/VoicePlayer.cpp/h` | 播放缓冲与进度 |
|
||||||
|
| `src/Apps/Game/src/audio/VoiceEffect.cpp/h` | 变调、增益、重采样、静音裁剪 |
|
||||||
|
| `src/Apps/Game/src/gameplay/VoiceInteractionController.cpp/h` | 语音交互流程调度 |
|
||||||
|
| `src/Apps/Game/src/recognition/KeywordRecognizer.cpp/h` | 识别接口与命令枚举 |
|
||||||
|
| `src/Apps/Game/src/recognition/TinyKwsRecognizer.cpp/h` | TinyKWS 端侧推理实现 |
|
||||||
|
| `src/Apps/Game/src/recognition/TinyKwsModelData.cpp/h` | 模型权重数据 |
|
||||||
|
| `src/Apps/Game/src/gameplay/KeywordCommandRouter.cpp/h` | 关键词到游戏动作映射 |
|
||||||
|
| `src/Core/Draw2D/DrawContext.cpp/h` | 统一绘制入口 |
|
||||||
|
| `src/Core/Core/FrameBuffer.cpp/h` | RGB565 帧缓冲 |
|
||||||
|
| `src/Core/Platform/IDisplay.h` | 显示后端接口 |
|
||||||
|
| `src/Core/Platform/IAudioInput.h / IAudioOutput.h` | 音频接口 |
|
||||||
|
| `src/Core/Platform/Timer.h / TimeSource.h` | 固定步长计时 |
|
||||||
|
| `CMakeLists.txt` | 顶层构建配置 |
|
||||||
|
|
||||||
|
### 附录 B 系统运行截图
|
||||||
|
|
||||||
|
[图 B-1：TomGame PC 端 Idle 状态运行截图]
|
||||||
|
[图 B-2：TomGame 录音中（tom_listhen）状态截图]
|
||||||
|
[图 B-3：TomGame 关键词识别成功后跳跃状态截图]
|
||||||
|
[图 B-4：TomGame 设置面板截图]
|
||||||
|
[图 B-5：IMX6ULL 板端运行实拍图]
|
||||||
|
|
||||||
|
### 附录 C 硬件连接示意图
|
||||||
|
|
||||||
|
[图 C-1：IMX6ULL 教学板硬件连接示意图]
|
||||||
|
[图 C-2：LCD、触摸屏、麦克风、扬声器接线示意]
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
# TomGame 实训报告正文草稿 —— 绪论背景（仿写范文）
|
||||||
|
|
||||||
|
## 1.1 课题背景与研究意义
|
||||||
|
|
||||||
|
随着物联网、人工智能以及嵌入式等相关技术应用的不断普及，智能终端的交互方式正从传统的按键、触屏向更加自然、便捷的语音交互演进。在资源受限的嵌入式平台上，将语音技术与图形渲染相结合，能够为用户带来全新的互动娱乐体验。以智能语音为核心的交互终端，不仅可以通过语音识别、触摸点击等方式响应用户操作，还能通过生动的角色动画、即时音效反馈实现情感化交互，使嵌入式游戏产品更加智能化、人性化和趣味化。
|
||||||
|
|
||||||
|
移动互联网与边缘计算的快速发展为嵌入式娱乐市场提供了新的思路和方向，开启了“智能终端 + 语音交互”的娱乐新时代。随着人们对碎片化娱乐和个性化交互体验需求的不断提升，嵌入式语音交互游戏应运而生。通过对用户多元化娱乐需求的深入挖掘，利用嵌入式语音处理、纯 CPU 软渲染与离线资源转换等技术，能够使嵌入式游戏产品在无 GPU、低算力的硬件平台上流畅运行，满足用户对趣味性和情感陪伴的需求，拓展智能终端在家庭、教育、陪伴等场景中的应用价值。
|
||||||
@@ -0,0 +1,125 @@
|
|||||||
|
# IMX6U-Game / TomGame 实训报告 —— 目录结构与摘要（初稿）
|
||||||
|
|
||||||
|
> 本文档依据往届《基于 STM32 与 IMX6ULL 的智能家居系统》实训报告的结构，结合本项目的实际技术内容，拟写一份适用于 TomGame 项目的报告目录结构与完整摘要。后续撰写正文时可直接套用该框架。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 一、报告题目建议
|
||||||
|
|
||||||
|
**中文题目：** 基于 IMX6ULL 的纯 CPU 渲染语音交互游戏设计与实现 —— TomGame 会说话的小猫
|
||||||
|
|
||||||
|
**英文题目：** Design and Implementation of a Pure-CPU Rendered Voice-Interactive Game on IMX6ULL —— TomGame
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 二、目录结构
|
||||||
|
|
||||||
|
```
|
||||||
|
基于 IMX6ULL 的纯 CPU 渲染语音交互游戏设计与实现
|
||||||
|
—— TomGame 会说话的小猫
|
||||||
|
|
||||||
|
摘要
|
||||||
|
ABSTRACT
|
||||||
|
|
||||||
|
目录
|
||||||
|
|
||||||
|
1 绪论
|
||||||
|
1.1 课题背景与研究意义
|
||||||
|
1.1.1 嵌入式游戏与语音交互的发展背景
|
||||||
|
1.1.2 在资源受限平台运行图形游戏的必要性
|
||||||
|
1.2 国内外研究现状
|
||||||
|
1.2.1 嵌入式图形渲染技术研究现状
|
||||||
|
1.2.2 端侧语音关键词识别技术研究现状
|
||||||
|
1.3 本文主要研究内容
|
||||||
|
|
||||||
|
2 系统总体方案与相关技术
|
||||||
|
2.1 系统需求分析
|
||||||
|
2.1.1 功能需求
|
||||||
|
2.1.2 性能需求
|
||||||
|
2.1.3 跨平台开发与部署需求
|
||||||
|
2.2 系统总体架构设计
|
||||||
|
2.2.1 Core / Apps 分层架构
|
||||||
|
2.2.2 平台抽象层设计
|
||||||
|
2.2.3 资源离线转换流程
|
||||||
|
2.3 关键开发平台与技术
|
||||||
|
2.3.1 IMX6ULL 开发板硬件平台
|
||||||
|
2.3.2 ARM Linux 与交叉编译环境
|
||||||
|
2.3.3 SDL2 / Framebuffer 双显示后端
|
||||||
|
2.3.4 C++11 与 CMake 构建系统
|
||||||
|
2.3.5 TinyKWS 关键词识别模型
|
||||||
|
2.3.6 音频采集与回放技术
|
||||||
|
|
||||||
|
3 硬件平台与运行环境设计
|
||||||
|
3.1 IMX6ULL 核心板硬件资源
|
||||||
|
3.1.1 CPU、内存与存储
|
||||||
|
3.1.2 LCD 显示接口与 framebuffer
|
||||||
|
3.1.3 音频输入输出接口
|
||||||
|
3.1.4 触摸与按键输入
|
||||||
|
3.2 开发环境搭建
|
||||||
|
3.2.1 Windows / Linux 主机开发环境
|
||||||
|
3.2.2 ARM 交叉编译工具链
|
||||||
|
3.2.3 板端运行环境配置
|
||||||
|
3.3 跨平台硬件抽象策略
|
||||||
|
|
||||||
|
4 系统软件设计
|
||||||
|
4.1 软件总体架构
|
||||||
|
4.1.1 模块划分与依赖关系
|
||||||
|
4.1.2 主循环与时间源设计
|
||||||
|
4.1.3 绘制调用链
|
||||||
|
4.2 底层图形引擎 Core 设计
|
||||||
|
4.2.1 FrameBuffer 与 DepthBuffer
|
||||||
|
4.2.2 颜色、图像与精灵数据结构
|
||||||
|
4.2.3 DrawContext 统一绘制接口
|
||||||
|
4.2.4 线段与三角形光栅化
|
||||||
|
4.2.5 精灵、Tilemap 与 BitmapFont 绘制
|
||||||
|
4.2.6 数学库：Vector / Matrix
|
||||||
|
4.3 平台适配层 Platform 设计
|
||||||
|
4.3.1 IDisplay 与 SDLDisplay / FBDisplay
|
||||||
|
4.3.2 IAudioInput / IAudioOutput 与 ALSA / SDL 后端
|
||||||
|
4.3.3 IButtonInput / IPointerInput 输入抽象
|
||||||
|
4.3.4 ITimeSource 独立时间源
|
||||||
|
4.4 离线资源转换工具链
|
||||||
|
4.4.1 PNG → RGBA5551 头文件
|
||||||
|
4.4.2 像素字体 → BitmapFont 头文件
|
||||||
|
4.4.3 Tom Atlas 自动生成流程
|
||||||
|
4.5 TomGame 应用层设计
|
||||||
|
4.5.1 TomGameApp 状态机
|
||||||
|
4.5.2 角色动画系统（TomAnimator）
|
||||||
|
4.5.3 语音交互控制器（VoiceInteractionController）
|
||||||
|
4.5.4 音频录制、变声与回放（VoiceRecorder / VoiceEffect / VoicePlayer）
|
||||||
|
4.5.5 嵌入式关键词识别（KeywordRecognizer / TinyKwsRecognizer）
|
||||||
|
4.5.6 命令路由与游戏反馈（KeywordCommandRouter）
|
||||||
|
4.5.7 HUD 与设置面板（TomHud / TomSettingsPanel）
|
||||||
|
4.6 程序入口与主循环
|
||||||
|
4.6.1 命令行参数解析
|
||||||
|
4.6.2 初始化流程
|
||||||
|
4.6.3 update / draw / present 循环
|
||||||
|
|
||||||
|
5 系统测试与结果分析
|
||||||
|
5.1 测试环境
|
||||||
|
5.2 跨平台编译测试
|
||||||
|
5.2.1 Windows / Linux x86 编译
|
||||||
|
5.2.2 ARM 交叉编译
|
||||||
|
5.3 功能测试
|
||||||
|
5.3.1 显示后端切换测试
|
||||||
|
5.3.2 精灵动画与 UI 交互测试
|
||||||
|
5.3.3 录音变声回放测试
|
||||||
|
5.3.4 关键词识别功能测试
|
||||||
|
5.4 性能测试
|
||||||
|
5.4.1 PC 端帧率与绘制耗时
|
||||||
|
5.4.2 IMX6ULL 端 framebuffer 提交性能
|
||||||
|
5.4.3 Release 与 Debug 构建对比
|
||||||
|
5.5 测试结果分析
|
||||||
|
|
||||||
|
6 总结与展望
|
||||||
|
6.1 工作总结
|
||||||
|
6.2 存在问题与改进方向
|
||||||
|
|
||||||
|
致谢
|
||||||
|
|
||||||
|
参考文献
|
||||||
|
|
||||||
|
附录
|
||||||
|
附录 A 关键源代码清单
|
||||||
|
附录 B 系统运行截图
|
||||||
|
附录 C 硬件连接示意图
|
||||||
@@ -0,0 +1,419 @@
|
|||||||
|
**基于 IMX6U 与光敏传感器的 2D 平台跳跃游戏设计**
|
||||||
|
|
||||||
|
**学生姓名：黄俊 指导老师：⟨指导老师姓名⟩**
|
||||||
|
|
||||||
|
**摘 要** 本实训针对无 GPU 或 GPU 能力极弱的嵌入式 SoC（IMX6ULL/Cortex-A7）上如何构建可玩性完整的图形化游戏这一问题，设计并实现了一个纯 CPU 软光栅化 2D 渲染框架 `Core`，以及一款以光敏传感器（AP3216C）作为核心输入的平台跳跃游戏 `LightGame`。渲染层采用统一 RGB565 帧缓冲、RGBA5551 精灵图集、1-bit 透明与整数定点亮度调制，将像素格式转换控制在显示后端边界，热路径全部使用整数运算与预分配缓冲；显示层抽象为 `Platform::IDisplay` 接口，分别在 Linux `/dev/fb0` 与 PC SDL2 后端上实现，通过 CMake 选项和条件编译一键切换。游戏层实现了亚像素积分物理、有限状态角色控制器、整数平滑跟随相机、房间网格分区与检查点重生等系统。传感器层绕开出厂 `ap3216c` 字符设备驱动，直接通过 `/dev/i2c-0` 以 `I2C_SLAVE_FORCE` 抢占地址、将 AP3216C 重配置为 ALS-only 连续采样与最大增益模式，随后在应用层引入 Q8 定点一阶指数移动平均滤波，将 12 位光照值以约 0.3 Hz 截止频率平滑后驱动关卡对象通断、地形几何显隐与 HUD 反馈，形成"环境光即输入"的独特玩法。实验结果表明，在 IMX6U 板上以 1024×600、30 FPS 稳定运行，光照变化到游戏世界响应延迟约 0.7 s，能够支撑完整的关卡切换与死亡重生流程，验证了纯 CPU 渲染 + 环境传感器输入的技术路线在教学级嵌入式设备上的可行性。
|
||||||
|
|
||||||
|
**关键词：** IMX6ULL；软光栅化；RGB565；AP3216C；I²C；定点数；平台跳跃游戏；嵌入式 Linux
|
||||||
|
|
||||||
|
**A 2D Platformer on IMX6U with a Photo-Sensor as Gameplay Input**
|
||||||
|
|
||||||
|
Student name: Huang Jun Advisor: ⟨Advisor Name⟩
|
||||||
|
|
||||||
|
**Abstract** This training addresses the problem of building a playable graphical game on an embedded SoC (IMX6ULL / Cortex-A7) that has no GPU or only extremely limited graphics acceleration. A pure-CPU software rasterisation 2D framework `Core` was designed and implemented, together with a platform-jumper game `LightGame` that uses an AP3216C ambient-light sensor as its primary gameplay input. The renderer standardises on an RGB565 framebuffer, RGBA5551 sprite atlases, 1-bit transparency and integer fixed-point brightness modulation, keeping pixel-format conversion at the display boundary and confining hot paths to integer math and pre-allocated buffers. The display layer is abstracted behind a `Platform::IDisplay` interface with Linux `/dev/fb0` and PC SDL2 back-ends, selected by a CMake option and conditional compilation. The game layer implements sub-pixel-accumulator physics, a finite-state character controller, an integer smoothing follow-camera, a room-grid partition and checkpoint respawn. The sensor layer bypasses the vendor `ap3216c` character driver, taking over the I²C address via `ioctl(I2C_SLAVE_FORCE)` on `/dev/i2c-0`, reconfigures the AP3216C to ALS-only continuous mode at maximum gain, and applies a Q8 fixed-point first-order EMA (≈0.3 Hz cut-off) on top. The smoothed 12-bit reading drives object solidity, tilemap hot-swap and HUD feedback, realising an "ambient light *is* the input" mechanic. On the IMX6U board the system runs at 1024×600, a steady 30 FPS, with an end-to-end light-to-world response of about 0.7 s, supporting a complete level-transition and death-respawn flow, and confirming that a pure-CPU renderer plus an ambient sensor is a viable technical route for educational embedded devices.
|
||||||
|
|
||||||
|
**Keywords:** IMX6ULL; Software rasterisation; RGB565; AP3216C; I²C; Fixed-point; Platformer; Embedded Linux
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**目 录**
|
||||||
|
|
||||||
|
1 专业方向综合实训概述 1
|
||||||
|
1.1 实训目的 1
|
||||||
|
1.2 实训任务 1
|
||||||
|
1.3 实训要求 1
|
||||||
|
1.4 环境及可持续性发展 2
|
||||||
|
2 平台与总体设计 3
|
||||||
|
2.1 IMX6U 平台约束 3
|
||||||
|
2.2 分层架构与依赖方向 3
|
||||||
|
2.3 关键技术选型 4
|
||||||
|
3 Core 底层渲染库设计 5
|
||||||
|
3.1 平台抽象与显示后端 5
|
||||||
|
3.2 FrameBuffer 与统一像素格式 6
|
||||||
|
3.3 DrawContext 绘制入口 7
|
||||||
|
3.4 Sprite 与 Tilemap 数据结构 7
|
||||||
|
3.5 整数定点与热路径规则 8
|
||||||
|
3.6 Timer 与固定帧率 9
|
||||||
|
4 LightGame 游戏系统实现 10
|
||||||
|
4.1 主循环与初始化 10
|
||||||
|
4.2 Physics2D：亚像素积分物理 10
|
||||||
|
4.3 PlayerController：状态机与二段可变高跳跃 11
|
||||||
|
4.4 Camera2D：整数平滑跟随 12
|
||||||
|
4.5 Level / Room：数据驱动的关卡与房间网格 12
|
||||||
|
4.6 LevelRenderer：分层合成顺序 13
|
||||||
|
4.7 GameStateManager 与 HUD 13
|
||||||
|
5 光敏传感器与游戏机制耦合 14
|
||||||
|
5.1 AP3216C 与出厂驱动的问题 14
|
||||||
|
5.2 i2c 直读方案与 I2C_SLAVE_FORCE 14
|
||||||
|
5.3 Q8 定点 EMA 滤波 15
|
||||||
|
5.4 光照 → 对象通断（LightEffectSystem） 16
|
||||||
|
5.5 光照 → 地形几何显隐 16
|
||||||
|
5.6 光照 → HUD 视觉反馈与手动调试通道 17
|
||||||
|
6 跨平台构建与调试 18
|
||||||
|
6.1 CMake 选项与条件编译 18
|
||||||
|
6.2 LevelEditor：PC + Debug 独占的关卡编辑器 18
|
||||||
|
6.3 调试流程与验证方法 19
|
||||||
|
7 功能测试与验证 20
|
||||||
|
7.1 渲染性能与帧率测试 ⟨待补⟩
|
||||||
|
7.2 传感器响应曲线测试 ⟨待补⟩
|
||||||
|
7.3 关卡完整通关测试 ⟨待补⟩
|
||||||
|
8 工作总结与展望 21
|
||||||
|
参考文献 22
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# 1. 专业方向综合实训概述
|
||||||
|
|
||||||
|
> 本节整体框架待课程正式发布"实训任务书"后按其标题与要点回填；当前保留与《MIPS 单周期处理器设计》一致的骨架。
|
||||||
|
|
||||||
|
## 1.1 实训目的
|
||||||
|
|
||||||
|
⟨待课程发布后按任务书表述回填；预期覆盖点：嵌入式 Linux 应用开发能力、外设驱动理解、软硬件协同的系统性思维、以中小型工程项目锻炼工程管理与文档能力。⟩
|
||||||
|
|
||||||
|
## 1.2 实训任务
|
||||||
|
|
||||||
|
在 IMX6ULL 平台上，独立完成一款可交互的图形化应用：包含图形渲染、外设输入、状态管理三块能力，并至少集成一路板载传感器作为输入源。
|
||||||
|
|
||||||
|
## 1.3 实训要求
|
||||||
|
|
||||||
|
⟨待课程发布后按任务书回填。当前预设要点：⟩
|
||||||
|
|
||||||
|
1. 使用 C/C++11，兼容嵌入式老工具链，代码经过 PC 与 ARM 交叉编译双路径验证；
|
||||||
|
2. 界面运行分辨率不低于 1024×600，图形帧率不低于 30 FPS；
|
||||||
|
3. 至少接入一路 I²C 或 SPI 外设并读取有效数据；
|
||||||
|
4. 项目需具备明确的分层结构与文档，能够独立部署到板端运行；
|
||||||
|
5. 提供实训报告与相关演示材料。
|
||||||
|
|
||||||
|
## 1.4 环境及可持续性发展
|
||||||
|
|
||||||
|
本次实训以 IMX6ULL 教学板 + PC 交叉开发环境为主，全过程使用软件仿真与实机验证结合。板端不需要额外硬件改造，桌面端使用 SDL2 模拟显示与输入，避免了反复烧写调试造成的电子元件损耗；同时项目采用离线资源转换（PNG → C 头文件、TTF → 位图字体 mask）方式，运行时不解码原始素材，减少 CPU 空转能耗，与嵌入式设备低功耗、长时运行的场景要求一致。项目分层清晰，`Core` 底层库可复用到后续其他嵌入式图形项目，符合可持续开发原则。
|
||||||
|
|
||||||
|
# 2. 平台与总体设计
|
||||||
|
|
||||||
|
## 2.1 IMX6U 平台约束
|
||||||
|
|
||||||
|
本项目的目标硬件为正点原子 IMX6ULL 教学开发板，其核心为 ARM Cortex-A7 单核处理器，未集成独立 GPU，显示输出通过直接向 `/dev/fb0` 写入帧缓冲完成，板载的 AP3216C 光/接近传感器则挂载在 I²C-0 总线上。与常见的移动 SoC 相比，该平台在图形计算、数值运算和输入通道三方面呈现出明显的资源约束，直接决定了后续渲染框架和游戏实现的技术路线。
|
||||||
|
|
||||||
|
在图形栈方面，IMX6ULL 不提供任何硬件加速，三角形填充、精灵位图、字体掩码等所有绘制操作都必须由 CPU 逐像素完成，因此渲染管线的热路径必须尽量降低计算开销。在数值运算方面，虽然 Cortex-A7 支持 VFPv4 浮点单元，但浮点指令的延迟和功耗均高于整数指令，在逐像素长循环中应避免使用浮点运算，以保证 30 FPS 的帧率预算。在输入通道方面，按键、触摸和传感器分别通过 evdev、tslib 和 i2c 字符设备或 sysfs 访问，内核并未提供统一的事件循环，应用层需要自行轮询各类设备状态。
|
||||||
|
|
||||||
|
上述约束共同决定了本项目的整体技术路线：核心渲染与游戏逻辑采用整数化计算，帧缓冲以单帧一次 memcpy 的方式提交，热路径避免堆分配，并将显示与输入细节完全抽象到平台层，从而使上层代码在 PC 与板端之间保持一致。
|
||||||
|
|
||||||
|
## 2.2 分层架构与依赖方向
|
||||||
|
|
||||||
|
为了保证代码在 PC 调试与 IMX6U 板端运行之间能够无缝迁移，项目采用严格的单向依赖分层，模块命名与目录结构一一对应（详见 `docs/APP_AND_CORE_ARCHITECTURE.md`），总体依赖方向为 `Apps -> Shared -> Core -> Platform`。其中，Platform 层位于最底层，负责屏蔽 SDL2、`/dev/fb0`、ALSA、evdev、I²C 等具体平台差异，向上以 `IDisplay`、`ITimeSource`、`IButtonInput`、`IKeyboardState`、`IPointerInput`、`IPhotoSensor` 等纯虚接口提供显示、时钟、按键、键盘、指针和光敏传感器能力；Core 层建立在 Platform 接口之上，包含 `FrameBuffer`、`DepthBuffer`、`DrawContext`、`Rasterizer`、`Tilemap`、`Sprite`、`BitmapFont`、`Timer` 等与具体游戏无关的运行时组件，为上层应用提供统一的 2D 渲染与定时服务；Apps 层位于最顶层，是各具体游戏的实现位置，本文的核心应用 `LightGame` 即位于 `src/Apps/LightGame/`，同项目中其他同学负责的 `Game`（Tom 游戏）与 `Demo` 也处于该层，但彼此独立、不存在直接依赖。需要说明的是，Shared 层按规划用于存放应用层共享的 UI、存档、配置等模块，当前目录结构中尚未填充具体实现，因此现有代码的实际依赖路径可视为 `Apps -> Core -> Platform`。
|
||||||
|
|
||||||
|
单向依赖通过 CMake target 与 include 路径共同约束：`Core -> Apps`、`Platform -> Apps` 以及 `GameA -> GameB` 这类反向引用都会在编译期直接暴露，从而避免平台代码或底层库被应用层细节污染。由于本文作者主要负责 `Core` 底层库与 `Apps/LightGame` 光敏平台跳跃游戏的实现，后续章节将围绕这两部分展开，仅在必要处引用另一位合作者负责的模块。
|
||||||
|
|
||||||
|
## 2.3 关键技术选型
|
||||||
|
|
||||||
|
在语言标准方面，项目选用 C++11，以兼容老旧的嵌入式交叉工具链（如 gcc-linaro-4.9.4），避免因引入 C++14/17 特性而导致板端编译失败。在帧缓冲格式方面，统一采用 RGB565，该格式与 IMX6ULL 板载 LCD 的物理显存布局一致，PC 端 SDL2 同样以 RGB565 streaming texture 接收，从而在全链路中省去像素格式转换。在精灵资源格式方面，采用 RGBA5551 的 1-bit 透明打包，足以表达像素风素材，同时把 5 位通道扩展到 RGB565 的转换成本限制在已知范围内。
|
||||||
|
|
||||||
|
在数值系统方面，核心路径统一使用整数运算，并在亮度调制和光敏滤波等场景引入 Q8/Q7 定点数，以屏蔽 VFP 浮点开销并消除浮点精度抖动。在显示后端方面，通过 `Platform::IDisplay` 抽象出 `FBDisplay` 与 `SDLDisplay` 两套实现，PC 与 ARM 共用同一份游戏和渲染代码。在输入抽象方面，按键、键盘、指针和光敏传感器分别对应 `IButtonInput`、`IKeyboardState`、`IPointerInput` 和 `IPhotoSensor` 接口，`DefaultHardware.h` 按平台做 typedef 切换，使得上层应用无需使用 `#ifdef`。在传感器读取方面，AP3216C 不依赖出厂字符设备驱动，而是直接通过 `/dev/i2c-0` 抢占地址并重配置寄存器，以获得稳定可用的 ALS 数据。
|
||||||
|
|
||||||
|
上述选型共同服务于一个目标：在嵌入式路径上保持整数化和固定像素格式，在 PC 路径上通过接口复用同一份实现，从而保证双平台下画面等价、行为一致，同时降低跨平台验证与后续维护的成本。
|
||||||
|
|
||||||
|
# 3. Core 底层渲染库设计
|
||||||
|
|
||||||
|
## 3.1 平台抽象与显示后端
|
||||||
|
|
||||||
|
`Platform::IDisplay` 是一个仅包含四个方法的纯虚接口：`init(width, height)`、`present(const Core::FrameBuffer*)`、`poll_events(bool& should_quit)` 与 `shutdown()`。它的核心思想是把“如何把像素缓冲送到屏幕”这一平台相关细节完全隔离到平台层，游戏逻辑只关心一块 RGB565 帧缓冲的内容。
|
||||||
|
|
||||||
|
### 3.1.1 FBDisplay（IMX6U 板端）
|
||||||
|
|
||||||
|
在 IMX6ULL 板端，`FBDisplay` 打开 `/dev/fb0`，通过 `ioctl` 读取 `FBIOGET_FSCREENINFO` 与 `FBIOGET_VSCREENINFO` 获取像素格式和行步长，随后用 `mmap` 将整块显存映射到用户空间指针 `fb_mem`。`present()` 根据 `vinfo` 中的 R/G/B 位掩码分三条路径提交：若面板本身就是 RGB565 且 `line_length == width * 2`，则一次性 `memcpy` 完成提交；若行步长不等，则改为逐行 `memcpy`。对于 ARGB8888 或 RGBA8888 面板，则对每个像素做 5→8 位扩展并重新排列字段。其余配置统一走 `convert_pixel()`，根据运行时读取的位掩码构造输出像素，保证对未知 fb 格式的兼容性。
|
||||||
|
|
||||||
|
板端没有 SDL 事件循环，`poll_events` 使用 `select(STDIN_FILENO, timeout=0)` 非阻塞检测 `q` 或 `Q` 键，使主循环不会被输入阻塞。`shutdown` 在 `munmap` 前先用 `memset` 清屏，避免程序退出后屏幕残留最后一帧画面。
|
||||||
|
|
||||||
|
### 3.1.2 SDLDisplay（PC 端）
|
||||||
|
|
||||||
|
PC 端 `SDLDisplay` 创建 `SDL_Window`、`SDL_Renderer(SDL_RENDERER_ACCELERATED)` 以及格式为 `SDL_PIXELFORMAT_RGB565`、访问模式为 `STREAMING` 的 `SDL_Texture`，与核心 `FrameBuffer` 的像素格式严格一一对应。`present()` 只需一次 `SDL_UpdateTexture`、`SDL_RenderCopy` 与 `SDL_RenderPresent`。这种设计让 PC 侧无需为像素格式转换编写第二条路径，确保 PC 与 ARM 的可见画面完全等价。
|
||||||
|
|
||||||
|
### 3.1.3 时间源与输入
|
||||||
|
|
||||||
|
`Platform::ITimeSource` 独立于显示层，提供单调递增的整数毫秒时钟：Windows 实现基于 `std::chrono::steady_clock`，Linux 实现基于 `clock_gettime(CLOCK_MONOTONIC)`，返回值均为 `uint32_t`，49 天回绕在一次游戏运行中可以忽略。按键、键盘与指针输入也采用同样模式——接口定义位于 `Core/Platform/`，ARM 端由 Evdev 系列实现，PC 端由 Sdl 系列实现，`DefaultHardware.h` 通过 typedef 决定调用点看到的具体类型。这种方式使得 `LightGameApp` 中不存在任何用于选择平台输入实现的 `#ifdef TARGET_IMX`。
|
||||||
|
|
||||||
|
## 3.2 FrameBuffer 与统一像素格式
|
||||||
|
|
||||||
|
`Core::FrameBuffer` 在构造时一次性分配大小为 `width * height` 的 `std::vector<uint16_t>` 作为 RGB565 像素缓冲，整个运行期间不再进行堆扩展。为了服务热路径绘制，它对外提供 `set_pixel_unsafe(x, y, rgba32)` 接口，调用方需自行保证坐标落在合法范围内，函数内部仅执行一次数组写入。颜色转换由头文件中的内联函数 `rgba_to_frame_pixel` 完成，其表达式仅包含 R 右移 3 位、G 右移 2 位、B 右移 3 位后的位或，编译器能够将这一短转换折叠进精灵和瓦片的内层循环。`get_buffer()` 返回缓冲区的原始指针，供 `FBDisplay` 或 `SDLDisplay` 直接 `memcpy` 或 `SDL_UpdateTexture` 使用，避免额外的数据拷贝。
|
||||||
|
|
||||||
|
与 `FrameBuffer` 配套的还有 `Core::DepthBuffer`，它同样使用 `uint16_t` 存储每个像素的深度值，清空时写入 `0xFFFF` 表示最远，语义为“值越小越近”。该缓冲主要服务于三角形光栅化路径，虽然 `LightGame` 作为纯 2D 游戏并不使用其内容，但 `DrawContext` 仍统一持有它，以保持渲染 API 的完整性和未来向 3D 扩展的可能性。
|
||||||
|
|
||||||
|
## 3.3 DrawContext 绘制入口
|
||||||
|
|
||||||
|
`Core::DrawContext` 是整个渲染管线的门面对象，构造时创建 `FrameBuffer`、`DepthBuffer`、`Rasterizer` 与 `TriangleRasterizer` 各一份，禁止拷贝和移动，所有动态分配集中在构造与析构阶段完成，运行期不再出现 new/delete。游戏层通过它访问的接口包括清屏、绘制线段与三角形、精灵与瓦片绘制、填充矩形与位图文本，以及最终的 `present(IDisplay*)` 帧提交。
|
||||||
|
|
||||||
|
在精灵绘制热路径上，源图集以 RGBA5551 格式存放，`blit_sprite_pixels` 在外层根据目标矩形完成一次屏幕裁剪，确定 `start_dx / dy` 与 `end_dx / dy` 区间；内层则是一个紧凑的 `for(sy) for(sx)` 双循环，行指针 `dst_row` 在外层预先算出，水平/垂直翻转通过对读索引的三元表达式实现，编译器可将其优化为条件传送，避免在内层引入分支。此外，`scale == 1` 与 `scale > 1` 两条路径分开实现，前者内层不含除法。1-bit 透明通过 `rgba5551_is_opaque` 的位测试跳过，路径上没有任何 alpha 混合或乘法，最短流程每像素仅一次读取、一次测试和一次写入。
|
||||||
|
|
||||||
|
瓦片绘制同样遵循整数化原则。`draw_tilemap_shaded` 以 `shade_numerator / (1 << shade_shift)` 的整数比例对每个颜色通道做定点缩放，具体实现细节见 §3.5。
|
||||||
|
|
||||||
|
## 3.4 Sprite 与 Tilemap 数据结构
|
||||||
|
|
||||||
|
`RenderData::Sprite` 被设计为一个轻量视图，仅保存指向源图集的 `const Image* atlas` 以及一个 `(x, y, w, h)` 子矩形，不持有像素数据本身。`Image` 则保存 `const void* pixels` 指针与宽高信息，像素默认按 RGBA5551 格式解释。所有精灵源图集在离线阶段由 `tools/png_to_header.py` 转换为 C 头文件中的 RGBA5551 `uint16_t` 数组，运行时不解码 PNG，也不进行格式转换，从而消除了运行时解析开销。
|
||||||
|
|
||||||
|
`RenderData::Tilemap` 用 `uint16_t tile_id` 表示地图网格，`0xFFFF` 作为空瓦片哨兵；当前实现支持单一 atlas 与固定 tile 尺寸，结构内同时保存地图宽高、tile 宽高和 atlas 列数。`draw_tilemap` 的裁剪分两层执行：首先在瓦片层根据相机位置计算 `start_tile_x = camera_x / tile_w`，并向左右、上下各多取一格，以覆盖像素级滚动时露出的一半瓦片；然后在像素层对每个瓦片按视口子矩形做进一步裁剪，只绘制实际可见部分。`DrawContext` 还提供了带 `viewport_w / viewport_h` 的重载，允许把地图渲染限制在屏幕上的任意子矩形内，`LightGame` 利用这一能力在屏幕顶部留出 24 px 的 HUD 黑边。
|
||||||
|
|
||||||
|
## 3.5 整数定点与热路径规则
|
||||||
|
|
||||||
|
为了在 IMX6ULL 上稳定达到 30 FPS，项目对核心渲染与游戏逻辑制定了严格的整数化约束：除导入导出、调试打印和上层表示等非热路径外，核心代码禁止使用浮点类型。这一约束贯穿于绘制、物理、光照滤波等各个环节。
|
||||||
|
|
||||||
|
在亮度调制场景中，`draw_tilemap_shaded` 接收 `(numerator, shift)` 参数，对每个颜色通道计算 `(component * numerator) >> shift`，其中绿通道还需做 5→6 位扩展 `(g << 1) | (g >> 4)`，以高位补齐低位的方式保持亮度分布均匀。整个乘法链仅包含整数乘法和右移，没有浮点参与。光敏值的平滑滤波采用 Q8 定点实现，HUD 光条颜色渐变同样使用纯整数运算，具体内容分别见 §5.3 和 §4.7。物理系统则以 `int32 px/s` 存储速度，并通过亚像素累加器完成位移积分，从而避免浮点累积误差。
|
||||||
|
|
||||||
|
## 3.6 Timer 与固定帧率
|
||||||
|
|
||||||
|
`Core::Timer` 负责控制主循环节奏，只接受 30、45、60 FPS 三档目标帧率，其它值会回退到 30。为了避免整数除法 `1000 / 30 = 33` 带来的截断误差累积，它采用余数累积法计算每帧的固定时间片：
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
tick_remainder_ += 1000;
|
||||||
|
fixed_delta_ms_ = tick_remainder_ / fps;
|
||||||
|
tick_remainder_ %= fps;
|
||||||
|
```
|
||||||
|
|
||||||
|
在 30 FPS 下，上述代码会依次产生 33、33、34、33、33、34……毫秒的 tick，每 30 帧累加恰好为 1000 ms，与真实时间保持零漂移。`remaining_frame_ms(now_ms)` 则给出当前帧剩余的睡眠预算，主循环通过一次 `std::this_thread::sleep_for` 消化这部分时间，从而把帧率稳定在目标值附近。
|
||||||
|
|
||||||
|
# 4. LightGame 游戏系统实现
|
||||||
|
|
||||||
|
## 4.1 主循环与初始化
|
||||||
|
|
||||||
|
`src/Apps/LightGame/src/main.cpp` 中的主循环遵循“初始化 → 固定时间片更新 → 绘制 → 帧提交 → 睡眠等待”的朴素结构。初始化阶段首先根据 `TARGET_IMX` 或 `TARGET_PC` 宏创建对应的 `Platform::IDisplay` 实现，随后实例化 `DefaultButtonInput`、`DefaultKeyboardState`、`DefaultPointerInput` 与 `DefaultPhotoSensor`，这些具体类型由 `DefaultHardware.h` 按平台 typedef 决定。`Core::DrawContext` 和 `Core::Timer` 随后以 1024×600 分辨率和 30 FPS 目标创建，`LightGameApp` 拿到输入、显示和时间源的指针后进入主循环。
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
Platform::IDisplay* display = CreateDisplay(); // 后端由 TARGET_IMX/PC 决定
|
||||||
|
display->init(ScreenWidth, ScreenHeight);
|
||||||
|
Platform::DefaultButtonInput buttonInput;
|
||||||
|
Platform::DefaultKeyboardState keyboardState;
|
||||||
|
Platform::DefaultPointerInput pointerInput;
|
||||||
|
Platform::DefaultPhotoSensor photoSensor;
|
||||||
|
// ... 初始化每个输入设备
|
||||||
|
Core::DrawContext ctx(ScreenWidth, ScreenHeight);
|
||||||
|
Core::Timer timer(30);
|
||||||
|
Platform::SteadyTimeSource time_source;
|
||||||
|
LightGame::LightGameApp app(...);
|
||||||
|
|
||||||
|
while (!should_quit) {
|
||||||
|
timer.begin_frame(time_source.get_time_ms());
|
||||||
|
display->poll_events(should_quit);
|
||||||
|
buttonInput.update();
|
||||||
|
pointerInput.update();
|
||||||
|
app.update(timer.fixed_delta_ms());
|
||||||
|
app.draw(ctx);
|
||||||
|
// 帧提交：IMX 走 ctx.present(display)，PC 直接 SDL_UpdateTexture
|
||||||
|
SleepRemainingFrameTime(timer, time_source);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`app.draw` 完成所有游戏对象的绘制，`ctx.present(display)` 在板端通过 `FBDisplay` 一次性提交到 `/dev/fb0`。PC 端的主循环则直接使用 `SDL_UpdateTexture` 与 `SDL_RenderCopy` 上传帧缓冲，以便与 SDL 渲染器和调试用的 ImGui 叠加；这是为 PC 调试生态刻意保留的分层例外，板端代码路径仍然严格遵守 `IDisplay` 抽象。`SleepRemainingFrameTime` 在每帧末尾消耗由 `timer.remaining_frame_ms` 计算出的剩余时间，从而把实际帧率稳定在 30 FPS 附近。
|
||||||
|
|
||||||
|
## 4.2 Physics2D：亚像素积分物理
|
||||||
|
|
||||||
|
`Physics2D` 是 LightGame 的物理层，默认参数为重力 `800 px/s²`、最大下落速度 `600 px/s`、地面摩擦力 `900 px/s²`。所有速度均以 `int32 px/s` 为单位存储，位移积分通过亚像素累加器完成。`Physics2D` 内部维护两个 `int32_t` 余数 `sub_pixel_x_` 与 `sub_pixel_y_`，每帧先把 `velocity * dt_ms` 累加到余数上，再除以 1000 得到整像素位移并保留余数：
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
sub_pixel_x_ += velocity.x * dt_ms; // 单位：px·ms/s
|
||||||
|
int32_t dx = sub_pixel_x_ / 1000;
|
||||||
|
sub_pixel_x_ -= dx * 1000; // 保留亚像素余数
|
||||||
|
```
|
||||||
|
|
||||||
|
以 50 px/s 的速度、33 ms 的帧时间为例，一帧的理论位移仅为 1.65 像素，若没有亚像素累加器，整除截断会导致物体静止或出现帧率相关的抖动；引入累加器后，这些微小位移会在多帧间累积并正确兑现。
|
||||||
|
|
||||||
|
碰撞解算采用最小分离轴策略。对于 Tilemap 碰撞，`Physics2D` 先根据 `world_collider` 算出覆盖的 tile 范围，再遍历每个 solid tile，分别计算左右上下四个方向的穿透深度，选择绝对值最小的方向将物体推出，同时清零该轴的速度和亚像素余数，防止冲量遗留。动态实体之间的碰撞同样使用四方向最小分离，但仅清零速度而保留亚像素余数，这样当多个移动平台轻微重叠时，余数可以在下一帧帮助抵消抖动。若对象掉出地图底端 128 像素以外的安全裕量，则会被标记为 `active = false`，避免进入无限下坠的死循环。
|
||||||
|
|
||||||
|
## 4.3 PlayerController：状态机与二段可变高跳跃
|
||||||
|
|
||||||
|
`PlayerController` 的运动参数默认值如表所示。
|
||||||
|
|
||||||
|
| 项 | 值 | 说明 |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| move_speed | 200 px/s | 水平最大速度 |
|
||||||
|
| acceleration | 1200 px/s² | 起步加速度 |
|
||||||
|
| deceleration | 1600 px/s² | 松开方向后的减速 |
|
||||||
|
| jump_velocity | −420 px/s | 一次跳跃的初速 |
|
||||||
|
| jump_cut_multiplier | 40 % | 松开跳跃时上升速度按 40% 截断 |
|
||||||
|
|
||||||
|
角色状态机在 `Idle`、`Running`、`Jumping`、`Falling`、`Dead` 五个状态之间转移，每帧根据 `grounded`、`move_dir` 和 `velocity.y` 的组合确定下一状态，没有隐藏状态。输入处理同时支持键盘与触摸屏：键盘使用 `←/→` 控制水平移动、`↑` 控制跳跃；触摸屏则将屏幕上半区映射为跳跃，左三分之一和右三分之一分别映射为向左和向右移动。当两者同时存在时键盘优先覆盖触摸，因此同一份 `PlayerController` 既能在 PC 上调试，也能在 IMX6U 触摸屏上直接运行，无需平台分支。
|
||||||
|
|
||||||
|
可变高跳跃通过检测跳跃键的按下沿和释放沿实现。起跳时若按键按住，角色获得 `−420 px/s` 的初速度；在上升阶段（`velocity.y < 0`）一旦检测到按键释放，立即将竖直速度按 `jump_cut_multiplier / 100` 截断为原来的 40%。轻按即小跳、长按即高跳，这是平台跳跃类游戏的常见手感设计。
|
||||||
|
|
||||||
|
死亡判定分为两种情况。第一种是角色位置越过关卡底部边界，立即进入 `Dead` 状态。第二种是碰到尖刺 tile，此时只有碰撞体进入 tile 顶部向下 14 px 的区域内才会判定死亡，边缘擦过不会触发。这一 14 px 的杀伤区在源码中以注释明确说明，是刻意保留的手感缓冲。
|
||||||
|
|
||||||
|
## 4.4 Camera2D：整数平滑跟随
|
||||||
|
|
||||||
|
`Camera2D` 是一个完全整数化的死区跟随相机。其实际视口为 1024 × (600 − 24) 像素，顶部留出 24 px 给 HUD 黑边；水平与垂直死区分别为 60 与 40 像素，平滑系数 `smooth_shift_` 固定为 3。跟随算法的核心思想是：先计算目标点与视口中心的偏移，若偏移超出死区的一半，则将偏移右移 3 位作为本次移动步长；由于右移结果可能为 0，此时用 ±1 像素兜底，防止相机停留在 1~7 像素的残余距离上无法追齐。
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
int32_t dx = target.x - (position.x + viewport.w / 2);
|
||||||
|
if (abs(dx) > dead_zone.w / 2) {
|
||||||
|
int32_t step = dx >> smooth_shift_; // 整数近似指数平滑
|
||||||
|
if (step == 0) step = (dx > 0 ? 1 : -1); // 避免残余不动
|
||||||
|
position.x += step;
|
||||||
|
}
|
||||||
|
// y 轴同理
|
||||||
|
clamp_to_bounds();
|
||||||
|
```
|
||||||
|
|
||||||
|
用右移代替除法（`dx >> 3` 等价于 `dx / 8`）不仅避免了浮点运算，在 ARM 上也是最廉价的分频方式。每帧更新后，相机位置还会被钳制到当前房间的边界内；当房间尺寸小于视口时，相机会直接吸附到房间的左上角，形成类似“回信箱”的取景效果，常用于竖直井道等狭小空间。
|
||||||
|
|
||||||
|
## 4.5 Level / Room：数据驱动的关卡与房间网格
|
||||||
|
|
||||||
|
LightGame 的关卡数据完全离线烘焙，运行时不需要任何文件 I/O。`LevelData` 是一个 POD 结构，包含前景/背景 tile 数组指针、`ObjectSpawn` 对象生成表、`TileLightRule` 光照规则表、出生点坐标与地图边界。所有这些内容都在 `src/Apps/LightGame/src/levels/*.h` 中以 C 数组形式给出，编译期即定案。
|
||||||
|
|
||||||
|
`Level` 类在内存中持有三份 tile 缓冲区视图：`original_tiles_` 保存只读的原始地图，`tile_buffer_` 作为可写的运行时前景，`background_tiles_` 作为可写背景。`LightEffectSystem::update_tilemap` 根据 `TileLightRule` 在这三份视图之间操作：当光照落在规则窗口内时，从 `original_tiles_` 恢复对应格子；否则写入 `EmptyTile`（0xFFFF）。由于每次操作只涉及规则数量而非整张地图，其复杂度为 O(rules)。
|
||||||
|
|
||||||
|
`RoomLayout` 则用整数网格把大关卡切分成若干矩形房间。`RoomGrid` 包含列数、行数、每个房间的像素宽高以及 `RoomDef` 数组，查询 `room_index_of(grid, world_pos)` 只需两次整除加上边界钳制即可得到当前房间索引。每个 `RoomDef` 携带 `RoomBounds` 与 `default_spawn`，切换房间时相机会立即更新边界，其它房间的检查点自动失效，从而形成类似 Metroidvania 的分区探索体验，同时避免了运行时加载关卡的额外开销。
|
||||||
|
|
||||||
|
## 4.6 LevelRenderer：分层合成顺序
|
||||||
|
|
||||||
|
`LevelRenderer` 每帧按照从远到近的顺序合成画面。首先绘制可选的背景 tilemap，调用 `draw_tilemap_shaded` 并指定 `shade_numerator=80`、`shade_shift=7`，即以 80/128 的整数比例将背景压暗到约 0.625 倍亮度，全部通过定点乘法完成，不引入浮点。接着绘制前景 tilemap，使用未经调暗的 `draw_tilemap` 快路径。然后遍历对象数组绘制非玩家对象，绘制前通过 `LightEffectSystem::should_render` 判断可见性：对于 `LightPlatform`、`ShadowPlatform` 和 `Door`，无论其当前 `solid` 状态如何都保持渲染，以便玩家读取光照规则；其它对象则以 `obj.solid` 作为是否绘制的依据。最后绘制玩家精灵，使其始终位于最上层。
|
||||||
|
|
||||||
|
在 `IMX6U_DEBUG` 调试构建中，`LevelRenderer` 还会追加一层 `draw_debug`，调用 `Rasterizer::DrawLine` 为每个活动对象绘制轴对齐碰撞盒，不同对象类型使用不同颜色。这也是当前项目中 `Rasterizer::DrawLine` 的主要使用场景。
|
||||||
|
|
||||||
|
## 4.7 GameStateManager 与 HUD
|
||||||
|
|
||||||
|
`GameStateManager` 管理游戏的高层状态机，包含 `Title`、`Playing`、`Paused`、`GameOver`、`LevelComplete` 五个状态。ESC 键通过 `esc_was_down_` 进行边沿检测，确保每次按键只触发一次状态切换；Title 状态下开始提示以 500 ms 为周期闪烁，引导玩家按键进入游戏。
|
||||||
|
|
||||||
|
HUD 由 `GameStateManager` 与 `LightGameApp` 协作绘制。屏幕左上角有一条宽 80 px 的光照指示条，其填充宽度为 `light_level * 78 / max_light_level`，填充颜色使用整数线性梯度：红色分量随光照增强而减小、绿色分量随光照增强而增大，黑暗时偏红、明亮时偏绿，玩家可以一眼判断当前光照区间。右上角显示当前关卡号 `LV{n}`，左下角显示金币数 `× n`，左侧还以小红块阵列表示剩余生命数。暂停时屏幕中央绘制一个半透明遮罩，当前使用 `Color(0, 0, 0, 200)`，但由于 RGB565 帧缓冲没有 alpha 通道，实际显示为纯黑遮罩，这是渲染管线带来的已知表现限制。
|
||||||
|
|
||||||
|
# 5. 光敏传感器与游戏机制耦合
|
||||||
|
|
||||||
|
**本节是本次实训中作者认为最有信息含量的部分：光敏传感器不是简单的"控件"，而是一条完整的传感-滤波-语义映射链。**
|
||||||
|
|
||||||
|
## 5.1 AP3216C 与出厂驱动的问题
|
||||||
|
|
||||||
|
AP3216C 是一颗集环境光（ALS）、接近（PS）和红外（IR）于一体的三合一传感器芯片。IMX6ULL 教学板的出厂内核提供了 `ap3216c` 字符设备驱动，用户空间理论上可以通过 `open("/dev/ap3216c")` 再 `read()` 获取 ALS/PS/IR 三个字节的数据。但在实测中，这条路径并不可靠，主要表现为两个方面。
|
||||||
|
|
||||||
|
一方面，`read()` 经常返回 `EINVAL` 或读到全零。其根本原因在于出厂驱动将 `SysConfig(0x00)` 配置为 `0x03`，即 ALS、PS、IR 同时开启，而 ALS 采样与 PS 中断共用同一 ADC 通道；当 PS 触发中断采样时，会打断 ALS 的转换过程，用户空间如果在两次触发之间读取，就可能拿到未完成转换的旧值或全零。
|
||||||
|
|
||||||
|
另一方面，该内核驱动已经占用了 I²C-0 总线上 0x1E 地址，`i2cdetect -y -r 0` 会显示该地址为 `UU`（被内核持有）。此时普通的 `ioctl(I2C_SLAVE, 0x1E)` 会返回 `EBUSY`，用户态程序无法再通过标准 I²C 从设备接口与该地址通信。
|
||||||
|
|
||||||
|
因此，为了获得稳定、随环境变化的 ALS 数据，项目选择绕开出厂字符设备驱动，直接通过 `/dev/i2c-0` 与 AP3216C 芯片寄存器对话。这也是后续 i2c 直读方案的设计出发点。
|
||||||
|
|
||||||
|
## 5.2 i2c 直读方案与 I2C_SLAVE_FORCE
|
||||||
|
|
||||||
|
`Ap3216cPhotoSensor` 的实现思路是直接打开 I²C 总线节点 `/dev/i2c-0`，然后使用 `ioctl(fd_, I2C_SLAVE_FORCE, 0x1E)` 强制占用 0x1E 地址，从而跳过内核驱动的锁检查。初始化阶段向 AP3216C 写入两组配置：一是将 `SysConfig(0x00)` 设为 `0x01`，切换到 ALS-only 连续采样模式，避免 PS 中断打断 ALS；二是将 `AlsConfig(0x10)` 设为 `0x30`，使增益位 `gain[5:4]=11`，选择 323 lux 满量程的最大增益档位，以充分利用 12 位 ADC 的室内光分辨率。
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
fd_ = open("/dev/i2c-0", O_RDWR);
|
||||||
|
ioctl(fd_, I2C_SLAVE_FORCE, 0x1E); // 强占地址，忽略 UU 状态
|
||||||
|
// SysConfig(0x00) = 0x01 → ALS-only 连续采样
|
||||||
|
write(fd_, {0x00, 0x01}, 2);
|
||||||
|
// AlsConfig(0x10) = 0x30 → gain[5:4]=11，full scale 323 lux，最大增益
|
||||||
|
write(fd_, {0x10, 0x30}, 2);
|
||||||
|
```
|
||||||
|
|
||||||
|
之后，每帧 `update()` 执行两次“写寄存器地址再读一字节”的事务，分别读取 `0x0C`（ALS 低字节）和 `0x0D`（ALS 高字节），组合成 `uint16_t als = (hi << 8) | lo` 并钳制到 4095。如果某次读取失败，驱动层保留上一次的值，防止单次 `ETIMEDOUT` 造成亮度归零跳变，其余平滑处理交由应用层完成。
|
||||||
|
|
||||||
|
这一方案有几个关键点。首先，`I2C_SLAVE_FORCE` 是必须的，因为普通 `I2C_SLAVE` 在地址已被内核驱动持有时会返回失败，只有 `_FORCE` 版本能跳过锁检查，实现用户态与内核驱动共享同一 I²C 地址。其次，ALS-only 模式让 ADC 只服务 ALS 通道，采样率稳定在数据手册标称的连续模式（约 100 ms/次），足以满足 30 Hz 主循环的需求。再次，最大增益 323 lux 的满量程比 20 kLux 低增益档位更适合室内环境，能把台灯、手电或手遮挡产生的光强变化映射到 12 位 ADC 的有效区间。最后，驱动层仅做“读失败保持上值”这一最小容错，不做额外平滑，避免在板端引入不必要的计算。
|
||||||
|
|
||||||
|
## 5.3 Q8 定点 EMA 滤波
|
||||||
|
|
||||||
|
原始 ALS 值在手指快速晃过或荧光灯高频闪烁时抖动明显，直接在 12 位裸值上驱动游戏会导致对象和地形频繁通断。为此，项目在 `LightGameApp` 中引入了一级 Q8 定点一阶指数移动平均（EMA）滤波，核心代码如下：
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
// 状态：smoothed_light_q8_（int32，= real 值 × 256）
|
||||||
|
int32_t target_q8 = static_cast<int32_t>(raw) << 8; // 12-bit → Q8
|
||||||
|
smoothed_light_q8_ += (target_q8 - smoothed_light_q8_) >> 4; // α = 1/16
|
||||||
|
uint16_t light = static_cast<uint16_t>(clamp(smoothed_light_q8_ >> 8, 0, 4095));
|
||||||
|
```
|
||||||
|
|
||||||
|
该滤波器的平滑系数 α 等于 2⁻⁴，即 1/16。在 30 Hz 采样率下，其截止频率约为 `f_c ≈ f_s · α / (2π) ≈ 0.30 Hz`，从 10% 上升到 90% 的响应时间约为 0.7 秒。这一速度对“举起手电”或“用手遮挡”这类交互而言较为舒适：既能滤除手指抖动和 ADC 量化噪声，又不会让玩家感到明显延迟。
|
||||||
|
|
||||||
|
采用 Q8 定点表示而非在 12 位裸值上直接滤波，是为了保留足够的头部空间。若直接对原始值执行 `smoothed += (raw - smoothed) >> 4`，当误差绝对值小于 16 时右移结果会归零，滤波器将停在错误值上不再更新。将数值左移 8 位到 Q8 后，即使差分很小也能被逐步吸收，从而保证缓慢变化的光照（例如拉窗帘）不会失锁。
|
||||||
|
|
||||||
|
在 PC 调试通道中，W/S 键以 ±256 的步长直接调节 `manual_light_level_`，并将 `smoothed_light_q8_` 同步设置到对应值，因此手动模式与传感器模式之间切换时不需要额外过渡，光照变化保持连续。
|
||||||
|
|
||||||
|
## 5.4 光照 → 对象通断（LightEffectSystem）
|
||||||
|
|
||||||
|
`LightEffectSystem::update(objects, light)` 将滤波后的光照值映射到三类关卡对象的实体状态，具体规则如下表所示。
|
||||||
|
|
||||||
|
| 对象类型 | 逻辑 | 直觉解释 |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `LightPlatform` | `solid = light >= threshold.min_level` | 被点亮时才凝固的平台 |
|
||||||
|
| `ShadowPlatform` | `solid = light <= threshold.max_level` | 只在阴影里存在的平台 |
|
||||||
|
| `Door` | `solid = !(min ≤ light ≤ max)` | 在给定亮度窗口内才开门 |
|
||||||
|
|
||||||
|
对象的 `solid` 字段直接决定 `Physics2D` 是否将其纳入碰撞解算：当 `solid == false` 时，玩家可以像穿过空气一样穿过该对象。视觉上，`LightPlatform` 和 `ShadowPlatform` 分别拥有 on/off 两组精灵，`Door` 拥有 closed/open 两组精灵，`LevelRenderer` 每帧根据当前 `solid` 状态切换贴图，实现“随光照凝固或消失”的效果。
|
||||||
|
|
||||||
|
这一设计的优势在于复用了 `GameObject` 已有的 `solid` 字段，物理层不需要为光敏对象编写特殊分支，代码路径保持统一；同时，关卡编辑器只需要为对象填写 `min_level` 和 `max_level` 两个 12 位整数，即可表达“在多亮时能够踩上去”的规则，制作过程直观且不易出错。
|
||||||
|
|
||||||
|
## 5.5 光照 → 地形几何显隐
|
||||||
|
|
||||||
|
除了移动对象的实体状态，光照还可以改变地形几何本身。`TileLightRule` 由 `tile_x`、`tile_y`、`light_min` 和 `light_max` 四个字段组成，`LightEffectSystem::update_tilemap(level, light)` 逐条规则进行判断：若当前光照值落在规则窗口内，则将 `tile_buffer_[y*w + x]` 恢复为 `original_tiles_[y*w + x]` 中的真实 tile；否则写入 `Tilemap::EmptyTile`（0xFFFF），使该格从前景中消失。
|
||||||
|
|
||||||
|
这里采用“原始 + 可变”双缓冲而不是原地翻转，是因为同一格子可能在不同光照窗口下多次进出，必须以 `original_tiles_` 作为权威真相。这种设计让地图本身能够随光照消失或浮现，例如光下才显形的桥、暗中才崩塌的地板等效果，都可以通过规则配置实现。由于每次更新只涉及规则数量而非整张地图，其时间复杂度为 O(rules)，与地图尺寸无关。
|
||||||
|
|
||||||
|
`Tilemap::EmptyTile` 同时被渲染层和物理层识别：渲染层遇到空瓦片跳过绘制，物理层的 `is_solid_tile` 对空瓦片返回 false。因此一次写入即同时完成“看不见”和“走得过去”两种语义，无需在其它系统中进行额外同步。
|
||||||
|
|
||||||
|
## 5.6 光照 → HUD 视觉反馈与手动调试通道
|
||||||
|
|
||||||
|
光照不仅影响游戏逻辑，也是玩家感知当前环境状态的第一视觉线索。HUD 左上角的光照指示条以颜色渐变实时展示当前 `light` 值，具体实现已在 §4.7 中说明。
|
||||||
|
|
||||||
|
在 PC 端开发时，由于没有真实的光敏传感器硬件，项目提供了手动调试通道：按下 `W` 或 `S` 键可以以 ±256 的步长调节 `manual_light_level_`，并置位 `has_manual_override_` 标志，使游戏绕过传感器读取而使用手动值。这样开发者可以在桌面环境中验证所有光敏关卡逻辑，这也是关卡编辑器在无硬件条件下的主要测试手段。
|
||||||
|
|
||||||
|
此外，`IPhotoSensor` 的 PC 实现 `SdlPhotoSensor` 默认返回 2048，即 12 位量程的中点。该默认值保证桌面构建启动时处于“中等亮度”，避免开局黑屏或过度曝光，使调试体验与板端真实光照环境保持接近。
|
||||||
|
|
||||||
|
# 6. 跨平台构建与调试
|
||||||
|
|
||||||
|
## 6.1 CMake 选项与条件编译
|
||||||
|
|
||||||
|
顶层 `CMakeLists.txt` 通过两组开关驱动条件编译。第一组是 `TARGET_IMX`，默认关闭；开启时定义 `TARGET_IMX` 宏，将 `FBDisplay`、`AlsaAudioInput`、`EvdevButtonInput` 和 `Ap3216cPhotoSensor` 等板端实现编入核心库；关闭时则定义 `TARGET_PC` 宏，编入 `SDLDisplay`、`SdlAudioInput`、`SdlKeyboardButtonInput`、`SdlPhotoSensor` 等 PC 实现，同时把 `third_party/imgui` 一并加入构建。第二组是 `$<CONFIG:Debug>` 生成器表达式，仅在 Debug 构建中定义 `IMX6U_DEBUG`，用于门控关卡编辑器、ImGui、F1 切换、调试 HUD 与碰撞盒可视化等调试功能。
|
||||||
|
|
||||||
|
`Apps/LightGame/CMakeLists.txt` 复用了同一套开关，并通过生成器表达式把 `LevelEditor.cpp` 限制在 `NOT TARGET_IMX AND CONFIG:Debug` 的条件下，确保编辑器只在 PC + Debug 构建中进入源码列表，Release 或 ARM 交叉编译构建中连相关符号都不会产生：
|
||||||
|
|
||||||
|
```cmake
|
||||||
|
set(LIGHTGAME_EDITOR_GUARD "$<AND:$<NOT:$<BOOL:${TARGET_IMX}>>,$<CONFIG:Debug>>")
|
||||||
|
target_sources(IMX6U-LightGame PRIVATE
|
||||||
|
"$<${LIGHTGAME_EDITOR_GUARD}:${CMAKE_CURRENT_SOURCE_DIR}/src/editor/LevelEditor.cpp>")
|
||||||
|
```
|
||||||
|
|
||||||
|
`DefaultHardware.h` 把平台差异收敛到类型定义处，因此 `LightGameApp` 中没有任何用于选择输入或传感器实现的 `#ifdef`。唯一的平台分支保留在 `main.cpp` 中，围绕显示帧提交与 ImGui 叠加这一调试需求展开，属于为 PC 调试生态刻意保留的分层例外。
|
||||||
|
|
||||||
|
## 6.2 LevelEditor：PC + Debug 独占的关卡编辑器
|
||||||
|
|
||||||
|
`LevelEditor` 是一个基于 ImGui 的关卡编辑器，直接在内存中编辑 `Level` 实例。它提供四类主要功能：工具切换，包括 `TileBrush`、`ObjectPlace`、`Select` 和 `Eraser`；图层切换，可在 `Foreground` 与 `Background` 之间选择；属性面板，用于修改选中对象的类型以及光照阈值 `min_level / max_level`；导出功能，通过 `export_to_header(name)` 将当前 tile 数组与对象列表按 `LevelData` 期望的 C 头文件结构序列化后写入磁盘。
|
||||||
|
|
||||||
|
由于 `LevelData` 是编译期常量，编辑器的“导出”实际上是“生成新头文件 → 重新编译”的循环。这一权衡的好处是运行时零 I/O、零解码；代价是快速迭代时需要重新编译，时间约为几十秒。
|
||||||
|
|
||||||
|
在主循环中，`F1` 键用于切换编辑器可见性。`ImGui_ImplSDL2_ProcessEvent` 与 `LightGameApp` 的输入并行处理，当编辑器处于活动状态时，ImGui 焦点会接管键盘和鼠标事件，从而避免点击面板时误操作角色。
|
||||||
|
|
||||||
|
## 6.3 调试流程与验证方法
|
||||||
|
|
||||||
|
项目推荐分四步完成开发与验证。第一步是 PC + Debug：在 VS Code 或 MSVC 中编辑代码后，执行 `cmake --build build-win --config Debug` 并直接 F5 调试；此阶段利用 W/S 手动光照通道遍历所有光敏关卡逻辑，并通过 F1 打开关卡编辑器修关。第二步是 PC + Release：使用 `--config Release` 构建，验证发布态下的帧率与关键路径行为是否稳定。第三步是 ARM 交叉编译：执行 `cmake -B build-arm-fb -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-arm-linux-gnueabihf.cmake -DTARGET_IMX=ON`，随后将二进制通过 `scp` 传到板上，`chmod +x` 后直接运行；`TARGET_IMX=ON` 会自动启用 framebuffer 后端，`/dev/fb0` 与 `/dev/i2c-0` 在设备节点权限已配置好的情况下无需 root。第四步是板端验证：主循环内已包含光照读取失败时保持上次值的容错，观察重点包括光条是否随环境变化、检查点重生是否正常、跳过尖刺的 14 px 手感是否合理，以及房间切换时相机吸附是否到位。
|
||||||
|
|
||||||
|
# 7. 功能测试与验证
|
||||||
|
|
||||||
|
⟨本节等课程要求出来后再回填详细数据。以下先给测试项模板与预期指标。⟩
|
||||||
|
|
||||||
|
## 7.1 渲染性能与帧率测试
|
||||||
|
|
||||||
|
- 目标平台：IMX6U 教学板，1024×600，30 FPS 目标；
|
||||||
|
- 度量方法：主循环中埋点 `time_source.get_time_ms()` 记录每帧的 `update / draw / present / sleep` 四段时长，采样若干秒的均值/最大值；
|
||||||
|
- 预期结果：全场景稳定 30 FPS，`sleep` 阶段应占多数（说明未跑满 CPU）；无掉帧、无撕裂。
|
||||||
|
|
||||||
|
## 7.2 传感器响应曲线测试
|
||||||
|
|
||||||
|
- 度量方法：以已知光源分别做 (a) 覆盖—打开、(b) 缓慢遮挡两组动作，记录 `raw` 与 `smoothed` 曲线；
|
||||||
|
- 预期结果：`raw` 抖动明显，`smoothed` 上升到 90% 的时间约 0.7 s，无过冲、无稳态偏差。
|
||||||
|
|
||||||
|
## 7.3 关卡完整通关测试
|
||||||
|
|
||||||
|
- 步骤：从 Title 开始，走过所有房间，触发若干 `LightPlatform / ShadowPlatform / Door` 与 `TileLightRule`，中途死亡 ≥ 1 次以验证检查点重生，最终触达 `LevelComplete`；
|
||||||
|
- 预期结果：状态机无死锁，房间切换无穿墙，检查点在预期位置激活，HUD 计数正确。
|
||||||
|
|
||||||
|
# 8. 工作总结与展望
|
||||||
|
|
||||||
|
本次实训完成了两块相互支撑的工作。其一是 `Core` 底层渲染库，它将“在 IMX6ULL 这类无 GPU 平台上，用 RGB565 帧缓冲和整数运算实现可玩 2D 画面”的实践，固化为 `IDisplay`、`DrawContext`、`Tilemap`、`Sprite`、`BitmapFont`、`Timer` 等可移植接口。其二是 `LightGame` 光敏平台跳跃游戏，围绕板载 AP3216C 光敏传感器，实现了从 i2c 直读、Q8 定点 EMA 滤波，到光照驱动对象通断、地形几何显隐与 HUD 反馈的完整闭环，并支持检查点重生与房间切换。
|
||||||
|
|
||||||
|
从技术层面看，本次实训有三点关键收获。第一，在嵌入式实践中，当出厂驱动的默认配置无法满足应用需求时，直接通过 `I2C_SLAVE_FORCE` 抢占地址并重配置传感器寄存器，是获得稳定输入的最短路径。第二，无 GPU 平台的热路径必须保持整数化与定点化：RGB565 帧缓冲、RGBA5551 1-bit 透明、Q8 EMA、Q7 亮度缩放、亚像素累加物理等任何环节引入浮点，都会给 CPU 带来不必要的额外开销。第三，通过接口加 typedef 隔离平台差异，比在主逻辑中大量使用 `#ifdef` 更易于维护；`DefaultHardware.h` 使得 `LightGameApp` 基本保持平台无关，PC 端的 W/S 手动光照、F1 关卡编辑器等调试通道也因此能够与板端代码 cleanly 共存。
|
||||||
|
|
||||||
|
当前实现仍存在一些局限，也为后续工作指明了方向。HUD 的半透明渲染受限于 RGB565 缺少 alpha 通道，暂停遮罩目前只能退化为不透明黑色，未来可以在 `DrawContext` 中增加 dither 或整数混合的 `fill_rect_blend` 路径。关卡数据全部编译期烘焙意味着修改关卡需要重新编译，后续可考虑将 `LevelData` 打包为板上可读的紧凑二进制并在启动时 mmap，以换取运行时改关能力，代价是增加一次 I/O 与解析。音频链路 `IAudioInput / IAudioOutput` 虽已抽象，但 LightGame 尚未接入，后续可让光照状态驱动音效，进一步强化“环境即输入”的沉浸感。此外，Launcher、LightGame 与合作者的 Game 目前仍是独立可执行文件，`docs/APP_AND_CORE_ARCHITECTURE.md` 中已规划 `IApp` 接口与单进程多应用切换，尚待落地。
|
||||||
|
|
||||||
|
总体来看，本次实训验证了在 IMX6U 这类无 GPU 嵌入式教学板上构建完整图形化游戏的可行性。通过把渲染、输入、传感器等每一层细节重新掌握在应用代码中，项目不仅实现了稳定的 1024×600@30 FPS 画面，也加深了对软硬件协同设计的理解。
|
||||||
|
|
||||||
|
# 参考文献
|
||||||
|
|
||||||
|
[1] Freescale Semiconductor. *i.MX 6UltraLite Applications Processor Reference Manual*. Rev. 2, 2016.
|
||||||
|
[2] LiteON. *AP3216C Ambient Light and Proximity Sensor Datasheet*. Rev. 1.5, 2015.
|
||||||
|
[3] Linux Kernel Documentation. *i2c/dev-interface — I²C/SMBus Character Device Interface*. https://www.kernel.org/doc/Documentation/i2c/dev-interface (accessed 2026-07).
|
||||||
|
[4] Linux Kernel Documentation. *fb/api.txt — The Linux Frame Buffer Device API*. https://www.kernel.org/doc/Documentation/fb/api.txt (accessed 2026-07).
|
||||||
|
[5] SDL Community. *Simple DirectMedia Layer 2.0 — Documentation Wiki*. https://wiki.libsdl.org/SDL2/ (accessed 2026-07).
|
||||||
|
[6] ⟨可选：正点原子.《I.MX6U 嵌入式 Linux 驱动开发指南》. 2020. ——若指导老师推荐加入⟩
|
||||||
|
[7] ⟨可选：唐佐林. 现代 C++ 嵌入式实战. 电子工业出版社, 2019. ——补充 C++11 相关引用⟩
|
||||||
Binary file not shown.
@@ -0,0 +1,41 @@
|
|||||||
|
# 实训日记 — IMX6U 光敏平台跳跃游戏
|
||||||
|
|
||||||
|
## 第 1 天（7 月 6 日，周一）：项目启动与选题
|
||||||
|
|
||||||
|
今天就要开始这个嵌入式的专业方向综合实训了。选题环节我几乎没有犹豫，直接定了做一个跑在 IMX6ULL 板子上的 2D 游戏——一方面自己之前用 C++ 写过一个跑在 CPU 上的软光栅小 demo，也用 Unity 做过一些小项目，对图形和游戏这一套流程比较熟；另一方面 IMX6ULL 没有 GPU，OpenGL 那一套走不通，反倒正好把之前软光栅的思路搬过来。翻了翻板子附带的手册，注意到板载有一颗 AP3216C 光敏传感器，就想干脆让传感器直接参与游戏机制，做一个"光照驱动"的平台跳跃：亮着的时候平台出现、暗下来时通道打开。今天大部分时间在读任务书、翻板载外设列表和写选题说明，正式的代码一行没写，但方向定下来之后心里踏实了不少。
|
||||||
|
|
||||||
|
## 第 2 天（7 月 7 日，周二）：交叉编译环境与双后端抽象
|
||||||
|
|
||||||
|
今天开始搭开发环境。先在 Ubuntu 上装了 `gcc-arm-linux-gnueabihf` 工具链，写了一份 `cmake/toolchain-arm-linux-gnueabihf.cmake`，让 CMake 能一键切到 ARM 交叉编译。考虑到板子上直接调试太慢，日常肯定要在 PC 上跑，所以早早就决定把显示后端做成可切换的：PC 端走 SDL2 开窗口，ARM 端直接写 `/dev/fb0`。这就是 `IDisplay` 这个纯虚接口的由来，通过 CMake 里的 `USE_FRAMEBUFFER` 选项在编译期二选一，核心渲染代码完全不知道自己底下是 SDL 还是 framebuffer。傍晚把两端的 hello world 都点亮了——PC 上开出一个纯色窗口，板子的 LCD 也刷出了预期的彩色横条，虽然还只是最基础的像素输出，但双端并行的开发节奏算是搭起来了。
|
||||||
|
|
||||||
|
## 第 3 天（7 月 8 日，周三）：Core 底层库的移植与重写
|
||||||
|
|
||||||
|
之前那份软光栅是 3D 的，齐次坐标、透视除法、深度缓冲一应俱全，这次做 2D 游戏根本用不到。今天的主要工作是把 `FrameBuffer`、`DrawContext`、`Timer` 这几个通用模块从旧项目里拆出来重写，只保留 2D 需要的部分。一个关键决定是：板载 LCD 是 RGB565 的，我把 `FrameBuffer` 的像素类型从原来的 `uint32_t` 换成了 `uint16_t`，Sprite atlas 也统一用 RGBA5551（1-bit alpha），这样一帧 1024×600 只占约 1.2 MB，比 32bpp 直接省一半带宽，对没有 GPU 的板子来说很重要。`Timer` 采用余数累积法维持 30 FPS，避免 `int` 除法舍入导致的帧率长期漂移。热路径规则也在今天正式立了下来：核心逻辑禁止浮点、禁止堆分配、SDL 相关的代码只允许出现在 `Core/Platform/` 目录，其他地方一概不许 `#include SDL.h`。
|
||||||
|
|
||||||
|
## 第 4 天（7 月 9 日，周四）：主循环与 Physics2D 亚像素积分
|
||||||
|
|
||||||
|
今天开始搭游戏本体。主循环结构比较常规：`poll_events → update(dt) → render → present`，`Timer` 负责把变化的真实时间切成固定步长交给 `update`。真正花时间的是 `Physics2D`——由于禁止浮点，速度、加速度全部用 Q8 定点存储，每帧把速度累加到 `sub_pixel_x_ / sub_pixel_y_` 亚像素累加器上，只有跨过整像素时对象位置才真正变化。这样即便每帧位移不足 1 像素，累积几帧也能得到平滑的移动效果，跳跃初速度、重力、最大下落速度都能用整数精细调参。中间踩了个小坑：我一开始把 `sub_pixel_x_ / sub_pixel_y_` 挂在 `GameObject` 上，写着写着发现所有物理体都在重复同一份亚像素状态，才反应过来这两个字段本来就应该属于 `Physics2D` 组件，物理逻辑才更内聚。返工不算大，但也提醒自己接下来在划分类职责时要更谨慎一点。
|
||||||
|
|
||||||
|
## 第 5 天（7 月 10 日，周五）：PlayerController 与 Camera2D
|
||||||
|
|
||||||
|
一周的最后一天，重点是让角色真正"活"起来。`PlayerController` 用状态机组织：Idle、Running、Jumping、Falling、Dead 五个状态之间根据输入和物理状态互相跳转。跳跃做成了"二段可变高"——按住跳键时持续施加上升力，松开或者达到最高高度切换到 Falling，这样短按能小跳、长按能高跳，手感一下就出来了。尖刺伤害判定选择了"tile 顶部向下 14 px"这样一个矩形区域，而不是整个 tile，避免玩家轻轻蹭到侧面就死掉。相机跟随算法花的时间比想象中久：直接线性插值会有浮点，改成 `dx >> smooth_shift_` 做整数右移，但发现 `dx` 只剩 1 时右移变 0，相机就再也追不上角色了，只好加一个 ±1 的兜底位移。周五收工的时候，小人已经能在临时关卡里跑跳、被尖刺秒杀、然后从头再来，虽然只是最裸的白盒关卡，但看着还是很有成就感的。周末先歇两天。
|
||||||
|
|
||||||
|
## 第 6 天（7 月 13 日，周一）：Level / Room 与关卡编辑器
|
||||||
|
|
||||||
|
回来第一天，先把关卡数据结构定下来。`Level` 分三层：foreground（碰撞层）、background（装饰层）、objects（平台、门、检查点等交互对象）；房间划分用 `RoomGrid`，每个房间对应一个 `RoomBounds`，`Camera2D` 在房间边界内 clamp，切换房间时相机做一次瞬移，形成类似 Metroidvania 的分屏效果。但用代码硬写 tile 数组实在太痛苦了，改一个空气块要跑一次编译、烧一次板子，效率完全不能忍。于是今天下午专门做了一个 PC 端的 `LevelEditor`：用 ImGui 显示 tile palette，鼠标左键刷 tile、右键擦除，F1 切换编辑器窗口，保存直接写回源文件的关卡数据数组。这个编辑器只在 PC 且 Debug 构建下编译，靠 CMake 里的 `LIGHTGAME_EDITOR_GUARD` 宏隔离，ARM Release 构建里连一个字节的 ImGui 都不会带进去，保证板端二进制体积干净。
|
||||||
|
|
||||||
|
## 第 7 天（7 月 14 日，周二）：AP3216C 攻坚
|
||||||
|
|
||||||
|
今天正式接光敏传感器，也是这个项目里最折腾的一天。板子出厂 dts 里 AP3216C 的驱动是加载好的，`/dev/ap3216c` 字符设备也在，我按最直觉的方式 `open + read`，结果一直返回 `EINVAL`。翻了半天板载 SDK 里的驱动源码也没定位到具体是哪一步出的问题，各种参数组合都试过，就是拿不到稳定的读数。走到快中午的时候干脆决定绕开驱动：既然板子上 i2c 总线是通的，那就直接 `open("/dev/i2c-0")`，用 `ioctl(fd, I2C_SLAVE_FORCE, 0x1E)` 抢占 AP3216C 的地址，然后手写寄存器——`SysConfig = 0x01` 打开 ALS-only 模式，`AlsConfig = 0x30` 选最大增益 323 lux 满量程，再从寄存器 `0x0C / 0x0D` 读两字节合成 16 位光照原始值。第一次稳定看到 lux 数值随着我用手挡光的动作实时变化时，一整天的憋闷才散了。收工时把这一坑详细记进了项目笔记，免得日后忘了为什么要用 `I2C_SLAVE_FORCE` 而不是普通的 `I2C_SLAVE`。
|
||||||
|
|
||||||
|
## 第 8 天（7 月 15 日，周三）：光照与游戏机制的耦合
|
||||||
|
|
||||||
|
有了稳定的 lux 输入，今天把光敏和游戏机制真正串起来。原始 lux 抖动不小，先在 `LightGameApp` 里加了一层 Q8 定点 EMA 滤波，`smoothed += (target - smoothed) >> 4`，也就是 α = 1/16，截止频率约 0.3 Hz、响应时间约 0.7 s，既能压掉手抖引起的高频噪声，也不至于慢到玩家察觉不到光变化。上层的 `LightEffectSystem` 定义了三类对象规则：`LightPlatform` 光照高于阈值时 `solid`、`ShadowPlatform` 光照低于阈值时 `solid`、`Door` 在特定区间内开启，一份规则表覆盖了几乎所有关卡玩法。地形几何显隐用 `TileLightRule` 加双缓冲实现——`original_tiles_` 永远保留关卡原始数据，`tile_buffer_` 是当前光照下重刷过的副本，光照跨越阈值时才重新生成一次 `tile_buffer_`，避免每帧都全图刷新。最后在 HUD 上加了一条光照条实时显示 smoothed lux，PC 端还挂了 W/S 手动调光通道方便调试。傍晚试玩的时候，看着小人跟着我用手挡光跳上跳下、又跟着灯光变化推开一扇门，那种"环境即输入"的味道终于对上了。
|
||||||
|
|
||||||
|
## 第 9 天（7 月 16 日，周四）：板端联调与性能打磨
|
||||||
|
|
||||||
|
离答辩还剩一天，今天必须把所有功能在板子上跑一遍。第一个冒出来的问题是帧率：PC 端 30 FPS 稳如老狗，板子上从一个大房间进另一个大房间时会掉到 20+ FPS。在 `DrawContext::present` 前后加了 `gettimeofday` 打时间戳定位瓶颈，发现 `draw_tilemap_shaded` 对每个可见 tile 都做了亮度整数缩放（Q7 乘法 + 位移），而当时相机可视区域外的 tile 也在参与计算。改成先按 `Camera2D` 视口裁剪、只把落在屏幕内的 tile 送进 shading 循环之后，帧率立即稳回 30 FPS，甚至还有余量。第二个是 HUD 半透明：RGB565 没有 alpha 通道，暂停画面的半透明遮罩暂时只能退化成不透明黑色，试了 dither 但边缘噪点比较明显，最终决定先按不透明处理，报告的"局限与展望"里如实记下来。晚上把主线路径又跑了两遍：起点出发、经过检查点、光照驱动开门、切换房间、被尖刺送回检查点，一切符合预期。收工时看了眼时间，离明天答辩还有大概 18 个小时。
|
||||||
|
|
||||||
|
## 第 10 天（7 月 17 日，周五）：报告收尾与答辩
|
||||||
|
|
||||||
|
今天就是答辩日了。上午把 `report_v2.md` 从大纲填成完整报告，中间反复对着源码核对每一处技术细节：Q8 EMA 的 α 值、AP3216C 的寄存器地址、亚像素累加公式、CMake 里的编译选项、`I2C_SLAVE_FORCE` 的宏值，尽量不让报告里出现一个和代码对不上的数字。下午答辩前又把 demo 视频重新录了一遍，特意在暗光和亮光两种环境下各跑一遍完整通关流程，把"环境即输入"这条主线讲清楚。回望这 10 天，从最开始定下"用 IMX6ULL + 软光栅 + 光敏传感器做游戏"，到双后端抽象、Core 库移植、Physics2D 亚像素积分、状态机角色控制、关卡编辑器，再到 AP3216C 那一天的独自踩坑和光照耦合出效果的傍晚，每一步都能对应到具体的代码和当时的想法。写文档这类事情我一直不太喜欢，但真的一行行把这 10 天梳理出来的时候，那种"东西是自己一点点搭起来的"的踏实感确实不太一样。答辩就在下午，剩下的交给现场发挥吧。
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
[Window][Debug##Default]
|
||||||
|
Pos=60,60
|
||||||
|
Size=400,400
|
||||||
|
|
||||||
|
[Window][Tools]
|
||||||
|
Pos=0,20
|
||||||
|
Size=140,580
|
||||||
|
|
||||||
|
[Window][Status]
|
||||||
|
Pos=140,576
|
||||||
|
Size=884,32
|
||||||
|
|
||||||
|
[Window][Export Level]
|
||||||
|
Pos=312,200
|
||||||
|
Size=400,200
|
||||||
|
|
||||||
|
[Window][Properties]
|
||||||
|
Pos=804,20
|
||||||
|
Size=220,300
|
||||||
|
|
||||||
@@ -15,9 +15,10 @@
|
|||||||
#include "font_atlas.h"
|
#include "font_atlas.h"
|
||||||
#include "test_sprite.h"
|
#include "test_sprite.h"
|
||||||
|
|
||||||
#ifdef USE_FRAMEBUFFER
|
#ifdef TARGET_IMX
|
||||||
#include "FBDisplay.h"
|
#include "FBDisplay.h"
|
||||||
#else
|
#endif
|
||||||
|
#ifdef TARGET_PC
|
||||||
#include "SDLDisplay.h"
|
#include "SDLDisplay.h"
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
@@ -47,9 +48,10 @@ namespace
|
|||||||
|
|
||||||
static Platform::IDisplay* CreateDisplay()
|
static Platform::IDisplay* CreateDisplay()
|
||||||
{
|
{
|
||||||
#ifdef USE_FRAMEBUFFER
|
#ifdef TARGET_IMX
|
||||||
return new Platform::FBDisplay();
|
return new Platform::FBDisplay();
|
||||||
#else
|
#endif
|
||||||
|
#ifdef TARGET_PC
|
||||||
return new Platform::SDLDisplay();
|
return new Platform::SDLDisplay();
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,101 @@
|
|||||||
|
set(DESKTOP_TARGET IMX6U-Desktop)
|
||||||
|
set(DESKTOP_ATLAS_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/generated/desktop_atlas.h")
|
||||||
|
set_source_files_properties(${DESKTOP_ATLAS_HEADER} PROPERTIES GENERATED TRUE)
|
||||||
|
|
||||||
|
add_executable(${DESKTOP_TARGET}
|
||||||
|
Main.cpp
|
||||||
|
src/app/DesktopApp.cpp
|
||||||
|
src/ui/GameCatalog.cpp
|
||||||
|
src/ui/GameCarousel.cpp
|
||||||
|
src/ui/GameLauncher.cpp
|
||||||
|
${DESKTOP_ATLAS_HEADER}
|
||||||
|
)
|
||||||
|
|
||||||
|
set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/src/ui/GameCatalog.cpp
|
||||||
|
PROPERTIES OBJECT_DEPENDS "${DESKTOP_ATLAS_HEADER}"
|
||||||
|
)
|
||||||
|
|
||||||
|
target_include_directories(${DESKTOP_TARGET} PRIVATE
|
||||||
|
${CMAKE_CURRENT_SOURCE_DIR}/src
|
||||||
|
${CMAKE_CURRENT_SOURCE_DIR}/src/app
|
||||||
|
${CMAKE_CURRENT_SOURCE_DIR}/src/ui
|
||||||
|
${CMAKE_CURRENT_SOURCE_DIR}/generated
|
||||||
|
)
|
||||||
|
|
||||||
|
imx6u_configure_app_target(${DESKTOP_TARGET})
|
||||||
|
|
||||||
|
if(CMAKE_CROSSCOMPILING AND NOT EXISTS "${DESKTOP_ATLAS_HEADER}")
|
||||||
|
message(FATAL_ERROR
|
||||||
|
"Desktop atlas header is missing. Run GenerateDesktopAtlasHeader in a host build before cross compiling."
|
||||||
|
)
|
||||||
|
endif()
|
||||||
|
|
||||||
|
if(NOT TARGET_IMX)
|
||||||
|
set(DESKTOP_ATLAS_TOOL_SOURCES
|
||||||
|
tools/asset_pipeline/DesktopAssetTool.cpp
|
||||||
|
)
|
||||||
|
|
||||||
|
if(WIN32)
|
||||||
|
add_executable(DesktopAtlasTool EXCLUDE_FROM_ALL ${DESKTOP_ATLAS_TOOL_SOURCES})
|
||||||
|
target_include_directories(DesktopAtlasTool PRIVATE
|
||||||
|
${PROJECT_SOURCE_DIR}/libs/Win/SDL2/include
|
||||||
|
${PROJECT_SOURCE_DIR}/libs/Win/SDL_image/include
|
||||||
|
)
|
||||||
|
target_link_directories(DesktopAtlasTool PRIVATE
|
||||||
|
${SDL2_LIB_DIR}
|
||||||
|
${SDL2_IMAGE_LIB_DIR}
|
||||||
|
)
|
||||||
|
target_link_libraries(DesktopAtlasTool PRIVATE SDL2main SDL2 SDL2_image)
|
||||||
|
|
||||||
|
add_custom_command(TARGET DesktopAtlasTool POST_BUILD
|
||||||
|
COMMAND ${CMAKE_COMMAND} -E copy_if_different
|
||||||
|
"${SDL2_DLL}"
|
||||||
|
"$<TARGET_FILE_DIR:DesktopAtlasTool>"
|
||||||
|
)
|
||||||
|
add_custom_command(TARGET DesktopAtlasTool POST_BUILD
|
||||||
|
COMMAND ${CMAKE_COMMAND} -E copy_if_different
|
||||||
|
"${SDL2_IMAGE_DLL}"
|
||||||
|
"$<TARGET_FILE_DIR:DesktopAtlasTool>"
|
||||||
|
)
|
||||||
|
elseif(SDL2_image_FOUND)
|
||||||
|
add_executable(DesktopAtlasTool EXCLUDE_FROM_ALL ${DESKTOP_ATLAS_TOOL_SOURCES})
|
||||||
|
target_include_directories(DesktopAtlasTool PRIVATE
|
||||||
|
${SDL2_image_INCLUDE_DIRS}
|
||||||
|
${SDL2_IMAGE_INCLUDE_DIRS}
|
||||||
|
)
|
||||||
|
target_link_libraries(DesktopAtlasTool PRIVATE SDL2::SDL2)
|
||||||
|
|
||||||
|
if(TARGET SDL2_image::SDL2_image)
|
||||||
|
target_link_libraries(DesktopAtlasTool PRIVATE SDL2_image::SDL2_image)
|
||||||
|
else()
|
||||||
|
target_link_libraries(DesktopAtlasTool PRIVATE
|
||||||
|
${SDL2_image_LIBRARIES}
|
||||||
|
${SDL2_IMAGE_LIBRARIES}
|
||||||
|
)
|
||||||
|
endif()
|
||||||
|
else()
|
||||||
|
message(STATUS "DesktopAtlasTool disabled: SDL2_image was not found")
|
||||||
|
endif()
|
||||||
|
|
||||||
|
if(TARGET DesktopAtlasTool)
|
||||||
|
if(NOT CMAKE_CROSSCOMPILING)
|
||||||
|
add_custom_target(GenerateDesktopAtlasHeader
|
||||||
|
COMMAND ${CMAKE_COMMAND} -E make_directory
|
||||||
|
"src/Apps/Desktop/generated"
|
||||||
|
COMMAND $<TARGET_FILE:DesktopAtlasTool>
|
||||||
|
WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}"
|
||||||
|
DEPENDS DesktopAtlasTool
|
||||||
|
COMMENT "Generating desktop atlas header from PNG assets"
|
||||||
|
VERBATIM
|
||||||
|
)
|
||||||
|
|
||||||
|
add_dependencies(${DESKTOP_TARGET} GenerateDesktopAtlasHeader)
|
||||||
|
else()
|
||||||
|
add_custom_target(GenerateDesktopAtlasHeader
|
||||||
|
COMMAND ${CMAKE_COMMAND} -E echo
|
||||||
|
"GenerateDesktopAtlasHeader is host-only. Run it in build-win or build-linux before cross compiling."
|
||||||
|
VERBATIM
|
||||||
|
)
|
||||||
|
endif()
|
||||||
|
endif()
|
||||||
|
endif()
|
||||||
@@ -0,0 +1,150 @@
|
|||||||
|
#include <chrono>
|
||||||
|
#include <cstdint>
|
||||||
|
#include <iostream>
|
||||||
|
#include <thread>
|
||||||
|
|
||||||
|
#include "DefaultHardware.h"
|
||||||
|
#include "DrawContext.h"
|
||||||
|
#include "IDisplay.h"
|
||||||
|
#include "TimeSource.h"
|
||||||
|
#include "Timer.h"
|
||||||
|
#include "app/DesktopApp.h"
|
||||||
|
#include "ui/GameLauncher.h"
|
||||||
|
|
||||||
|
#ifdef TARGET_IMX
|
||||||
|
#include "FBDisplay.h"
|
||||||
|
#endif
|
||||||
|
#ifdef TARGET_PC
|
||||||
|
#include "SDLDisplay.h"
|
||||||
|
#endif
|
||||||
|
|
||||||
|
namespace
|
||||||
|
{
|
||||||
|
const int32_t ScreenWidth = 1024;
|
||||||
|
const int32_t ScreenHeight = 600;
|
||||||
|
const int32_t RestoreRetryCount = 5;
|
||||||
|
const uint32_t RestoreRetryDelayMs = 300;
|
||||||
|
|
||||||
|
static Platform::IDisplay* CreateDisplay()
|
||||||
|
{
|
||||||
|
#ifdef TARGET_IMX
|
||||||
|
return new Platform::FBDisplay();
|
||||||
|
#endif
|
||||||
|
#ifdef TARGET_PC
|
||||||
|
return new Platform::SDLDisplay();
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
static void SleepRemainingFrameTime(const Core::Timer& timer, const Platform::ITimeSource& timeSource)
|
||||||
|
{
|
||||||
|
const uint32_t sleepMs = timer.remaining_frame_ms(timeSource.get_time_ms());
|
||||||
|
if (sleepMs > 0u)
|
||||||
|
{
|
||||||
|
std::this_thread::sleep_for(std::chrono::milliseconds(sleepMs));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool RestoreDesktopHardware(
|
||||||
|
Platform::IDisplay* display,
|
||||||
|
Platform::DefaultPointerInput& pointerInput)
|
||||||
|
{
|
||||||
|
for (int32_t attempt = 1; attempt <= RestoreRetryCount; ++attempt)
|
||||||
|
{
|
||||||
|
if (display->init(ScreenWidth, ScreenHeight))
|
||||||
|
{
|
||||||
|
if (!pointerInput.init("", ScreenWidth, ScreenHeight))
|
||||||
|
{
|
||||||
|
std::cerr << "[WARN] Pointer input restore failed; desktop touch is disabled." << std::endl;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::cerr << "[WARN] Failed to restore desktop display, retry "
|
||||||
|
<< attempt << "/" << RestoreRetryCount << "." << std::endl;
|
||||||
|
std::this_thread::sleep_for(std::chrono::milliseconds(RestoreRetryDelayMs));
|
||||||
|
}
|
||||||
|
|
||||||
|
std::cerr << "[ERROR] Failed to restore desktop display after retries." << std::endl;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool LaunchGameAndRestoreDesktop(
|
||||||
|
const Desktop::GameEntry& game,
|
||||||
|
Desktop::GameLauncher& launcher,
|
||||||
|
Platform::IDisplay* display,
|
||||||
|
Platform::DefaultPointerInput& pointerInput)
|
||||||
|
{
|
||||||
|
pointerInput.shutdown();
|
||||||
|
display->shutdown();
|
||||||
|
|
||||||
|
const bool launched = launcher.launch_and_wait(game);
|
||||||
|
if (!launched)
|
||||||
|
{
|
||||||
|
std::cerr << "[WARN] Game launch failed: " << (game.title ? game.title : "(unnamed)") << std::endl;
|
||||||
|
}
|
||||||
|
|
||||||
|
return RestoreDesktopHardware(display, pointerInput);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int argc, char* argv[])
|
||||||
|
{
|
||||||
|
(void)argc;
|
||||||
|
(void)argv;
|
||||||
|
|
||||||
|
Platform::IDisplay* display = CreateDisplay();
|
||||||
|
if (!display->init(ScreenWidth, ScreenHeight))
|
||||||
|
{
|
||||||
|
delete display;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
Platform::DefaultPointerInput pointerInput;
|
||||||
|
if (!pointerInput.init("", ScreenWidth, ScreenHeight))
|
||||||
|
{
|
||||||
|
std::cerr << "[WARN] Pointer input init failed; desktop touch is disabled." << std::endl;
|
||||||
|
}
|
||||||
|
|
||||||
|
Core::DrawContext ctx(ScreenWidth, ScreenHeight);
|
||||||
|
Core::Timer timer(30);
|
||||||
|
Platform::SteadyTimeSource timeSource;
|
||||||
|
Desktop::DesktopApp app(ScreenWidth, ScreenHeight, &pointerInput);
|
||||||
|
Desktop::GameLauncher launcher;
|
||||||
|
|
||||||
|
std::cout << "[INFO] Desktop started. Target FPS: " << timer.target_fps() << std::endl;
|
||||||
|
|
||||||
|
bool isRunning = true;
|
||||||
|
while (isRunning)
|
||||||
|
{
|
||||||
|
timer.begin_frame(timeSource.get_time_ms());
|
||||||
|
|
||||||
|
bool shouldQuit = false;
|
||||||
|
display->poll_events(shouldQuit);
|
||||||
|
if (shouldQuit)
|
||||||
|
{
|
||||||
|
isRunning = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
app.update(timer.fixed_delta_ms());
|
||||||
|
const Desktop::GameEntry* launchGame = app.consume_pending_launch();
|
||||||
|
if (launchGame != nullptr)
|
||||||
|
{
|
||||||
|
if (!LaunchGameAndRestoreDesktop(*launchGame, launcher, display, pointerInput))
|
||||||
|
{
|
||||||
|
isRunning = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
timer.begin_frame(timeSource.get_time_ms());
|
||||||
|
}
|
||||||
|
|
||||||
|
app.draw(ctx);
|
||||||
|
ctx.present(display);
|
||||||
|
SleepRemainingFrameTime(timer, timeSource);
|
||||||
|
}
|
||||||
|
|
||||||
|
pointerInput.shutdown();
|
||||||
|
display->shutdown();
|
||||||
|
delete display;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
IMX6U Desktop
|
||||||
|
=============
|
||||||
|
|
||||||
|
Game-console desktop application for the 1024x600 screen.
|
||||||
|
|
||||||
|
Interaction:
|
||||||
|
- Tap the left or right screen edge to switch games.
|
||||||
|
- Swipe horizontally to switch games.
|
||||||
|
- Tap the centered cover card to launch the selected game.
|
||||||
|
|
||||||
|
Adding a game:
|
||||||
|
1. Add a PNG cover to `assets/raw`.
|
||||||
|
2. Add it to `DesktopAssetTool.cpp`.
|
||||||
|
3. Add a `GameEntry` in `src/ui/GameCatalog.cpp`.
|
||||||
|
4. Rebuild the Desktop target to regenerate `generated/desktop_atlas.h`.
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 4.7 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.5 MiB |
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,58 @@
|
|||||||
|
#include "DesktopApp.h"
|
||||||
|
|
||||||
|
#include <iostream>
|
||||||
|
|
||||||
|
#include "DrawContext.h"
|
||||||
|
#include "PointerInput.h"
|
||||||
|
#include "font_atlas.h"
|
||||||
|
|
||||||
|
namespace Desktop
|
||||||
|
{
|
||||||
|
DesktopApp::DesktopApp(int32_t width, int32_t height, Platform::IPointerInput* input)
|
||||||
|
: pointerInput(input),
|
||||||
|
screenWidth(width),
|
||||||
|
screenHeight(height),
|
||||||
|
pendingLaunchGame(nullptr)
|
||||||
|
{
|
||||||
|
font.mask_bits = font_atlas_mask;
|
||||||
|
font.atlas_width = font_atlas_width;
|
||||||
|
font.atlas_height = font_atlas_height;
|
||||||
|
font.char_w = font_char_w;
|
||||||
|
font.char_h = font_char_h;
|
||||||
|
font.columns = font_columns;
|
||||||
|
font.first_char = font_first_char;
|
||||||
|
|
||||||
|
carousel.configure(GetGameCatalog(), GetGameCatalogCount(), screenWidth, screenHeight);
|
||||||
|
}
|
||||||
|
|
||||||
|
void DesktopApp::update(uint32_t deltaMs)
|
||||||
|
{
|
||||||
|
if (pointerInput != nullptr)
|
||||||
|
{
|
||||||
|
pointerInput->update();
|
||||||
|
}
|
||||||
|
|
||||||
|
const CarouselAction action = carousel.update(deltaMs, pointerInput);
|
||||||
|
if (action == CarouselAction::Launch)
|
||||||
|
{
|
||||||
|
const GameEntry* game = carousel.get_current_game();
|
||||||
|
if (game != nullptr)
|
||||||
|
{
|
||||||
|
std::cout << "[INFO] Launching " << game->title << std::endl;
|
||||||
|
pendingLaunchGame = game;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void DesktopApp::draw(Core::DrawContext& ctx)
|
||||||
|
{
|
||||||
|
carousel.draw(ctx, font);
|
||||||
|
}
|
||||||
|
|
||||||
|
const GameEntry* DesktopApp::consume_pending_launch()
|
||||||
|
{
|
||||||
|
const GameEntry* game = pendingLaunchGame;
|
||||||
|
pendingLaunchGame = nullptr;
|
||||||
|
return game;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <cstdint>
|
||||||
|
|
||||||
|
#include "BitmapFont.h"
|
||||||
|
#include "GameCatalog.h"
|
||||||
|
#include "GameCarousel.h"
|
||||||
|
|
||||||
|
namespace Core
|
||||||
|
{
|
||||||
|
class DrawContext;
|
||||||
|
}
|
||||||
|
|
||||||
|
namespace Platform
|
||||||
|
{
|
||||||
|
class IPointerInput;
|
||||||
|
}
|
||||||
|
|
||||||
|
namespace Desktop
|
||||||
|
{
|
||||||
|
class DesktopApp
|
||||||
|
{
|
||||||
|
private:
|
||||||
|
Platform::IPointerInput* pointerInput;
|
||||||
|
GameCarousel carousel;
|
||||||
|
RenderData::BitmapFont font;
|
||||||
|
int32_t screenWidth;
|
||||||
|
int32_t screenHeight;
|
||||||
|
const GameEntry* pendingLaunchGame;
|
||||||
|
|
||||||
|
public:
|
||||||
|
DesktopApp(int32_t width, int32_t height, Platform::IPointerInput* pointerInput);
|
||||||
|
|
||||||
|
void update(uint32_t deltaMs);
|
||||||
|
void draw(Core::DrawContext& ctx);
|
||||||
|
const GameEntry* consume_pending_launch();
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,333 @@
|
|||||||
|
#include "GameCarousel.h"
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
|
||||||
|
#include "Color.h"
|
||||||
|
#include "DrawContext.h"
|
||||||
|
#include "PointerInput.h"
|
||||||
|
|
||||||
|
namespace Desktop
|
||||||
|
{
|
||||||
|
namespace
|
||||||
|
{
|
||||||
|
const int32_t EdgeHotZone = 120;
|
||||||
|
const int32_t SwipeThreshold = 120;
|
||||||
|
const int32_t TapSlop = 18;
|
||||||
|
const uint32_t SlideDurationMs = 220;
|
||||||
|
|
||||||
|
static int32_t AbsInt(int32_t value)
|
||||||
|
{
|
||||||
|
return value < 0 ? -value : value;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int32_t LerpInt(int32_t from, int32_t to, uint32_t elapsed, uint32_t duration)
|
||||||
|
{
|
||||||
|
if (duration == 0 || elapsed >= duration)
|
||||||
|
{
|
||||||
|
return to;
|
||||||
|
}
|
||||||
|
const int32_t delta = to - from;
|
||||||
|
return from + static_cast<int32_t>((static_cast<int64_t>(delta) * static_cast<int64_t>(elapsed)) / static_cast<int64_t>(duration));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
GameCarousel::GameCarousel()
|
||||||
|
: games(nullptr),
|
||||||
|
gameCount(0),
|
||||||
|
currentIndex(0),
|
||||||
|
targetIndex(0),
|
||||||
|
screenWidth(0),
|
||||||
|
screenHeight(0),
|
||||||
|
dragStartX(0),
|
||||||
|
dragCurrentX(0),
|
||||||
|
dragStartY(0),
|
||||||
|
dragging(false),
|
||||||
|
pointerWasDown(false),
|
||||||
|
animationFromOffset(0),
|
||||||
|
animationToOffset(0),
|
||||||
|
animationElapsedMs(0),
|
||||||
|
animationDurationMs(SlideDurationMs)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
void GameCarousel::configure(const GameEntry* catalog, size_t catalogCount, int32_t width, int32_t height)
|
||||||
|
{
|
||||||
|
games = catalog;
|
||||||
|
gameCount = catalogCount;
|
||||||
|
screenWidth = width;
|
||||||
|
screenHeight = height;
|
||||||
|
currentIndex = 0;
|
||||||
|
targetIndex = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t GameCarousel::previous_index(size_t index) const
|
||||||
|
{
|
||||||
|
if (gameCount == 0)
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return index == 0 ? gameCount - 1 : index - 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t GameCarousel::next_index(size_t index) const
|
||||||
|
{
|
||||||
|
return gameCount == 0 ? 0 : (index + 1) % gameCount;
|
||||||
|
}
|
||||||
|
|
||||||
|
int32_t GameCarousel::get_card_width() const
|
||||||
|
{
|
||||||
|
return screenWidth - 144;
|
||||||
|
}
|
||||||
|
|
||||||
|
int32_t GameCarousel::get_card_height() const
|
||||||
|
{
|
||||||
|
return screenHeight - 156;
|
||||||
|
}
|
||||||
|
|
||||||
|
int32_t GameCarousel::get_card_x() const
|
||||||
|
{
|
||||||
|
return (screenWidth - get_card_width()) / 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
int32_t GameCarousel::get_card_y() const
|
||||||
|
{
|
||||||
|
return 70;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool GameCarousel::is_animating() const
|
||||||
|
{
|
||||||
|
return animationElapsedMs < animationDurationMs && currentIndex != targetIndex;
|
||||||
|
}
|
||||||
|
|
||||||
|
int32_t GameCarousel::get_current_offset() const
|
||||||
|
{
|
||||||
|
if (dragging)
|
||||||
|
{
|
||||||
|
return dragCurrentX - dragStartX;
|
||||||
|
}
|
||||||
|
if (is_animating())
|
||||||
|
{
|
||||||
|
return LerpInt(animationFromOffset, animationToOffset, animationElapsedMs, animationDurationMs);
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool GameCarousel::is_inside_card(int32_t x, int32_t y) const
|
||||||
|
{
|
||||||
|
const int32_t cardX = get_card_x();
|
||||||
|
const int32_t cardY = get_card_y();
|
||||||
|
return x >= cardX && x < cardX + get_card_width() && y >= cardY && y < cardY + get_card_height();
|
||||||
|
}
|
||||||
|
|
||||||
|
void GameCarousel::begin_slide(size_t index, int32_t direction)
|
||||||
|
{
|
||||||
|
if (gameCount <= 1 || index == currentIndex)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
targetIndex = index;
|
||||||
|
animationFromOffset = direction * screenWidth;
|
||||||
|
animationToOffset = 0;
|
||||||
|
animationElapsedMs = 0;
|
||||||
|
animationDurationMs = SlideDurationMs;
|
||||||
|
}
|
||||||
|
|
||||||
|
CarouselAction GameCarousel::update(uint32_t deltaMs, Platform::IPointerInput* pointerInput)
|
||||||
|
{
|
||||||
|
if (is_animating())
|
||||||
|
{
|
||||||
|
animationElapsedMs += deltaMs;
|
||||||
|
if (animationElapsedMs >= animationDurationMs)
|
||||||
|
{
|
||||||
|
currentIndex = targetIndex;
|
||||||
|
animationElapsedMs = animationDurationMs;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (gameCount == 0 || pointerInput == nullptr)
|
||||||
|
{
|
||||||
|
return CarouselAction::None;
|
||||||
|
}
|
||||||
|
|
||||||
|
const bool down = pointerInput->is_down();
|
||||||
|
const int32_t x = pointerInput->get_x();
|
||||||
|
const int32_t y = pointerInput->get_y();
|
||||||
|
|
||||||
|
if (down && !pointerWasDown && !is_animating())
|
||||||
|
{
|
||||||
|
dragging = true;
|
||||||
|
dragStartX = x;
|
||||||
|
dragCurrentX = x;
|
||||||
|
dragStartY = y;
|
||||||
|
}
|
||||||
|
else if (down && dragging)
|
||||||
|
{
|
||||||
|
dragCurrentX = x;
|
||||||
|
}
|
||||||
|
else if (!down && pointerWasDown && dragging)
|
||||||
|
{
|
||||||
|
const int32_t dx = x - dragStartX;
|
||||||
|
const int32_t dy = y - dragStartY;
|
||||||
|
dragging = false;
|
||||||
|
|
||||||
|
if (is_animating())
|
||||||
|
{
|
||||||
|
pointerWasDown = down;
|
||||||
|
return CarouselAction::None;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (AbsInt(dx) > SwipeThreshold && AbsInt(dx) > AbsInt(dy))
|
||||||
|
{
|
||||||
|
begin_slide(dx < 0 ? next_index(currentIndex) : previous_index(currentIndex), dx < 0 ? 1 : -1);
|
||||||
|
}
|
||||||
|
else if (AbsInt(dx) <= TapSlop && AbsInt(dy) <= TapSlop)
|
||||||
|
{
|
||||||
|
if (dragStartX < EdgeHotZone)
|
||||||
|
{
|
||||||
|
begin_slide(previous_index(currentIndex), -1);
|
||||||
|
}
|
||||||
|
else if (dragStartX > screenWidth - EdgeHotZone)
|
||||||
|
{
|
||||||
|
begin_slide(next_index(currentIndex), 1);
|
||||||
|
}
|
||||||
|
else if (is_inside_card(dragStartX, dragStartY))
|
||||||
|
{
|
||||||
|
pointerWasDown = down;
|
||||||
|
return CarouselAction::Launch;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pointerWasDown = down;
|
||||||
|
return CarouselAction::None;
|
||||||
|
}
|
||||||
|
|
||||||
|
void GameCarousel::draw_card(Core::DrawContext& ctx, const GameEntry& game, int32_t x, int32_t y, bool active) const
|
||||||
|
{
|
||||||
|
const int32_t cardW = get_card_width();
|
||||||
|
const int32_t cardH = get_card_height();
|
||||||
|
const RenderData::Color shadow(0, 0, 0, 255);
|
||||||
|
const RenderData::Color border = active ? RenderData::Color(250, 230, 145, 255) : RenderData::Color(74, 82, 105, 255);
|
||||||
|
const RenderData::Color panel(20, 24, 36, 255);
|
||||||
|
|
||||||
|
ctx.fill_rect(x + 12, y + 14, cardW, cardH, shadow);
|
||||||
|
ctx.fill_rect(x - 4, y - 4, cardW + 8, cardH + 8, border);
|
||||||
|
ctx.fill_rect(x, y, cardW, cardH, panel);
|
||||||
|
|
||||||
|
if (game.cover != nullptr && game.cover->atlas != nullptr)
|
||||||
|
{
|
||||||
|
const int32_t coverX = x + (cardW - game.cover->width) / 2;
|
||||||
|
const int32_t coverY = y + (cardH - game.cover->height) / 2;
|
||||||
|
ctx.draw_sprite(coverX, coverY, *game.cover);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
ctx.fill_rect(x + 28, y + 28, cardW - 56, cardH - 56, RenderData::Color(48, 56, 82, 255));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void GameCarousel::draw_text_centered(Core::DrawContext& ctx, const RenderData::BitmapFont& font, int32_t y, const char* text) const
|
||||||
|
{
|
||||||
|
if (text == nullptr)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
int32_t len = 0;
|
||||||
|
for (const char* p = text; *p; ++p)
|
||||||
|
{
|
||||||
|
++len;
|
||||||
|
}
|
||||||
|
|
||||||
|
const int32_t x = (screenWidth - len * font.char_w) / 2;
|
||||||
|
ctx.draw_text(font, x, y, RenderData::Color(245, 248, 255, 255), text);
|
||||||
|
}
|
||||||
|
|
||||||
|
void GameCarousel::draw_edge_hints(Core::DrawContext& ctx) const
|
||||||
|
{
|
||||||
|
ctx.fill_rect(0, 0, 42, screenHeight, RenderData::Color(9, 12, 20, 255));
|
||||||
|
ctx.fill_rect(screenWidth - 42, 0, 42, screenHeight, RenderData::Color(9, 12, 20, 255));
|
||||||
|
|
||||||
|
const int32_t cy = screenHeight / 2;
|
||||||
|
for (int32_t i = 0; i < 24; ++i)
|
||||||
|
{
|
||||||
|
ctx.fill_rect(24 - i / 2, cy - 24 + i, 3, 3, RenderData::Color(154, 168, 210, 255));
|
||||||
|
ctx.fill_rect(screenWidth - 27 + i / 2, cy - 24 + i, 3, 3, RenderData::Color(154, 168, 210, 255));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void GameCarousel::draw_page_dots(Core::DrawContext& ctx) const
|
||||||
|
{
|
||||||
|
if (gameCount == 0)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const int32_t dotSize = 10;
|
||||||
|
const int32_t gap = 18;
|
||||||
|
const int32_t totalW = static_cast<int32_t>(gameCount) * dotSize + static_cast<int32_t>(gameCount - 1) * gap;
|
||||||
|
const int32_t startX = (screenWidth - totalW) / 2;
|
||||||
|
const int32_t y = screenHeight - 42;
|
||||||
|
|
||||||
|
for (size_t i = 0; i < gameCount; ++i)
|
||||||
|
{
|
||||||
|
const bool active = i == currentIndex || (is_animating() && i == targetIndex);
|
||||||
|
const RenderData::Color color = active ? RenderData::Color(250, 230, 145, 255) : RenderData::Color(90, 100, 128, 255);
|
||||||
|
ctx.fill_rect(startX + static_cast<int32_t>(i) * (dotSize + gap), y, dotSize, dotSize, color);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void GameCarousel::draw(Core::DrawContext& ctx, const RenderData::BitmapFont& font) const
|
||||||
|
{
|
||||||
|
ctx.clear_color(RenderData::Color(10, 14, 24, 255));
|
||||||
|
ctx.fill_rect(0, 0, screenWidth, 56, RenderData::Color(16, 22, 35, 255));
|
||||||
|
draw_text_centered(ctx, font, 20, "IMX6U GAME DESKTOP");
|
||||||
|
|
||||||
|
if (gameCount == 0)
|
||||||
|
{
|
||||||
|
draw_text_centered(ctx, font, screenHeight / 2, "NO GAMES");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
draw_edge_hints(ctx);
|
||||||
|
|
||||||
|
const int32_t baseX = get_card_x();
|
||||||
|
const int32_t baseY = get_card_y();
|
||||||
|
const int32_t offset = get_current_offset();
|
||||||
|
|
||||||
|
if (is_animating())
|
||||||
|
{
|
||||||
|
const int32_t currentOffset = animationFromOffset > 0 ? offset - screenWidth : offset + screenWidth;
|
||||||
|
draw_card(ctx, games[currentIndex], baseX + currentOffset, baseY, false);
|
||||||
|
draw_card(ctx, games[targetIndex], baseX + offset, baseY, true);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (dragging && gameCount > 1)
|
||||||
|
{
|
||||||
|
if (offset > 0)
|
||||||
|
{
|
||||||
|
draw_card(ctx, games[previous_index(currentIndex)], baseX + offset - screenWidth, baseY, false);
|
||||||
|
}
|
||||||
|
else if (offset < 0)
|
||||||
|
{
|
||||||
|
draw_card(ctx, games[next_index(currentIndex)], baseX + offset + screenWidth, baseY, false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
draw_card(ctx, games[currentIndex], baseX + offset, baseY, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
draw_text_centered(ctx, font, screenHeight - 78, games[currentIndex].title);
|
||||||
|
draw_page_dots(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
const GameEntry* GameCarousel::get_current_game() const
|
||||||
|
{
|
||||||
|
if (games == nullptr || gameCount == 0)
|
||||||
|
{
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
return &games[currentIndex];
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <cstddef>
|
||||||
|
#include <cstdint>
|
||||||
|
|
||||||
|
#include "BitmapFont.h"
|
||||||
|
#include "GameCatalog.h"
|
||||||
|
|
||||||
|
namespace Core
|
||||||
|
{
|
||||||
|
class DrawContext;
|
||||||
|
}
|
||||||
|
|
||||||
|
namespace Platform
|
||||||
|
{
|
||||||
|
class IPointerInput;
|
||||||
|
}
|
||||||
|
|
||||||
|
namespace Desktop
|
||||||
|
{
|
||||||
|
enum class CarouselAction
|
||||||
|
{
|
||||||
|
None,
|
||||||
|
Launch
|
||||||
|
};
|
||||||
|
|
||||||
|
class GameCarousel
|
||||||
|
{
|
||||||
|
private:
|
||||||
|
const GameEntry* games;
|
||||||
|
size_t gameCount;
|
||||||
|
size_t currentIndex;
|
||||||
|
size_t targetIndex;
|
||||||
|
int32_t screenWidth;
|
||||||
|
int32_t screenHeight;
|
||||||
|
int32_t dragStartX;
|
||||||
|
int32_t dragCurrentX;
|
||||||
|
int32_t dragStartY;
|
||||||
|
bool dragging;
|
||||||
|
bool pointerWasDown;
|
||||||
|
int32_t animationFromOffset;
|
||||||
|
int32_t animationToOffset;
|
||||||
|
uint32_t animationElapsedMs;
|
||||||
|
uint32_t animationDurationMs;
|
||||||
|
|
||||||
|
size_t previous_index(size_t index) const;
|
||||||
|
size_t next_index(size_t index) const;
|
||||||
|
int32_t get_card_width() const;
|
||||||
|
int32_t get_card_height() const;
|
||||||
|
int32_t get_card_x() const;
|
||||||
|
int32_t get_card_y() const;
|
||||||
|
int32_t get_current_offset() const;
|
||||||
|
bool is_animating() const;
|
||||||
|
bool is_inside_card(int32_t x, int32_t y) const;
|
||||||
|
void begin_slide(size_t index, int32_t direction);
|
||||||
|
void draw_card(Core::DrawContext& ctx, const GameEntry& game, int32_t x, int32_t y, bool active) const;
|
||||||
|
void draw_text_centered(Core::DrawContext& ctx, const RenderData::BitmapFont& font, int32_t y, const char* text) const;
|
||||||
|
void draw_edge_hints(Core::DrawContext& ctx) const;
|
||||||
|
void draw_page_dots(Core::DrawContext& ctx) const;
|
||||||
|
|
||||||
|
public:
|
||||||
|
GameCarousel();
|
||||||
|
|
||||||
|
void configure(const GameEntry* catalog, size_t catalogCount, int32_t width, int32_t height);
|
||||||
|
CarouselAction update(uint32_t deltaMs, Platform::IPointerInput* pointerInput);
|
||||||
|
void draw(Core::DrawContext& ctx, const RenderData::BitmapFont& font) const;
|
||||||
|
const GameEntry* get_current_game() const;
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
#include "GameCatalog.h"
|
||||||
|
|
||||||
|
#include "desktop_atlas.h"
|
||||||
|
|
||||||
|
namespace Desktop
|
||||||
|
{
|
||||||
|
namespace
|
||||||
|
{
|
||||||
|
const GameEntry Games[] = {
|
||||||
|
{ "tom", "TomGame", &DesktopAtlas::TomGame, "IMX6U-Game", true },
|
||||||
|
{ "light", "LightGame", &DesktopAtlas::LightGame, "IMX6U-LightGame", true },
|
||||||
|
{ "demo", "Demo", &DesktopAtlas::Demo, "IMX6U-Demo", true },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const GameEntry* GetGameCatalog()
|
||||||
|
{
|
||||||
|
return Games;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t GetGameCatalogCount()
|
||||||
|
{
|
||||||
|
return sizeof(Games) / sizeof(Games[0]);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <cstddef>
|
||||||
|
|
||||||
|
#include "Sprite.h"
|
||||||
|
|
||||||
|
namespace Desktop
|
||||||
|
{
|
||||||
|
struct GameEntry
|
||||||
|
{
|
||||||
|
const char* id;
|
||||||
|
const char* title;
|
||||||
|
const RenderData::Sprite* cover;
|
||||||
|
const char* executable;
|
||||||
|
bool enabled;
|
||||||
|
};
|
||||||
|
|
||||||
|
const GameEntry* GetGameCatalog();
|
||||||
|
size_t GetGameCatalogCount();
|
||||||
|
}
|
||||||
@@ -0,0 +1,135 @@
|
|||||||
|
#include "GameLauncher.h"
|
||||||
|
|
||||||
|
#include <cstdlib>
|
||||||
|
#include <iostream>
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#ifdef _WIN32
|
||||||
|
#include <direct.h>
|
||||||
|
#include <windows.h>
|
||||||
|
#include <process.h>
|
||||||
|
#else
|
||||||
|
#include <errno.h>
|
||||||
|
#include <limits.h>
|
||||||
|
#include <sys/wait.h>
|
||||||
|
#include <sys/types.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
|
namespace Desktop
|
||||||
|
{
|
||||||
|
namespace
|
||||||
|
{
|
||||||
|
static bool HasExeExtension(const std::string& command)
|
||||||
|
{
|
||||||
|
if (command.size() < 4)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const size_t offset = command.size() - 4;
|
||||||
|
return (command[offset] == '.') &&
|
||||||
|
(command[offset + 1] == 'e' || command[offset + 1] == 'E') &&
|
||||||
|
(command[offset + 2] == 'x' || command[offset + 2] == 'X') &&
|
||||||
|
(command[offset + 3] == 'e' || command[offset + 3] == 'E');
|
||||||
|
}
|
||||||
|
|
||||||
|
static std::string BuildExecutableName(const char* executable)
|
||||||
|
{
|
||||||
|
std::string command = executable ? executable : "";
|
||||||
|
#ifdef _WIN32
|
||||||
|
if (!HasExeExtension(command))
|
||||||
|
{
|
||||||
|
command += ".exe";
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
return command;
|
||||||
|
}
|
||||||
|
|
||||||
|
static std::string GetExecutableDirectory()
|
||||||
|
{
|
||||||
|
#ifdef _WIN32
|
||||||
|
char path[MAX_PATH];
|
||||||
|
const DWORD length = GetModuleFileNameA(nullptr, path, MAX_PATH);
|
||||||
|
if (length == 0 || length >= MAX_PATH)
|
||||||
|
{
|
||||||
|
char cwd[1024];
|
||||||
|
return _getcwd(cwd, sizeof(cwd)) == nullptr ? std::string(".") : std::string(cwd);
|
||||||
|
}
|
||||||
|
std::string fullPath(path, length);
|
||||||
|
const size_t slash = fullPath.find_last_of("\\/");
|
||||||
|
return slash == std::string::npos ? std::string(".") : fullPath.substr(0, slash);
|
||||||
|
#else
|
||||||
|
char path[PATH_MAX];
|
||||||
|
const ssize_t length = readlink("/proc/self/exe", path, sizeof(path) - 1);
|
||||||
|
if (length <= 0)
|
||||||
|
{
|
||||||
|
char cwd[PATH_MAX];
|
||||||
|
return getcwd(cwd, sizeof(cwd)) == nullptr ? std::string(".") : std::string(cwd);
|
||||||
|
}
|
||||||
|
path[length] = '\0';
|
||||||
|
std::string fullPath(path);
|
||||||
|
const size_t slash = fullPath.find_last_of('/');
|
||||||
|
return slash == std::string::npos ? std::string(".") : fullPath.substr(0, slash);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
static std::string BuildLocalExecutablePath(const std::string& executable)
|
||||||
|
{
|
||||||
|
#ifdef _WIN32
|
||||||
|
return GetExecutableDirectory() + "\\" + executable;
|
||||||
|
#else
|
||||||
|
return GetExecutableDirectory() + "/" + executable;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool GameLauncher::launch_and_wait(const GameEntry& game) const
|
||||||
|
{
|
||||||
|
if (!game.enabled || game.executable == nullptr || game.executable[0] == '\0')
|
||||||
|
{
|
||||||
|
std::cerr << "[WARN] Game is not launchable: " << (game.title ? game.title : "(unnamed)") << std::endl;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::string executable = BuildExecutableName(game.executable);
|
||||||
|
const std::string localPath = BuildLocalExecutablePath(executable);
|
||||||
|
|
||||||
|
#ifdef _WIN32
|
||||||
|
intptr_t result = _spawnl(_P_WAIT, localPath.c_str(), executable.c_str(), nullptr);
|
||||||
|
if (result == -1)
|
||||||
|
{
|
||||||
|
result = _spawnlp(_P_WAIT, executable.c_str(), executable.c_str(), nullptr);
|
||||||
|
}
|
||||||
|
if (result == -1)
|
||||||
|
{
|
||||||
|
std::cerr << "[WARN] Failed to launch " << executable << std::endl;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
#else
|
||||||
|
const pid_t pid = fork();
|
||||||
|
if (pid < 0)
|
||||||
|
{
|
||||||
|
std::cerr << "[WARN] fork failed for " << executable << std::endl;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (pid == 0)
|
||||||
|
{
|
||||||
|
execl(localPath.c_str(), executable.c_str(), static_cast<char*>(nullptr));
|
||||||
|
execlp(executable.c_str(), executable.c_str(), static_cast<char*>(nullptr));
|
||||||
|
std::exit(127);
|
||||||
|
}
|
||||||
|
|
||||||
|
int status = 0;
|
||||||
|
while (waitpid(pid, &status, 0) < 0)
|
||||||
|
{
|
||||||
|
if (errno != EINTR)
|
||||||
|
{
|
||||||
|
std::cerr << "[WARN] waitpid failed for " << executable << std::endl;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "GameCatalog.h"
|
||||||
|
|
||||||
|
namespace Desktop
|
||||||
|
{
|
||||||
|
class GameLauncher
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
bool launch_and_wait(const GameEntry& game) const;
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,439 @@
|
|||||||
|
#include <SDL.h>
|
||||||
|
#include <SDL_image.h>
|
||||||
|
#include <algorithm>
|
||||||
|
#include <cstdint>
|
||||||
|
#include <cstring>
|
||||||
|
#include <fstream>
|
||||||
|
#include <iomanip>
|
||||||
|
#include <iostream>
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
namespace
|
||||||
|
{
|
||||||
|
const char* InputDirectory = "src/Apps/Desktop/assets/raw";
|
||||||
|
const char* OutputHeaderPath = "src/Apps/Desktop/generated/desktop_atlas.h";
|
||||||
|
const int32_t AtlasWidth = 1024;
|
||||||
|
const int32_t AtlasPadding = 1;
|
||||||
|
const int32_t CoverMaxWidth = 860;
|
||||||
|
const int32_t CoverMaxHeight = 420;
|
||||||
|
|
||||||
|
struct SourceSpec
|
||||||
|
{
|
||||||
|
const char* file_name;
|
||||||
|
const char* region_name;
|
||||||
|
bool allow_generated_fallback;
|
||||||
|
};
|
||||||
|
|
||||||
|
const SourceSpec Sources[] = {
|
||||||
|
{ "TomGame.png", "TomGame", false },
|
||||||
|
{ "LightGame.png", "LightGame", false },
|
||||||
|
{ "Demo.png", "Demo", true },
|
||||||
|
};
|
||||||
|
|
||||||
|
struct Image
|
||||||
|
{
|
||||||
|
std::vector<uint16_t> pixels;
|
||||||
|
int32_t width;
|
||||||
|
int32_t height;
|
||||||
|
|
||||||
|
Image() : width(0), height(0) {}
|
||||||
|
|
||||||
|
bool is_valid() const
|
||||||
|
{
|
||||||
|
return width > 0 && height > 0 && !pixels.empty();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
struct AtlasRegion
|
||||||
|
{
|
||||||
|
const SourceSpec* source;
|
||||||
|
Image image;
|
||||||
|
int32_t x;
|
||||||
|
int32_t y;
|
||||||
|
|
||||||
|
AtlasRegion() : source(nullptr), x(0), y(0) {}
|
||||||
|
};
|
||||||
|
|
||||||
|
static std::string JoinPath(const std::string& directory, const std::string& fileName)
|
||||||
|
{
|
||||||
|
if (directory.empty())
|
||||||
|
{
|
||||||
|
return fileName;
|
||||||
|
}
|
||||||
|
const char last = directory[directory.size() - 1];
|
||||||
|
if (last == '/' || last == '\\')
|
||||||
|
{
|
||||||
|
return directory + fileName;
|
||||||
|
}
|
||||||
|
return directory + "/" + fileName;
|
||||||
|
}
|
||||||
|
|
||||||
|
static uint16_t PackRgba5551(uint8_t r, uint8_t g, uint8_t b, uint8_t a)
|
||||||
|
{
|
||||||
|
return static_cast<uint16_t>(
|
||||||
|
((static_cast<uint16_t>(r) >> 3) << 11) |
|
||||||
|
((static_cast<uint16_t>(g) >> 3) << 6) |
|
||||||
|
((static_cast<uint16_t>(b) >> 3) << 1) |
|
||||||
|
(a ? 1u : 0u));
|
||||||
|
}
|
||||||
|
|
||||||
|
static Uint32 ReadSurfacePixel(const SDL_Surface* surface, int32_t x, int32_t y)
|
||||||
|
{
|
||||||
|
const uint8_t* row = static_cast<const uint8_t*>(surface->pixels) + static_cast<size_t>(y) * static_cast<size_t>(surface->pitch);
|
||||||
|
const uint8_t* src = row + static_cast<size_t>(x) * static_cast<size_t>(surface->format->BytesPerPixel);
|
||||||
|
|
||||||
|
switch (surface->format->BytesPerPixel)
|
||||||
|
{
|
||||||
|
case 1:
|
||||||
|
return src[0];
|
||||||
|
case 2:
|
||||||
|
{
|
||||||
|
uint16_t pixel = 0;
|
||||||
|
std::memcpy(&pixel, src, sizeof(pixel));
|
||||||
|
return static_cast<Uint32>(pixel);
|
||||||
|
}
|
||||||
|
case 3:
|
||||||
|
if (SDL_BYTEORDER == SDL_BIG_ENDIAN)
|
||||||
|
{
|
||||||
|
return (static_cast<Uint32>(src[0]) << 16) | (static_cast<Uint32>(src[1]) << 8) | static_cast<Uint32>(src[2]);
|
||||||
|
}
|
||||||
|
return static_cast<Uint32>(src[0]) | (static_cast<Uint32>(src[1]) << 8) | (static_cast<Uint32>(src[2]) << 16);
|
||||||
|
case 4:
|
||||||
|
{
|
||||||
|
Uint32 pixel = 0;
|
||||||
|
std::memcpy(&pixel, src, sizeof(pixel));
|
||||||
|
return pixel;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool LoadPngImage(const std::string& path, Image& image)
|
||||||
|
{
|
||||||
|
SDL_Surface* surface = IMG_Load(path.c_str());
|
||||||
|
if (surface == nullptr)
|
||||||
|
{
|
||||||
|
std::cerr << "IMG_Load failed: " << path << " : " << IMG_GetError() << std::endl;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (SDL_LockSurface(surface) != 0)
|
||||||
|
{
|
||||||
|
std::cerr << "SDL_LockSurface failed: " << SDL_GetError() << std::endl;
|
||||||
|
SDL_FreeSurface(surface);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
image.width = surface->w;
|
||||||
|
image.height = surface->h;
|
||||||
|
image.pixels.assign(static_cast<size_t>(image.width) * static_cast<size_t>(image.height), 0);
|
||||||
|
|
||||||
|
for (int32_t y = 0; y < image.height; ++y)
|
||||||
|
{
|
||||||
|
for (int32_t x = 0; x < image.width; ++x)
|
||||||
|
{
|
||||||
|
uint8_t r = 0;
|
||||||
|
uint8_t g = 0;
|
||||||
|
uint8_t b = 0;
|
||||||
|
uint8_t a = 0;
|
||||||
|
SDL_GetRGBA(ReadSurfacePixel(surface, x, y), surface->format, &r, &g, &b, &a);
|
||||||
|
image.pixels[static_cast<size_t>(y) * static_cast<size_t>(image.width) + static_cast<size_t>(x)] =
|
||||||
|
a == 0 ? 0u : PackRgba5551(r, g, b, a);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
SDL_UnlockSurface(surface);
|
||||||
|
SDL_FreeSurface(surface);
|
||||||
|
return image.is_valid();
|
||||||
|
}
|
||||||
|
|
||||||
|
static Image ResizeNearest(const Image& source, int32_t width, int32_t height)
|
||||||
|
{
|
||||||
|
Image result;
|
||||||
|
if (!source.is_valid() || width <= 0 || height <= 0)
|
||||||
|
{
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
result.width = width;
|
||||||
|
result.height = height;
|
||||||
|
result.pixels.assign(static_cast<size_t>(width) * static_cast<size_t>(height), 0);
|
||||||
|
|
||||||
|
for (int32_t y = 0; y < height; ++y)
|
||||||
|
{
|
||||||
|
const int32_t sourceY = y * source.height / height;
|
||||||
|
for (int32_t x = 0; x < width; ++x)
|
||||||
|
{
|
||||||
|
const int32_t sourceX = x * source.width / width;
|
||||||
|
result.pixels[static_cast<size_t>(y) * static_cast<size_t>(width) + static_cast<size_t>(x)] =
|
||||||
|
source.pixels[static_cast<size_t>(sourceY) * static_cast<size_t>(source.width) + static_cast<size_t>(sourceX)];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
static Image ResizeToFit(const Image& source, int32_t maxWidth, int32_t maxHeight)
|
||||||
|
{
|
||||||
|
if (!source.is_valid() || maxWidth <= 0 || maxHeight <= 0)
|
||||||
|
{
|
||||||
|
return Image();
|
||||||
|
}
|
||||||
|
|
||||||
|
const float scaleX = static_cast<float>(maxWidth) / static_cast<float>(source.width);
|
||||||
|
const float scaleY = static_cast<float>(maxHeight) / static_cast<float>(source.height);
|
||||||
|
const float scale = std::min(scaleX, scaleY);
|
||||||
|
const int32_t width = std::max(1, static_cast<int32_t>(static_cast<float>(source.width) * scale));
|
||||||
|
const int32_t height = std::max(1, static_cast<int32_t>(static_cast<float>(source.height) * scale));
|
||||||
|
return ResizeNearest(source, width, height);
|
||||||
|
}
|
||||||
|
|
||||||
|
static Image GenerateDemoCover()
|
||||||
|
{
|
||||||
|
Image image;
|
||||||
|
image.width = CoverMaxWidth;
|
||||||
|
image.height = CoverMaxHeight;
|
||||||
|
image.pixels.assign(static_cast<size_t>(image.width) * static_cast<size_t>(image.height), 0u);
|
||||||
|
|
||||||
|
for (int32_t y = 0; y < image.height; ++y)
|
||||||
|
{
|
||||||
|
for (int32_t x = 0; x < image.width; ++x)
|
||||||
|
{
|
||||||
|
const uint8_t r = static_cast<uint8_t>(24 + (x * 48) / image.width);
|
||||||
|
const uint8_t g = static_cast<uint8_t>(36 + (y * 72) / image.height);
|
||||||
|
const uint8_t b = static_cast<uint8_t>(80 + ((x + y) * 64) / (image.width + image.height));
|
||||||
|
image.pixels[static_cast<size_t>(y) * static_cast<size_t>(image.width) + static_cast<size_t>(x)] =
|
||||||
|
PackRgba5551(r, g, b, 255);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const int32_t boxX = image.width / 2 - 170;
|
||||||
|
const int32_t boxY = image.height / 2 - 62;
|
||||||
|
for (int32_t y = 0; y < 124; ++y)
|
||||||
|
{
|
||||||
|
for (int32_t x = 0; x < 340; ++x)
|
||||||
|
{
|
||||||
|
const bool border = x < 6 || y < 6 || x >= 334 || y >= 118;
|
||||||
|
const uint16_t pixel = border ? PackRgba5551(250, 230, 145, 255) : PackRgba5551(18, 24, 42, 255);
|
||||||
|
const int32_t dstX = boxX + x;
|
||||||
|
const int32_t dstY = boxY + y;
|
||||||
|
if (dstX >= 0 && dstX < image.width && dstY >= 0 && dstY < image.height)
|
||||||
|
{
|
||||||
|
image.pixels[static_cast<size_t>(dstY) * static_cast<size_t>(image.width) + static_cast<size_t>(dstX)] = pixel;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return image;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool LoadAndResizeSource(const SourceSpec& source, AtlasRegion& region)
|
||||||
|
{
|
||||||
|
region.source = &source;
|
||||||
|
Image image;
|
||||||
|
const std::string inputPath = JoinPath(InputDirectory, source.file_name);
|
||||||
|
if (!LoadPngImage(inputPath, image))
|
||||||
|
{
|
||||||
|
if (!source.allow_generated_fallback)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
region.image = GenerateDemoCover();
|
||||||
|
std::cout << "(generated fallback) -> " << source.region_name << " ("
|
||||||
|
<< region.image.width << "x" << region.image.height << ")" << std::endl;
|
||||||
|
return region.image.is_valid();
|
||||||
|
}
|
||||||
|
|
||||||
|
region.image = ResizeToFit(image, CoverMaxWidth, CoverMaxHeight);
|
||||||
|
if (!region.image.is_valid())
|
||||||
|
{
|
||||||
|
std::cerr << "Resize failed: " << inputPath << std::endl;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::cout << source.file_name << " -> " << source.region_name << " ("
|
||||||
|
<< region.image.width << "x" << region.image.height << ")" << std::endl;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool PackAtlas(std::vector<AtlasRegion>& regions, int32_t& atlasHeight)
|
||||||
|
{
|
||||||
|
int32_t cursorX = 0;
|
||||||
|
int32_t cursorY = 0;
|
||||||
|
int32_t rowHeight = 0;
|
||||||
|
|
||||||
|
for (size_t i = 0; i < regions.size(); ++i)
|
||||||
|
{
|
||||||
|
AtlasRegion& region = regions[i];
|
||||||
|
if (region.image.width > AtlasWidth)
|
||||||
|
{
|
||||||
|
std::cerr << "AtlasWidth is smaller than source image: " << region.source->file_name << std::endl;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (cursorX > 0 && cursorX + region.image.width > AtlasWidth)
|
||||||
|
{
|
||||||
|
cursorY += rowHeight + AtlasPadding;
|
||||||
|
cursorX = 0;
|
||||||
|
rowHeight = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
region.x = cursorX;
|
||||||
|
region.y = cursorY;
|
||||||
|
cursorX += region.image.width + AtlasPadding;
|
||||||
|
rowHeight = std::max(rowHeight, region.image.height);
|
||||||
|
}
|
||||||
|
|
||||||
|
atlasHeight = cursorY + rowHeight;
|
||||||
|
return atlasHeight > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static std::vector<uint16_t> BuildAtlasPixels(const std::vector<AtlasRegion>& regions, int32_t atlasHeight)
|
||||||
|
{
|
||||||
|
std::vector<uint16_t> atlasPixels(static_cast<size_t>(AtlasWidth) * static_cast<size_t>(atlasHeight), 0u);
|
||||||
|
|
||||||
|
for (size_t i = 0; i < regions.size(); ++i)
|
||||||
|
{
|
||||||
|
const AtlasRegion& region = regions[i];
|
||||||
|
for (int32_t y = 0; y < region.image.height; ++y)
|
||||||
|
{
|
||||||
|
const size_t srcOffset = static_cast<size_t>(y) * static_cast<size_t>(region.image.width);
|
||||||
|
const size_t dstOffset = static_cast<size_t>(region.y + y) * static_cast<size_t>(AtlasWidth) + static_cast<size_t>(region.x);
|
||||||
|
std::copy(
|
||||||
|
region.image.pixels.begin() + static_cast<std::vector<uint16_t>::difference_type>(srcOffset),
|
||||||
|
region.image.pixels.begin() + static_cast<std::vector<uint16_t>::difference_type>(srcOffset + static_cast<size_t>(region.image.width)),
|
||||||
|
atlasPixels.begin() + static_cast<std::vector<uint16_t>::difference_type>(dstOffset));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return atlasPixels;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void WritePixel5551(std::ofstream& file, uint16_t pixel)
|
||||||
|
{
|
||||||
|
file << "0x"
|
||||||
|
<< std::uppercase
|
||||||
|
<< std::hex
|
||||||
|
<< std::setw(4)
|
||||||
|
<< std::setfill('0')
|
||||||
|
<< pixel
|
||||||
|
<< std::dec
|
||||||
|
<< std::nouppercase
|
||||||
|
<< std::setfill(' ');
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool WriteAtlasHeader(const std::vector<AtlasRegion>& regions, const std::vector<uint16_t>& atlasPixels, int32_t atlasHeight)
|
||||||
|
{
|
||||||
|
std::ofstream file(OutputHeaderPath, std::ios::binary);
|
||||||
|
if (!file.good())
|
||||||
|
{
|
||||||
|
std::cerr << "Open output header failed: " << OutputHeaderPath << std::endl;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
file << "// Auto-generated by DesktopAssetTool.cpp\n";
|
||||||
|
file << "#pragma once\n";
|
||||||
|
file << "#include <cstdint>\n";
|
||||||
|
file << "#include \"Image.h\"\n";
|
||||||
|
file << "#include \"Sprite.h\"\n\n";
|
||||||
|
file << "namespace DesktopAtlas\n";
|
||||||
|
file << "{\n";
|
||||||
|
file << "\tstatic const int32_t desktop_atlas_width = " << AtlasWidth << ";\n";
|
||||||
|
file << "\tstatic const int32_t desktop_atlas_height = " << atlasHeight << ";\n\n";
|
||||||
|
file << "\tstatic const uint16_t desktop_atlas_pixels[] = {\n";
|
||||||
|
|
||||||
|
for (size_t i = 0; i < atlasPixels.size(); i += 12)
|
||||||
|
{
|
||||||
|
file << "\t\t";
|
||||||
|
const size_t end = std::min(i + 12, atlasPixels.size());
|
||||||
|
for (size_t j = i; j < end; ++j)
|
||||||
|
{
|
||||||
|
if (j > i)
|
||||||
|
{
|
||||||
|
file << " ";
|
||||||
|
}
|
||||||
|
WritePixel5551(file, atlasPixels[j]);
|
||||||
|
if (j + 1 < atlasPixels.size())
|
||||||
|
{
|
||||||
|
file << ",";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
file << "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
file << "\t};\n\n";
|
||||||
|
file << "\tstatic const RenderData::Image image(desktop_atlas_pixels, desktop_atlas_width, desktop_atlas_height, 0x0000, RenderData::PixelFormat::RGBA5551);\n\n";
|
||||||
|
|
||||||
|
for (size_t i = 0; i < regions.size(); ++i)
|
||||||
|
{
|
||||||
|
const AtlasRegion& region = regions[i];
|
||||||
|
file << "\tstatic const RenderData::Sprite "
|
||||||
|
<< region.source->region_name
|
||||||
|
<< "(&image, "
|
||||||
|
<< region.x << ", "
|
||||||
|
<< region.y << ", "
|
||||||
|
<< region.image.width << ", "
|
||||||
|
<< region.image.height << ");\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
file << "}\n";
|
||||||
|
return file.good();
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool GenerateDesktopAtlasHeader()
|
||||||
|
{
|
||||||
|
std::vector<AtlasRegion> regions;
|
||||||
|
const size_t sourceCount = sizeof(Sources) / sizeof(Sources[0]);
|
||||||
|
regions.reserve(sourceCount);
|
||||||
|
|
||||||
|
for (size_t i = 0; i < sourceCount; ++i)
|
||||||
|
{
|
||||||
|
AtlasRegion region;
|
||||||
|
if (!LoadAndResizeSource(Sources[i], region))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
regions.push_back(region);
|
||||||
|
}
|
||||||
|
|
||||||
|
int32_t atlasHeight = 0;
|
||||||
|
if (!PackAtlas(regions, atlasHeight))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::vector<uint16_t> atlasPixels = BuildAtlasPixels(regions, atlasHeight);
|
||||||
|
if (!WriteAtlasHeader(regions, atlasPixels, atlasHeight))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::cout << "Generated " << OutputHeaderPath << " (" << AtlasWidth << "x" << atlasHeight << ", "
|
||||||
|
<< regions.size() << " regions)" << std::endl;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int argc, char* argv[])
|
||||||
|
{
|
||||||
|
if (argc > 1)
|
||||||
|
{
|
||||||
|
std::cout << "Usage: DesktopAssetTool\n";
|
||||||
|
std::cout << "Generates " << OutputHeaderPath << " from PNG files in " << InputDirectory << "." << std::endl;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const int imageFlags = IMG_INIT_PNG;
|
||||||
|
if ((IMG_Init(imageFlags) & imageFlags) != imageFlags)
|
||||||
|
{
|
||||||
|
std::cerr << "IMG_Init failed: " << IMG_GetError() << std::endl;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const bool ok = GenerateDesktopAtlasHeader();
|
||||||
|
IMG_Quit();
|
||||||
|
return ok ? 0 : 1;
|
||||||
|
}
|
||||||
@@ -2,31 +2,85 @@ set(TOM_GAME_TARGET IMX6U-Game)
|
|||||||
set(TOM_ATLAS_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/generated/tom_atlas.h")
|
set(TOM_ATLAS_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/generated/tom_atlas.h")
|
||||||
set_source_files_properties(${TOM_ATLAS_HEADER} PROPERTIES GENERATED TRUE)
|
set_source_files_properties(${TOM_ATLAS_HEADER} PROPERTIES GENERATED TRUE)
|
||||||
|
|
||||||
|
set(TOM_AUDIO_GENERATED_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated")
|
||||||
|
set(TOM_AUDIO_HEADER "${TOM_AUDIO_GENERATED_DIR}/TomAudioData.h")
|
||||||
|
set(TOM_AUDIO_SOURCE "${TOM_AUDIO_GENERATED_DIR}/TomAudioData.cpp")
|
||||||
|
add_custom_command(
|
||||||
|
OUTPUT ${TOM_AUDIO_HEADER} ${TOM_AUDIO_SOURCE}
|
||||||
|
COMMAND ${CMAKE_COMMAND}
|
||||||
|
"-DOPEN_LIGHT_WAV=${CMAKE_CURRENT_SOURCE_DIR}/assets/audio/OpenLight.wav"
|
||||||
|
"-DCLOSE_LIGHT_WAV=${CMAKE_CURRENT_SOURCE_DIR}/assets/audio/CloseLight.wav"
|
||||||
|
"-DOUTPUT_HEADER=${TOM_AUDIO_HEADER}"
|
||||||
|
"-DOUTPUT_SOURCE=${TOM_AUDIO_SOURCE}"
|
||||||
|
-P "${CMAKE_CURRENT_SOURCE_DIR}/tools/EmbedTomAudio.cmake"
|
||||||
|
DEPENDS
|
||||||
|
"${CMAKE_CURRENT_SOURCE_DIR}/assets/audio/OpenLight.wav"
|
||||||
|
"${CMAKE_CURRENT_SOURCE_DIR}/assets/audio/CloseLight.wav"
|
||||||
|
"${CMAKE_CURRENT_SOURCE_DIR}/tools/EmbedTomAudio.cmake"
|
||||||
|
COMMENT "Embedding Tom voice response WAV files"
|
||||||
|
VERBATIM
|
||||||
|
)
|
||||||
|
set_source_files_properties(${TOM_AUDIO_HEADER} ${TOM_AUDIO_SOURCE} PROPERTIES GENERATED TRUE)
|
||||||
|
|
||||||
add_executable(${TOM_GAME_TARGET}
|
add_executable(${TOM_GAME_TARGET}
|
||||||
Main.cpp
|
Main.cpp
|
||||||
src/app/TomGameApp.cpp
|
src/app/TomGameApp.cpp
|
||||||
src/audio/VoiceEffect.cpp
|
src/audio/VoiceEffect.cpp
|
||||||
src/audio/VoicePlayer.cpp
|
src/audio/VoicePlayer.cpp
|
||||||
src/audio/VoiceRecorder.cpp
|
src/audio/VoiceRecorder.cpp
|
||||||
|
src/audio/WavAudioData.cpp
|
||||||
|
src/gameplay/KeywordCommandRouter.cpp
|
||||||
|
src/gameplay/SpriteButton.cpp
|
||||||
|
src/gameplay/TomAnimator.cpp
|
||||||
|
src/gameplay/TomHud.cpp
|
||||||
|
src/gameplay/VoiceInteractionController.cpp
|
||||||
|
src/recognition/KeywordRecognizer.cpp
|
||||||
|
src/recognition/ResBnKwsCnRecognizer.cpp
|
||||||
|
src/recognition/ResBnKwsCnModelData.cpp
|
||||||
|
src/ui/TomSettingsPanel.cpp
|
||||||
${TOM_ATLAS_HEADER}
|
${TOM_ATLAS_HEADER}
|
||||||
|
${TOM_AUDIO_HEADER}
|
||||||
|
${TOM_AUDIO_SOURCE}
|
||||||
)
|
)
|
||||||
|
|
||||||
target_include_directories(${TOM_GAME_TARGET} PRIVATE
|
target_include_directories(${TOM_GAME_TARGET} PRIVATE
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/src
|
${CMAKE_CURRENT_SOURCE_DIR}/src
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/src/app
|
${CMAKE_CURRENT_SOURCE_DIR}/src/app
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/src/audio
|
${CMAKE_CURRENT_SOURCE_DIR}/src/audio
|
||||||
|
${CMAKE_CURRENT_SOURCE_DIR}/src/gameplay
|
||||||
|
${CMAKE_CURRENT_SOURCE_DIR}/src/recognition
|
||||||
|
${CMAKE_CURRENT_SOURCE_DIR}/src/ui
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/generated
|
${CMAKE_CURRENT_SOURCE_DIR}/generated
|
||||||
|
${TOM_AUDIO_GENERATED_DIR}
|
||||||
)
|
)
|
||||||
|
|
||||||
imx6u_configure_app_target(${TOM_GAME_TARGET})
|
imx6u_configure_app_target(${TOM_GAME_TARGET})
|
||||||
|
|
||||||
|
if(BUILD_TESTING AND NOT TARGET_IMX)
|
||||||
|
add_executable(TomAudioDataTests EXCLUDE_FROM_ALL
|
||||||
|
tests/TomAudioDataTests.cpp
|
||||||
|
src/audio/WavAudioData.cpp
|
||||||
|
src/gameplay/KeywordCommandRouter.cpp
|
||||||
|
src/recognition/KeywordRecognizer.cpp
|
||||||
|
${TOM_AUDIO_HEADER}
|
||||||
|
${TOM_AUDIO_SOURCE}
|
||||||
|
)
|
||||||
|
target_include_directories(TomAudioDataTests PRIVATE
|
||||||
|
${CMAKE_CURRENT_SOURCE_DIR}/src/audio
|
||||||
|
${CMAKE_CURRENT_SOURCE_DIR}/src/gameplay
|
||||||
|
${CMAKE_CURRENT_SOURCE_DIR}/src/recognition
|
||||||
|
${TOM_AUDIO_GENERATED_DIR}
|
||||||
|
)
|
||||||
|
add_test(NAME tom_audio_data COMMAND TomAudioDataTests)
|
||||||
|
endif()
|
||||||
|
|
||||||
if(CMAKE_CROSSCOMPILING AND NOT EXISTS "${TOM_ATLAS_HEADER}")
|
if(CMAKE_CROSSCOMPILING AND NOT EXISTS "${TOM_ATLAS_HEADER}")
|
||||||
message(FATAL_ERROR
|
message(FATAL_ERROR
|
||||||
"Tom atlas header is missing. Run GenerateTomAtlasHeader in a host build before cross compiling."
|
"Tom atlas header is missing. Run GenerateTomAtlasHeader in a host build before cross compiling."
|
||||||
)
|
)
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
if(NOT USE_FRAMEBUFFER)
|
if(NOT TARGET_IMX)
|
||||||
set(TOM_ATLAS_TOOL_SOURCES
|
set(TOM_ATLAS_TOOL_SOURCES
|
||||||
tools/asset_pipeline/SpriteAssetTool.cpp
|
tools/asset_pipeline/SpriteAssetTool.cpp
|
||||||
)
|
)
|
||||||
|
|||||||
+247
-8
@@ -3,18 +3,22 @@
|
|||||||
#include <cstdlib>
|
#include <cstdlib>
|
||||||
#include <cstring>
|
#include <cstring>
|
||||||
#include <iostream>
|
#include <iostream>
|
||||||
|
#include <string>
|
||||||
#include <thread>
|
#include <thread>
|
||||||
|
|
||||||
#include "DefaultHardware.h"
|
#include "DefaultHardware.h"
|
||||||
#include "Display.h"
|
#include "IDisplay.h"
|
||||||
#include "DrawContext.h"
|
#include "DrawContext.h"
|
||||||
#include "TimeSource.h"
|
#include "TimeSource.h"
|
||||||
#include "Timer.h"
|
#include "Timer.h"
|
||||||
#include "app/TomGameApp.h"
|
#include "app/TomGameApp.h"
|
||||||
|
#include "recognition/KeywordRecognizer.h"
|
||||||
|
#include "recognition/ResBnKwsCnRecognizer.h"
|
||||||
|
|
||||||
#ifdef USE_FRAMEBUFFER
|
#ifdef TARGET_IMX
|
||||||
#include "FBDisplay.h"
|
#include "FBDisplay.h"
|
||||||
#else
|
#endif
|
||||||
|
#ifdef TARGET_PC
|
||||||
#include "SDLDisplay.h"
|
#include "SDLDisplay.h"
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
@@ -26,10 +30,28 @@ namespace
|
|||||||
struct ProgramOptions
|
struct ProgramOptions
|
||||||
{
|
{
|
||||||
uint32_t target_fps;
|
uint32_t target_fps;
|
||||||
|
uint32_t audio_sample_rate;
|
||||||
|
uint32_t audio_channels;
|
||||||
|
float kws_threshold;
|
||||||
|
float kws_input_gain;
|
||||||
|
std::string audio_input_device;
|
||||||
|
std::string audio_output_device;
|
||||||
|
std::string settings_path;
|
||||||
bool show_help;
|
bool show_help;
|
||||||
|
|
||||||
ProgramOptions()
|
ProgramOptions()
|
||||||
: target_fps(Core::Timer::DefaultFps),
|
: target_fps(Core::Timer::DefaultFps),
|
||||||
|
audio_sample_rate(16000),
|
||||||
|
audio_channels(1),
|
||||||
|
kws_threshold(0.75f),
|
||||||
|
kws_input_gain(1.0f),
|
||||||
|
audio_input_device("default"),
|
||||||
|
audio_output_device("default"),
|
||||||
|
#ifdef TARGET_IMX
|
||||||
|
settings_path("/opt/imx6u-game/tom_settings.cfg"),
|
||||||
|
#else
|
||||||
|
settings_path("tom_settings.cfg"),
|
||||||
|
#endif
|
||||||
show_help(false)
|
show_help(false)
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
@@ -37,9 +59,10 @@ namespace
|
|||||||
|
|
||||||
static Platform::IDisplay *CreateDisplay()
|
static Platform::IDisplay *CreateDisplay()
|
||||||
{
|
{
|
||||||
#ifdef USE_FRAMEBUFFER
|
#ifdef TARGET_IMX
|
||||||
return new Platform::FBDisplay();
|
return new Platform::FBDisplay();
|
||||||
#else
|
#endif
|
||||||
|
#ifdef TARGET_PC
|
||||||
return new Platform::SDLDisplay();
|
return new Platform::SDLDisplay();
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
@@ -47,8 +70,10 @@ namespace
|
|||||||
static void PrintUsage(const char *program_name)
|
static void PrintUsage(const char *program_name)
|
||||||
{
|
{
|
||||||
std::cout
|
std::cout
|
||||||
<< "Usage: " << program_name << " [--fps 30|45|60]\n"
|
<< "Usage: " << program_name << " [--fps 30|45|60] [--kws-threshold 0.75] [--kws-input-gain 1.0]\n"
|
||||||
<< " " << program_name << " [--fps=30|45|60]\n";
|
<< " " << program_name << " [--audio-input-device default] [--audio-output-device default]\n"
|
||||||
|
<< " " << program_name << " [--audio-rate 16000] [--audio-channels 1]\n"
|
||||||
|
<< " " << program_name << " [--settings-path tom_settings.cfg]\n";
|
||||||
}
|
}
|
||||||
|
|
||||||
static ProgramOptions ParseProgramOptions(int argc, char *argv[])
|
static ProgramOptions ParseProgramOptions(int argc, char *argv[])
|
||||||
@@ -59,6 +84,13 @@ namespace
|
|||||||
{
|
{
|
||||||
const char *arg = argv[i];
|
const char *arg = argv[i];
|
||||||
const char *fps_value = nullptr;
|
const char *fps_value = nullptr;
|
||||||
|
const char *audio_rate_value = nullptr;
|
||||||
|
const char *audio_channels_value = nullptr;
|
||||||
|
const char *kws_threshold_value = nullptr;
|
||||||
|
const char *kws_input_gain_value = nullptr;
|
||||||
|
const char *audio_input_device_value = nullptr;
|
||||||
|
const char *audio_output_device_value = nullptr;
|
||||||
|
const char *settings_path_value = nullptr;
|
||||||
|
|
||||||
if (std::strcmp(arg, "--help") == 0 || std::strcmp(arg, "-h") == 0)
|
if (std::strcmp(arg, "--help") == 0 || std::strcmp(arg, "-h") == 0)
|
||||||
{
|
{
|
||||||
@@ -80,6 +112,120 @@ namespace
|
|||||||
{
|
{
|
||||||
fps_value = arg + 6;
|
fps_value = arg + 6;
|
||||||
}
|
}
|
||||||
|
else if (std::strcmp(arg, "--audio-rate") == 0)
|
||||||
|
{
|
||||||
|
if (i + 1 < argc)
|
||||||
|
{
|
||||||
|
audio_rate_value = argv[++i];
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
std::cerr << "Missing value for --audio-rate, using default 16000.\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (std::strncmp(arg, "--audio-rate=", 13) == 0)
|
||||||
|
{
|
||||||
|
audio_rate_value = arg + 13;
|
||||||
|
}
|
||||||
|
else if (std::strcmp(arg, "--audio-channels") == 0)
|
||||||
|
{
|
||||||
|
if (i + 1 < argc)
|
||||||
|
{
|
||||||
|
audio_channels_value = argv[++i];
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
std::cerr << "Missing value for --audio-channels, using default 1.\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (std::strncmp(arg, "--audio-channels=", 17) == 0)
|
||||||
|
{
|
||||||
|
audio_channels_value = arg + 17;
|
||||||
|
}
|
||||||
|
else if (std::strcmp(arg, "--audio-input-device") == 0)
|
||||||
|
{
|
||||||
|
if (i + 1 < argc)
|
||||||
|
{
|
||||||
|
audio_input_device_value = argv[++i];
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
std::cerr << "Missing value for --audio-input-device, using default.\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (std::strncmp(arg, "--audio-input-device=", 21) == 0)
|
||||||
|
{
|
||||||
|
audio_input_device_value = arg + 21;
|
||||||
|
}
|
||||||
|
else if (std::strcmp(arg, "--audio-output-device") == 0)
|
||||||
|
{
|
||||||
|
if (i + 1 < argc)
|
||||||
|
{
|
||||||
|
audio_output_device_value = argv[++i];
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
std::cerr << "Missing value for --audio-output-device, using default.\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (std::strncmp(arg, "--audio-output-device=", 22) == 0)
|
||||||
|
{
|
||||||
|
audio_output_device_value = arg + 22;
|
||||||
|
}
|
||||||
|
else if (std::strcmp(arg, "--settings-path") == 0)
|
||||||
|
{
|
||||||
|
if (i + 1 < argc)
|
||||||
|
{
|
||||||
|
settings_path_value = argv[++i];
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
std::cerr << "Missing value for --settings-path, using default.\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (std::strncmp(arg, "--settings-path=", 16) == 0)
|
||||||
|
{
|
||||||
|
settings_path_value = arg + 16;
|
||||||
|
}
|
||||||
|
else if (std::strcmp(arg, "--kws-model") == 0 ||
|
||||||
|
std::strncmp(arg, "--kws-model=", 12) == 0)
|
||||||
|
{
|
||||||
|
std::cerr << "--kws-model is ignored: Tiny KWS weights are embedded in the executable.\n";
|
||||||
|
if (std::strcmp(arg, "--kws-model") == 0 && i + 1 < argc)
|
||||||
|
{
|
||||||
|
++i;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (std::strcmp(arg, "--kws-threshold") == 0)
|
||||||
|
{
|
||||||
|
if (i + 1 < argc)
|
||||||
|
{
|
||||||
|
kws_threshold_value = argv[++i];
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
std::cerr << "Missing value for --kws-threshold, using default 0.75.\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (std::strncmp(arg, "--kws-threshold=", 16) == 0)
|
||||||
|
{
|
||||||
|
kws_threshold_value = arg + 16;
|
||||||
|
}
|
||||||
|
else if (std::strcmp(arg, "--kws-input-gain") == 0)
|
||||||
|
{
|
||||||
|
if (i + 1 < argc)
|
||||||
|
{
|
||||||
|
kws_input_gain_value = argv[++i];
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
std::cerr << "Missing value for --kws-input-gain, using default 1.0.\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (std::strncmp(arg, "--kws-input-gain=", 17) == 0)
|
||||||
|
{
|
||||||
|
kws_input_gain_value = arg + 17;
|
||||||
|
}
|
||||||
|
|
||||||
if (fps_value != nullptr)
|
if (fps_value != nullptr)
|
||||||
{
|
{
|
||||||
@@ -94,6 +240,79 @@ namespace
|
|||||||
options.target_fps = Core::Timer::DefaultFps;
|
options.target_fps = Core::Timer::DefaultFps;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (audio_rate_value != nullptr)
|
||||||
|
{
|
||||||
|
const uint32_t parsed_rate = static_cast<uint32_t>(std::strtoul(audio_rate_value, nullptr, 10));
|
||||||
|
if (parsed_rate > 0u)
|
||||||
|
{
|
||||||
|
options.audio_sample_rate = parsed_rate;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
std::cerr << "Unsupported audio rate '" << audio_rate_value << "', using default 16000.\n";
|
||||||
|
options.audio_sample_rate = 16000;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (audio_channels_value != nullptr)
|
||||||
|
{
|
||||||
|
const uint32_t parsed_channels = static_cast<uint32_t>(std::strtoul(audio_channels_value, nullptr, 10));
|
||||||
|
if (parsed_channels > 0u)
|
||||||
|
{
|
||||||
|
options.audio_channels = parsed_channels;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
std::cerr << "Unsupported audio channels '" << audio_channels_value << "', using default 1.\n";
|
||||||
|
options.audio_channels = 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (audio_input_device_value != nullptr && audio_input_device_value[0] != '\0')
|
||||||
|
{
|
||||||
|
options.audio_input_device = audio_input_device_value;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (audio_output_device_value != nullptr && audio_output_device_value[0] != '\0')
|
||||||
|
{
|
||||||
|
options.audio_output_device = audio_output_device_value;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (settings_path_value != nullptr && settings_path_value[0] != '\0')
|
||||||
|
{
|
||||||
|
options.settings_path = settings_path_value;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (kws_threshold_value != nullptr)
|
||||||
|
{
|
||||||
|
char *threshold_end = nullptr;
|
||||||
|
const float parsed_threshold = std::strtof(kws_threshold_value, &threshold_end);
|
||||||
|
if (threshold_end != kws_threshold_value && *threshold_end == '\0' && parsed_threshold >= 0.0f)
|
||||||
|
{
|
||||||
|
options.kws_threshold = parsed_threshold;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
std::cerr << "Unsupported KWS threshold '" << kws_threshold_value << "', using default 0.75.\n";
|
||||||
|
options.kws_threshold = 0.75f;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (kws_input_gain_value != nullptr)
|
||||||
|
{
|
||||||
|
char *gain_end = nullptr;
|
||||||
|
const float parsed_gain = std::strtof(kws_input_gain_value, &gain_end);
|
||||||
|
if (gain_end != kws_input_gain_value && *gain_end == '\0' && parsed_gain > 0.0f)
|
||||||
|
{
|
||||||
|
options.kws_input_gain = parsed_gain;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
std::cerr << "Unsupported KWS input gain '" << kws_input_gain_value << "', using default 1.0.\n";
|
||||||
|
options.kws_input_gain = 1.0f;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return options;
|
return options;
|
||||||
@@ -129,6 +348,11 @@ int main(int argc, char* argv[])
|
|||||||
Platform::DefaultAudioOutput audioOutput;
|
Platform::DefaultAudioOutput audioOutput;
|
||||||
Platform::DefaultButtonInput buttonInput;
|
Platform::DefaultButtonInput buttonInput;
|
||||||
Platform::DefaultPointerInput pointerInput;
|
Platform::DefaultPointerInput pointerInput;
|
||||||
|
Platform::DefaultBoardLed boardLed;
|
||||||
|
|
||||||
|
Game::ResBnKwsCnRecognizer keywordRecognizer;
|
||||||
|
keywordRecognizer.set_confidence_threshold(options.kws_threshold);
|
||||||
|
keywordRecognizer.set_input_gain(options.kws_input_gain);
|
||||||
|
|
||||||
if (!buttonInput.init())
|
if (!buttonInput.init())
|
||||||
{
|
{
|
||||||
@@ -138,6 +362,10 @@ int main(int argc, char* argv[])
|
|||||||
{
|
{
|
||||||
std::cerr << "[WARN] Pointer input init failed; on-screen button trigger is disabled." << std::endl;
|
std::cerr << "[WARN] Pointer input init failed; on-screen button trigger is disabled." << std::endl;
|
||||||
}
|
}
|
||||||
|
if (!boardLed.init())
|
||||||
|
{
|
||||||
|
std::cerr << "[WARN] Board LED init failed; voice responses remain enabled." << std::endl;
|
||||||
|
}
|
||||||
|
|
||||||
Core::DrawContext ctx(ScreenWidth, ScreenHeight);
|
Core::DrawContext ctx(ScreenWidth, ScreenHeight);
|
||||||
Core::Timer timer(options.target_fps);
|
Core::Timer timer(options.target_fps);
|
||||||
@@ -148,7 +376,12 @@ int main(int argc, char* argv[])
|
|||||||
&audioInput,
|
&audioInput,
|
||||||
&audioOutput,
|
&audioOutput,
|
||||||
&buttonInput,
|
&buttonInput,
|
||||||
&pointerInput);
|
&pointerInput,
|
||||||
|
&boardLed,
|
||||||
|
&keywordRecognizer,
|
||||||
|
options.settings_path);
|
||||||
|
app.configure_audio(options.audio_sample_rate, options.audio_channels);
|
||||||
|
app.configure_audio_devices(options.audio_input_device, options.audio_output_device);
|
||||||
|
|
||||||
std::cout << "[INFO] Tom game started. Target FPS: " << timer.target_fps() << std::endl;
|
std::cout << "[INFO] Tom game started. Target FPS: " << timer.target_fps() << std::endl;
|
||||||
|
|
||||||
@@ -165,6 +398,12 @@ int main(int argc, char* argv[])
|
|||||||
}
|
}
|
||||||
|
|
||||||
app.update(timer.fixed_delta_ms());
|
app.update(timer.fixed_delta_ms());
|
||||||
|
if (app.should_exit())
|
||||||
|
{
|
||||||
|
is_running = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
app.draw(ctx);
|
app.draw(ctx);
|
||||||
ctx.present(display);
|
ctx.present(display);
|
||||||
SleepRemainingFrameTime(timer, time_source);
|
SleepRemainingFrameTime(timer, time_source);
|
||||||
|
|||||||
@@ -0,0 +1,381 @@
|
|||||||
|
# TomGame、Desktop 与板端部署
|
||||||
|
|
||||||
|
本文说明 TomGame 当前的关键词识别方式，以及 Windows 验证、WSL
|
||||||
|
交叉编译、Windows SCP 部署和 IMX6ULL 板端运行流程。
|
||||||
|
|
||||||
|
TomGame 使用内嵌的 `ResBnKwsCnRecognizer` INT8 模型，不需要 Python、
|
||||||
|
TensorFlow、TensorFlow Lite 运行库或外部模型文件。
|
||||||
|
|
||||||
|
模型支持以下 14 个类别：
|
||||||
|
|
||||||
|
```text
|
||||||
|
Down, Go, Left, No, Off, On, Right, Stop, Up, Yes,
|
||||||
|
OpenLight, CloseLight, Silence, Unknown
|
||||||
|
```
|
||||||
|
|
||||||
|
当前游戏动作映射：
|
||||||
|
|
||||||
|
```text
|
||||||
|
Up / On -> Jump
|
||||||
|
Stop -> Idle
|
||||||
|
OpenLight -> 打开 ALPHA 板载 LED，播放 Tom 音效语音
|
||||||
|
CloseLight -> 关闭 ALPHA 板载 LED，播放 Tom 音效语音
|
||||||
|
```
|
||||||
|
|
||||||
|
`OpenLight.wav` 和 `CloseLight.wav` 会在构建阶段转换成只读 C++ 数据并
|
||||||
|
编译进可执行文件，板端部署时不需要额外复制 WAV 文件。板载 LED 使用
|
||||||
|
正点原子 I.MX6U-ALPHA 的 `GPIO1_IO03`，低电平点亮；程序优先使用
|
||||||
|
`/dev/gpioled` 等正点原子教学驱动设备，并兼容 GPIO sysfs。
|
||||||
|
|
||||||
|
## Windows 验证
|
||||||
|
|
||||||
|
Windows 版本和板端版本使用同一套 C++ 特征提取、INT8 模型推理和
|
||||||
|
多窗口融合逻辑，不再需要 Python 对照脚本。
|
||||||
|
|
||||||
|
在项目根目录构建 TomGame：
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
cmake --build build --config Release --target IMX6U-Game
|
||||||
|
```
|
||||||
|
|
||||||
|
直接运行：
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
.\build\Release\IMX6U-Game.exe `
|
||||||
|
--fps 30 `
|
||||||
|
--kws-threshold 0.75 `
|
||||||
|
--kws-input-gain 1.0
|
||||||
|
```
|
||||||
|
|
||||||
|
程序支持以下关键词识别参数：
|
||||||
|
|
||||||
|
```text
|
||||||
|
--kws-threshold 识别置信度阈值，默认 0.75
|
||||||
|
--kws-input-gain MFCC 输入增益，范围 (0, 1]，默认 1.0
|
||||||
|
```
|
||||||
|
|
||||||
|
识别成功或未达到阈值时，控制台会打印类似日志：
|
||||||
|
|
||||||
|
```text
|
||||||
|
[INFO] ResBN-CN INT8 KWS result: class=Up, confidence=0.92, windows=...
|
||||||
|
```
|
||||||
|
|
||||||
|
Windows Desktop 和全部游戏可以一起构建：
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
cmake --build build --config Release --target `
|
||||||
|
IMX6U-Desktop `
|
||||||
|
IMX6U-Game `
|
||||||
|
IMX6U-LightGame `
|
||||||
|
IMX6U-Demo
|
||||||
|
```
|
||||||
|
|
||||||
|
运行主页：
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
.\build\Release\IMX6U-Desktop.exe
|
||||||
|
```
|
||||||
|
|
||||||
|
Desktop 会从自身所在目录寻找并启动以下程序：
|
||||||
|
|
||||||
|
```text
|
||||||
|
IMX6U-Game.exe
|
||||||
|
IMX6U-LightGame.exe
|
||||||
|
IMX6U-Demo.exe
|
||||||
|
```
|
||||||
|
|
||||||
|
因此 Desktop 和各游戏的可执行文件需要位于同一个目录。
|
||||||
|
|
||||||
|
## WSL 交叉编译
|
||||||
|
|
||||||
|
### 1. 安装工具链
|
||||||
|
|
||||||
|
首次编译时安装 ARM 交叉编译器和 ALSA 开发文件：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo dpkg --add-architecture armhf
|
||||||
|
sudo apt update
|
||||||
|
sudo apt install -y \
|
||||||
|
cmake \
|
||||||
|
build-essential \
|
||||||
|
gcc-arm-linux-gnueabihf \
|
||||||
|
g++-arm-linux-gnueabihf \
|
||||||
|
libasound2-dev:armhf
|
||||||
|
```
|
||||||
|
|
||||||
|
检查交叉编译器：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
arm-linux-gnueabihf-g++ --version
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. 确认生成资源
|
||||||
|
|
||||||
|
交叉编译不会在板端生成 PNG 图集。开始编译前确认以下文件存在：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ls src/Apps/Game/generated/tom_atlas.h
|
||||||
|
ls src/Apps/Desktop/generated/desktop_atlas.h
|
||||||
|
```
|
||||||
|
|
||||||
|
如果文件不存在，先在 Windows 主机构建一次对应目标，生成图集后再同步到
|
||||||
|
WSL 源码目录。
|
||||||
|
|
||||||
|
### 3. 配置并编译
|
||||||
|
|
||||||
|
在 WSL 项目根目录执行：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
rm -rf build-arm-fb
|
||||||
|
|
||||||
|
cmake -S . -B build-arm-fb \
|
||||||
|
-DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-arm-linux-gnueabihf.cmake \
|
||||||
|
-DTARGET_IMX=ON \
|
||||||
|
-DCMAKE_BUILD_TYPE=Release
|
||||||
|
|
||||||
|
cmake --build build-arm-fb \
|
||||||
|
--target \
|
||||||
|
IMX6U-Desktop \
|
||||||
|
IMX6U-Game \
|
||||||
|
IMX6U-LightGame \
|
||||||
|
IMX6U-Demo \
|
||||||
|
-j"$(nproc)"
|
||||||
|
```
|
||||||
|
|
||||||
|
查找编译产物：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
find build-arm-fb -type f \
|
||||||
|
\( -name IMX6U-Desktop \
|
||||||
|
-o -name IMX6U-Game \
|
||||||
|
-o -name IMX6U-LightGame \
|
||||||
|
-o -name IMX6U-Demo \)
|
||||||
|
```
|
||||||
|
|
||||||
|
使用 `file` 确认产物是 ARM 程序：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
file "$(find build-arm-fb -type f -name IMX6U-Desktop | head -n 1)"
|
||||||
|
file "$(find build-arm-fb -type f -name IMX6U-Game | head -n 1)"
|
||||||
|
file "$(find build-arm-fb -type f -name IMX6U-LightGame | head -n 1)"
|
||||||
|
file "$(find build-arm-fb -type f -name IMX6U-Demo | head -n 1)"
|
||||||
|
```
|
||||||
|
|
||||||
|
正确结果应包含：
|
||||||
|
|
||||||
|
```text
|
||||||
|
ELF 32-bit LSB executable, ARM, EABI5
|
||||||
|
```
|
||||||
|
|
||||||
|
如果显示 `x86-64`，说明没有使用 ARM 工具链。
|
||||||
|
|
||||||
|
## Windows SCP 部署
|
||||||
|
|
||||||
|
将 WSL 编译出的四个程序复制到 Windows。以下示例假设文件位于：
|
||||||
|
|
||||||
|
```text
|
||||||
|
E:\嵌入式实验\IMX6U-Desktop
|
||||||
|
E:\嵌入式实验\IMX6U-Game
|
||||||
|
E:\嵌入式实验\IMX6U-LightGame
|
||||||
|
E:\嵌入式实验\IMX6U-Demo
|
||||||
|
```
|
||||||
|
|
||||||
|
板子地址示例：
|
||||||
|
|
||||||
|
```text
|
||||||
|
root@192.168.0.200
|
||||||
|
```
|
||||||
|
|
||||||
|
板端统一部署目录：
|
||||||
|
|
||||||
|
```text
|
||||||
|
/home/root/opt
|
||||||
|
```
|
||||||
|
|
||||||
|
先创建目录：
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
ssh `
|
||||||
|
-o HostKeyAlgorithms=+ssh-rsa `
|
||||||
|
root@192.168.0.200 `
|
||||||
|
"mkdir -p /home/root/opt"
|
||||||
|
```
|
||||||
|
|
||||||
|
批量上传：
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
$files = @(
|
||||||
|
"IMX6U-Desktop",
|
||||||
|
"IMX6U-Game",
|
||||||
|
"IMX6U-LightGame",
|
||||||
|
"IMX6U-Demo"
|
||||||
|
)
|
||||||
|
|
||||||
|
foreach ($file in $files) {
|
||||||
|
scp -O `
|
||||||
|
-o HostKeyAlgorithms=+ssh-rsa `
|
||||||
|
"E:\嵌入式实验\$file" `
|
||||||
|
"root@192.168.0.200:/home/root/opt/$file"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
其中：
|
||||||
|
|
||||||
|
```text
|
||||||
|
-O 强制使用旧版 SCP 协议
|
||||||
|
HostKeyAlgorithms=+ssh-rsa 兼容板子上的旧 SSH 服务
|
||||||
|
```
|
||||||
|
|
||||||
|
ResBN 模型已经编译进 `IMX6U-Game`，不需要上传 `.tflite` 或其他模型文件。
|
||||||
|
|
||||||
|
## 板端运行
|
||||||
|
|
||||||
|
### 1. 登录板子
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
ssh -o HostKeyAlgorithms=+ssh-rsa root@192.168.0.200
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. 初始化音频
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /home/root/shell/audio
|
||||||
|
./mic_in_config.sh
|
||||||
|
amixer cset numid=41 1
|
||||||
|
```
|
||||||
|
|
||||||
|
`amixer cset numid=41 1` 会将 WM8960 的左 ADC 同时输出到左右声道。
|
||||||
|
板载 MIC 实测主要连接在 Left ADC，未设置时可能只有单声道有效。
|
||||||
|
|
||||||
|
### 3. 设置执行权限
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /home/root/opt
|
||||||
|
chmod +x IMX6U-Desktop
|
||||||
|
chmod +x IMX6U-Game
|
||||||
|
chmod +x IMX6U-LightGame
|
||||||
|
chmod +x IMX6U-Demo
|
||||||
|
```
|
||||||
|
|
||||||
|
也可以统一执行：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
chmod +x IMX6U-*
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4. 启动 Desktop
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /home/root/opt
|
||||||
|
./IMX6U-Desktop
|
||||||
|
```
|
||||||
|
|
||||||
|
Desktop 操作方式：
|
||||||
|
|
||||||
|
```text
|
||||||
|
点击屏幕左侧或右侧 切换游戏
|
||||||
|
水平滑动 切换游戏
|
||||||
|
点击中间游戏封面 启动当前游戏
|
||||||
|
```
|
||||||
|
|
||||||
|
Desktop 启动游戏前会释放 framebuffer 和触摸设备。游戏退出后，
|
||||||
|
Desktop 会重新初始化硬件并恢复主页。
|
||||||
|
|
||||||
|
Desktop 使用相对路径启动游戏，因此以下程序必须与 `IMX6U-Desktop`
|
||||||
|
处于同一目录：
|
||||||
|
|
||||||
|
```text
|
||||||
|
/home/root/opt/IMX6U-Desktop
|
||||||
|
/home/root/opt/IMX6U-Game
|
||||||
|
/home/root/opt/IMX6U-LightGame
|
||||||
|
/home/root/opt/IMX6U-Demo
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5. 单独运行 TomGame
|
||||||
|
|
||||||
|
排查语音或音频问题时，可以绕过 Desktop 直接启动 TomGame：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /home/root/opt
|
||||||
|
|
||||||
|
./IMX6U-Game \
|
||||||
|
--fps 30 \
|
||||||
|
--kws-threshold 0.75 \
|
||||||
|
--kws-input-gain 1.0 \
|
||||||
|
--audio-input-device sysdefault:CARD=wm8960audio \
|
||||||
|
--audio-output-device sysdefault:CARD=wm8960audio
|
||||||
|
```
|
||||||
|
|
||||||
|
如果 `sysdefault:CARD=wm8960audio` 初始化失败，改用：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./IMX6U-Game \
|
||||||
|
--fps 30 \
|
||||||
|
--kws-threshold 0.75 \
|
||||||
|
--kws-input-gain 1.0 \
|
||||||
|
--audio-input-device plughw:0,0 \
|
||||||
|
--audio-output-device plughw:0,0
|
||||||
|
```
|
||||||
|
|
||||||
|
Desktop 启动 TomGame 时不会附加命令行参数，因此 TomGame 会使用 ALSA
|
||||||
|
的 `"default"` 输入和输出设备。如果直接运行 TomGame 正常，但从 Desktop
|
||||||
|
启动后音频初始化失败，需要将板子的 ALSA default 配置映射到 WM8960。
|
||||||
|
|
||||||
|
## 常见问题
|
||||||
|
|
||||||
|
### SSH 提示不支持 ssh-rsa
|
||||||
|
|
||||||
|
错误示例：
|
||||||
|
|
||||||
|
```text
|
||||||
|
Unable to negotiate ... no matching host key type found.
|
||||||
|
Their offer: ssh-rsa
|
||||||
|
```
|
||||||
|
|
||||||
|
连接时增加：
|
||||||
|
|
||||||
|
```text
|
||||||
|
-o HostKeyAlgorithms=+ssh-rsa
|
||||||
|
```
|
||||||
|
|
||||||
|
SCP 还建议增加 `-O`，以兼容板子上的旧版 SSH/Dropbear。
|
||||||
|
|
||||||
|
### 找不到 libasound
|
||||||
|
|
||||||
|
项目会优先使用：
|
||||||
|
|
||||||
|
```text
|
||||||
|
third_party/arm-linux-gnueabihf/alsa/lib/libasound.so
|
||||||
|
```
|
||||||
|
|
||||||
|
如果该文件不存在，需要安装兼容的 ARMHF ALSA 包，或者从板子复制匹配的
|
||||||
|
`libasound.so*` 到该目录。不要使用与板端系统版本不兼容的新版库。
|
||||||
|
|
||||||
|
### Desktop 可以启动，但点击游戏没有反应
|
||||||
|
|
||||||
|
检查四个程序是否位于同一个目录，并具有执行权限：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /home/root/opt
|
||||||
|
ls -l IMX6U-*
|
||||||
|
```
|
||||||
|
|
||||||
|
Desktop 的游戏目录配置为：
|
||||||
|
|
||||||
|
```text
|
||||||
|
TomGame -> IMX6U-Game
|
||||||
|
LightGame -> IMX6U-LightGame
|
||||||
|
Demo -> IMX6U-Demo
|
||||||
|
```
|
||||||
|
|
||||||
|
### TomGame 没有识别结果
|
||||||
|
|
||||||
|
按以下顺序检查：
|
||||||
|
|
||||||
|
1. 执行 `mic_in_config.sh` 和 `amixer cset numid=41 1`。
|
||||||
|
2. 使用 `arecord -l`、`arecord -L` 确认 WM8960 设备存在。
|
||||||
|
3. 使用 `arecord -vv` 检查说话时是否有明显音量变化。
|
||||||
|
4. 确认游戏已切换到关键词识别模式。
|
||||||
|
5. 检查日志中是否出现 `ResBN INT8 KWS result`。
|
||||||
|
6. 保持 `--kws-input-gain` 在 `(0, 1]` 范围。
|
||||||
|
7. 必要时将 `--kws-threshold` 从 `0.75` 临时降低到 `0.70` 进行链路测试。
|
||||||
|
|||||||
Binary file not shown.
Binary file not shown.
@@ -1 +0,0 @@
|
|||||||
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
|
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 244 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 223 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 212 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 240 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 266 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 244 KiB |
@@ -1 +0,0 @@
|
|||||||
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
|
|
||||||
+88184
-11207
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,56 @@
|
|||||||
|
TI Text File License
|
||||||
|
|
||||||
|
Copyright (c) 2023 Texas Instruments Incorporated
|
||||||
|
|
||||||
|
All rights reserved not granted herein.
|
||||||
|
Limited License.
|
||||||
|
|
||||||
|
Texas Instruments Incorporated grants a world-wide, royalty-free,non-exclusive
|
||||||
|
license under copyrights and patents it now or hereafter owns or controls to
|
||||||
|
make, have made, use, import, offer to sell and sell ("Utilize") this software
|
||||||
|
subject to the terms herein. With respect to the foregoing patent license,
|
||||||
|
such license is granted solely to the extent that any such patent is
|
||||||
|
necessary to Utilize the software alone. The patent license shall not apply
|
||||||
|
to any combinations which include this software, other than combinations with
|
||||||
|
devices manufactured by or for TI ("TI Devices").No hardware patent is licensed
|
||||||
|
hereunder.
|
||||||
|
|
||||||
|
Redistributions must preserve existing copyright notices and reproduce this
|
||||||
|
license (including the above copyright notice and the disclaimer and (if
|
||||||
|
applicable) source code license limitations below) in the documentation and/or
|
||||||
|
other materials provided with the distribution
|
||||||
|
|
||||||
|
Redistribution and use in binary form, without modification, are permitted
|
||||||
|
provided that the following conditions are met:
|
||||||
|
|
||||||
|
* No reverse engineering, decompilation, or disassembly of this software is
|
||||||
|
permitted with respect to any software provided in binary form.
|
||||||
|
* any redistribution and use are licensed by TI for use only with TI Devices.
|
||||||
|
* Nothing shall obligate TI to provide you with source code for the software
|
||||||
|
licensed and provided to you in object code.
|
||||||
|
|
||||||
|
If software source code is provided to you, modification and redistribution of
|
||||||
|
the source code are permitted provided that the following conditions are met:
|
||||||
|
|
||||||
|
* any redistribution and use of the source code, including any resulting
|
||||||
|
derivative works, are licensed by TI for use only with TI Devices.
|
||||||
|
* any redistribution and use of any object code compiled from the source
|
||||||
|
code and any resulting derivative works, are licensed by TI for use only
|
||||||
|
with TI Devices.
|
||||||
|
|
||||||
|
Neither the name of Texas Instruments Incorporated nor the names of its
|
||||||
|
suppliers may be used to endorse or promote products derived from this
|
||||||
|
software without specific prior written permission.
|
||||||
|
|
||||||
|
DISCLAIMER.
|
||||||
|
|
||||||
|
THIS SOFTWARE IS PROVIDED BY TI AND TI"S LICENSORS "AS IS" AND ANY EXPRESS OR
|
||||||
|
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||||
|
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
|
||||||
|
EVENT SHALL TI AND TI"S LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||||
|
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
|
||||||
|
OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||||
|
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||||
|
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
||||||
|
EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# Audio keyword spotting on Sitara MPU
|
||||||
|
|
||||||
|
This repository has been validated on Texas Insruments AM62x and AM62Ax microprocessors for the 8.6 SDK.
|
||||||
|
|
||||||
|
## What is this repo?
|
||||||
|
|
||||||
|
Keyword spotting allows automatic recognition of speech words within a limited vocabulary to assist human-machine interaction. This task often uses machine learning models like neural networks, and can run in a limited memory and CPU processing footprint on a microcontroller or microprocessor.
|
||||||
|
|
||||||
|
This repo hosts python3 code for audio keyword spotting using two (tinyml-kws and matchboxnet) separate models trained on the Google Speech commands dataset (v1 and v2, respectively) running on the Am62x/AM62Ax SoC (aka 'the device') with the Linux SDK. It has been validated on these devices, but will likely run on other linux-based devices since there are very few hardware restrictions.
|
||||||
|
|
||||||
|
## What is required
|
||||||
|
|
||||||
|
* An [AM62](https://www.ti.com/tool/SK-AM62) or [AM62A](https://www.ti.com/tool/SK-AM62A-LP) starter kit EVM
|
||||||
|
* SD card
|
||||||
|
* USB to USB-micro cable
|
||||||
|
* ethernet network connection
|
||||||
|
* USB microphone
|
||||||
|
|
||||||
|
## How to run the demo
|
||||||
|
|
||||||
|
1. Setup the SDK on an SD card for the starter kit EVM's selected device/SoC according to the respective device's user guide ([AM62x](https://dev.ti.com/tirex/explore/node?node=A__AdoyIZ2jtLBUfHZNVmgFBQ__am62x-devtools__FUz-xrs__LATEST&search=am62x) and [AM62Ax](https://dev.ti.com/tirex/explore/node?node=A__AQniYj7pI2aoPAFMxWtKDQ__am62ax-devtools__FUz-xrs__LATEST). Follow that guide until a linux terminal session is available through serial/USB or internet/SSH
|
||||||
|
2. Clone this repository onto the device. This may require setting proxy variables like HTTPS_PROXY if the device is behind a firewall
|
||||||
|
3. Run the 'install_on_evm.sh' script. This will download and build the portaudio dependency, and then setup a few python libraries for audio processing
|
||||||
|
4. "Source" the 'quick_setup.sh' script to set an environment variable that is needed to load pyaudio in python: ```source ./quick_setup.sh```
|
||||||
|
5. Within the two subfolders, run either of the two python scripts starting with 'audio-inference'. Note that you will need to have the USB microphone plugged in and know the device index within linux. This is 1 by default, but the 'detect_microphone.py' script will also help identify them.
|
||||||
|
|
||||||
|
### Words recognized
|
||||||
|
|
||||||
|
The tinyml model can recognize 12 words: Down, Go, Left, No, Off, On, Right, Stop, Up, Yes, Silence, Unknown
|
||||||
|
|
||||||
|
The matchboxnet model is capable of recognizing a larger set of words: visual, wow, learn, backward, dog, two, left, happy, nine, go, up, bed, stop, one, zero, tree, seven, on, four, bird, right, eight, no, six, forward, house, marvin, sheila, five, off, three, down, cat, follow, yes
|
||||||
|
|
||||||
|
## Support and Resources
|
||||||
|
|
||||||
|
Please direct questions to the [Processors e2e](https://e2e.ti.com/support/processors-group/processors/f/processors-forum)
|
||||||
|
|
||||||
|
* Main TI Edge AI page: [https://ti.com/edgeai](https://ti.com/edgeai)
|
||||||
|
* [TI Arm Processors](https://www.ti.com/microcontrollers-mcus-processors/arm-based-processors/overview.html)
|
||||||
|
* [ML Commons Tiny repo](https://github.com/mlcommons/tiny)
|
||||||
|
* [Nemo keyword spotting models using Matchboxnet](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nemo/models/commandrecognition_en_matchboxnet3x2x64_v2)
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
#
|
||||||
|
# Copyright (C) 2023 Texas Instruments Incorporated - http://www.ti.com/
|
||||||
|
#
|
||||||
|
#
|
||||||
|
# Redistribution and use in source and binary forms, with or without
|
||||||
|
# modification, are permitted provided that the following conditions
|
||||||
|
# are met:
|
||||||
|
#
|
||||||
|
# Redistributions of source code must retain the above copyright
|
||||||
|
# notice, this list of conditions and the following disclaimer.
|
||||||
|
#
|
||||||
|
# Redistributions in binary form must reproduce the above copyright
|
||||||
|
# notice, this list of conditions and the following disclaimer in the
|
||||||
|
# documentation and/or other materials provided with the
|
||||||
|
# distribution.
|
||||||
|
#
|
||||||
|
# Neither the name of Texas Instruments Incorporated nor the names of
|
||||||
|
# its contributors may be used to endorse or promote products derived
|
||||||
|
# from this software without specific prior written permission.
|
||||||
|
#
|
||||||
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||||
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||||
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||||
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||||
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||||
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||||
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||||
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
#
|
||||||
|
import pyaudio
|
||||||
|
|
||||||
|
audio = pyaudio.PyAudio()
|
||||||
|
|
||||||
|
dev_info = audio.get_host_api_info_by_index(0)
|
||||||
|
num_dev = dev_info.get('deviceCount')
|
||||||
|
|
||||||
|
for k in range(0, num_dev):
|
||||||
|
if (audio.get_device_info_by_host_api_device_index(0, k).get('maxInputChannels')) > 0:
|
||||||
|
dev_name = audio.get_device_info_by_host_api_device_index(0, k).get('name')
|
||||||
|
print("Audio Input ID and name: ", k, " - ", dev_name)
|
||||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,52 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Copyright (C) 2023 Texas Instruments Incorporated - http://www.ti.com/
|
||||||
|
#
|
||||||
|
#
|
||||||
|
# Redistribution and use in source and binary forms, with or without
|
||||||
|
# modification, are permitted provided that the following conditions
|
||||||
|
# are met:
|
||||||
|
#
|
||||||
|
# Redistributions of source code must retain the above copyright
|
||||||
|
# notice, this list of conditions and the following disclaimer.
|
||||||
|
#
|
||||||
|
# Redistributions in binary form must reproduce the above copyright
|
||||||
|
# notice, this list of conditions and the following disclaimer in the
|
||||||
|
# documentation and/or other materials provided with the
|
||||||
|
# distribution.
|
||||||
|
#
|
||||||
|
# Neither the name of Texas Instruments Incorporated nor the names of
|
||||||
|
# its contributors may be used to endorse or promote products derived
|
||||||
|
# from this software without specific prior written permission.
|
||||||
|
#
|
||||||
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||||
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||||
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||||
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||||
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||||
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||||
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||||
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
#
|
||||||
|
echo "Running setup for audio processing. This requires a network connection"
|
||||||
|
|
||||||
|
if [[ ! -d ./portaudio ]]; then
|
||||||
|
echo "portaudio not found; buidling portaudio"
|
||||||
|
git clone https://github.com/PortAudio/portaudio/
|
||||||
|
cd portaudio
|
||||||
|
./configure
|
||||||
|
make -j
|
||||||
|
make install
|
||||||
|
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
|
||||||
|
echo "built and installed portaudio to /usr/local/lib, and exported to library path"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "install librosa and pyaudio for audio processing and audio capture, respectively"
|
||||||
|
PROXY_INFO=""
|
||||||
|
#PROXY_INFO= " --proxy http://webproxy.ext.ti.com:80 "
|
||||||
|
pip3 --trusted-host pypi.org --trusted-host pypi.python.org --trusted-host files.pythonhosted.org $PROXY_INFO install librosa pyaudio soundfile
|
||||||
|
|
||||||
|
echo "Finished setup"
|
||||||
|
|
||||||
@@ -0,0 +1,227 @@
|
|||||||
|
#
|
||||||
|
# Copyright (C) 2023 Texas Instruments Incorporated - http://www.ti.com/
|
||||||
|
#
|
||||||
|
#
|
||||||
|
# Redistribution and use in source and binary forms, with or without
|
||||||
|
# modification, are permitted provided that the following conditions
|
||||||
|
# are met:
|
||||||
|
#
|
||||||
|
# Redistributions of source code must retain the above copyright
|
||||||
|
# notice, this list of conditions and the following disclaimer.
|
||||||
|
#
|
||||||
|
# Redistributions in binary form must reproduce the above copyright
|
||||||
|
# notice, this list of conditions and the following disclaimer in the
|
||||||
|
# documentation and/or other materials provided with the
|
||||||
|
# distribution.
|
||||||
|
#
|
||||||
|
# Neither the name of Texas Instruments Incorporated nor the names of
|
||||||
|
# its contributors may be used to endorse or promote products derived
|
||||||
|
# from this software without specific prior written permission.
|
||||||
|
#
|
||||||
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||||
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||||
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||||
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||||
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||||
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||||
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||||
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
#
|
||||||
|
'''
|
||||||
|
This source file is a standalone application for running live inference for
|
||||||
|
keyword spotting on audio data from a microphone connected in linux
|
||||||
|
'''
|
||||||
|
import os, time, sys
|
||||||
|
import pyaudio
|
||||||
|
import numpy as np
|
||||||
|
import librosa
|
||||||
|
import tflite_runtime.interpreter as tflite
|
||||||
|
import soundfile
|
||||||
|
import math
|
||||||
|
import scipy.fftpack
|
||||||
|
|
||||||
|
p = pyaudio.PyAudio()
|
||||||
|
|
||||||
|
class AudioInference(object):
|
||||||
|
PROCESSING_RATE = 16000 #Hz
|
||||||
|
NUM_MFCC_BINS = 49
|
||||||
|
NUM_MFCC_PER_BIN = 10
|
||||||
|
BIN_WINDOW_SIZE = int(PROCESSING_RATE * 0.03)
|
||||||
|
BIN_WINDOW_STEP = int(PROCESSING_RATE * 0.02)
|
||||||
|
def __init__(self, modeldir, modelname, rate=48000, data_format=pyaudio.paInt16, channels=1, device_index=1):
|
||||||
|
'''
|
||||||
|
@param modeldir: The directory the model is contained in
|
||||||
|
@param modelname: the name of the modelfile within the modeldir
|
||||||
|
@param rate: the sampling rate of the microphone
|
||||||
|
@param data_format: the type of the incoming audio samples
|
||||||
|
@param channels: The number of channels supplied by the microphone. In general, only 1 should be used
|
||||||
|
@param device_index: The index of the microphone as understood by linux. Use detect_microphone.py is this is unknown, but it should be 1 in general
|
||||||
|
'''
|
||||||
|
print('initializing inference session')
|
||||||
|
self.rate=rate
|
||||||
|
self.format=data_format
|
||||||
|
self.channels=channels
|
||||||
|
self.device_index=device_index
|
||||||
|
|
||||||
|
self.input_stream = None
|
||||||
|
|
||||||
|
modelpath = os.path.join(modeldir, modelname)
|
||||||
|
|
||||||
|
# the set of words within this KWS model's dictionary. The ordering is important
|
||||||
|
self.word_labels = ["Down", "Go", "Left", "No", "Off", "On", "Right", "Stop", "Up", "Yes", "Silence", "Unknown"]
|
||||||
|
|
||||||
|
print('loading KWS model...')
|
||||||
|
self.interpreter = tflite.Interpreter(modelpath, num_threads=1)
|
||||||
|
self.interpreter.allocate_tensors()
|
||||||
|
self.input_details = self.interpreter.get_input_details()
|
||||||
|
self.output_details = self.interpreter.get_output_details()
|
||||||
|
print('loaded.')
|
||||||
|
|
||||||
|
def setup(self):
|
||||||
|
'''
|
||||||
|
Configure the input data stream
|
||||||
|
'''
|
||||||
|
self.inference_session = None
|
||||||
|
|
||||||
|
seconds_per_chunk = 0.5
|
||||||
|
chunk_size = int(self.rate * seconds_per_chunk)
|
||||||
|
self.last_chunk = None
|
||||||
|
print('opening input audio stream...')
|
||||||
|
self.input_stream = p.open(rate=self.rate, channels=self.channels, format=self.format, input=True, input_device_index=self.device_index, output=False, stream_callback=self.inference_callback, frames_per_buffer=chunk_size)
|
||||||
|
print('opened.')
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
self.input_stream.close()
|
||||||
|
|
||||||
|
def calculate_features(self, audio_data, sr=PROCESSING_RATE):
|
||||||
|
'''
|
||||||
|
The default configuration of MFCC using librosa does not match what tensorflow's recommended method of calculating MFFC's.
|
||||||
|
|
||||||
|
Through trial, error, and analysis, the following differences were found between the implementation in the keyword-spotting preprocessing script (which is almost identical to the TF MFCC code (https://www.tensorflow.org/api_docs/python/tf/signal/mfccs_from_log_mel_spectrograms)) and the recommend/default librosa implementation:
|
||||||
|
1. STFT differs when n_fft samples != window_size
|
||||||
|
2. Mel filters in frequency space are different due to normalization
|
||||||
|
3. In mel-spectrogram, librosa uses power-decibel scale to get log-mel whereas tensorflow implementation uses natural log
|
||||||
|
|
||||||
|
|
||||||
|
Since preprocessing must match to achieve the same results, this implementation of MFCC calculation changes several parameters to match how it was done in tensorflow
|
||||||
|
|
||||||
|
'''
|
||||||
|
n_fft = np.log2(AudioInference.BIN_WINDOW_SIZE)
|
||||||
|
n_fft = 2**math.ceil(n_fft)
|
||||||
|
|
||||||
|
#although the actual win_length should not be =n_fft per the preprocessing instructions, the reality is that librosa's result DOES NOT MATCH tensorflow unless n_fft=win_length
|
||||||
|
stft = librosa.core.stft(y=audio_data.reshape((-1)),
|
||||||
|
n_fft=n_fft,
|
||||||
|
hop_length=AudioInference.BIN_WINDOW_STEP,
|
||||||
|
win_length=n_fft,
|
||||||
|
center=False)
|
||||||
|
|
||||||
|
spectrogram = np.abs(stft) # don't compute **2 for power since TF implmentation skipped this
|
||||||
|
|
||||||
|
#calculate a mel filter transformation matrix
|
||||||
|
lower_edge_hertz, upper_edge_hertz, num_mel_bins = 20.0, 4000.0, 40
|
||||||
|
lin_to_mel_matrix = librosa.filters.mel(sr=sr, n_fft=n_fft, n_mels=num_mel_bins, fmin=lower_edge_hertz, fmax=upper_edge_hertz, htk=True, norm=None)
|
||||||
|
# multiply matrices to get melspectorgram
|
||||||
|
melspectrograms = np.dot(lin_to_mel_matrix, spectrogram)
|
||||||
|
|
||||||
|
# use natrual log instead of dB power scale, per TF implementation
|
||||||
|
log_mel_spectrograms = np.log(melspectrograms + 1e-6)
|
||||||
|
|
||||||
|
# DCT with orthogonal normalization convert log-mel-spectrogram to mel cepstrum coefficients (MFCC)
|
||||||
|
mfcc = scipy.fftpack.dct(log_mel_spectrograms, axis=-2, type=2, norm='ortho')[...,:10,:]
|
||||||
|
return mfcc
|
||||||
|
|
||||||
|
def run_inference(self, mfcc):
|
||||||
|
'''
|
||||||
|
Run inference on the MFCC's that represent a second of audio
|
||||||
|
'''
|
||||||
|
tensor = np.zeros((1, AudioInference.NUM_MFCC_BINS, AudioInference.NUM_MFCC_PER_BIN, 1)) #shape 49,10
|
||||||
|
tensor[0,:,:,0] = mfcc.transpose() # extend shape to agree with model expectation
|
||||||
|
self.interpreter.set_tensor(self.input_details[0]['index'], tensor.astype(np.float32))
|
||||||
|
|
||||||
|
t1 = time.time_ns()//1000 / 1000
|
||||||
|
self.interpreter.invoke()
|
||||||
|
t2 = time.time_ns()//1000 / 1000
|
||||||
|
print("Inference Time is %0.4f ms" % (t2-t1))
|
||||||
|
|
||||||
|
class_probs = self.interpreter.get_tensor(self.output_details[0]['index'])[0]
|
||||||
|
return class_probs
|
||||||
|
|
||||||
|
def convert_audio_for_features(self, raw_input, input_rate, output_rate=PROCESSING_RATE):
|
||||||
|
'''
|
||||||
|
Convert the input from the microphone into a form that the primary preprocessing function (calculate features) is prepared to use
|
||||||
|
|
||||||
|
:param raw_input: The raw buffer of audio samples
|
||||||
|
:param input_rate: the sampling rate of the audio samples
|
||||||
|
:param output_rate: The desired sample-rate of the output buffer of audio
|
||||||
|
'''
|
||||||
|
audio_data = raw_input / max([np.max(raw_input),abs(np.min(raw_input))]) #normalize to [-1:1]
|
||||||
|
|
||||||
|
audio_resample = librosa.resample(audio_data.astype(np.float32), orig_sr=input_rate, target_sr=output_rate)
|
||||||
|
|
||||||
|
return audio_resample
|
||||||
|
|
||||||
|
|
||||||
|
def inference_callback(self, audio_buffer, frame_count, time_info, flag):
|
||||||
|
'''
|
||||||
|
A callback registered to pyaudio for running the main body of this application. Most parameters are unused. It is required to return and audio sample and a 'continuation' flag
|
||||||
|
'''
|
||||||
|
if self.last_chunk is None:
|
||||||
|
print('Skipping first chunk... typically takes a moment for librosa to initialize')
|
||||||
|
else:
|
||||||
|
t1 = time.time_ns()//1000/1000
|
||||||
|
#audio data is collected in 500 ms samples and processed in 1000 ms chunks, i.e. a 50% sliding window
|
||||||
|
audio_data = np.frombuffer(self.last_chunk+audio_buffer, dtype=np.int16)
|
||||||
|
|
||||||
|
|
||||||
|
audio_resample = self.convert_audio_for_features(audio_data, input_rate = self.rate, output_rate=AudioInference.PROCESSING_RATE)
|
||||||
|
|
||||||
|
mfcc = self.calculate_features(audio_resample)
|
||||||
|
|
||||||
|
t2 = time.time_ns()//1000/1000
|
||||||
|
print("Preprocess Time is %0.3f ms" % (t2-t1))
|
||||||
|
print('run inference next..')
|
||||||
|
|
||||||
|
class_probs = self.run_inference(mfcc)
|
||||||
|
|
||||||
|
c = np.argmax(class_probs)
|
||||||
|
print('******\ndetected class: ' + str(self.word_labels[c]) + '\n******')
|
||||||
|
|
||||||
|
print('finished with inference\n')
|
||||||
|
|
||||||
|
self.last_chunk = audio_buffer
|
||||||
|
|
||||||
|
return self.last_chunk, pyaudio.paContinue
|
||||||
|
|
||||||
|
|
||||||
|
def main(modeldir, modelname):
|
||||||
|
print('main in audio-inference-tinykws')
|
||||||
|
audio = AudioInference(modeldir=modeldir, modelname=modelname, device_index=1, )
|
||||||
|
audio.setup()
|
||||||
|
|
||||||
|
while (audio.input_stream.is_active()): time.sleep(2)
|
||||||
|
|
||||||
|
audio.stop()
|
||||||
|
|
||||||
|
def test_on_file(modeldir, modelname, audio_filename='no_0cb74144_nohash_1.wav'):
|
||||||
|
'''
|
||||||
|
Test the model on an input file instead of live input from microphone. This file should be 1 second of audio long.
|
||||||
|
'''
|
||||||
|
audio_inf = AudioInference(modeldir=modeldir, modelname=modelname, device_index=14, )
|
||||||
|
|
||||||
|
audio_data, sr = soundfile.read(audio_filename)
|
||||||
|
|
||||||
|
|
||||||
|
audio_resampled = audio_inf.convert_audio_for_features(audio_data, sr)
|
||||||
|
mfcc = audio_inf.calculate_features(audio_resampled)
|
||||||
|
|
||||||
|
class_prob = audio_inf.run_inference(mfcc)
|
||||||
|
c = np.argmax(class_prob)
|
||||||
|
print('******\ndetected class: ' + str(audio_inf.word_labels[c]) + '\n******')
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main('./models', 'kws_ref_model_float32.tflite')
|
||||||
|
# test_on_file('./models', 'kws_ref_model_float32.tflite')
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,67 @@
|
|||||||
|
#
|
||||||
|
# Copyright (C) 2023 Texas Instruments Incorporated - http://www.ti.com/
|
||||||
|
#
|
||||||
|
#
|
||||||
|
# Redistribution and use in source and binary forms, with or without
|
||||||
|
# modification, are permitted provided that the following conditions
|
||||||
|
# are met:
|
||||||
|
#
|
||||||
|
# Redistributions of source code must retain the above copyright
|
||||||
|
# notice, this list of conditions and the following disclaimer.
|
||||||
|
#
|
||||||
|
# Redistributions in binary form must reproduce the above copyright
|
||||||
|
# notice, this list of conditions and the following disclaimer in the
|
||||||
|
# documentation and/or other materials provided with the
|
||||||
|
# distribution.
|
||||||
|
#
|
||||||
|
# Neither the name of Texas Instruments Incorporated nor the names of
|
||||||
|
# its contributors may be used to endorse or promote products derived
|
||||||
|
# from this software without specific prior written permission.
|
||||||
|
#
|
||||||
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||||
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||||
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||||
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||||
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||||
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||||
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||||
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
#
|
||||||
|
'''
|
||||||
|
This source file is intended to test audio input using pyaudio and save collected
|
||||||
|
audio to a .wav file. The tested device was a FIFINE K669 USB microphone with
|
||||||
|
48kHz sample rate
|
||||||
|
'''
|
||||||
|
|
||||||
|
import pyaudio
|
||||||
|
import numpy as np
|
||||||
|
import soundfile
|
||||||
|
import time
|
||||||
|
|
||||||
|
device_index = 1
|
||||||
|
input_rate = 48000
|
||||||
|
seconds_per_run = 5
|
||||||
|
|
||||||
|
p = pyaudio.PyAudio()
|
||||||
|
|
||||||
|
#get audio
|
||||||
|
stream = p.open(rate=input_rate, channels=1, format=pyaudio.paInt16, input=True, input_device_index=device_index)
|
||||||
|
|
||||||
|
print('Start recording')
|
||||||
|
audio_data = stream.read(num_frames=input_rate*seconds_per_run, exception_on_overflow = False)
|
||||||
|
|
||||||
|
print('Stopped recording')
|
||||||
|
|
||||||
|
t1 = time.time_ns()
|
||||||
|
audio_formatted = np.frombuffer(audio_data, dtype=np.int16) #convert from byte stream to audio samples. Littleendian signed 16bit ints#output into floats
|
||||||
|
t2 = time.time_ns()
|
||||||
|
print(audio_formatted.shape)
|
||||||
|
|
||||||
|
|
||||||
|
print('Processed audio in %f ms' % ((t2-t1)/1e6))
|
||||||
|
print('Audio resampled')
|
||||||
|
|
||||||
|
soundfile.write('processed_audio.wav', audio_formatted, input_rate)
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
#
|
||||||
|
# Copyright (C) 2023 Texas Instruments Incorporated - http://www.ti.com/
|
||||||
|
#
|
||||||
|
#
|
||||||
|
# Redistribution and use in source and binary forms, with or without
|
||||||
|
# modification, are permitted provided that the following conditions
|
||||||
|
# are met:
|
||||||
|
#
|
||||||
|
# Redistributions of source code must retain the above copyright
|
||||||
|
# notice, this list of conditions and the following disclaimer.
|
||||||
|
#
|
||||||
|
# Redistributions in binary form must reproduce the above copyright
|
||||||
|
# notice, this list of conditions and the following disclaimer in the
|
||||||
|
# documentation and/or other materials provided with the
|
||||||
|
# distribution.
|
||||||
|
#
|
||||||
|
# Neither the name of Texas Instruments Incorporated nor the names of
|
||||||
|
# its contributors may be used to endorse or promote products derived
|
||||||
|
# from this software without specific prior written permission.
|
||||||
|
#
|
||||||
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||||
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||||
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||||
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||||
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||||
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||||
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||||
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
#
|
||||||
|
'''
|
||||||
|
This source file is used to compare MFCC generation methods between tensorflow
|
||||||
|
and librosa, since the latter was found to give different results for effectively
|
||||||
|
identical parameters. This file looks at each step of the preprocessing algorithm
|
||||||
|
to understand where the two methods deviate. It is not crucial to use this file,
|
||||||
|
but it does provide some interesting insight into the MFCC calculation process
|
||||||
|
|
||||||
|
This is intended to run on a development machine, not an EVM / TI SoC
|
||||||
|
'''
|
||||||
|
|
||||||
|
import tensorflow as tf
|
||||||
|
import numpy as np
|
||||||
|
import librosa
|
||||||
|
import soundfile
|
||||||
|
import scipy
|
||||||
|
from matplotlib import pyplot as plt
|
||||||
|
|
||||||
|
audio_data, sr = soundfile.read('down_0c40e715_nohash_0.wav')
|
||||||
|
audio_data = audio_data.astype(np.float32)
|
||||||
|
|
||||||
|
np.set_printoptions(precision=4, suppress=True)
|
||||||
|
|
||||||
|
fft_size = 512
|
||||||
|
frame_size = 512
|
||||||
|
hop_size = 320
|
||||||
|
|
||||||
|
tf_stft = tf.signal.stft(audio_data, frame_length=frame_size, fft_length=fft_size, frame_step=hop_size)
|
||||||
|
|
||||||
|
tf_spectrogram = magnitudes = tf.abs(tf_stft)
|
||||||
|
|
||||||
|
lr_stft = librosa.core.stft(y=audio_data.reshape((-1)),
|
||||||
|
n_fft=fft_size,
|
||||||
|
hop_length=hop_size,
|
||||||
|
win_length=frame_size,
|
||||||
|
center=False)
|
||||||
|
|
||||||
|
lr_spectrogram = np.abs(lr_stft)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
difference = np.abs(tf_spectrogram - lr_spectrogram.T)
|
||||||
|
print("STFT Differences:\nmin:", np.min(difference),
|
||||||
|
"max:", np.max(difference),
|
||||||
|
"mean:", np.mean(difference),
|
||||||
|
"std:", np.std(difference))
|
||||||
|
|
||||||
|
print('do TF melspec')
|
||||||
|
lower_edge_hertz, upper_edge_hertz, num_mel_bins = 20.0, 4000.0, 40
|
||||||
|
tf_lin_to_mel_matrix = tf.signal.linear_to_mel_weight_matrix( num_mel_bins, fft_size//2+1, sr,lower_edge_hertz, upper_edge_hertz)
|
||||||
|
tf_mel_spectrograms = tf.tensordot(tf_spectrogram, tf_lin_to_mel_matrix, 1)
|
||||||
|
print(tf_mel_spectrograms.shape)
|
||||||
|
tf_mel_spectrograms.set_shape(tf_spectrogram.shape[:-1].concatenate(
|
||||||
|
tf_lin_to_mel_matrix.shape[-1:]))
|
||||||
|
print(tf_mel_spectrograms.shape)
|
||||||
|
|
||||||
|
print('do LR melspec')
|
||||||
|
# lr_melspectrograms = lr_melspectrograms.T
|
||||||
|
|
||||||
|
lr_melspectrograms = librosa.feature.melspectrogram(S=lr_spectrogram**2, center=False, sr=sr, n_fft=fft_size, fmin=lower_edge_hertz, fmax=upper_edge_hertz, n_mels=num_mel_bins, htk=True)
|
||||||
|
difference = np.abs(tf_mel_spectrograms - lr_melspectrograms.T)
|
||||||
|
print("\nmelspectrogram Differences (LR melspectrogram function):\nmin:", np.min(difference),
|
||||||
|
"max:", np.max(difference),
|
||||||
|
"mean:", np.mean(difference),
|
||||||
|
"std:", np.std(difference))
|
||||||
|
|
||||||
|
lr_melspectrograms = np.dot(lr_spectrogram.T, tf_lin_to_mel_matrix.numpy())
|
||||||
|
difference = np.abs(tf_mel_spectrograms - lr_melspectrograms)
|
||||||
|
print("\nmelspectrogram Differences (tf mel filters mtx):\nmin:", np.min(difference),
|
||||||
|
"max:", np.max(difference),
|
||||||
|
"mean:", np.mean(difference),
|
||||||
|
"std:", np.std(difference))
|
||||||
|
|
||||||
|
lr_lin_to_mel_matrix = librosa.filters.mel(sr=sr, n_fft=fft_size, n_mels=num_mel_bins, fmin=lower_edge_hertz, fmax=upper_edge_hertz, htk=True, norm=None)
|
||||||
|
lr_melspectrograms = np.dot(lr_spectrogram.T, lr_lin_to_mel_matrix.T)
|
||||||
|
lr_melspectrograms = np.dot(lr_lin_to_mel_matrix, lr_spectrogram)
|
||||||
|
difference = np.abs(tf_mel_spectrograms - lr_melspectrograms.T)
|
||||||
|
print("\nmelspectrogram Differences (filters.mel):\nmin:", np.min(difference),
|
||||||
|
"max:", np.max(difference),
|
||||||
|
"mean:", np.mean(difference),
|
||||||
|
"std:", np.std(difference))
|
||||||
|
|
||||||
|
print('do log mel spectrograms')
|
||||||
|
tf_log_mel_spectrograms = tf.math.log(tf_mel_spectrograms + 1e-6)
|
||||||
|
lr_log_mel_spectrograms = np.log(lr_melspectrograms + 1e-6)
|
||||||
|
|
||||||
|
difference = np.abs(tf_log_mel_spectrograms - lr_log_mel_spectrograms.T)
|
||||||
|
print("\nlog-mel Differences :\nmin:", np.min(difference),
|
||||||
|
"max:", np.max(difference),
|
||||||
|
"mean:", np.mean(difference),
|
||||||
|
"std:", np.std(difference))
|
||||||
|
|
||||||
|
|
||||||
|
tf_mfcc = tf.signal.mfccs_from_log_mel_spectrograms(tf_log_mel_spectrograms)
|
||||||
|
lr_mfcc = scipy.fftpack.dct(lr_log_mel_spectrograms, axis=-2, type=2, norm='ortho')
|
||||||
|
|
||||||
|
difference = np.abs(tf_mfcc - lr_mfcc.T)
|
||||||
|
print("\nmfcc Differences :\nmin:", np.min(difference),
|
||||||
|
"max:", np.max(difference),
|
||||||
|
"mean:", np.mean(difference),
|
||||||
|
"std:", np.std(difference))
|
||||||
|
|
||||||
|
print(tf_mfcc[0,:])
|
||||||
|
print(lr_mfcc[0,:])
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
fig, ax = plt.subplots(nrows=2, sharex=True, sharey=True)
|
||||||
|
tf_mfcc_plot = tf_mfcc.numpy()
|
||||||
|
print(tf_mfcc_plot.shape)
|
||||||
|
print(lr_mfcc.shape)
|
||||||
|
|
||||||
|
img1 = librosa.display.specshow(tf_mfcc_plot, x_axis='time', ax=ax[0])
|
||||||
|
|
||||||
|
ax[0].set(title='tensorflow mfcc')
|
||||||
|
|
||||||
|
fig.colorbar(img1, ax=[ax[0]])
|
||||||
|
|
||||||
|
img2 = librosa.display.specshow(lr_mfcc.T, x_axis='time', ax=ax[1])
|
||||||
|
|
||||||
|
ax[1].set(title='librosa mfcc')
|
||||||
|
|
||||||
|
fig.colorbar(img2, ax=[ax[1]])
|
||||||
|
|
||||||
|
plt.show()
|
||||||
@@ -0,0 +1,180 @@
|
|||||||
|
#
|
||||||
|
# Copyright (C) 2023 Texas Instruments Incorporated - http://www.ti.com/
|
||||||
|
#
|
||||||
|
#
|
||||||
|
# Redistribution and use in source and binary forms, with or without
|
||||||
|
# modification, are permitted provided that the following conditions
|
||||||
|
# are met:
|
||||||
|
#
|
||||||
|
# Redistributions of source code must retain the above copyright
|
||||||
|
# notice, this list of conditions and the following disclaimer.
|
||||||
|
#
|
||||||
|
# Redistributions in binary form must reproduce the above copyright
|
||||||
|
# notice, this list of conditions and the following disclaimer in the
|
||||||
|
# documentation and/or other materials provided with the
|
||||||
|
# distribution.
|
||||||
|
#
|
||||||
|
# Neither the name of Texas Instruments Incorporated nor the names of
|
||||||
|
# its contributors may be used to endorse or promote products derived
|
||||||
|
# from this software without specific prior written permission.
|
||||||
|
#
|
||||||
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||||
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||||
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||||
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||||
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||||
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||||
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||||
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
#
|
||||||
|
'''
|
||||||
|
This source file is a standalone application for running live inference for
|
||||||
|
keyword spotting on audio data from a microphone connected in linux
|
||||||
|
'''
|
||||||
|
|
||||||
|
import os, time
|
||||||
|
import pyaudio
|
||||||
|
import numpy as np
|
||||||
|
import librosa
|
||||||
|
import onnxruntime as ort
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
p = pyaudio.PyAudio()
|
||||||
|
|
||||||
|
class AudioInference(object):
|
||||||
|
PROCESSING_RATE = 16000 #Hz
|
||||||
|
NUM_MFCC_BINS = 101
|
||||||
|
NUM_MFCC_PER_BIN = 64
|
||||||
|
BIN_WINDOW_SIZE = int(PROCESSING_RATE * 0.025)
|
||||||
|
BIN_WINDOW_STEP = int(PROCESSING_RATE * 0.01)
|
||||||
|
LOGIT_THRESHOLD = 12 #arbitrary..
|
||||||
|
def __init__(self, modeldir, modelname, rate=48000, data_format=pyaudio.paInt16, channels=1, device_index=1, labels_file='labels.yaml'):
|
||||||
|
print('setup')
|
||||||
|
self.rate=rate
|
||||||
|
self.format=data_format
|
||||||
|
self.channels=channels
|
||||||
|
self.device_index=device_index
|
||||||
|
|
||||||
|
self.input_stream = None
|
||||||
|
|
||||||
|
modelpath = os.path.join(modeldir, modelname)
|
||||||
|
|
||||||
|
with open(labels_file,'r') as f:
|
||||||
|
self.word_labels = yaml.safe_load(f)['labels']
|
||||||
|
|
||||||
|
self.sess_options = ort.SessionOptions()
|
||||||
|
self.interpreter = interpreter = ort.InferenceSession(modelpath, providers=['CPUExecutionProvider'], provider_options=[{}], sess_options=self.sess_options)
|
||||||
|
self.input_details = self.interpreter.get_inputs()
|
||||||
|
|
||||||
|
|
||||||
|
def setup(self):
|
||||||
|
self.inference_session = None
|
||||||
|
|
||||||
|
seconds_per_chunk = 0.5
|
||||||
|
chunk_size = int(self.rate * seconds_per_chunk)
|
||||||
|
self.last_chunk = None
|
||||||
|
print('open input stream')
|
||||||
|
self.input_stream = p.open(rate=self.rate, channels=self.channels, format=self.format, input=True, input_device_index=self.device_index, output=False, stream_callback=self.inference_callback, frames_per_buffer=chunk_size)
|
||||||
|
print('opened..')
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
self.input_stream.close()
|
||||||
|
|
||||||
|
def calculate_features(self, audio_data, sr=PROCESSING_RATE):
|
||||||
|
'''
|
||||||
|
Calculate features from one second of audio data at sampling rate sr
|
||||||
|
|
||||||
|
|
||||||
|
'''
|
||||||
|
n_fft = 512
|
||||||
|
n_mels = 64
|
||||||
|
n_mfcc = 64
|
||||||
|
|
||||||
|
melspec = librosa.feature.melspectrogram(y=audio_data, sr=sr, n_fft=n_fft, win_length=AudioInference.BIN_WINDOW_SIZE, hop_length=AudioInference.BIN_WINDOW_STEP, n_mels=n_mels, power=2, center=True, htk=True, norm=None)
|
||||||
|
|
||||||
|
# print(melspec)
|
||||||
|
# S = lr.power_to_db(melspec)
|
||||||
|
S = np.log(melspec + 1e-6)
|
||||||
|
|
||||||
|
mfcc = librosa.feature.mfcc(S=S, norm='ortho', n_mfcc=n_mfcc)
|
||||||
|
return mfcc
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def run_inference(self, mfcc):
|
||||||
|
|
||||||
|
mfcc = mfcc[None,:]
|
||||||
|
t1 = time.time_ns()//1000/1000
|
||||||
|
result = self.interpreter.run(None, {self.input_details[0].name: mfcc})
|
||||||
|
t2 = time.time_ns()//1000/1000
|
||||||
|
print("Inference Time is %0.3f ms" % (t2-t1))
|
||||||
|
|
||||||
|
if np.max(result[0][0,:]) > AudioInference.LOGIT_THRESHOLD:
|
||||||
|
best_class = int(np.argmax(result[0][0,:]))
|
||||||
|
else: best_class = -1
|
||||||
|
return best_class, result
|
||||||
|
|
||||||
|
def convert_audio_for_features(self, raw_input, input_rate, output_rate=PROCESSING_RATE):
|
||||||
|
audio_data = raw_input / max([np.max(raw_input),abs(np.min(raw_input))]) #normalize to [-1:1]
|
||||||
|
|
||||||
|
audio_resample = librosa.resample(audio_data.astype(np.float32), orig_sr=input_rate, target_sr=output_rate)
|
||||||
|
|
||||||
|
return audio_resample
|
||||||
|
|
||||||
|
|
||||||
|
def inference_callback(self, audio_buffer, frame_count, time_info, flag):
|
||||||
|
if self.last_chunk is None:
|
||||||
|
print('Skipping first chunk... typically takes a moment for librosa to initialize')
|
||||||
|
else:
|
||||||
|
t1 = time.time_ns()//1000/1000
|
||||||
|
audio_data = np.frombuffer(self.last_chunk+audio_buffer, dtype=np.int16)
|
||||||
|
|
||||||
|
|
||||||
|
audio_resample = self.convert_audio_for_features(audio_data, input_rate = self.rate, output_rate=AudioInference.PROCESSING_RATE)
|
||||||
|
|
||||||
|
mfcc = self.calculate_features(audio_resample)
|
||||||
|
|
||||||
|
t2 = time.time_ns()//1000/1000
|
||||||
|
print("Preprocess Time is %0.3f ms" % (t2-t1))
|
||||||
|
|
||||||
|
best_class, class_logits = self.run_inference(mfcc)
|
||||||
|
class_name = 'unknown' if best_class < 0 else self.word_labels[best_class]
|
||||||
|
|
||||||
|
print('******detected speech: ' + class_name + '******\n')
|
||||||
|
# print(class_logits)
|
||||||
|
# print(np.max(class_logits))
|
||||||
|
self.last_chunk = audio_buffer
|
||||||
|
|
||||||
|
return self.last_chunk, pyaudio.paContinue
|
||||||
|
|
||||||
|
# audio_data = stream.read(num_frames=input_rate*seconds_per_run, exception_on_overflow = False)
|
||||||
|
|
||||||
|
def main(modeldir, modelname):
|
||||||
|
print('main')
|
||||||
|
audio = AudioInference(modeldir=modeldir, modelname=modelname, device_index=14, )
|
||||||
|
audio.setup()
|
||||||
|
|
||||||
|
while (audio.input_stream.is_active()): time.sleep(5)
|
||||||
|
|
||||||
|
audio.stop()
|
||||||
|
|
||||||
|
def test_on_file(modeldir, modelname):
|
||||||
|
import soundfile
|
||||||
|
audio_inf = AudioInference(modeldir=modeldir, modelname=modelname, device_index=1, )
|
||||||
|
|
||||||
|
audio_data, sr = soundfile.read('no_0cb74144_nohash_1.wav')
|
||||||
|
|
||||||
|
|
||||||
|
audio_resampled = audio_inf.convert_audio_for_features(audio_data, sr)
|
||||||
|
mfcc = audio_inf.calculate_features(audio_resampled)
|
||||||
|
|
||||||
|
best_class, class_logits = audio_inf.run_inference(mfcc)
|
||||||
|
class_name = 'unknown' if best_class < 0 else audio_inf.word_labels[best_class]
|
||||||
|
print('******\ndetected class: ' + class_name + '\n******')
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main('.', 'matchboxnet.onnx')
|
||||||
|
# test_on_file('./models', 'kws_ref_model_float32.tflite')
|
||||||
BIN
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user