over

CMakeLists.txt

@@ -35,10 +35,11 @@ set(CORE_SOURCES
 )
 
 if(TARGET_IMX)
-    list(APPEND CORE_SOURCES
+	list(APPEND CORE_SOURCES
-        src/Core/Platform/FBDisplay.cpp
+		src/Core/Platform/FBDisplay.cpp
-        src/Core/Platform/Ap3216cPhotoSensor.cpp
+		src/Core/Platform/Ap3216cPhotoSensor.cpp
-    )
+		src/Core/Platform/Imx6ullAlphaLed.cpp
+	)
 else()
     list(APPEND CORE_SOURCES
         src/Core/Platform/SDLDisplay.cpp

src/Apps/Game/CMakeLists.txt

@@ -2,22 +2,45 @@ set(TOM_GAME_TARGET IMX6U-Game)
 set(TOM_ATLAS_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/generated/tom_atlas.h")
 set_source_files_properties(${TOM_ATLAS_HEADER} PROPERTIES GENERATED TRUE)
 
+set(TOM_AUDIO_GENERATED_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated")
+set(TOM_AUDIO_HEADER "${TOM_AUDIO_GENERATED_DIR}/TomAudioData.h")
+set(TOM_AUDIO_SOURCE "${TOM_AUDIO_GENERATED_DIR}/TomAudioData.cpp")
+add_custom_command(
+    OUTPUT ${TOM_AUDIO_HEADER} ${TOM_AUDIO_SOURCE}
+    COMMAND ${CMAKE_COMMAND}
+        "-DOPEN_LIGHT_WAV=${CMAKE_CURRENT_SOURCE_DIR}/assets/audio/OpenLight.wav"
+        "-DCLOSE_LIGHT_WAV=${CMAKE_CURRENT_SOURCE_DIR}/assets/audio/CloseLight.wav"
+        "-DOUTPUT_HEADER=${TOM_AUDIO_HEADER}"
+        "-DOUTPUT_SOURCE=${TOM_AUDIO_SOURCE}"
+        -P "${CMAKE_CURRENT_SOURCE_DIR}/tools/EmbedTomAudio.cmake"
+    DEPENDS
+        "${CMAKE_CURRENT_SOURCE_DIR}/assets/audio/OpenLight.wav"
+        "${CMAKE_CURRENT_SOURCE_DIR}/assets/audio/CloseLight.wav"
+        "${CMAKE_CURRENT_SOURCE_DIR}/tools/EmbedTomAudio.cmake"
+    COMMENT "Embedding Tom voice response WAV files"
+    VERBATIM
+)
+set_source_files_properties(${TOM_AUDIO_HEADER} ${TOM_AUDIO_SOURCE} PROPERTIES GENERATED TRUE)
+
 add_executable(${TOM_GAME_TARGET}
     Main.cpp
     src/app/TomGameApp.cpp
     src/audio/VoiceEffect.cpp
     src/audio/VoicePlayer.cpp
     src/audio/VoiceRecorder.cpp
+    src/audio/WavAudioData.cpp
     src/gameplay/KeywordCommandRouter.cpp
     src/gameplay/SpriteButton.cpp
     src/gameplay/TomAnimator.cpp
     src/gameplay/TomHud.cpp
     src/gameplay/VoiceInteractionController.cpp
     src/recognition/KeywordRecognizer.cpp
-    src/recognition/ResBnKwsRecognizer.cpp
+    src/recognition/ResBnKwsCnRecognizer.cpp
-    src/recognition/ResBnKwsModelData.cpp
+    src/recognition/ResBnKwsCnModelData.cpp
     src/ui/TomSettingsPanel.cpp
     ${TOM_ATLAS_HEADER}
+    ${TOM_AUDIO_HEADER}
+    ${TOM_AUDIO_SOURCE}
 )
 
 target_include_directories(${TOM_GAME_TARGET} PRIVATE
@@ -28,10 +51,29 @@ target_include_directories(${TOM_GAME_TARGET} PRIVATE
     ${CMAKE_CURRENT_SOURCE_DIR}/src/recognition
     ${CMAKE_CURRENT_SOURCE_DIR}/src/ui
     ${CMAKE_CURRENT_SOURCE_DIR}/generated
+    ${TOM_AUDIO_GENERATED_DIR}
 )
 
 imx6u_configure_app_target(${TOM_GAME_TARGET})
 
+if(BUILD_TESTING AND NOT TARGET_IMX)
+    add_executable(TomAudioDataTests EXCLUDE_FROM_ALL
+        tests/TomAudioDataTests.cpp
+        src/audio/WavAudioData.cpp
+        src/gameplay/KeywordCommandRouter.cpp
+        src/recognition/KeywordRecognizer.cpp
+        ${TOM_AUDIO_HEADER}
+        ${TOM_AUDIO_SOURCE}
+    )
+    target_include_directories(TomAudioDataTests PRIVATE
+        ${CMAKE_CURRENT_SOURCE_DIR}/src/audio
+        ${CMAKE_CURRENT_SOURCE_DIR}/src/gameplay
+        ${CMAKE_CURRENT_SOURCE_DIR}/src/recognition
+        ${TOM_AUDIO_GENERATED_DIR}
+    )
+    add_test(NAME tom_audio_data COMMAND TomAudioDataTests)
+endif()
+
 if(CMAKE_CROSSCOMPILING AND NOT EXISTS "${TOM_ATLAS_HEADER}")
     message(FATAL_ERROR
         "Tom atlas header is missing. Run GenerateTomAtlasHeader in a host build before cross compiling."

src/Apps/Game/Main.cpp

@@ -13,7 +13,7 @@
 #include "Timer.h"
 #include "app/TomGameApp.h"
 #include "recognition/KeywordRecognizer.h"
-#include "recognition/ResBnKwsRecognizer.h"
+#include "recognition/ResBnKwsCnRecognizer.h"
 
 #ifdef TARGET_IMX
 #include "FBDisplay.h"
@@ -348,8 +348,9 @@ int main(int argc, char *argv[])
 	Platform::DefaultAudioOutput audioOutput;
 	Platform::DefaultButtonInput buttonInput;
 	Platform::DefaultPointerInput pointerInput;
+	Platform::DefaultBoardLed boardLed;
 
-	Game::ResBnKwsRecognizer keywordRecognizer;
+	Game::ResBnKwsCnRecognizer keywordRecognizer;
 	keywordRecognizer.set_confidence_threshold(options.kws_threshold);
 	keywordRecognizer.set_input_gain(options.kws_input_gain);
 
@@ -361,6 +362,10 @@ int main(int argc, char *argv[])
 	{
 		std::cerr << "[WARN] Pointer input init failed; on-screen button trigger is disabled." << std::endl;
 	}
+	if (!boardLed.init())
+	{
+		std::cerr << "[WARN] Board LED init failed; voice responses remain enabled." << std::endl;
+	}
 
 	Core::DrawContext ctx(ScreenWidth, ScreenHeight);
 	Core::Timer timer(options.target_fps);
@@ -372,6 +377,7 @@ int main(int argc, char *argv[])
 		&audioOutput,
 		&buttonInput,
 		&pointerInput,
+		&boardLed,
 		&keywordRecognizer,
 		options.settings_path);
 	app.configure_audio(options.audio_sample_rate, options.audio_channels);

src/Apps/Game/README.md

@@ -3,13 +3,14 @@
 本文说明 TomGame 当前的关键词识别方式，以及 Windows 验证、WSL
 交叉编译、Windows SCP 部署和 IMX6ULL 板端运行流程。
 
-TomGame 使用内嵌的 `ResBnKwsRecognizer` INT8 模型，不需要 Python、
+TomGame 使用内嵌的 `ResBnKwsCnRecognizer` INT8 模型，不需要 Python、
 TensorFlow、TensorFlow Lite 运行库或外部模型文件。
 
-模型支持以下 12 个类别：
+模型支持以下 14 个类别：
 
 ```text
-Down, Go, Left, No, Off, On, Right, Stop, Up, Yes, Silence, Unknown
+Down, Go, Left, No, Off, On, Right, Stop, Up, Yes,
+OpenLight, CloseLight, Silence, Unknown
 ```
 
 当前游戏动作映射：
@@ -17,8 +18,15 @@ Down, Go, Left, No, Off, On, Right, Stop, Up, Yes, Silence, Unknown
 ```text
 Up / On -> Jump
 Stop    -> Idle
+OpenLight  -> 打开 ALPHA 板载 LED，播放 Tom 音效语音
+CloseLight -> 关闭 ALPHA 板载 LED，播放 Tom 音效语音
 ```
 
+`OpenLight.wav` 和 `CloseLight.wav` 会在构建阶段转换成只读 C++ 数据并
+编译进可执行文件，板端部署时不需要额外复制 WAV 文件。板载 LED 使用
+正点原子 I.MX6U-ALPHA 的 `GPIO1_IO03`，低电平点亮；程序优先使用
+`/dev/gpioled` 等正点原子教学驱动设备，并兼容 GPIO sysfs。
+
 ## Windows 验证
 
 Windows 版本和板端版本使用同一套 C++ 特征提取、INT8 模型推理和
@@ -49,7 +57,7 @@ cmake --build build --config Release --target IMX6U-Game
 识别成功或未达到阈值时，控制台会打印类似日志：
 
 ```text
-[INFO] ResBN INT8 KWS result: command=Up, confidence=0.92, windows=...
+[INFO] ResBN-CN INT8 KWS result: class=Up, confidence=0.92, windows=...
 ```
 
 Windows Desktop 和全部游戏可以一起构建：

src/Apps/Game/src/app/TomGameApp.cpp

@@ -1,4 +1,6 @@
 #include "TomGameApp.h"
+#include "IBoardLed.h"
+#include "TomAudioData.h"
 #include "IButtonInput.h"
 #include "Color.h"
 #include "DrawContext.h"
@@ -56,10 +58,12 @@ namespace Game
 		Platform::IAudioOutput *audioOutput,
 		Platform::IButtonInput *buttonInput,
 		Platform::IPointerInput *pointerInput,
+		Platform::IBoardLed *boardLed,
 		IKeywordRecognizer *keywordRecognizer,
 		const std::string& settingsPath)
 		: buttonInput(buttonInput),
 		  pointerInput(pointerInput),
+		  boardLed(boardLed),
 		  state(TomGameState::Idle),
 		  voiceMode(VoiceMode::PitchRepeat),
 		  recordingVoiceMode(VoiceMode::PitchRepeat),
@@ -384,6 +388,28 @@ namespace Game
 			back_to_idle();
 			break;
 
+		case KeywordGameAction::OpenLight:
+			if (boardLed != nullptr)
+			{
+				boardLed->set_on(true);
+			}
+			handle_voice_result(voiceController.play_effected_wav(
+				EmbeddedAudio::OpenLightWav,
+				EmbeddedAudio::OpenLightWavSize,
+				"OpenLight.wav"));
+			break;
+
+		case KeywordGameAction::CloseLight:
+			if (boardLed != nullptr)
+			{
+				boardLed->set_on(false);
+			}
+			handle_voice_result(voiceController.play_effected_wav(
+				EmbeddedAudio::CloseLightWav,
+				EmbeddedAudio::CloseLightWavSize,
+				"CloseLight.wav"));
+			break;
+
 		case KeywordGameAction::None:
 		default:
 			back_to_idle();

src/Apps/Game/src/app/TomGameApp.h

@@ -19,6 +19,7 @@ namespace Platform
 {
 	class IAudioInput;
 	class IAudioOutput;
+	class IBoardLed;
 	class IButtonInput;
 	class IPointerInput;
 }
@@ -32,6 +33,7 @@ namespace Game
 	private:
 		Platform::IButtonInput* buttonInput;
 		Platform::IPointerInput* pointerInput;
+		Platform::IBoardLed* boardLed;
 
 		TomGameState state;
 		VoiceMode voiceMode;
@@ -75,6 +77,7 @@ namespace Game
 			Platform::IAudioOutput* audioOutput,
 			Platform::IButtonInput* buttonInput,
 			Platform::IPointerInput* pointerInput,
+			Platform::IBoardLed* boardLed,
 			IKeywordRecognizer* keywordRecognizer,
 			const std::string& settingsPath);
 

src/Apps/Game/src/audio/WavAudioData.cpp

@@ -0,0 +1,95 @@
+#include "WavAudioData.h"
+#include <cstring>
+
+namespace
+{
+	static uint16_t ReadU16(const uint8_t* bytes)
+	{
+		return static_cast<uint16_t>(bytes[0]) |
+			(static_cast<uint16_t>(bytes[1]) << 8u);
+	}
+
+	static uint32_t ReadU32(const uint8_t* bytes)
+	{
+		return static_cast<uint32_t>(bytes[0]) |
+			(static_cast<uint32_t>(bytes[1]) << 8u) |
+			(static_cast<uint32_t>(bytes[2]) << 16u) |
+			(static_cast<uint32_t>(bytes[3]) << 24u);
+	}
+
+	static bool HasTag(const uint8_t* bytes, const char* tag)
+	{
+		return std::memcmp(bytes, tag, 4u) == 0;
+	}
+}
+
+namespace Game
+{
+	bool DecodePcm16Wav(
+		const uint8_t* bytes,
+		size_t byteCount,
+		WavAudioData& audio)
+	{
+		audio = WavAudioData();
+		if (bytes == nullptr || byteCount < 12u ||
+			!HasTag(bytes, "RIFF") || !HasTag(bytes + 8u, "WAVE"))
+		{
+			return false;
+		}
+
+		uint16_t audioFormat = 0u;
+		uint16_t bitsPerSample = 0u;
+		uint16_t blockAlign = 0u;
+		const uint8_t* sampleBytes = nullptr;
+		size_t sampleByteCount = 0u;
+
+		size_t offset = 12u;
+		while (offset + 8u <= byteCount)
+		{
+			const uint8_t* chunk = bytes + offset;
+			const uint32_t chunkSize = ReadU32(chunk + 4u);
+			offset += 8u;
+			if (static_cast<size_t>(chunkSize) > byteCount - offset)
+			{
+				return false;
+			}
+
+			if (HasTag(chunk, "fmt "))
+			{
+				if (chunkSize < 16u)
+				{
+					return false;
+				}
+				audioFormat = ReadU16(bytes + offset);
+				audio.channels = ReadU16(bytes + offset + 2u);
+				audio.sampleRate = ReadU32(bytes + offset + 4u);
+				blockAlign = ReadU16(bytes + offset + 12u);
+				bitsPerSample = ReadU16(bytes + offset + 14u);
+			}
+			else if (HasTag(chunk, "data"))
+			{
+				sampleBytes = bytes + offset;
+				sampleByteCount = chunkSize;
+			}
+
+			offset += static_cast<size_t>(chunkSize);
+			if ((chunkSize & 1u) != 0u && offset < byteCount)
+			{
+				++offset;
+			}
+		}
+
+		if (audioFormat != 1u || bitsPerSample != 16u ||
+			audio.channels == 0u || audio.sampleRate == 0u ||
+			blockAlign != audio.channels * sizeof(int16_t) ||
+			sampleBytes == nullptr || sampleByteCount == 0u ||
+			(sampleByteCount % blockAlign) != 0u)
+		{
+			return false;
+		}
+
+		audio.samples.resize(sampleByteCount / sizeof(int16_t));
+		std::memcpy(audio.samples.data(), sampleBytes, sampleByteCount);
+		return true;
+	}
+}

src/Apps/Game/src/audio/WavAudioData.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
+namespace Game
+{
+	struct WavAudioData
+	{
+		std::vector<int16_t> samples;
+		uint32_t sampleRate;
+		uint32_t channels;
+
+		WavAudioData() : sampleRate(0u), channels(0u) {}
+	};
+
+	bool DecodePcm16Wav(
+		const uint8_t* bytes,
+		size_t byteCount,
+		WavAudioData& audio);
+}

src/Apps/Game/src/gameplay/KeywordCommandRouter.cpp

@@ -13,12 +13,19 @@ namespace Game
 		case KeywordCommand::Stop:
 			return KeywordGameAction::BackToIdle;
 
+		case KeywordCommand::OpenLight:
+			return KeywordGameAction::OpenLight;
+
+		case KeywordCommand::CloseLight:
+			return KeywordGameAction::CloseLight;
+
 		case KeywordCommand::Down:
 		case KeywordCommand::Left:
 		case KeywordCommand::Right:
 		case KeywordCommand::Go:
 		case KeywordCommand::Yes:
 		case KeywordCommand::No:
+		case KeywordCommand::Off:
 		case KeywordCommand::None:
 		default:
 			return KeywordGameAction::None;

src/Apps/Game/src/gameplay/KeywordCommandRouter.h

@@ -8,7 +8,9 @@ namespace Game
 	{
 		None,
 		Jump,
-		BackToIdle
+		BackToIdle,
+		OpenLight,
+		CloseLight
 	};
 
 	class KeywordCommandRouter

src/Apps/Game/src/gameplay/VoiceInteractionController.cpp

@@ -1,5 +1,6 @@
 #include "VoiceInteractionController.h"
 #include "../audio/VoiceEffect.h"
+#include "../audio/WavAudioData.h"
 #include "IAudioInput.h"
 #include "IAudioOutput.h"
 #include <algorithm>
@@ -233,9 +234,13 @@ namespace Game
 		return VoiceInteractionResult(VoiceInteractionEvent::KeywordRecognized, result);
 	}
 
-	VoiceInteractionResult VoiceInteractionController::start_speaking(const std::vector<int16_t>& samples)
+	VoiceInteractionResult VoiceInteractionController::start_speaking(
+		const std::vector<int16_t>& samples,
+		uint32_t sourceSampleRate,
+		uint32_t sourceChannels)
 	{
-		if (samples.empty() || audioOutput == nullptr)
+		if (samples.empty() || sourceSampleRate == 0u ||
+			sourceChannels == 0u || audioOutput == nullptr)
 		{
 			return VoiceInteractionResult(VoiceInteractionEvent::IdleRequested);
 		}
@@ -256,13 +261,20 @@ namespace Game
 		const uint32_t playbackSampleRate = audioOutput->get_sample_rate();
 		const uint32_t playbackChannels = audioOutput->get_channels();
 		std::vector<int16_t> playbackSamples = samples;
-		if (playbackChannels != audioChannels)
+		if (playbackChannels != sourceChannels)
 		{
-			playbackSamples = VoiceEffect::convert_channels(playbackSamples, audioChannels, playbackChannels);
+			playbackSamples = VoiceEffect::convert_channels(
+				playbackSamples,
+				sourceChannels,
+				playbackChannels);
 		}
-		if (playbackSampleRate != audioSampleRate)
+		if (playbackSampleRate != sourceSampleRate)
 		{
-			playbackSamples = VoiceEffect::resample(playbackSamples, audioSampleRate, playbackSampleRate, playbackChannels);
+			playbackSamples = VoiceEffect::resample(
+				playbackSamples,
+				sourceSampleRate,
+				playbackSampleRate,
+				playbackChannels);
 		}
 
 		player.set_voice(playbackSamples, playbackSampleRate, playbackChannels);
@@ -311,21 +323,10 @@ namespace Game
 			<< recorder.get_last_volume()
 			<< std::endl;
 
-		std::vector<int16_t> samples = VoiceEffect::amplify(recordedSamples, inputGain);
+		std::vector<int16_t> samples = apply_voice_effect(
-		const std::vector<int16_t> trimmedSamples = VoiceEffect::trim_silence(samples, 0.02f, audioChannels);
+			recordedSamples,
-		const size_t minPlaybackSamples = static_cast<size_t>(audioSampleRate * audioChannels / 4u);
+			audioSampleRate,
-		if (!samples.empty() && trimmedSamples.size() < minPlaybackSamples)
+			audioChannels);
-		{
-			std::cerr
-				<< "[WARN] Trimmed recording is too short ("
-				<< trimmedSamples.size()
-				<< " samples); playing untrimmed samples for diagnostics."
-				<< std::endl;
-		}
-		else
-		{
-			samples = trimmedSamples;
-		}
 
 		std::cout
 			<< "[INFO] Playback prepared: samples="
@@ -336,9 +337,62 @@ namespace Game
 			<< audioChannels
 			<< std::endl;
 
-		samples = VoiceEffect::pitch_up(samples, pitchFactor, audioChannels);
+		return start_speaking(samples, audioSampleRate, audioChannels);
-		samples = VoiceEffect::amplify(samples, outputGain);
+	}
-		return start_speaking(samples);
+
+	VoiceInteractionResult VoiceInteractionController::play_effected_wav(
+		const uint8_t* wavBytes,
+		size_t wavByteCount,
+		const char* name)
+	{
+		WavAudioData audio;
+		if (!DecodePcm16Wav(wavBytes, wavByteCount, audio))
+		{
+			std::cerr
+				<< "[WARN] Failed to decode embedded WAV: "
+				<< (name != nullptr ? name : "(unnamed)")
+				<< std::endl;
+			return VoiceInteractionResult(VoiceInteractionEvent::IdleRequested);
+		}
+
+		std::cout
+			<< "[INFO] Embedded WAV loaded: "
+			<< (name != nullptr ? name : "(unnamed)")
+			<< ", samples=" << audio.samples.size()
+			<< ", rate=" << audio.sampleRate
+			<< ", channels=" << audio.channels
+			<< std::endl;
+
+		const std::vector<int16_t> effectedSamples = apply_voice_effect(
+			audio.samples,
+			audio.sampleRate,
+			audio.channels);
+		return start_speaking(effectedSamples, audio.sampleRate, audio.channels);
+	}
+
+	std::vector<int16_t> VoiceInteractionController::apply_voice_effect(
+		const std::vector<int16_t>& sourceSamples,
+		uint32_t sampleRate,
+		uint32_t channels) const
+	{
+		std::vector<int16_t> samples = VoiceEffect::amplify(sourceSamples, inputGain);
+		const std::vector<int16_t> trimmedSamples = VoiceEffect::trim_silence(samples, 0.02f, channels);
+		const size_t minPlaybackSamples = static_cast<size_t>(sampleRate * channels / 4u);
+		if (!samples.empty() && trimmedSamples.size() < minPlaybackSamples)
+		{
+			std::cerr
+				<< "[WARN] Trimmed voice is too short ("
+				<< trimmedSamples.size()
+				<< " samples); using untrimmed audio."
+				<< std::endl;
+		}
+		else
+		{
+			samples = trimmedSamples;
+		}
+
+		samples = VoiceEffect::pitch_up(samples, pitchFactor, channels);
+		return VoiceEffect::amplify(samples, outputGain);
 	}
 
 	void VoiceInteractionController::stop_all()

src/Apps/Game/src/gameplay/VoiceInteractionController.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <cstddef>
 #include <cstdint>
 #include <string>
 #include <vector>
@@ -61,7 +62,14 @@ namespace Game
 		VoiceInteractionResult finish_pitch_repeat_recording();
 		VoiceInteractionResult finish_keyword_recording();
 		VoiceInteractionResult try_recognize_keyword(const std::vector<int16_t>& samples);
-		VoiceInteractionResult start_speaking(const std::vector<int16_t>& samples);
+		VoiceInteractionResult start_speaking(
+			const std::vector<int16_t>& samples,
+			uint32_t sourceSampleRate,
+			uint32_t sourceChannels);
+		std::vector<int16_t> apply_voice_effect(
+			const std::vector<int16_t>& samples,
+			uint32_t sampleRate,
+			uint32_t channels) const;
 
 	public:
 		VoiceInteractionController(
@@ -76,6 +84,10 @@ namespace Game
 		VoiceInteractionResult update_keyword_recording(uint32_t deltaMs);
 		VoiceInteractionResult update_speaking(uint32_t deltaMs);
 		VoiceInteractionResult repeat_last_recording();
+		VoiceInteractionResult play_effected_wav(
+			const uint8_t* wavBytes,
+			size_t wavByteCount,
+			const char* name);
 		void stop_all();
 		void set_input_gain(float gain);
 		void set_output_gain(float gain);

src/Apps/Game/src/recognition/KeywordRecognizer.cpp

@@ -36,6 +36,12 @@ namespace Game
 			return "No";
 		case KeywordCommand::On:
 			return "On";
+		case KeywordCommand::Off:
+			return "Off";
+		case KeywordCommand::OpenLight:
+			return "OpenLight";
+		case KeywordCommand::CloseLight:
+			return "CloseLight";
 		case KeywordCommand::None:
 		default:
 			return "None";

src/Apps/Game/src/recognition/KeywordRecognizer.h

@@ -16,7 +16,10 @@ namespace Game
 		Stop,
 		Yes,
 		No,
-		On
+		On,
+		Off,
+		OpenLight,
+		CloseLight
 	};
 
 	struct KeywordRecognitionResult

src/Apps/Game/src/recognition/MODLE.md

@@ -0,0 +1,392 @@
+# ResBN INT8 关键词识别模型研究总结
+
+## 1. 研究目标
+
+本项目面向 IMX6ULL-ALPHA 的离线关键词识别部署，核心约束包括：
+
+- 使用 Google Speech Commands v2 数据集；
+- 保持 1 秒、16 kHz、单声道输入；
+- 输入特征固定为 `49 x 10 x 1` MFCC，兼容现有识别流程；
+- 支持 10 个命令词，以及 `_silence_` 和 `_unknown_`，共 12 类；
+- 模型必须能够转换为 INT8，并在不依赖 TensorFlow Lite/LiteRT 运行时的情况下部署；
+- 长音频识别不能为每个滑动窗口重复计算完整 MFCC；
+- 在精度、模型体积、内存占用和 Cortex-A7 推理开销之间取得平衡。
+
+最终选择的方案是：
+
+> **1 秒 ResBN DS-CNN + 全 INT8 量化 + 整段 mel 特征缓存 + 多窗口 logits 融合。**
+
+---
+
+## 2. Baseline 模型
+
+初始模型采用标准轻量 DS-CNN：
+
+```text
+49 x 10 x 1 MFCC
+    ↓
+Conv2D 10x4, stride 2x2, 64 channels
+    ↓
+Depthwise 3x3 + Pointwise 1x1，共 4 个 DS-CNN 块
+    ↓
+Global Average Pooling
+    ↓
+Dense 12 + Softmax
+```
+
+特征参数为：
+
+- 采样率：16 kHz；
+- 窗长/FFT：512 点，即 32 ms；
+- 帧移：320 点，即 20 ms；
+- mel 滤波器：40 维；
+- MFCC：保留前 10 维；
+- 1 秒音频生成 49 帧 MFCC。
+
+Baseline 在 4,890 条独立测试样本上的准确率为 **88.92%**，模型大小为 **44,856 bytes**。
+
+Baseline 的主要问题不是模型过大，而是：
+
+- 4 个 DS-CNN 块串联后缺少跨层信息通道；
+- 深层特征只能依赖连续卷积传播，训练稳定性和特征复用能力有限；
+- 输出直接经过 Softmax，不利于后续对多个窗口的 logits 做统一融合；
+- 长音频处理会重复执行归一化、FFT、mel 滤波和 DCT。
+
+---
+
+## 3. 模型迭代研究过程
+
+### 3.1 引入残差连接和 Batch Normalization
+
+我们首先设计了 ResBN 模型，保持输入尺寸、通道数和 DS-CNN 主体不变，只修改每个深度可分离卷积块：
+
+```text
+输入 identity
+    ├──────────────────────────────┐
+    ↓                              │
+DepthwiseConv 3x3                  │
+    ↓                              │
+BatchNorm                          │
+    ↓                              │
+ReLU                               │
+    ↓                              │
+PointwiseConv 1x1                  │
+    ↓                              │
+BatchNorm                          │
+    ↓                              │
+Add(identity)  ←───────────────────┘
+    ↓
+ReLU
+```
+
+该设计的价值：
+
+- 残差连接不增加卷积参数，仅增加低成本逐元素加法；
+- 保留浅层信息，改善 4 层 DS-CNN 堆叠后的梯度传播；
+- BN 提高训练稳定性，并在导出时折叠进卷积权重和 bias，部署时不需要单独执行 BN；
+- 输出改为原始 logits，Softmax 移到多窗口融合阶段执行；
+- 不改变 `49 x 10` 输入合同，不需要重新设计前端采集和窗口协议。
+
+ResBN Dynamic TFLite 的测试准确率达到 **91.10%**，相比 Baseline 提升 **2.19 个百分点**。
+
+### 3.2 评估 SE 通道注意力
+
+在 ResBN 基础上，我们继续设计了 SE 模型：
+
+- 每个残差 DS-CNN 块增加通道注意力；
+- SE ratio 为 8；
+- 使用 `hard_sigmoid`，降低量化和 Cortex-A7 上的激活开销；
+- 通过全局平均池化、降维、升维和逐通道乘法调整 64 个通道的重要性。
+
+SE 模型的测试准确率为 **91.19%**，只比 ResBN Dynamic 高 **0.08 个百分点**，但模型大小从 **46,392 bytes** 增加到 **71,040 bytes**，增长约 **53.13%**。
+
+SE 的最佳验证准确率较高，但在独立测试集上的提升很小，而且类别收益不均衡：
+
+- `go`、`right`、`no` 有改善；
+- `down`、`up`、`unknown` 出现回退；
+- 增加了额外的全局池化、两层通道映射、激活和逐通道乘法。
+
+因此，SE 在本次数据和模型规模下没有形成足够高的精度/开销收益比，最终没有作为部署模型。
+
+### 3.3 保留 1 秒模型，而不是扩展为 2 秒输入
+
+研究中对比了两种长音频方案：
+
+1. 将模型输入扩大为 2 秒，减少推理窗口数量；
+2. 保留已验证的 1 秒模型，对整段音频只计算一次频谱特征，再切分特征窗口。
+
+最终选择第二种方案，原因是：
+
+- 1 秒模型已经完成训练、量化和精度验证；
+- 不改变输入尺寸和卷积结构，部署风险更低；
+- 2 秒模型会增加单次推理 MAC、激活内存和重新训练成本；
+- 长音频的主要重复开销来自 FFT 和 mel，而不只是模型推理；
+- 特征缓存可以直接消除相邻窗口之间的频谱重复计算。
+
+---
+
+## 4. 核心创新
+
+### 4.1 残差 DS-CNN 与 BN 折叠
+
+我们不是简单增加通道数或卷积层，而是在不明显增加推理复杂度的情况下改善网络信息流：
+
+- 4 个 DS-CNN 块全部增加 identity 残差；
+- 每个 Depthwise 和 Pointwise 卷积后加入 BN；
+- BN 在 TFLite 转换时折叠进卷积参数；
+- 保持 64 通道和原始空间尺寸，残差无需投影层。
+
+这种设计将提升重点放在训练稳定性和特征复用，而不是通过扩大模型换取精度。
+
+### 4.2 从“重复 MFCC”改为“整段 mel cache”
+
+传统滑窗实现会对每个 1 秒窗口重复执行：
+
+```text
+窗口归一化 → Hann window → FFT → magnitude → mel → log → DCT → inference
+```
+
+我们的实现改为：
+
+```text
+整段音频
+    ↓
+一次性计算所有 STFT 帧和 mel energy
+    ↓
+缓存每一帧的 40 维 mel energy
+    ↓
+按 frameStart 切出 49 帧窗口
+    ↓
+按该 1 秒窗口的 maxAbs 恢复训练时的窗口归一化语义
+    ↓
+log + DCT
+    ↓
+INT8 inference
+```
+
+这里没有直接缓存最终 MFCC，而是缓存**归一化之前的 mel energy**。原因是训练时每个 1 秒窗口都独立按最大绝对值归一化。如果直接缓存整段归一化后的 MFCC，不同窗口的幅度语义会与训练不一致。
+
+通过“原始 mel energy cache + 每窗口线性缩放”，既复用了 FFT/mel，又保留了训练时的逐窗口归一化。
+
+### 4.3 预计算 DCT 基矩阵
+
+MFCC 的 DCT 原本每个窗口需要执行：
+
+```text
+49 frames x 10 coefficients x 40 mel bins = 19,600 次 cos()
+```
+
+我们将 `10 x 40` DCT 基矩阵首次初始化后缓存：
+
+- 只需计算 400 次 `cos()`；
+- 后续窗口仅执行矩阵乘加；
+- 5 秒、21 个窗口时，三角函数计算量从 411,600 次降到 400 次；
+- `cos()` 调用减少约 **99.90%**。
+
+### 4.4 多窗口 logits 融合
+
+模型输出改为 logits，并在识别器中增加四层判定：
+
+1. **EMA logits 平滑**：默认 `alpha = 0.60`；
+2. **Top-1 confidence**：默认阈值 `0.75`；
+3. **Top-1/Top-2 margin**：默认阈值 `0.15`；
+4. **连续命中**：多窗口时默认至少连续命中 2 次。
+
+与单次 argmax 相比，该策略能够：
+
+- 抑制瞬时噪声造成的高置信误判；
+- 避免 Top-1 和 Top-2 非常接近时过早输出；
+- 利用相邻窗口的一致性；
+- 保留窗口步长、阈值和平滑系数的板端可调能力。
+
+### 4.5 从 `.tflite` 自动导出无运行时 C++ 模型
+
+为了避免在 IMX6ULL 上集成完整 TensorFlow Lite/LiteRT，我们实现了 `export_cpp.py`：
+
+- 解析 INT8 TFLite 图；
+- 提取 Stem、4 个 Depthwise、4 个 Pointwise、4 个 Add、Global Mean 和 Dense；
+- 导出 INT8 weights、INT32 bias；
+- 导出 per-channel multiplier 和 shift；
+- 将 BN 融合后的参数直接写入 `ResBnKwsModelData.cpp`；
+- 生成手写 INT8 Conv、Depthwise、Add、GAP 和 Dense 推理路径。
+
+最终板端只需要：
+
+- `ResBnKwsRecognizer.h/.cpp`
+- `ResBnKwsModelData.h/.cpp`
+
+不需要在运行时加载 `.tflite`，也不依赖 TFLite Interpreter。
+
+### 4.6 逐层复刻 TFLite INT8 舍入语义
+
+量化部署中最容易被忽略的问题不是权重，而是整数乘法、舍入、饱和和零点处理。
+
+我们针对以下细节进行了逐层校正：
+
+- Saturating Rounding Doubling High Mul；
+- Rounding Divide by Power-of-Two；
+- per-channel Conv multiplier/shift；
+- Add 算子的输入缩放与 left shift；
+- Global Mean 的 TFLite reference multiplier 调整方式；
+- INT8 activation clamp；
+- 输出零点和 scale 的还原。
+
+Global Mean 最初只有少量 1 LSB 偏差，但仍继续对照 TensorFlow Lite 2.21 内核源码，最终做到全部中间张量完全一致。
+
+### 4.7 训练与 C++ 特征前端严格一致
+
+部署验证中还发现并修正了两个容易造成精度损失的前端差异：
+
+1. TensorFlow mel 滤波器是在 **mel 域**做三角插值，而不是在 Hz 域；
+2. `tf.signal.mfccs_from_log_mel_spectrograms()` 使用 HTK 缩放，`C0` 同样采用 `sqrt(2/N)`，不是正交 DCT 的 `sqrt(1/N)`。
+
+修正后，C++ 特征经过 INT8 量化，与 TensorFlow 训练侧输入完全一致。
+
+---
+
+## 5. 模型结果
+
+测试集包含 4,890 条样本。
+
+| 模型 | 测试准确率 | TFLite 大小 | 结论 |
+|---|---:|---:|---|
+| Baseline DS-CNN Dynamic | 88.92% | 44,856 B | 初始基线 |
+| ResBN Dynamic | 91.10% | 46,392 B | 残差和 BN 带来明显提升 |
+| ResBN + SE Dynamic | 91.19% | 71,040 B | 仅提升 0.08pp，体积增加 53.13% |
+| **ResBN INT8** | **91.68%** | **48,368 B** | **最终部署模型** |
+
+最终 ResBN INT8 相比 Baseline：
+
+- 准确率提升 **2.76 个百分点**；
+- 错误率从 11.08% 降至 8.32%；
+- 相对错误率下降约 **24.91%**；
+- 相比 ResBN Dynamic 还提升 **0.57 个百分点**；
+- 模型保持约 48 KB，适合嵌入式固化。
+
+INT8 文件不一定比 Dynamic TFLite 更小，因为文件中还包含量化参数和算子元数据。选择 INT8 的主要价值是：
+
+- 权重和激活均为 INT8；
+- 卷积累加使用 INT32；
+- 适合 Cortex-A7 的整数执行；
+- 更容易实现无 TFLite 运行时的静态 C++ 推理。
+
+### 关键类别提升
+
+| 类别 | Baseline | ResBN INT8 | 提升 |
+|---|---:|---:|---:|
+| right | 78.54% | 87.88% | +9.34pp |
+| no | 84.69% | 93.58% | +8.89pp |
+| go | 76.37% | 84.08% | +7.71pp |
+| down | 84.98% | 88.67% | +3.69pp |
+| left | 91.02% | 93.69% | +2.67pp |
+| yes | 92.36% | 94.75% | +2.39pp |
+
+`on` 和 `_unknown_` 相比 Baseline 有小幅回退，说明后续仍需使用真实麦克风噪声和板端场景数据校准类别阈值。
+
+---
+
+## 6. 长音频优化效果
+
+以 5 秒音频、1 秒窗口、200 ms 步长为例：
+
+- 推理窗口数：21；
+- 每个窗口包含 49 个 STFT/MFCC 帧；
+- 传统逐窗口方式需要计算 `21 x 49 = 1,029` 个 FFT/mel 帧；
+- 整段 5 秒缓存只需要计算 249 个连续 STFT/mel 帧；
+- FFT/mel 帧计算减少约 **75.80%**；
+- 相当于频谱前端计算次数降低约 **4.13 倍**。
+
+该优化不减少 21 次模型推理，但消除了相邻窗口 80% 重叠区域的 FFT 和 mel 重复计算。
+
+如果后续板端实测表明模型推理仍是主要瓶颈，可以通过接口将窗口步长从 10 帧调整为 25 帧或 50 帧，在响应速度和计算量之间继续权衡，而不需要重新训练模型。
+
+---
+
+## 7. 部署一致性验证
+
+为了保证桌面测试结果能够迁移到 IMX6ULL，我们没有只验证最终分类结果，而是验证了完整链路。
+
+### 7.1 INT8 算子逐层验证
+
+在 `down`、`go`、`right`、`stop`、`yes` 五类样本上，对以下 15 个算子输出逐层比较：
+
+- Stem Conv；
+- 4 个 Depthwise Conv；
+- 4 个 Pointwise Conv；
+- 4 个 Residual Add；
+- Global Mean；
+- Dense。
+
+结果：
+
+> 所有层的 INT8 输出与 TensorFlow Lite reference kernel 完全一致，mismatch 为 0。
+
+### 7.2 C++ MFCC 验证
+
+使用真实 C++ FFT、mel cache、窗口归一化和 DCT，与 TensorFlow 训练前端比较：
+
+- 单窗口 490 个 INT8 MFCC 输入：`0/490` mismatch；
+- 5 秒音频选取 7 个不同 frameStart：每个窗口均为 `0/490` mismatch；
+- 浮点 MFCC 最大误差不超过约 `2.83e-4`；
+- C++ logits 与 TFLite reference 输出最大误差约 `9.09e-7`。
+
+这说明缓存切窗没有改变模型实际接收到的 INT8 特征。
+
+### 7.3 编译验证
+
+最终 C++ 模型数据和识别器通过 MSVC `/W4` 语法编译检查。
+
+---
+
+## 8. 最终交付
+
+### 训练与测试
+
+- `ResBNKWS/model.py`
+- `ResBNKWS/train.py`
+- `ResBNKWS/test.py`
+- `ResBNKWS/artifacts/resbn_kws_1s_int8.tflite`
+
+### C++ 导出与验证
+
+- `ResBNKWS/export_cpp.py`
+- `ResBNKWS/validate_cpp_export.py`
+- `ResBNKWS/DEPLOYMENT.md`
+
+### IMX6ULL 项目需要迁移
+
+- `ResBnKwsRecognizer.h`
+- `ResBnKwsRecognizer.cpp`
+- `ResBnKwsModelData.h`
+- `ResBnKwsModelData.cpp`
+
+运行时不需要迁移 `.tflite`，因为模型参数已经转换为 C++ 常量数组。
+
+---
+
+## 9. 当前限制与后续工作
+
+目前已经完成模型、量化、特征缓存、融合和 C++ 数值一致性验证，但仍有以下板端工作：
+
+1. 在 IMX6ULL 上测量单窗口推理时间、5 秒总处理时间和峰值内存；
+2. 使用实际麦克风录音重新标定 confidence、margin 和连续命中阈值；
+3. 根据实时性要求评估 200 ms、500 ms 和 1 s 三种窗口步长；
+4. 如 FFT 成为瓶颈，可替换为 NEON 优化 FFT 或固定点 FFT；
+5. 当前 `off` 类需要目标项目的 `KeywordCommand` 增加 `Off`，并定义 `RESBN_KWS_HAS_OFF_COMMAND`；
+6. 如果需要连续输出多个命令，而不是从整段音频返回一个最佳命令，需要在上层增加事件时间戳、冷却时间和去重状态机。
+
+---
+
+## 10. 结论
+
+本项目的主要创新不是单纯扩大网络，而是联合优化了模型结构、量化实现、特征前端和长音频决策逻辑：
+
+- 使用残差和 BN 提升 DS-CNN 精度，同时保持轻量结构；
+- 通过实验否决收益不足的 SE 部署方案；
+- 保留 1 秒模型，利用 mel cache 消除长音频重复 FFT；
+- 使用 logits EMA、margin 和连续命中提高输出稳定性；
+- 将 INT8 TFLite 自动转换为无运行时依赖的 C++ 推理代码；
+- 对齐 TFLite 整数舍入和 TensorFlow MFCC 细节；
+- 最终达到 **91.68%** 测试准确率、约 **48 KB** 模型大小，并完成逐层零误差验证。
+
+最终方案在精度、体积、可解释性、长音频计算量和 IMX6ULL 部署可控性之间取得了更好的综合平衡。

src/Apps/Game/src/recognition/ResBnKwsCnModelData.h

@@ -0,0 +1,106 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+namespace Game
+{
+namespace ResBnKwsCnModelData
+{
+	const size_t InputFrameCount = 49;
+	const size_t InputMfccCount = 10;
+	const size_t ChannelCount = 64;
+	const size_t ClassCount = 14;
+	const size_t BlockCount = 4;
+
+	struct Quantization
+	{
+		float scale;
+		int32_t zeroPoint;
+	};
+
+	struct ConvLayerData
+	{
+		const int8_t* weights;
+		const int32_t* bias;
+		const int32_t* multipliers;
+		const int32_t* shifts;
+		Quantization input;
+		Quantization output;
+		bool relu;
+	};
+
+	struct AddLayerData
+	{
+		Quantization input0;
+		Quantization input1;
+		Quantization output;
+		int32_t input0Multiplier;
+		int32_t input0Shift;
+		int32_t input1Multiplier;
+		int32_t input1Shift;
+		int32_t outputMultiplier;
+		int32_t outputShift;
+		int32_t leftShift;
+	};
+
+	struct MeanLayerData
+	{
+		Quantization input;
+		Quantization output;
+		int32_t multiplier;
+		int32_t shift;
+		int32_t elementCount;
+	};
+
+	extern const int8_t StemWeights[2560];
+	extern const int32_t StemBias[64];
+	extern const int32_t StemMultipliers[64];
+	extern const int32_t StemShifts[64];
+	extern const int8_t Dw0Weights[576];
+	extern const int32_t Dw0Bias[64];
+	extern const int32_t Dw0Multipliers[64];
+	extern const int32_t Dw0Shifts[64];
+	extern const int8_t Dw1Weights[576];
+	extern const int32_t Dw1Bias[64];
+	extern const int32_t Dw1Multipliers[64];
+	extern const int32_t Dw1Shifts[64];
+	extern const int8_t Dw2Weights[576];
+	extern const int32_t Dw2Bias[64];
+	extern const int32_t Dw2Multipliers[64];
+	extern const int32_t Dw2Shifts[64];
+	extern const int8_t Dw3Weights[576];
+	extern const int32_t Dw3Bias[64];
+	extern const int32_t Dw3Multipliers[64];
+	extern const int32_t Dw3Shifts[64];
+	extern const int8_t Pw0Weights[4096];
+	extern const int32_t Pw0Bias[64];
+	extern const int32_t Pw0Multipliers[64];
+	extern const int32_t Pw0Shifts[64];
+	extern const int8_t Pw1Weights[4096];
+	extern const int32_t Pw1Bias[64];
+	extern const int32_t Pw1Multipliers[64];
+	extern const int32_t Pw1Shifts[64];
+	extern const int8_t Pw2Weights[4096];
+	extern const int32_t Pw2Bias[64];
+	extern const int32_t Pw2Multipliers[64];
+	extern const int32_t Pw2Shifts[64];
+	extern const int8_t Pw3Weights[4096];
+	extern const int32_t Pw3Bias[64];
+	extern const int32_t Pw3Multipliers[64];
+	extern const int32_t Pw3Shifts[64];
+	extern const int8_t DenseWeights[896];
+	extern const int32_t DenseBias[14];
+	extern const int32_t DenseMultipliers[14];
+	extern const int32_t DenseShifts[14];
+
+	extern const ConvLayerData Stem;
+	extern const ConvLayerData Depthwise[BlockCount];
+	extern const ConvLayerData Pointwise[BlockCount];
+	extern const AddLayerData Adds[BlockCount];
+	extern const MeanLayerData GlobalMean;
+	extern const ConvLayerData Dense;
+	extern const Quantization ModelInput;
+	extern const Quantization ModelOutput;
+}
+}

src/Apps/Game/src/recognition/ResBnKwsCnRecognizer.h

@@ -0,0 +1,50 @@
+#pragma once
+
+#include "KeywordRecognizer.h"
+#include <cstddef>
+#include <string>
+#include <vector>
+
+namespace Game
+{
+	class ResBnKwsCnRecognizer : public IKeywordRecognizer
+	{
+	private:
+		float confidenceThreshold;
+		float marginThreshold;
+		float smoothingAlpha;
+		float inputGain;
+		size_t minConsecutiveHits;
+		size_t featureWindowStepFrames;
+		bool initialized;
+
+		bool run_embedded_model_logits(
+			const std::vector<float>& features,
+			std::vector<float>& logits) const;
+
+	public:
+		ResBnKwsCnRecognizer();
+		explicit ResBnKwsCnRecognizer(const std::string& unusedModelPath);
+		~ResBnKwsCnRecognizer();
+
+		bool init() override;
+		KeywordRecognitionResult recognize(
+			const std::vector<int16_t>& samples,
+			uint32_t sampleRate,
+			uint32_t channels) override;
+
+		void set_confidence_threshold(float threshold);
+		float get_confidence_threshold() const { return confidenceThreshold; }
+		void set_margin_threshold(float threshold);
+		float get_margin_threshold() const { return marginThreshold; }
+		void set_smoothing_alpha(float alpha);
+		float get_smoothing_alpha() const { return smoothingAlpha; }
+		void set_min_consecutive_hits(size_t hits);
+		size_t get_min_consecutive_hits() const { return minConsecutiveHits; }
+		void set_feature_window_step_frames(size_t frames);
+		size_t get_feature_window_step_frames() const { return featureWindowStepFrames; }
+		void set_input_gain(float gain) override;
+		float get_input_gain() const override { return inputGain; }
+		bool is_initialized() const { return initialized; }
+	};
+}

src/Apps/Game/tests/TomAudioDataTests.cpp

@@ -0,0 +1,53 @@
+#include "TomAudioData.h"
+#include "WavAudioData.h"
+#include "KeywordCommandRouter.h"
+#include <iostream>
+
+namespace
+{
+	static bool CheckAudio(
+		const uint8_t* bytes,
+		size_t byteCount,
+		const char* name)
+	{
+		Game::WavAudioData audio;
+		if (!Game::DecodePcm16Wav(bytes, byteCount, audio))
+		{
+			std::cerr << name << " failed to decode" << std::endl;
+			return false;
+		}
+		if (audio.sampleRate != 48000u || audio.channels != 2u || audio.samples.empty())
+		{
+			std::cerr << name << " has unexpected audio format" << std::endl;
+			return false;
+		}
+		return true;
+	}
+}
+
+int main()
+{
+	if (!CheckAudio(
+			Game::EmbeddedAudio::OpenLightWav,
+			Game::EmbeddedAudio::OpenLightWavSize,
+			"OpenLight.wav") ||
+		!CheckAudio(
+			Game::EmbeddedAudio::CloseLightWav,
+			Game::EmbeddedAudio::CloseLightWavSize,
+			"CloseLight.wav"))
+	{
+		return 1;
+	}
+
+	const Game::KeywordCommandRouter router;
+	if (router.route(Game::KeywordRecognitionResult(Game::KeywordCommand::OpenLight, 1.0f)) !=
+			Game::KeywordGameAction::OpenLight ||
+		router.route(Game::KeywordRecognitionResult(Game::KeywordCommand::CloseLight, 1.0f)) !=
+			Game::KeywordGameAction::CloseLight)
+	{
+		std::cerr << "Light keyword routing failed" << std::endl;
+		return 2;
+	}
+
+	return 0;
+}

src/Apps/Game/tools/EmbedTomAudio.cmake

@@ -0,0 +1,55 @@
+if(NOT DEFINED OPEN_LIGHT_WAV OR
+   NOT DEFINED CLOSE_LIGHT_WAV OR
+   NOT DEFINED OUTPUT_HEADER OR
+   NOT DEFINED OUTPUT_SOURCE)
+    message(FATAL_ERROR "EmbedTomAudio.cmake is missing an input or output path")
+endif()
+
+get_filename_component(OUTPUT_DIRECTORY "${OUTPUT_HEADER}" DIRECTORY)
+file(MAKE_DIRECTORY "${OUTPUT_DIRECTORY}")
+
+file(WRITE "${OUTPUT_HEADER}" [=[#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+namespace Game
+{
+namespace EmbeddedAudio
+{
+	extern const uint8_t OpenLightWav[];
+	extern const size_t OpenLightWavSize;
+	extern const uint8_t CloseLightWav[];
+	extern const size_t CloseLightWavSize;
+}
+}
+]=])
+
+file(WRITE "${OUTPUT_SOURCE}" [=[#include "TomAudioData.h"
+
+namespace Game
+{
+namespace EmbeddedAudio
+{
+]=])
+
+function(append_embedded_wav SYMBOL INPUT_PATH)
+    file(READ "${INPUT_PATH}" WAV_HEX HEX)
+    string(TOLOWER "${WAV_HEX}" WAV_HEX)
+    string(LENGTH "${WAV_HEX}" WAV_HEX_LENGTH)
+    math(EXPR WAV_SIZE "${WAV_HEX_LENGTH} / 2")
+    string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\1," WAV_BYTES "${WAV_HEX}")
+    set(BYTE_TOKEN "0x[0-9a-f][0-9a-f],")
+    set(LINE_PATTERN "(${BYTE_TOKEN}${BYTE_TOKEN}${BYTE_TOKEN}${BYTE_TOKEN}${BYTE_TOKEN}${BYTE_TOKEN}${BYTE_TOKEN}${BYTE_TOKEN}${BYTE_TOKEN}${BYTE_TOKEN}${BYTE_TOKEN}${BYTE_TOKEN}${BYTE_TOKEN}${BYTE_TOKEN}${BYTE_TOKEN}${BYTE_TOKEN})")
+    string(REGEX REPLACE "${LINE_PATTERN}" "\\1\n\t" WAV_BYTES "${WAV_BYTES}")
+
+    file(APPEND "${OUTPUT_SOURCE}" "\tconst uint8_t ${SYMBOL}[] = {\n\t${WAV_BYTES}\n\t};\n")
+    file(APPEND "${OUTPUT_SOURCE}" "\tconst size_t ${SYMBOL}Size = ${WAV_SIZE}u;\n\n")
+endfunction()
+
+append_embedded_wav("OpenLightWav" "${OPEN_LIGHT_WAV}")
+append_embedded_wav("CloseLightWav" "${CLOSE_LIGHT_WAV}")
+
+file(APPEND "${OUTPUT_SOURCE}" [=[}
+}
+]=])

src/Core/Platform/DefaultHardware.h

@@ -7,6 +7,7 @@
 #include "EvdevKeyboardState.h"
 #include "EvdevTouchInput.h"
 #include "Ap3216cPhotoSensor.h"
+#include "Imx6ullAlphaLed.h"
 #endif
 #ifdef TARGET_PC
 #include "SdlAudioInput.h"
@@ -15,6 +16,7 @@
 #include "SdlKeyboardState.h"
 #include "SdlPointerInput.h"
 #include "SdlPhotoSensor.h"
+#include "SimulatedBoardLed.h"
 #endif
 
 namespace Platform
@@ -26,6 +28,7 @@ namespace Platform
 	typedef EvdevKeyboardState DefaultKeyboardState;
 	typedef EvdevTouchInput DefaultPointerInput;
 	typedef Ap3216cPhotoSensor DefaultPhotoSensor;
+	typedef Imx6ullAlphaLed DefaultBoardLed;
 #endif
 #ifdef TARGET_PC
 	typedef SdlAudioInput DefaultAudioInput;
@@ -34,5 +37,6 @@ namespace Platform
 	typedef SdlKeyboardState DefaultKeyboardState;
 	typedef SdlPointerInput DefaultPointerInput;
 	typedef SdlPhotoSensor DefaultPhotoSensor;
+	typedef SimulatedBoardLed DefaultBoardLed;
 #endif
 }

src/Core/Platform/IBoardLed.h

@@ -0,0 +1,14 @@
+#pragma once
+
+namespace Platform
+{
+	class IBoardLed
+	{
+	public:
+		virtual ~IBoardLed() {}
+
+		virtual bool init() = 0;
+		virtual bool set_on(bool on) = 0;
+		virtual bool is_on() const = 0;
+	};
+}

src/Core/Platform/Imx6ullAlphaLed.cpp

@@ -0,0 +1,172 @@
+#include "Imx6ullAlphaLed.h"
+#include <cerrno>
+#include <chrono>
+#include <fcntl.h>
+#include <iostream>
+#include <sys/stat.h>
+#include <thread>
+#include <unistd.h>
+
+namespace
+{
+	const char* const CharacterDevices[] = {
+		"/dev/gpioled",
+		"/dev/dtsled",
+		"/dev/led"};
+
+	const char* const LedClassBrightnessPaths[] = {
+		"/sys/class/leds/sys-led/brightness",
+		"/sys/class/leds/user-led/brightness",
+		"/sys/class/leds/user/brightness",
+		"/sys/class/leds/led0/brightness"};
+
+	const char* const GpioNumber = "3";
+	const char* const GpioDirectory = "/sys/class/gpio/gpio3";
+	const char* const GpioDirectionPath = "/sys/class/gpio/gpio3/direction";
+	const char* const GpioValuePath = "/sys/class/gpio/gpio3/value";
+
+	static bool PathExists(const char* path)
+	{
+		struct stat info;
+		return path != nullptr && stat(path, &info) == 0;
+	}
+
+	static ssize_t WriteFile(const std::string& path, const void* data, size_t size)
+	{
+		const int fd = open(path.c_str(), O_WRONLY | O_CLOEXEC);
+		if (fd < 0)
+		{
+			return -1;
+		}
+
+		const ssize_t written = write(fd, data, size);
+		const int savedErrno = errno;
+		close(fd);
+		errno = savedErrno;
+		return written;
+	}
+
+	static bool WriteTextFile(const std::string& path, const char* value, size_t size)
+	{
+		return WriteFile(path, value, size) == static_cast<ssize_t>(size);
+	}
+}
+
+namespace Platform
+{
+	Imx6ullAlphaLed::Imx6ullAlphaLed()
+		: backend_(Backend::None),
+		  path_(),
+		  on_(false)
+	{
+	}
+
+	bool Imx6ullAlphaLed::init()
+	{
+		if (backend_ != Backend::None)
+		{
+			return true;
+		}
+
+		for (size_t i = 0; i < sizeof(CharacterDevices) / sizeof(CharacterDevices[0]); ++i)
+		{
+			if (PathExists(CharacterDevices[i]))
+			{
+				backend_ = Backend::CharacterDevice;
+				path_ = CharacterDevices[i];
+				std::cout << "[INFO] Board LED backend: " << path_ << std::endl;
+				return set_on(false);
+			}
+		}
+
+		for (size_t i = 0; i < sizeof(LedClassBrightnessPaths) / sizeof(LedClassBrightnessPaths[0]); ++i)
+		{
+			if (PathExists(LedClassBrightnessPaths[i]))
+			{
+				backend_ = Backend::LedClass;
+				path_ = LedClassBrightnessPaths[i];
+				std::cout << "[INFO] Board LED backend: " << path_ << std::endl;
+				return set_on(false);
+			}
+		}
+
+		if (configure_sysfs_gpio())
+		{
+			backend_ = Backend::SysfsGpio;
+			path_ = GpioValuePath;
+			std::cout << "[INFO] Board LED backend: GPIO1_IO03 via sysfs" << std::endl;
+			return set_on(false);
+		}
+
+		std::cerr
+			<< "[WARN] Board LED is unavailable. Load the ALPHA gpioled driver or enable GPIO sysfs."
+			<< std::endl;
+		return false;
+	}
+
+	bool Imx6ullAlphaLed::set_on(bool on)
+	{
+		if (backend_ == Backend::None && !init())
+		{
+			return false;
+		}
+
+		bool success = false;
+		if (backend_ == Backend::CharacterDevice)
+		{
+			const unsigned char command = on ? 1u : 0u;
+			// The teaching driver consumes the byte but returns zero from write().
+			success = WriteFile(path_, &command, sizeof(command)) >= 0;
+		}
+		else if (backend_ == Backend::LedClass)
+		{
+			const char value = on ? '1' : '0';
+			success = WriteTextFile(path_, &value, 1u);
+		}
+		else if (backend_ == Backend::SysfsGpio)
+		{
+			// GPIO1_IO03 is active-low on the ALIENTEK I.MX6U-ALPHA board.
+			const char value = on ? '0' : '1';
+			success = WriteTextFile(path_, &value, 1u);
+		}
+
+		if (!success)
+		{
+			std::cerr
+				<< "[WARN] Failed to turn board LED "
+				<< (on ? "on" : "off")
+				<< " using " << path_
+				<< ": errno=" << errno
+				<< std::endl;
+			return false;
+		}
+
+		on_ = on;
+		std::cout << "[INFO] Board LED: " << (on_ ? "on" : "off") << std::endl;
+		return true;
+	}
+
+	bool Imx6ullAlphaLed::configure_sysfs_gpio()
+	{
+		if (!PathExists(GpioDirectory))
+		{
+			const ssize_t exported = WriteFile("/sys/class/gpio/export", GpioNumber, 1u);
+			if (exported < 0 && errno != EBUSY)
+			{
+				return false;
+			}
+
+			for (size_t attempt = 0u; attempt < 50u && !PathExists(GpioDirectory); ++attempt)
+			{
+				std::this_thread::sleep_for(std::chrono::milliseconds(10));
+			}
+		}
+
+		if (!PathExists(GpioValuePath))
+		{
+			return false;
+		}
+
+		return WriteTextFile(GpioDirectionPath, "out", 3u);
+	}
+}

src/Core/Platform/Imx6ullAlphaLed.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "IBoardLed.h"
+#include <string>
+
+namespace Platform
+{
+	class Imx6ullAlphaLed : public IBoardLed
+	{
+	private:
+		enum class Backend
+		{
+			None,
+			CharacterDevice,
+			LedClass,
+			SysfsGpio
+		};
+
+		Backend backend_;
+		std::string path_;
+		bool on_;
+
+		bool configure_sysfs_gpio();
+
+	public:
+		Imx6ullAlphaLed();
+
+		bool init() override;
+		bool set_on(bool on) override;
+		bool is_on() const override { return on_; }
+	};
+}

src/Core/Platform/SimulatedBoardLed.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#include "IBoardLed.h"
+#include <iostream>
+
+namespace Platform
+{
+	class SimulatedBoardLed : public IBoardLed
+	{
+	private:
+		bool on_;
+
+	public:
+		SimulatedBoardLed() : on_(false) {}
+
+		bool init() override { return true; }
+
+		bool set_on(bool on) override
+		{
+			on_ = on;
+			std::cout << "[INFO] Simulated board LED: " << (on_ ? "on" : "off") << std::endl;
+			return true;
+		}
+
+		bool is_on() const override { return on_; }
+	};
+}