diff --git a/app_main.py b/app_main.py index ade88df..6e4f5e2 100644 --- a/app_main.py +++ b/app_main.py @@ -224,22 +224,26 @@ async def lifespan(app: FastAPI): # 4. Day 21: 预加载新 AI 管道模型（避免首次使用时延迟） if USE_NEW_AI_PIPELINE: - async def _preload_models(): + # Day 28: VAD 同步预加载，避免第一句话不识别 + try: + print("[PRELOAD] 预加载 Silero VAD...") + from server_vad import get_vad_model + get_vad_model() # 直接加载 VAD 模型 + print("[PRELOAD] Silero VAD 预加载完成") + except Exception as e: + print(f"[PRELOAD] VAD 预加载失败: {e}") + + # SenseVoice 异步加载（不阻塞启动） + async def _preload_sensevoice(): try: - print("[PRELOAD] 预加载 Silero VAD...") - from server_vad import get_server_vad - get_server_vad() # 触发 VAD 模型加载 - print("[PRELOAD] 预加载 SenseVoice ASR...") from sensevoice_asr import init_sensevoice - await init_sensevoice() # 异步加载 ASR 模型 - + await init_sensevoice() print("[PRELOAD] 新 AI 管道模型预加载完成") except Exception as e: - print(f"[PRELOAD] 模型预加载失败: {e}") + print(f"[PRELOAD] SenseVoice 预加载失败: {e}") - # 后台预加载，不阻塞启动 - asyncio.create_task(_preload_models()) + asyncio.create_task(_preload_sensevoice()) print("[LIFESPAN] 应用启动完成") @@ -349,7 +353,9 @@ def load_navigation_models(): # global yolo_seg_model, obstacle_detector (Moved to ctx) try: - seg_model_path = os.getenv("BLIND_PATH_MODEL", "model/yolo-seg.pt") + # 使用基于当前文件的绝对路径 + default_seg_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "model", "yolo-seg.pt") + seg_model_path = os.getenv("BLIND_PATH_MODEL", default_seg_path) # Day 20: 优先使用 TensorRT 引擎 seg_model_path = get_best_model_path(seg_model_path) #print(f"[NAVIGATION] 尝试加载模型: {seg_model_path}") @@ -401,7 +407,8 @@ def load_navigation_models(): print(f"[NAVIGATION] 请检查文件路径是否正确") # 【修改开始】使用 ObstacleDetectorClient 替代直接的 YOLO - obstacle_model_path = os.getenv("OBSTACLE_MODEL", "model/yoloe-11l-seg.pt") + default_obs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "model", "yoloe-11l-seg.pt") + obstacle_model_path = os.getenv("OBSTACLE_MODEL", default_obs_path) # Day 20: 优先使用 TensorRT 引擎 obstacle_model_path = get_best_model_path(obstacle_model_path) print(f"[NAVIGATION] 尝试加载障碍物检测模型: {obstacle_model_path}") @@ -483,7 +490,10 @@ def load_indoor_model(): from model_utils import is_tensorrt_engine # Imported here for usage try: - indoor_model_path = os.getenv("INDOOR_MODEL", "model/yolo11l-seg-indoor.engine") + # Day 28: 使用新训练的 14 类模型 (用户请求切换) + # 使用基于当前文件的绝对路径，确保在服务器任意目录启动都能找到模型 + default_model_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "model", "yolo11l-seg-indoor14.engine") + indoor_model_path = os.getenv("INDOOR_MODEL", default_model_path) # 优先使用 TensorRT 引擎 indoor_model_path = get_best_model_path(indoor_model_path) print(f"[INDOOR] 尝试加载室内导盲模型: {indoor_model_path}") @@ -751,7 +761,8 @@ async def start_ai_with_text_custom(user_text: str): if ctx.orchestrator: current_state = ctx.orchestrator.get_state() # 如果在导航模式或红绿灯检测模式（非CHAT模式） - if current_state not in ["CHAT", "IDLE"]: + # Day 28: 允许 INDOOR_NAV 模式下进行对话，但其他模式（盲道、过马路）依然严格屏蔽 + if current_state not in ["CHAT", "IDLE", "INDOOR_NAV"]: # 检查是否是允许的对话触发词 allowed_keywords = ["帮我看", "帮我看下", "帮我找", "找一下", "看看", "识别一下"] is_allowed_query = any(keyword in user_text for keyword in allowed_keywords) @@ -759,7 +770,9 @@ async def start_ai_with_text_custom(user_text: str): # 检查是否是导航控制命令 nav_control_keywords = ["开始过马路", "过马路结束", "开始导航", "盲道导航", "停止导航", "结束导航", "检测红绿灯", "看红绿灯", "停止检测", "停止红绿灯", - "室内导航", "室内导盲"] # 新增室内导航 + "室内导航", "室内导盲", "四内导航", "思维导航", "失内导航", "时内导航", + "室类导航", "类导航", + "退出导航", "关闭导航", "别导了", "别念了", "停止", "导航"] # Day 28: 增强停止命令识别 + 单独"导航" is_nav_control = any(keyword in user_text for keyword in nav_control_keywords) # 如果既不是允许的查询，也不是导航控制命令，则丢弃 @@ -843,7 +856,8 @@ async def start_ai_with_text_custom(user_text: str): return # 【修改】检查是否是导航相关命令 - 使用orchestrator控制 - if "开始导航" in user_text or "盲道导航" in user_text or "帮我导航" in user_text: + # Day 28: 支持单独说"导航"作为盲道导航启动命令（防止因 AS R吞字变成聊天） + if "开始导航" in user_text or "盲道导航" in user_text or "帮我导航" in user_text or user_text.strip() == "导航": # 【新增】如果正在找物品，先停止 if ctx.yolomedia_running: stop_yolomedia() @@ -858,8 +872,11 @@ async def start_ai_with_text_custom(user_text: str): await ui_broadcast_final("[系统] 导航系统未就绪") return - # 【新增】检查是否是室内导航命令 - if "室内导航" in user_text or "室内导盲" in user_text: + # 【新增】检查是否是室内导航命令（包含ASR误识别别名） + # Day 28: 添加更多同音误识别别名 + indoor_nav_aliases = ["室内导航", "室内导盲", "四内导航", "思维导航", "失内导航", "时内导航", + "室类导航", "类导航"] # Day 28: 新增误识别 + if any(alias in user_text for alias in indoor_nav_aliases): # 如果正在找物品，先停止 if ctx.yolomedia_running: stop_yolomedia() @@ -876,7 +893,8 @@ async def start_ai_with_text_custom(user_text: str): # 【修改】停止导航优先判断 # 只要包含"停止导航"或"结束导航"，无论是否包含"室内"，都视为停止指令 - if "停止导航" in user_text or "结束导航" in user_text: + stop_keywords = ["停止导航", "结束导航", "退出导航", "关闭导航", "别导了", "别念了", "停止"] + if any(k in user_text for k in stop_keywords): if ctx.orchestrator: ctx.orchestrator.stop_navigation() print(f"[NAVIGATION] 导航已停止，状态: {ctx.orchestrator.get_state()}") @@ -1060,8 +1078,15 @@ async def start_ai_with_text(user_text: str): from audio_stream import stream_clients for sc in list(stream_clients): if not sc.abort_event.is_set(): - try: sc.q.put_nowait(b"\x00"*BYTES_PER_20MS_16K) - except Exception: pass + # Day 28: 添加少量静音填充防止结尾爆音 (Pop noise fix) + # 增加到 10 帧 (200ms) 以确保完全淡出 + try: + silence_frame = b'\x00' * 640 # 20ms silence (16k * 2 bytes * 0.02) + for _ in range(10): # 200ms silence + sc.q.put_nowait(silence_frame) + except Exception: + pass + try: sc.q.put_nowait(None) except Exception: pass @@ -1128,8 +1153,9 @@ async def start_ai_with_text(user_text: str): from audio_stream import stream_clients for sc in list(stream_clients): if not sc.abort_event.is_set(): - try: sc.q.put_nowait(b"\x00"*BYTES_PER_20MS_16K) - except Exception: pass + # Day 28: 移除静音填充包以消除杂音 + # try: sc.q.put_nowait(b"\x00"*BYTES_PER_20MS_16K) + # except Exception: pass try: sc.q.put_nowait(None) except Exception: pass diff --git a/asr_core.py b/asr_core.py index b67a826..3dcc337 100644 --- a/asr_core.py +++ b/asr_core.py @@ -64,7 +64,9 @@ NAV_CONTROL_WHITELIST = [ "停止导航", "结束导航", "停止检测", "停止红绿灯", "开始导航", "盲道导航", "开始过马路", "过马路结束", "帮我导航", "帮我过马路", - "室内导航", "室内导盲", # Day 25: 新增室内导航命令 + "室内导航", "室内导盲", "四内导航", "思维导航", "失内导航", "时内导航", # Day 28: 室内导航 + 同音误识别 + "室类导航", "类导航", # Day 28: 新增误识别 + "退出导航", "关闭导航", "别导了", "别念了", "停止", # Day 28: 增强停止命令 ] diff --git a/audio_player.py b/audio_player.py index 8dddfa4..29b63d7 100644 --- a/audio_player.py +++ b/audio_player.py @@ -225,6 +225,14 @@ async def _broadcast_audio_optimized(pcm_data: bytes): # 注意：录制在 broadcast_pcm16_realtime 中统一完成，避免重复 + # Day 28: 播放期间全局暂停 VAD，防止系统听到自己的声音 + # 这对于没有回声消除(AEC)的系统至关重要，否则导航提示语音会触发 VAD + # 导致 VAD 误判为用户说话，从而一直占用识别通道 + from server_vad import get_server_vad + vad = get_server_vad() + if vad: + vad.set_tts_playing(True) + # 单次调用交给底层 pacing（20ms节拍在 broadcast_pcm16_realtime 内部实现） await broadcast_pcm16_realtime(full_audio) @@ -232,6 +240,12 @@ async def _broadcast_audio_optimized(pcm_data: bytes): except Exception as e: print(f"[AUDIO] 广播音频失败: {e}") finally: + # 恢复 VAD 检测 + from server_vad import get_server_vad + vad = get_server_vad() + if vad: + vad.set_tts_playing(False) + # 清除播放标志 with _playing_lock: _is_playing = False diff --git a/audio_stream.py b/audio_stream.py index d8ffd91..0e8eb8e 100644 --- a/audio_stream.py +++ b/audio_stream.py @@ -102,6 +102,19 @@ async def hard_reset_audio(reason: str = ""): # 2) 取消当前AI任务 await cancel_current_ai() + # Day 28: 强制重置 VAD TTS 状态，防止因任务取消导致计数器未归零（VAD 冻结） + try: + # Safe import to avoid circular dependency + import sys + if 'server_vad' in sys.modules: + server_vad = sys.modules['server_vad'] + if hasattr(server_vad, 'get_server_vad'): + vad = server_vad.get_server_vad() + if vad: + vad.reset_tts_state() + except Exception as e: + print(f"[HARD-RESET] 重置 VAD 状态失败: {e}") + # 3) 日志 if reason: print(f"[HARD-RESET] {reason}") diff --git a/navigation_master.py b/navigation_master.py index 71f6253..f2a9ade 100644 --- a/navigation_master.py +++ b/navigation_master.py @@ -293,6 +293,38 @@ class NavigationMaster: def get_state(self) -> str: return self.state + # Day 28: 室内导航可视化绘制 + def _draw_indoor_visualizations(self, image: np.ndarray, visualizations: list): + if not visualizations: + return + + for viz in visualizations: + v_type = viz.get('type') + + if v_type == 'walkable_mask': + mask = viz.get('mask') + color_str = viz.get('color', 'rgba(0, 255, 0, 0.3)') + # 这里简单处理，只画绿色轮廓和半透明填充 + if mask is not None: + # 1. 绿色覆盖 + green_mask = np.zeros_like(image) + green_mask[mask > 0] = [0, 255, 0] # BGR + image[:] = cv2.addWeighted(image, 1.0, green_mask, 0.3, 0) + + # 2. 轮廓 + contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + cv2.drawContours(image, contours, -1, (0, 255, 0), 2) + + elif v_type in ('obstacle', 'poi', 'person'): + center = viz.get('center') + label = viz.get('class_name_cn', '?') + if center: + cx, cy = center + color = (0, 0, 255) if v_type == 'obstacle' else (255, 255, 0) + cv2.circle(image, (cx, cy), 5, color, -1) + cv2.putText(image, label, (cx + 10, cy), cv2.FONT_HERSHEY_SIMPLEX, + 0.6, color, 2, cv2.LINE_AA) + def start_blind_path_navigation(self): """启动盲道导航模式""" self.state = BLINDPATH_NAV @@ -330,8 +362,9 @@ class NavigationMaster: """启动室内导航模式（使用室内导盲模型）""" self.state = INDOOR_NAV self.cooldown_until = time.time() + self.COOLDOWN_SEC - if self.blind: - self.blind.reset() + # Day 28: 应该重置室内导航器，而不是盲道导航器 + if self.indoor: + self.indoor.reset() def is_in_navigation_mode(self): """检查是否在导航模式（非对话模式）""" @@ -481,18 +514,28 @@ class NavigationMaster: if self.state == INDOOR_NAV: # 优先使用室内导航器，如果没有则 fallback 到盲道导航器 nav = self.indoor if self.indoor else self.blind + # Day 28: 添加警告日志 + if self.indoor is None: + print("[NAV MASTER] 警告: 室内导航器未初始化，fallback 到盲道导航器！") try: result = nav.process_frame(bgr) except Exception as e: - self.state = RECOVERY + # Day 28: 室内导航出错时，保持在室内模式，不要切到 RECOVERY (会导致自动切回盲道) + print(f"[INDOOR ERROR] 室内导航异常: {e}") + # self.state = RECOVERY <-- 禁止切换! ann_err = bgr.copy() - return OrchestratorResult(ann_err, self._say(now, ""), self.state, {"error": str(e)}) + return OrchestratorResult(ann_err, self._say(now, ""), INDOOR_NAV, {"error": str(e)}) ann = result.annotated_image if result.annotated_image is not None else bgr.copy() say = result.guidance_text or "" state_info = result.state_info if hasattr(result, 'state_info') else {} - return OrchestratorResult(ann, self._say(now, say), self.state, + # Day 28: 绘制室内导航可视化 + visualizations = result.visualizations if hasattr(result, 'visualizations') else [] + self._draw_indoor_visualizations(ann, visualizations) + + # Day 28: 确保返回正确的状态 INDOOR_NAV + return OrchestratorResult(ann, self._say(now, say), INDOOR_NAV, {"source": "indoor", "state_info": state_info}) # 各状态处理 diff --git a/server_vad.py b/server_vad.py index 21ef7f3..3f7f23d 100644 --- a/server_vad.py +++ b/server_vad.py @@ -96,7 +96,8 @@ class SileroVAD: self.speech_audio = bytearray() # 存储语音音频 # TTS 播放状态 - 播放期间暂停 VAD - self.tts_playing = False + # Day 28: 使用引用计数处理并发播放的情况 + self.tts_playing_count = 0 self.tts_end_time = 0 # TTS 结束时间 self.tts_cooldown_ms = 500 # TTS 结束后等待 500ms 再开始检测 @@ -105,9 +106,9 @@ class SileroVAD: self.window_size = 5 # 滑动窗口大小 self.frame_threshold = 3 # 至少多少帧语音才算开始说话 - # Day 23: Pre-speech buffer (Lookback) to fix "cut-off" start of words - # 300ms lookback approx. (each chunk is 32ms) -> 10 chunks - self.pre_speech_buffer = collections.deque(maxlen=10) + # Day 23+28: Pre-speech buffer (Lookback) to fix "cut-off" start of words + # Day 28: 增加到 768ms (24 chunks) 以捕获 "室内导航" 等较长开头，防止 ASR 吞字 + self.pre_speech_buffer = collections.deque(maxlen=24) print(f"[VAD] 初始化: threshold={threshold}, threshold_low={threshold_low}, " f"min_silence_ms={min_silence_ms}, min_speech_ms={min_speech_ms}") @@ -120,29 +121,46 @@ class SileroVAD: self.last_speech_time = 0 self.speech_start_time = 0 self.voice_window.clear() - self.tts_playing = False + self.tts_playing_count = 0 self.tts_end_time = 0 if self.model: self.model.reset_states() + if hasattr(self, 'pre_speech_buffer'): + self.pre_speech_buffer.clear() + + def reset_tts_state(self): + """强制重置 TTS 播放状态 (用于硬重置)""" + self.tts_playing_count = 0 + print("[VAD] 强制重置 TTS 状态 (VAD 恢复)") def set_tts_playing(self, playing: bool): - """设置 TTS 播放状态""" - self.tts_playing = playing - if not playing: - # TTS 结束，记录时间 - self.tts_end_time = time.time() * 1000 - print("[VAD] TTS 结束，等待冷却期...") + """设置 TTS 播放状态 (引用计数)""" + if playing: + self.tts_playing_count += 1 + if self.tts_playing_count == 1: + print("[VAD] TTS 开始播放，暂停 VAD 检测") + # TTS 开始播放时，如果正在录音则中断 + if self.is_speaking: + self.is_speaking = False + self.speech_audio.clear() + self.voice_window.clear() + # Day 23: Clear lookback buffer + if hasattr(self, 'pre_speech_buffer'): + self.pre_speech_buffer.clear() + # Day 28: 重置模型状态 + if self.model: + self.model.reset_states() + print("[VAD] TTS 播放打断语音录制") else: - print("[VAD] TTS 开始播放，暂停 VAD 检测") - # TTS 开始播放时，如果正在录音则中断 - if self.is_speaking: - self.is_speaking = False - self.speech_audio.clear() - self.voice_window.clear() - # Day 23: Clear lookback buffer - if hasattr(self, 'pre_speech_buffer'): - self.pre_speech_buffer.clear() - print("[VAD] TTS 播放打断语音录制") + if self.tts_playing_count > 0: + self.tts_playing_count -= 1 + if self.tts_playing_count == 0: + # TTS 结束，记录时间 + self.tts_end_time = time.time() * 1000 + print("[VAD] TTS 完全结束，等待冷却期...") + else: + # 已经是0了，忽略 + pass def process(self, audio_bytes: bytes) -> dict: """ @@ -172,7 +190,7 @@ class SileroVAD: # TTS 播放期间，跳过 VAD 检测 current_time = time.time() * 1000 - if self.tts_playing: + if self.tts_playing_count > 0: return result # TTS 刚结束，等待冷却期 diff --git a/workflow_indoor.py b/workflow_indoor.py index 129681d..b9ee0f9 100644 --- a/workflow_indoor.py +++ b/workflow_indoor.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ 室内导航工作流 (Indoor Navigation Workflow) -Day 26: 专为室内导盲模型 (yolo11l-seg-indoor) 设计 +Day 26: 专为室内导盲模型 (yolo11l-seg-indoor14) 设计 类别映射 (14 classes from MIT Indoor): - 可行走区域: floor(0), corridor(1), sidewalk(2) @@ -22,85 +22,74 @@ from collections import deque logger = logging.getLogger(__name__) -# ========== 类别常量 ========== -# 可行走区域 +# ========== 类别常量 (14类模型 - yolo11l-seg-indoor14) ========== +# Day 28: 使用 14 类模型 (MIT Indoor Subset) + +# 可行走区域 (0-2) WALKABLE_CLASSES = {0, 1, 2} # floor, corridor, sidewalk CLASS_FLOOR = 0 CLASS_CORRIDOR = 1 CLASS_SIDEWALK = 2 -# 静态障碍物 (家具 + 杂物) -OBSTACLE_CLASSES = {3, 4, 5, 11, 12, 14, 15, 16, 17, 18} +# 静态障碍物 (3-5, 11-12) +OBSTACLE_CLASSES = {3, 4, 5, 11, 12, 13} # window 只要是障碍物也算? window(13)是墙? +# Wait, Window is 13. Is window an obstacle? Usually yes (don't walk into it). +# Cabinet 11, Trash 12. CLASS_CHAIR = 3 CLASS_TABLE = 4 CLASS_SOFA_BED = 5 CLASS_CABINET = 11 CLASS_TRASH_CAN = 12 -CLASS_TRASH_CAN = 12 -# CLASS_CUP_BOTTLE = 14 (Removed) -CLASS_BAG = 14 -CLASS_ELECTRONICS = 15 -CLASS_PLANT = 16 -CLASS_OBSTACLE = 17 -CLASS_APPLIANCE = 18 +CLASS_WINDOW = 13 # 窗户通常视为边界或障碍 +CLASS_WALL = 9 # Wall 9 -# 兴趣点 -POI_CLASSES = {6, 7, 8, 19, 20} # door, elevator, stairs, toilet, sink +# 兴趣点 (6-8) +POI_CLASSES = {6, 7, 8} # door, elevator, stairs CLASS_DOOR = 6 CLASS_ELEVATOR = 7 CLASS_STAIRS = 8 -CLASS_TOILET = 19 -CLASS_SINK = 20 + +# 动态障碍 (10) +CLASS_PERSON = 10 # 边界 -BOUNDARY_CLASSES = {9, 10} # wall, window -CLASS_WALL = 9 -CLASS_WINDOW = 10 - -# 动态障碍 -CLASS_PERSON = 13 +BOUNDARY_CLASSES = {9, 13} # wall(9), window(13) # 类别名称映射 CLASS_NAMES = { 0: 'floor', 1: 'corridor', 2: 'sidewalk', 3: 'chair', 4: 'table', 5: 'sofa_bed', 6: 'door', 7: 'elevator', 8: 'stairs', - 9: 'wall', 10: 'window', 11: 'cabinet', - 12: 'trash_can', 13: 'person', - 14: 'bag', 15: 'electronics', 16: 'plant', - 17: 'obstacle', 18: 'appliance', - 19: 'toilet', 20: 'sink', - 21: 'tableware' + 9: 'wall', 10: 'person', 11: 'cabinet', + 12: 'trash_can', 13: 'window' } - # 中文名称（用于语音） CLASS_NAMES_CN = { 0: '地面', 1: '走廊', 2: '人行道', 3: '椅子', 4: '桌子', 5: '沙发', 6: '门', 7: '电梯', 8: '楼梯', - 9: '墙壁', 10: '窗户', 11: '柜子', - 12: '垃圾桶', 13: '行人', - 14: '包', 15: '电子设备', 16: '绿植', - 17: '障碍物', 18: '家电', - 19: '卫生间', 20: '洗手台', - 21: '餐具' + 9: '墙壁', 10: '行人', 11: '柜子', + 12: '垃圾桶', 13: '窗户' } -# 物品类 (不播报，除非寻找模式) -ITEM_CLASSES = {21} -CLASS_TABLEWARE = 21 +# 物品类 (无) +ITEM_CLASSES = set() # ========== 配置参数 ========== -CONF_THRESHOLD = float(os.getenv('INDOOR_CONF_THRESHOLD', '0.25')) -WALKABLE_MIN_AREA = int(os.getenv('INDOOR_WALKABLE_MIN_AREA', '3000')) -OBSTACLE_MIN_AREA = int(os.getenv('INDOOR_OBSTACLE_MIN_AREA', '500')) +# Day 28: 进一步降低阈值以提升木地板检测率 +# Day 28: 进一步降低阈值以提升木地板检测率 +CONF_THRESHOLD = float(os.getenv('INDOOR_CONF_THRESHOLD', '0.05')) # 全局极低阈值，由后续逻辑二次过滤 +WALKABLE_MIN_AREA = int(os.getenv('INDOOR_WALKABLE_MIN_AREA', '50')) # 极端降低最小面积以进行调试 (原 1000) +OBSTACLE_MIN_AREA = int(os.getenv('INDOOR_OBSTACLE_MIN_AREA', '300')) # 语音间隔 GUIDE_INTERVAL = float(os.getenv('INDOOR_GUIDE_INTERVAL', '3.0')) DIRECTION_INTERVAL = float(os.getenv('INDOOR_DIRECTION_INTERVAL', '2.5')) POI_INTERVAL = float(os.getenv('INDOOR_POI_INTERVAL', '5.0')) OBSTACLE_INTERVAL = float(os.getenv('INDOOR_OBSTACLE_INTERVAL', '2.0')) +# Day 28: “未检测到可行走区域”播报间隔（8秒） +NO_WALKABLE_INTERVAL = float(os.getenv('INDOOR_NO_WALKABLE_INTERVAL', '8.0')) # ========== 可视化颜色 (BGR) ========== VIS_COLORS = { @@ -135,6 +124,10 @@ class IndoorNavigator: self.seg_model = seg_model self.device_id = device_id self.frame_counter = 0 + + # Day 28: 持久化缓冲参数 + self.no_walkable_persistence_sec = 2.0 + self.last_walkable_detected_time = 0 # 语音节流 self.last_guide_time = 0 @@ -150,11 +143,14 @@ class IndoorNavigator: # 缓存 self.last_walkable_mask = None + self.last_valid_walkable_mask = None + self.last_no_walkable_time = 0 + self.last_obstacles = [] self.last_obstacles = [] self.last_pois = [] - # 灰度图（用于光流等） - self.prev_gray = None + # Day 28: 移除未使用的灰度图转换 (光流功能未启用) + # self.prev_gray = None # 日志间隔 self.log_interval = int(os.getenv('AIGLASS_LOG_INTERVAL', '30')) @@ -172,6 +168,9 @@ class IndoorNavigator: self.last_obstacle_time = 0 self.last_guidance_text = "" self.last_direction_text = "" + self.last_valid_walkable_mask = None + self.last_no_walkable_time = 0 # Day 28: "未检测到可行走区域"节流 + self.last_walkable_detected_time = 0 self.last_walkable_mask = None self.last_obstacles = [] self.last_pois = [] @@ -207,13 +206,27 @@ class IndoorNavigator: obstacles = self.last_obstacles pois = self.last_pois - # 生成导航引导 + + # 3. 缓存有效的 mask (用于可视化防抖) + walkable_area = int(np.count_nonzero(walkable_mask)) if walkable_mask is not None else 0 + if walkable_area > WALKABLE_MIN_AREA: + self.last_valid_walkable_mask = walkable_mask + + # 4. 生成导航引导 if walkable_mask is not None: guidance_text = self._generate_guidance(walkable_mask, obstacles, pois, h, w, now) - - # 添加可视化 - self._add_mask_visualization(walkable_mask, frame_visualizations, - "walkable_mask", "rgba(0, 255, 0, 0.3)") + + # 5. 可视化 (带持久化防抖) + viz_mask = walkable_mask + + # 如果当前没有检测到路，但还在持久化时间内，使用缓存的 mask 进行可视化 + if (viz_mask is None or walkable_area < WALKABLE_MIN_AREA) and \ + (now - self.last_walkable_detected_time) < self.no_walkable_persistence_sec and \ + self.last_valid_walkable_mask is not None: + viz_mask = self.last_valid_walkable_mask + + self._add_mask_visualization(viz_mask, frame_visualizations, + "walkable_mask", "rgba(0, 255, 0, 0.3)") # 障碍物可视化 for obs in obstacles: @@ -225,7 +238,8 @@ class IndoorNavigator: # 日志 if self.frame_counter % self.log_interval == 0: - walkable_area = int(walkable_mask.sum()) if walkable_mask is not None else 0 + # Day 28: 修复面积计算 - 使用 count_nonzero 而不是 sum (mask 值是 0 或 255) + walkable_area = int(np.count_nonzero(walkable_mask)) if walkable_mask is not None else 0 logger.info(f"[INDOOR] Frame={self.frame_counter} | 可行走面积={walkable_area} | " f"障碍物={len(obstacles)} | 兴趣点={len(pois)}") @@ -237,11 +251,12 @@ class IndoorNavigator: 'pois_count': len(pois), } - # 更新灰度图 - self.prev_gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + # Day 28: 移除未使用的灰度图转换 + # self.prev_gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + # Day 28: 避免每帧复制图像，直接传递原图像（下游如需可视化再复制） return IndoorResult( - annotated_image=image.copy(), + annotated_image=image, # 不再 copy，节省内存/CPU guidance_text=guidance_text, state_info=state_info, visualizations=frame_visualizations @@ -275,23 +290,41 @@ class IndoorNavigator: cls_id = int(cls_id.item()) conf_val = float(conf.item()) - # 过滤物品类 (默认不参与导航逻辑，防止刷屏) + # 过滤物品类 (默认不参与导航逻辑，避免刷屏) if cls_id in ITEM_CLASSES: - # 可以选择存入特定的 items 列表供"找东西"功能使用 - # 这里暂时忽略，避免干扰避障 - continue + continue + + # Day 28: 混合阈值策略 + # 地面类(WALKABLE)使用全局低阈值(0.05)以提高召回率 + # 障碍物(OBSTACLE/POI/BOUNDARY)使用较高阈值(0.25)以拒绝误报 + filter_threshold = 0.25 + if cls_id in WALKABLE_CLASSES: + filter_threshold = 0.05 + + if conf_val < filter_threshold: + continue # 调整 mask 尺寸 mask_resized = cv2.resize(mask, (w, h), interpolation=cv2.INTER_NEAREST) mask_bin = (mask_resized > 0.5).astype(np.uint8) area = int(mask_bin.sum()) - if area < 100: # 过滤小碎片 + + # Day 28: 调试日志 - 查看检测到的类别 (ALL detections) + if area > 10: # 几乎记录所有检测 + cls_name = CLASS_NAMES.get(cls_id, f'unknown_{cls_id}') + logger.info(f"[INDOOR DEBUG] 检测到 {cls_name}(id={cls_id}) conf={conf_val:.2f} area={area}") + + if area < 50: # 极端小的才过滤 continue # 可行走区域 if cls_id in WALKABLE_CLASSES and area > WALKABLE_MIN_AREA: - walkable_mask = cv2.bitwise_or(walkable_mask, mask_bin * 255) + # Day 28: 确保类型一致，避免 bitwise_or 失败 + mask_add = (mask_bin * 255).astype(np.uint8) + walkable_mask = cv2.bitwise_or(walkable_mask, mask_add) + if area > 10000: # 调试：记录大面积添加 + logger.info(f"[INDOOR DEBUG] 添加可行走区域: class={cls_id} area={area} current_total={np.count_nonzero(walkable_mask)}") # 障碍物 elif cls_id in OBSTACLE_CLASSES or cls_id == CLASS_PERSON: @@ -351,8 +384,15 @@ class IndoorNavigator: self.last_obstacle_time = now self.last_guidance_text = guidance_text elif direction_guidance: + # Day 28: "未检测到可行走区域" 降低播报频率 + # Day 28: "未检测到可行走区域" 降低播报频率 + if direction_guidance == "未检测到可行走区域": + # 首次检测到（last_no_walkable_time == 0）或者间隔已过8秒 + if self.last_no_walkable_time == 0 or (now - self.last_no_walkable_time) > NO_WALKABLE_INTERVAL: + guidance_text = direction_guidance + self.last_no_walkable_time = now # 方向引导节流 - if direction_guidance != self.last_direction_text: + elif direction_guidance != self.last_direction_text: if (now - self.last_direction_time) > DIRECTION_INTERVAL: guidance_text = direction_guidance self.last_direction_time = now @@ -369,13 +409,25 @@ class IndoorNavigator: def _compute_direction_guidance(self, walkable_mask, h, w): """计算方向引导""" - if walkable_mask is None or walkable_mask.sum() < WALKABLE_MIN_AREA: + # Day 28: 使用 count_nonzero 替代 sum (mask 值是 0 或 255) + walkable_area = np.count_nonzero(walkable_mask) if walkable_mask is not None else 0 + now = time.time() + + if walkable_area < WALKABLE_MIN_AREA: + # 缓冲逻辑：如果最近才看到过路，不要立刻报错 + if (now - self.last_walkable_detected_time) < self.no_walkable_persistence_sec: + return None # 保持沉默，或者返回 "保持直行" (更稳妥是沉默) return "未检测到可行走区域" + # 如果检测到了，更新时间戳 + self.last_walkable_detected_time = now + # 分析下半部分（更近的区域） lower_half = walkable_mask[int(h * 0.5):, :] - if lower_half.sum() < 1000: + if np.count_nonzero(lower_half) < 1000: + if (now - self.last_walkable_detected_time) < self.no_walkable_persistence_sec: + return None return "前方可行走区域较小，请小心" # 计算左中右分布 @@ -448,8 +500,8 @@ class IndoorNavigator: if cy > h * 0.5: # 比较近 return f"注意前方有{name_cn}" - # 门/电梯/卫生间/洗手台提示 - elif cls_id in (CLASS_DOOR, CLASS_ELEVATOR, CLASS_TOILET, CLASS_SINK): + # 门/电梯提示 + elif cls_id in (CLASS_DOOR, CLASS_ELEVATOR): if cy > h * 0.3: # 在视野内 position = "左侧" if cx < w * 0.4 else ("右侧" if cx > w * 0.6 else "前方") return f"{position}有{name_cn}" diff --git a/yoloe_backend.py b/yoloe_backend.py index 223c10c..0ab595f 100644 --- a/yoloe_backend.py +++ b/yoloe_backend.py @@ -15,7 +15,9 @@ except Exception: from ultralytics import YOLO as _MODEL # Day 20: 优先使用 TensorRT 引擎 -DEFAULT_MODEL_PATH = get_best_model_path(os.getenv("YOLOE_MODEL_PATH", "model/yoloe-11l-seg.pt")) +# Day 28: 使用基于当前文件的绝对路径 +_DEFAULT_YOLOE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "model", "yoloe-11l-seg.pt") +DEFAULT_MODEL_PATH = get_best_model_path(os.getenv("YOLOE_MODEL_PATH", _DEFAULT_YOLOE_PATH)) TRACKER_CFG = os.getenv("YOLO_TRACKER_YAML", "bytetrack.yaml") class YoloEBackend: