diff --git a/README.md b/README.md index 3ae7a35..e73276a 100644 --- a/README.md +++ b/README.md @@ -22,11 +22,9 @@ ### 编译 ```bash -cd /workspace # pip install -r requirements.txt +cd /workspace colcon build --symlink-install -source /opt/ros/humble/setup.bash -source install/setup.bash ``` ### 注意事项 @@ -34,63 +32,55 @@ source install/setup.bash ```bash source /opt/ros/humble/setup.bash -source install/setup.bash -export TURTLEBOT3_MODEL=burger +source /workspace/install/setup.bash export NAV2_MAP_PATH=/workspace/src/turtlebot3_simulations/turtlebot3_gazebo/map/office_map.yaml ``` -为了方便,也可以写入 `~/.bashrc`: - -```bash -echo 'source /opt/ros/humble/setup.bash' >> ~/.bashrc -echo 'source /workspace/install/setup.bash' >> ~/.bashrc -echo 'export TURTLEBOT3_MODEL=burger' >> ~/.bashrc -echo 'export NAV2_MAP_PATH=/workspace/src/turtlebot3_simulations/turtlebot3_gazebo/map/office_map.yaml' >> ~/.bashrc -source ~/.bashrc -``` -1. 启动仿真, Nav2导航栈, AMCL节点,rviz +## 2. 启动仿真(后面每一个步骤都在新终端执行) ```bash ./gzsim_run.sh - +``` + +## 3. 启动 AMCL节点(负责 “map_server + amcl”) + +```bash ros2 launch nav2_rrtstar_planner bringup_localization_with_initial_pose.launch.py \ use_sim_time:=true \ - map:=$NAV2_MAP_PATH + map:=$NAV2_MAP_PATH \ + params_file:=/workspace/src/TurtleBot-RRT-Star/nav2_params.yaml +``` +## 4. 启动 Nav2导航栈(负责 “RRT* 规划器 + controller + bt_navigator) + +```bash ros2 launch nav2_bringup navigation_launch.py \ use_sim_time:=True \ - params_file:=/workspace/src/TurtleBot-RRT-Star/nav2_params.yaml \ - map:=$NAV2_MAP_PATH + params_file:=/workspace/src/TurtleBot-RRT-Star/nav2_params.yaml +``` +## 5. 打开rviz + +```bash export __NV_PRIME_RENDER_OFFLOAD=1 export __GLX_VENDOR_LIBRARY_NAME=nvidia ros2 run rviz2 rviz2 -d /opt/ros/humble/share/nav2_bringup/rviz/nav2_default_view.rviz --ros-args -p use_sim_time:=true - ``` -2. 新开终端,执行环境生效后启动vlm_perception_pkg的语义检测节点,开始目标语义标注并构建语义记忆库,在机器人导航过程中,节点实时检测环境中的目标并生成语义标注结果,保存JSON 格式的语义记忆库到指定路径。 + +新开终端,执行环境生效后启动vlm_perception_pkg的语义检测节点,开始目标语义标注并构建语义记忆库,在机器人导航过程中,节点实时检测环境中的目标并生成语义标注结果,保存JSON 格式的语义记忆库到指定路径。 +## 6. 启动目标检测节点 + ```bash ros2 run vlm_perception_pkg vlm_detection --ros-args -p use_sim_time:=true -# 默认:启动时加载历史 JSON(同一个语义记忆库文件) -# 可选:启动时重置语义记忆库,避免历史最高分长期不更新 +# 默认:启动时加载历史 JSON(同一个语义记忆库文件)可选:启动时重置语义记忆库,避免历史最高分长期不更新 # ros2 run vlm_perception_pkg vlm_detection --ros-args -p use_sim_time:=true -p reset_records_on_startup:=true ``` -3. 加载生成的语义记忆库文件(in /workspace/src/vlm_nav_pkg/config),通过 CLIP 将语义标签编码为shared embedding space,当用户输入自然语言查询时,系统匹配最接近的已知目标,提取其坐标并发送至 Nav2,实现基于自然语言的语义导航。以下是验证步骤: -4. 重新启动仿真, Nav2导航栈, AMCL节点,rviz + +检测完成后可查看目标标注结果(in /workspace/src/vlm_nav_pkg/config,带时间戳),加载生成的语义记忆库文件,通过 CLIP 将语义标签编码为shared embedding space,当用户输入自然语言查询时,系统匹配最接近的已知目标,提取其坐标并发送至 Nav2,实现基于自然语言的语义导航。以下是验证步骤: +## 7. 启动目标检测节点 + ```bash - -./gzsim_run.sh - -ros2 launch nav2_rrtstar_planner bringup_localization_with_initial_pose.launch.py \ - use_sim_time:=true \ - map:=$NAV2_MAP_PATH - -ros2 launch nav2_bringup navigation_launch.py \ - use_sim_time:=True \ - params_file:=/workspace/src/TurtleBot-RRT-Star/nav2_params.yaml \ - map:=$NAV2_MAP_PATH - -ros2 run rviz2 rviz2 -d /opt/ros/humble/share/nav2_bringup/rviz/nav2_default_view.rviz --ros-args -p use_sim_time:=true - +# 关闭检测节点后 pip3 install "numpy<2" ros2 run vlm_nav_pkg semantic_nav ``` diff --git a/assets/picture07.png b/assets/picture07.png index ed383af..1596a90 100644 Binary files a/assets/picture07.png and b/assets/picture07.png differ diff --git a/src/TurtleBot-RRT-Star/CMakeLists.txt b/src/TurtleBot-RRT-Star/CMakeLists.txt index 4700401..6eb216f 100755 --- a/src/TurtleBot-RRT-Star/CMakeLists.txt +++ b/src/TurtleBot-RRT-Star/CMakeLists.txt @@ -79,10 +79,6 @@ install(DIRECTORY launch/ DESTINATION share/${PROJECT_NAME}/launch ) -install(DIRECTORY scripts/ - DESTINATION share/${PROJECT_NAME}/scripts -) - if(BUILD_TESTING) find_package(ament_lint_auto REQUIRED) ament_lint_auto_find_test_dependencies() diff --git a/src/vlm_nav_pkg/config/example_object_detection_vlm.json b/src/vlm_nav_pkg/config/example_object_detection_vlm.json index 2befd64..739e61d 100755 --- a/src/vlm_nav_pkg/config/example_object_detection_vlm.json +++ b/src/vlm_nav_pkg/config/example_object_detection_vlm.json @@ -1,7 +1,7 @@ { "Refrigerator": { - "highest_score": 0.301513671875, - "last_detected": "2026-05-18T14:13:25.085735", + "highest_score": 0.271240234375, + "last_detected": "2026-05-18T10:25:26.258502", "position": { "x": -1.3181, "y": 0.5165, @@ -9,39 +9,39 @@ } }, "water dispenser": { - "highest_score": 0.316162109375, - "last_detected": "2026-05-18T14:15:56.440127", + "highest_score": 0.31689453125, + "last_detected": "2026-05-18T10:26:11.990891", "position": { - "x": -1.6292, - "y": -1.7906, - "z": 0.661 + "x": -1.4587, + "y": -1.8551, + "z": 0.6153 } }, "sofa": { - "highest_score": 0.307861328125, - "last_detected": "2026-05-18T14:17:27.585956", + "highest_score": 0.311767578125, + "last_detected": "2026-05-18T10:26:51.065460", "position": { - "x": 1.4535, - "y": -0.4709, - "z": 0.5445 + "x": 1.6362, + "y": 0.2425, + "z": 0.4759 } }, "white toilet": { - "highest_score": 0.269287109375, - "last_detected": "2026-05-18T14:13:07.308074", + "highest_score": 0.2568359375, + "last_detected": "2026-05-18T10:25:18.120319", "position": { - "x": -4.3598, - "y": -1.1593, - "z": 0.3363 + "x": -4.4027, + "y": -1.2225, + "z": 0.1715 } }, "office chair with wheels": { - "highest_score": 0.3037109375, - "last_detected": "2026-05-18T14:12:39.416197", + "highest_score": 0.302734375, + "last_detected": "2026-05-18T10:26:35.889506", "position": { - "x": -2.8461, - "y": 2.6732, - "z": 0.4811 + "x": -2.9779, + "y": 2.3836, + "z": 0.4007 } } } \ No newline at end of file diff --git a/src/vlm_perception_pkg/vlm_perception_pkg/vlm_detection.py b/src/vlm_perception_pkg/vlm_perception_pkg/vlm_detection.py index f8998dc..048e67f 100755 --- a/src/vlm_perception_pkg/vlm_perception_pkg/vlm_detection.py +++ b/src/vlm_perception_pkg/vlm_perception_pkg/vlm_detection.py @@ -55,7 +55,12 @@ class ClipDetectionNode(Node): # CLIP + Faster R-CNN self.clip_model, self.clip_preprocess = clip.load("ViT-B/32", device=self.device) - self.frcnn_model = torchvision.models.detection.fasterrcnn_resnet50_fpn_v2(weights="DEFAULT").eval() + self.frcnn_model = torchvision.models.detection.fasterrcnn_resnet50_fpn_v2( + weights="DEFAULT" + ).to(self.device).eval() + self.get_logger().info( + f"Detection models initialized on device={self.device}" + ) # Labels and score tracking self.labels = ["Refrigerator","water dispenser", "sofa", "white toilet", "office chair with wheels"] @@ -208,7 +213,7 @@ class ClipDetectionNode(Node): depth_img = self.latest_depth.copy() frame_stamp = self.latest_depth_stamp camera_info = self.camera_info - image_tensor = transforms.ToTensor()(cv_img) + image_tensor = transforms.ToTensor()(cv_img).to(self.device) frcnn_min_score = self.get_parameter( 'frcnn_min_score' ).get_parameter_value().double_value