test_dengjun
/
ros2_office_vlm
forked from hq/ros2_office_vlm
1
0
Fork 0
Code Pull Requests Activity

update

This commit is contained in:
äHuang77 2026-05-18 11:34:58 +00:00
parent 29af363b4c
commit 2f9bcbb8ad
5 changed files with 58 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
README.md
View File

@ -22,11 +22,9 @@
### 编译
```bash
cd /workspace
# pip install -r requirements.txt
cd /workspace
colcon build --symlink-install
source /opt/ros/humble/setup.bash
source install/setup.bash
```
### 注意事项
@ -34,63 +32,55 @@ source install/setup.bash
```bash
source /opt/ros/humble/setup.bash
source install/setup.bash
export TURTLEBOT3_MODEL=burger
source /workspace/install/setup.bash
export NAV2_MAP_PATH=/workspace/src/turtlebot3_simulations/turtlebot3_gazebo/map/office_map.yaml
```
为了方便，也可以写入 `~/.bashrc`：
```bash
echo 'source /opt/ros/humble/setup.bash' >> ~/.bashrc
echo 'source /workspace/install/setup.bash' >> ~/.bashrc
echo 'export TURTLEBOT3_MODEL=burger' >> ~/.bashrc
echo 'export NAV2_MAP_PATH=/workspace/src/turtlebot3_simulations/turtlebot3_gazebo/map/office_map.yaml' >> ~/.bashrc
source ~/.bashrc
```
1. 启动仿真， Nav2导航栈， AMCL节点，rviz
## 2. 启动仿真(后面每一个步骤都在新终端执行)
```bash
./gzsim_run.sh
```
## 3. 启动 AMCL节点（负责 “map_server + amcl”）
```bash
ros2 launch nav2_rrtstar_planner bringup_localization_with_initial_pose.launch.py \
use_sim_time:=true \
map:=$NAV2_MAP_PATH
map:=$NAV2_MAP_PATH \
params_file:=/workspace/src/TurtleBot-RRT-Star/nav2_params.yaml
```
## 4. 启动 Nav2导航栈（负责 “RRT* 规划器 + controller + bt_navigator）
```bash
ros2 launch nav2_bringup navigation_launch.py \
use_sim_time:=True \
params_file:=/workspace/src/TurtleBot-RRT-Star/nav2_params.yaml \
map:=$NAV2_MAP_PATH
params_file:=/workspace/src/TurtleBot-RRT-Star/nav2_params.yaml
```
## 5. 打开rviz
```bash
export __NV_PRIME_RENDER_OFFLOAD=1
export __GLX_VENDOR_LIBRARY_NAME=nvidia
ros2 run rviz2 rviz2 -d /opt/ros/humble/share/nav2_bringup/rviz/nav2_default_view.rviz --ros-args -p use_sim_time:=true
```
2. 新开终端，执行环境生效后启动vlm_perception_pkg的语义检测节点，开始目标语义标注并构建语义记忆库，在机器人导航过程中，节点实时检测环境中的目标并生成语义标注结果，保存JSON 格式的语义记忆库到指定路径。
新开终端，执行环境生效后启动vlm_perception_pkg的语义检测节点，开始目标语义标注并构建语义记忆库，在机器人导航过程中，节点实时检测环境中的目标并生成语义标注结果，保存JSON 格式的语义记忆库到指定路径。
## 6. 启动目标检测节点
```bash
ros2 run vlm_perception_pkg vlm_detection --ros-args -p use_sim_time:=true
# 默认：启动时加载历史 JSON（同一个语义记忆库文件）
# 可选：启动时重置语义记忆库，避免历史最高分长期不更新
# 默认：启动时加载历史 JSON（同一个语义记忆库文件）可选：启动时重置语义记忆库，避免历史最高分长期不更新
# ros2 run vlm_perception_pkg vlm_detection --ros-args -p use_sim_time:=true -p reset_records_on_startup:=true
```
3. 加载生成的语义记忆库文件（in /workspace/src/vlm_nav_pkg/config），通过 CLIP 将语义标签编码为shared embedding space，当用户输入自然语言查询时，系统匹配最接近的已知目标，提取其坐标并发送至 Nav2，实现基于自然语言的语义导航。以下是验证步骤：
4. 重新启动仿真， Nav2导航栈， AMCL节点，rviz
检测完成后可查看目标标注结果（in /workspace/src/vlm_nav_pkg/config，带时间戳），加载生成的语义记忆库文件，通过 CLIP 将语义标签编码为shared embedding space，当用户输入自然语言查询时，系统匹配最接近的已知目标，提取其坐标并发送至 Nav2，实现基于自然语言的语义导航。以下是验证步骤：
## 7. 启动目标检测节点
```bash
./gzsim_run.sh
ros2 launch nav2_rrtstar_planner bringup_localization_with_initial_pose.launch.py \
use_sim_time:=true \
map:=$NAV2_MAP_PATH
ros2 launch nav2_bringup navigation_launch.py \
use_sim_time:=True \
params_file:=/workspace/src/TurtleBot-RRT-Star/nav2_params.yaml \
map:=$NAV2_MAP_PATH
ros2 run rviz2 rviz2 -d /opt/ros/humble/share/nav2_bringup/rviz/nav2_default_view.rviz --ros-args -p use_sim_time:=true
# 关闭检测节点后
pip3 install "numpy<2"
ros2 run vlm_nav_pkg semantic_nav
```

BIN
assets/picture07.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 230 KiB

After

Width:  |  Height:  |  Size: 377 KiB

4
src/TurtleBot-RRT-Star/CMakeLists.txt
View File

@ -79,10 +79,6 @@ install(DIRECTORY launch/
DESTINATION share/${PROJECT_NAME}/launch
)
install(DIRECTORY scripts/
DESTINATION share/${PROJECT_NAME}/scripts
)
if(BUILD_TESTING)
find_package(ament_lint_auto REQUIRED)
ament_lint_auto_find_test_dependencies()

44
src/vlm_nav_pkg/config/example_object_detection_vlm.json
View File

@ -1,7 +1,7 @@
{
"Refrigerator": {
"highest_score": 0.301513671875,
"last_detected": "2026-05-18T14:13:25.085735",
"highest_score": 0.271240234375,
"last_detected": "2026-05-18T10:25:26.258502",
"position": {
"x": -1.3181,
"y": 0.5165,
@ -9,39 +9,39 @@
}
},
"water dispenser": {
"highest_score": 0.316162109375,
"last_detected": "2026-05-18T14:15:56.440127",
"highest_score": 0.31689453125,
"last_detected": "2026-05-18T10:26:11.990891",
"position": {
"x": -1.6292,
"y": -1.7906,
"z": 0.661
"x": -1.4587,
"y": -1.8551,
"z": 0.6153
}
},
"sofa": {
"highest_score": 0.307861328125,
"last_detected": "2026-05-18T14:17:27.585956",
"highest_score": 0.311767578125,
"last_detected": "2026-05-18T10:26:51.065460",
"position": {
"x": 1.4535,
"y": -0.4709,
"z": 0.5445
"x": 1.6362,
"y": 0.2425,
"z": 0.4759
}
},
"white toilet": {
"highest_score": 0.269287109375,
"last_detected": "2026-05-18T14:13:07.308074",
"highest_score": 0.2568359375,
"last_detected": "2026-05-18T10:25:18.120319",
"position": {
"x": -4.3598,
"y": -1.1593,
"z": 0.3363
"x": -4.4027,
"y": -1.2225,
"z": 0.1715
}
},
"office chair with wheels": {
"highest_score": 0.3037109375,
"last_detected": "2026-05-18T14:12:39.416197",
"highest_score": 0.302734375,
"last_detected": "2026-05-18T10:26:35.889506",
"position": {
"x": -2.8461,
"y": 2.6732,
"z": 0.4811
"x": -2.9779,
"y": 2.3836,
"z": 0.4007
}
}
}

9
src/vlm_perception_pkg/vlm_perception_pkg/vlm_detection.py
View File

@ -55,7 +55,12 @@ class ClipDetectionNode(Node):
# CLIP + Faster R-CNN
self.clip_model, self.clip_preprocess = clip.load("ViT-B/32", device=self.device)
self.frcnn_model = torchvision.models.detection.fasterrcnn_resnet50_fpn_v2(weights="DEFAULT").eval()
self.frcnn_model = torchvision.models.detection.fasterrcnn_resnet50_fpn_v2(
weights="DEFAULT"
).to(self.device).eval()
self.get_logger().info(
f"Detection models initialized on device={self.device}"
)
# Labels and score tracking
self.labels = ["Refrigerator","water dispenser", "sofa", "white toilet", "office chair with wheels"]
@ -208,7 +213,7 @@ class ClipDetectionNode(Node):
depth_img = self.latest_depth.copy()
frame_stamp = self.latest_depth_stamp
camera_info = self.camera_info
image_tensor = transforms.ToTensor()(cv_img)
image_tensor = transforms.ToTensor()(cv_img).to(self.device)
frcnn_min_score = self.get_parameter(
'frcnn_min_score'
).get_parameter_value().double_value