Windows 11 系统配置 Intel RealSense D435i 深度相机与 Python

4. 配置 opencv 环境，安装 opencv。

## python3.6 的版本必须装这个版本，此处必须按照指定版本安装
pip install -i https://mirrors.aliyun.com/pypi/simple/ opencv-python==4.3.0.38 pip install opencv-contrib-python==4.4.0.46

pip install opencv-python pip install opencv-contrib-python 

5. 接着安装 open3d 和 pyrealsense2，open3d 需要等很长时间。

pip install pywinpty==1.1.4 pip install open3d==0.15.1 pip install pyrealsense2==2.48.0.3891

pip install open3d pip install pyrealsense2 

3.2 可视化

可视化脚本：

import pyrealsense2 as rs
import numpy as np
import cv2

if __name__ == "__main__":
    # Configure depth and color streams
    pipeline = rs.pipeline()
    config = rs.config()
    config.enable_stream(rs.stream.depth, 640, 480, rs.format.z16, 30)
    config.enable_stream(rs.stream.color, 640, 480, rs.format.bgr8, 30)
    # Start streaming
    pipeline.start(config)
    try:
        while True:
            # Wait for a coherent pair of frames: depth and color
            frames = pipeline.wait_for_frames()
            depth_frame = frames.get_depth_frame()
            color_frame = frames.get_color_frame()
            if not depth_frame or not color_frame:
                continue
            # Convert images to numpy arrays
            depth_image = np.asanyarray(depth_frame.get_data())
            color_image = np.asanyarray(color_frame.get_data())
            # Apply colormap on depth image (image must be converted to 8-bit per pixel first)
            depth_colormap = cv2.applyColorMap(cv2.convertScaleAbs(depth_image, alpha=0.03), cv2.COLORMAP_JET)
            # Stack both images horizontally
            images = np.hstack((color_image, depth_colormap))
            # Show images
            cv2.namedWindow('RealSense', cv2.WINDOW_AUTOSIZE)
            cv2.imshow('RealSense', images)
            key = cv2.waitKey(1)
            # Press esc or 'q' to close the image window
            if key & 0xFF == ord('q') or key == 27:
                cv2.destroyAllWindows()
                break
    finally:
        # Stop streaming
        pipeline.stop()

四、结合 python+Vscode 实现数据采集

4.1 实现 RGB 数据采集

脚本：

import os
import cv2
import numpy as np
import pyrealsense2 as rs
import time

# 初始化 RealSense
pipeline = rs.pipeline()
config = rs.config()
config.enable_stream(rs.stream.color, 1920, 1080, rs.format.bgr8, 15)
profile = pipeline.start(config)
cv2.namedWindow('RGB Image', cv2.WINDOW_AUTOSIZE)

# 设置保存目录
save_dir = 'E:/my_images/'
os.makedirs(save_dir, exist_ok=True)
counter = 0

try:
    while True:
        frames = pipeline.wait_for_frames()
        color_frame = frames.get_color_frame()
        if not color_frame:
            continue
        # 获取 RGB 图像数据
        color_image = np.asanyarray(color_frame.get_data())
        cv2.imshow('RGB Image', color_image)
        # 按 't' 键保存 RGB 图像
        key = cv2.waitKey(1)
        if key == ord('t'):
            rgb_file = os.path.join(save_dir, f'rgb_{counter:04d}.png')
            cv2.imwrite(rgb_file, color_image)
            print(f'Saved: {rgb_file}')
            counter += 1
        # 按 'q' 或 ESC 退出
        if key in [27, ord('q')]:
            cv2.destroyAllWindows()
            break
finally:
    pipeline.stop()

五、一些其他扩展玩法

5.1 YOLOv8 结合相机

具体脚本如下：

import cv2
import pyrealsense2 as rs
import time
import numpy as np
import math
from ultralytics import YOLO

# 加载 YOLOv8 模型
model = YOLO("runs/detect/train/weights/best.pt")

# 配置 RealSense
pipeline = rs.pipeline()
config = rs.config()
config.enable_stream(rs.stream.depth, 848, 480, rs.format.z16, 30)
config.enable_stream(rs.stream.color, 848, 480, rs.format.bgr8, 30)
# 启动相机流
pipeline.start(config)
align_to = rs.stream.color # 与 color 流对齐
align = rs.align(align_to)

def get_aligned_images():
    frames = pipeline.wait_for_frames()
    # 等待获取图像帧
    aligned_frames = align.process(frames)
    # 获取对齐帧
    aligned_depth_frame = aligned_frames.get_depth_frame()
    # 获取对齐帧中的 depth 帧
    color_frame = aligned_frames.get_color_frame()
    # 获取对齐帧中的 color 帧
    # 相机参数的获取
    intr = color_frame.profile.as_video_stream_profile().intrinsics
    # 获取相机内参
    depth_intrin = aligned_depth_frame.profile.as_video_stream_profile().intrinsics
    # 获取深度参数（像素坐标系转相机坐标系会用到）
    '''camera_parameters = {'fx': intr.fx, 'fy': intr.fy, 'ppx': intr.ppx, 'ppy': intr.ppy, 'height': intr.height, 'width': intr.width, 'depth_scale': profile.get_device().first_depth_sensor().get_depth_scale() }'''
    # 保存内参到本地
    # with open('./intrinsics.json', 'w') as fp:
    # json.dump(camera_parameters, fp)
    #######################################################
    depth_image = np.asanyarray(aligned_depth_frame.get_data())
    # 深度图（默认 16 位）
    depth_image_8bit = cv2.convertScaleAbs(depth_image, alpha=0.03)
    # 深度图（8 位）
    depth_image_3d = np.dstack((depth_image_8bit, depth_image_8bit, depth_image_8bit))
    # 3 通道深度图
    color_image = np.asanyarray(color_frame.get_data())
    # RGB 图
    # 返回相机内参、深度参数、彩色图、深度图、齐帧中的 depth 帧
    return intr, depth_intrin, color_image, depth_image, aligned_depth_frame

def get_3d_camera_coordinate(depth_pixel, aligned_depth_frame, depth_intrin):
    x = depth_pixel[0]
    y = depth_pixel[1]
    dis = aligned_depth_frame.get_distance(x, y)
    # 获取该像素点对应的深度
    # print ('depth: ',dis) # 深度单位是 m
    camera_coordinate = rs.rs2_deproject_pixel_to_point(depth_intrin, depth_pixel, dis)
    # print ('camera_coordinate: ',camera_coordinate)
    return dis, camera_coordinate

# 初始化 FPS 计算
fps = 0
frame_count = 0
start_time = time.time()

try:
    while True:
        # 等待获取一对连续的帧：深度和颜色
        intr, depth_intrin, color_image, depth_image, aligned_depth_frame = get_aligned_images()
        if not depth_image.any() or not color_image.any():
            continue
        # 获取当前时间
        time1 = time.time()
        # 将图像转为 numpy 数组
        depth_colormap = cv2.applyColorMap(cv2.convertScaleAbs(depth_image, alpha=0.03), cv2.COLORMAP_JET)
        images = np.hstack((color_image, depth_colormap))
        # 使用 YOLOv8 进行目标检测
        results = model.predict(color_image, conf=0.5)
        annotated_frame = results[0].plot()
        detected_boxes = results[0].boxes.xyxy
        # 获取边界框坐标
        # print('方框坐标', detected_boxes)
        for i, box in enumerate(detected_boxes):
            x1, y1, x2, y2 = map(int, box)
            # 获取边界框坐标
            # 计算步长
            xrange = max(1, math.ceil(abs((x1 - x2)/30)))
            yrange = max(1, math.ceil(abs((y1 - y2)/30)))
            # xrange = 1
            # yrange = 1
            point_cloud_data = []
            # 获取范围内点的三维坐标
            for x_position in range(x1, x2, xrange):
                for y_position in range(y1, y2, yrange):
                    depth_pixel = [x_position, y_position]
                    dis, camera_coordinate = get_3d_camera_coordinate(depth_pixel, aligned_depth_frame, depth_intrin)
                    # 获取对应像素点的三维坐标
                    point_cloud_data.append(f"{camera_coordinate} ")
            # 一次性写入所有数据
            with open("point_cloud_data.txt", "a") as file:
                file.write(f"\nTime: {time.time()}\n")
                file.write(" ".join(point_cloud_data))
            # 显示中心点坐标
            ux = int((x1 + x2)/2)
            uy = int((y1 + y2)/2)
            dis, camera_coordinate = get_3d_camera_coordinate([ux, uy], aligned_depth_frame, depth_intrin)
            # 获取对应像素点的三维坐标
            formatted_camera_coordinate = f"({camera_coordinate[0]:.2f}, {camera_coordinate[1]:.2f}, {camera_coordinate[2]:.2f})"
            cv2.circle(annotated_frame, (ux, uy), 4, (255, 255, 255), 5)
            # 标出中心点
            cv2.putText(annotated_frame, formatted_camera_coordinate, (ux + 20, uy + 10), 0, 1, [225, 255, 255], thickness=1, lineType=cv2.LINE_AA)
            # 标出坐标
            # 计算 FPS
            frame_count += 1
            time2 = time.time()
            fps = int(1/(time2 - time1))
            # 显示 FPS
            cv2.putText(annotated_frame, f'FPS: {fps:.2f}', (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA)
            # 显示结果
            cv2.imshow('YOLOv8 RealSense', annotated_frame)
            if cv2.waitKey(1) & 0xFF == ord('q'):
                break
finally:
    # 停止流
    pipeline.stop()

5.2 关键点识别结合相机

脚本思路：

1. 获取相机对齐图像帧与相机参数

frames = pipeline.wait_for_frames()
# 等待获取图像帧，获取颜色和深度的框架集
aligned_frames = align.process(frames)
# 获取对齐帧，将深度框与颜色框对齐
aligned_depth_frame = aligned_frames.get_depth_frame()
# 获取对齐帧中的的 depth 帧
aligned_color_frame = aligned_frames.get_color_frame()
# 获取对齐帧中的的 color 帧
#### 获取相机参数 ####
depth_intrin = aligned_depth_frame.profile.as_video_stream_profile().intrinsics
# 获取深度参数（像素坐标系转相机坐标系会用到）
color_intrin = aligned_color_frame.profile.as_video_stream_profile().intrinsics
# 获取相机内参
#### 将 images 转为 numpy arrays ####
img_color = np.asanyarray(aligned_color_frame.get_data())
# RGB 图
img_depth = np.asanyarray(aligned_depth_frame.get_data())
# 深度图（默认 16 位）

2. 检测

# 检测
result = list(model(image, conf=0.3, stream=True))[0]
# inference
boxes = result.boxes # Boxes object for bbox outputs
boxes = boxes.cpu().numpy()
# convert to numpy array

3. 遍历所有的 Keypoints

# 遍历 keypoints
keypoints = result.keypoints # Keypoints object for pose outputs
keypoints = keypoints.cpu().numpy()
# convert to numpy array

4. 获得所有关键点二维坐标

point1 = (int(x0), int(y0))
point2 = (int(x1), int(y1))

5. 计算所有点的深度值

point1_dis = depth_frame.get_distance(x0, y0)
# 获取第一个点的深度值
point2_dis = depth_frame.get_distance(x1, y1)
# 获取第二个点的深度值

6. 计算所有点的三维坐标

point1_dis = depth_frame.get_distance(x0, y0)
# 获取第一个点的深度值
point2_dis = depth_frame.get_distance(x1, y1)
# 获取第二个点的深度值