rk3588和fpga的pcie通信
pcie xdma
驱动编译
SHELL = /bin/bash # # optional makefile parameters: # - DEBUG=<0|1>, enable verbose debug print-out in the driver # - config_bar_num=, xdma pci config bar number # - xvc_bar_num=, xvc pci bar # # - xvc_bar_offset=, xvc register base offset # DEBUG=0 #config_bar_num=1 # xvc_bar_num=1 ifneq ($(xvc_bar_num),) XVC_FLAGS += -D__XVC_BAR_NUM__=$(xvc_bar_num) endif ifneq ($(xvc_bar_offset),) XVC_FLAGS += -D__XVC_BAR_OFFSET__=$(xvc_bar_offset) endif $(warning XVC_FLAGS: $(XVC_FLAGS).) topdir := $(shell cd $(src)/.. && pwd) TARGET_MODULE:=xdma EXTRA_CFLAGS := -I$(topdir)/include $(XVC_FLAGS) ifeq ($(DEBUG),1) EXTRA_CFLAGS += -D__LIBXDMA_DEBUG__ endif ifneq ($(config_bar_num),) EXTRA_CFLAGS += -DXDMA_CONFIG_BAR_NUM=$(config_bar_num) endif #EXTRA_CFLAGS += -DINTERNAL_TESTING KERNELRELEASE:=/3588/kernel # ifneq ($(KERNELRELEASE),) # $(TARGET_MODULE)-objs := libxdma.o xdma_cdev.o cdev_ctrl.o cdev_events.o cdev_sgdma.o cdev_xvc.o cdev_bypass.o xdma_mod.o xdma_thread.o # obj-m := $(TARGET_MODULE).o # else # BUILDSYSTEM_DIR:=/lib/modules/$(shell uname -r)/build # PWD:=$(shell pwd) $(TARGET_MODULE)-objs := libxdma.o xdma_cdev.o cdev_ctrl.o cdev_events.o cdev_sgdma.o cdev_xvc.o cdev_bypass.o xdma_mod.o xdma_thread.o obj-m := $(TARGET_MODULE).o # linux 源码目录 BUILDSYSTEM_DIR:=/sdk/06_rk3588_241027/61_moEr_d2k_3588/kernel PWD:=$(shell pwd) CROSS_COMPLIE_3588:=/3588/prebuilts/gcc/linux-x86/aarch64/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu- all : $(MAKE) -C $(BUILDSYSTEM_DIR) M=$(PWD) modules ARCH=arm64 CROSS_COMPILE=$(CROSS_COMPLIE_3588) clean: #$(MAKE) -C $(BUILDSYSTEM_DIR) M=$(PWD) clean @/bin/rm -f *.ko modules.order *.mod.c *.o *.o.ur-safe .*.o.cmd install: all @rm -f /lib/modules/5.15.0-67-generic/extra/xdma.ko @echo "installing kernel modules to /lib/modules/$(shell uname -r)/xdma ..." @mkdir -p -m 755 /lib/modules/$(shell uname -r)/xdma @install -v -m 644 *.ko /lib/modules/$(shell uname -r)/xdma @depmod -a || true uninstall: @echo "Un-installing /lib/modules/$(shell uname -r)/xdma ..." @/bin/rm -rf /lib/modules/$(shell uname -r)/xdma @depmod -a
驱动加载
[root@rk3588-buildroot /opt/00_test/pcie]#insmod xdma.ko [ 20.349214] xdma: loading out-of-tree module taints kernel. [ 20.351795] xdma:xdma_mod_init: Xilinx XDMA Reference Driver xdma v2020.2.2 [ 20.351818] xdma:xdma_mod_init[root@rk3588-buildroot /opt/00_test/pcie]#: desc_blen_max: 0xfffffff/268435455, timeout: h2c 10 c2h 10 sec. [ 20.352450] xdma:xdma_device_open: xdma device 0000:01:00.0, 0x00000000405154b7. [ 20.352498] xdma 0000:01:00.0: enabling device (0000 -> 0002) [ 20.352599] xdma:map_single_bar: BAR0 at 0xf0200000 mapped at 0x00000000708655f0, length=1048576(/1048576) [ 20.352616] xdma:map_single_bar: BAR1 at 0xf0300000 mapped at 0x000000006b5b942f, length=65536(/65536) [ 20.352625] xdma:map_bars: config bar 1, pos 1. [ 20.352633] xdma:identify_bars: 2 BARs: config 1, user 0, bypass -1. [ 20.354467] xdma:pci_keep_intx_enabled: 0000:01:00.0: clear INTX_DISABLE, 0x406 -> 0x6. [ 20.354548] xdma:probe_one: 0000:01:00.0 xdma0, pdev 0x00000000405154b7, xdev 0x00000000a58ffaf7, 0x00000000cc9a9c39, usr 16, ch 2,2. [ 20.358004] xdma:cdev_xvc_init: xcdev 0x0000000005eef4e4, bar 0, offset 0x40000. [root@rk3588-buildroot /opt/00_test/pcie]#
设备节点
[root@rk3588-buildroot /opt/00_test/pcie]#ls /dev/xdma0_* -lh crw------- 1 root root 234, 36 Jan 1 08:00 /dev/xdma0_c2h_0 crw------- 1 root root 234, 37 Jan 1 08:00 /dev/xdma0_c2h_1 crw------- 1 root root 234, 1 Jan 1 08:00 /dev/xdma0_control crw------- 1 root root 234, 10 Jan 1 08:00 /dev/xdma0_events_0 crw------- 1 root root 234, 11 Jan 1 08:00 /dev/xdma0_events_1 crw------- 1 root root 234, 20 Jan 1 08:00 /dev/xdma0_events_10 crw------- 1 root root 234, 21 Jan 1 08:00 /dev/xdma0_events_11 crw------- 1 root root 234, 22 Jan 1 08:00 /dev/xdma0_events_12 crw------- 1 root root 234, 23 Jan 1 08:00 /dev/xdma0_events_13 crw------- 1 root root 234, 24 Jan 1 08:00 /dev/xdma0_events_14 crw------- 1 root root 234, 25 Jan 1 08:00 /dev/xdma0_events_15 crw------- 1 root root 234, 12 Jan 1 08:00 /dev/xdma0_events_2 crw------- 1 root root 234, 13 Jan 1 08:00 /dev/xdma0_events_3 crw------- 1 root root 234, 14 Jan 1 08:00 /dev/xdma0_events_4 crw------- 1 root root 234, 15 Jan 1 08:00 /dev/xdma0_events_5 crw------- 1 root root 234, 16 Jan 1 08:00 /dev/xdma0_events_6 crw------- 1 root root 234, 17 Jan 1 08:00 /dev/xdma0_events_7 crw------- 1 root root 234, 18 Jan 1 08:00 /dev/xdma0_events_8 crw------- 1 root root 234, 19 Jan 1 08:00 /dev/xdma0_events_9 crw------- 1 root root 234, 32 Jan 1 08:00 /dev/xdma0_h2c_0 crw------- 1 root root 234, 33 Jan 1 08:00 /dev/xdma0_h2c_1 crw------- 1 root root 234, 0 Jan 1 08:00 /dev/xdma0_user crw------- 1 root root 234, 2 Jan 1 08:00 /dev/xdma0_xvc [root@rk3588-buildroot /opt/00_test/pcie]#xdma0_c2h 是card ---> host 有两个通道 xdma0_h2c 是host ---> card 有两个通道 xdma0_control 驱动里对应的是dma相关的bar1 BAR1 at 0xf0300000 xdma0_user 驱动里对应的是自定义的bar0 BAR0 at 0xf0200000 xdma0_events 感觉是自定义的一些中断，类似于card拉高某个gpio，card端然后触发中断相关的操作，继续触发pcie中断通知host，驱动里就响应中断，判断相关 event中断寄存器。个人感觉需要card配合才能使用。
关键api解读
if (aperture) { struct xdma_aperture_ioctl io; io.buffer = (unsigned long)buffer; io.len = size; io.ep_addr = addr; io.aperture = aperture; io.done = 0UL; rc = ioctl(fpga_fd, IOCTL_XDMA_APERTURE_R, &io); if (rc < 0 || io.error) { log_pcie("aperture R failed %d,%d.\n", rc, io.error); goto out; } bytes_done = io.done; } else { rc = read_to_buffer(devname, fpga_fd, buffer, size, addr); if (rc < 0) goto out; bytes_done = rc; } 简单测试了下aperture的，传输速度要比下面的慢，后面就没用上面的接口传输了。暂时没理解上面传输的优缺点。read_to_buffer(devname, fpga_fd, buffer, size, addr) write_from_buffer(devname, fpga_fd, buffer, size, addr); 两个api类似，注意最后的addr是card端的地址，从该addr获取或者放数据。假设card是fpga，该addr是ddr的地址。reg_rw工具移植到代码里的修改 第一次是这么修改的，只map一次，在map上偏移地址，去读取bar0上的值，发现不能读取的不对，应该是没有映射到bar0。 m_file_fd_user = open(m_devname_user, O_RDWR | O_SYNC); off_t target = 0x00; off_t pgsz, target_aligned, offset; /* check for target page alignment */ pgsz = sysconf(_SC_PAGESIZE); offset = target & (pgsz - 1); target_aligned = target & (~(pgsz - 1)); //address: 0x0 (0x0+0x0), access read. m_map_user = mmap(NULL, offset + 4, PROT_READ | PROT_WRITE, MAP_SHARED, m_file_fd_user, target_aligned); 第二次，每次读取或者设置寄存器都先mmap，再用指针去操作，最后unmap，发现还是不行。 第三次，参考海思的工具himm，先知道bar0的物理地址，然后每次操作该bar空间的内存时就先open（/dev/mem）,然后mmap，再偏移，操作完后unmap、close。这样子操作和使用reg_rw工具操作的值是一样的。
h2c 和 c2h的数据传输应该是h和c之间是有互动的，即通过user（bar0）来传递信息，大数据就通过dma传输。
