PCIe SR-IOV 虚拟化详解

# 1. 查看设备是否支持SR-IOV
$ lspci -s 03:00.0 -vvv | grep -i sriov
	Capabilities: [160] Single Root I/O Virtualization (SR-IOV)
		IOVCap: Migration-, Interrupt Message Number: 000
		IOVCtl: Enable- Migration- Interrupt- MSE- ARIHierarchy-
		IOVSta: Migration-
		Initial VFs: 64, Total VFs: 64, Number of VFs: 0

# 2. 查看当前VF数量
$ cat /sys/bus/pci/devices/0000:03:00.0/sriov_numvfs
0

# 3. 创建4个VF
$ echo 4 | sudo tee /sys/bus/pci/devices/0000:03:00.0/sriov_numvfs

# 4. 查看创建的VF
$ lspci | grep Virtual
03:00.1 Ethernet controller: Intel Corporation Virtual Function
03:00.2 Ethernet controller: Intel Corporation Virtual Function
03:00.3 Ethernet controller: Intel Corporation Virtual Function
03:00.4 Ethernet controller: Intel Corporation Virtual Function

# 5. 查看VF详细信息
$ lspci -s 03:00.1 -vvv

# 6. 删除VF（设置为0）
$ echo 0 | sudo tee /sys/bus/pci/devices/0000:03:00.0/sriov_numvfs
                    

# 1. 解绑VF from host driver
$ echo 0000:03:00.1 | sudo tee /sys/bus/pci/drivers/ixgbevf/unbind

# 2. 绑定VF to vfio-pci（用于直通）
$ echo vfio-pci | sudo tee /sys/bus/pci/devices/0000:03:00.1/driver_override
$ echo 0000:03:00.1 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind

# 3. 查看VF的IOMMU组
$ readlink /sys/bus/pci/devices/0000:03:00.1/iommu_group
../../../../../../kernel/iommu_groups/16

# 4. 启动虚拟机时附加VF
$ qemu-system-x86_64 \
    -device vfio-pci,host=03:00.1,id=net0 \
    ...

# 或使用libvirt
$ virsh attach-device vm1 vf.xml
                    

# 查看设备的ACS能力
$ lspci -vvv -s 00:01.0 | grep -A 10 "Access Control Services"

# 强制启用ACS（某些情况下需要）
$ echo 1 | sudo tee /sys/bus/pci/devices/0000:00:01.0/acs_enable

# 检查IOMMU是否启用
$ dmesg | grep -i iommu
[    0.000000] Command line: ... intel_iommu=on iommu=pt ...

# 查看IOMMU组
$ find /sys/kernel/iommu_groups/ -type l
                    

# 测试VF网络性能（使用iperf3）
# Host端
$ iperf3 -s

# VM端（通过VF）
$ iperf3 -c  -t 30 -P 4

# 对比：使用virtio-net
$ iperf3 -c  -t 30 -P 4

# 查看VF统计
$ ethtool -S eth0

# 查看中断分布
$ cat /proc/interrupts | grep eth0
                    

/* VFIO 典型用户态序（摘自 Documentation/driver-api/vfio.rst 的骨架） */
container = open("/dev/vfio/vfio", O_RDWR);
ioctl(container, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU);

group = open("/dev/vfio/<group id>", O_RDWR);
ioctl(group, VFIO_GROUP_SET_CONTAINER, &container);

/* 把客户机内存段注册为设备可见的 IOVA */
struct vfio_iommu_type1_dma_map dma_map = {
    .vaddr = (uintptr_t)guest_ram, .iova = 0x0,
    .size  = guest_ram_size, .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE,
};
ioctl(container, VFIO_IOMMU_MAP_DMA, &dma_map);

/* 取设备 fd，逐一 mmap region（BAR 空间） */
device = ioctl(group, VFIO_GROUP_GET_DEVICE_FD, "0000:06:0d.0");
ioctl(device, VFIO_DEVICE_GET_INFO, &device_info);
for (i = 0; i < device_info.num_regions; i++) {
    struct vfio_region_info reg = { .argsz = sizeof(reg), .index = i };
    ioctl(device, VFIO_DEVICE_GET_REGION_INFO, ®);
    /* reg.offset/size/flags：mmap 到用户态直接操作 BAR */
}