From 666f38f41899a2c481905bc7992610355c53662d Mon Sep 17 00:00:00 2001 From: Shingo Omura Date: Sat, 14 Jun 2025 03:06:19 +0900 Subject: [PATCH] DRA: Introduce a helper function producing standardized device attributes for DRA drivers The shared function can provide a way for DRA drivers to produce standardized device attributes in a consistent way. Signed-off-by: Shingo Omura --- .../api/standard_attributes.go | 13 ++ .../deviceattribute/attribute.go | 42 ++++ .../deviceattribute/attribute_test.go | 87 +++++++ .../deviceattribute/pci.go | 62 +++++ .../deviceattribute/pci_test.go | 142 ++++++++++++ .../testing/file.go | 27 +++ .../dynamic-resource-allocation/utils/pci.go | 213 ++++++++++++++++++ .../utils/pci_test.go | 195 ++++++++++++++++ .../utils/sysfs.go | 97 ++++++++ 9 files changed, 878 insertions(+) create mode 100644 staging/src/k8s.io/dynamic-resource-allocation/api/standard_attributes.go create mode 100644 staging/src/k8s.io/dynamic-resource-allocation/deviceattribute/attribute.go create mode 100644 staging/src/k8s.io/dynamic-resource-allocation/deviceattribute/attribute_test.go create mode 100644 staging/src/k8s.io/dynamic-resource-allocation/deviceattribute/pci.go create mode 100644 staging/src/k8s.io/dynamic-resource-allocation/deviceattribute/pci_test.go create mode 100644 staging/src/k8s.io/dynamic-resource-allocation/testing/file.go create mode 100644 staging/src/k8s.io/dynamic-resource-allocation/utils/pci.go create mode 100644 staging/src/k8s.io/dynamic-resource-allocation/utils/pci_test.go create mode 100644 staging/src/k8s.io/dynamic-resource-allocation/utils/sysfs.go diff --git a/staging/src/k8s.io/dynamic-resource-allocation/api/standard_attributes.go b/staging/src/k8s.io/dynamic-resource-allocation/api/standard_attributes.go new file mode 100644 index 00000000000..0d0a0f51616 --- /dev/null +++ b/staging/src/k8s.io/dynamic-resource-allocation/api/standard_attributes.go @@ -0,0 +1,13 @@ +package api + +const ( + // StandardDeviceAttributePrefix is the prefix used for standard device attributes. + StandardDeviceAttributePrefix = "resource.kubernetes.io/" + + // StandardDeviceAttributePCIeRoot is a standard device attribute name + // which describe PCIe Root Complex of the PCIe device. + // The value is a string value in the format `pci:`, + // referring to a PCIe (Peripheral Component Interconnect Express) Root Complex. + // This attribute can be used to identify devices that share the same PCIe Root Complex. + StandardDeviceAttributePCIeRoot = StandardDeviceAttributePrefix + "pcieRoot" +) diff --git a/staging/src/k8s.io/dynamic-resource-allocation/deviceattribute/attribute.go b/staging/src/k8s.io/dynamic-resource-allocation/deviceattribute/attribute.go new file mode 100644 index 00000000000..f6f23ab099c --- /dev/null +++ b/staging/src/k8s.io/dynamic-resource-allocation/deviceattribute/attribute.go @@ -0,0 +1,42 @@ +package deviceattribute + +import resourceapi "k8s.io/dynamic-resource-allocation/api" + +type StandardDeviceAttributesOption func(args *StandardDeviceAttributesOptions) + +// WithStandardPCIDeviceAttributesOpts sets the standard PCI device attributes options +func WithStandardPCIDeviceAttributesOpts(opts ...StandardPCIDeviceAttributesOption) StandardDeviceAttributesOption { + return func(args *StandardDeviceAttributesOptions) { + args.pciDeviceAttributeOpts = append(args.pciDeviceAttributeOpts, opts...) + } +} + +type StandardDeviceAttributesOptions struct { + pciDeviceAttributeOpts []StandardPCIDeviceAttributesOption +} + +// StandardDeviceAttributes generates standard device attributes based on the provided options. +// It currently supports standard PCI device attributes if options are provided. +func StandardDeviceAttributes( + opts ...StandardDeviceAttributesOption, +) (map[resourceapi.QualifiedName]resourceapi.DeviceAttribute, error) { + options := &StandardDeviceAttributesOptions{} + for _, opt := range opts { + opt(options) + } + + attrs := map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{} + + if len(options.pciDeviceAttributeOpts) > 0 { + // Standard PCI device attributes + pciAttrs, err := StandardPCIDeviceAttributes(options.pciDeviceAttributeOpts...) + if err != nil { + return nil, err + } + for k, v := range pciAttrs { + attrs[k] = v + } + } + + return attrs, nil +} diff --git a/staging/src/k8s.io/dynamic-resource-allocation/deviceattribute/attribute_test.go b/staging/src/k8s.io/dynamic-resource-allocation/deviceattribute/attribute_test.go new file mode 100644 index 00000000000..f3be4b84bf8 --- /dev/null +++ b/staging/src/k8s.io/dynamic-resource-allocation/deviceattribute/attribute_test.go @@ -0,0 +1,87 @@ +package deviceattribute + +import ( + "os" + "reflect" + "testing" + + resourceapi "k8s.io/dynamic-resource-allocation/api" + dratesting "k8s.io/dynamic-resource-allocation/testing" + "k8s.io/dynamic-resource-allocation/utils" + "k8s.io/utils/ptr" +) + +func TestAddStandardDeviceAttributes(t *testing.T) { + // Create mock sysfs for testing + testSysfsDir, err := os.MkdirTemp("", "test-sysfs") + if err != nil { + t.Fatalf("failed to create temp sysfs directory: %v", err) + } + t.Cleanup(func() { + if err := os.RemoveAll(testSysfsDir); err != nil { + t.Fatalf("failed to clean up temp sysfs directory: %v", err) + } + }) + + testSysFs := utils.NewSysfsWithRoot(testSysfsDir) + // "0123:45:67.8" + testPCIAddress := utils.PCIAddress{ + Domain: 0x0123, + Bus: 0x45, + Device: 0x67, + Function: 0x8, + } + // "0123:45" + testPCIRoot := utils.PCIRoot{ + Domain: 0x1234, + Bus: 0x56, + } + // /sys/bus/pci/devices/0123:45:67.8 --> /sys/devices/pci1234:56/0123:45:67.8 + testPCIDevicePath := testSysFs.Devices("pci" + testPCIRoot.String() + "/" + testPCIAddress.String()) + dratesting.TouchFile(t, testPCIDevicePath) + testPCIBusPath := testSysFs.Bus("pci/devices/" + testPCIAddress.String()) + dratesting.CreateSymlink(t, testPCIDevicePath, testPCIBusPath) + + tests := map[string]struct { + opts []StandardDeviceAttributesOption + expectedAttrs map[resourceapi.QualifiedName]resourceapi.DeviceAttribute + }{ + "pci attrs": { + opts: []StandardDeviceAttributesOption{ + WithStandardPCIDeviceAttributesOpts( + WithPCIDeviceAddress(&testPCIAddress), + withSysfs(testSysFs), // Use the mock sysfs + ), + }, + expectedAttrs: map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{ + qualifiedNamePCIeRoot: { + StringValue: ptr.To("pci" + testPCIRoot.String()), + }, + }, + }, + "nil attrs": { + opts: nil, + expectedAttrs: map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{}, + }, + "empty attrs": { + opts: []StandardDeviceAttributesOption{}, + expectedAttrs: map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{}, + }, + "empty pci attrs": { + opts: []StandardDeviceAttributesOption{WithStandardPCIDeviceAttributesOpts()}, + expectedAttrs: map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{}, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + attrs, err := StandardDeviceAttributes(test.opts...) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !reflect.DeepEqual(attrs, test.expectedAttrs) { + t.Errorf("expected %v, got %v", test.expectedAttrs, attrs) + } + }) + } +} diff --git a/staging/src/k8s.io/dynamic-resource-allocation/deviceattribute/pci.go b/staging/src/k8s.io/dynamic-resource-allocation/deviceattribute/pci.go new file mode 100644 index 00000000000..7a739c7065b --- /dev/null +++ b/staging/src/k8s.io/dynamic-resource-allocation/deviceattribute/pci.go @@ -0,0 +1,62 @@ +package deviceattribute + +import ( + "k8s.io/utils/ptr" + + resourceapi "k8s.io/dynamic-resource-allocation/api" + utils "k8s.io/dynamic-resource-allocation/utils" +) + +var ( + qualifiedNamePCIeRoot = resourceapi.QualifiedName(resourceapi.StandardDeviceAttributePCIeRoot) +) + +type StandardPCIDeviceAttributesOption func(args *StandardPCIDeviceAttributeArgs) + +// WithPCIDeviceAddress sets the PCI address for the PCI device attributes. +func WithPCIDeviceAddress(pciAddress *utils.PCIAddress) StandardPCIDeviceAttributesOption { + return func(args *StandardPCIDeviceAttributeArgs) { + args.pciAddress = pciAddress + } +} + +// StandardPCIDeviceAttributeArgs holds the arguments for generating standard PCI device attributes. +type StandardPCIDeviceAttributeArgs struct { + pciAddress *utils.PCIAddress + sysfs utils.Sysfs +} + +// StandardPCIDeviceAttributes returns standard device attributes for a PCI device. +func StandardPCIDeviceAttributes(opts ...StandardPCIDeviceAttributesOption) (map[resourceapi.QualifiedName]resourceapi.DeviceAttribute, error) { + args := &StandardPCIDeviceAttributeArgs{ + sysfs: utils.NewSysfs(), + } + + for _, opt := range opts { + opt(args) + } + + if args.pciAddress == nil { + return nil, nil + } + + attrs := make(map[resourceapi.QualifiedName]resourceapi.DeviceAttribute) + + pciRoot, err := args.pciAddress.ResolvePCIRoot(args.sysfs) + if err != nil { + return nil, err + } + + attrs[qualifiedNamePCIeRoot] = resourceapi.DeviceAttribute{ + StringValue: ptr.To("pci" + pciRoot.String()), + } + + return attrs, nil +} + +// This is only for testing purposes. +func withSysfs(sysfs utils.Sysfs) StandardPCIDeviceAttributesOption { + return func(args *StandardPCIDeviceAttributeArgs) { + args.sysfs = sysfs + } +} diff --git a/staging/src/k8s.io/dynamic-resource-allocation/deviceattribute/pci_test.go b/staging/src/k8s.io/dynamic-resource-allocation/deviceattribute/pci_test.go new file mode 100644 index 00000000000..a2989343cf4 --- /dev/null +++ b/staging/src/k8s.io/dynamic-resource-allocation/deviceattribute/pci_test.go @@ -0,0 +1,142 @@ +package deviceattribute + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "k8s.io/utils/ptr" + + resourceapi "k8s.io/dynamic-resource-allocation/api" + dratesting "k8s.io/dynamic-resource-allocation/testing" + utils "k8s.io/dynamic-resource-allocation/utils" +) + +func TestStandardPCIDeviceAttributes(t *testing.T) { + pcieRootAttrName := resourceapi.QualifiedName(resourceapi.StandardDeviceAttributePCIeRoot) + + address := &utils.PCIAddress{Domain: 0, Bus: 1, Device: 2, Function: 3} + root := &utils.PCIRoot{Domain: 0, Bus: 1} + standardAttributes := map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{ + pcieRootAttrName: { + StringValue: ptr.To("pci" + root.String()), + }, + } + + tests := map[string]struct { + pciAddress *utils.PCIAddress + sysfs func(*testing.T, string) utils.Sysfs + expectedAttributes map[resourceapi.QualifiedName]resourceapi.DeviceAttribute + expectsErr bool + expectedErrMsg string + }{ + "nil address": { + pciAddress: nil, + sysfs: func(t *testing.T, testSysfsRoot string) utils.Sysfs { + return utils.NewSysfs() + }, + expectedAttributes: nil, + }, + "valid address": { + pciAddress: address, + sysfs: func(t *testing.T, testSysfsRoot string) utils.Sysfs { + sysfs := utils.NewSysfsWithRoot(testSysfsRoot) + devicePath := sysfs.Devices(filepath.Join("pci"+root.String(), address.String())) + dratesting.TouchFile(t, devicePath) + + busPath := sysfs.Bus(filepath.Join("pci", "devices", address.String())) + dratesting.CreateSymlink(t, devicePath, busPath) + return sysfs + }, + expectedAttributes: standardAttributes, + }, + "non-existent address": { + pciAddress: address, + sysfs: func(t *testing.T, testSysfsRoot string) utils.Sysfs { + return utils.NewSysfsWithRoot(testSysfsRoot) + }, + expectsErr: true, + expectedErrMsg: "no such file or directory", + }, + "devices path not a symlink": { + pciAddress: address, + sysfs: func(t *testing.T, testSysfsRoot string) utils.Sysfs { + sysfs := utils.NewSysfsWithRoot(testSysfsRoot) + devicePath := sysfs.Devices(filepath.Join("pci"+root.String(), address.String())) + dratesting.TouchFile(t, devicePath) + + // Create a file instead of a symlink + busPath := sysfs.Bus(filepath.Join("pci", "devices", address.String())) + dratesting.TouchFile(t, busPath) + return sysfs + }, + expectsErr: true, + expectedErrMsg: "invalid argument", + }, + "invalid symlink target": { + pciAddress: address, + sysfs: func(t *testing.T, testSysfsRoot string) utils.Sysfs { + sysfs := utils.NewSysfsWithRoot(testSysfsRoot) + invalidDevicePath := sysfs.Devices(filepath.Join("invalid", "pci"+root.String(), address.String())) + dratesting.TouchFile(t, invalidDevicePath) + + // Create a symlink that points to an invalid path + busPath := sysfs.Bus(filepath.Join("pci", "devices", address.String())) + dratesting.CreateSymlink(t, invalidDevicePath, busPath) + return sysfs + }, + expectsErr: true, + expectedErrMsg: "invalid symlink target for PCI device", + }, + "invalid root format": { + pciAddress: address, + sysfs: func(t *testing.T, testSysfsRoot string) utils.Sysfs { + sysfs := utils.NewSysfsWithRoot(testSysfsRoot) + devicePath := sysfs.Devices(filepath.Join("pci-invalid-root", address.String())) + dratesting.TouchFile(t, devicePath) + + // Create a symlink with an invalid root format + busPath := sysfs.Bus(filepath.Join("pci", "devices", address.String())) + dratesting.CreateSymlink(t, devicePath, busPath) + return sysfs + }, + expectsErr: true, + expectedErrMsg: "invalid PCIRoot format", + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + testSysfsRoot, err := os.MkdirTemp("", "sysfs-test") + if err != nil { + t.Fatalf("Failed to create temp sysfs directory: %v", err) + } + t.Cleanup(func() { + if err := os.RemoveAll(testSysfsRoot); err != nil { + t.Errorf("Failed to remove temp sysfs directory: %v", err) + } + }) + result, err := StandardPCIDeviceAttributes( + WithPCIDeviceAddress(test.pciAddress), + withSysfs(test.sysfs(t, testSysfsRoot)), + ) + if test.expectsErr { + if err == nil { + t.Fatalf("Expected error but got none") + } + if !strings.Contains(err.Error(), test.expectedErrMsg) { + t.Fatalf("Expected error message %q contains %q", err.Error(), test.expectedErrMsg) + } + return + } + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if !reflect.DeepEqual(result, test.expectedAttributes) { + t.Fatalf("Expected attributes %v, got %v", test.expectedAttributes, result) + } + }) + } +} diff --git a/staging/src/k8s.io/dynamic-resource-allocation/testing/file.go b/staging/src/k8s.io/dynamic-resource-allocation/testing/file.go new file mode 100644 index 00000000000..27eb56aaf9b --- /dev/null +++ b/staging/src/k8s.io/dynamic-resource-allocation/testing/file.go @@ -0,0 +1,27 @@ +package testing + +import ( + "os" + "path/filepath" + "testing" +) + +func TouchFile(t *testing.T, path string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatalf("Failed to create directory %s: %v", filepath.Dir(path), err) + } + if _, err := os.Create(path); err != nil { + t.Fatalf("Failed to create file %s: %v", path, err) + } +} + +func CreateSymlink(t *testing.T, target, link string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(link), 0755); err != nil { + t.Fatalf("Failed to create directory for symlink %s: %v", filepath.Dir(link), err) + } + if err := os.Symlink(target, link); err != nil { + t.Fatalf("Failed to create symlink from %s to %s: %v", target, link, err) + } +} diff --git a/staging/src/k8s.io/dynamic-resource-allocation/utils/pci.go b/staging/src/k8s.io/dynamic-resource-allocation/utils/pci.go new file mode 100644 index 00000000000..69f765a6a33 --- /dev/null +++ b/staging/src/k8s.io/dynamic-resource-allocation/utils/pci.go @@ -0,0 +1,213 @@ +package utils + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" +) + +var ( + // bdfRegexp matches PCI address in BDF notation. + // The format is ::. + // where: + // - domain: 4-digit hexadecimal number representing the PCI domain + // - bus: 2-digit hexadecimal number representing the PCI bus + // - device: 2-digit hexadecimal number representing the PCI device + // - function: 1-digit hexadecimal number representing the PCI function + // Example: "0000:0e:1f.0" represents domain 0, bus 14, device 31, function 0. + // ref: https://wiki.xenproject.org/wiki/Bus:Device.Function_(BDF)_Notation + bdfRegexp = regexp.MustCompile(`^([0-9a-f]{4}):([0-9a-f]{2}):([0-9a-f]{2})\.([0-9a-f]{1})$`) +) + +// PCIRoot represents a PCI root address in the combination of Domain and Bus. +// This represents the top-level element in sysfs PCI device hierarchy. +// (i.e. /sys/devices/pci:) +// +// ref: https://docs.kernel.org/PCI/sysfs-pci.html +type PCIRoot struct { + Domain uint32 + Bus uint32 +} + +func (p *PCIRoot) String() string { + return fmt.Sprintf("%04x:%02x", p.Domain, p.Bus) +} + +// The PCIAddress holds PCI address components in BDF notation. +// ::. +// where: +// - domain: 4-digit hexadecimal number representing the PCI domain +// - bus: 2-digit hexadecimal number representing the PCI bus +// - device: 2-digit hexadecimal number representing the PCI device +// - function: 1-digit hexadecimal number representing the PCI function +// +// Example: "0000:0e:1f.0" represents domain 0, bus 14, device 31, function 0. +// +// ref: https://wiki.xenproject.org/wiki/Bus:Device.Function_(BDF)_Notation +type PCIAddress struct { + Domain uint32 + Bus uint32 + Device uint32 + Function uint32 +} + +func (p *PCIAddress) String() string { + return fmt.Sprintf("%04x:%02x:%02x.%01x", p.Domain, p.Bus, p.Device, p.Function) +} + +// ResolvePCIRoot resolves the PCI root for the PCIAddress. +// It returns a PCIRoot object that contains the domain and bus of the PCI root. +// The PCIRoot is derived from the sysfs path of the PCIAddress. +// +// sysfs argument is optional and only for testing purposes. +func (p *PCIAddress) ResolvePCIRoot(sysfs Sysfs) (*PCIRoot, error) { + if sysfs == nil { + sysfs = NewSysfs() + } + + // e.g. /sys/devices/pci0000:01/....../0000:00:1f.0, + sysDevicesPath, err := p.resolveSysDevicesPath(sysfs) + if err != nil { + return nil, fmt.Errorf("failed to resolve sysfs path for PCI device %s: %w", p.String(), err) + } + + pciRootPart := strings.Split(strings.TrimPrefix(sysDevicesPath, sysfs.Devices("")+"/"), "/")[0] + pciRoot, err := parsePCIRoot( + pciRootPart[3:], // skip "pci" prefix + ) + if err != nil { + return nil, fmt.Errorf("failed to resolve PCI root for %s: %w", p.String(), err) + } + return pciRoot, nil +} + +// resolveSysDevicesPath resolves the /sys/devices path for the PCIAddress. +// +// /sys/devices has directory structure which reflects the hardware hierarchy in the system. +// Therefore, the device path may contains intermediate directories(devices). +// Thus, we can not simply find the device path from the PCIAddress. +// Fortunately, /sys/bus/pci/devices/
is a symlink to the actual device path in /sys/devices. +// So we can resolve the actual device path by reading the symlink at /sys/bus/pci/devices/
. +// +// For example, if the PCIAddress is "0000:00:1f.0", +// /sys/bus/pci/devices/0000:00:1f.0 points to +// /sys/devices/pci0000:01/....../0000:00:1f.0, +func (p *PCIAddress) resolveSysDevicesPath(sysfs Sysfs) (string, error) { + // e.g. /sys/bus/pci/devices/0000:00:1f.0 + sysBusPath := sysfs.Bus(filepath.Join("pci", "devices", p.String())) + + targetRelative, err := os.Readlink(sysBusPath) + if err != nil { + return "", fmt.Errorf("failed to read symlink for PCI device %s: %w", sysBusPath, err) + } + var targetAbs string + if filepath.IsAbs(targetRelative) { + targetAbs = targetRelative + } else { + // If the target is a relative path, we need to resolve it relative to the symlink's directory. + targetAbs = filepath.Join(filepath.Dir(sysBusPath), targetRelative) + } + + // targetAbs must be /sys/devices/pci0000:01/....../0000:00:1f.0 + devicePathPrefix := sysfs.Devices("pci") + if !strings.HasPrefix(targetAbs, devicePathPrefix) || !(filepath.Base(targetAbs) == p.String()) { + return "", fmt.Errorf("invalid symlink target for PCI device %s: %s", sysBusPath, targetAbs) + } + + return targetAbs, nil +} + +// parsePCIRoot parses a PCIRoot from a string in the format "domain:bus". +// The format is: +// - domain: 4-digit hexadecimal number representing the PCI domain +// - bus: 2-digit hexadecimal number representing the PCI bus +// Example: "0000:01" represents domain 0, bus 1. +func parsePCIRoot(str string) (*PCIRoot, error) { + parts := strings.Split(str, ":") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid PCIRoot format: %s", str) + } + + parsePart := func(name, part string) (uint32, error) { + value, err := parseHexToUint32(part) + if err != nil { + return 0, fmt.Errorf("invalid value %s in PCIRoot %s: %w", name, str, err) + } + return uint32(value), nil + } + + var domain, bus uint32 + var err error + if domain, err = parsePart("domain", parts[0]); err != nil { + return nil, err + } + if bus, err = parsePart("bus", parts[1]); err != nil { + return nil, err + } + + return &PCIRoot{ + Domain: domain, + Bus: bus, + }, nil +} + +// ParsePCIAddress parses a PCI address in BDF notation. +// The format is ::. +// where: +// - domain: 4-digit hexadecimal number representing the PCI domain +// - bus: 2-digit hexadecimal number representing the PCI bus +// - device: 2-digit hexadecimal number representing the PCI device +// - function: 1-digit hexadecimal number representing the PCI function +// The function returns a PCIAddress object or an error if the format is invalid. +// +// Example: "0000:0e:1f.0" represents domain 0, bus 14, device 31, function 0. +// +// ref: https://wiki.xenproject.org/wiki/Bus:Device.Function_(BDF)_Notation +func ParsePCIAddress(bdfString string) (*PCIAddress, error) { + match := bdfRegexp.FindStringSubmatch(bdfString) + + if len(match) == 0 { + return nil, fmt.Errorf("invalid PCI address format: %s", bdfString) + } + + parsePart := func(name, part string) (uint32, error) { + value, err := parseHexToUint32(part) + if err != nil { + return 0, fmt.Errorf("invalid value %s in PCI address %s: %w", name, bdfString, err) + } + return uint32(value), nil + } + + var domain, bus, device, function uint32 + var err error + if domain, err = parsePart("domain", match[1]); err != nil { + return nil, err + } + if bus, err = parsePart("bus", match[2]); err != nil { + return nil, err + } + if device, err = parsePart("device", match[3]); err != nil { + return nil, err + } + if function, err = parsePart("function", match[4]); err != nil { + return nil, err + } + + return &PCIAddress{ + Domain: uint32(domain), + Bus: uint32(bus), + Device: uint32(device), + Function: uint32(function), + }, nil +} + +func parseHexToUint32(s string) (uint32, error) { + value, err := strconv.ParseUint(s, 16, 32) + if err != nil { + return 0, err + } + return uint32(value), nil +} diff --git a/staging/src/k8s.io/dynamic-resource-allocation/utils/pci_test.go b/staging/src/k8s.io/dynamic-resource-allocation/utils/pci_test.go new file mode 100644 index 00000000000..e0a79bd0a61 --- /dev/null +++ b/staging/src/k8s.io/dynamic-resource-allocation/utils/pci_test.go @@ -0,0 +1,195 @@ +package utils + +import ( + "os" + "path/filepath" + "strings" + "testing" + + dratesting "k8s.io/dynamic-resource-allocation/testing" +) + +func TestPCIAddressString(t *testing.T) { + tests := map[string]struct { + address PCIAddress + expected string + }{ + "zero": {PCIAddress{Domain: 0, Bus: 0, Device: 0, Function: 0}, "0000:00:00.0"}, + "simple": {PCIAddress{Domain: 1, Bus: 2, Device: 3, Function: 4}, "0001:02:03.4"}, + "complex": {PCIAddress{Domain: 0x1234, Bus: 0x56, Device: 0x78, Function: 0x9}, "1234:56:78.9"}, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + if got := test.address.String(); got != test.expected { + t.Errorf("Expected %s, got %s", test.expected, got) + } + }) + } +} + +func TestParsePCIAddress(t *testing.T) { + tests := map[string]struct { + input string + expected *PCIAddress + expectsErr bool + }{ + "zero": {"0000:00:00.0", &PCIAddress{Domain: 0, Bus: 0, Device: 0, Function: 0}, false}, + "simple": {"0001:02:03.4", &PCIAddress{Domain: 1, Bus: 2, Device: 3, Function: 4}, false}, + "complex": {"1234:56:78.9", &PCIAddress{Domain: 0x1234, Bus: 0x56, Device: 0x78, Function: 0x9}, false}, + "invalid format": {"0000:00:00", nil, true}, + "too many parts": {"0000:00:00.0.1", nil, true}, + "invalid hex value": {"0000:00:00.0a", nil, true}, + "completely invalid": {"invalid", nil, true}, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + result, err := ParsePCIAddress(test.input) + if (err != nil) != test.expectsErr { + t.Errorf("ParsePCIAddress(%s) error = %v, wantErr %v", test.input, err, test.expectsErr) + return + } + if !test.expectsErr && result.String() != test.expected.String() { + t.Errorf("ParsePCIAddress(%s) = %s, want %s", test.input, result.String(), test.expected.String()) + return + } + }) + } +} + +func TestPCIRootString(t *testing.T) { + tests := map[string]struct { + root PCIRoot + expected string + }{ + "zero": {PCIRoot{Domain: 0, Bus: 0}, "0000:00"}, + "simple": {PCIRoot{Domain: 1, Bus: 2}, "0001:02"}, + "complex": {PCIRoot{Domain: 0x1234, Bus: 0x56}, "1234:56"}, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + if got := test.root.String(); got != test.expected { + t.Errorf("Expected %s, got %s", test.expected, got) + } + }) + } +} + +func TestPCIAddressResolvePCIRoot(t *testing.T) { + address := PCIAddress{ + Domain: 0x1234, + Bus: 0x56, + Device: 0x78, + Function: 0x9, + } + root := PCIRoot{ + Domain: 0x2345, + Bus: 0x67, + } + + tests := map[string]struct { + sysfs func(*testing.T, string) Sysfs + expectsErr bool + expectedErrMsg string + }{ + "valid address": { + sysfs: func(t *testing.T, testSysfsRoot string) Sysfs { + sysfs := NewSysfsWithRoot(testSysfsRoot) + devicePath := sysfs.Devices(filepath.Join("pci"+root.String(), address.String())) + dratesting.TouchFile(t, devicePath) + + busPath := sysfs.Bus(filepath.Join("pci", "devices", address.String())) + dratesting.CreateSymlink(t, devicePath, busPath) + return sysfs + }, + }, + "non-existent address": { + sysfs: func(t *testing.T, testSysfsRoot string) Sysfs { + return NewSysfsWithRoot(testSysfsRoot) + }, + expectsErr: true, + expectedErrMsg: "no such file or directory", + }, + "devices path not a symlink": { + sysfs: func(t *testing.T, testSysfsRoot string) Sysfs { + sysfs := NewSysfsWithRoot(testSysfsRoot) + devicePath := sysfs.Devices(filepath.Join("pci"+root.String(), address.String())) + dratesting.TouchFile(t, devicePath) + + // Create a file instead of a symlink + busPath := sysfs.Bus(filepath.Join("pci", "devices", address.String())) + dratesting.TouchFile(t, busPath) + return sysfs + }, + expectsErr: true, + expectedErrMsg: "invalid argument", + }, + "invalid symlink target": { + sysfs: func(t *testing.T, testSysfsRoot string) Sysfs { + sysfs := NewSysfsWithRoot(testSysfsRoot) + invalidDevicePath := sysfs.Devices(filepath.Join("invalid", "pci"+root.String(), address.String())) + dratesting.TouchFile(t, invalidDevicePath) + + // Create a symlink that points to an invalid path + busPath := sysfs.Bus(filepath.Join("pci", "devices", address.String())) + dratesting.CreateSymlink(t, invalidDevicePath, busPath) + return sysfs + }, + expectsErr: true, + expectedErrMsg: "invalid symlink target for PCI device", + }, + "invalid root format": { + sysfs: func(t *testing.T, testSysfsRoot string) Sysfs { + sysfs := NewSysfsWithRoot(testSysfsRoot) + devicePath := sysfs.Devices(filepath.Join("pci-invalid-root", address.String())) + dratesting.TouchFile(t, devicePath) + + // Create a symlink with an invalid root format + busPath := sysfs.Bus(filepath.Join("pci", "devices", address.String())) + dratesting.CreateSymlink(t, devicePath, busPath) + return sysfs + }, + expectsErr: true, + expectedErrMsg: "invalid PCIRoot format", + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + testSysfsRoot, err := os.MkdirTemp("", "sysfs_test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + t.Cleanup(func() { + if err := os.RemoveAll(testSysfsRoot); err != nil { + t.Errorf("Failed to remove temp sysfs directory: %v", err) + } + }) + + ret, err := address.ResolvePCIRoot(test.sysfs(t, testSysfsRoot)) + + if test.expectsErr { + if err == nil { + t.Errorf("Expected error but got none") + return + } + if !strings.Contains(err.Error(), test.expectedErrMsg) { + t.Errorf("Expected error message %q contains %q", err.Error(), test.expectedErrMsg) + return + } + return + } + + if err != nil { + t.Errorf("Unexpected error: %v", err) + return + } + if ret == nil { + t.Errorf("Expected non-nil PCIRoot, got nil") + return + } + if ret.String() != root.String() { + t.Errorf("Expected PCIRoot %s, got %s", root.String(), ret.String()) + return + } + }) + } +} diff --git a/staging/src/k8s.io/dynamic-resource-allocation/utils/sysfs.go b/staging/src/k8s.io/dynamic-resource-allocation/utils/sysfs.go new file mode 100644 index 00000000000..f1822c5982c --- /dev/null +++ b/staging/src/k8s.io/dynamic-resource-allocation/utils/sysfs.go @@ -0,0 +1,97 @@ +package utils + +import "path/filepath" + +const ( + sysfsPath = "/sys" +) + +// Sysfs provides methods to access various sysfs paths. +// It abstracts the sysfs directory structure, allowing for easy retrieval of paths +// ref: https://linux-kernel-labs.github.io/refs/heads/master/labs/device_model.html#sysfs +type Sysfs interface { + // Devices returns the full path to the devices directory with the specified path appended. + // The path is relative to /sys/devices. + Devices(path string) string + + // Bus returns the full path to the bus directory with the specified path appended. + // The path is relative to /sys/bus. + Bus(path string) string + + // Block returns the full path to the block directory with the specified path appended. + // The path is relative to /sys/block. + Block(path string) string + + // Class returns the full path to the class directory with the specified path appended. + // The path is relative to /sys/class. + Class(path string) string + + // Dev returns the full path to the dev directory with the specified path appended. + // The path is relative to /sys/dev. + Dev(path string) string + + // Firmware returns the full path to the firmware directory with the specified path appended. + // The path is relative to /sys/firmware. + Firmware(path string) string + + // Kernel returns the full path to the kernel directory with the specified path appended. + // The path is relative to /sys/kernel. + Kernel(path string) string + + // Module returns the full path to the module directory with the specified path appended. + // The path is relative to /sys/module. + Module(path string) string +} + +// sysfs provides methods to access sysfs paths +// root is the root path of the sysfs filesystem, typically "/sys". +// Setting non-"/sys" root is only for testing purposes. +type sysfs struct { + root string +} + +// NewSysfs creates a new Sysfs instance(/sys) +func NewSysfs() Sysfs { + return &sysfs{root: sysfsPath} +} + +// NewSysfsWithRoot creates a new Sysfs instance with the specified root path. +// If the root path is empty, it defaults to "/sys". +// This method is only for testing purposes. +func NewSysfsWithRoot(root string) Sysfs { + if root == "" { + root = sysfsPath + } + return &sysfs{root: root} +} + +func (s *sysfs) Devices(path string) string { + return filepath.Join(s.root, "devices", path) +} + +func (s *sysfs) Bus(path string) string { + return filepath.Join(s.root, "bus", path) +} + +func (s *sysfs) Block(path string) string { + return filepath.Join(s.root, "block", path) +} + +func (s *sysfs) Class(path string) string { + return filepath.Join(s.root, "class", path) +} + +func (s *sysfs) Dev(path string) string { + return filepath.Join(s.root, "dev", path) +} + +func (s *sysfs) Firmware(path string) string { + return filepath.Join(s.root, "firmware", path) +} +func (s *sysfs) Kernel(path string) string { + return filepath.Join(s.root, "kernel", path) +} + +func (s *sysfs) Module(path string) string { + return filepath.Join(s.root, "module", path) +}