From b2fe74fba89d2234cb7379e8cd7f1d88f70f9855 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Thu, 18 Dec 2025 19:19:49 +0300 Subject: [PATCH 01/29] add dra usbip Signed-off-by: Yaroslav Borbat --- .../virtualization-dra-plugin/werf.inc.yaml | 10 +- .../mount-points.yaml | 4 + .../werf.inc.yaml | 28 + images/virtualization-dra/Taskfile.yaml | 12 + images/virtualization-dra/api/doc.go | 20 + images/virtualization-dra/api/types.go | 48 ++ .../api/zz_generated.deepcopy.go | 51 ++ .../cmd/go-usbip/app/app.go | 51 ++ .../cmd/go-usbip/app/attach.go | 59 ++ .../cmd/go-usbip/app/bind.go | 49 ++ .../cmd/go-usbip/app/detach.go | 55 ++ .../cmd/go-usbip/app/ports.go | 59 ++ .../cmd/go-usbip/app/run.go | 89 +++ .../cmd/go-usbip/app/unbind.go | 49 ++ .../virtualization-dra/cmd/go-usbip/main.go | 36 ++ .../cmd/usb-gateway/app/app.go | 183 ++++++ .../cmd/usb-gateway/app/init.go | 52 ++ .../cmd/usb-gateway/main.go | 36 ++ images/virtualization-dra/go.mod | 57 +- images/virtualization-dra/go.sum | 126 ++-- .../hack/boilerplate.go.txt | 15 + .../virtualization-dra/hack/update-codegen.sh | 28 + .../internal/common/consts.go | 29 + .../internal/featuregates/featuregates.go | 76 +++ .../internal/plugin/driver.go | 6 +- .../controller/resourceclaim/controller.go | 610 ++++++++++++++++++ .../internal/usb-gateway/informer/informer.go | 150 +++++ .../internal/usb-gateway/labeler/labeler.go | 83 +++ .../internal/usb-gateway/prepare/labels.go | 32 + .../internal/usb-gateway/tlsproxy/proxy.go | 97 +++ .../internal/usb/convert.go | 41 +- .../virtualization-dra/internal/usb/device.go | 178 +---- .../internal/usb/discovery.go | 44 +- .../virtualization-dra/internal/usb/store.go | 134 ++-- .../internal/usbip/attacher.go | 302 +++++++++ .../internal/usbip/binder.go | 224 +++++++ .../internal/usbip/interfaces.go | 80 +++ .../internal/usbip/protocol/common.go | 145 +++++ .../internal/usbip/protocol/convert.go | 52 ++ .../internal/usbip/protocol/device_list.go | 249 +++++++ .../internal/usbip/protocol/import.go | 88 +++ .../internal/usbip/sysfs.go | 79 +++ .../internal/usbip/usbip.go | 17 + .../internal/usbip/usbipd.go | 442 +++++++++++++ .../internal/usbip/usbipd_config.go | 164 +++++ .../virtualization-dra/internal/usbip/vhci.go | 213 ++++++ .../pkg/modprobe/modprobe.go | 58 ++ .../virtualization-dra/pkg/usb/discovery.go | 87 +++ images/virtualization-dra/pkg/usb/monitor.go | 180 ++++++ images/virtualization-dra/pkg/usb/speed.go | 46 ++ images/virtualization-dra/pkg/usb/usb.go | 452 +++++++++++++ .../test/pod-with-template-3.yaml | 16 + .../test/resourceclaim-template-2.yaml | 17 + images/virtualization-dra/werf.inc.yaml | 11 +- .../_helper.tpl | 5 + .../daemonset.yaml | 126 ++++ .../rbac-for-us.yaml | 34 + 57 files changed, 5379 insertions(+), 305 deletions(-) create mode 100644 images/virtualization-dra-usb-gateway/mount-points.yaml create mode 100644 images/virtualization-dra-usb-gateway/werf.inc.yaml create mode 100644 images/virtualization-dra/api/doc.go create mode 100644 images/virtualization-dra/api/types.go create mode 100644 images/virtualization-dra/api/zz_generated.deepcopy.go create mode 100644 images/virtualization-dra/cmd/go-usbip/app/app.go create mode 100644 images/virtualization-dra/cmd/go-usbip/app/attach.go create mode 100644 images/virtualization-dra/cmd/go-usbip/app/bind.go create mode 100644 images/virtualization-dra/cmd/go-usbip/app/detach.go create mode 100644 images/virtualization-dra/cmd/go-usbip/app/ports.go create mode 100644 images/virtualization-dra/cmd/go-usbip/app/run.go create mode 100644 images/virtualization-dra/cmd/go-usbip/app/unbind.go create mode 100644 images/virtualization-dra/cmd/go-usbip/main.go create mode 100644 images/virtualization-dra/cmd/usb-gateway/app/app.go create mode 100644 images/virtualization-dra/cmd/usb-gateway/app/init.go create mode 100644 images/virtualization-dra/cmd/usb-gateway/main.go create mode 100644 images/virtualization-dra/hack/boilerplate.go.txt create mode 100755 images/virtualization-dra/hack/update-codegen.sh create mode 100644 images/virtualization-dra/internal/common/consts.go create mode 100644 images/virtualization-dra/internal/featuregates/featuregates.go create mode 100644 images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go create mode 100644 images/virtualization-dra/internal/usb-gateway/informer/informer.go create mode 100644 images/virtualization-dra/internal/usb-gateway/labeler/labeler.go create mode 100644 images/virtualization-dra/internal/usb-gateway/prepare/labels.go create mode 100644 images/virtualization-dra/internal/usb-gateway/tlsproxy/proxy.go create mode 100644 images/virtualization-dra/internal/usbip/attacher.go create mode 100644 images/virtualization-dra/internal/usbip/binder.go create mode 100644 images/virtualization-dra/internal/usbip/interfaces.go create mode 100644 images/virtualization-dra/internal/usbip/protocol/common.go create mode 100644 images/virtualization-dra/internal/usbip/protocol/convert.go create mode 100644 images/virtualization-dra/internal/usbip/protocol/device_list.go create mode 100644 images/virtualization-dra/internal/usbip/protocol/import.go create mode 100644 images/virtualization-dra/internal/usbip/sysfs.go create mode 100644 images/virtualization-dra/internal/usbip/usbip.go create mode 100644 images/virtualization-dra/internal/usbip/usbipd.go create mode 100644 images/virtualization-dra/internal/usbip/usbipd_config.go create mode 100644 images/virtualization-dra/internal/usbip/vhci.go create mode 100644 images/virtualization-dra/pkg/modprobe/modprobe.go create mode 100644 images/virtualization-dra/pkg/usb/discovery.go create mode 100644 images/virtualization-dra/pkg/usb/monitor.go create mode 100644 images/virtualization-dra/pkg/usb/speed.go create mode 100644 images/virtualization-dra/pkg/usb/usb.go create mode 100644 images/virtualization-dra/test/pod-with-template-3.yaml create mode 100644 images/virtualization-dra/test/resourceclaim-template-2.yaml create mode 100644 templates/virtualization-dra-usb-gateway/_helper.tpl create mode 100644 templates/virtualization-dra-usb-gateway/daemonset.yaml create mode 100644 templates/virtualization-dra-usb-gateway/rbac-for-us.yaml diff --git a/images/virtualization-dra-plugin/werf.inc.yaml b/images/virtualization-dra-plugin/werf.inc.yaml index 7615500ef5..dc3f154093 100644 --- a/images/virtualization-dra-plugin/werf.inc.yaml +++ b/images/virtualization-dra-plugin/werf.inc.yaml @@ -9,10 +9,10 @@ import: to: /app/virtualization-dra-plugin after: install {{- if eq $.DEBUG_COMPONENT "delve/virtualization-dra-plugin" }} -- image: debugger - add: /app/dlv - to: /app/dlv - after: install + - image: debugger + add: /app/dlv + to: /app/dlv + after: install {{- end }} imageSpec: config: @@ -22,7 +22,7 @@ imageSpec: env: PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/app/dlv" XDG_CONFIG_HOME: "/tmp" - entrypoint: ["/app/dlv", "--listen=:2345", "--headless=true", "--continue", "--log=true", "--log-output=debugger,debuglineerr,gdbwire,lldbout,rpc", "--accept-multiclient", "--api-version=2", "exec", "/app/virtualization-dra-plugin", "--", "--leader-election=false"] + entrypoint: ["/app/dlv", "--listen=:2345", "--headless=true", "--continue", "--log=true", "--log-output=debugger,debuglineerr,gdbwire,lldbout,rpc", "--accept-multiclient", "--api-version=2", "exec", "/app/virtualization-dra-plugin", "--"] {{- else }} entrypoint: ["/app/virtualization-dra-plugin"] {{- end }} diff --git a/images/virtualization-dra-usb-gateway/mount-points.yaml b/images/virtualization-dra-usb-gateway/mount-points.yaml new file mode 100644 index 0000000000..448596c34b --- /dev/null +++ b/images/virtualization-dra-usb-gateway/mount-points.yaml @@ -0,0 +1,4 @@ +# A list of pre-created mount points for containerd strict mode. + +dirs: [] + diff --git a/images/virtualization-dra-usb-gateway/werf.inc.yaml b/images/virtualization-dra-usb-gateway/werf.inc.yaml new file mode 100644 index 0000000000..e5fbc673e7 --- /dev/null +++ b/images/virtualization-dra-usb-gateway/werf.inc.yaml @@ -0,0 +1,28 @@ +--- +image: {{ .ModuleNamePrefix }}{{ .ImageName }} +fromImage: {{ .ModuleNamePrefix }}distroless +git: + {{- include "image mount points" . }} +import: + - image: {{ .ModuleNamePrefix }}virtualization-dra-builder + add: /out/virtualization-dra-usb-gateway + to: /app/virtualization-dra-usb-gateway + after: install + {{- if eq $.DEBUG_COMPONENT "delve/virtualization-dra-usb-gateway" }} + - image: debugger + add: /app/dlv + to: /app/dlv + after: install + {{- end }} +imageSpec: + config: + user: 64535 + workingDir: "/app" + {{- if eq $.DEBUG_COMPONENT "delve/virtualization-dra-usb-gateway" }} + env: + PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/app/dlv" + XDG_CONFIG_HOME: "/tmp" + entrypoint: ["/app/dlv", "--listen=:2345", "--headless=true", "--continue", "--log=true", "--log-output=debugger,debuglineerr,gdbwire,lldbout,rpc", "--accept-multiclient", "--api-version=2", "exec", "/app/virtualization-dra-usb-gateway", "--"] + {{- else }} + entrypoint: ["/app/virtualization-dra-usb-gateway"] + {{- end }} diff --git a/images/virtualization-dra/Taskfile.yaml b/images/virtualization-dra/Taskfile.yaml index a28e5823b4..145a70c862 100644 --- a/images/virtualization-dra/Taskfile.yaml +++ b/images/virtualization-dra/Taskfile.yaml @@ -57,3 +57,15 @@ tasks: cmds: - | golangci-lint run + + + build:go-usbip: + desc: "Build go-usbip binary" + cmds: + - go build -o bin/go-usbip cmd/go-usbip/main.go + + api:generate: + desc: "Generate API code" + cmds: + - hack/update-codegen.sh + diff --git a/images/virtualization-dra/api/doc.go b/images/virtualization-dra/api/doc.go new file mode 100644 index 0000000000..01a9011a22 --- /dev/null +++ b/images/virtualization-dra/api/doc.go @@ -0,0 +1,20 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:deepcopy-gen=package +// +groupName=dra.virtualization.deckhouse.io + +package api diff --git a/images/virtualization-dra/api/types.go b/images/virtualization-dra/api/types.go new file mode 100644 index 0000000000..abdadcbaa6 --- /dev/null +++ b/images/virtualization-dra/api/types.go @@ -0,0 +1,48 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +type USBGatewayStatus struct { + metav1.TypeMeta `json:",inline"` + + BusNum int `json:"busNum"` + DeviceNum int `json:"deviceNum"` + DevicePath string `json:"devicePath"` + + TargetIP string `json:"targetIP"` + TargetPort int `json:"targetPort"` + Bound bool `json:"bound"` + Attached bool `json:"attached"` +} + +func FromData(data *runtime.RawExtension) *USBGatewayStatus { + if data == nil { + return nil + } + status, ok := data.Object.(*USBGatewayStatus) + if !ok { + return nil + } + return status +} diff --git a/images/virtualization-dra/api/zz_generated.deepcopy.go b/images/virtualization-dra/api/zz_generated.deepcopy.go new file mode 100644 index 0000000000..21ac5d1595 --- /dev/null +++ b/images/virtualization-dra/api/zz_generated.deepcopy.go @@ -0,0 +1,51 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package api + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *USBGatewayStatus) DeepCopyInto(out *USBGatewayStatus) { + *out = *in + out.TypeMeta = in.TypeMeta + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new USBGatewayStatus. +func (in *USBGatewayStatus) DeepCopy() *USBGatewayStatus { + if in == nil { + return nil + } + out := new(USBGatewayStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *USBGatewayStatus) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} diff --git a/images/virtualization-dra/cmd/go-usbip/app/app.go b/images/virtualization-dra/cmd/go-usbip/app/app.go new file mode 100644 index 0000000000..fbb1fe327f --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/app/app.go @@ -0,0 +1,51 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import "github.com/spf13/cobra" + +const long = ` + _ _ + __ _ ___ _ _ ___| |__ (_)_ __ + / _' |/ _ \ _____| | | / __| '_ \| | '_ \ +| (_| | (_) |_____| |_| \__ \ |_) | | |_) | +\__, | \___/ \__,_|___/_.__/|_| .__/ +|___/ |_| + + go-usbip is a implementation of USBIP server and client. +` + +func NewUSBIPCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "usbip", + Short: "USBIP command line tool", + Long: long, + SilenceUsage: true, + SilenceErrors: true, + } + + cmd.AddCommand( + NewRunCommand(), + NewBindCommand(), + NewUnbindCommand(), + NewAttachCommand(), + NewDetachCommand(), + NewUsedPortsCommand(), + ) + + return cmd +} diff --git a/images/virtualization-dra/cmd/go-usbip/app/attach.go b/images/virtualization-dra/cmd/go-usbip/app/attach.go new file mode 100644 index 0000000000..818f983a00 --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/app/attach.go @@ -0,0 +1,59 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/deckhouse/virtualization-dra/internal/usbip" +) + +func NewAttachCommand() *cobra.Command { + o := &attachOptions{} + cmd := &cobra.Command{ + Use: "attach [:host:] [:busID:]", + Short: "Attach USB devices to USBIP server", + Example: o.Usage(), + RunE: o.Run, + Args: cobra.ExactArgs(2), + } + + o.AddFlags(cmd.Flags()) + + return cmd +} + +type attachOptions struct { + port int +} + +func (o *attachOptions) Usage() string { + return ` # Attach USB devices to USBIP server + $ go-usbip attach 192.168.1.1 3-2.1.1 +` +} + +func (o *attachOptions) AddFlags(fs *pflag.FlagSet) { + fs.IntVar(&o.port, "port", 3240, "Remote port for attaching") +} + +func (o *attachOptions) Run(_ *cobra.Command, args []string) error { + host := args[0] + busID := args[1] + return usbip.NewUSBAttacher().Attach(host, busID, o.port) +} diff --git a/images/virtualization-dra/cmd/go-usbip/app/bind.go b/images/virtualization-dra/cmd/go-usbip/app/bind.go new file mode 100644 index 0000000000..31b505ac5b --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/app/bind.go @@ -0,0 +1,49 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "github.com/spf13/cobra" + + "github.com/deckhouse/virtualization-dra/internal/usbip" +) + +func NewBindCommand() *cobra.Command { + o := &bindOptions{} + cmd := &cobra.Command{ + Use: "bind [:busID:]", + Short: "Bind USB devices to USBIP server", + Example: o.Usage(), + RunE: o.Run, + Args: cobra.ExactArgs(1), + } + + return cmd +} + +type bindOptions struct{} + +func (o *bindOptions) Usage() string { + return ` # Bind USB devices to USBIP server + $ go-usbip bind 3-2.1.1 +` +} + +func (o *bindOptions) Run(_ *cobra.Command, args []string) error { + busID := args[0] + return usbip.NewUSBBinder().Bind(busID) +} diff --git a/images/virtualization-dra/cmd/go-usbip/app/detach.go b/images/virtualization-dra/cmd/go-usbip/app/detach.go new file mode 100644 index 0000000000..15c2654407 --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/app/detach.go @@ -0,0 +1,55 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + "strconv" + + "github.com/spf13/cobra" + + "github.com/deckhouse/virtualization-dra/internal/usbip" +) + +func NewDetachCommand() *cobra.Command { + o := &detachOptions{} + cmd := &cobra.Command{ + Use: "detach [:port:]", + Short: "Detach USB devices from USBIP server", + Example: o.Usage(), + RunE: o.Run, + Args: cobra.ExactArgs(1), + } + + return cmd +} + +type detachOptions struct{} + +func (o *detachOptions) Usage() string { + return ` # Detach USB devices from USBIP server + $ go-usbip detach 0 +` +} + +func (o *detachOptions) Run(_ *cobra.Command, args []string) error { + port, err := strconv.Atoi(args[0]) + if err != nil { + return fmt.Errorf("invalid port: %w", err) + } + return usbip.NewUSBAttacher().Detach(port) +} diff --git a/images/virtualization-dra/cmd/go-usbip/app/ports.go b/images/virtualization-dra/cmd/go-usbip/app/ports.go new file mode 100644 index 0000000000..634e519071 --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/app/ports.go @@ -0,0 +1,59 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/deckhouse/virtualization-dra/internal/usbip" +) + +func NewUsedPortsCommand() *cobra.Command { + o := &usedPortsOptions{} + cmd := &cobra.Command{ + Use: "ports", + Short: "List used ports", + Example: o.Usage(), + RunE: o.Run, + } + + return cmd +} + +type usedPortsOptions struct{} + +func (o *usedPortsOptions) Usage() string { + return ` # List used ports + $ go-usbip ports +` +} + +func (o *usedPortsOptions) Run(cmd *cobra.Command, _ []string) error { + ports, err := usbip.NewUSBAttacher().GetUsedPorts() + if err != nil { + return err + } + + cmd.Println("Used ports:") + for _, port := range ports { + cmd.Println(fmt.Sprintf("- %d", port)) + } + + return nil +} diff --git a/images/virtualization-dra/cmd/go-usbip/app/run.go b/images/virtualization-dra/cmd/go-usbip/app/run.go new file mode 100644 index 0000000000..0962f174b2 --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/app/run.go @@ -0,0 +1,89 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "context" + "time" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/deckhouse/virtualization-dra/internal/usbip" + "github.com/deckhouse/virtualization-dra/pkg/usb" +) + +func NewRunCommand() *cobra.Command { + o := &runOptions{} + cmd := &cobra.Command{ + Use: "run", + Short: "Run USBIP server", + Example: o.Usage(), + RunE: o.Run, + Args: cobra.NoArgs, + } + + o.AddFlags(cmd.Flags()) + + return cmd +} + +type runOptions struct { + port int + resyncPeriod time.Duration +} + +func (o *runOptions) Usage() string { + return ` # Run USBIP server + $ go-usbip run +` +} + +func (o *runOptions) AddFlags(fs *pflag.FlagSet) { + fs.IntVar(&o.port, "port", 3240, "Port to listen on") + fs.DurationVar(&o.resyncPeriod, "resync-period", time.Second*300, "Resync period") +} + +func (o *runOptions) Run(cmd *cobra.Command, _ []string) error { + monitor, err := usb.NewMonitor(context.Background(), o.resyncPeriod) + if err != nil { + return err + } + + config := usbip.USBIPDConfig{ + Port: o.port, + Monitor: monitor, + } + err = config.Validate() + if err != nil { + return err + } + + usbipd, err := config.Complete() + if err != nil { + return err + } + + err = usbipd.Start(cmd.Context()) + if err != nil { + return err + } + + <-cmd.Context().Done() + + return nil +} diff --git a/images/virtualization-dra/cmd/go-usbip/app/unbind.go b/images/virtualization-dra/cmd/go-usbip/app/unbind.go new file mode 100644 index 0000000000..a2bc72d792 --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/app/unbind.go @@ -0,0 +1,49 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "github.com/spf13/cobra" + + "github.com/deckhouse/virtualization-dra/internal/usbip" +) + +func NewUnbindCommand() *cobra.Command { + o := &unbindOptions{} + cmd := &cobra.Command{ + Use: "unbind [:busID:]", + Short: "Unbind USB devices from USBIP server", + Example: o.Usage(), + RunE: o.Run, + Args: cobra.ExactArgs(1), + } + + return cmd +} + +type unbindOptions struct{} + +func (o *unbindOptions) Usage() string { + return ` # Unbind USB devices from USBIP server + $ go-usbip unbind 3-2.1.1 +` +} + +func (o *unbindOptions) Run(_ *cobra.Command, args []string) error { + busID := args[0] + return usbip.NewUSBBinder().Unbind(busID) +} diff --git a/images/virtualization-dra/cmd/go-usbip/main.go b/images/virtualization-dra/cmd/go-usbip/main.go new file mode 100644 index 0000000000..ee2334bb7b --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/main.go @@ -0,0 +1,36 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "log/slog" + "os" + "os/signal" + "syscall" + + "github.com/deckhouse/virtualization-dra/cmd/go-usbip/app" +) + +func main() { + ctx, _ := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + + if err := app.NewUSBIPCommand().ExecuteContext(ctx); err != nil { + slog.Error("failed to execute command", slog.Any("err", err)) + os.Exit(1) + } +} diff --git a/images/virtualization-dra/cmd/usb-gateway/app/app.go b/images/virtualization-dra/cmd/usb-gateway/app/app.go new file mode 100644 index 0000000000..0bd715c0df --- /dev/null +++ b/images/virtualization-dra/cmd/usb-gateway/app/app.go @@ -0,0 +1,183 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + "net" + "os" + "time" + + "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/component-base/cli/flag" + + "github.com/deckhouse/virtualization-dra/internal/featuregates" + "github.com/deckhouse/virtualization-dra/internal/usb-gateway/controller/resourceclaim" + "github.com/deckhouse/virtualization-dra/internal/usb-gateway/informer" + "github.com/deckhouse/virtualization-dra/internal/usb-gateway/prepare" + "github.com/deckhouse/virtualization-dra/internal/usbip" + "github.com/deckhouse/virtualization-dra/pkg/logger" + "github.com/deckhouse/virtualization-dra/pkg/usb" +) + +func NewUSBGatewayCommand() *cobra.Command { + o := newUsbOptions() + + cmd := &cobra.Command{ + Use: "usb-gateway", + Short: "USB gateway", + Long: "USB gateway", + SilenceUsage: true, + SilenceErrors: true, + PreRunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + log := o.Logging.Complete() + logger.SetDefaultLogger(log) + return nil + }, + RunE: o.Run, + } + + cmd.AddCommand(NewInitCommand()) + + fs := cmd.Flags() + for _, f := range o.NamedFlags().FlagSets { + fs.AddFlagSet(f) + } + + return cmd +} + +func newUsbOptions() *usbOptions { + return &usbOptions{ + Logging: &logger.Options{}, + featureGates: featuregates.AddFlags, + } +} + +type usbOptions struct { + Kubeconfig string + NodeName string + PodIP string + USBIPPort int + USBResyncPeriod time.Duration + Logging *logger.Options + featureGates featuregates.AddFlagsFunc +} + +func (o *usbOptions) NamedFlags() (fs flag.NamedFlagSets) { + mfs := fs.FlagSet("usb-gateway") + mfs.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "Path to kubeconfig file") + mfs.StringVar(&o.NodeName, "node-name", os.Getenv("NODE_NAME"), "Node name") + mfs.StringVar(&o.PodIP, "pod-ip", os.Getenv("POD_IP"), "Pod IP") + mfs.IntVar(&o.USBIPPort, "usbip-port", 3240, "USBIP port") + mfs.DurationVar(&o.USBResyncPeriod, "usb-resync-period", 5*time.Minute, "USB resync period") + + o.Logging.AddFlags(fs.FlagSet("logging")) + + o.featureGates(fs.FlagSet("feature-gates")) + + return fs +} + +func (o *usbOptions) Validate() error { + if o.NodeName == "" { + return fmt.Errorf("NodeName is required") + } + if o.PodIP == "" { + return fmt.Errorf("PodIP is required") + } + if net.ParseIP(o.PodIP) == nil { + return fmt.Errorf("PodIP is not a valid IP address") + } + if o.USBIPPort < 1 || o.USBIPPort > 65535 { + return fmt.Errorf("USBIPPort is not a valid port number") + } + + return nil +} + +func (o *usbOptions) Run(cmd *cobra.Command, _ []string) error { + monitor, err := usb.NewMonitor(cmd.Context(), o.USBResyncPeriod) + if err != nil { + return err + } + config := usbip.USBIPDConfig{ + Port: o.USBIPPort, + Monitor: monitor, + } + err = config.Validate() + if err != nil { + return err + } + + usbipd, err := config.Complete() + if err != nil { + return err + } + + cfg, err := clientcmd.BuildConfigFromFlags("", o.Kubeconfig) + if err != nil { + return fmt.Errorf("failed to get rest config: %w", err) + } + + client, err := kubernetes.NewForConfig(cfg) + if err != nil { + return fmt.Errorf("failed to create kubernetes client: %w", err) + } + + dynamicClient, err := dynamic.NewForConfig(cfg) + if err != nil { + return fmt.Errorf("failed to create dynamic client: %w", err) + } + + if err = prepare.MarkNodeForUSBGateway(cmd.Context(), o.NodeName, dynamicClient); err != nil { + return fmt.Errorf("failed to mark node for USB gateway: %w", err) + } + + f := informer.NewFactory(client, nil) + resourceClaimInformer := f.ResourceClaim() + resourceSliceInformer := f.ResourceSlice() + nodeInformer := f.Nodes() + podInformer := f.Pods() + + f.Start(cmd.Context().Done()) + f.WaitForCacheSync(cmd.Context().Done()) + + ip := net.ParseIP(o.PodIP) + usbIPInterface := usbip.New() + c, err := resourceclaim.NewController(o.NodeName, ip, o.USBIPPort, client, resourceClaimInformer, resourceSliceInformer, nodeInformer, podInformer, usbIPInterface) + if err != nil { + return fmt.Errorf("failed to create resourceclaim controller: %w", err) + } + + group, ctx := errgroup.WithContext(cmd.Context()) + group.Go(func() error { + return usbipd.Run(ctx) + }) + group.Go(func() error { + return c.Run(ctx, 1) + }) + + return group.Wait() +} diff --git a/images/virtualization-dra/cmd/usb-gateway/app/init.go b/images/virtualization-dra/cmd/usb-gateway/app/init.go new file mode 100644 index 0000000000..a435e880dd --- /dev/null +++ b/images/virtualization-dra/cmd/usb-gateway/app/init.go @@ -0,0 +1,52 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/deckhouse/virtualization-dra/pkg/modprobe" +) + +func NewInitCommand() *cobra.Command { + o := &initOptions{} + + cmd := &cobra.Command{ + Use: "init", + Short: "Init USB gateway", + RunE: o.Run, + } + + return cmd +} + +type initOptions struct{} + +func (o *initOptions) Run(_ *cobra.Command, _ []string) error { + modules := []string{ + "kernel/drivers/usb/usbip/usbip-core.ko", + "kernel/drivers/usb/usbip/vhci-hcd.ko", + } + + if err := modprobe.LoadModules(modules); err != nil { + return fmt.Errorf("failed to load modules: %w", err) + } + + return nil +} diff --git a/images/virtualization-dra/cmd/usb-gateway/main.go b/images/virtualization-dra/cmd/usb-gateway/main.go new file mode 100644 index 0000000000..4ab9d51646 --- /dev/null +++ b/images/virtualization-dra/cmd/usb-gateway/main.go @@ -0,0 +1,36 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "log/slog" + "os" + "os/signal" + "syscall" + + "github.com/deckhouse/virtualization-dra/cmd/usb-gateway/app" +) + +func main() { + ctx, _ := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + + if err := app.NewUSBGatewayCommand().ExecuteContext(ctx); err != nil { + slog.Error("failed to execute command", slog.Any("err", err)) + os.Exit(1) + } +} diff --git a/images/virtualization-dra/go.mod b/images/virtualization-dra/go.mod index a6363e2f74..9bac9000d9 100644 --- a/images/virtualization-dra/go.mod +++ b/images/virtualization-dra/go.mod @@ -1,38 +1,47 @@ module github.com/deckhouse/virtualization-dra -go 1.24.7 +go 1.25.0 -tool github.com/onsi/ginkgo/v2/ginkgo +tool ( + github.com/onsi/ginkgo/v2/ginkgo + k8s.io/code-generator +) require ( github.com/containerd/nri v0.10.0 github.com/deckhouse/deckhouse/pkg/log v0.1.0 - github.com/go-logr/logr v1.4.2 + github.com/fsnotify/fsnotify v1.5.1 + github.com/go-logr/logr v1.4.3 github.com/godbus/dbus/v5 v5.2.0 - github.com/onsi/ginkgo/v2 v2.21.0 - github.com/onsi/gomega v1.35.1 + github.com/onsi/ginkgo/v2 v2.27.2 + github.com/onsi/gomega v1.38.2 github.com/spf13/cobra v1.10.1 github.com/spf13/pflag v1.0.9 + golang.org/x/sync v0.18.0 + golang.org/x/sys v0.38.0 google.golang.org/grpc v1.72.1 k8s.io/api v0.34.2 - k8s.io/apimachinery v0.34.2 + k8s.io/apimachinery v0.35.0 k8s.io/client-go v0.34.2 k8s.io/component-base v0.34.2 k8s.io/dynamic-resource-allocation v0.34.2 k8s.io/klog/v2 v2.130.1 k8s.io/kubelet v0.34.2 - k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 tags.cncf.io/container-device-interface v1.0.1 tags.cncf.io/container-device-interface/specs-go v1.0.0 ) require ( github.com/DataDog/gostackparse v0.7.0 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/ttrpc v1.2.7 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect - github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -41,7 +50,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect + github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -53,32 +62,38 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/tetratelabs/wazero v1.9.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.etcd.io/etcd/client/pkg/v3 v3.6.4 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/mod v0.21.0 // indirect - golang.org/x/net v0.38.0 // indirect + golang.org/x/mod v0.29.0 // indirect + golang.org/x/net v0.47.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect - golang.org/x/sys v0.31.0 // indirect - golang.org/x/term v0.30.0 // indirect - golang.org/x/text v0.23.0 // indirect + golang.org/x/term v0.37.0 // indirect + golang.org/x/text v0.31.0 // indirect golang.org/x/time v0.9.0 // indirect - golang.org/x/tools v0.26.0 // indirect + golang.org/x/tools v0.38.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect - google.golang.org/protobuf v1.36.5 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect - sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + k8s.io/code-generator v0.35.0 // indirect + k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b // indirect + k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect sigs.k8s.io/yaml v1.6.0 // indirect diff --git a/images/virtualization-dra/go.sum b/images/virtualization-dra/go.sum index dea9d5d69f..8127885373 100644 --- a/images/virtualization-dra/go.sum +++ b/images/virtualization-dra/go.sum @@ -1,7 +1,13 @@ github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4= github.com/DataDog/gostackparse v0.7.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/nri v0.10.0 h1:bt2NzfvlY6OJE0i+fB5WVeGQEycxY7iFVQpEbh7J3Go= @@ -21,8 +27,14 @@ github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWp github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= @@ -35,6 +47,8 @@ github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+Gr github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/godbus/dbus/v5 v5.2.0 h1:3WexO+U+yg9T70v9FdHr9kCxYlazaAXUhx2VMkbfax8= github.com/godbus/dbus/v5 v5.2.0/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -46,8 +60,8 @@ github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7O github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -59,10 +73,14 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/knqyf263/go-plugin v0.9.0 h1:CQs2+lOPIlkZVtcb835ZYDEoyyWJWLbSTWeCs0EwTwI= github.com/knqyf263/go-plugin v0.9.0/go.mod h1:2z5lCO1/pez6qGo8CvCxSlBFSEat4MEp1DrnA+f7w8Q= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -72,8 +90,14 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -84,10 +108,10 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWu github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= -github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= -github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= -github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= +github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= +github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -96,14 +120,18 @@ github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626/go. github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opencontainers/selinux v1.10.0 h1:rAiKF8hTcgLI3w0DHm6i0ylVVcOrlgR1kK99DRLDhyU= github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= @@ -123,12 +151,20 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I= github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= @@ -160,8 +196,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= -go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -169,19 +205,21 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= -golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -189,22 +227,26 @@ golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= +golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -213,13 +255,13 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA= google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -228,24 +270,28 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.34.2 h1:fsSUNZhV+bnL6Aqrp6O7lMTy6o5x2C4XLjnh//8SLYY= k8s.io/api v0.34.2/go.mod h1:MMBPaWlED2a8w4RSeanD76f7opUoypY8TFYkSM+3XHw= -k8s.io/apimachinery v0.34.2 h1:zQ12Uk3eMHPxrsbUJgNF8bTauTVR2WgqJsTmwTE/NW4= -k8s.io/apimachinery v0.34.2/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8= +k8s.io/apimachinery v0.35.0/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= k8s.io/client-go v0.34.2 h1:Co6XiknN+uUZqiddlfAjT68184/37PS4QAzYvQvDR8M= k8s.io/client-go v0.34.2/go.mod h1:2VYDl1XXJsdcAxw7BenFslRQX28Dxz91U9MWKjX97fE= +k8s.io/code-generator v0.35.0 h1:TvrtfKYZTm9oDF2z+veFKSCcgZE3Igv0svY+ehCmjHQ= +k8s.io/code-generator v0.35.0/go.mod h1:iS1gvVf3c/T71N5DOGYO+Gt3PdJ6B9LYSvIyQ4FHzgc= k8s.io/component-base v0.34.2 h1:HQRqK9x2sSAsd8+R4xxRirlTjowsg6fWCPwWYeSvogQ= k8s.io/component-base v0.34.2/go.mod h1:9xw2FHJavUHBFpiGkZoKuYZ5pdtLKe97DEByaA+hHbM= k8s.io/dynamic-resource-allocation v0.34.2 h1:SjlRGSWl6CZXoJwQNL+Y0wRfdH8PkJ4mHRNK6MMj0bY= k8s.io/dynamic-resource-allocation v0.34.2/go.mod h1:ul6I+gfrCmC+OCuVdN0/iykyB2sPrIqh2WyKQ3RQPCU= +k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b h1:gMplByicHV/TJBizHd9aVEsTYoJBnnUAT5MHlTkbjhQ= +k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b/go.mod h1:CgujABENc3KuTrcsdpGmrrASjtQsWCT7R99mEV4U/fM= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= k8s.io/kubelet v0.34.2 h1:Dl+1uh7xwJr70r+SHKyIpvu6XvzuoPu0uDIC4cqgJUs= k8s.io/kubelet v0.34.2/go.mod h1:RfwR03iuKeVV7Z1qD9XKH98c3tlPImJpQ3qHIW40htM= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= diff --git a/images/virtualization-dra/hack/boilerplate.go.txt b/images/virtualization-dra/hack/boilerplate.go.txt new file mode 100644 index 0000000000..b8911cc97b --- /dev/null +++ b/images/virtualization-dra/hack/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ diff --git a/images/virtualization-dra/hack/update-codegen.sh b/images/virtualization-dra/hack/update-codegen.sh new file mode 100755 index 0000000000..def707d625 --- /dev/null +++ b/images/virtualization-dra/hack/update-codegen.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyright 2025 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +API_ROOT="${SCRIPT_DIR}/../api" +CODEGEN_PKG="$(go env GOMODCACHE)/$(go list -f '{{.Path}}@{{.Version}}' -m k8s.io/code-generator)" +source "${CODEGEN_PKG}/kube_codegen.sh" + +function generate { + kube::codegen::gen_helpers \ + --boilerplate "${SCRIPT_DIR}/boilerplate.go.txt" \ + "${API_ROOT}" +} + +generate diff --git a/images/virtualization-dra/internal/common/consts.go b/images/virtualization-dra/internal/common/consts.go new file mode 100644 index 0000000000..e9a5bdc0eb --- /dev/null +++ b/images/virtualization-dra/internal/common/consts.go @@ -0,0 +1,29 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +const ( + USBGatewayLabel = "virtualization.deckhouse.io/usb-gateway" +) + +const ( + USBGatewayAnnotation = "virtualization.deckhouse.io/usb-gateway" +) + +const ( + VirtualizationDraPluginName = "virtualization-dra" +) diff --git a/images/virtualization-dra/internal/featuregates/featuregates.go b/images/virtualization-dra/internal/featuregates/featuregates.go new file mode 100644 index 0000000000..91fac715a1 --- /dev/null +++ b/images/virtualization-dra/internal/featuregates/featuregates.go @@ -0,0 +1,76 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package featuregates + +import ( + "github.com/spf13/pflag" + "k8s.io/component-base/featuregate" +) + +const ( + USBGateway featuregate.Feature = "USBGateway" +) + +var featureSpecs = map[featuregate.Feature]featuregate.FeatureSpec{ + USBGateway: { + Default: false, + PreRelease: featuregate.Alpha, + }, +} + +var ( + instance *FeatureGate + addFlags func(fs *pflag.FlagSet) +) + +func init() { + gate, gateAddFlags, _, err := New() + if err != nil { + panic(err) + } + instance = gate + addFlags = gateAddFlags +} + +func AddFlags(fs *pflag.FlagSet) { + addFlags(fs) +} + +func Default() *FeatureGate { + return instance +} + +type ( + AddFlagsFunc func(fs *pflag.FlagSet) + SetFromMapFunc func(m map[string]bool) error +) + +func New() (*FeatureGate, AddFlagsFunc, SetFromMapFunc, error) { + gate := featuregate.NewFeatureGate() + if err := gate.Add(featureSpecs); err != nil { + return nil, nil, nil, err + } + return &FeatureGate{gate}, gate.AddFlag, gate.SetFromMap, nil +} + +type FeatureGate struct { + featuregate.FeatureGate +} + +func (f *FeatureGate) USBGatewayEnabled() bool { + return f.Enabled(USBGateway) +} diff --git a/images/virtualization-dra/internal/plugin/driver.go b/images/virtualization-dra/internal/plugin/driver.go index c8251e2020..a8605d29a9 100644 --- a/images/virtualization-dra/internal/plugin/driver.go +++ b/images/virtualization-dra/internal/plugin/driver.go @@ -30,9 +30,11 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/dynamic-resource-allocation/kubeletplugin" "k8s.io/dynamic-resource-allocation/resourceslice" + + "github.com/deckhouse/virtualization-dra/internal/common" ) -const DriverName = "virtualization-dra" +const DriverName = common.VirtualizationDraPluginName func NewDriver(nodeName string, kubeClient kubernetes.Interface, allocator Allocator, log *slog.Logger) *Driver { return &Driver{ @@ -169,8 +171,6 @@ func (d *Driver) startPublisher(ctx context.Context) { return case devices := <-ch: d.log.Info("Publishing devices", slog.Any("devices", devices)) - if len(devices) > 0 { - } resources := d.makeResources(devices) err := d.helper.PublishResources(ctx, resources) if err != nil { diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go new file mode 100644 index 0000000000..85ad158e98 --- /dev/null +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go @@ -0,0 +1,610 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourceclaim + +import ( + "context" + "fmt" + "log/slog" + "net" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + resourcev1beta1 "k8s.io/api/resource/v1beta1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + + vdraapi "github.com/deckhouse/virtualization-dra/api" + "github.com/deckhouse/virtualization-dra/internal/common" + "github.com/deckhouse/virtualization-dra/internal/usb-gateway/informer" + "github.com/deckhouse/virtualization-dra/internal/usbip" +) + +const controllerName = "resourceclaim-controller" +const finalizer = "virtualization.deckhouse.io/usb-gateway" + +var ( + keyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc +) + +func controllerKeyFunc(namespace, name string) string { + return types.NamespacedName{ + Namespace: namespace, + Name: name, + }.String() +} + +type Controller struct { + nodeName string + podIP net.IP + usbipPort int + client kubernetes.Interface + resourceClaimIndexer cache.Indexer + resourceSliceIndexer cache.Indexer + nodeIndexer cache.Indexer + podIndexer cache.Indexer + usbIP usbip.Interface + queue workqueue.TypedRateLimitingInterface[string] + log *slog.Logger + hasSynced cache.InformerSynced +} + +func NewController(nodeName string, podIP net.IP, usbipPort int, client kubernetes.Interface, resourceClaimInformer, resourceSliceInformer, nodeInformer, podInformer cache.SharedIndexInformer, usbIP usbip.Interface) (*Controller, error) { + queue := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[string](), + workqueue.TypedRateLimitingQueueConfig[string]{Name: controllerName}, + ) + + c := &Controller{ + nodeName: nodeName, + podIP: podIP, + usbipPort: usbipPort, + client: client, + resourceClaimIndexer: resourceClaimInformer.GetIndexer(), + resourceSliceIndexer: resourceSliceInformer.GetIndexer(), + nodeIndexer: nodeInformer.GetIndexer(), + podIndexer: podInformer.GetIndexer(), + usbIP: usbIP, + queue: queue, + log: slog.With(slog.String("controller", controllerName)), + } + + _, err := resourceClaimInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: c.addResourceClaim, + UpdateFunc: c.updateResourceClaim, + DeleteFunc: c.deleteResourceClaim, + }) + if err != nil { + return nil, fmt.Errorf("unable to add event handler to resourceclaim informer: %w", err) + } + + c.hasSynced = func() bool { + return resourceClaimInformer.HasSynced() && nodeInformer.HasSynced() && podInformer.HasSynced() && resourceSliceInformer.HasSynced() + } + + return c, nil +} + +func (c *Controller) addResourceClaim(obj interface{}) { + if rc, ok := obj.(*resourcev1beta1.ResourceClaim); ok { + c.enqueueResourceClaim(rc) + } +} + +func (c *Controller) deleteResourceClaim(obj interface{}) { + if rc, ok := obj.(*resourcev1beta1.ResourceClaim); ok { + c.enqueueResourceClaim(rc) + } +} + +func (c *Controller) updateResourceClaim(_, newObj interface{}) { + newRC, ok := newObj.(*resourcev1beta1.ResourceClaim) + if !ok { + return + } + + if newRC.Status.Allocation != nil { + c.enqueueResourceClaim(newRC) + } +} + +func (c *Controller) enqueueResourceClaim(rc *resourcev1beta1.ResourceClaim) { + key, err := keyFunc(rc) + if err != nil { + utilruntime.HandleError(fmt.Errorf("couldn't get key for object %#v: %w", rc, err)) + return + } + c.queueAdd(key) +} + +func (c *Controller) queueAdd(key string) { + c.queue.Add(key) +} + +func (c *Controller) queueAfterAdd(key string, duration time.Duration) { + c.queue.AddAfter(key, duration) +} + +func (c *Controller) Run(ctx context.Context, workers int) error { + defer utilruntime.HandleCrash() + defer c.queue.ShutDown() + + c.log.Info("Starting controller") + defer c.log.Info("Shutting down controller") + + if !cache.WaitForCacheSync(ctx.Done(), c.hasSynced) { + return fmt.Errorf("failed to wait for caches to sync") + } + + c.log.Info("Starting workers controller") + for i := 0; i < workers; i++ { + go wait.UntilWithContext(ctx, c.worker, time.Second) + } + + <-ctx.Done() + return nil +} + +func (c *Controller) worker(ctx context.Context) { + workFunc := func(ctx context.Context) bool { + key, quit := c.queue.Get() + if quit { + return true + } + defer c.queue.Done(key) + + if err := c.sync(key); err != nil { + c.log.Error("re-enqueuing", slog.String("key", key), slog.Any("err", err)) + c.queue.AddRateLimited(key) + } else { + c.log.Info(fmt.Sprintf("processed ResourceClaim %v", key)) + c.queue.Forget(key) + } + return false + } + for { + quit := workFunc(ctx) + + if quit { + return + } + } +} + +func (c *Controller) sync(key string) error { + log := c.log.With("key", key) + log.Info("syncing resource claim") + + rc, err := c.getResourceClaim(key) + if err != nil { + return err + } + if rc == nil { + return nil + } + if !rc.GetDeletionTimestamp().IsZero() { + return c.handleDelete(rc) + } + + pod, err := c.getReservedFor(rc) + if err != nil { + return err + } + if pod == nil { + c.log.Info("no reserved pod found for resource claim, re-enqueue after 10s") + c.queueAfterAdd(key, time.Second*10) + return nil + } + + myAllocationDevices, otherAllocationDevices, err := c.getAllocationDevices(rc) + if err != nil { + return err + } + + onMyNode := c.podOnMyNode(pod) + shouldShare := !onMyNode && len(myAllocationDevices) > 0 + shouldAttach := onMyNode && len(otherAllocationDevices) > 0 + + switch { + case shouldShare: + log.Info("sharing usb to other node") + if err = c.handleServer(rc, myAllocationDevices); err != nil { + return fmt.Errorf("failed to handle server: %w", err) + } + case shouldAttach: + log.Info("attaching usb to my node") + if err = c.handleClient(rc, otherAllocationDevices); err != nil { + return fmt.Errorf("failed to handle client: %w", err) + } + + } + + return nil +} + +// TODO: handle detach too +func (c *Controller) handleDelete(rc *resourcev1beta1.ResourceClaim) error { + if c.allUnBound(rc) { + return c.removeFinalizer(rc) + } + + myAllocationDevices, _, err := c.getAllocationDevices(rc) + if err != nil { + return err + } + + myAllocationDevicesByName := make(map[string]resourcev1beta1.Device) + for _, device := range myAllocationDevices { + myAllocationDevicesByName[device.Name] = device + } + + shouldUpdate := false + + for i := range rc.Status.Devices { + allocatedDeviceStatus := &rc.Status.Devices[i] + + usbGatewayStatus := vdraapi.FromData(allocatedDeviceStatus.Data) + if usbGatewayStatus == nil { + continue + } + if !usbGatewayStatus.Bound { + continue + } + + device, ok := myAllocationDevicesByName[allocatedDeviceStatus.Device] + if !ok { + continue + } + + busID := "" + if attr, ok := device.Basic.Attributes["busID"]; ok && attr.StringValue != nil { + busID = *attr.StringValue + } else { + continue + } + + // TODO: device can be added to other resource claims. Not supported yet. + c.log.Info("unbinding usb") + if err = c.usbIP.Unbind(busID); err != nil { + return fmt.Errorf("failed to unbind usb: %w", err) + } + + usbGatewayStatus.Bound = false + allocatedDeviceStatus.Data.Object = usbGatewayStatus + shouldUpdate = true + } + + if shouldUpdate { + _, err = c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).UpdateStatus(context.Background(), rc, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update resource claim status: %w", err) + } + } + + return nil +} + +func (c *Controller) allUnBound(rc *resourcev1beta1.ResourceClaim) bool { + for _, deviceStatus := range rc.Status.Devices { + usbGatewayStatus := vdraapi.FromData(deviceStatus.Data) + if usbGatewayStatus == nil { + continue + } + if usbGatewayStatus.Bound { + return false + } + } + + return true +} + +func (c *Controller) addFinalizer(rc *resourcev1beta1.ResourceClaim) error { + var newFinalizers []string + for _, fin := range rc.GetFinalizers() { + if fin == finalizer { + return nil + } + newFinalizers = append(newFinalizers, fin) + } + newFinalizers = append(newFinalizers, finalizer) + rc.SetFinalizers(newFinalizers) + _, err := c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).Update(context.Background(), rc, metav1.UpdateOptions{}) + return err +} + +func (c *Controller) removeFinalizer(rc *resourcev1beta1.ResourceClaim) error { + var newFinalizers []string + for _, fin := range rc.GetFinalizers() { + if fin == finalizer { + continue + } + newFinalizers = append(newFinalizers, fin) + } + if len(newFinalizers) == len(rc.GetFinalizers()) { + return nil + } + + rc.SetFinalizers(newFinalizers) + _, err := c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).Update(context.Background(), rc, metav1.UpdateOptions{}) + return err +} + +func (c *Controller) handleServer(rc *resourcev1beta1.ResourceClaim, myAllocationDevices []resourcev1beta1.Device) error { + indexAllocDevice := make(map[string]int) + for i, allocDeviceStatus := range rc.Status.Devices { + indexAllocDevice[allocDeviceStatus.Device] = i + } + + shouldUpdate := false + + for _, device := range myAllocationDevices { + if device.Basic == nil { + continue + } + + index, ok := indexAllocDevice[device.Name] + if !ok { + continue + } + + allocDeviceStatus := &rc.Status.Devices[index] + + usbGatewayStatus := vdraapi.FromData(allocDeviceStatus.Data) + + targetIPAlreadySet := usbGatewayStatus != nil && usbGatewayStatus.TargetIP != "" + if targetIPAlreadySet { + continue + } + usbGatewayStatus = &vdraapi.USBGatewayStatus{} + + busID := "" + if attr, ok := device.Basic.Attributes["busID"]; ok && attr.StringValue != nil { + busID = *attr.StringValue + } else { + continue + } + + bound, err := c.usbIP.IsBound(busID) + if err != nil { + return fmt.Errorf("failed to check if usb is bound: %w", err) + } + + if !bound { + if err = c.usbIP.Bind(busID); err != nil { + return fmt.Errorf("failed to bind usb: %w", err) + } + } + + usbGatewayStatus.TargetIP = c.podIP.String() + usbGatewayStatus.TargetPort = c.usbipPort + usbGatewayStatus.Bound = true + + if allocDeviceStatus.Data == nil { + allocDeviceStatus.Data = &runtime.RawExtension{} + } + allocDeviceStatus.Data.Object = usbGatewayStatus + shouldUpdate = true + } + + if shouldUpdate { + err := c.addFinalizer(rc) + if err != nil { + return err + } + _, err = c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).UpdateStatus(context.Background(), rc, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update resource claim status: %w", err) + } + } + + return nil +} + +func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAllocationDevices []resourcev1beta1.Device) error { + indexAllocDevice := make(map[string]int) + for i, allocDeviceStatus := range rc.Status.Devices { + indexAllocDevice[allocDeviceStatus.Device] = i + } + + shouldUpdate := false + + for _, device := range otherAllocationDevices { + if device.Basic == nil { + continue + } + index, ok := indexAllocDevice[device.Name] + if !ok { + continue + } + allocDeviceStatus := &rc.Status.Devices[index] + usbGatewayStatus := vdraapi.FromData(allocDeviceStatus.Data) + if usbGatewayStatus == nil { + continue + } + if !usbGatewayStatus.Bound { + continue + } + if usbGatewayStatus.Attached { + continue + } + busID := "" + if attr, ok := device.Basic.Attributes["busID"]; ok && attr.StringValue != nil { + busID = *attr.StringValue + } else { + continue + } + + err := c.usbIP.Attach(usbGatewayStatus.TargetIP, busID, usbGatewayStatus.TargetPort) + if err != nil { + return fmt.Errorf("failed to attach usb: %w", err) + } + + usbGatewayStatus.Attached = true + if allocDeviceStatus.Data == nil { + allocDeviceStatus.Data = &runtime.RawExtension{} + } + allocDeviceStatus.Data.Object = usbGatewayStatus + shouldUpdate = true + } + + if shouldUpdate { + _, err := c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).UpdateStatus(context.Background(), rc, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update resource claim status: %w", err) + } + } + + return nil +} + +func (c *Controller) getResourceClaim(key string) (*resourcev1beta1.ResourceClaim, error) { + obj, exists, err := c.resourceClaimIndexer.GetByKey(key) + if err != nil && !k8serrors.IsNotFound(err) { + return nil, fmt.Errorf("failed to get resourceclaim: %w", err) + } + if !exists { + return nil, nil + } + + rc, ok := obj.(*resourcev1beta1.ResourceClaim) + if !ok { + return nil, fmt.Errorf("unexpected type of resourceclaim: %T", obj) + } + + return rc.DeepCopy(), nil +} + +func (c *Controller) getPod(name, namespace string) (*corev1.Pod, error) { + obj, exists, err := c.podIndexer.GetByKey(controllerKeyFunc(namespace, name)) + if err != nil && !k8serrors.IsNotFound(err) { + return nil, err + } + if !exists { + return nil, nil + } + + pod, ok := obj.(*corev1.Pod) + if !ok { + return nil, fmt.Errorf("unexpected type of pod: %T", obj) + } + + return pod.DeepCopy(), nil +} + +func (c *Controller) getVirtualizationDraResourceSlices() ([]resourcev1beta1.ResourceSlice, error) { + slicesObj, err := c.resourceSliceIndexer.ByIndex(informer.DriverIndex, common.VirtualizationDraPluginName) + if err != nil { + return nil, err + } + var slices []resourcev1beta1.ResourceSlice + for _, obj := range slicesObj { + slice, ok := obj.(resourcev1beta1.ResourceSlice) + if !ok { + return nil, fmt.Errorf("unexpected type of resource slice: %T", obj) + } + slices = append(slices, *slice.DeepCopy()) + } + return slices, nil +} + +// TODO: refactor me. only one pod supports now +func (c *Controller) getReservedFor(rc *resourcev1beta1.ResourceClaim) (*corev1.Pod, error) { + for _, rFor := range rc.Status.ReservedFor { + if rFor.Resource == "pods" { + pod, err := c.getPod(rFor.Name, rc.Namespace) + if err != nil { + return nil, err + } + if pod == nil || pod.GetUID() != rFor.UID { + return nil, nil + } + return pod, nil + } + } + return nil, nil +} + +func (c *Controller) podOnMyNode(pod *corev1.Pod) bool { + return pod.Spec.NodeName == c.nodeName +} + +func (c *Controller) getAllocationDevices(rc *resourcev1beta1.ResourceClaim) ([]resourcev1beta1.Device, []resourcev1beta1.Device, error) { + if rc.Status.Allocation == nil { + return nil, nil, fmt.Errorf("resource claim %s/%s has no allocation", rc.Namespace, rc.Name) + } + + virtualizationDraSlices, err := c.getVirtualizationDraResourceSlices() + if err != nil { + return nil, nil, err + } + + byPoolSlices := make(map[string][]resourcev1beta1.ResourceSlice) + for _, slice := range virtualizationDraSlices { + byPoolSlices[slice.Spec.Pool.Name] = append(byPoolSlices[slice.Spec.Pool.Name], slice) + } + + allocResultsByPool := make(map[string]map[string]resourcev1beta1.DeviceRequestAllocationResult) + + for _, status := range rc.Status.Allocation.Devices.Results { + if status.Driver != common.VirtualizationDraPluginName { + continue + } + // now, driver virtualization-dra supports only usb, but we can add more devices later + // so we need to check if the device is usb + if strings.HasPrefix(status.Device, "usb") { + continue + } + + allocResultsByPool[status.Pool][status.Device] = status + } + + var myDevices []resourcev1beta1.Device + var otherDevices []resourcev1beta1.Device + + for pool, allocResultsByDevice := range allocResultsByPool { + slices, ok := byPoolSlices[pool] + if !ok { + return nil, nil, fmt.Errorf("no resource slices found for pool %s", pool) + } + + for _, slice := range slices { + for _, device := range slice.Spec.Devices { + allocResult, ok := allocResultsByDevice[device.Name] + if !ok { + continue + } + + // virtualization-dra creates slices with pool name by node name + if allocResult.Pool == c.nodeName { + myDevices = append(myDevices, device) + } else { + otherDevices = append(otherDevices, device) + } + } + } + } + + return myDevices, otherDevices, nil +} diff --git a/images/virtualization-dra/internal/usb-gateway/informer/informer.go b/images/virtualization-dra/internal/usb-gateway/informer/informer.go new file mode 100644 index 0000000000..3d32df4fb6 --- /dev/null +++ b/images/virtualization-dra/internal/usb-gateway/informer/informer.go @@ -0,0 +1,150 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package informer + +import ( + "log/slog" + "math/rand/v2" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + resourcev1beta1 "k8s.io/api/resource/v1beta1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" +) + +const ( + NodeIndex = "node" + PoolIndex = "pool" + DriverIndex = "driver" +) + +func NewFactory(clientSet *kubernetes.Clientset, resync *time.Duration) *Factory { + var defaultResync time.Duration + if resync != nil { + defaultResync = *resync + } else { + defaultResync = resyncPeriod(12 * time.Hour) + } + + return &Factory{ + clientSet: clientSet, + defaultResync: defaultResync, + informers: make(map[string]cache.SharedIndexInformer), + } +} + +type Factory struct { + clientSet *kubernetes.Clientset + defaultResync time.Duration + + informers map[string]cache.SharedIndexInformer + startedInformers map[string]struct{} + mu sync.Mutex +} + +func (f *Factory) Start(stopCh <-chan struct{}) { + f.mu.Lock() + defer f.mu.Unlock() + + for name, informer := range f.informers { + if _, found := f.startedInformers[name]; found { + // skip informers that have already started. + slog.Info("SKIPPING informer", slog.String("name", name)) + continue + } + slog.Info("STARTING informer", slog.String("name", name)) + go informer.Run(stopCh) + f.startedInformers[name] = struct{}{} + } +} + +func (f *Factory) WaitForCacheSync(stopCh <-chan struct{}) { + var syncs []cache.InformerSynced + + f.mu.Lock() + for name, informer := range f.informers { + slog.Info("Waiting for cache sync of informer", slog.String("name", name)) + syncs = append(syncs, informer.HasSynced) + } + f.mu.Unlock() + + cache.WaitForCacheSync(stopCh, syncs...) +} + +func (f *Factory) ResourceClaim() cache.SharedIndexInformer { + return f.getInformer("resourceClaimInformer", func() cache.SharedIndexInformer { + lw := cache.NewListWatchFromClient(f.clientSet.ResourceV1beta1().RESTClient(), "resourceclaims", corev1.NamespaceAll, fields.Everything()) + return cache.NewSharedIndexInformer(lw, &resourcev1beta1.ResourceClaim{}, f.defaultResync, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + }) +} + +func (f *Factory) ResourceSlice() cache.SharedIndexInformer { + return f.getInformer("resourceSliceInformer", func() cache.SharedIndexInformer { + lw := cache.NewListWatchFromClient(f.clientSet.ResourceV1beta1().RESTClient(), "resourceslices", corev1.NamespaceAll, fields.Everything()) + return cache.NewSharedIndexInformer(lw, &resourcev1beta1.ResourceSlice{}, f.defaultResync, cache.Indexers{ + PoolIndex: func(obj interface{}) ([]string, error) { + return []string{obj.(*resourcev1beta1.ResourceSlice).Spec.Pool.Name}, nil + }, + DriverIndex: func(obj interface{}) ([]string, error) { + return []string{obj.(*resourcev1beta1.ResourceSlice).Spec.Driver}, nil + }, + }) + }) +} + +func (f *Factory) Nodes() cache.SharedIndexInformer { + return f.getInformer("nodesInformer", func() cache.SharedIndexInformer { + lw := cache.NewListWatchFromClient(f.clientSet.CoreV1().RESTClient(), "nodes", corev1.NamespaceAll, fields.Everything()) + return cache.NewSharedIndexInformer(lw, &corev1.Node{}, f.defaultResync, cache.Indexers{}) + }) +} + +func (f *Factory) Pods() cache.SharedIndexInformer { + return f.getInformer("podsInformer", func() cache.SharedIndexInformer { + lw := cache.NewListWatchFromClient(f.clientSet.CoreV1().RESTClient(), "pods", corev1.NamespaceAll, fields.Everything()) + return cache.NewSharedIndexInformer(lw, &corev1.Pod{}, f.defaultResync, cache.Indexers{ + cache.NamespaceIndex: cache.MetaNamespaceIndexFunc, + NodeIndex: func(obj interface{}) ([]string, error) { + return []string{obj.(*corev1.Pod).Spec.NodeName}, nil + }, + }) + }) +} + +func (f *Factory) getInformer(key string, newFunc func() cache.SharedIndexInformer) cache.SharedIndexInformer { + f.mu.Lock() + defer f.mu.Unlock() + + informer, ok := f.informers[key] + if ok { + return informer + } + + informer = newFunc() + f.informers[key] = informer + + return informer +} + +// resyncPeriod computes the time interval a shared informer waits before resyncing with the api server +func resyncPeriod(minResyncPeriod time.Duration) time.Duration { + factor := rand.Float64() + 1 + return time.Duration(float64(minResyncPeriod.Nanoseconds()) * factor) +} diff --git a/images/virtualization-dra/internal/usb-gateway/labeler/labeler.go b/images/virtualization-dra/internal/usb-gateway/labeler/labeler.go new file mode 100644 index 0000000000..2e36e3fdfc --- /dev/null +++ b/images/virtualization-dra/internal/usb-gateway/labeler/labeler.go @@ -0,0 +1,83 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package labeler + +import ( + "context" + "encoding/json" + "fmt" + "maps" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" +) + +type Labeler interface { + Label(ctx context.Context, name, namespace string, labels map[string]string) error +} + +type genericLabeler struct { + client dynamic.Interface + gvr schema.GroupVersionResource +} + +func NewGenericLabeler(client dynamic.Interface, gvr schema.GroupVersionResource) Labeler { + return &genericLabeler{ + client: client, + gvr: gvr, + } +} + +func (l *genericLabeler) Label(ctx context.Context, name, namespace string, newLabels map[string]string) error { + obj, err := l.client.Resource(l.gvr).Namespace(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return err + } + + labels := obj.GetLabels() + maps.Copy(labels, newLabels) + + value, err := json.Marshal(labels) + if err != nil { + return err + } + + patch := []byte(fmt.Sprintf("[{'op': 'replace', 'path': '/metadata/labels', 'value': %s}]", value)) + + _, err = l.client.Resource(l.gvr).Namespace(namespace).Patch(ctx, name, types.JSONPatchType, patch, metav1.PatchOptions{}) + return err +} + +type NodeLabeler struct { + generic Labeler +} + +func NewNodeLabeler(client dynamic.Interface) NodeLabeler { + return NodeLabeler{ + generic: NewGenericLabeler(client, schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "nodes", + }), + } +} + +func (l NodeLabeler) Label(ctx context.Context, name, namespace string, newLabels map[string]string) error { + return l.generic.Label(ctx, name, namespace, newLabels) +} diff --git a/images/virtualization-dra/internal/usb-gateway/prepare/labels.go b/images/virtualization-dra/internal/usb-gateway/prepare/labels.go new file mode 100644 index 0000000000..904c288f85 --- /dev/null +++ b/images/virtualization-dra/internal/usb-gateway/prepare/labels.go @@ -0,0 +1,32 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prepare + +import ( + "context" + + "k8s.io/client-go/dynamic" + + "github.com/deckhouse/virtualization-dra/internal/common" + "github.com/deckhouse/virtualization-dra/internal/usb-gateway/labeler" +) + +func MarkNodeForUSBGateway(ctx context.Context, nodeName string, dynamicClient dynamic.Interface) error { + return labeler.NewNodeLabeler(dynamicClient).Label(ctx, nodeName, "", map[string]string{ + common.USBGatewayLabel: "true", + }) +} diff --git a/images/virtualization-dra/internal/usb-gateway/tlsproxy/proxy.go b/images/virtualization-dra/internal/usb-gateway/tlsproxy/proxy.go new file mode 100644 index 0000000000..c502bde742 --- /dev/null +++ b/images/virtualization-dra/internal/usb-gateway/tlsproxy/proxy.go @@ -0,0 +1,97 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tlsproxy + +import ( + "context" + "crypto/tls" + "io" + "log/slog" + "net" + "strconv" +) + +type TLSProxy struct { + tlsConfig *tls.Config + port int +} + +func NewTLSProxy(tlsConfig *tls.Config, port int) *TLSProxy { + return &TLSProxy{ + tlsConfig: tlsConfig, + port: port, + } +} + +func (p *TLSProxy) Start(ctx context.Context, plainConn net.Conn) error { + listener, err := tls.Listen("tcp", net.JoinHostPort("", strconv.Itoa(p.port)), p.tlsConfig) + if err != nil { + return err + } + + go func() { + defer listener.Close() + defer plainConn.Close() + + tlsConn, err := listener.Accept() + if err != nil { + select { + case <-ctx.Done(): + return + default: + slog.Error("accept error", slog.Any("error", err)) + return + } + } + defer tlsConn.Close() + + done := make(chan struct{}, 2) + + // TLS -> plain + go func() { + _, err = io.Copy(plainConn, tlsConn) + if err != nil { + slog.Error("copy error from TLS to plain", + slog.Any("error", err), + slog.String("tlsRemoteAddr", tlsConn.RemoteAddr().String()), + slog.String("proxyRemoteAddr", plainConn.RemoteAddr().String()), + ) + } + done <- struct{}{} + }() + + // plain -> TLS + go func() { + _, err = io.Copy(tlsConn, plainConn) + if err != nil { + slog.Error("copy error from plain to TLS", + slog.Any("error", err), + slog.String("tlsRemoteAddr", tlsConn.RemoteAddr().String()), + slog.String("proxyRemoteAddr", plainConn.RemoteAddr().String()), + ) + } + done <- struct{}{} + }() + + select { + case <-ctx.Done(): + case <-done: + } + }() + + return nil +} diff --git a/images/virtualization-dra/internal/usb/convert.go b/images/virtualization-dra/internal/usb/convert.go index 95f4cb65aa..8cbeb53f9a 100644 --- a/images/virtualization-dra/internal/usb/convert.go +++ b/images/virtualization-dra/internal/usb/convert.go @@ -17,16 +17,27 @@ limitations under the License. package usb import ( + corev1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1" "k8s.io/utils/ptr" + + "github.com/deckhouse/virtualization-dra/internal/common" + "github.com/deckhouse/virtualization-dra/internal/featuregates" ) -func convertToAPIDevice(usbDevice Device) *resourceapi.Device { - return &resourceapi.Device{ - Name: usbDevice.GetName(), +func convertToAPIDevice(usbDevice Device, nodeName string) *resourceapi.Device { + name := usbDevice.GetName(nodeName) + device := &resourceapi.Device{ + Name: name, Attributes: map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{ "name": { - StringValue: ptr.To(usbDevice.Name), + StringValue: ptr.To(name), + }, + "path": { + StringValue: ptr.To(usbDevice.Path), + }, + "busID": { + StringValue: ptr.To(usbDevice.BusID), }, "manufacturer": { StringValue: ptr.To(usbDevice.Manufacturer), @@ -69,4 +80,26 @@ func convertToAPIDevice(usbDevice Device) *resourceapi.Device { }, }, } + + if featuregates.Default().USBGatewayEnabled() { + device.NodeSelector = &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: common.USBGatewayLabel, + Operator: corev1.NodeSelectorOpExists, + Values: []string{"true"}, + }, + }, + }, + }, + } + // TODO: add support for multiple allocations + // device.AllowMultipleAllocations = ptr.To(true) + } else { + device.NodeName = ptr.To(nodeName) + } + + return device } diff --git a/images/virtualization-dra/internal/usb/device.go b/images/virtualization-dra/internal/usb/device.go index 534a9d3465..0a53c8d303 100644 --- a/images/virtualization-dra/internal/usb/device.go +++ b/images/virtualization-dra/internal/usb/device.go @@ -17,15 +17,12 @@ limitations under the License. package usb import ( - "bufio" "fmt" - "log/slog" - "os" - "path/filepath" "strconv" "strings" "github.com/deckhouse/virtualization-dra/pkg/set" + "github.com/deckhouse/virtualization-dra/pkg/usb" ) type DeviceSet = set.Set[Device] @@ -35,7 +32,8 @@ func NewDeviceSet() *DeviceSet { } type Device struct { - Name string + Path string + BusID string Manufacturer string Product string VendorID int4x @@ -49,13 +47,16 @@ type Device struct { DevicePath string } -func (d *Device) GetName() string { - // usb---- +func (d *Device) GetName(nodeName string) string { + // usb----- // usb-003-005-e39-f100 - return fmt.Sprintf("usb-%s-%s-%s-%s", d.Bus.String(), d.DeviceNumber.String(), d.VendorID.String(), d.ProductID.String()) + return fmt.Sprintf("usb-%s-%s-%s-%s-%s", d.Bus.String(), d.DeviceNumber.String(), d.VendorID.String(), d.ProductID.String(), nodeName) } func (d *Device) Validate() error { + if d.BusID == "" { + return fmt.Errorf("BusID is required") + } if d.VendorID == 0 { return fmt.Errorf("VendorID is required") } @@ -80,153 +81,22 @@ func (d *Device) Validate() error { return nil } -func LoadDevice(path string) (device Device, err error) { - if err = parseSysUeventFile(path, &device); err != nil { - return - } - if err = parseSerial(path, &device); err != nil { - return - } - if err = parseManufacturer(path, &device); err != nil { - return - } - if err = parseProduct(path, &device); err != nil { - return - } - return -} - -func parseSysUeventFile(path string, device *Device) error { - // Example uevent file: - // MAJOR=189 - // MINOR=257 - // DEVNAME=bus/usb/003/002 - // DEVTYPE=usb_device - // DRIVER=usb - // PRODUCT=e39/f100/35d - // TYPE=0/0/0 - // BUSNUM=003 - // DEVNUM=002 - file, err := os.Open(filepath.Join(path, "uevent")) - if err != nil { - return fmt.Errorf("unable to open the file %s: %w", path, err) - } - defer file.Close() - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - values := strings.Split(line, "=") - if len(values) != 2 { - slog.Info("Skipping %s due not being key=value", slog.String("line", line)) - continue - } - switch values[0] { - case "MAJOR": - val, err := strconv.ParseInt(values[1], 10, 32) - if err != nil { - slog.Error("Failed to parse MAJOR", slog.String("value", values[1]), slog.Any("err", err)) - return nil - } - device.Major = int(val) - case "MINOR": - val, err := strconv.ParseInt(values[1], 10, 32) - if err != nil { - slog.Error("Failed to parse MINOR", slog.String("value", values[1]), slog.Any("err", err)) - return nil - } - device.Minor = int(val) - case "BUSNUM": - val, err := strconv.ParseInt(values[1], 10, 32) - if err != nil { - slog.Error("Failed to parse BUSNUM", slog.String("value", values[1]), slog.Any("err", err)) - return nil - } - device.Bus = int3d(val) - case "DEVNUM": - val, err := strconv.ParseInt(values[1], 10, 32) - if err != nil { - slog.Error("Failed to parse DEVNUM", slog.String("value", values[1]), slog.Any("err", err)) - return nil - } - device.DeviceNumber = int3d(val) - case "PRODUCT": - products := strings.Split(values[1], "/") - if len(products) != 3 { - slog.Error("Failed to parse PRODUCT", slog.String("value", values[1]), slog.Any("err", err)) - return nil - } - - val, err := strconv.ParseInt(products[0], 16, 32) - if err != nil { - slog.Error("Failed to parse PRODUCT", slog.String("value", values[1]), slog.Any("err", err)) - return nil - } - device.VendorID = int4x(val) - - val, err = strconv.ParseInt(products[1], 16, 32) - if err != nil { - slog.Error("Failed to parse PRODUCT", slog.String("value", values[1]), slog.Any("err", err)) - return nil - } - device.ProductID = int4x(val) - - val, err = strconv.ParseInt(products[2], 16, 32) - if err != nil { - slog.Error("Failed to parse PRODUCT", slog.String("value", values[1]), slog.Any("err", err)) - return nil - } - device.BCD = int4x(val) - case "DEVNAME": - device.DevicePath = filepath.Join("/dev", values[1]) - default: - slog.Info("Skipping unhandled line", slog.String("line", line)) - } - } - return nil -} - -func parseSerial(path string, device *Device) error { - b, err := os.ReadFile(filepath.Join(path, "serial")) - if err != nil { - return err - } - lines := strings.Split(string(b), "\n") - if len(lines) >= 1 { - device.Serial = strings.TrimSpace(lines[0]) - } else { - device.Serial = "unknown" +func toDevice(device *usb.Device) Device { + return Device{ + Path: device.Path, + BusID: device.BusID, + Manufacturer: device.Manufacturer, + Product: device.Product, + VendorID: int4x(device.VendorID), + ProductID: int4x(device.ProductID), + BCD: int4x(device.BCD), + Bus: int3d(device.Bus), + DeviceNumber: int3d(device.DeviceNumber), + Major: int(device.Major), + Minor: int(device.Minor), + Serial: device.Serial, + DevicePath: device.DevicePath, } - - return nil -} - -func parseManufacturer(path string, device *Device) error { - b, err := os.ReadFile(filepath.Join(path, "manufacturer")) - if err != nil { - return err - } - lines := strings.Split(string(b), "\n") - if len(lines) >= 1 { - device.Manufacturer = strings.TrimSpace(lines[0]) - } else { - device.Manufacturer = "unknown" - } - return nil -} - -func parseProduct(path string, device *Device) error { - b, err := os.ReadFile(filepath.Join(path, "product")) - if err != nil { - return err - } - lines := strings.Split(string(b), "\n") - if len(lines) >= 1 { - device.Product = strings.TrimSpace(lines[0]) - } else { - device.Product = "unknown" - } - return nil } type int4x int diff --git a/images/virtualization-dra/internal/usb/discovery.go b/images/virtualization-dra/internal/usb/discovery.go index 304ea83d88..28b72771d3 100644 --- a/images/virtualization-dra/internal/usb/discovery.go +++ b/images/virtualization-dra/internal/usb/discovery.go @@ -17,46 +17,20 @@ limitations under the License. package usb import ( - "fmt" - "log/slog" - "os" - "path/filepath" - "strings" + "github.com/deckhouse/virtualization-dra/pkg/usb" ) -const PathToUSBDevices = "/sys/bus/usb/devices" +const PathToUSBDevices = usb.PathToUSBDevices func discoverPluggedUSBDevices(pathToUSBDevices string) (*DeviceSet, error) { - usbDeviceSet := NewDeviceSet() - err := filepath.Walk(pathToUSBDevices, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - // Ignore named usb controllers - if strings.HasPrefix(info.Name(), "usb") { - return nil - } - // We are interested in actual USB devices information that - // contains idVendor and idProduct. We can skip all others. - if _, err := os.Stat(filepath.Join(path, "idVendor")); err != nil { - return nil - } - - // Get device information - device, err := LoadDevice(path) - if err = device.Validate(); err != nil { - slog.Error("failed to validate device, skip...", slog.Any("device", device), slog.String("error", err.Error())) - return nil - } - if err != nil { - return err - } - usbDeviceSet.Add(device) - return nil - }) - + devices, err := usb.DiscoverPluggedUSBDevices(pathToUSBDevices) if err != nil { - return nil, fmt.Errorf("failed when walking usb devices tree: %w", err) + return nil, err } + usbDeviceSet := NewDeviceSet() + for _, device := range devices { + usbDeviceSet.Add(toDevice(device)) + } + return usbDeviceSet, nil } diff --git a/images/virtualization-dra/internal/usb/store.go b/images/virtualization-dra/internal/usb/store.go index 5ae5d2b50d..aa658170d5 100644 --- a/images/virtualization-dra/internal/usb/store.go +++ b/images/virtualization-dra/internal/usb/store.go @@ -20,18 +20,24 @@ import ( "context" "fmt" "log/slog" + "strconv" "strings" "sync" "time" "github.com/containerd/nri/pkg/api" resourceapi "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" drapbv1 "k8s.io/kubelet/pkg/apis/dra/v1beta1" "k8s.io/utils/ptr" cdiapi "tags.cncf.io/container-device-interface/pkg/cdi" cdispec "tags.cncf.io/container-device-interface/specs-go" + vdraapi "github.com/deckhouse/virtualization-dra/api" + "github.com/deckhouse/virtualization-dra/internal/common" + "github.com/deckhouse/virtualization-dra/internal/featuregates" + "github.com/deckhouse/virtualization-dra/internal/cdi" "github.com/deckhouse/virtualization-dra/pkg/set" ) @@ -103,7 +109,7 @@ func (s *AllocationStore) sync() error { allocatableDevices := make([]resourceapi.Device, discoverPluggedUSBDevices.Len()) for i, usbDevice := range discoverPluggedUSBDevices.Slice() { - allocatableDevices[i] = *convertToAPIDevice(usbDevice) + allocatableDevices[i] = *convertToAPIDevice(usbDevice, s.nodeName) } allocatableDevicesByName := make(map[string]resourceapi.Device, len(allocatableDevices)) @@ -183,7 +189,12 @@ func (s *AllocationStore) Prepare(_ context.Context, claim *resourceapi.Resource return nil, fmt.Errorf("device %v is already allocated", result.Device) } - edits, err := s.makeContainerEdits(claimUID, &usbDevice) + containerEditsOptions, err := newContainerEditsOptions(&usbDevice, claim) + if err != nil { + return nil, err + } + + edits, err := s.makeContainerEdits(claimUID, containerEditsOptions) if err != nil { return nil, err } @@ -213,75 +224,114 @@ func (s *AllocationStore) Prepare(_ context.Context, claim *resourceapi.Resource return devices, nil } -// TODO: refactor me -func (s *AllocationStore) makeContainerEdits(claimUID string, device *resourceapi.Device) (*cdiapi.ContainerEdits, error) { - var ( - devicePath string - deviceNum string - bus string - major int64 - minor int64 - ) - - if attr, ok := device.Attributes["devicePath"]; ok { - if val := attr.StringValue; val != nil { - devicePath = *val - } else { - return nil, fmt.Errorf("devicePath attribute is not exist") - } +func newContainerEditsOptions(device *resourceapi.Device, claim *resourceapi.ResourceClaim) (containerEditsOptions, error) { + opts := containerEditsOptions{ + Name: device.Name, } - if attr, ok := device.Attributes["deviceNumber"]; ok { - if val := attr.StringValue; val != nil { - deviceNum = *val - } else { - return nil, fmt.Errorf("deviceNum attribute is not exist") + if featuregates.Default().USBGatewayEnabled() && isUSBGateway(claim) { + var data *runtime.RawExtension + for _, deviceStatus := range claim.Status.Devices { + if deviceStatus.Device == device.Name { + data = deviceStatus.Data + break + } + } + if data == nil { + return opts, fmt.Errorf("device status data is not found") } - } - if attr, ok := device.Attributes["bus"]; ok { - if val := attr.StringValue; val != nil { - bus = *val - } else { - return nil, fmt.Errorf("bus attribute is not exist") + usbGatewayStatus, ok := data.Object.(*vdraapi.USBGatewayStatus) + if !ok { + return opts, fmt.Errorf("device status data is not a USBGatewayStatus") + } + if usbGatewayStatus == nil { + return opts, fmt.Errorf("device status data Object is nil") + } + + opts.Bus = strconv.Itoa(usbGatewayStatus.BusNum) + opts.DeviceNum = strconv.Itoa(usbGatewayStatus.DeviceNum) + opts.DevicePath = usbGatewayStatus.DevicePath + + } else { + if attr, ok := device.Attributes["devicePath"]; ok { + if val := attr.StringValue; val != nil { + opts.DevicePath = *val + } else { + return opts, fmt.Errorf("devicePath attribute is not exist") + } + } + + if attr, ok := device.Attributes["deviceNumber"]; ok { + if val := attr.StringValue; val != nil { + opts.DeviceNum = *val + } else { + return opts, fmt.Errorf("deviceNum attribute is not exist") + } + } + + if attr, ok := device.Attributes["bus"]; ok { + if val := attr.StringValue; val != nil { + opts.Bus = *val + } else { + return opts, fmt.Errorf("bus attribute is not exist") + } } } if attr, ok := device.Attributes["major"]; ok { if val := attr.IntValue; val != nil { - major = *val + opts.Major = *val } else { - return nil, fmt.Errorf("major attribute is not exist") + return opts, fmt.Errorf("major attribute is not exist") } } if attr, ok := device.Attributes["minor"]; ok { if val := attr.IntValue; val != nil { - minor = *val + opts.Minor = *val } else { - return nil, fmt.Errorf("minor attribute is not exist") + return opts, fmt.Errorf("minor attribute is not exist") } } + return opts, nil +} + +func isUSBGateway(claim *resourceapi.ResourceClaim) bool { + return claim.Annotations[common.USBGatewayAnnotation] == "true" +} + +type containerEditsOptions struct { + Name string + DevicePath string + DeviceNum string + Bus string + Major int64 + Minor int64 +} + +// TODO: refactor me +func (s *AllocationStore) makeContainerEdits(claimUID string, opts containerEditsOptions) (*cdiapi.ContainerEdits, error) { claimUIDUpper := strings.ToUpper(claimUID) - deviceNameUpper := strings.ToUpper(device.Name) + deviceNameUpper := strings.ToUpper(opts.Name) edits := &cdiapi.ContainerEdits{ ContainerEdits: &cdispec.ContainerEdits{ Env: []string{ fmt.Sprintf("DRA_USB_CLAIM_UID_%s=%s", claimUIDUpper, claimUID), - fmt.Sprintf("DRA_USB_DEVICE_NAME_%s=%s", deviceNameUpper, device.Name), - fmt.Sprintf("DRA_USB_CLAIM_UID_%s_DEVICE_NAME=%s", claimUIDUpper, device.Name), - fmt.Sprintf("DRA_USB_%s_DEVICE_PATH=%s", deviceNameUpper, devicePath), - fmt.Sprintf("DRA_USB_%s_BUS_DEVICENUMBER=%s:%s", deviceNameUpper, bus, deviceNum), + fmt.Sprintf("DRA_USB_DEVICE_NAME_%s=%s", deviceNameUpper, opts.Name), + fmt.Sprintf("DRA_USB_CLAIM_UID_%s_DEVICE_NAME=%s", claimUIDUpper, opts.Name), + fmt.Sprintf("DRA_USB_%s_DEVICE_PATH=%s", deviceNameUpper, opts.DevicePath), + fmt.Sprintf("DRA_USB_%s_BUS_DEVICENUMBER=%s:%s", deviceNameUpper, opts.Bus, opts.DeviceNum), }, DeviceNodes: []*cdispec.DeviceNode{ { - Path: devicePath, - HostPath: devicePath, + Path: opts.DevicePath, + HostPath: opts.DevicePath, Type: "c", - Major: major, - Minor: minor, + Major: opts.Major, + Minor: opts.Minor, Permissions: "mrw", UID: ptr.To(uint32(107)), // qemu user. TODO: make this configurable GID: ptr.To(uint32(107)), // qemu group. TODO: make this configurable diff --git a/images/virtualization-dra/internal/usbip/attacher.go b/images/virtualization-dra/internal/usbip/attacher.go new file mode 100644 index 0000000000..c23f24d1c9 --- /dev/null +++ b/images/virtualization-dra/internal/usbip/attacher.go @@ -0,0 +1,302 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usbip + +import ( + "fmt" + "log/slog" + "net" + "os" + "strconv" + "syscall" + + "github.com/deckhouse/virtualization-dra/internal/usbip/protocol" + "github.com/deckhouse/virtualization-dra/pkg/usb" +) + +func NewUSBAttacher() USBAttacher { + return &usbAttacher{} +} + +type usbAttacher struct{} + +// https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_attach.c#L174 +func (a usbAttacher) Attach(host, busID string, port int) error { + conn, err := a.usbipNetTCPConnect(host, fmt.Sprintf("%d", port)) + if err != nil { + return fmt.Errorf("failed to connect to usbipd: %w", err) + } + + rhport, err := a.queryImportDevice(conn, busID) + if err != nil { + return fmt.Errorf("failed to query import device: %w", err) + } + + err = a.recordConnection(host, strconv.Itoa(port), busID, rhport) + if err != nil { + return fmt.Errorf("failed to record connection: %w", err) + } + + return nil +} + +// https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_detach.c#L32 +func (a usbAttacher) Detach(port int) error { + driver, err := newVhciDriver() + if err != nil { + return fmt.Errorf("failed to get vhci driver: %w", err) + } + + found := false + for i := 0; i < driver.nports; i++ { + idev := &driver.idevs[i] + + if idev.port == port { + found = true + vstatus := protocol.DeviceStatus(idev.status) + if vstatus != protocol.VDeviceStatusNull { + break + } + + slog.Info("Port is already detached", slog.Int("port", port)) + return fmt.Errorf("port is already detached") + + } + } + + if !found { + slog.Error("Invalid port > maxports", slog.Int("port", port), slog.Int("maxports", driver.nports)) + return fmt.Errorf("port %d not found", port) + } + + path := vhciStatePortPath(port) + + if err = os.Remove(path); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove vhci state port file: %w", err) + } + + if err = os.RemoveAll(vhciStatePath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove vhci state path: %w", err) + } + + if err = writeSysfsAttr(vhciHcdDetach, detachAttr{port: port}); err != nil { + return fmt.Errorf("failed to write detach attribute: %w", err) + } + + slog.Info("Port detached", slog.Int("port", port)) + return nil +} + +func (a usbAttacher) GetUsedPorts() ([]int, error) { + driver, err := newVhciDriver() + if err != nil { + return nil, fmt.Errorf("failed to get vhci driver: %w", err) + } + + var ports []int + + for i := 0; i < driver.nports; i++ { + idev := &driver.idevs[i] + + vstatus := protocol.DeviceStatus(idev.status) + if vstatus == protocol.VDeviceStatusUsed { + ports = append(ports, idev.port) + } + } + + return ports, nil +} + +// https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_network.c#L261 +func (a usbAttacher) usbipNetTCPConnect(host, port string) (*net.TCPConn, error) { + tcpAddr, err := net.ResolveTCPAddr("tcp", net.JoinHostPort(host, port)) + if err != nil { + return nil, fmt.Errorf("resolve TCP address: %w", err) + } + + conn, err := net.DialTCP("tcp", nil, tcpAddr) + if err != nil { + return nil, fmt.Errorf("dial TCP: %w", err) + } + + if err := conn.SetNoDelay(true); err != nil { + conn.Close() + return nil, fmt.Errorf("set TCP_NODELAY: %w", err) + } + + if err := conn.SetKeepAlive(true); err != nil { + conn.Close() + return nil, fmt.Errorf("set keepalive: %w", err) + } + + return conn, nil +} + +// https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_attach.c#L120 +func (a usbAttacher) queryImportDevice(conn *net.TCPConn, busID string) (int, error) { + opCommon := protocol.NewOpCommon(protocol.OpReqImport, protocol.OpStatusOk) + importReq := protocol.NewImportRequest(busID) + + if err := opCommon.Encode(conn); err != nil { + return -1, fmt.Errorf("failed to encode OpCommon: %w", err) + } + + if err := importReq.Encode(conn); err != nil { + return -1, fmt.Errorf("failed to encode ImportRequest: %w", err) + } + + importReply := &protocol.ImportReply{} + if err := importReply.Decode(conn); err != nil { + return -1, fmt.Errorf("failed to decode ImportReply: %w", err) + } + + if importReply.Version != protocol.Version { + return -1, fmt.Errorf("unsupported USBIP version: %d", importReply.Version) + } + + if importReply.Status != protocol.OpStatusOk { + return -1, fmt.Errorf("reply failed: %d", importReply.Status) + } + + if importReply.USBDevice.GetBusID() != busID { + return -1, fmt.Errorf("busID mismatch: %s != %s", importReply.USBDevice.GetBusID(), busID) + } + + return a.importDevice(conn, importReply.USBDevice) +} + +// https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_attach.c#L81 +func (a usbAttacher) importDevice(conn *net.TCPConn, usbDevice protocol.USBDevice) (int, error) { + port, err := a.getFreePort(usbDevice.Speed) + if err != nil { + return -1, fmt.Errorf("failed to get free port: %w", err) + } + + sockFd, err := a.getSockFd(conn) + if err != nil { + return -1, fmt.Errorf("failed to get socket fd: %w", err) + } + + devID := getDevId(usbDevice.Busnum, usbDevice.Devnum) + + attr := attachAttr{ + port: port, + sockFd: sockFd, + devId: devID, + speed: usbDevice.Speed, + } + + err = writeSysfsAttr(vhciHcdAttach, attr) + if err != nil { + return -1, fmt.Errorf("failed to write attach attribute: %w", err) + } + + return port, nil +} + +// https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/libsrc/vhci_driver.c#L334 +func (a usbAttacher) getFreePort(speed uint32) (int, error) { + driver, err := newVhciDriver() + if err != nil { + return -1, err + } + + deviceSpeed := usb.DeviceSpeed(speed) + + for i := 0; i < driver.nports; i++ { + switch deviceSpeed { + case usb.DeviceSpeedSuper: + if driver.idevs[i].hub != hubSpeedSuper { + continue + } + break + default: + if driver.idevs[i].hub != hubSpeedHigh { + continue + } + break + } + vstatus := protocol.DeviceStatus(driver.idevs[i].status) + if vstatus == protocol.VDeviceStatusNull { + return driver.idevs[i].port, nil + + } + } + + return -1, nil +} + +func (a usbAttacher) getSockFd(conn *net.TCPConn) (int, error) { + file, err := conn.File() + if err != nil { + return -1, err + } + defer file.Close() + + fd := int(file.Fd()) + + newFd, err := syscall.Dup(fd) + if err != nil { + return -1, err + } + + return newFd, nil +} + +// https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_attach.c#L39 +func (a usbAttacher) recordConnection(host, port, busID string, rhport int) error { + err := os.MkdirAll(vhciStatePath, 0700) + if err != nil { + return fmt.Errorf("failed to create vhci state path: %w", err) + } + + path := vhciStatePortPath(rhport) + + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0700) + if err != nil { + return fmt.Errorf("failed to open vhci state port file: %w", err) + } + defer file.Close() + + value := fmt.Sprintf("%s %s %s", host, port, busID) + + _, err = file.WriteString(value) + if err != nil { + return fmt.Errorf("failed to write vhci state port file: %w", err) + } + + return nil +} + +type attachAttr struct { + port int + sockFd int + devId int + speed uint32 +} + +func (a attachAttr) Complete() string { + return fmt.Sprintf("%d %d %d %d", a.port, a.sockFd, a.devId, a.speed) +} + +type detachAttr struct { + port int +} + +func (a detachAttr) Complete() string { + return fmt.Sprintf("%d", a.port) +} diff --git a/images/virtualization-dra/internal/usbip/binder.go b/images/virtualization-dra/internal/usbip/binder.go new file mode 100644 index 0000000000..226eabc6a8 --- /dev/null +++ b/images/virtualization-dra/internal/usbip/binder.go @@ -0,0 +1,224 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usbip + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +func NewUSBBinder() USBBinder { + return &usbBinder{} +} + +type usbBinder struct{} + +// Bind binds the USB device to the USBIP server. +// https://github.com/torvalds/linux/blob/40fbbd64bba6c6e7a72885d2f59b6a3be9991eeb/tools/usb/usbip/src/usbip_bind.c#L130 +func (b *usbBinder) Bind(busID string) error { + devInfo, err := b.getUSBDeviceInfo(busID) + if err != nil { + return fmt.Errorf("device with bus ID %s does not exist: %w", busID, err) + } + + if strings.Contains(devInfo.DevPath, "vhci_hcd") { + return fmt.Errorf("bind loop detected: device %s is already attached to vhci_hcd", busID) + } + + err = b.unbindOther(devInfo) + if err != nil { + return fmt.Errorf("failed to unbind other devices: %w", err) + } + + if err = b.modifyMatchBusID(busID, true); err != nil { + return err + } + + if err = b.bindUsbip(busID); err != nil { + return fmt.Errorf("failed to bind usb device: %w: %w", err, b.modifyMatchBusID(busID, false)) + } + + return b.storeBind(busID, true) +} + +// Unbind unbinds the USB device from the USBIP server. +// https://github.com/torvalds/linux/blob/40fbbd64bba6c6e7a72885d2f59b6a3be9991eeb/tools/usb/usbip/src/usbip_unbind.c#L30 +func (b *usbBinder) Unbind(busID string) error { + devInfo, err := b.getUSBDeviceInfo(busID) + if err != nil { + return fmt.Errorf("device with bus ID %s does not exist: %w", busID, err) + } + + if devInfo.Driver != usbipHostDriverName { + return fmt.Errorf("device %s is not bound to %s driver", devInfo.BusID, usbipHostDriverName) + } + + if err = b.unbindUsbip(busID); err != nil { + return fmt.Errorf("failed to unbind usb device %s: %w", busID, err) + } + + // notify driver of unbind + if err = b.modifyMatchBusID(busID, false); err != nil { + return fmt.Errorf("failed to modify match bus ID %s: %w", busID, err) + } + + // Trigger new probing + if err = b.rebindUsbip(busID); err != nil { + return fmt.Errorf("failed to rebind usb device %s: %w", busID, err) + } + + return b.storeBind(busID, false) +} + +func (b *usbBinder) IsBound(busID string) (bool, error) { + return b.isBound(busID) +} + +type usbDeviceInfo struct { + BusID string + Driver string + DevPath string + IsHub bool +} + +func (b *usbBinder) getUSBDeviceInfo(busID string) (*usbDeviceInfo, error) { + path := getUSBDevicePath(busID) + + if _, err := os.Stat(path); err != nil { + return nil, err + } + + info := &usbDeviceInfo{ + BusID: busID, + } + + bDevClassPath := filepath.Join(path, "bDeviceClass") + if data, err := os.ReadFile(bDevClassPath); err == nil { + info.IsHub = strings.TrimSpace(string(data)) == "09" // 09 = USB Hub class + } + + driverLink := filepath.Join(path, "driver") + if link, err := os.Readlink(driverLink); err == nil { + info.Driver = filepath.Base(link) + } + + ueventPath := filepath.Join(path, "uevent") + if data, err := os.ReadFile(ueventPath); err == nil { + lines := strings.Split(string(data), "\n") + for _, line := range lines { + if strings.HasPrefix(line, "DEVNAME=") { + info.DevPath = filepath.Join("/dev", strings.TrimPrefix(line, "DEVNAME=")) + break + } + } + } + + return info, nil +} + +func (b *usbBinder) storeBind(busID string, bind bool) error { + bound, err := b.isBound(busID) + if err != nil { + return err + } + if bound == bind { + return nil + } + path := bindPath(busID) + if bind { + _, err = os.Create(path) + return err + } + return os.Remove(path) +} + +func (b *usbBinder) isBound(busID string) (bool, error) { + path := bindPath(busID) + _, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func bindPath(busID string) string { + return filepath.Join(getUSBDevicePath(busID), "usbip_bound") +} + +func (b *usbBinder) unbindOther(devInfo *usbDeviceInfo) error { + if devInfo.IsHub { + return fmt.Errorf("skip unbinding of hub %s", devInfo.BusID) + } + + if devInfo.Driver == "" { + // no driver bound to the device + return nil + } + + if devInfo.Driver == usbipHostDriverName { + return fmt.Errorf("device %s is already bound to %s", devInfo.BusID, usbipHostDriverName) + } + + unbindPath := unbindAttrPath(devInfo.Driver) + + if err := writeSysfsAttr(unbindPath, busIDAttr{busID: devInfo.BusID}); err != nil { + return fmt.Errorf("error unbinding device %s from driver %s: %w", devInfo.BusID, devInfo.Driver, err) + } + + return nil +} + +func (b *usbBinder) bindUsbip(busID string) error { + return writeSysfsAttr(bindAttrPath(usbipHostDriverName), busIDAttr{busID: busID}) +} + +func (b *usbBinder) unbindUsbip(busID string) error { + return writeSysfsAttr(unbindAttrPath(usbipHostDriverName), busIDAttr{busID: busID}) +} + +func (b *usbBinder) rebindUsbip(busID string) error { + return writeSysfsAttr(rebindAttrPath(usbipHostDriverName), busIDAttr{busID: busID}) +} + +func (b *usbBinder) modifyMatchBusID(busID string, add bool) error { + return writeSysfsAttr(matchBusIDAttrPath(usbipHostDriverName), modifyMatchBusIDAttr{busID: busID, add: add}) +} + +type modifyMatchBusIDAttr struct { + busID string + add bool +} + +func (a modifyMatchBusIDAttr) Complete() string { + if a.add { + return fmt.Sprintf("add %s", a.busID) + } + return fmt.Sprintf("del %s", a.busID) +} + +type busIDAttr struct { + busID string +} + +func (a busIDAttr) Complete() string { + return a.busID +} diff --git a/images/virtualization-dra/internal/usbip/interfaces.go b/images/virtualization-dra/internal/usbip/interfaces.go new file mode 100644 index 0000000000..17062585d6 --- /dev/null +++ b/images/virtualization-dra/internal/usbip/interfaces.go @@ -0,0 +1,80 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usbip + +type Interface interface { + ServerInterface + ClientInterface +} + +type ServerInterface interface { + USBBinder +} + +type ClientInterface interface { + USBAttacher +} + +type USBBinder interface { + Bind(busID string) error + Unbind(busID string) error + IsBound(busID string) (bool, error) +} + +type USBAttacher interface { + Attach(host, busID string, port int) error + Detach(port int) error + GetUsedPorts() ([]int, error) +} + +type serverImpl struct { + USBBinder +} + +func NewServer(binder USBBinder) ServerInterface { + return &serverImpl{USBBinder: binder} +} + +type clientImpl struct { + USBAttacher +} + +func NewClient(attacher USBAttacher) ClientInterface { + return &clientImpl{USBAttacher: attacher} +} + +type interfaceImpl struct { + ServerInterface + ClientInterface +} + +func NewInterface(server ServerInterface, client ClientInterface) Interface { + return &interfaceImpl{ + ServerInterface: server, + ClientInterface: client, + } +} + +func New() Interface { + binder := NewUSBBinder() + attacher := NewUSBAttacher() + + server := NewServer(binder) + client := NewClient(attacher) + + return NewInterface(server, client) +} diff --git a/images/virtualization-dra/internal/usbip/protocol/common.go b/images/virtualization-dra/internal/usbip/protocol/common.go new file mode 100644 index 0000000000..c1e3c5f714 --- /dev/null +++ b/images/virtualization-dra/internal/usbip/protocol/common.go @@ -0,0 +1,145 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package protocol + +import ( + "encoding/binary" + "fmt" + "io" +) + +type USBVersion uint16 + +const ( + Version USBVersion = 0x0111 +) + +type Op uint16 + +// Common header for all the kinds of PDUs. +const ( + OpRequest Op = 0x80 << 8 + OpReply Op = 0x00 << 8 +) + +// Dummy Code +const ( + OpUnspec Op = 0x00 + OpReqUnspec Op = OpRequest | OpUnspec + OpRepUnspec Op = OpReply | OpUnspec +) + +// Retrieve USB device information. (still not used) +const ( + OpDevInfo Op = 0x02 + OpReqDevInfo Op = OpRequest | OpDevInfo + OpRepDevInfo Op = OpReply | OpDevInfo +) + +// Import a remote USB device. +const ( + OpImport Op = 0x03 + OpReqImport Op = OpRequest | OpImport + OpRepImport Op = OpReply | OpImport +) + +// Negotiate IPSec encryption key. (still not used) +const ( + OpCrypkey Op = 0x04 + OpReqCrypkey Op = OpRequest | OpCrypkey + OpRepCrypkey Op = OpReply | OpCrypkey +) + +// Retrieve the list of exported USB devices. +const ( + OpDevList Op = 0x05 + OpReqDevList Op = OpRequest | OpDevList + OpRepDevList Op = OpReply | OpDevList +) + +// Export a USB device to a remote host. +const ( + OpExport Op = 0x06 + OpReqExport Op = OpRequest | OpExport + OpRepExport Op = OpReply | OpExport +) + +// un-Export a USB device from a remote host. +const ( + OpUnexport Op = 0x07 + OpReqUnexport Op = OpRequest | OpUnexport + OpRepUnexport Op = OpReply | OpUnexport +) + +type OpStatus uint32 + +const ( + OpStatusOk OpStatus = 0x00 + OpStatusNA OpStatus = 0x01 + OpStatusDevBusy OpStatus = 0x02 + OpStatusDevErr OpStatus = 0x03 + OpStatusNoDev OpStatus = 0x04 + OpStatusError OpStatus = 0x05 +) + +type DeviceStatus uint32 + +const ( + DeviceStatusAvailable DeviceStatus = iota + 0x01 + DeviceStatusUsed + DeviceStatusError + VDeviceStatusNull + VDeviceStatusNotAssigned + VDeviceStatusUsed + VDeviceStatusError +) + +func NewOpCommon(code Op, status OpStatus) *OpCommon { + return &OpCommon{ + Version: Version, + Code: code, + Status: status, + } +} + +type OpCommon struct { + Version USBVersion + Code Op + Status OpStatus +} + +func (op *OpCommon) Decode(r io.Reader) error { + buf := make([]byte, 8) + _, err := io.ReadFull(r, buf) + if err != nil { + return fmt.Errorf("failed to read OpCommon: %w", err) + } + + op.Version = USBVersion(binary.BigEndian.Uint16(buf[0:2])) + op.Code = Op(binary.BigEndian.Uint16(buf[2:4])) + op.Status = OpStatus(binary.BigEndian.Uint32(buf[4:8])) + return nil +} + +func (op *OpCommon) Encode(w io.Writer) error { + buf := make([]byte, 8) + binary.BigEndian.PutUint16(buf[0:2], uint16(op.Version)) + binary.BigEndian.PutUint16(buf[2:4], uint16(op.Code)) + binary.BigEndian.PutUint32(buf[4:8], uint32(op.Status)) + _, err := w.Write(buf) + return err +} diff --git a/images/virtualization-dra/internal/usbip/protocol/convert.go b/images/virtualization-dra/internal/usbip/protocol/convert.go new file mode 100644 index 0000000000..21d3b5ad43 --- /dev/null +++ b/images/virtualization-dra/internal/usbip/protocol/convert.go @@ -0,0 +1,52 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package protocol + +import "bytes" + +func ToDevicePath(path string) [256]byte { + var result [256]byte + writeCString(result[:], path) + return result +} + +func ToBusID(busID string) [32]byte { + var result [32]byte + writeCString(result[:], busID) + return result +} + +func fromCString(buf []byte) string { + newBytes := buf[:] + if ib := bytes.IndexByte(newBytes, 0); ib != -1 { + newBytes = newBytes[:ib] + } + return string(newBytes) +} + +func writeCString(dst []byte, s string) { + for i := range dst { + dst[i] = 0 + } + + n := len(s) + if n >= len(dst) { + n = len(dst) - 1 + } + + copy(dst[:n], s) +} diff --git a/images/virtualization-dra/internal/usbip/protocol/device_list.go b/images/virtualization-dra/internal/usbip/protocol/device_list.go new file mode 100644 index 0000000000..2f25a56fff --- /dev/null +++ b/images/virtualization-dra/internal/usbip/protocol/device_list.go @@ -0,0 +1,249 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package protocol + +import ( + "encoding/binary" + "fmt" + "io" +) + +func NewDeviceList(status OpStatus, devices []USBDeviceInfo) *DeviceList { + return &DeviceList{ + OpCommon: OpCommon{ + Version: Version, + Code: OpReqDevList, + Status: status, + }, + Ndev: uint32(len(devices)), + Devices: devices, + } +} + +type DeviceList struct { + OpCommon + + Ndev uint32 + Devices []USBDeviceInfo +} + +func (d *DeviceList) Encode(w io.Writer) error { + if err := d.OpCommon.Encode(w); err != nil { + return fmt.Errorf("failed to encode OpCommon: %w", err) + } + + buf := make([]byte, 4) + binary.BigEndian.PutUint32(buf[0:4], d.Ndev) + + if _, err := w.Write(buf); err != nil { + return fmt.Errorf("failed to write Ndev to writer: %w", err) + } + + for _, dev := range d.Devices { + if err := dev.Encode(w); err != nil { + return fmt.Errorf("failed to encode USBDeviceInfo: %w", err) + } + } + + return nil +} + +const ( + sysfsPathMax = 256 + sysfsBusIdMax = 32 +) + +type USBDeviceInfo struct { + USBDevice + Interfaces []USBDeviceInterface +} + +func (d *USBDeviceInfo) Decode(r io.Reader) error { + if err := d.USBDevice.Decode(r); err != nil { + return fmt.Errorf("unable to decode USBDevice: %w", err) + } + + d.Interfaces = make([]USBDeviceInterface, d.BNumInterfaces) + for i := 0; i < int(d.BNumInterfaces); i++ { + if err := d.Interfaces[i].Decode(r); err != nil { + return fmt.Errorf("unable to decode USBDeviceInterface: %w", err) + } + } + + return nil +} + +func (d *USBDeviceInfo) Encode(w io.Writer) error { + if err := d.USBDevice.Encode(w); err != nil { + return fmt.Errorf("unable to encode USBDevice: %w", err) + } + + for _, iface := range d.Interfaces { + if err := iface.Encode(w); err != nil { + return fmt.Errorf("unable to encode USBDeviceInterface: %w", err) + } + } + + return nil +} + +type USBDevice struct { + Path [sysfsPathMax]byte + BusID [sysfsBusIdMax]byte + + Busnum uint32 + Devnum uint32 + Speed uint32 + + IDVendor uint16 + IDProduct uint16 + BcdDevice uint16 + + BDeviceClass uint8 + BDeviceSubClass uint8 + BDeviceProtocol uint8 + BConfigurationValue uint8 + BNumConfigurations uint8 + BNumInterfaces uint8 +} + +func (u *USBDevice) GetPath() string { + return fromCString(u.Path[:]) +} + +func (u *USBDevice) GetBusID() string { + return fromCString(u.BusID[:]) +} + +func (u *USBDevice) Decode(r io.Reader) error { + buf := make([]byte, sysfsPathMax+sysfsBusIdMax+12+6+6) + _, err := io.ReadFull(r, buf) + if err != nil { + return fmt.Errorf("failed to read USBDevice from reader: %w", err) + } + + copy(u.Path[:], buf[0:sysfsPathMax]) + copy(u.BusID[:], buf[sysfsPathMax:sysfsPathMax+sysfsBusIdMax]) + + pass := sysfsPathMax + sysfsBusIdMax + + u.Busnum = binary.BigEndian.Uint32(buf[pass : pass+4]) + pass += 4 + u.Devnum = binary.BigEndian.Uint32(buf[pass : pass+4]) + pass += 4 + u.Speed = binary.BigEndian.Uint32(buf[pass : pass+4]) + pass += 4 + + u.IDVendor = binary.BigEndian.Uint16(buf[pass : pass+2]) + pass += 2 + u.IDProduct = binary.BigEndian.Uint16(buf[pass : pass+2]) + pass += 2 + u.BcdDevice = binary.BigEndian.Uint16(buf[pass : pass+2]) + pass += 2 + + u.BDeviceClass = buf[pass] + pass += 1 + u.BDeviceSubClass = buf[pass] + pass += 1 + u.BDeviceProtocol = buf[pass] + pass += 1 + u.BConfigurationValue = buf[pass] + pass += 1 + u.BNumConfigurations = buf[pass] + pass += 1 + u.BNumInterfaces = buf[pass] + + return nil +} + +func (u *USBDevice) Encode(w io.Writer) error { + buf := make([]byte, sysfsPathMax+sysfsBusIdMax+12+6+6) + + copy(buf[0:sysfsPathMax], u.Path[:]) + copy(buf[sysfsPathMax:sysfsPathMax+sysfsBusIdMax], u.BusID[:]) + + pass := sysfsPathMax + sysfsBusIdMax + + binary.BigEndian.PutUint32(buf[pass:pass+4], u.Busnum) + pass += 4 + binary.BigEndian.PutUint32(buf[pass:pass+4], u.Devnum) + pass += 4 + binary.BigEndian.PutUint32(buf[pass:pass+4], u.Speed) + pass += 4 + + binary.BigEndian.PutUint16(buf[pass:pass+2], u.IDVendor) + pass += 2 + binary.BigEndian.PutUint16(buf[pass:pass+2], u.IDProduct) + pass += 2 + binary.BigEndian.PutUint16(buf[pass:pass+2], u.BcdDevice) + pass += 2 + + buf[pass] = u.BDeviceClass + pass += 1 + buf[pass] = u.BDeviceSubClass + pass += 1 + buf[pass] = u.BDeviceProtocol + pass += 1 + buf[pass] = u.BConfigurationValue + pass += 1 + buf[pass] = u.BNumConfigurations + pass += 1 + buf[pass] = u.BNumInterfaces + + _, err := w.Write(buf) + if err != nil { + return fmt.Errorf("failed to write USBDevice to writer: %w", err) + } + return nil +} + +type USBDeviceInterface struct { + BInterfaceClass uint8 + BInterfaceSubClass uint8 + BInterfaceProtocol uint8 + padding uint8 +} + +func (u *USBDeviceInterface) Decode(r io.Reader) error { + buf := make([]byte, 4) + _, err := io.ReadFull(r, buf) + if err != nil { + return fmt.Errorf("failed to read USBDeviceInterface from reader: %w", err) + } + + u.BInterfaceClass = buf[0] + u.BInterfaceSubClass = buf[1] + u.BInterfaceProtocol = buf[2] + u.padding = buf[3] + + return nil +} + +func (u *USBDeviceInterface) Encode(w io.Writer) error { + buf := make([]byte, 4) + + buf[0] = u.BInterfaceClass + buf[1] = u.BInterfaceSubClass + buf[2] = u.BInterfaceProtocol + buf[3] = u.padding + + _, err := w.Write(buf) + if err != nil { + return fmt.Errorf("failed to write USBDeviceInterface to writer: %w", err) + } + return nil +} diff --git a/images/virtualization-dra/internal/usbip/protocol/import.go b/images/virtualization-dra/internal/usbip/protocol/import.go new file mode 100644 index 0000000000..783e659936 --- /dev/null +++ b/images/virtualization-dra/internal/usbip/protocol/import.go @@ -0,0 +1,88 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package protocol + +import ( + "fmt" + "io" +) + +func NewImportRequest(busID string) *ImportRequest { + return &ImportRequest{ + busID: ToBusID(busID), + } +} + +type ImportRequest struct { + busID [sysfsBusIdMax]byte +} + +func (i *ImportRequest) BusID() string { + return fromCString(i.busID[:]) +} + +func (i *ImportRequest) Encode(w io.Writer) error { + _, err := w.Write(i.busID[:]) + return err +} + +func (i *ImportRequest) Decode(r io.Reader) error { + buf := make([]byte, sysfsBusIdMax) + _, err := io.ReadFull(r, buf) + if err != nil { + return fmt.Errorf("failed to read ImportRequest from reader: %w", err) + } + + copy(i.busID[:], buf) + return nil +} + +type ImportReply struct { + OpCommon + USBDevice +} + +func NewImportReply(status OpStatus, device USBDevice) *ImportReply { + return &ImportReply{ + OpCommon: OpCommon{ + Version: Version, + Code: OpRepImport, + Status: status, + }, + USBDevice: device, + } +} + +func (i *ImportReply) Encode(w io.Writer) error { + if err := i.OpCommon.Encode(w); err != nil { + return fmt.Errorf("failed to encode OpCommon: %w", err) + } + if err := i.USBDevice.Encode(w); err != nil { + return fmt.Errorf("failed to encode USBDevice: %w", err) + } + return nil +} + +func (i *ImportReply) Decode(r io.Reader) error { + if err := i.OpCommon.Decode(r); err != nil { + return fmt.Errorf("failed to decode OpCommon: %w", err) + } + if err := i.USBDevice.Decode(r); err != nil { + return fmt.Errorf("failed to decode USBDevice: %w", err) + } + return nil +} diff --git a/images/virtualization-dra/internal/usbip/sysfs.go b/images/virtualization-dra/internal/usbip/sysfs.go new file mode 100644 index 0000000000..cb41335791 --- /dev/null +++ b/images/virtualization-dra/internal/usbip/sysfs.go @@ -0,0 +1,79 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usbip + +import ( + "fmt" + "os" +) + +type sysfsAttr interface { + Complete() string +} + +func writeSysfsAttr(attrPath string, value sysfsAttr) error { + f, err := os.OpenFile(attrPath, os.O_WRONLY, 0644) + if err != nil { + return err + } + defer f.Close() + + _, err = f.WriteString(value.Complete()) + return err +} + +const ( + bindAttrPathTmpl = "/sys/bus/usb/drivers/%s/bind" + unbindAttrPathTmpl = "/sys/bus/usb/drivers/%s/unbind" + rebindAttrPathTmpl = "/sys/bus/usb/drivers/%s/rebind" + matchBusIDAttrPathTmpl = "/sys/bus/usb/drivers/%s/match_busid" + + usbDevicesTmpl = "/sys/bus/usb/devices/%s" + + usbipStatusPathTmpl = "/sys/bus/usb/devices/%s/usbip_status" + usbipSockFdPathTmpl = "/sys/bus/usb/devices/%s/usbip_sockfd" + + usbipHostDriverName = "usbip-host" +) + +func getUSBDevicePath(busID string) string { + return fmt.Sprintf(usbDevicesTmpl, busID) +} + +func bindAttrPath(driver string) string { + return fmt.Sprintf(bindAttrPathTmpl, driver) +} + +func unbindAttrPath(driver string) string { + return fmt.Sprintf(unbindAttrPathTmpl, driver) +} + +func rebindAttrPath(driver string) string { + return fmt.Sprintf(rebindAttrPathTmpl, driver) +} + +func matchBusIDAttrPath(driver string) string { + return fmt.Sprintf(matchBusIDAttrPathTmpl, driver) +} + +func usbipStatusPath(busID string) string { + return fmt.Sprintf(usbipStatusPathTmpl, busID) +} + +func usbipSockFdPath(busID string) string { + return fmt.Sprintf(usbipSockFdPathTmpl, busID) +} diff --git a/images/virtualization-dra/internal/usbip/usbip.go b/images/virtualization-dra/internal/usbip/usbip.go new file mode 100644 index 0000000000..959d25ae5f --- /dev/null +++ b/images/virtualization-dra/internal/usbip/usbip.go @@ -0,0 +1,17 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usbip diff --git a/images/virtualization-dra/internal/usbip/usbipd.go b/images/virtualization-dra/internal/usbip/usbipd.go new file mode 100644 index 0000000000..055fbd13a2 --- /dev/null +++ b/images/virtualization-dra/internal/usbip/usbipd.go @@ -0,0 +1,442 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usbip + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "log/slog" + "net" + "os" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/deckhouse/virtualization-dra/internal/usbip/protocol" + "github.com/deckhouse/virtualization-dra/pkg/usb" +) + +const ( + defaultMaxTCPConnection = 100 + defaultGracefulShutdownTimeout = 30 * time.Second +) + +func makeOptions(opts ...Option) *options { + options := &options{ + maxTCPConnection: defaultMaxTCPConnection, + gracefulShutdownTimeout: defaultGracefulShutdownTimeout, + } + + for _, opt := range opts { + opt(options) + } + + return options +} + +type options struct { + tlsConfig *tls.Config + gracefulShutdownTimeout time.Duration + maxTCPConnection int +} + +type Option func(*options) + +func WithTLSConfig(tlsConfig *tls.Config) Option { + return func(o *options) { + o.tlsConfig = tlsConfig + } +} +func WithGracefulShutdownTimeout(timeout time.Duration) Option { + return func(o *options) { + o.gracefulShutdownTimeout = timeout + } +} + +func WithMaxTCPConnection(maxTCPConnection int) Option { + return func(o *options) { + o.maxTCPConnection = maxTCPConnection + } +} + +func NewUSBIPD(port int, monitor *usb.Monitor, opts ...Option) *USBIPD { + options := makeOptions(opts...) + return &USBIPD{ + addr: ":" + strconv.Itoa(port), + tlsConfig: options.tlsConfig, + gracefulShutdownTimeout: options.gracefulShutdownTimeout, + logger: slog.Default().With(slog.String("component", "usbipd")), + maxTCPConnection: options.maxTCPConnection, + monitor: monitor, + } + +} + +type USBIPD struct { + addr string + tlsConfig *tls.Config + gracefulShutdownTimeout time.Duration + logger *slog.Logger + maxTCPConnection int + + listener net.Listener + connWg sync.WaitGroup + connCount atomic.Int64 + quit chan struct{} + + monitor *usb.Monitor +} + +func (u *USBIPD) Start(ctx context.Context) error { + if err := u.setup(); err != nil { + return err + } + + u.connWg.Add(1) + go u.run(ctx) + + return nil +} + +func (u *USBIPD) Run(ctx context.Context) error { + if err := u.setup(); err != nil { + return err + } + + u.connWg.Add(1) + u.run(ctx) + + return nil +} + +func (u *USBIPD) setup() (err error) { + if u.tlsConfig != nil { + u.listener, err = tls.Listen("tcp", u.addr, u.tlsConfig) + if err != nil { + return err + } + } else { + u.listener, err = net.Listen("tcp", u.addr) + if err != nil { + return err + } + } + return nil +} + +func (u *USBIPD) run(ctx context.Context) { + var connCount atomic.Int64 + defer u.connWg.Done() + for { + conn, err := u.listener.Accept() + // Error occurred when + // 1. Connection error + // 2. The listener is closed (quit channel is closed) + if err != nil { + select { + case <-u.quit: + return + default: + u.logger.Error("unsable to accept request", slog.String("address", u.addr), slog.Any("err", err)) + } + } else { + // Check if TCP connection reached the limit specified in given config + count := connCount.Load() + if count+1 > int64(u.maxTCPConnection) { + u.logger.Error("maximum TCP connection reached, drop the connection", slog.Int64("count", count)) + conn.Close() + continue + } + + // TCP connection handler + u.connWg.Add(1) + connCount.Add(1) + go func() { + defer connCount.Add(-1) + defer u.connWg.Done() + defer conn.Close() + + u.logger.Info("new connection established", slog.String("addr", conn.RemoteAddr().String())) + keepConn, err := u.handleConnection(conn) + if err != nil { + if !errors.Is(err, io.EOF) { + u.logger.Error("failed to handle connection", slog.Any("err", err), slog.String("addr", conn.RemoteAddr().String())) + } else { + u.logger.Info("connection EOF", slog.String("addr", conn.RemoteAddr().String())) + } + } + if keepConn { + // don't handle and read from the socket. other work doing a kernel module + <-ctx.Done() + } + u.logger.Info("connection closed", slog.String("addr", conn.RemoteAddr().String())) + }() + } + } +} + +// https://docs.kernel.org/usb/usbip_protocol.html +// https://github.com/torvalds/linux/blob/9448598b22c50c8a5bb77a9103e2d49f134c9578/tools/usb/usbip/src/usbipd.c#L251 +func (u *USBIPD) handleConnection(conn net.Conn) (bool, error) { + opCommon := &protocol.OpCommon{} + if err := opCommon.Decode(conn); err != nil { + return false, fmt.Errorf("failed to decode OpCommon: %w", err) + } + + if opCommon.Version != protocol.Version { + return false, fmt.Errorf("unsupported USBIP version: %d", opCommon.Version) + } + + if opCommon.Status != protocol.OpStatusOk { + return false, fmt.Errorf("request failed: %d", opCommon.Status) + } + + switch opCommon.Code { + case protocol.OpReqDevList: + if err := u.handleDeviceList(conn); err != nil { + return false, fmt.Errorf("failed to handle OpReqDevList: %w", err) + } + case protocol.OpReqImport: + if err := u.handleImportRequest(conn); err != nil { + return false, fmt.Errorf("failed to handle OpReqImport: %w", err) + } + return true, nil + case protocol.OpReqDevInfo, protocol.OpReqCrypkey: + // nothing to do + default: + return false, fmt.Errorf("unsupported OpCommon.Code: %d", opCommon.Code) + } + + return false, nil +} + +// https://github.com/torvalds/linux/blob/9448598b22c50c8a5bb77a9103e2d49f134c9578/tools/usb/usbip/src/usbipd.c#L229 +func (u *USBIPD) handleDeviceList(conn net.Conn) error { + info := u.getUSBDeviceInfo() + if len(info) == 0 { + slog.Info("no USB devices found") + } + devList := protocol.NewDeviceList(protocol.OpStatusOk, info) + return devList.Encode(conn) +} + +// https://github.com/torvalds/linux/blob/9448598b22c50c8a5bb77a9103e2d49f134c9578/tools/usb/usbip/src/usbipd.c#L91 +func (u *USBIPD) handleImportRequest(conn net.Conn) error { + importReq := &protocol.ImportRequest{} + if err := importReq.Decode(conn); err != nil { + return fmt.Errorf("failed to decode ImportRequest: %w", err) + } + + busID := importReq.BusID() + log := u.logger.With(slog.String("busID", busID)) + log.Info("import request") + + devices := u.monitor.GetDevices() + + var bindDevice *usb.Device + for _, device := range devices { + if device.BusID == busID { + log.Info("found device for export") + bindDevice = &device + break + } + } + + status := protocol.OpStatusOk + + if bindDevice != nil { + // should set TCP_NODELAY for usbip + u.setNotDelay(conn) + + status = u.exportDevice(conn, bindDevice) + if status != protocol.OpStatusOk { + log.Error("failed to export device") + } + + } else { + // not found + status = protocol.OpStatusNoDev + } + + u.logger.Info("export device", slog.Any("device", bindDevice)) + + usbDevice := toUSBDeviceInfo(bindDevice).USBDevice + reply := protocol.NewImportReply(status, usbDevice) + + return reply.Encode(conn) +} + +// https://github.com/torvalds/linux/blob/9448598b22c50c8a5bb77a9103e2d49f134c9578/tools/usb/usbip/libsrc/usbip_host_common.c#L212 +func (u *USBIPD) exportDevice(conn net.Conn, device *usb.Device) protocol.OpStatus { + + log := u.logger.With(slog.String("busID", device.BusID)) + log.Info("export request") + + usbIpStatus, err := u.getUSBIPStatus(device) + if err != nil { + log.Error("failed to get USBIP status", slog.Any("error", err)) + return protocol.OpStatusError + } + + if usbIpStatus != protocol.DeviceStatusAvailable { + log.Info("USBIP status is not available") + switch usbIpStatus { + case protocol.DeviceStatusError: + log.Debug("USBIP status is error") + return protocol.OpStatusDevErr + case protocol.DeviceStatusUsed: + log.Debug("USBIP status is used") + return protocol.OpStatusDevBusy + default: + log.Debug("USBIP status unknown") + return protocol.OpStatusNA + } + } + + syscallConn, ok := conn.(syscall.Conn) + if !ok { + log.Error("conn does not implement syscall.Conn") + return protocol.OpStatusNA + } + + var sockFd int + rawConn, err := syscallConn.SyscallConn() + if err != nil { + log.Error("failed to get raw connection", slog.Any("error", err)) + return protocol.OpStatusNA + } + err = rawConn.Control(func(fd uintptr) { + sockFd = int(fd) + }) + if err != nil { + log.Error("failed to get socket fd", slog.Any("error", err)) + return protocol.OpStatusNA + } + + err = writeSysfsAttr(usbipSockFdPath(device.BusID), sockFdAttr{sockFd: sockFd}) + if err != nil { + log.Error("failed to write usbip_sockfd", slog.Any("error", err)) + return protocol.OpStatusNA + } + + log.Info("Connect") + + return protocol.OpStatusOk +} + +type sockFdAttr struct { + sockFd int +} + +func (a sockFdAttr) Complete() string { + return fmt.Sprintf("%d\n", a.sockFd) +} + +func (u *USBIPD) getUSBIPStatus(device *usb.Device) (protocol.DeviceStatus, error) { + statusPath := usbipStatusPath(device.BusID) + + data, err := os.ReadFile(statusPath) + if err != nil { + return 0, fmt.Errorf("failed to read %s: %w", statusPath, err) + } + + statusStr := strings.TrimSpace(string(data)) + + value, err := strconv.ParseUint(statusStr, 10, 32) + if err != nil { + return 0, fmt.Errorf("invalid status value %q: %w", statusStr, err) + } + + status := protocol.DeviceStatus(value) + + return status, nil +} + +func (u *USBIPD) setNotDelay(conn net.Conn) { + tcpConn, ok := conn.(*net.TCPConn) + if ok { + err := tcpConn.SetNoDelay(true) + if err != nil { + u.logger.Error("failed to set TCP_NODELAY", slog.String("error", err.Error())) + } + return + } + u.logger.Error("failed to cast connection to TCPConn") +} + +// TODO: check already used devices +func (u *USBIPD) getUSBDeviceInfo() []protocol.USBDeviceInfo { + devices := u.monitor.GetDevices() + + var bindDevices []protocol.USBDeviceInfo + + for _, device := range devices { + if device.Driver == usbipHostDriverName { + bindDevice := toUSBDeviceInfo(&device) + bindDevices = append(bindDevices, bindDevice) + } + } + + return bindDevices +} + +func toUSBDeviceInfo(device *usb.Device) protocol.USBDeviceInfo { + if device == nil { + return protocol.USBDeviceInfo{} + } + return protocol.USBDeviceInfo{ + USBDevice: protocol.USBDevice{ + Path: protocol.ToDevicePath(device.DevicePath), + BusID: protocol.ToBusID(device.BusID), + Busnum: device.Bus, + Devnum: device.DeviceNumber, + Speed: toSpeed(device.Speed), + IDVendor: device.VendorID, + IDProduct: device.ProductID, + BcdDevice: device.BCD, + BDeviceClass: device.BDeviceClass, + BDeviceSubClass: device.BDeviceSubClass, + BDeviceProtocol: device.BDeviceProtocol, + BConfigurationValue: device.BConfigurationValue, + BNumConfigurations: device.BNumConfigurations, + BNumInterfaces: device.BNumInterfaces, + }, + Interfaces: toInterfaces(device.Interfaces), + } +} + +func toInterfaces(interfaces []usb.DeviceInterface) []protocol.USBDeviceInterface { + result := make([]protocol.USBDeviceInterface, len(interfaces)) + for i, iface := range interfaces { + result[i] = protocol.USBDeviceInterface{ + BInterfaceClass: iface.BInterfaceClass, + BInterfaceSubClass: iface.BInterfaceSubClass, + BInterfaceProtocol: iface.BInterfaceProtocol, + } + } + return result +} + +func toSpeed(speed uint32) uint32 { + return uint32(usb.ResolveDeviceSpeed(speed)) +} diff --git a/images/virtualization-dra/internal/usbip/usbipd_config.go b/images/virtualization-dra/internal/usbip/usbipd_config.go new file mode 100644 index 0000000000..49ddadcea6 --- /dev/null +++ b/images/virtualization-dra/internal/usbip/usbipd_config.go @@ -0,0 +1,164 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usbip + +import ( + "crypto/tls" + "crypto/x509" + "flag" + "fmt" + "os" + "time" + + "github.com/deckhouse/virtualization-dra/pkg/usb" +) + +type ClientAuthType tls.ClientAuthType + +func (c *ClientAuthType) String() string { + cc := tls.ClientAuthType(*c) + return cc.String() +} + +func (c *ClientAuthType) Set(s string) error { + switch s { + case "NoClientCert": + *c = ClientAuthType(tls.NoClientCert) + case "RequestClientCert": + *c = ClientAuthType(tls.RequestClientCert) + case "RequireAnyClientCert": + *c = ClientAuthType(tls.RequireAnyClientCert) + case "VerifyClientCertIfGiven": + *c = ClientAuthType(tls.VerifyClientCertIfGiven) + case "RequireAndVerifyClientCert": + *c = ClientAuthType(tls.RequireAndVerifyClientCert) + default: + return fmt.Errorf("invalid client auth type: %s", s) + } + return nil +} + +type USBIPDConfig struct { + ServerCertificateFile string + ServerKeyFile string + + RootCAFile string + + ClientCAFile string + ClientAuthType *ClientAuthType + clientAuthType int + InsecureSkipVerify bool + + Port int + GracefulShutdownTimeout time.Duration + + Monitor *usb.Monitor +} + +func (c *USBIPDConfig) AddFlags(fs *flag.FlagSet) { + fs.IntVar(&c.Port, "usbipd-port", 0, "USBIPD port") + fs.StringVar(&c.ServerCertificateFile, "usbipd-server-certificate-file", "", "USBIPD server certificate file") + fs.StringVar(&c.ServerKeyFile, "usbipd-server-key-file", "", "USBIPD server key file") + fs.StringVar(&c.RootCAFile, "usbipd-root-ca-file", "", "USBIPD root CA file") + fs.StringVar(&c.ClientCAFile, "usbipd-client-ca-file", "", "USBIPD client CA file") + fs.Var(c.ClientAuthType, "usbipd-client-auth-type", "USBIPD client auth type") + fs.BoolVar(&c.InsecureSkipVerify, "usbipd-insecure-skip-verify", false, "USBIPD insecure skip verify") + fs.DurationVar(&c.GracefulShutdownTimeout, "usbipd-graceful-shutdown-timeout", 0, "USBIPD graceful shutdown timeout") +} + +func (c *USBIPDConfig) Validate() error { + if c.Port == 0 { + return fmt.Errorf("port is required") + } + + if c.ServerCertificateFile != "" && c.ServerKeyFile == "" { + return fmt.Errorf("server key file is required if server certificate file is provided") + } + + if c.ServerCertificateFile == "" && c.ServerKeyFile != "" { + return fmt.Errorf("server certificate file is required if server key file is provided") + } + + if c.Monitor == nil { + return fmt.Errorf("monitor is required") + } + + return nil +} + +func (c *USBIPDConfig) Complete() (*USBIPD, error) { + var opts []Option + if c.GracefulShutdownTimeout != 0 { + opts = append(opts, WithGracefulShutdownTimeout(c.GracefulShutdownTimeout)) + } + + var serverCertificate *tls.Certificate + if c.ServerCertificateFile != "" && c.ServerKeyFile != "" { + certificate, err := tls.LoadX509KeyPair(c.ServerCertificateFile, c.ServerKeyFile) + if err != nil { + return nil, err + } + serverCertificate = &certificate + } + + rootCACertPool, err := loadCAPoolFromFile(c.RootCAFile) + if err != nil { + return nil, err + } + + clientCACertPool, err := loadCAPoolFromFile(c.ClientCAFile) + if err != nil { + return nil, err + } + + if serverCertificate != nil || rootCACertPool != nil || clientCACertPool != nil { + tlsConfig := &tls.Config{ + RootCAs: rootCACertPool, + ClientCAs: clientCACertPool, + InsecureSkipVerify: c.InsecureSkipVerify, + } + if serverCertificate != nil { + tlsConfig.Certificates = []tls.Certificate{*serverCertificate} + } + if c.ClientAuthType != nil { + tlsConfig.ClientAuth = tls.ClientAuthType(*c.ClientAuthType) + } + + opts = append(opts, WithTLSConfig(tlsConfig)) + } + + return NewUSBIPD(c.Port, c.Monitor, opts...), nil + +} + +func loadCAPoolFromFile(file string) (*x509.CertPool, error) { + if file == "" { + return nil, nil + } + + caCertPool := x509.NewCertPool() + caCertPEMBlock, err := os.ReadFile(file) + if err != nil { + return nil, fmt.Errorf("failed to read CA certificate: %w", err) + } + + if !caCertPool.AppendCertsFromPEM(caCertPEMBlock) { + return nil, fmt.Errorf("failed to parse CA certificate") + } + + return caCertPool, nil +} diff --git a/images/virtualization-dra/internal/usbip/vhci.go b/images/virtualization-dra/internal/usbip/vhci.go new file mode 100644 index 0000000000..f6439f1825 --- /dev/null +++ b/images/virtualization-dra/internal/usbip/vhci.go @@ -0,0 +1,213 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usbip + +import ( + "bytes" + "fmt" + "os" + "strconv" + "strings" +) + +const ( + vhciStatePath = "/var/run/vhci_hcd" + platformPath = "/sys/devices/platform" + usbipVhciHcdNPortsPath = "/sys/devices/platform/vhci_hcd.0/nports" + + vhciHcdAttach = "/sys/devices/platform/vhci_hcd.0/attach" + vhciHcdDetach = "/sys/devices/platform/vhci_hcd.0/detach" + vhciHcdStatus = "/sys/devices/platform/vhci_hcd.0/status" + secondaryVhciHcdStatusTmpl = "/sys/devices/platform/vhci_hcd.%d/status.%d" + + vhciStatePortTmpl = "/var/run/vhci_hcd/port%d" +) + +func vhciStatePortPath(port int) string { + return fmt.Sprintf(vhciStatePortTmpl, port) +} + +func secondaryVhciHcdStatusPath(count int) string { + return fmt.Sprintf(secondaryVhciHcdStatusTmpl, count, count) +} + +type vhciDriver struct { + nports int + ncontrollers int + idevs []importDevice +} + +type importDevice struct { + hub hubSpeed + port, status, devID, busnum, devnum int +} + +type hubSpeed int + +const ( + hubSpeedHigh hubSpeed = iota + hubSpeedSuper +) + +// https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/libsrc/vhci_driver.c#L243 +func newVhciDriver() (*vhciDriver, error) { + nports, err := getNPorts() + if err != nil { + return nil, err + } + ncontrollers, err := getNControllers() + if err != nil { + return nil, err + } + + driver := &vhciDriver{ + nports: nports, + ncontrollers: ncontrollers, + } + + err = driver.refreshImportDeviceList() + if err != nil { + return nil, fmt.Errorf("failed to refresh import device list: %w", err) + } + + return driver, nil +} + +func getNPorts() (int, error) { + data, err := os.ReadFile(usbipVhciHcdNPortsPath) + if err != nil { + return -1, err + } + + nports, err := strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil { + return -1, err + } + + return nports, nil +} + +func getNControllers() (int, error) { + entries, err := os.ReadDir(platformPath) + if err != nil { + return -1, err + } + count := 0 + for _, entry := range entries { + if entry.IsDir() && strings.HasPrefix(entry.Name(), "vhci_hcd") { + count++ + } + } + + return count, nil +} + +// https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/libsrc/vhci_driver.c#L111 +func (d *vhciDriver) refreshImportDeviceList() error { + + status := vhciHcdStatus + + for i := 0; i < d.ncontrollers; i++ { + if i > 0 { + status = secondaryVhciHcdStatusPath(i) + } + + attrStatus, err := os.ReadFile(status) + if err != nil { + return fmt.Errorf("failed to read %s: %w", status, err) + } + + err = d.parseStatus(attrStatus) + if err != nil { + return fmt.Errorf("failed to parse attr status %s: %w", status, err) + } + } + + return nil +} + +// https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/libsrc/vhci_driver.c#L40 +func (d *vhciDriver) parseStatus(statusBytes []byte) error { + lines := strings.Split(string(statusBytes), "\n") + + // hub port sta spd dev sockfd local_busid + // hs 0000 004 000 00000000 000000 0-0 + // hs 0001 004 000 00000000 000000 0-0 + // hs 0002 004 000 00000000 000000 0-0 + + head := true + for _, line := range lines { + if head { + // skip header + head = false + continue + } + + if strings.TrimSpace(line) == "" { + continue + } + + var ( + hub string + port, status, speed, devID, sockFd int + localBusID string + ) + + buf := bytes.NewBufferString(line) + _, err := fmt.Fscanf(buf, "%2s %d %d %d %x %d %31s", &hub, &port, &status, &speed, &devID, &sockFd, &localBusID) + if err != nil { + return fmt.Errorf("failed to parse status: %w", err) + } + + if len(d.idevs) <= port { + idevs := make([]importDevice, port+1) + for i, idev := range d.idevs { + idevs[i] = idev + } + d.idevs = idevs + } + + busnum, devnum := getBusNumDevNum(devID) + + idev := &d.idevs[port] + + idev.port = port + idev.status = status + idev.devID = devID + idev.busnum = busnum + idev.devnum = devnum + + if hub == "hs" { + idev.hub = hubSpeedHigh + } else if hub == "ss" { + idev.hub = hubSpeedSuper + } + } + + return nil +} + +func getDevId(busnum, devnum uint32) int { + return int((busnum << 16) | devnum) +} + +func getBusNumDevNum(devID int) (int, int) { + busnum := devID >> 16 + devnum := devID & 0x0000ffff + + return busnum, devnum +} diff --git a/images/virtualization-dra/pkg/modprobe/modprobe.go b/images/virtualization-dra/pkg/modprobe/modprobe.go new file mode 100644 index 0000000000..838285a926 --- /dev/null +++ b/images/virtualization-dra/pkg/modprobe/modprobe.go @@ -0,0 +1,58 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package modprobe + +import ( + "fmt" + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +func LoadModules(modules []string) error { + var uts unix.Utsname + if err := unix.Uname(&uts); err != nil { + return fmt.Errorf("uname: %w", err) + } + + kernel := unix.ByteSliceToString(uts.Release[:]) + base := filepath.Join("/lib/modules", kernel) + + for _, m := range modules { + path := filepath.Join(base, m) + if err := loadModule(path); err != nil { + return fmt.Errorf("load module %s: %w", path, err) + } + } + + return nil +} + +func loadModule(path string) error { + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("open %s: %w", path, err) + } + defer f.Close() + + if err = unix.FinitModule(int(f.Fd()), "", 0); err != nil { + return fmt.Errorf("finit_module %s: %w", path, err) + } + + return nil +} diff --git a/images/virtualization-dra/pkg/usb/discovery.go b/images/virtualization-dra/pkg/usb/discovery.go new file mode 100644 index 0000000000..f3a7a19b4e --- /dev/null +++ b/images/virtualization-dra/pkg/usb/discovery.go @@ -0,0 +1,87 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usb + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" +) + +func DiscoverPluggedUSBDevices(pathToUSBDevices string) (map[string]*Device, error) { + devices := make(map[string]*Device) + + err := filepath.Walk(pathToUSBDevices, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if !isUsbPath(path) { + return nil + } + + // Get device information + device, err := LoadDevice(path) + if err != nil { + return err + } + if err = device.Validate(); err != nil { + slog.Error("failed to validate device, skip...", slog.Any("device", device), slog.String("error", err.Error())) + return nil + } + + devices[path] = &device + return nil + }) + + if err != nil { + return nil, fmt.Errorf("failed when walking usb devices tree: %w", err) + } + + return devices, nil +} + +func isUsbPath(path string) bool { + // Ignore named usb controllers + if strings.HasPrefix(filepath.Base(path), "usb") { + return false + } + // We are interested in actual USB devices information that + // contains idVendor and idProduct. We can skip all others. + for _, file := range requiredFiles { + if _, err := os.Stat(filepath.Join(path, file)); err != nil { + return false + } + } + + return true +} + +var requiredFiles = []string{ + "idVendor", + "idProduct", + "uevent", + "serial", + "manufacturer", + "product", + "bConfigurationValue", + "bNumConfigurations", + "bNumInterfaces", + "speed", +} diff --git a/images/virtualization-dra/pkg/usb/monitor.go b/images/virtualization-dra/pkg/usb/monitor.go new file mode 100644 index 0000000000..d5b12204e5 --- /dev/null +++ b/images/virtualization-dra/pkg/usb/monitor.go @@ -0,0 +1,180 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usb + +import ( + "context" + "fmt" + "log/slog" + "maps" + "slices" + "strings" + "sync" + "time" + + "github.com/fsnotify/fsnotify" +) + +type Monitor struct { + mu sync.RWMutex + devices map[string]*Device + watcher *fsnotify.Watcher + notifiers []Notifier +} + +func NewMonitor(ctx context.Context, resyncPeriod time.Duration) (*Monitor, error) { + devices, err := DiscoverPluggedUSBDevices(PathToUSBDevices) + if err != nil { + return nil, err + } + if devices == nil { + devices = make(map[string]*Device) + } + + // TODO: recursive watcher + watcher, err := fsnotify.NewWatcher() + if err != nil { + return nil, err + } + if err = watcher.Add(PathToUSBDevices); err != nil { + _ = watcher.Close() + return nil, fmt.Errorf("failed to add USB devices path to fsnotify watcher: %w", err) + + } + + monitor := &Monitor{ + devices: devices, + watcher: watcher, + } + + go func() { + monitor.run(ctx, resyncPeriod) + }() + + return monitor, nil +} + +func (m *Monitor) run(ctx context.Context, resyncPeriod time.Duration) { + for { + select { + case <-ctx.Done(): + _ = m.watcher.Close() + return + case event := <-m.watcher.Events: + switch event.Op { + case fsnotify.Create, fsnotify.Write: + if err := m.handleUpdate(event); err != nil { + slog.Error("failed to handle update", slog.String("error", err.Error())) + } + case fsnotify.Remove: + m.handleRemove(event) + default: + continue + } + case err := <-m.watcher.Errors: + slog.Error("error watching USB devices", slog.String("error", err.Error())) + + case <-time.After(resyncPeriod): + devices, err := DiscoverPluggedUSBDevices(PathToUSBDevices) + if err != nil { + slog.Error("failed to discover USB devices", slog.String("error", err.Error())) + continue + } + if devices == nil { + devices = make(map[string]*Device) + } + m.mu.Lock() + if !maps.Equal(m.devices, devices) { + m.devices = devices + m.notify() + } + m.mu.Unlock() + } + } +} + +func (m *Monitor) handleUpdate(event fsnotify.Event) error { + path := event.Name + if !isUsbPath(path) { + return nil + } + + // Get device information + device, err := LoadDevice(path) + if err != nil { + return err + } + if err = device.Validate(); err != nil { + slog.Error("failed to validate device, skip...", slog.Any("device", device), slog.String("error", err.Error())) + return nil + } + + m.mu.Lock() + defer m.mu.Unlock() + + oldDevice, ok := m.devices[path] + if !ok || !device.Equal(oldDevice) { + m.devices[path] = &device + m.notify() + } + + return nil +} + +func (m *Monitor) handleRemove(event fsnotify.Event) { + path := event.Name + + m.mu.Lock() + defer m.mu.Unlock() + + if _, ok := m.devices[path]; ok { + delete(m.devices, path) + m.notify() + } +} + +func (m *Monitor) GetDevices() []Device { + m.mu.RLock() + devices := make([]Device, 0, len(m.devices)) + for _, device := range m.devices { + devices = append(devices, *device) + } + m.mu.RUnlock() + + slices.SortFunc(devices, func(a, b Device) int { + return strings.Compare(a.DevicePath, b.DevicePath) + }) + + return devices +} + +func (m *Monitor) AddNotifier(notifier Notifier) { + m.mu.Lock() + defer m.mu.Unlock() + + m.notifiers = append(m.notifiers, notifier) +} + +func (m *Monitor) notify() { + for _, notifier := range m.notifiers { + notifier.Notify() + } +} + +type Notifier interface { + Notify() +} diff --git a/images/virtualization-dra/pkg/usb/speed.go b/images/virtualization-dra/pkg/usb/speed.go new file mode 100644 index 0000000000..874d45b72e --- /dev/null +++ b/images/virtualization-dra/pkg/usb/speed.go @@ -0,0 +1,46 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usb + +type DeviceSpeed uint32 + +const ( + DeviceSpeedUnknown DeviceSpeed = iota // enumerating + DeviceSpeedLow // usb 1.1 + DeviceSpeedFull // usb 1.1 + DeviceSpeedHigh // usb 2.0 + DeviceSpeedSuper // usb 3.0 + DeviceSpeedSuperPlus // usb 3.1 +) + +// https://mjmwired.net/kernel/Documentation/ABI/testing/sysfs-bus-usb#502 +func ResolveDeviceSpeed(speed uint32) DeviceSpeed { + switch speed { + case 1: + return DeviceSpeedLow + case 12, 15: + return DeviceSpeedFull + case 480: + return DeviceSpeedHigh + case 5000: + return DeviceSpeedSuper + case 10000, 20000: + return DeviceSpeedSuperPlus + default: + return DeviceSpeedUnknown + } +} diff --git a/images/virtualization-dra/pkg/usb/usb.go b/images/virtualization-dra/pkg/usb/usb.go new file mode 100644 index 0000000000..ae220ea329 --- /dev/null +++ b/images/virtualization-dra/pkg/usb/usb.go @@ -0,0 +1,452 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usb + +import ( + "bufio" + "fmt" + "log/slog" + "os" + "path/filepath" + "slices" + "strconv" + "strings" +) + +const PathToUSBDevices = "/sys/bus/usb/devices" + +type Device struct { + Path string + BusID string + Manufacturer string + Product string + Serial string + DevicePath string + Driver string + IsHub bool + VendorID uint16 + ProductID uint16 + BCD uint16 + Bus uint32 + DeviceNumber uint32 + Speed uint32 + Major uint64 + Minor uint64 + BDeviceClass uint8 + BDeviceSubClass uint8 + BDeviceProtocol uint8 + BConfigurationValue uint8 + BNumConfigurations uint8 + BNumInterfaces uint8 + Interfaces []DeviceInterface +} + +type DeviceInterface struct { + BInterfaceClass uint8 + BInterfaceSubClass uint8 + BInterfaceProtocol uint8 +} + +func (d *Device) Equal(other *Device) bool { + return d.Path == other.Path && + d.BusID == other.BusID && + d.Manufacturer == other.Manufacturer && + d.Product == other.Product && + d.Serial == other.Serial && + d.DevicePath == other.DevicePath && + d.Driver == other.Driver && + d.IsHub == other.IsHub && + d.VendorID == other.VendorID && + d.ProductID == other.ProductID && + d.BCD == other.BCD && + d.Bus == other.Bus && + d.DeviceNumber == other.DeviceNumber && + d.Major == other.Major && + d.Minor == other.Minor && + d.BDeviceClass == other.BDeviceClass && + d.BDeviceSubClass == other.BDeviceSubClass && + d.BDeviceProtocol == other.BDeviceProtocol && + d.BConfigurationValue == other.BConfigurationValue && + d.BNumConfigurations == other.BNumConfigurations && + d.BNumInterfaces == other.BNumInterfaces && + slices.Equal(d.Interfaces, other.Interfaces) +} + +func (d *Device) Validate() error { + if d.VendorID == 0 { + return fmt.Errorf("VendorID is required") + } + if d.ProductID == 0 { + return fmt.Errorf("ProductID is required") + } + if d.Bus == 0 { + return fmt.Errorf("Bus is required") + } + if d.DeviceNumber == 0 { + return fmt.Errorf("DeviceNumber is required") + } + if d.DevicePath == "" { + return fmt.Errorf("DevicePath is required") + } + if d.Major == 0 { + return fmt.Errorf("Major is required") + } + if d.Minor == 0 { + return fmt.Errorf("Minor is required") + } + return nil +} + +func LoadDevice(path string) (device Device, err error) { + if !strings.HasPrefix(path, PathToUSBDevices) { + return device, fmt.Errorf("path %s is not a usb device", path) + } + + device.Path = path + device.BusID = filepath.Base(path) + + if err = parseSysUeventFile(path, &device); err != nil { + return + } + if err = parseSerial(path, &device); err != nil { + return + } + if err = parseManufacturer(path, &device); err != nil { + return + } + if err = parseProduct(path, &device); err != nil { + return + } + if err = parseBConfigurationValue(path, &device); err != nil { + return + } + if err = parseBNumConfigurations(path, &device); err != nil { + return + } + if err = parseBNumInterfaces(path, &device); err != nil { + return + } + if err = parseSpeed(path, &device); err != nil { + return + } + if err = parseSysUeventInterfaces(path, &device); err != nil { + return + } + + return +} + +func parseSysUeventFile(path string, device *Device) error { + // Example uevent file: + // MAJOR=189 + // MINOR=257 + // DEVNAME=bus/usb/003/002 + // DEVTYPE=usb_device + // DRIVER=usb + // PRODUCT=e39/f100/35d + // TYPE=0/0/0 + // BUSNUM=003 + // DEVNUM=002 + file, err := os.Open(filepath.Join(path, "uevent")) + if err != nil { + return fmt.Errorf("unable to open the file %s: %w", path, err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + values := strings.Split(line, "=") + if len(values) != 2 { + slog.Info("Skipping %s due not being key=value", slog.String("line", line)) + continue + } + switch values[0] { + case "MAJOR": + val, err := strconv.ParseUint(values[1], 10, 32) + if err != nil { + return fmt.Errorf("failed to parse MAJOR: %s", values[1]) + } + device.Major = val + case "MINOR": + val, err := strconv.ParseUint(values[1], 10, 32) + if err != nil { + return fmt.Errorf("failed to parse MINOR: %s", values[1]) + } + device.Minor = val + case "DEVNAME": + device.DevicePath = filepath.Join("/dev", values[1]) + case "DRIVER": + device.Driver = values[1] + case "PRODUCT": + products := strings.Split(values[1], "/") + if len(products) != 3 { + slog.Error("Failed to parse PRODUCT", slog.String("value", values[1]), slog.Any("err", err)) + return fmt.Errorf("failed to parse PRODUCT: %s", values[1]) + } + + val, err := strconv.ParseUint(products[0], 16, 32) + if err != nil { + return fmt.Errorf("failed to parse PRODUCT: %s", values[1]) + } + device.VendorID = uint16(val) + + val, err = strconv.ParseUint(products[1], 16, 32) + if err != nil { + return fmt.Errorf("failed to parse PRODUCT: %s", values[1]) + } + device.ProductID = uint16(val) + + val, err = strconv.ParseUint(products[2], 16, 32) + if err != nil { + return fmt.Errorf("failed to parse PRODUCT: %s", values[1]) + } + device.BCD = uint16(val) + case "TYPE": + types := strings.Split(values[1], "/") + if len(types) != 3 { + return fmt.Errorf("failed to parse TYPE: %s", values[1]) + } + val, err := strconv.ParseUint(types[0], 10, 8) + if err != nil { + return fmt.Errorf("failed to parse TYPE: %s", values[1]) + } + device.BDeviceClass = uint8(val) + device.IsHub = device.BDeviceClass == 9 // 09 = USB Hub class + + val, err = strconv.ParseUint(types[1], 10, 8) + if err != nil { + return fmt.Errorf("failed to parse TYPE: %s", values[1]) + } + device.BDeviceSubClass = uint8(val) + + val, err = strconv.ParseUint(types[2], 10, 8) + if err != nil { + return fmt.Errorf("failed to parse TYPE: %s", values[1]) + } + device.BDeviceProtocol = uint8(val) + case "BUSNUM": + val, err := strconv.ParseUint(values[1], 10, 32) + if err != nil { + return fmt.Errorf("failed to parse BUSNUM: %s", values[1]) + } + device.Bus = uint32(val) + case "DEVNUM": + val, err := strconv.ParseUint(values[1], 10, 32) + if err != nil { + return fmt.Errorf("failed to parse DEVNUM: %s", values[1]) + } + device.DeviceNumber = uint32(val) + default: + slog.Info("Skipping unhandled line", slog.String("line", line)) + } + } + return nil +} + +func parseSerial(path string, device *Device) error { + serial, err := parseStringValue(path, "serial") + if err != nil { + return err + } + device.Serial = serial + return nil +} + +func parseManufacturer(path string, device *Device) error { + manufacturer, err := parseStringValue(path, "manufacturer") + if err != nil { + return err + } + device.Manufacturer = manufacturer + return nil +} + +func parseProduct(path string, device *Device) error { + product, err := parseStringValue(path, "product") + if err != nil { + return err + } + device.Product = product + return nil +} + +func parseBConfigurationValue(path string, device *Device) error { + val, err := parseUintValue(path, "bConfigurationValue", 8, true) + if err != nil { + return err + } + device.BConfigurationValue = uint8(val) + return nil +} + +func parseBNumConfigurations(path string, device *Device) error { + val, err := parseUintValue(path, "bNumConfigurations", 8, false) + if err != nil { + return err + } + device.BNumConfigurations = uint8(val) + return nil +} + +func parseBNumInterfaces(path string, device *Device) error { + val, err := parseUintValue(path, "bNumInterfaces", 8, true) + if err != nil { + return err + } + device.BNumInterfaces = uint8(val) + return nil +} + +func parseSpeed(path string, device *Device) error { + val, err := parseUintValue(path, "speed", 32, false) + if err != nil { + return err + } + device.Speed = uint32(val) + return nil +} + +func parseSysUeventInterfaces(path string, device *Device) error { + // 3-2.1.1:1.0 + // | | | + // │ | |- bInterfaceNumber + // | |--- bConfigurationValue + // |----------- busID usb_device + + // path - /sys/bus/usb/devices/3-2.1.1 + // uevent path - /sys/bus/usb/devices/3-2.1.1:1.0/uevent + + if device.BConfigurationValue == 0 || device.BNumInterfaces == 0 { + device.Interfaces = nil + return nil + } + + var deviceInterfaces []DeviceInterface + + parent := filepath.Dir(path) + entries, err := os.ReadDir(parent) + if err != nil { + return fmt.Errorf("unable to read the directory %s: %w", path, err) + } + + prefix := fmt.Sprintf("%s:%d.", device.BusID, device.BConfigurationValue) + for _, entry := range entries { + if !entry.IsDir() { + continue + } + if !strings.HasPrefix(entry.Name(), prefix) { + continue + } + interfaceNumberStr := strings.TrimPrefix(entry.Name(), prefix) + _, err := strconv.Atoi(interfaceNumberStr) + if err != nil { + // not a valid interface number + continue + } + + ueventPath := filepath.Join(path, entry.Name(), "uevent") + file, err := os.Open(ueventPath) + if err != nil { + return fmt.Errorf("unable to open the file %s: %w", ueventPath, err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + values := strings.Split(line, "=") + if len(values) != 2 { + slog.Info("Skipping %s due not being key=value", slog.String("line", line)) + continue + } + switch values[0] { + case "INTERFACE": + deviceInterface := DeviceInterface{} + + interfaces := strings.Split(values[1], "/") + if len(interfaces) != 3 { + return fmt.Errorf("failed to parse INTERFACE: %s", values[1]) + } + val, err := strconv.ParseUint(interfaces[0], 10, 8) + if err != nil { + return fmt.Errorf("failed to parse INTERFACE: %s", values[1]) + } + deviceInterface.BInterfaceClass = uint8(val) + + val, err = strconv.ParseUint(interfaces[1], 10, 8) + if err != nil { + return fmt.Errorf("failed to parse INTERFACE: %s", values[1]) + } + deviceInterface.BInterfaceSubClass = uint8(val) + + val, err = strconv.ParseUint(interfaces[2], 10, 8) + if err != nil { + return fmt.Errorf("failed to parse INTERFACE: %s", values[1]) + } + deviceInterface.BInterfaceProtocol = uint8(val) + + deviceInterfaces = append(deviceInterfaces, deviceInterface) + + break + } + } + } + + device.Interfaces = deviceInterfaces + + return nil +} + +func parseStringValue(path, valueName string) (string, error) { + valuePath := filepath.Join(path, valueName) + data, err := os.ReadFile(valuePath) + if err != nil { + return "", fmt.Errorf("failed to read %s: %w", valuePath, err) + } + + value := strings.TrimSpace(string(data)) + if value == "" { + return "", fmt.Errorf("invalid %s: empty", valueName) + } + + return value, nil +} + +func parseUintValue(path, valueName string, bitSize int, ignoreNotExist bool) (uint64, error) { + valuePath := filepath.Join(path, valueName) + data, err := os.ReadFile(valuePath) + if err != nil { + return 0, fmt.Errorf("failed to read %s: %w", path, err) + } + + value := strings.TrimSpace(string(data)) + if value == "" && ignoreNotExist { + return 0, nil + } + + val, err := strconv.ParseUint(value, 10, bitSize) + if err != nil { + return 0, fmt.Errorf("failed to parse %s: %w", valueName, err) + } + + if val == 0 { + return 0, fmt.Errorf("invalid %s: 0", valueName) + } + + return val, nil +} diff --git a/images/virtualization-dra/test/pod-with-template-3.yaml b/images/virtualization-dra/test/pod-with-template-3.yaml new file mode 100644 index 0000000000..0c4c603333 --- /dev/null +++ b/images/virtualization-dra/test/pod-with-template-3.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-pod-with-usb-template-3 + namespace: usb +spec: + containers: + - name: test-container + image: nicolaka/netshoot:latest + command: ["sleep", "3600"] + resources: + claims: + - name: usb-device + resourceClaims: + - name: usb-device + resourceClaimTemplateName: usb-product-0951-vendor-0104-template diff --git a/images/virtualization-dra/test/resourceclaim-template-2.yaml b/images/virtualization-dra/test/resourceclaim-template-2.yaml new file mode 100644 index 0000000000..e51ab2ab84 --- /dev/null +++ b/images/virtualization-dra/test/resourceclaim-template-2.yaml @@ -0,0 +1,17 @@ +apiVersion: resource.k8s.io/v1beta1 +kind: ResourceClaimTemplate +metadata: + name: usb-product-0951-vendor-0104-template +spec: + spec: + devices: + requests: + - name: req-0 + allocationMode: "ExactCount" + count: 1 + deviceClassName: usb-devices.virtualization.deckhouse.io + selectors: + - cel: + expression: |- + device.attributes["virtualization-dra"].productID == "0104" && + device.attributes["virtualization-dra"].vendorID == "0951" diff --git a/images/virtualization-dra/werf.inc.yaml b/images/virtualization-dra/werf.inc.yaml index 919a0adcbb..f3549fa388 100644 --- a/images/virtualization-dra/werf.inc.yaml +++ b/images/virtualization-dra/werf.inc.yaml @@ -1,7 +1,7 @@ --- image: {{ .ModuleNamePrefix }}{{ .ImageName }}-builder final: false -fromImage: {{ eq $.SVACE_ENABLED "false" | ternary "builder/golang-bookworm-1.24" "builder/golang-alt-svace-1.24.9" }} +fromImage: {{ eq $.SVACE_ENABLED "false" | ternary "builder/golang-bookworm-1.25" "builder/golang-alt-svace-1.25.4" }} git: - add: {{ .ModuleDir }}/images/{{ .ImageName }} to: /src/images/virtualization-dra @@ -39,3 +39,12 @@ shell: {{- include "image-build.build" (set $ "BuildCommand" `go build -ldflags="-s -w" -v -o /out/virtualization-dra-plugin ./cmd/virtualization-dra-plugin`) | nindent 4 }} {{- end }} + - | + echo "Build virtualization-dra-usb-gateway binary" + {{- $_ := set $ "ProjectName" (list $.ImageName "virtualization-dra-usb-gateway" | join "/") }} + + {{- if eq $.DEBUG_COMPONENT "delve/virtualization-dra-usb-gateway" }} + go build -v -o /out/virtualization-dra-usb-gateway ./cmd/usb-gateway + {{- else }} + {{- include "image-build.build" (set $ "BuildCommand" `go build -ldflags="-s -w" -v -o /out/virtualization-dra-usb-gateway ./cmd/usb-gateway`) | nindent 4 }} + {{- end }} diff --git a/templates/virtualization-dra-usb-gateway/_helper.tpl b/templates/virtualization-dra-usb-gateway/_helper.tpl new file mode 100644 index 0000000000..26caf8ac46 --- /dev/null +++ b/templates/virtualization-dra-usb-gateway/_helper.tpl @@ -0,0 +1,5 @@ +{{- define "virtualization-dra-usb-gateway.isEnabled" -}} +{{- if eq (include "hasValidModuleConfig" .) "true" -}} +true +{{- end -}} +{{- end -}} diff --git a/templates/virtualization-dra-usb-gateway/daemonset.yaml b/templates/virtualization-dra-usb-gateway/daemonset.yaml new file mode 100644 index 0000000000..4c8c6a79d2 --- /dev/null +++ b/templates/virtualization-dra-usb-gateway/daemonset.yaml @@ -0,0 +1,126 @@ +{{- $priorityClassName := include "priorityClassName" . }} +{{- $delve := (include "delve" . | fromYaml) -}} +{{- define "virtualization-dra-usb-gateway_resources" }} +cpu: 10m +memory: 25Mi +{{- end }} + + +{{- if eq (include "virtualization-dra-usb-gateway.isEnabled" .) "true"}} + +{{- if (.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} +--- +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: virtualization-dra-usb-gateway + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-dra-usb-gateway" "workload-resource-policy.deckhouse.io" "every-node")) | nindent 2 }} +spec: + targetRef: + apiVersion: "apps/v1" + kind: DaemonSet + name: virtualization-dra-usb-gateway + updatePolicy: + updateMode: "Auto" + resourcePolicy: + containerPolicies: + - containerName: virtualization-dra-usb-gateway + minAllowed: + {{- include "virtualization-dra-usb-gateway_resources" . | nindent 8 }} + maxAllowed: + cpu: 20m + memory: 25Mi +{{- end }} + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: virtualization-dra-usb-gateway + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-dra-usb-gateway")) | nindent 2 }} +spec: + selector: + matchLabels: + app: virtualization-dra-usb-gateway + template: + metadata: + labels: + app: virtualization-dra-usb-gateway + spec: + {{- include "helm_lib_priority_class" (tuple . $priorityClassName) | nindent 6 }} + {{- include "helm_lib_tolerations" (tuple . "any-node") | nindent 6 }} + {{- include "helm_lib_module_pod_security_context_run_as_user_root" . | nindent 6 }} + imagePullSecrets: + - name: virtualization-module-registry + serviceAccountName: virtualization-dra-usb-gateway + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + kubernetes.io/os: linux + initContainers: + - name: virtualization-dra-usb-gateway-init + image: {{ include "helm_lib_module_image" (list . "virtualizationDraUsbGateway") }} + imagePullPolicy: "IfNotPresent" + args: ["init"] + securityContext: + privileged: false + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + add: + - SYS_MODULE + mounts: + - name: lib-modules + mountPath: /lib/modules + containers: + - name: virtualization-dra-usb-gateway + {{- include "helm_lib_module_container_security_context_privileged_read_only_root_filesystem" . | nindent 10 }} + image: {{ include "helm_lib_module_image" (list . "virtualizationDraUsbGateway") }} + imagePullPolicy: "IfNotPresent" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if eq (include "moduleLogLevel" .) "debug" }} + - name: VERBOSITY + value: "10" + {{- end }} + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 14 }} + {{- if not ( .Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} + {{- include "virtualization-dra-usb-gateway_resources" . | nindent 14 }} + {{- end }} + ports: + - containerPort: 51515 + name: health + protocol: TCP + {{- include "delvePorts" (list $delve "delve/virtualization-dra-usb-gateway") | nindent 12 }} + {{- if ne "delve/virtualization-dra-usb-gateway" ($delve | dig "debug" "component" "") }} + # TODO: add readinessProbe and livenessProbe + #readinessProbe: + #livenessProbe: + {{- end }} + volumeMounts: + - name: sys + mountPath: /sys + - name: var-run + mountPath: /var/run + volumes: + - name: sys + hostPath: + path: /sys + - name: var-run + hostPath: + path: /var/run + - name: lib-modules + hostPath: + path: /lib/modules +{{- end }} diff --git a/templates/virtualization-dra-usb-gateway/rbac-for-us.yaml b/templates/virtualization-dra-usb-gateway/rbac-for-us.yaml new file mode 100644 index 0000000000..8b2b2097f4 --- /dev/null +++ b/templates/virtualization-dra-usb-gateway/rbac-for-us.yaml @@ -0,0 +1,34 @@ +{{- if eq (include "virtualization-dra-usb-gateway.isEnabled" .) "true"}} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: virtualization-dra-usb-gateway + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-dra-usb-gateway")) | nindent 2 }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: d8:{{ .Chart.Name }}:virtualization-dra-usb-gateway + {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-dra-usb-gateway")) | nindent 2 }} +rules: + # TODO: fix me + - apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: d8:{{ .Chart.Name }}:virtualization-dra-usb-gateway + {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-dra-usb-gateway")) | nindent 2 }} +subjects: + - kind: ServiceAccount + name: virtualization-dra-usb-gateway + namespace: d8-{{ .Chart.Name }} +roleRef: + kind: ClusterRole + name: d8:{{ .Chart.Name }}:virtualization-dra-usb-gateway + apiGroup: rbac.authorization.k8s.io +{{- end }} From 79b5c7715f3e158f4d82880a44f9c96a816b5d1f Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Mon, 29 Dec 2025 13:09:56 +0300 Subject: [PATCH 02/29] add Signed-off-by: Yaroslav Borbat --- images/virtualization-dra/Taskfile.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/images/virtualization-dra/Taskfile.yaml b/images/virtualization-dra/Taskfile.yaml index 145a70c862..c0c37e7377 100644 --- a/images/virtualization-dra/Taskfile.yaml +++ b/images/virtualization-dra/Taskfile.yaml @@ -58,7 +58,6 @@ tasks: - | golangci-lint run - build:go-usbip: desc: "Build go-usbip binary" cmds: @@ -68,4 +67,3 @@ tasks: desc: "Generate API code" cmds: - hack/update-codegen.sh - From 96bd388a1409a289e939a62aeec87458db51bfff Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 12:57:26 +0300 Subject: [PATCH 03/29] fix templates Signed-off-by: Yaroslav Borbat --- templates/virtualization-dra-usb-gateway/daemonset.yaml | 2 +- tools/kubeconform/fixtures/module-values.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/templates/virtualization-dra-usb-gateway/daemonset.yaml b/templates/virtualization-dra-usb-gateway/daemonset.yaml index 4c8c6a79d2..49e7d0e72e 100644 --- a/templates/virtualization-dra-usb-gateway/daemonset.yaml +++ b/templates/virtualization-dra-usb-gateway/daemonset.yaml @@ -71,7 +71,7 @@ spec: - ALL add: - SYS_MODULE - mounts: + volumeMounts: - name: lib-modules mountPath: /lib/modules containers: diff --git a/tools/kubeconform/fixtures/module-values.yaml b/tools/kubeconform/fixtures/module-values.yaml index 6dfa410372..546cd53ba2 100644 --- a/tools/kubeconform/fixtures/module-values.yaml +++ b/tools/kubeconform/fixtures/module-values.yaml @@ -331,7 +331,8 @@ global: virtualizationApi: sha256:0000000000000000000000000000000000000000000000000000000000000000 virtualizationController: sha256:0000000000000000000000000000000000000000000000000000000000000000 vmRouteForge: sha256:0000000000000000000000000000000000000000000000000000000000000000 - virtualizationDraPlugin: sha256:0000000000000000000000000000000000000000000000000000000000000000w + virtualizationDraPlugin: sha256:0000000000000000000000000000000000000000000000000000000000000000 + virtualizationDraUsbGateway: sha256:0000000000000000000000000000000000000000000000000000000000000000 registry: CA: "" address: some-registry.io From 5d19d3a12eddae229bc5aa090bf2385cde7db107 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 13:33:12 +0300 Subject: [PATCH 04/29] fix json patch Signed-off-by: Yaroslav Borbat --- .../virtualization-dra/internal/usb-gateway/labeler/labeler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/virtualization-dra/internal/usb-gateway/labeler/labeler.go b/images/virtualization-dra/internal/usb-gateway/labeler/labeler.go index 2e36e3fdfc..6f8af80b21 100644 --- a/images/virtualization-dra/internal/usb-gateway/labeler/labeler.go +++ b/images/virtualization-dra/internal/usb-gateway/labeler/labeler.go @@ -58,7 +58,7 @@ func (l *genericLabeler) Label(ctx context.Context, name, namespace string, newL return err } - patch := []byte(fmt.Sprintf("[{'op': 'replace', 'path': '/metadata/labels', 'value': %s}]", value)) + patch := []byte(fmt.Sprintf(`[{"op": "replace", "path": "/metadata/labels", "value": %s}]`, string(value))) _, err = l.client.Resource(l.gvr).Namespace(namespace).Patch(ctx, name, types.JSONPatchType, patch, metav1.PatchOptions{}) return err From c2d034427c69d2a4604e5b6a70091d038b884d5a Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 13:37:49 +0300 Subject: [PATCH 05/29] fix load modules Signed-off-by: Yaroslav Borbat --- images/virtualization-dra/pkg/modprobe/modprobe.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/images/virtualization-dra/pkg/modprobe/modprobe.go b/images/virtualization-dra/pkg/modprobe/modprobe.go index 838285a926..30b61127ed 100644 --- a/images/virtualization-dra/pkg/modprobe/modprobe.go +++ b/images/virtualization-dra/pkg/modprobe/modprobe.go @@ -17,7 +17,9 @@ limitations under the License. package modprobe import ( + "errors" "fmt" + "log/slog" "os" "path/filepath" @@ -51,8 +53,14 @@ func loadModule(path string) error { defer f.Close() if err = unix.FinitModule(int(f.Fd()), "", 0); err != nil { + if errors.Is(err, unix.EEXIST) { + slog.Info("Module already loaded", slog.String("path", path)) + return nil + } return fmt.Errorf("finit_module %s: %w", path, err) } + slog.Info("Module loaded", slog.String("path", path)) + return nil } From a7b93c12b108bb60487ef594ebfe9e2c55248b46 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 13:49:36 +0300 Subject: [PATCH 06/29] fix map nil panic Signed-off-by: Yaroslav Borbat --- .../internal/usb-gateway/informer/informer.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/images/virtualization-dra/internal/usb-gateway/informer/informer.go b/images/virtualization-dra/internal/usb-gateway/informer/informer.go index 3d32df4fb6..ea703ba826 100644 --- a/images/virtualization-dra/internal/usb-gateway/informer/informer.go +++ b/images/virtualization-dra/internal/usb-gateway/informer/informer.go @@ -44,9 +44,10 @@ func NewFactory(clientSet *kubernetes.Clientset, resync *time.Duration) *Factory } return &Factory{ - clientSet: clientSet, - defaultResync: defaultResync, - informers: make(map[string]cache.SharedIndexInformer), + clientSet: clientSet, + defaultResync: defaultResync, + informers: make(map[string]cache.SharedIndexInformer), + startedInformers: make(map[string]struct{}), } } From 747d9c9a62717723bab8dc8b397409fe7f9bd059 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 14:19:20 +0300 Subject: [PATCH 07/29] fix loading modules for fresh kernels Signed-off-by: Yaroslav Borbat --- .../pkg/modprobe/modprobe.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/images/virtualization-dra/pkg/modprobe/modprobe.go b/images/virtualization-dra/pkg/modprobe/modprobe.go index 30b61127ed..c4867c1e0d 100644 --- a/images/virtualization-dra/pkg/modprobe/modprobe.go +++ b/images/virtualization-dra/pkg/modprobe/modprobe.go @@ -22,6 +22,7 @@ import ( "log/slog" "os" "path/filepath" + "strings" "golang.org/x/sys/unix" ) @@ -36,7 +37,23 @@ func LoadModules(modules []string) error { base := filepath.Join("/lib/modules", kernel) for _, m := range modules { - path := filepath.Join(base, m) + + var path string + if strings.HasSuffix(m, ".zst") { + path = filepath.Join(base, m) + } else { + pathZst := filepath.Join(base, m+".zst") + pathKo := filepath.Join(base, m) + + if _, err := os.Stat(pathZst); err == nil { + path = pathZst + } else if _, err := os.Stat(pathKo); err == nil { + path = pathKo + } else { + return fmt.Errorf("module file not found: %s or %s", pathKo, pathZst) + } + } + if err := loadModule(path); err != nil { return fmt.Errorf("load module %s: %w", path, err) } From 194656f606435109d995b8057542710dfff68a02 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 14:46:11 +0300 Subject: [PATCH 08/29] fix loading modules for fresh kernels Signed-off-by: Yaroslav Borbat --- .../cmd/usb-gateway/app/init.go | 18 ++++- .../pkg/modprobe/modprobe.go | 66 ++++++++++--------- 2 files changed, 51 insertions(+), 33 deletions(-) diff --git a/images/virtualization-dra/cmd/usb-gateway/app/init.go b/images/virtualization-dra/cmd/usb-gateway/app/init.go index a435e880dd..ebba3886ff 100644 --- a/images/virtualization-dra/cmd/usb-gateway/app/init.go +++ b/images/virtualization-dra/cmd/usb-gateway/app/init.go @@ -18,6 +18,7 @@ package app import ( "fmt" + "path/filepath" "github.com/spf13/cobra" @@ -39,12 +40,23 @@ func NewInitCommand() *cobra.Command { type initOptions struct{} func (o *initOptions) Run(_ *cobra.Command, _ []string) error { + kernelRelease, err := modprobe.KernelRelease() + if err != nil { + return fmt.Errorf("failed to get kernel release: %w", err) + } + modules := []string{ - "kernel/drivers/usb/usbip/usbip-core.ko", - "kernel/drivers/usb/usbip/vhci-hcd.ko", + filepath.Join("/lib/modules", kernelRelease, "kernel/drivers/usb/usbip/usbip-core.ko"), + filepath.Join("/lib/modules", kernelRelease, "kernel/drivers/usb/usbip/vhci-hcd.ko"), + } + + if modprobe.KernelSupportsZst(kernelRelease) { + for i := range modules { + modules[i] += ".zst" + } } - if err := modprobe.LoadModules(modules); err != nil { + if err := modprobe.LoadModules(modules...); err != nil { return fmt.Errorf("failed to load modules: %w", err) } diff --git a/images/virtualization-dra/pkg/modprobe/modprobe.go b/images/virtualization-dra/pkg/modprobe/modprobe.go index c4867c1e0d..67eb61ffb9 100644 --- a/images/virtualization-dra/pkg/modprobe/modprobe.go +++ b/images/virtualization-dra/pkg/modprobe/modprobe.go @@ -21,41 +21,16 @@ import ( "fmt" "log/slog" "os" - "path/filepath" + "strconv" "strings" "golang.org/x/sys/unix" ) -func LoadModules(modules []string) error { - var uts unix.Utsname - if err := unix.Uname(&uts); err != nil { - return fmt.Errorf("uname: %w", err) - } - - kernel := unix.ByteSliceToString(uts.Release[:]) - base := filepath.Join("/lib/modules", kernel) - - for _, m := range modules { - - var path string - if strings.HasSuffix(m, ".zst") { - path = filepath.Join(base, m) - } else { - pathZst := filepath.Join(base, m+".zst") - pathKo := filepath.Join(base, m) - - if _, err := os.Stat(pathZst); err == nil { - path = pathZst - } else if _, err := os.Stat(pathKo); err == nil { - path = pathKo - } else { - return fmt.Errorf("module file not found: %s or %s", pathKo, pathZst) - } - } - - if err := loadModule(path); err != nil { - return fmt.Errorf("load module %s: %w", path, err) +func LoadModules(modules ...string) error { + for _, module := range modules { + if err := loadModule(module); err != nil { + return fmt.Errorf("load module %s: %w", module, err) } } @@ -81,3 +56,34 @@ func loadModule(path string) error { return nil } + +func KernelRelease() (string, error) { + var uts unix.Utsname + if err := unix.Uname(&uts); err != nil { + return "", fmt.Errorf("uname: %w", err) + } + return unix.ByteSliceToString(uts.Release[:]), nil + +} + +func KernelSupportsZst(release string) bool { + parts := strings.Split(release, ".") + if len(parts) < 2 { + return false + } + + major, err1 := strconv.Atoi(parts[0]) + minor, err2 := strconv.Atoi(parts[1]) + if err1 != nil || err2 != nil { + return false + } + + // ZST is supported since 5.16 + if major > 5 { + return true + } + if major == 5 && minor >= 16 { + return true + } + return false +} From e17000ab1262db02493499bdffd05a44af9db0a1 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 15:40:25 +0300 Subject: [PATCH 09/29] fix loading modules for fresh kernels Signed-off-by: Yaroslav Borbat --- .../pkg/modprobe/modprobe.go | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/images/virtualization-dra/pkg/modprobe/modprobe.go b/images/virtualization-dra/pkg/modprobe/modprobe.go index 67eb61ffb9..62f3893434 100644 --- a/images/virtualization-dra/pkg/modprobe/modprobe.go +++ b/images/virtualization-dra/pkg/modprobe/modprobe.go @@ -19,11 +19,14 @@ package modprobe import ( "errors" "fmt" + "io" "log/slog" "os" + "path/filepath" "strconv" "strings" + "github.com/klauspost/compress/zstd" "golang.org/x/sys/unix" ) @@ -38,6 +41,15 @@ func LoadModules(modules ...string) error { } func loadModule(path string) error { + if strings.HasSuffix(path, ".zst") { + uncompressedPath, err := uncompressModuleToTmp(path) + if err != nil { + return fmt.Errorf("uncompress module %s: %w", path, err) + } + defer os.Remove(uncompressedPath) + path = uncompressedPath + } + f, err := os.Open(path) if err != nil { return fmt.Errorf("open %s: %w", path, err) @@ -57,6 +69,33 @@ func loadModule(path string) error { return nil } +func uncompressModuleToTmp(path string) (string, error) { + pattern := filepath.Base(path) + "-*" + uncompress, err := os.CreateTemp("", pattern) + if err != nil { + return "", err + } + defer uncompress.Close() + + in, err := os.Open(path) + if err != nil { + return "", err + } + defer in.Close() + + decoder, err := zstd.NewReader(in) + if err != nil { + return "", err + } + defer decoder.Close() + + if _, err := io.Copy(uncompress, decoder); err != nil { + return "", err + } + + return uncompress.Name(), nil +} + func KernelRelease() (string, error) { var uts unix.Utsname if err := unix.Uname(&uts); err != nil { From 6eacc3a0e3994643c61588904195b88c8aa32b4a Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 15:41:00 +0300 Subject: [PATCH 10/29] go tidy Signed-off-by: Yaroslav Borbat --- images/virtualization-dra/go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/images/virtualization-dra/go.mod b/images/virtualization-dra/go.mod index 9bac9000d9..705cf01feb 100644 --- a/images/virtualization-dra/go.mod +++ b/images/virtualization-dra/go.mod @@ -13,6 +13,7 @@ require ( github.com/fsnotify/fsnotify v1.5.1 github.com/go-logr/logr v1.4.3 github.com/godbus/dbus/v5 v5.2.0 + github.com/klauspost/compress v1.18.0 github.com/onsi/ginkgo/v2 v2.27.2 github.com/onsi/gomega v1.38.2 github.com/spf13/cobra v1.10.1 From d2bcb2dcd87a1b3c396e985907dd62208c90ece8 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 15:56:42 +0300 Subject: [PATCH 11/29] fix pointer resourceSlice Signed-off-by: Yaroslav Borbat --- .../internal/usb-gateway/controller/resourceclaim/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go index 85ad158e98..bf6c216a4c 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go @@ -520,7 +520,7 @@ func (c *Controller) getVirtualizationDraResourceSlices() ([]resourcev1beta1.Res } var slices []resourcev1beta1.ResourceSlice for _, obj := range slicesObj { - slice, ok := obj.(resourcev1beta1.ResourceSlice) + slice, ok := obj.(*resourcev1beta1.ResourceSlice) if !ok { return nil, fmt.Errorf("unexpected type of resource slice: %T", obj) } From 2f936692a7c048968864609e8ca98be857192ee8 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 16:20:15 +0300 Subject: [PATCH 12/29] enable USBGateway featureGate Signed-off-by: Yaroslav Borbat --- templates/virtualization-dra/daemonset.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/virtualization-dra/daemonset.yaml b/templates/virtualization-dra/daemonset.yaml index 47c1e7543f..8300a6b418 100644 --- a/templates/virtualization-dra/daemonset.yaml +++ b/templates/virtualization-dra/daemonset.yaml @@ -63,6 +63,8 @@ spec: {{- include "helm_lib_module_container_security_context_privileged_read_only_root_filesystem" . | nindent 10 }} image: {{ include "helm_lib_module_image" (list . "virtualizationDraPlugin") }} imagePullPolicy: "IfNotPresent" + args: + - --feature-gates=USBGateway=true env: - name: NODE_NAME valueFrom: From 116b8275c64dceaa1efe5e8227a3d64ba07b46d4 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 16:43:27 +0300 Subject: [PATCH 13/29] fix dra feature-gates flags Signed-off-by: Yaroslav Borbat --- images/virtualization-dra/cmd/usb-gateway/app/app.go | 7 +------ .../cmd/virtualization-dra-plugin/app/app.go | 7 ++++++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/images/virtualization-dra/cmd/usb-gateway/app/app.go b/images/virtualization-dra/cmd/usb-gateway/app/app.go index 0bd715c0df..4f0a92dcf4 100644 --- a/images/virtualization-dra/cmd/usb-gateway/app/app.go +++ b/images/virtualization-dra/cmd/usb-gateway/app/app.go @@ -29,7 +29,6 @@ import ( "k8s.io/client-go/tools/clientcmd" "k8s.io/component-base/cli/flag" - "github.com/deckhouse/virtualization-dra/internal/featuregates" "github.com/deckhouse/virtualization-dra/internal/usb-gateway/controller/resourceclaim" "github.com/deckhouse/virtualization-dra/internal/usb-gateway/informer" "github.com/deckhouse/virtualization-dra/internal/usb-gateway/prepare" @@ -70,8 +69,7 @@ func NewUSBGatewayCommand() *cobra.Command { func newUsbOptions() *usbOptions { return &usbOptions{ - Logging: &logger.Options{}, - featureGates: featuregates.AddFlags, + Logging: &logger.Options{}, } } @@ -82,7 +80,6 @@ type usbOptions struct { USBIPPort int USBResyncPeriod time.Duration Logging *logger.Options - featureGates featuregates.AddFlagsFunc } func (o *usbOptions) NamedFlags() (fs flag.NamedFlagSets) { @@ -95,8 +92,6 @@ func (o *usbOptions) NamedFlags() (fs flag.NamedFlagSets) { o.Logging.AddFlags(fs.FlagSet("logging")) - o.featureGates(fs.FlagSet("feature-gates")) - return fs } diff --git a/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go b/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go index 088c76c204..a7a69110b1 100644 --- a/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go +++ b/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go @@ -29,6 +29,7 @@ import ( "k8s.io/component-base/cli/flag" "github.com/deckhouse/virtualization-dra/internal/cdi" + "github.com/deckhouse/virtualization-dra/internal/featuregates" "github.com/deckhouse/virtualization-dra/internal/plugin" "github.com/deckhouse/virtualization-dra/internal/usb" "github.com/deckhouse/virtualization-dra/pkg/logger" @@ -79,6 +80,7 @@ func newDraOptions() *draOptions { HealthzPort: 51515, USBResyncPeriod: usb.DefaultResyncPeriod, Logging: &logger.Options{}, + featureGates: featuregates.AddFlags, } if healthzPort := os.Getenv("HEALTHZ_PORT"); healthzPort != "" { @@ -101,7 +103,8 @@ type draOptions struct { HealthzPort int USBResyncPeriod time.Duration - Logging *logger.Options + Logging *logger.Options + featureGates featuregates.AddFlagsFunc } func (o *draOptions) NamedFlags() (fs flag.NamedFlagSets) { @@ -117,6 +120,8 @@ func (o *draOptions) NamedFlags() (fs flag.NamedFlagSets) { o.Logging.AddFlags(fs.FlagSet("logging")) + o.featureGates(fs.FlagSet("feature-gates")) + return fs } From c66f6b6ca45b474f84f9cd48dc2970ebe65a4867 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 17:32:16 +0300 Subject: [PATCH 14/29] support NodeSelector in ResourceSlice Signed-off-by: Yaroslav Borbat --- .../virtualization-dra/internal/plugin/driver.go | 15 ++++++++++----- images/virtualization-dra/internal/usb/convert.go | 1 + 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/images/virtualization-dra/internal/plugin/driver.go b/images/virtualization-dra/internal/plugin/driver.go index a8605d29a9..458d37b1ba 100644 --- a/images/virtualization-dra/internal/plugin/driver.go +++ b/images/virtualization-dra/internal/plugin/driver.go @@ -30,8 +30,10 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/dynamic-resource-allocation/kubeletplugin" "k8s.io/dynamic-resource-allocation/resourceslice" + "k8s.io/utils/ptr" "github.com/deckhouse/virtualization-dra/internal/common" + "github.com/deckhouse/virtualization-dra/internal/featuregates" ) const DriverName = common.VirtualizationDraPluginName @@ -182,14 +184,17 @@ func (d *Driver) startPublisher(ctx context.Context) { } func (d *Driver) makeResources(devices []resourceapi.Device) resourceslice.DriverResources { + slice := resourceslice.Slice{ + Devices: devices, + } + if featuregates.Default().USBGatewayEnabled() { + slice.PerDeviceNodeSelection = ptr.To(true) + } + return resourceslice.DriverResources{ Pools: map[string]resourceslice.Pool{ d.nodeName: { - Slices: []resourceslice.Slice{ - { - Devices: devices, - }, - }, + Slices: []resourceslice.Slice{slice}, }, }, } diff --git a/images/virtualization-dra/internal/usb/convert.go b/images/virtualization-dra/internal/usb/convert.go index 8cbeb53f9a..deb52202a5 100644 --- a/images/virtualization-dra/internal/usb/convert.go +++ b/images/virtualization-dra/internal/usb/convert.go @@ -82,6 +82,7 @@ func convertToAPIDevice(usbDevice Device, nodeName string) *resourceapi.Device { } if featuregates.Default().USBGatewayEnabled() { + // TODO: need pr to deckhouse for enable DRAPartitionableDevices feature gate on ApiServer device.NodeSelector = &corev1.NodeSelector{ NodeSelectorTerms: []corev1.NodeSelectorTerm{ { From b6f3b08f2494275fe0aa7ae117707bc1cca4cc2e Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 17:42:57 +0300 Subject: [PATCH 15/29] fix nodeSelector operator Signed-off-by: Yaroslav Borbat --- images/virtualization-dra/internal/usb/convert.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/virtualization-dra/internal/usb/convert.go b/images/virtualization-dra/internal/usb/convert.go index deb52202a5..542c8b11bf 100644 --- a/images/virtualization-dra/internal/usb/convert.go +++ b/images/virtualization-dra/internal/usb/convert.go @@ -89,7 +89,7 @@ func convertToAPIDevice(usbDevice Device, nodeName string) *resourceapi.Device { MatchExpressions: []corev1.NodeSelectorRequirement{ { Key: common.USBGatewayLabel, - Operator: corev1.NodeSelectorOpExists, + Operator: corev1.NodeSelectorOpIn, Values: []string{"true"}, }, }, From be2fb2332e8e79bb0bc9bc2c97cf74d6a0e7c1b1 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 18:05:01 +0300 Subject: [PATCH 16/29] fix poolname Signed-off-by: Yaroslav Borbat --- images/virtualization-dra-plugin/debug/dlv.Dockerfile | 5 +++-- images/virtualization-dra/internal/plugin/driver.go | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/images/virtualization-dra-plugin/debug/dlv.Dockerfile b/images/virtualization-dra-plugin/debug/dlv.Dockerfile index 9e85ba1d99..5ee5803b5f 100644 --- a/images/virtualization-dra-plugin/debug/dlv.Dockerfile +++ b/images/virtualization-dra-plugin/debug/dlv.Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.24.7-bookworm@sha256:2c5f7a0c252a17cf6aa30ddee15caa0f485ee29410a6ea64cddb62eea2b07bdf AS builder +FROM golang:1.25-bookworm@sha256:019c22232e57fda8ded2b10a8f201989e839f3d3f962d4931375069bbb927e03 AS builder ARG TARGETOS ARG TARGETARCH @@ -13,13 +13,14 @@ RUN go mod download COPY ./images/virtualization-dra/cmd /app/images/virtualization-dra/cmd COPY ./images/virtualization-dra/internal /app/images/virtualization-dra/internal COPY ./images/virtualization-dra/pkg /app/images/virtualization-dra/pkg +COPY ./images/virtualization-dra/api /app/images/virtualization-dra/api ENV GO111MODULE=on ENV GOOS=${TARGETOS:-linux} ENV GOARCH=${TARGETARCH:-amd64} ENV CGO_ENABLED=0 -RUN go build -tags EE -gcflags "all=-N -l" -a -o virtualization-dra-plugin ./cmd/virtualization-dra-plugin +RUN go build -gcflags "all=-N -l" -a -o virtualization-dra-plugin ./cmd/virtualization-dra-plugin/main.go FROM busybox:1.36.1-glibc diff --git a/images/virtualization-dra/internal/plugin/driver.go b/images/virtualization-dra/internal/plugin/driver.go index 458d37b1ba..1bc5773c8c 100644 --- a/images/virtualization-dra/internal/plugin/driver.go +++ b/images/virtualization-dra/internal/plugin/driver.go @@ -187,13 +187,16 @@ func (d *Driver) makeResources(devices []resourceapi.Device) resourceslice.Drive slice := resourceslice.Slice{ Devices: devices, } + poolName := d.nodeName + if featuregates.Default().USBGatewayEnabled() { slice.PerDeviceNodeSelection = ptr.To(true) + poolName = "global" } return resourceslice.DriverResources{ Pools: map[string]resourceslice.Pool{ - d.nodeName: { + poolName: { Slices: []resourceslice.Slice{slice}, }, }, From 1afdc8c3d6d247be66997cf999a537d297f1b91d Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 20:59:08 +0300 Subject: [PATCH 17/29] add new publisher Signed-off-by: Yaroslav Borbat --- .../internal/plugin/driver.go | 34 ++----- .../internal/plugin/interfaces.go | 3 +- .../internal/plugin/publish.go | 98 +++++++++++++++++++ .../virtualization-dra/internal/usb/store.go | 29 +++++- 4 files changed, 132 insertions(+), 32 deletions(-) create mode 100644 images/virtualization-dra/internal/plugin/publish.go diff --git a/images/virtualization-dra/internal/plugin/driver.go b/images/virtualization-dra/internal/plugin/driver.go index 1bc5773c8c..d7e58a14c7 100644 --- a/images/virtualization-dra/internal/plugin/driver.go +++ b/images/virtualization-dra/internal/plugin/driver.go @@ -29,11 +29,8 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes" "k8s.io/dynamic-resource-allocation/kubeletplugin" - "k8s.io/dynamic-resource-allocation/resourceslice" - "k8s.io/utils/ptr" "github.com/deckhouse/virtualization-dra/internal/common" - "github.com/deckhouse/virtualization-dra/internal/featuregates" ) const DriverName = common.VirtualizationDraPluginName @@ -54,6 +51,7 @@ type Driver struct { allocator Allocator log *slog.Logger + publisher resourcePublisher helper *kubeletplugin.Helper pluginCtx context.Context pluginCancel context.CancelCauseFunc @@ -64,6 +62,8 @@ func (d *Driver) Start(ctx context.Context) error { d.pluginCtx = ctx d.pluginCancel = cancel + d.publisher = newNonOwnerPublisher(ctx, d.kubeClient, d.HandleError) + log.Info("Starting dra plugin") helper, err := kubeletplugin.Start( ctx, @@ -92,6 +92,7 @@ func (d *Driver) Wait() { } func (d *Driver) Shutdown() { + d.publisher.Stop() if d.helper != nil { d.log.Info("Stopping dra plugin") d.helper.Stop() @@ -171,10 +172,9 @@ func (d *Driver) startPublisher(ctx context.Context) { select { case <-ctx.Done(): return - case devices := <-ch: - d.log.Info("Publishing devices", slog.Any("devices", devices)) - resources := d.makeResources(devices) - err := d.helper.PublishResources(ctx, resources) + case resources := <-ch: + d.log.Info("Publishing devices", slog.Any("resources", resources)) + err := d.publisher.PublishResources(ctx, resources) if err != nil { d.log.Error("Failed to publish devices", slog.Any("err", err)) } @@ -182,23 +182,3 @@ func (d *Driver) startPublisher(ctx context.Context) { } }() } - -func (d *Driver) makeResources(devices []resourceapi.Device) resourceslice.DriverResources { - slice := resourceslice.Slice{ - Devices: devices, - } - poolName := d.nodeName - - if featuregates.Default().USBGatewayEnabled() { - slice.PerDeviceNodeSelection = ptr.To(true) - poolName = "global" - } - - return resourceslice.DriverResources{ - Pools: map[string]resourceslice.Pool{ - poolName: { - Slices: []resourceslice.Slice{slice}, - }, - }, - } -} diff --git a/images/virtualization-dra/internal/plugin/interfaces.go b/images/virtualization-dra/internal/plugin/interfaces.go index 4ef9d1bf9d..3f757fc3a5 100644 --- a/images/virtualization-dra/internal/plugin/interfaces.go +++ b/images/virtualization-dra/internal/plugin/interfaces.go @@ -22,11 +22,12 @@ import ( "github.com/containerd/nri/pkg/api" resourceapi "k8s.io/api/resource/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/dynamic-resource-allocation/resourceslice" drapbv1 "k8s.io/kubelet/pkg/apis/dra/v1beta1" ) type Allocator interface { - UpdateChannel() chan []resourceapi.Device + UpdateChannel() chan resourceslice.DriverResources Prepare(ctx context.Context, claim *resourceapi.ResourceClaim) ([]*drapbv1.Device, error) Unprepare(ctx context.Context, claimUID types.UID) error Synchronize(ctx context.Context, pods []*api.PodSandbox, containers []*api.Container) ([]*api.ContainerUpdate, error) diff --git a/images/virtualization-dra/internal/plugin/publish.go b/images/virtualization-dra/internal/plugin/publish.go new file mode 100644 index 0000000000..ecb338b75d --- /dev/null +++ b/images/virtualization-dra/internal/plugin/publish.go @@ -0,0 +1,98 @@ +package plugin + +import ( + "context" + "errors" + "fmt" + "sync" + + "k8s.io/client-go/kubernetes" + "k8s.io/dynamic-resource-allocation/kubeletplugin" + "k8s.io/dynamic-resource-allocation/resourceslice" + "k8s.io/klog/v2" +) + +type resourcePublisher interface { + PublishResources(ctx context.Context, resources resourceslice.DriverResources) error + Stop() +} +type errorHandler func(ctx context.Context, err error, msg string) + +func newNonOwnerPublisher(ctx context.Context, kubeClient kubernetes.Interface, errorHandler errorHandler) resourcePublisher { + ctx, cancel := context.WithCancelCause(ctx) + return &nonOwnerPublisher{ + driverName: DriverName, + kubeClient: kubeClient, + errorHandler: errorHandler, + backgroundCtx: ctx, + cancel: cancel, + } +} + +type nonOwnerPublisher struct { + driverName string + kubeClient kubernetes.Interface + backgroundCtx context.Context + cancel func(cause error) + errorHandler errorHandler + + mutex sync.Mutex + resourceSliceController *resourceslice.Controller +} + +func (p *nonOwnerPublisher) PublishResources(_ context.Context, resources resourceslice.DriverResources) error { + p.mutex.Lock() + defer p.mutex.Unlock() + + driverResources := &resourceslice.DriverResources{ + Pools: resources.Pools, + } + + if p.resourceSliceController == nil { + // Start publishing the information. The controller is using + // our background context, not the one passed into this + // function, and thus is connected to the lifecycle of the + // plugin. + controllerCtx := p.backgroundCtx + controllerLogger := klog.FromContext(controllerCtx) + controllerLogger = klog.LoggerWithName(controllerLogger, "ResourceSlice controller") + controllerCtx = klog.NewContext(controllerCtx, controllerLogger) + var err error + if p.resourceSliceController, err = resourceslice.StartController(controllerCtx, + resourceslice.Options{ + DriverName: p.driverName, + KubeClient: p.kubeClient, + Resources: driverResources, + ErrorHandler: func(ctx context.Context, err error, msg string) { + // ResourceSlice publishing errors like dropped fields or + // invalid spec are not going to get resolved by retrying, + // but neither is restarting the process going to help + // -> all errors are recoverable. + p.errorHandler(ctx, recoverableError{error: err}, msg) + }, + }); err != nil { + return fmt.Errorf("start ResourceSlice controller: %w", err) + } + } else { + // Inform running controller about new information. + p.resourceSliceController.Update(driverResources) + } + + return nil +} + +func (p *nonOwnerPublisher) Stop() { + if p == nil { + return + } + p.cancel(errors.New("nonOwnerPublisher was stopped")) +} + +type recoverableError struct { + error +} + +var _ error = recoverableError{} + +func (err recoverableError) Is(other error) bool { return other == kubeletplugin.ErrRecoverable } +func (err recoverableError) Unwrap() error { return err.error } diff --git a/images/virtualization-dra/internal/usb/store.go b/images/virtualization-dra/internal/usb/store.go index aa658170d5..7190f29200 100644 --- a/images/virtualization-dra/internal/usb/store.go +++ b/images/virtualization-dra/internal/usb/store.go @@ -29,6 +29,7 @@ import ( resourceapi "k8s.io/api/resource/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/dynamic-resource-allocation/resourceslice" drapbv1 "k8s.io/kubelet/pkg/apis/dra/v1beta1" "k8s.io/utils/ptr" cdiapi "tags.cncf.io/container-device-interface/pkg/cdi" @@ -57,7 +58,7 @@ func NewAllocationStore(nodeName, devicesPath string, resyncPeriod time.Duration resyncPeriod: resyncPeriod, cdi: cdiManager, log: log.With(slog.String("component", "usb-allocation-store")), - updateChannel: make(chan []resourceapi.Device, 2), + updateChannel: make(chan resourceslice.DriverResources, 2), discoverPluggedUSBDevices: NewDeviceSet(), allocatableDevices: make(map[string]resourceapi.Device), allocatedDevices: set.New[string](), @@ -85,7 +86,7 @@ type AllocationStore struct { monitor *monitor - updateChannel chan []resourceapi.Device + updateChannel chan resourceslice.DriverResources mu sync.RWMutex discoverPluggedUSBDevices *DeviceSet @@ -119,7 +120,7 @@ func (s *AllocationStore) sync() error { s.allocatableDevices = allocatableDevicesByName - s.updateChannel <- allocatableDevices + s.updateChannel <- s.makeResources(allocatableDevices) return nil } @@ -159,7 +160,7 @@ func (s *AllocationStore) Start(ctx context.Context) error { return nil } -func (s *AllocationStore) UpdateChannel() chan []resourceapi.Device { +func (s *AllocationStore) UpdateChannel() chan resourceslice.DriverResources { return s.updateChannel } @@ -417,3 +418,23 @@ func parseDraEnvToClaimAllocations(envs []string) (map[types.UID][]string, error return result, nil } + +func (s *AllocationStore) makeResources(devices []resourceapi.Device) resourceslice.DriverResources { + slice := resourceslice.Slice{ + Devices: devices, + } + poolName := s.nodeName + + if featuregates.Default().USBGatewayEnabled() { + slice.PerDeviceNodeSelection = ptr.To(true) + poolName = "usb-gateway" + } + + return resourceslice.DriverResources{ + Pools: map[string]resourceslice.Pool{ + poolName: { + Slices: []resourceslice.Slice{slice}, + }, + }, + } +} From ccf5a565fda97a3ef5cc4f902f2b1f9dcb4ec9da Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Wed, 14 Jan 2026 11:39:56 +0300 Subject: [PATCH 18/29] fix usb gateway status Signed-off-by: Yaroslav Borbat --- Taskfile.yaml | 36 ++++++- .../debug/dlv.Dockerfile | 32 +++++++ images/virtualization-dra/api/sheme.go | 36 +++++++ images/virtualization-dra/api/types.go | 36 ++++++- .../cmd/usb-gateway/app/init.go | 1 + .../controller/resourceclaim/controller.go | 94 ++++++++++++++----- .../internal/usb/convert.go | 30 +++--- .../virtualization-dra/internal/usb/store.go | 16 +++- .../internal/usbip/binder.go | 64 ++++++------- 9 files changed, 267 insertions(+), 78 deletions(-) create mode 100644 images/virtualization-dra-usb-gateway/debug/dlv.Dockerfile create mode 100644 images/virtualization-dra/api/sheme.go diff --git a/Taskfile.yaml b/Taskfile.yaml index 6703c2ad6e..ea1475e669 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -314,5 +314,39 @@ tasks: } } }' - kubectl -n d8-virtualization port-forward deploy/virtualization-dra 2345:2345 + kubectl -n d8-virtualization port-forward pod/ 2345:2345 + EOF + + dlv:virtualization-dra-usb-gateway:build: + desc: "Build image virtualization-dra-usb-gateway with dlv" + cmds: + - docker build --build-arg BRANCH=$BRANCH -f ./images/virtualization-dra-usb-gateway/debug/dlv.Dockerfile -t "{{ .DLV_IMAGE }}" --platform linux/amd64 . + + dlv:virtualization-dra-usb-gateway:build-push: + desc: "Build and Push image virtualization-dra-usb-gateway with dlv" + cmds: + - task: dlv:virtualization-dra-usb-gateway:build + - docker push "{{ .DLV_IMAGE }}" + - task: dlv:virtualization-dra-usb-gateway:print + + dlv:virtualization-dra-usb-gateway:print: + desc: "Print commands for debug" + env: + IMAGE: "{{ .DLV_IMAGE }}" + cmd: | + cat < 2345:2345 EOF diff --git a/images/virtualization-dra-usb-gateway/debug/dlv.Dockerfile b/images/virtualization-dra-usb-gateway/debug/dlv.Dockerfile new file mode 100644 index 0000000000..99c8469601 --- /dev/null +++ b/images/virtualization-dra-usb-gateway/debug/dlv.Dockerfile @@ -0,0 +1,32 @@ +FROM golang:1.25-bookworm@sha256:019c22232e57fda8ded2b10a8f201989e839f3d3f962d4931375069bbb927e03 AS builder +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /app/images/virtualization-dra +RUN go install github.com/go-delve/delve/cmd/dlv@latest + +COPY ./images/virtualization-dra/go.mod /app/images/virtualization-dra/ +COPY ./images/virtualization-dra/go.sum /app/images/virtualization-dra/ + +RUN go mod download + +COPY ./images/virtualization-dra/cmd /app/images/virtualization-dra/cmd +COPY ./images/virtualization-dra/internal /app/images/virtualization-dra/internal +COPY ./images/virtualization-dra/pkg /app/images/virtualization-dra/pkg +COPY ./images/virtualization-dra/api /app/images/virtualization-dra/api + +ENV GO111MODULE=on +ENV GOOS=${TARGETOS:-linux} +ENV GOARCH=${TARGETARCH:-amd64} +ENV CGO_ENABLED=0 + +RUN go build -gcflags "all=-N -l" -a -o virtualization-dra-usb-gateway ./cmd/usb-gateway/main.go + +FROM busybox:1.36.1-glibc + +WORKDIR /app +COPY --from=builder /go/bin/dlv /app/dlv +COPY --from=builder /app/images/virtualization-dra/virtualization-dra-usb-gateway /app/virtualization-dra-usb-gateway +USER 65532:65532 + +ENTRYPOINT ["./dlv", "--listen=:2345", "--headless=true", "--continue", "--log=true", "--log-output=debugger,debuglineerr,gdbwire,lldbout,rpc", "--accept-multiclient", "--api-version=2", "exec", "./virtualization-dra-usb-gateway", "--"] diff --git a/images/virtualization-dra/api/sheme.go b/images/virtualization-dra/api/sheme.go new file mode 100644 index 0000000000..2a6eeb290f --- /dev/null +++ b/images/virtualization-dra/api/sheme.go @@ -0,0 +1,36 @@ +package api + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +const Version = "v1alpha2" +const Group = "usb-gateway.virtualization.deckhouse.io" + +var SchemeGroupVersion = schema.GroupVersion{Group: Group, Version: Version} + +var ( + Scheme = runtime.NewScheme() + Codecs = serializer.NewCodecFactory(Scheme) +) + +var ( + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + AddToScheme = SchemeBuilder.AddToScheme +) + +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &USBGatewayStatus{}, + ) + return nil +} + +func init() { + metav1.AddToGroupVersion(Scheme, SchemeGroupVersion) + utilruntime.Must(AddToScheme(Scheme)) +} diff --git a/images/virtualization-dra/api/types.go b/images/virtualization-dra/api/types.go index abdadcbaa6..8f09b29f99 100644 --- a/images/virtualization-dra/api/types.go +++ b/images/virtualization-dra/api/types.go @@ -17,10 +17,14 @@ limitations under the License. package api import ( + "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) +const USBGatewayStatusKind = "USBGatewayStatus" + // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object type USBGatewayStatus struct { @@ -36,13 +40,35 @@ type USBGatewayStatus struct { Attached bool `json:"attached"` } -func FromData(data *runtime.RawExtension) *USBGatewayStatus { +func FromData(data *runtime.RawExtension) (*USBGatewayStatus, error) { if data == nil { - return nil + return nil, nil + } + + obj, err := runtime.Decode(Codecs.UniversalDecoder(SchemeGroupVersion), data.Raw) + if err != nil { + return nil, fmt.Errorf("failed to decode USBGatewayStatus: %w", err) } - status, ok := data.Object.(*USBGatewayStatus) + status, ok := obj.(*USBGatewayStatus) if !ok { - return nil + return nil, fmt.Errorf("failed to decode USBGatewayStatus: unexpected object type: %T", obj) } - return status + + return status, nil +} + +func ToData(status *USBGatewayStatus) (*runtime.RawExtension, error) { + if status == nil { + return nil, nil + } + + raw, err := runtime.Encode(Codecs.LegacyCodec(SchemeGroupVersion), status) + if err != nil { + return nil, fmt.Errorf("failed to encode USBGatewayStatus: %w", err) + } + + return &runtime.RawExtension{ + Raw: raw, + Object: status, + }, nil } diff --git a/images/virtualization-dra/cmd/usb-gateway/app/init.go b/images/virtualization-dra/cmd/usb-gateway/app/init.go index ebba3886ff..83511a71dc 100644 --- a/images/virtualization-dra/cmd/usb-gateway/app/init.go +++ b/images/virtualization-dra/cmd/usb-gateway/app/init.go @@ -47,6 +47,7 @@ func (o *initOptions) Run(_ *cobra.Command, _ []string) error { modules := []string{ filepath.Join("/lib/modules", kernelRelease, "kernel/drivers/usb/usbip/usbip-core.ko"), + filepath.Join("/lib/modules", kernelRelease, "kernel/drivers/usb/usbip/usbip-host.ko"), filepath.Join("/lib/modules", kernelRelease, "kernel/drivers/usb/usbip/vhci-hcd.ko"), } diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go index bf6c216a4c..9b57c957bd 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go @@ -28,7 +28,6 @@ import ( resourcev1beta1 "k8s.io/api/resource/v1beta1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" @@ -246,7 +245,11 @@ func (c *Controller) sync(key string) error { // TODO: handle detach too func (c *Controller) handleDelete(rc *resourcev1beta1.ResourceClaim) error { - if c.allUnBound(rc) { + unbound, err := c.allUnBound(rc) + if err != nil { + return err + } + if unbound { return c.removeFinalizer(rc) } @@ -265,7 +268,10 @@ func (c *Controller) handleDelete(rc *resourcev1beta1.ResourceClaim) error { for i := range rc.Status.Devices { allocatedDeviceStatus := &rc.Status.Devices[i] - usbGatewayStatus := vdraapi.FromData(allocatedDeviceStatus.Data) + usbGatewayStatus, err := vdraapi.FromData(allocatedDeviceStatus.Data) + if err != nil { + return err + } if usbGatewayStatus == nil { continue } @@ -292,7 +298,10 @@ func (c *Controller) handleDelete(rc *resourcev1beta1.ResourceClaim) error { } usbGatewayStatus.Bound = false - allocatedDeviceStatus.Data.Object = usbGatewayStatus + allocatedDeviceStatus.Data, err = vdraapi.ToData(usbGatewayStatus) + if err != nil { + return err + } shouldUpdate = true } @@ -306,18 +315,21 @@ func (c *Controller) handleDelete(rc *resourcev1beta1.ResourceClaim) error { return nil } -func (c *Controller) allUnBound(rc *resourcev1beta1.ResourceClaim) bool { +func (c *Controller) allUnBound(rc *resourcev1beta1.ResourceClaim) (bool, error) { for _, deviceStatus := range rc.Status.Devices { - usbGatewayStatus := vdraapi.FromData(deviceStatus.Data) + usbGatewayStatus, err := vdraapi.FromData(deviceStatus.Data) + if err != nil { + return false, err + } if usbGatewayStatus == nil { continue } if usbGatewayStatus.Bound { - return false + return false, nil } } - return true + return true, nil } func (c *Controller) addFinalizer(rc *resourcev1beta1.ResourceClaim) error { @@ -366,18 +378,49 @@ func (c *Controller) handleServer(rc *resourcev1beta1.ResourceClaim, myAllocatio index, ok := indexAllocDevice[device.Name] if !ok { - continue + var ( + driver string + pool string + ) + if rc.Status.Allocation != nil { + for _, result := range rc.Status.Allocation.Devices.Results { + if result.Device == device.Name { + driver = result.Driver + pool = result.Pool + } + } + } + if driver == "" || pool == "" { + return fmt.Errorf("device %s is not allocated, driver or pool is empty", device.Name) + } + + rc.Status.Devices = append(rc.Status.Devices, resourcev1beta1.AllocatedDeviceStatus{ + Driver: driver, + Pool: pool, + Device: device.Name, + }) } allocDeviceStatus := &rc.Status.Devices[index] - usbGatewayStatus := vdraapi.FromData(allocDeviceStatus.Data) + usbGatewayStatus, err := vdraapi.FromData(allocDeviceStatus.Data) + if err != nil { + return err + } targetIPAlreadySet := usbGatewayStatus != nil && usbGatewayStatus.TargetIP != "" - if targetIPAlreadySet { + targetIPWrong := usbGatewayStatus != nil && usbGatewayStatus.TargetIP != c.podIP.String() + + if targetIPAlreadySet && !targetIPWrong { continue } - usbGatewayStatus = &vdraapi.USBGatewayStatus{} + + usbGatewayStatus = &vdraapi.USBGatewayStatus{ + TypeMeta: metav1.TypeMeta{ + APIVersion: vdraapi.SchemeGroupVersion.String(), + Kind: vdraapi.USBGatewayStatusKind, + }, + } busID := "" if attr, ok := device.Basic.Attributes["busID"]; ok && attr.StringValue != nil { @@ -401,10 +444,12 @@ func (c *Controller) handleServer(rc *resourcev1beta1.ResourceClaim, myAllocatio usbGatewayStatus.TargetPort = c.usbipPort usbGatewayStatus.Bound = true - if allocDeviceStatus.Data == nil { - allocDeviceStatus.Data = &runtime.RawExtension{} + data, err := vdraapi.ToData(usbGatewayStatus) + if err != nil { + return err } - allocDeviceStatus.Data.Object = usbGatewayStatus + + allocDeviceStatus.Data = data shouldUpdate = true } @@ -439,7 +484,10 @@ func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAlloca continue } allocDeviceStatus := &rc.Status.Devices[index] - usbGatewayStatus := vdraapi.FromData(allocDeviceStatus.Data) + usbGatewayStatus, err := vdraapi.FromData(allocDeviceStatus.Data) + if err != nil { + return err + } if usbGatewayStatus == nil { continue } @@ -456,16 +504,16 @@ func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAlloca continue } - err := c.usbIP.Attach(usbGatewayStatus.TargetIP, busID, usbGatewayStatus.TargetPort) + err = c.usbIP.Attach(usbGatewayStatus.TargetIP, busID, usbGatewayStatus.TargetPort) if err != nil { return fmt.Errorf("failed to attach usb: %w", err) } usbGatewayStatus.Attached = true - if allocDeviceStatus.Data == nil { - allocDeviceStatus.Data = &runtime.RawExtension{} + allocDeviceStatus.Data, err = vdraapi.ToData(usbGatewayStatus) + if err != nil { + return err } - allocDeviceStatus.Data.Object = usbGatewayStatus shouldUpdate = true } @@ -573,10 +621,14 @@ func (c *Controller) getAllocationDevices(rc *resourcev1beta1.ResourceClaim) ([] } // now, driver virtualization-dra supports only usb, but we can add more devices later // so we need to check if the device is usb - if strings.HasPrefix(status.Device, "usb") { + if !strings.HasPrefix(status.Device, "usb") { continue } + if _, exists := allocResultsByPool[status.Pool]; !exists { + allocResultsByPool[status.Pool] = make(map[string]resourcev1beta1.DeviceRequestAllocationResult) + } + allocResultsByPool[status.Pool][status.Device] = status } diff --git a/images/virtualization-dra/internal/usb/convert.go b/images/virtualization-dra/internal/usb/convert.go index 542c8b11bf..500136d3a2 100644 --- a/images/virtualization-dra/internal/usb/convert.go +++ b/images/virtualization-dra/internal/usb/convert.go @@ -83,19 +83,7 @@ func convertToAPIDevice(usbDevice Device, nodeName string) *resourceapi.Device { if featuregates.Default().USBGatewayEnabled() { // TODO: need pr to deckhouse for enable DRAPartitionableDevices feature gate on ApiServer - device.NodeSelector = &corev1.NodeSelector{ - NodeSelectorTerms: []corev1.NodeSelectorTerm{ - { - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: common.USBGatewayLabel, - Operator: corev1.NodeSelectorOpIn, - Values: []string{"true"}, - }, - }, - }, - }, - } + //device.NodeSelector = getNodeSelector() // TODO: add support for multiple allocations // device.AllowMultipleAllocations = ptr.To(true) } else { @@ -104,3 +92,19 @@ func convertToAPIDevice(usbDevice Device, nodeName string) *resourceapi.Device { return device } + +func getNodeSelector() *corev1.NodeSelector { + return &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: common.USBGatewayLabel, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"true"}, + }, + }, + }, + }, + } +} diff --git a/images/virtualization-dra/internal/usb/store.go b/images/virtualization-dra/internal/usb/store.go index 7190f29200..0e6fefa704 100644 --- a/images/virtualization-dra/internal/usb/store.go +++ b/images/virtualization-dra/internal/usb/store.go @@ -420,21 +420,27 @@ func parseDraEnvToClaimAllocations(envs []string) (map[types.UID][]string, error } func (s *AllocationStore) makeResources(devices []resourceapi.Device) resourceslice.DriverResources { + //time.Sleep(20 * time.Second) slice := resourceslice.Slice{ Devices: devices, } poolName := s.nodeName if featuregates.Default().USBGatewayEnabled() { - slice.PerDeviceNodeSelection = ptr.To(true) - poolName = "usb-gateway" + //slice.PerDeviceNodeSelection = ptr.To(true) + } + + pool := resourceslice.Pool{ + Slices: []resourceslice.Slice{slice}, + } + + if featuregates.Default().USBGatewayEnabled() { + pool.NodeSelector = getNodeSelector() } return resourceslice.DriverResources{ Pools: map[string]resourceslice.Pool{ - poolName: { - Slices: []resourceslice.Slice{slice}, - }, + poolName: pool, }, } } diff --git a/images/virtualization-dra/internal/usbip/binder.go b/images/virtualization-dra/internal/usbip/binder.go index 226eabc6a8..027a54e242 100644 --- a/images/virtualization-dra/internal/usbip/binder.go +++ b/images/virtualization-dra/internal/usbip/binder.go @@ -54,7 +54,8 @@ func (b *usbBinder) Bind(busID string) error { return fmt.Errorf("failed to bind usb device: %w: %w", err, b.modifyMatchBusID(busID, false)) } - return b.storeBind(busID, true) + return nil + // return b.storeBind(busID, true) } // Unbind unbinds the USB device from the USBIP server. @@ -65,7 +66,7 @@ func (b *usbBinder) Unbind(busID string) error { return fmt.Errorf("device with bus ID %s does not exist: %w", busID, err) } - if devInfo.Driver != usbipHostDriverName { + if b.isBound(devInfo) { return fmt.Errorf("device %s is not bound to %s driver", devInfo.BusID, usbipHostDriverName) } @@ -83,11 +84,16 @@ func (b *usbBinder) Unbind(busID string) error { return fmt.Errorf("failed to rebind usb device %s: %w", busID, err) } - return b.storeBind(busID, false) + return nil + // return b.storeBind(busID, false) } func (b *usbBinder) IsBound(busID string) (bool, error) { - return b.isBound(busID) + devInfo, err := b.getUSBDeviceInfo(busID) + if err != nil { + return false, fmt.Errorf("device with bus ID %s does not exist: %w", busID, err) + } + return b.isBound(devInfo), nil } type usbDeviceInfo struct { @@ -132,37 +138,29 @@ func (b *usbBinder) getUSBDeviceInfo(busID string) (*usbDeviceInfo, error) { return info, nil } -func (b *usbBinder) storeBind(busID string, bind bool) error { - bound, err := b.isBound(busID) - if err != nil { - return err - } - if bound == bind { - return nil - } - path := bindPath(busID) - if bind { - _, err = os.Create(path) - return err - } - return os.Remove(path) -} +//func (b *usbBinder) storeBind(busID string, bind bool) error { +// bound, err := b.isBound(busID) +// if err != nil { +// return err +// } +// if bound == bind { +// return nil +// } +// path := bindPath(busID) +// if bind { +// _, err = os.Create(path) +// return err +// } +// return os.Remove(path) +//} -func (b *usbBinder) isBound(busID string) (bool, error) { - path := bindPath(busID) - _, err := os.Stat(path) - if err != nil { - if os.IsNotExist(err) { - return false, nil - } - return false, err - } - return true, nil +func (b *usbBinder) isBound(devInfo *usbDeviceInfo) bool { + return devInfo.Driver == usbipHostDriverName } -func bindPath(busID string) string { - return filepath.Join(getUSBDevicePath(busID), "usbip_bound") -} +//func bindPath(busID string) string { +// return filepath.Join(getUSBDevicePath(busID), "usbip_bound") +//} func (b *usbBinder) unbindOther(devInfo *usbDeviceInfo) error { if devInfo.IsHub { @@ -174,7 +172,7 @@ func (b *usbBinder) unbindOther(devInfo *usbDeviceInfo) error { return nil } - if devInfo.Driver == usbipHostDriverName { + if b.isBound(devInfo) { return fmt.Errorf("device %s is already bound to %s", devInfo.BusID, usbipHostDriverName) } From 9c7c2bb04275ad2dfa8cbc8b4c8353fdbc4916e1 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Wed, 14 Jan 2026 15:49:25 +0300 Subject: [PATCH 19/29] fix dra usbgateway devices Signed-off-by: Yaroslav Borbat --- images/virtualization-dra/api/types.go | 15 +- .../cmd/go-usbip/app/app.go | 1 + .../cmd/go-usbip/app/attach.go | 3 +- .../cmd/go-usbip/app/info.go | 46 +++++ .../controller/resourceclaim/controller.go | 28 ++- .../internal/usb/discovery.go | 38 +++- .../virtualization-dra/internal/usb/store.go | 170 +++++++++++------- .../internal/usbip/attacher.go | 35 +++- .../internal/usbip/interfaces.go | 12 +- .../virtualization-dra/internal/usbip/vhci.go | 2 + 10 files changed, 260 insertions(+), 90 deletions(-) create mode 100644 images/virtualization-dra/cmd/go-usbip/app/info.go diff --git a/images/virtualization-dra/api/types.go b/images/virtualization-dra/api/types.go index 8f09b29f99..7a729f0dd5 100644 --- a/images/virtualization-dra/api/types.go +++ b/images/virtualization-dra/api/types.go @@ -30,14 +30,13 @@ const USBGatewayStatusKind = "USBGatewayStatus" type USBGatewayStatus struct { metav1.TypeMeta `json:",inline"` - BusNum int `json:"busNum"` - DeviceNum int `json:"deviceNum"` - DevicePath string `json:"devicePath"` - - TargetIP string `json:"targetIP"` - TargetPort int `json:"targetPort"` - Bound bool `json:"bound"` - Attached bool `json:"attached"` + BusID string `json:"busID"` + + RemoteIP string `json:"remoteIP"` + RemotePort int `json:"remotePort"` + + Bound bool `json:"bound"` + Attached bool `json:"attached"` } func FromData(data *runtime.RawExtension) (*USBGatewayStatus, error) { diff --git a/images/virtualization-dra/cmd/go-usbip/app/app.go b/images/virtualization-dra/cmd/go-usbip/app/app.go index fbb1fe327f..e4801ed497 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/app.go +++ b/images/virtualization-dra/cmd/go-usbip/app/app.go @@ -45,6 +45,7 @@ func NewUSBIPCommand() *cobra.Command { NewAttachCommand(), NewDetachCommand(), NewUsedPortsCommand(), + NewUsedInfoCommand(), ) return cmd diff --git a/images/virtualization-dra/cmd/go-usbip/app/attach.go b/images/virtualization-dra/cmd/go-usbip/app/attach.go index 818f983a00..1e73b92519 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/attach.go +++ b/images/virtualization-dra/cmd/go-usbip/app/attach.go @@ -55,5 +55,6 @@ func (o *attachOptions) AddFlags(fs *pflag.FlagSet) { func (o *attachOptions) Run(_ *cobra.Command, args []string) error { host := args[0] busID := args[1] - return usbip.NewUSBAttacher().Attach(host, busID, o.port) + _, err := usbip.NewUSBAttacher().Attach(host, busID, o.port) + return err } diff --git a/images/virtualization-dra/cmd/go-usbip/app/info.go b/images/virtualization-dra/cmd/go-usbip/app/info.go new file mode 100644 index 0000000000..c386ca1ee5 --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/app/info.go @@ -0,0 +1,46 @@ +package app + +import ( + "encoding/json" + "fmt" + + "github.com/spf13/cobra" + + "github.com/deckhouse/virtualization-dra/internal/usbip" +) + +func NewUsedInfoCommand() *cobra.Command { + o := &usedInfoOptions{} + cmd := &cobra.Command{ + Use: "info", + Short: "Get used info", + Example: o.Usage(), + RunE: o.Run, + } + + return cmd +} + +type usedInfoOptions struct{} + +func (o *usedInfoOptions) Usage() string { + return ` # Get used info + $ go-usbip info +` +} + +func (o *usedInfoOptions) Run(cmd *cobra.Command, _ []string) error { + infos, err := usbip.NewUSBAttacher().GetUsedInfo() + if err != nil { + return err + } + + bytes, err := json.Marshal(infos) + if err != nil { + return fmt.Errorf("failed to marshal json: %w", err) + } + + cmd.Println(string(bytes)) + + return nil +} diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go index 9b57c957bd..a00f4d6587 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go @@ -408,8 +408,8 @@ func (c *Controller) handleServer(rc *resourcev1beta1.ResourceClaim, myAllocatio return err } - targetIPAlreadySet := usbGatewayStatus != nil && usbGatewayStatus.TargetIP != "" - targetIPWrong := usbGatewayStatus != nil && usbGatewayStatus.TargetIP != c.podIP.String() + targetIPAlreadySet := usbGatewayStatus != nil && usbGatewayStatus.RemoteIP != "" + targetIPWrong := usbGatewayStatus != nil && usbGatewayStatus.RemoteIP != c.podIP.String() if targetIPAlreadySet && !targetIPWrong { continue @@ -440,8 +440,8 @@ func (c *Controller) handleServer(rc *resourcev1beta1.ResourceClaim, myAllocatio } } - usbGatewayStatus.TargetIP = c.podIP.String() - usbGatewayStatus.TargetPort = c.usbipPort + usbGatewayStatus.RemoteIP = c.podIP.String() + usbGatewayStatus.RemotePort = c.usbipPort usbGatewayStatus.Bound = true data, err := vdraapi.ToData(usbGatewayStatus) @@ -504,12 +504,30 @@ func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAlloca continue } - err = c.usbIP.Attach(usbGatewayStatus.TargetIP, busID, usbGatewayStatus.TargetPort) + rhport, err := c.usbIP.Attach(usbGatewayStatus.RemoteIP, busID, usbGatewayStatus.RemotePort) if err != nil { return fmt.Errorf("failed to attach usb: %w", err) } + infos, err := c.usbIP.GetUsedInfo() + if err != nil { + return fmt.Errorf("failed to get used info: %w", err) + } + + var usedInfo *usbip.UsedInfo + for _, info := range infos { + if info.Port == rhport { + usedInfo = &info + break + } + } + if usedInfo == nil { + return fmt.Errorf("failed to find used info for port %d", rhport) + } + usbGatewayStatus.Attached = true + usbGatewayStatus.BusID = usedInfo.LocalBusID + allocDeviceStatus.Data, err = vdraapi.ToData(usbGatewayStatus) if err != nil { return err diff --git a/images/virtualization-dra/internal/usb/discovery.go b/images/virtualization-dra/internal/usb/discovery.go index 28b72771d3..6d787d97a0 100644 --- a/images/virtualization-dra/internal/usb/discovery.go +++ b/images/virtualization-dra/internal/usb/discovery.go @@ -17,20 +17,50 @@ limitations under the License. package usb import ( + "github.com/deckhouse/virtualization-dra/internal/featuregates" + "github.com/deckhouse/virtualization-dra/internal/usbip" "github.com/deckhouse/virtualization-dra/pkg/usb" ) const PathToUSBDevices = usb.PathToUSBDevices -func discoverPluggedUSBDevices(pathToUSBDevices string) (*DeviceSet, error) { +func newDiscoverer() discoverer { + return discoverer{ + getter: usbip.NewUSBAttacher(), + } +} + +type discoverer struct { + getter usbip.USBInfoGetter +} + +func (d *discoverer) DiscoveryPluggedUSBDevices(pathToUSBDevices string) (*DeviceSet, *DeviceSet, error) { devices, err := usb.DiscoverPluggedUSBDevices(pathToUSBDevices) if err != nil { - return nil, err + return nil, nil, err } + + busIdMaps := make(map[string]struct{}) + if featuregates.Default().USBGatewayEnabled() { + infos, err := d.getter.GetUsedInfo() + if err != nil { + return nil, nil, err + } + for _, info := range infos { + busIdMaps[info.LocalBusID] = struct{}{} + } + } + usbDeviceSet := NewDeviceSet() + usbipDeviceSet := NewDeviceSet() + for _, device := range devices { - usbDeviceSet.Add(toDevice(device)) + if _, ok := busIdMaps[device.BusID]; ok { + usbipDeviceSet.Add(toDevice(device)) + } else { + usbDeviceSet.Add(toDevice(device)) + } } - return usbDeviceSet, nil + return usbDeviceSet, usbipDeviceSet, nil } diff --git a/images/virtualization-dra/internal/usb/store.go b/images/virtualization-dra/internal/usb/store.go index 0e6fefa704..7d9217a882 100644 --- a/images/virtualization-dra/internal/usb/store.go +++ b/images/virtualization-dra/internal/usb/store.go @@ -20,14 +20,12 @@ import ( "context" "fmt" "log/slog" - "strconv" "strings" "sync" "time" "github.com/containerd/nri/pkg/api" resourceapi "k8s.io/api/resource/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/dynamic-resource-allocation/resourceslice" drapbv1 "k8s.io/kubelet/pkg/apis/dra/v1beta1" @@ -36,10 +34,8 @@ import ( cdispec "tags.cncf.io/container-device-interface/specs-go" vdraapi "github.com/deckhouse/virtualization-dra/api" - "github.com/deckhouse/virtualization-dra/internal/common" - "github.com/deckhouse/virtualization-dra/internal/featuregates" - "github.com/deckhouse/virtualization-dra/internal/cdi" + "github.com/deckhouse/virtualization-dra/internal/featuregates" "github.com/deckhouse/virtualization-dra/pkg/set" ) @@ -63,6 +59,7 @@ func NewAllocationStore(nodeName, devicesPath string, resyncPeriod time.Duration allocatableDevices: make(map[string]resourceapi.Device), allocatedDevices: set.New[string](), resourceClaimAllocations: make(map[types.UID][]string), + discoverer: newDiscoverer(), } monitor := newUSBMonitor(monitorCallback{ @@ -89,20 +86,26 @@ type AllocationStore struct { updateChannel chan resourceslice.DriverResources mu sync.RWMutex - discoverPluggedUSBDevices *DeviceSet - allocatableDevices map[string]resourceapi.Device - allocatedDevices *set.Set[string] - resourceClaimAllocations map[types.UID][]string + discoverer discoverer + + discoverPluggedUSBDevices *DeviceSet + discoverUsbIpPluggedUSBDevices *DeviceSet + allocatableDevices map[string]resourceapi.Device + + allocatedDevices *set.Set[string] + resourceClaimAllocations map[types.UID][]string } func (s *AllocationStore) sync() error { s.mu.Lock() defer s.mu.Unlock() - discoverPluggedUSBDevices, err := discoverPluggedUSBDevices(s.devicesPath) + discoverPluggedUSBDevices, discoverUsbIpPluggedUSBDevices, err := s.discoverer.DiscoveryPluggedUSBDevices(s.devicesPath) if err != nil { return err } + s.discoverUsbIpPluggedUSBDevices = discoverUsbIpPluggedUSBDevices + if discoverPluggedUSBDevices.Equal(s.discoverPluggedUSBDevices) { return nil } @@ -175,11 +178,10 @@ func (s *AllocationStore) Prepare(_ context.Context, claim *resourceapi.Resource claimUID := string(claim.UID) preparedDevices := make(cdi.PreparedDevices, len(claim.Status.Allocation.Devices.Results)) + usbGatewayEnabled := featuregates.Default().USBGatewayEnabled() + for i, result := range claim.Status.Allocation.Devices.Results { - usbDevice, exists := s.allocatableDevices[result.Device] - if !exists { - return nil, fmt.Errorf("requested device is not allocatable: %v", result.Device) - } + // TODO: unnecessary? // kubernetes check allocatable devices // Warning FailedScheduling 8s default-scheduler 0/3 nodes are available: @@ -190,9 +192,45 @@ func (s *AllocationStore) Prepare(_ context.Context, claim *resourceapi.Resource return nil, fmt.Errorf("device %v is already allocated", result.Device) } - containerEditsOptions, err := newContainerEditsOptions(&usbDevice, claim) - if err != nil { - return nil, err + isUSBGatewayRequest := s.isUSBGatewayRequest(&result) + + if !usbGatewayEnabled && isUSBGatewayRequest { + return nil, fmt.Errorf("claim %s/%s has usb gateway request but usb gateway is disabled", claim.Namespace, claim.Name) + } + + var containerEditsOptions containerEditsOptions + + if isUSBGatewayRequest { + + usbGatewayStatus, err := s.getUsbGatewayStatus(claim, result.Device) + if err != nil { + return nil, err + } + + if !usbGatewayStatus.Attached { + return nil, fmt.Errorf("claim %s/%s has usb gateway request but usb gateway is not attached", claim.Namespace, claim.Name) + } + + usbDevice := s.getUsbGatewayUsbDevice(usbGatewayStatus.BusID) + if usbDevice == nil { + return nil, fmt.Errorf("usb device %s is not found", usbGatewayStatus.BusID) + } + + containerEditsOptions = newContainerEditsOptionsForUSBGateway(result.Device, usbDevice) + + } else { + + usbDevice, exists := s.allocatableDevices[result.Device] + if !exists { + return nil, fmt.Errorf("requested device is not allocatable: %v", result.Device) + } + + opts, err := newContainerEditsOptions(&usbDevice) + if err != nil { + return nil, err + } + containerEditsOptions = opts + } edits, err := s.makeContainerEdits(claimUID, containerEditsOptions) @@ -225,58 +263,61 @@ func (s *AllocationStore) Prepare(_ context.Context, claim *resourceapi.Resource return devices, nil } -func newContainerEditsOptions(device *resourceapi.Device, claim *resourceapi.ResourceClaim) (containerEditsOptions, error) { - opts := containerEditsOptions{ - Name: device.Name, +func (s *AllocationStore) getUsbGatewayStatus(claim *resourceapi.ResourceClaim, deviceName string) (*vdraapi.USBGatewayStatus, error) { + for _, allocDeviceStatus := range claim.Status.Devices { + if allocDeviceStatus.Device == deviceName { + return vdraapi.FromData(allocDeviceStatus.Data) + } } + return nil, fmt.Errorf("device %s is not allocated", deviceName) +} - if featuregates.Default().USBGatewayEnabled() && isUSBGateway(claim) { - var data *runtime.RawExtension - for _, deviceStatus := range claim.Status.Devices { - if deviceStatus.Device == device.Name { - data = deviceStatus.Data - break - } - } - if data == nil { - return opts, fmt.Errorf("device status data is not found") +func (s *AllocationStore) getUsbGatewayUsbDevice(busID string) *Device { + for _, device := range s.discoverUsbIpPluggedUSBDevices.Slice() { + if device.BusID == busID { + return &device } + } + return nil +} - usbGatewayStatus, ok := data.Object.(*vdraapi.USBGatewayStatus) - if !ok { - return opts, fmt.Errorf("device status data is not a USBGatewayStatus") - } - if usbGatewayStatus == nil { - return opts, fmt.Errorf("device status data Object is nil") - } +func newContainerEditsOptionsForUSBGateway(deviceName string, usbDevice *Device) containerEditsOptions { + return containerEditsOptions{ + Name: deviceName, + DevicePath: usbDevice.DevicePath, + DeviceNum: usbDevice.DeviceNumber.String(), + Bus: usbDevice.Bus.String(), + Major: int64(usbDevice.Major), + Minor: int64(usbDevice.Minor), + } +} - opts.Bus = strconv.Itoa(usbGatewayStatus.BusNum) - opts.DeviceNum = strconv.Itoa(usbGatewayStatus.DeviceNum) - opts.DevicePath = usbGatewayStatus.DevicePath +func newContainerEditsOptions(device *resourceapi.Device) (containerEditsOptions, error) { + opts := containerEditsOptions{ + Name: device.Name, + } - } else { - if attr, ok := device.Attributes["devicePath"]; ok { - if val := attr.StringValue; val != nil { - opts.DevicePath = *val - } else { - return opts, fmt.Errorf("devicePath attribute is not exist") - } + if attr, ok := device.Attributes["devicePath"]; ok { + if val := attr.StringValue; val != nil { + opts.DevicePath = *val + } else { + return opts, fmt.Errorf("devicePath attribute is not exist") } + } - if attr, ok := device.Attributes["deviceNumber"]; ok { - if val := attr.StringValue; val != nil { - opts.DeviceNum = *val - } else { - return opts, fmt.Errorf("deviceNum attribute is not exist") - } + if attr, ok := device.Attributes["deviceNumber"]; ok { + if val := attr.StringValue; val != nil { + opts.DeviceNum = *val + } else { + return opts, fmt.Errorf("deviceNum attribute is not exist") } + } - if attr, ok := device.Attributes["bus"]; ok { - if val := attr.StringValue; val != nil { - opts.Bus = *val - } else { - return opts, fmt.Errorf("bus attribute is not exist") - } + if attr, ok := device.Attributes["bus"]; ok { + if val := attr.StringValue; val != nil { + opts.Bus = *val + } else { + return opts, fmt.Errorf("bus attribute is not exist") } } @@ -299,8 +340,10 @@ func newContainerEditsOptions(device *resourceapi.Device, claim *resourceapi.Res return opts, nil } -func isUSBGateway(claim *resourceapi.ResourceClaim) bool { - return claim.Annotations[common.USBGatewayAnnotation] == "true" +func (s *AllocationStore) isUSBGatewayRequest(result *resourceapi.DeviceRequestAllocationResult) bool { + // virtualization-dra creates slices with pool name by node name + // if pool not equal our node name, it is usb gateway request + return result.Pool != s.nodeName } type containerEditsOptions struct { @@ -420,16 +463,11 @@ func parseDraEnvToClaimAllocations(envs []string) (map[types.UID][]string, error } func (s *AllocationStore) makeResources(devices []resourceapi.Device) resourceslice.DriverResources { - //time.Sleep(20 * time.Second) slice := resourceslice.Slice{ Devices: devices, } poolName := s.nodeName - if featuregates.Default().USBGatewayEnabled() { - //slice.PerDeviceNodeSelection = ptr.To(true) - } - pool := resourceslice.Pool{ Slices: []resourceslice.Slice{slice}, } diff --git a/images/virtualization-dra/internal/usbip/attacher.go b/images/virtualization-dra/internal/usbip/attacher.go index c23f24d1c9..0bec243d73 100644 --- a/images/virtualization-dra/internal/usbip/attacher.go +++ b/images/virtualization-dra/internal/usbip/attacher.go @@ -35,23 +35,23 @@ func NewUSBAttacher() USBAttacher { type usbAttacher struct{} // https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_attach.c#L174 -func (a usbAttacher) Attach(host, busID string, port int) error { +func (a usbAttacher) Attach(host, busID string, port int) (int, error) { conn, err := a.usbipNetTCPConnect(host, fmt.Sprintf("%d", port)) if err != nil { - return fmt.Errorf("failed to connect to usbipd: %w", err) + return -1, fmt.Errorf("failed to connect to usbipd: %w", err) } rhport, err := a.queryImportDevice(conn, busID) if err != nil { - return fmt.Errorf("failed to query import device: %w", err) + return -1, fmt.Errorf("failed to query import device: %w", err) } err = a.recordConnection(host, strconv.Itoa(port), busID, rhport) if err != nil { - return fmt.Errorf("failed to record connection: %w", err) + return -1, fmt.Errorf("failed to record connection: %w", err) } - return nil + return rhport, nil } // https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_detach.c#L32 @@ -121,6 +121,31 @@ func (a usbAttacher) GetUsedPorts() ([]int, error) { return ports, nil } +func (a usbAttacher) GetUsedInfo() ([]UsedInfo, error) { + driver, err := newVhciDriver() + if err != nil { + return nil, fmt.Errorf("failed to get vhci driver: %w", err) + } + + var usedInfos []UsedInfo + + for i := 0; i < driver.nports; i++ { + idev := &driver.idevs[i] + + vstatus := protocol.DeviceStatus(idev.status) + if vstatus == protocol.VDeviceStatusUsed { + usedInfos = append(usedInfos, UsedInfo{ + Port: idev.port, + Busnum: idev.busnum, + Devnum: idev.devnum, + LocalBusID: idev.localBusID, + }) + } + } + + return usedInfos, nil +} + // https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_network.c#L261 func (a usbAttacher) usbipNetTCPConnect(host, port string) (*net.TCPConn, error) { tcpAddr, err := net.ResolveTCPAddr("tcp", net.JoinHostPort(host, port)) diff --git a/images/virtualization-dra/internal/usbip/interfaces.go b/images/virtualization-dra/internal/usbip/interfaces.go index 17062585d6..48ef560530 100644 --- a/images/virtualization-dra/internal/usbip/interfaces.go +++ b/images/virtualization-dra/internal/usbip/interfaces.go @@ -36,9 +36,19 @@ type USBBinder interface { } type USBAttacher interface { - Attach(host, busID string, port int) error + Attach(host, busID string, port int) (int, error) Detach(port int) error + USBInfoGetter +} + +type USBInfoGetter interface { GetUsedPorts() ([]int, error) + GetUsedInfo() ([]UsedInfo, error) +} + +type UsedInfo struct { + Port, Busnum, Devnum int + LocalBusID string } type serverImpl struct { diff --git a/images/virtualization-dra/internal/usbip/vhci.go b/images/virtualization-dra/internal/usbip/vhci.go index f6439f1825..5945bb6b41 100644 --- a/images/virtualization-dra/internal/usbip/vhci.go +++ b/images/virtualization-dra/internal/usbip/vhci.go @@ -54,6 +54,7 @@ type vhciDriver struct { type importDevice struct { hub hubSpeed port, status, devID, busnum, devnum int + localBusID string } type hubSpeed int @@ -190,6 +191,7 @@ func (d *vhciDriver) parseStatus(statusBytes []byte) error { idev.devID = devID idev.busnum = busnum idev.devnum = devnum + idev.localBusID = localBusID if hub == "hs" { idev.hub = hubSpeedHigh From 2dd6552af994b5636e73f58cf33ea7e6f09b7a19 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Wed, 14 Jan 2026 22:13:17 +0300 Subject: [PATCH 20/29] improve attach Signed-off-by: Yaroslav Borbat --- .../mount-points.yaml | 3 +- .../controller/resourceclaim/controller.go | 105 ++++++++--- .../controller/resourceclaim/record.go | 167 ++++++++++++++++++ .../internal/usbip/attacher.go | 2 +- .../internal/usbip/protocol/common.go | 19 ++ .../internal/usbip/usbipd.go | 4 +- .../daemonset.yaml | 4 + 7 files changed, 279 insertions(+), 25 deletions(-) create mode 100644 images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go diff --git a/images/virtualization-dra-usb-gateway/mount-points.yaml b/images/virtualization-dra-usb-gateway/mount-points.yaml index 448596c34b..dfccfafd0c 100644 --- a/images/virtualization-dra-usb-gateway/mount-points.yaml +++ b/images/virtualization-dra-usb-gateway/mount-points.yaml @@ -1,4 +1,5 @@ # A list of pre-created mount points for containerd strict mode. -dirs: [] +dirs: + - /var/run/usb-gateway diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go index a00f4d6587..3a9e118046 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go @@ -68,6 +68,7 @@ type Controller struct { queue workqueue.TypedRateLimitingInterface[string] log *slog.Logger hasSynced cache.InformerSynced + recordManager *recordManager } func NewController(nodeName string, podIP net.IP, usbipPort int, client kubernetes.Interface, resourceClaimInformer, resourceSliceInformer, nodeInformer, podInformer cache.SharedIndexInformer, usbIP usbip.Interface) (*Controller, error) { @@ -76,6 +77,11 @@ func NewController(nodeName string, podIP net.IP, usbipPort int, client kubernet workqueue.TypedRateLimitingQueueConfig[string]{Name: controllerName}, ) + recordManager, err := newRecordManager(DefaultRecordStateDir, usbIP) + if err != nil { + return nil, err + } + c := &Controller{ nodeName: nodeName, podIP: podIP, @@ -88,9 +94,10 @@ func NewController(nodeName string, podIP net.IP, usbipPort int, client kubernet usbIP: usbIP, queue: queue, log: slog.With(slog.String("controller", controllerName)), + recordManager: recordManager, } - _, err := resourceClaimInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + _, err = resourceClaimInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: c.addResourceClaim, UpdateFunc: c.updateResourceClaim, DeleteFunc: c.deleteResourceClaim, @@ -217,12 +224,13 @@ func (c *Controller) sync(key string) error { return nil } + onMyNode := c.podOnMyNode(pod) + myAllocationDevices, otherAllocationDevices, err := c.getAllocationDevices(rc) if err != nil { return err } - onMyNode := c.podOnMyNode(pod) shouldShare := !onMyNode && len(myAllocationDevices) > 0 shouldAttach := onMyNode && len(otherAllocationDevices) > 0 @@ -504,30 +512,15 @@ func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAlloca continue } - rhport, err := c.usbIP.Attach(usbGatewayStatus.RemoteIP, busID, usbGatewayStatus.RemotePort) - if err != nil { - return fmt.Errorf("failed to attach usb: %w", err) - } - - infos, err := c.usbIP.GetUsedInfo() + err = c.recordManager.Refresh() if err != nil { - return fmt.Errorf("failed to get used info: %w", err) + return fmt.Errorf("failed to Refresh record: %w", err) } - var usedInfo *usbip.UsedInfo - for _, info := range infos { - if info.Port == rhport { - usedInfo = &info - break - } - } - if usedInfo == nil { - return fmt.Errorf("failed to find used info for port %d", rhport) + if err = c.attach(busID, usbGatewayStatus); err != nil { + return fmt.Errorf("failed to attach usb: %w", err) } - usbGatewayStatus.Attached = true - usbGatewayStatus.BusID = usedInfo.LocalBusID - allocDeviceStatus.Data, err = vdraapi.ToData(usbGatewayStatus) if err != nil { return err @@ -545,6 +538,76 @@ func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAlloca return nil } +func (c *Controller) attach(busID string, usbGatewayStatus *vdraapi.USBGatewayStatus) error { + entries := c.recordManager.GetEntries() + for _, entry := range entries { + if entry.BusID == busID { + if entry.RemoteBusID != usbGatewayStatus.RemoteIP || entry.RemotePort != usbGatewayStatus.RemotePort { + if err := c.detach(entry.Port); err != nil { + return err + } + } + return nil + } + } + + var attachErr error + var rhport int + + defer func() { + if attachErr != nil { + if err := c.detach(rhport); err != nil { + c.log.Error("failed to detach usb", slog.String("error", err.Error()), slog.Int("port", rhport)) + } + } + }() + + rhport, attachErr = c.usbIP.Attach(usbGatewayStatus.RemoteIP, busID, usbGatewayStatus.RemotePort) + if attachErr != nil { + return fmt.Errorf("failed to attach usb: %w", attachErr) + } + + infos, err := c.usbIP.GetUsedInfo() + if err != nil { + return fmt.Errorf("failed to get used info: %w", err) + } + + var usedInfo *usbip.UsedInfo + for _, info := range infos { + if info.Port == rhport { + usedInfo = &info + break + } + } + if usedInfo == nil { + return fmt.Errorf("failed to find used info for port %d", rhport) + } + + entry := Entry{ + Port: rhport, + RemotePort: usbGatewayStatus.RemotePort, + RemoteIP: usbGatewayStatus.RemoteIP, + RemoteBusID: busID, + BusID: usedInfo.LocalBusID, + } + + if err = c.recordManager.AddEntry(entry); err != nil { + return fmt.Errorf("failed to add entry: %w", err) + } + + usbGatewayStatus.Attached = true + usbGatewayStatus.BusID = usedInfo.LocalBusID + + return nil +} + +func (c *Controller) detach(port int) error { + if err := c.usbIP.Detach(port); err != nil { + return fmt.Errorf("failed to detach usb: %w", err) + } + return nil +} + func (c *Controller) getResourceClaim(key string) (*resourcev1beta1.ResourceClaim, error) { obj, exists, err := c.resourceClaimIndexer.GetByKey(key) if err != nil && !k8serrors.IsNotFound(err) { diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go new file mode 100644 index 0000000000..732e006c83 --- /dev/null +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go @@ -0,0 +1,167 @@ +package resourceclaim + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "sync" + + "github.com/deckhouse/virtualization-dra/internal/usbip" +) + +const DefaultRecordStateDir = "/var/run/usb-gateway" + +type record struct { + Entries []Entry `json:"entries,omitempty"` +} + +type Entry struct { + Port int `json:"port"` + RemotePort int `json:"remotePort" json:"remotePort"` + RemoteIP string `json:"remoteIP" json:"remoteIP"` + RemoteBusID string `json:"remoteBusID" json:"remoteBusID"` + BusID string `json:"busID" json:"busID"` +} + +func (e Entry) Validate() error { + if e.Port <= 0 { + return fmt.Errorf("port is required") + } + if e.RemotePort <= 0 { + return fmt.Errorf("remotePort is required") + } + if e.RemoteIP == "" { + return fmt.Errorf("remoteIP is required") + } + if e.RemoteBusID == "" { + return fmt.Errorf("remoteBusID is required") + } + if e.BusID == "" { + return fmt.Errorf("busID is required") + } + return nil +} + +type recordManager struct { + recordFile string + getter usbip.USBInfoGetter + + mu sync.RWMutex + record record +} + +func newRecordManager(stateDir string, getter usbip.USBInfoGetter) (*recordManager, error) { + err := os.MkdirAll(stateDir, 0700) + if err != nil { + return nil, err + } + + recordFile := filepath.Join(stateDir, "record.json") + if _, err = os.Stat(recordFile); err != nil { + if os.IsNotExist(err) { + f, err := os.Create(recordFile) + if err != nil { + return nil, err + } + _ = f.Close() + } else { + return nil, err + } + } + + r := recordManager{ + recordFile: recordFile, + getter: getter, + } + + if err = r.Refresh(); err != nil { + return nil, fmt.Errorf("failed to Refresh record: %w", err) + } + + return &r, nil +} + +func (r *recordManager) Refresh() error { + r.mu.Lock() + defer r.mu.Unlock() + + infos, err := r.getter.GetUsedInfo() + if err != nil { + return err + } + + byBusId := make(map[string]*usbip.UsedInfo, len(infos)) + for _, info := range infos { + byBusId[info.LocalBusID] = &info + } + + b, err := os.ReadFile(r.recordFile) + if err != nil { + return err + } + + record := record{} + if err = json.Unmarshal(b, &record); err != nil { + return err + } + + // keep only real entries + var newEntries []Entry + for _, e := range record.Entries { + if _, ok := byBusId[e.RemoteBusID]; ok { + newEntries = append(newEntries, e) + } + } + record.Entries = newEntries + + r.record = record + + return nil +} + +func (r *recordManager) GetEntries() []Entry { + r.mu.RLock() + defer r.mu.RUnlock() + + return slices.Clone(r.record.Entries) +} + +func (r *recordManager) AddEntry(e Entry) error { + if err := e.Validate(); err != nil { + return err + } + + r.mu.Lock() + defer r.mu.Unlock() + + for _, entry := range r.record.Entries { + if entry.RemoteBusID == e.RemoteBusID { + return fmt.Errorf("entry with RemoteBusID %s already exists", e.RemoteBusID) + } + if entry.BusID == e.BusID { + return fmt.Errorf("entry with BusID %s already exists", e.BusID) + } + if entry.Port == e.Port { + return fmt.Errorf("entry with Port %d already exists", e.Port) + } + } + + newEntries := slices.Clone(r.record.Entries) + newEntries = append(newEntries, e) + + record := record{Entries: newEntries} + + b, err := json.Marshal(record) + if err != nil { + return err + } + + if err = os.WriteFile(r.recordFile, b, 0600); err != nil { + return err + } + + r.record = record + return nil +} diff --git a/images/virtualization-dra/internal/usbip/attacher.go b/images/virtualization-dra/internal/usbip/attacher.go index 0bec243d73..19b043314a 100644 --- a/images/virtualization-dra/internal/usbip/attacher.go +++ b/images/virtualization-dra/internal/usbip/attacher.go @@ -194,7 +194,7 @@ func (a usbAttacher) queryImportDevice(conn *net.TCPConn, busID string) (int, er } if importReply.Status != protocol.OpStatusOk { - return -1, fmt.Errorf("reply failed: %d", importReply.Status) + return -1, fmt.Errorf("reply failed: %s", importReply.Status.String()) } if importReply.USBDevice.GetBusID() != busID { diff --git a/images/virtualization-dra/internal/usbip/protocol/common.go b/images/virtualization-dra/internal/usbip/protocol/common.go index c1e3c5f714..76e48cf81e 100644 --- a/images/virtualization-dra/internal/usbip/protocol/common.go +++ b/images/virtualization-dra/internal/usbip/protocol/common.go @@ -96,6 +96,25 @@ const ( OpStatusError OpStatus = 0x05 ) +func (o OpStatus) String() string { + switch o { + case OpStatusOk: + return "OK" + case OpStatusNA: + return "NA" + case OpStatusDevBusy: + return "DevBusy" + case OpStatusDevErr: + return "DevErr" + case OpStatusNoDev: + return "NoDev" + case OpStatusError: + return "Error" + default: + return "Unknown" + } +} + type DeviceStatus uint32 const ( diff --git a/images/virtualization-dra/internal/usbip/usbipd.go b/images/virtualization-dra/internal/usbip/usbipd.go index 055fbd13a2..d538232d1d 100644 --- a/images/virtualization-dra/internal/usbip/usbipd.go +++ b/images/virtualization-dra/internal/usbip/usbipd.go @@ -208,7 +208,7 @@ func (u *USBIPD) handleConnection(conn net.Conn) (bool, error) { } if opCommon.Status != protocol.OpStatusOk { - return false, fmt.Errorf("request failed: %d", opCommon.Status) + return false, fmt.Errorf("request failed: %s", opCommon.Status.String()) } switch opCommon.Code { @@ -270,7 +270,7 @@ func (u *USBIPD) handleImportRequest(conn net.Conn) error { status = u.exportDevice(conn, bindDevice) if status != protocol.OpStatusOk { - log.Error("failed to export device") + log.Error("failed to export device", slog.String("status", status.String())) } } else { diff --git a/templates/virtualization-dra-usb-gateway/daemonset.yaml b/templates/virtualization-dra-usb-gateway/daemonset.yaml index 49e7d0e72e..1dfaaf5cca 100644 --- a/templates/virtualization-dra-usb-gateway/daemonset.yaml +++ b/templates/virtualization-dra-usb-gateway/daemonset.yaml @@ -113,6 +113,8 @@ spec: mountPath: /sys - name: var-run mountPath: /var/run + - name: usb-gateway + mountPath: /var/run/usb-gateway volumes: - name: sys hostPath: @@ -123,4 +125,6 @@ spec: - name: lib-modules hostPath: path: /lib/modules + - name: usb-gateway + emptyDir: {} {{- end }} From d4df853ee60992a8cd236cfe2d10a66ee4f65413 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Wed, 14 Jan 2026 22:24:40 +0300 Subject: [PATCH 21/29] fix store empty record Signed-off-by: Yaroslav Borbat --- .../usb-gateway/controller/resourceclaim/record.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go index 732e006c83..f95bf2206b 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go @@ -61,11 +61,14 @@ func newRecordManager(stateDir string, getter usbip.USBInfoGetter) (*recordManag recordFile := filepath.Join(stateDir, "record.json") if _, err = os.Stat(recordFile); err != nil { if os.IsNotExist(err) { - f, err := os.Create(recordFile) + b, err := json.Marshal(record{}) if err != nil { return nil, err } - _ = f.Close() + err = os.WriteFile(recordFile, b, 0600) + if err == nil { + return nil, err + } } else { return nil, err } From 0f17e13bd1bb92d8b1369aaf2908c4a9491530f0 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Wed, 14 Jan 2026 22:37:26 +0300 Subject: [PATCH 22/29] fix recordManager panic Signed-off-by: Yaroslav Borbat --- .../internal/usb-gateway/controller/resourceclaim/record.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go index f95bf2206b..e39b5da480 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go @@ -66,7 +66,7 @@ func newRecordManager(stateDir string, getter usbip.USBInfoGetter) (*recordManag return nil, err } err = os.WriteFile(recordFile, b, 0600) - if err == nil { + if err != nil { return nil, err } } else { From 4bb332a7397ffe5d2760cbdd9c1156ba1a7a3702 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Wed, 14 Jan 2026 23:27:43 +0300 Subject: [PATCH 23/29] refactor attach Signed-off-by: Yaroslav Borbat --- .../cmd/go-usbip/app/app.go | 3 +- .../go-usbip/app/{info.go => attach-info.go} | 20 +++---- .../cmd/go-usbip/app/ports.go | 59 ------------------- .../controller/resourceclaim/controller.go | 14 ++--- .../controller/resourceclaim/record.go | 10 ++-- .../internal/usb/discovery.go | 4 +- .../internal/usbip/attacher.go | 26 +------- .../internal/usbip/binder.go | 20 ------- .../internal/usbip/interfaces.go | 9 ++- 9 files changed, 32 insertions(+), 133 deletions(-) rename images/virtualization-dra/cmd/go-usbip/app/{info.go => attach-info.go} (51%) delete mode 100644 images/virtualization-dra/cmd/go-usbip/app/ports.go diff --git a/images/virtualization-dra/cmd/go-usbip/app/app.go b/images/virtualization-dra/cmd/go-usbip/app/app.go index e4801ed497..85a3043e26 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/app.go +++ b/images/virtualization-dra/cmd/go-usbip/app/app.go @@ -44,8 +44,7 @@ func NewUSBIPCommand() *cobra.Command { NewUnbindCommand(), NewAttachCommand(), NewDetachCommand(), - NewUsedPortsCommand(), - NewUsedInfoCommand(), + NewAttachInfoCommand(), ) return cmd diff --git a/images/virtualization-dra/cmd/go-usbip/app/info.go b/images/virtualization-dra/cmd/go-usbip/app/attach-info.go similarity index 51% rename from images/virtualization-dra/cmd/go-usbip/app/info.go rename to images/virtualization-dra/cmd/go-usbip/app/attach-info.go index c386ca1ee5..c5879f8f2c 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/info.go +++ b/images/virtualization-dra/cmd/go-usbip/app/attach-info.go @@ -9,11 +9,11 @@ import ( "github.com/deckhouse/virtualization-dra/internal/usbip" ) -func NewUsedInfoCommand() *cobra.Command { - o := &usedInfoOptions{} +func NewAttachInfoCommand() *cobra.Command { + o := &attachInfoOptions{} cmd := &cobra.Command{ - Use: "info", - Short: "Get used info", + Use: "attach-info", + Short: "Get attach info", Example: o.Usage(), RunE: o.Run, } @@ -21,16 +21,16 @@ func NewUsedInfoCommand() *cobra.Command { return cmd } -type usedInfoOptions struct{} +type attachInfoOptions struct{} -func (o *usedInfoOptions) Usage() string { - return ` # Get used info - $ go-usbip info +func (o *attachInfoOptions) Usage() string { + return ` # Get attach info + $ go-usbip attach-info ` } -func (o *usedInfoOptions) Run(cmd *cobra.Command, _ []string) error { - infos, err := usbip.NewUSBAttacher().GetUsedInfo() +func (o *attachInfoOptions) Run(cmd *cobra.Command, _ []string) error { + infos, err := usbip.NewUSBAttacher().GetAttachInfo() if err != nil { return err } diff --git a/images/virtualization-dra/cmd/go-usbip/app/ports.go b/images/virtualization-dra/cmd/go-usbip/app/ports.go deleted file mode 100644 index 634e519071..0000000000 --- a/images/virtualization-dra/cmd/go-usbip/app/ports.go +++ /dev/null @@ -1,59 +0,0 @@ -/* -Copyright 2025 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/deckhouse/virtualization-dra/internal/usbip" -) - -func NewUsedPortsCommand() *cobra.Command { - o := &usedPortsOptions{} - cmd := &cobra.Command{ - Use: "ports", - Short: "List used ports", - Example: o.Usage(), - RunE: o.Run, - } - - return cmd -} - -type usedPortsOptions struct{} - -func (o *usedPortsOptions) Usage() string { - return ` # List used ports - $ go-usbip ports -` -} - -func (o *usedPortsOptions) Run(cmd *cobra.Command, _ []string) error { - ports, err := usbip.NewUSBAttacher().GetUsedPorts() - if err != nil { - return err - } - - cmd.Println("Used ports:") - for _, port := range ports { - cmd.Println(fmt.Sprintf("- %d", port)) - } - - return nil -} diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go index 3a9e118046..3b9738d727 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go @@ -551,28 +551,28 @@ func (c *Controller) attach(busID string, usbGatewayStatus *vdraapi.USBGatewaySt } } - var attachErr error + var err error var rhport int defer func() { - if attachErr != nil { + if err != nil && rhport >= 0 { if err := c.detach(rhport); err != nil { c.log.Error("failed to detach usb", slog.String("error", err.Error()), slog.Int("port", rhport)) } } }() - rhport, attachErr = c.usbIP.Attach(usbGatewayStatus.RemoteIP, busID, usbGatewayStatus.RemotePort) - if attachErr != nil { - return fmt.Errorf("failed to attach usb: %w", attachErr) + rhport, err = c.usbIP.Attach(usbGatewayStatus.RemoteIP, busID, usbGatewayStatus.RemotePort) + if err != nil { + return fmt.Errorf("failed to attach usb: %w", err) } - infos, err := c.usbIP.GetUsedInfo() + infos, err := c.usbIP.GetAttachInfo() if err != nil { return fmt.Errorf("failed to get used info: %w", err) } - var usedInfo *usbip.UsedInfo + var usedInfo *usbip.AttachInfo for _, info := range infos { if info.Port == rhport { usedInfo = &info diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go index e39b5da480..3afdd5fa10 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go @@ -26,7 +26,7 @@ type Entry struct { } func (e Entry) Validate() error { - if e.Port <= 0 { + if e.Port < 0 { return fmt.Errorf("port is required") } if e.RemotePort <= 0 { @@ -46,13 +46,13 @@ func (e Entry) Validate() error { type recordManager struct { recordFile string - getter usbip.USBInfoGetter + getter usbip.AttachInfoGetter mu sync.RWMutex record record } -func newRecordManager(stateDir string, getter usbip.USBInfoGetter) (*recordManager, error) { +func newRecordManager(stateDir string, getter usbip.AttachInfoGetter) (*recordManager, error) { err := os.MkdirAll(stateDir, 0700) if err != nil { return nil, err @@ -90,12 +90,12 @@ func (r *recordManager) Refresh() error { r.mu.Lock() defer r.mu.Unlock() - infos, err := r.getter.GetUsedInfo() + infos, err := r.getter.GetAttachInfo() if err != nil { return err } - byBusId := make(map[string]*usbip.UsedInfo, len(infos)) + byBusId := make(map[string]*usbip.AttachInfo, len(infos)) for _, info := range infos { byBusId[info.LocalBusID] = &info } diff --git a/images/virtualization-dra/internal/usb/discovery.go b/images/virtualization-dra/internal/usb/discovery.go index 6d787d97a0..74d8dca3d2 100644 --- a/images/virtualization-dra/internal/usb/discovery.go +++ b/images/virtualization-dra/internal/usb/discovery.go @@ -31,7 +31,7 @@ func newDiscoverer() discoverer { } type discoverer struct { - getter usbip.USBInfoGetter + getter usbip.AttachInfoGetter } func (d *discoverer) DiscoveryPluggedUSBDevices(pathToUSBDevices string) (*DeviceSet, *DeviceSet, error) { @@ -42,7 +42,7 @@ func (d *discoverer) DiscoveryPluggedUSBDevices(pathToUSBDevices string) (*Devic busIdMaps := make(map[string]struct{}) if featuregates.Default().USBGatewayEnabled() { - infos, err := d.getter.GetUsedInfo() + infos, err := d.getter.GetAttachInfo() if err != nil { return nil, nil, err } diff --git a/images/virtualization-dra/internal/usbip/attacher.go b/images/virtualization-dra/internal/usbip/attacher.go index 19b043314a..9c7df3daf6 100644 --- a/images/virtualization-dra/internal/usbip/attacher.go +++ b/images/virtualization-dra/internal/usbip/attacher.go @@ -101,40 +101,20 @@ func (a usbAttacher) Detach(port int) error { return nil } -func (a usbAttacher) GetUsedPorts() ([]int, error) { +func (a usbAttacher) GetAttachInfo() ([]AttachInfo, error) { driver, err := newVhciDriver() if err != nil { return nil, fmt.Errorf("failed to get vhci driver: %w", err) } - var ports []int + var usedInfos []AttachInfo for i := 0; i < driver.nports; i++ { idev := &driver.idevs[i] vstatus := protocol.DeviceStatus(idev.status) if vstatus == protocol.VDeviceStatusUsed { - ports = append(ports, idev.port) - } - } - - return ports, nil -} - -func (a usbAttacher) GetUsedInfo() ([]UsedInfo, error) { - driver, err := newVhciDriver() - if err != nil { - return nil, fmt.Errorf("failed to get vhci driver: %w", err) - } - - var usedInfos []UsedInfo - - for i := 0; i < driver.nports; i++ { - idev := &driver.idevs[i] - - vstatus := protocol.DeviceStatus(idev.status) - if vstatus == protocol.VDeviceStatusUsed { - usedInfos = append(usedInfos, UsedInfo{ + usedInfos = append(usedInfos, AttachInfo{ Port: idev.port, Busnum: idev.busnum, Devnum: idev.devnum, diff --git a/images/virtualization-dra/internal/usbip/binder.go b/images/virtualization-dra/internal/usbip/binder.go index 027a54e242..38f09d2236 100644 --- a/images/virtualization-dra/internal/usbip/binder.go +++ b/images/virtualization-dra/internal/usbip/binder.go @@ -138,30 +138,10 @@ func (b *usbBinder) getUSBDeviceInfo(busID string) (*usbDeviceInfo, error) { return info, nil } -//func (b *usbBinder) storeBind(busID string, bind bool) error { -// bound, err := b.isBound(busID) -// if err != nil { -// return err -// } -// if bound == bind { -// return nil -// } -// path := bindPath(busID) -// if bind { -// _, err = os.Create(path) -// return err -// } -// return os.Remove(path) -//} - func (b *usbBinder) isBound(devInfo *usbDeviceInfo) bool { return devInfo.Driver == usbipHostDriverName } -//func bindPath(busID string) string { -// return filepath.Join(getUSBDevicePath(busID), "usbip_bound") -//} - func (b *usbBinder) unbindOther(devInfo *usbDeviceInfo) error { if devInfo.IsHub { return fmt.Errorf("skip unbinding of hub %s", devInfo.BusID) diff --git a/images/virtualization-dra/internal/usbip/interfaces.go b/images/virtualization-dra/internal/usbip/interfaces.go index 48ef560530..493529d232 100644 --- a/images/virtualization-dra/internal/usbip/interfaces.go +++ b/images/virtualization-dra/internal/usbip/interfaces.go @@ -38,15 +38,14 @@ type USBBinder interface { type USBAttacher interface { Attach(host, busID string, port int) (int, error) Detach(port int) error - USBInfoGetter + AttachInfoGetter } -type USBInfoGetter interface { - GetUsedPorts() ([]int, error) - GetUsedInfo() ([]UsedInfo, error) +type AttachInfoGetter interface { + GetAttachInfo() ([]AttachInfo, error) } -type UsedInfo struct { +type AttachInfo struct { Port, Busnum, Devnum int LocalBusID string } From ff2ee73d91bf72b6da09c521f5dc1121d9dfdb55 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Thu, 15 Jan 2026 11:54:43 +0300 Subject: [PATCH 24/29] use sets util Signed-off-by: Yaroslav Borbat --- .../virtualization-dra/internal/usb/device.go | 9 +-- .../internal/usb/discovery.go | 6 +- .../virtualization-dra/internal/usb/store.go | 22 +++---- images/virtualization-dra/pkg/set/set.go | 64 ------------------- 4 files changed, 19 insertions(+), 82 deletions(-) delete mode 100644 images/virtualization-dra/pkg/set/set.go diff --git a/images/virtualization-dra/internal/usb/device.go b/images/virtualization-dra/internal/usb/device.go index 0a53c8d303..efd8f5acb0 100644 --- a/images/virtualization-dra/internal/usb/device.go +++ b/images/virtualization-dra/internal/usb/device.go @@ -21,14 +21,15 @@ import ( "strconv" "strings" - "github.com/deckhouse/virtualization-dra/pkg/set" + "k8s.io/apimachinery/pkg/util/sets" + "github.com/deckhouse/virtualization-dra/pkg/usb" ) -type DeviceSet = set.Set[Device] +type DeviceSet = sets.Set[Device] -func NewDeviceSet() *DeviceSet { - return set.New[Device]() +func NewDeviceSet() DeviceSet { + return sets.New[Device]() } type Device struct { diff --git a/images/virtualization-dra/internal/usb/discovery.go b/images/virtualization-dra/internal/usb/discovery.go index 74d8dca3d2..0eb5a59245 100644 --- a/images/virtualization-dra/internal/usb/discovery.go +++ b/images/virtualization-dra/internal/usb/discovery.go @@ -34,7 +34,7 @@ type discoverer struct { getter usbip.AttachInfoGetter } -func (d *discoverer) DiscoveryPluggedUSBDevices(pathToUSBDevices string) (*DeviceSet, *DeviceSet, error) { +func (d *discoverer) DiscoveryPluggedUSBDevices(pathToUSBDevices string) (DeviceSet, DeviceSet, error) { devices, err := usb.DiscoverPluggedUSBDevices(pathToUSBDevices) if err != nil { return nil, nil, err @@ -56,9 +56,9 @@ func (d *discoverer) DiscoveryPluggedUSBDevices(pathToUSBDevices string) (*Devic for _, device := range devices { if _, ok := busIdMaps[device.BusID]; ok { - usbipDeviceSet.Add(toDevice(device)) + usbipDeviceSet.Insert(toDevice(device)) } else { - usbDeviceSet.Add(toDevice(device)) + usbDeviceSet.Insert(toDevice(device)) } } diff --git a/images/virtualization-dra/internal/usb/store.go b/images/virtualization-dra/internal/usb/store.go index 7d9217a882..1d3a55689c 100644 --- a/images/virtualization-dra/internal/usb/store.go +++ b/images/virtualization-dra/internal/usb/store.go @@ -27,6 +27,7 @@ import ( "github.com/containerd/nri/pkg/api" resourceapi "k8s.io/api/resource/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/dynamic-resource-allocation/resourceslice" drapbv1 "k8s.io/kubelet/pkg/apis/dra/v1beta1" "k8s.io/utils/ptr" @@ -36,7 +37,6 @@ import ( vdraapi "github.com/deckhouse/virtualization-dra/api" "github.com/deckhouse/virtualization-dra/internal/cdi" "github.com/deckhouse/virtualization-dra/internal/featuregates" - "github.com/deckhouse/virtualization-dra/pkg/set" ) const DefaultResyncPeriod = 10 * time.Minute @@ -57,7 +57,7 @@ func NewAllocationStore(nodeName, devicesPath string, resyncPeriod time.Duration updateChannel: make(chan resourceslice.DriverResources, 2), discoverPluggedUSBDevices: NewDeviceSet(), allocatableDevices: make(map[string]resourceapi.Device), - allocatedDevices: set.New[string](), + allocatedDevices: sets.New[string](), resourceClaimAllocations: make(map[types.UID][]string), discoverer: newDiscoverer(), } @@ -88,11 +88,11 @@ type AllocationStore struct { discoverer discoverer - discoverPluggedUSBDevices *DeviceSet - discoverUsbIpPluggedUSBDevices *DeviceSet + discoverPluggedUSBDevices DeviceSet + discoverUsbIpPluggedUSBDevices DeviceSet allocatableDevices map[string]resourceapi.Device - allocatedDevices *set.Set[string] + allocatedDevices sets.Set[string] resourceClaimAllocations map[types.UID][]string } @@ -112,7 +112,7 @@ func (s *AllocationStore) sync() error { s.discoverPluggedUSBDevices = discoverPluggedUSBDevices allocatableDevices := make([]resourceapi.Device, discoverPluggedUSBDevices.Len()) - for i, usbDevice := range discoverPluggedUSBDevices.Slice() { + for i, usbDevice := range discoverPluggedUSBDevices.UnsortedList() { allocatableDevices[i] = *convertToAPIDevice(usbDevice, s.nodeName) } @@ -188,7 +188,7 @@ func (s *AllocationStore) Prepare(_ context.Context, claim *resourceapi.Resource // 1 node(s) had tolerated taint {node-role.kubernetes.io/control-plane: }, // 2 cannot allocate all claims. // still not schedulable, preemption: 0/3 nodes are available: 3 Preemption is not helpful for scheduling. - if s.allocatedDevices.Contains(result.Device) { + if s.allocatedDevices.Has(result.Device) { return nil, fmt.Errorf("device %v is already allocated", result.Device) } @@ -256,7 +256,7 @@ func (s *AllocationStore) Prepare(_ context.Context, claim *resourceapi.Resource devices := preparedDevices.GetDevices() for _, device := range devices { - s.allocatedDevices.Add(device.DeviceName) + s.allocatedDevices.Insert(device.DeviceName) s.resourceClaimAllocations[claim.UID] = append(s.resourceClaimAllocations[claim.UID], device.DeviceName) } @@ -273,7 +273,7 @@ func (s *AllocationStore) getUsbGatewayStatus(claim *resourceapi.ResourceClaim, } func (s *AllocationStore) getUsbGatewayUsbDevice(busID string) *Device { - for _, device := range s.discoverUsbIpPluggedUSBDevices.Slice() { + for _, device := range s.discoverUsbIpPluggedUSBDevices.UnsortedList() { if device.BusID == busID { return &device } @@ -397,7 +397,7 @@ func (s *AllocationStore) Unprepare(_ context.Context, claimUID types.UID) error allocatedDevices := s.resourceClaimAllocations[claimUID] for _, device := range allocatedDevices { - s.allocatedDevices.Remove(device) + s.allocatedDevices.Delete(device) } delete(s.resourceClaimAllocations, claimUID) @@ -426,7 +426,7 @@ func (s *AllocationStore) Synchronize(_ context.Context, pods []*api.PodSandbox, for claimUID, deviceNames := range claimUIDDeviceNames { s.resourceClaimAllocations[claimUID] = append(s.resourceClaimAllocations[claimUID], deviceNames...) for _, deviceName := range deviceNames { - s.allocatedDevices.Add(deviceName) + s.allocatedDevices.Insert(deviceName) } } diff --git a/images/virtualization-dra/pkg/set/set.go b/images/virtualization-dra/pkg/set/set.go deleted file mode 100644 index 4b7f4378a5..0000000000 --- a/images/virtualization-dra/pkg/set/set.go +++ /dev/null @@ -1,64 +0,0 @@ -/* -Copyright 2025 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package set - -type Set[T comparable] struct { - m map[T]struct{} -} - -func New[T comparable]() *Set[T] { - return &Set[T]{ - m: make(map[T]struct{}), - } -} -func (s *Set[T]) Add(v T) { - s.m[v] = struct{}{} -} - -func (s *Set[T]) Remove(v T) { - delete(s.m, v) -} - -func (s *Set[T]) Contains(v T) bool { - _, ok := s.m[v] - return ok -} - -func (s *Set[T]) Len() int { - return len(s.m) -} - -func (s *Set[T]) Slice() []T { - out := make([]T, 0, len(s.m)) - for k := range s.m { - out = append(out, k) - } - return out -} - -func (s *Set[T]) Equal(other *Set[T]) bool { - if s.Len() != other.Len() { - return false - } - - for k := range s.m { - if !other.Contains(k) { - return false - } - } - return true -} From 7b76b10c0cc23193d1ae4d8f916178702b25e195 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Thu, 15 Jan 2026 12:48:27 +0300 Subject: [PATCH 25/29] add bindinfo Signed-off-by: Yaroslav Borbat --- .../cmd/go-usbip/app/app.go | 1 + .../cmd/go-usbip/app/bind-info.go | 46 +++++++++++++++++++ .../internal/usbip/binder.go | 30 ++++++++++++ .../internal/usbip/interfaces.go | 13 ++++++ .../virtualization-dra/pkg/usb/discovery.go | 4 ++ 5 files changed, 94 insertions(+) create mode 100644 images/virtualization-dra/cmd/go-usbip/app/bind-info.go diff --git a/images/virtualization-dra/cmd/go-usbip/app/app.go b/images/virtualization-dra/cmd/go-usbip/app/app.go index 85a3043e26..90710d44a2 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/app.go +++ b/images/virtualization-dra/cmd/go-usbip/app/app.go @@ -45,6 +45,7 @@ func NewUSBIPCommand() *cobra.Command { NewAttachCommand(), NewDetachCommand(), NewAttachInfoCommand(), + NewBindInfoCommand(), ) return cmd diff --git a/images/virtualization-dra/cmd/go-usbip/app/bind-info.go b/images/virtualization-dra/cmd/go-usbip/app/bind-info.go new file mode 100644 index 0000000000..defdcbe703 --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/app/bind-info.go @@ -0,0 +1,46 @@ +package app + +import ( + "encoding/json" + "fmt" + + "github.com/spf13/cobra" + + "github.com/deckhouse/virtualization-dra/internal/usbip" +) + +func NewBindInfoCommand() *cobra.Command { + o := &bindInfoOptions{} + cmd := &cobra.Command{ + Use: "bind-info", + Short: "Get bind info", + Example: o.Usage(), + RunE: o.Run, + } + + return cmd +} + +type bindInfoOptions struct{} + +func (o *bindInfoOptions) Usage() string { + return ` # Get bind info + $ go-usbip bind-info +` +} + +func (o *bindInfoOptions) Run(cmd *cobra.Command, _ []string) error { + infos, err := usbip.NewUSBBinder().GetBindInfo() + if err != nil { + return err + } + + bytes, err := json.Marshal(infos) + if err != nil { + return fmt.Errorf("failed to marshal json: %w", err) + } + + cmd.Println(string(bytes)) + + return nil +} diff --git a/images/virtualization-dra/internal/usbip/binder.go b/images/virtualization-dra/internal/usbip/binder.go index 38f09d2236..647f1fc6f0 100644 --- a/images/virtualization-dra/internal/usbip/binder.go +++ b/images/virtualization-dra/internal/usbip/binder.go @@ -21,6 +21,8 @@ import ( "os" "path/filepath" "strings" + + "github.com/deckhouse/virtualization-dra/pkg/usb" ) func NewUSBBinder() USBBinder { @@ -96,6 +98,34 @@ func (b *usbBinder) IsBound(busID string) (bool, error) { return b.isBound(devInfo), nil } +func (b *usbBinder) GetBindInfo() ([]BindInfo, error) { + usbDevices, err := usb.DefaultDiscoverPluggedUSBDevices() + if err != nil { + return nil, fmt.Errorf("failed to discover USB devices: %w", err) + } + + var infos []BindInfo + + for _, device := range usbDevices { + devInfo := usbDeviceInfo{ + BusID: device.BusID, + Driver: device.Driver, + DevPath: device.DevicePath, + IsHub: device.IsHub, + } + + infos = append(infos, BindInfo{ + DevicePath: device.DevicePath, + BusID: device.BusID, + Busnum: int(device.Bus), + Devnum: int(device.DeviceNumber), + Bound: b.isBound(&devInfo), + }) + } + + return infos, err +} + type usbDeviceInfo struct { BusID string Driver string diff --git a/images/virtualization-dra/internal/usbip/interfaces.go b/images/virtualization-dra/internal/usbip/interfaces.go index 493529d232..537a5004bb 100644 --- a/images/virtualization-dra/internal/usbip/interfaces.go +++ b/images/virtualization-dra/internal/usbip/interfaces.go @@ -33,6 +33,19 @@ type USBBinder interface { Bind(busID string) error Unbind(busID string) error IsBound(busID string) (bool, error) + BindInfoGetter +} + +type BindInfoGetter interface { + GetBindInfo() ([]BindInfo, error) +} + +type BindInfo struct { + DevicePath string + BusID string + Busnum int + Devnum int + Bound bool } type USBAttacher interface { diff --git a/images/virtualization-dra/pkg/usb/discovery.go b/images/virtualization-dra/pkg/usb/discovery.go index f3a7a19b4e..ebe1e357fd 100644 --- a/images/virtualization-dra/pkg/usb/discovery.go +++ b/images/virtualization-dra/pkg/usb/discovery.go @@ -24,6 +24,10 @@ import ( "strings" ) +func DefaultDiscoverPluggedUSBDevices() (map[string]*Device, error) { + return DiscoverPluggedUSBDevices(PathToUSBDevices) +} + func DiscoverPluggedUSBDevices(pathToUSBDevices string) (map[string]*Device, error) { devices := make(map[string]*Device) From f40cc559f6dde332599e3c9faa91cc21fa91b770 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Thu, 15 Jan 2026 12:49:08 +0300 Subject: [PATCH 26/29] add license Signed-off-by: Yaroslav Borbat --- images/virtualization-dra/api/sheme.go | 16 ++++++++++++++++ .../cmd/go-usbip/app/attach-info.go | 16 ++++++++++++++++ .../cmd/go-usbip/app/bind-info.go | 16 ++++++++++++++++ .../internal/plugin/publish.go | 16 ++++++++++++++++ .../controller/resourceclaim/record.go | 16 ++++++++++++++++ 5 files changed, 80 insertions(+) diff --git a/images/virtualization-dra/api/sheme.go b/images/virtualization-dra/api/sheme.go index 2a6eeb290f..573f52a64e 100644 --- a/images/virtualization-dra/api/sheme.go +++ b/images/virtualization-dra/api/sheme.go @@ -1,3 +1,19 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package api import ( diff --git a/images/virtualization-dra/cmd/go-usbip/app/attach-info.go b/images/virtualization-dra/cmd/go-usbip/app/attach-info.go index c5879f8f2c..a76bddbc93 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/attach-info.go +++ b/images/virtualization-dra/cmd/go-usbip/app/attach-info.go @@ -1,3 +1,19 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package app import ( diff --git a/images/virtualization-dra/cmd/go-usbip/app/bind-info.go b/images/virtualization-dra/cmd/go-usbip/app/bind-info.go index defdcbe703..5f0814f180 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/bind-info.go +++ b/images/virtualization-dra/cmd/go-usbip/app/bind-info.go @@ -1,3 +1,19 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package app import ( diff --git a/images/virtualization-dra/internal/plugin/publish.go b/images/virtualization-dra/internal/plugin/publish.go index ecb338b75d..d93e2bb189 100644 --- a/images/virtualization-dra/internal/plugin/publish.go +++ b/images/virtualization-dra/internal/plugin/publish.go @@ -1,3 +1,19 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package plugin import ( diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go index 3afdd5fa10..e75262faa0 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go @@ -1,3 +1,19 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package resourceclaim import ( From a11a8b0b99997ef37f5676cbb03358d6713b8edf Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Thu, 15 Jan 2026 12:49:51 +0300 Subject: [PATCH 27/29] fix long Signed-off-by: Yaroslav Borbat --- images/virtualization-dra/cmd/go-usbip/app/app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/virtualization-dra/cmd/go-usbip/app/app.go b/images/virtualization-dra/cmd/go-usbip/app/app.go index 90710d44a2..9fec58bb34 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/app.go +++ b/images/virtualization-dra/cmd/go-usbip/app/app.go @@ -24,7 +24,7 @@ const long = ` / _' |/ _ \ _____| | | / __| '_ \| | '_ \ | (_| | (_) |_____| |_| \__ \ |_) | | |_) | \__, | \___/ \__,_|___/_.__/|_| .__/ -|___/ |_| +|___/ |_| go-usbip is a implementation of USBIP server and client. ` From bd77b30c39521890637d2fe49e9911539d7de064 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Thu, 15 Jan 2026 20:46:23 +0300 Subject: [PATCH 28/29] impl detach unbound Signed-off-by: Yaroslav Borbat --- .../controller/resourceclaim/controller.go | 278 ++++++++++++------ .../controller/resourceclaim/record.go | 20 +- 2 files changed, 195 insertions(+), 103 deletions(-) diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go index 3b9738d727..6b225b4728 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go @@ -34,6 +34,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + "k8s.io/utils/strings/slices" vdraapi "github.com/deckhouse/virtualization-dra/api" "github.com/deckhouse/virtualization-dra/internal/common" @@ -106,6 +107,13 @@ func NewController(nodeName string, podIP net.IP, usbipPort int, client kubernet return nil, fmt.Errorf("unable to add event handler to resourceclaim informer: %w", err) } + _, err = podInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + DeleteFunc: c.deletePod, + }) + if err != nil { + return nil, fmt.Errorf("unable to add event handler to pod informer: %w", err) + } + c.hasSynced = func() bool { return resourceClaimInformer.HasSynced() && nodeInformer.HasSynced() && podInformer.HasSynced() && resourceSliceInformer.HasSynced() } @@ -136,6 +144,18 @@ func (c *Controller) updateResourceClaim(_, newObj interface{}) { } } +func (c *Controller) deletePod(obj interface{}) { + pod, ok := obj.(*corev1.Pod) + if !ok { + return + } + for _, status := range pod.Status.ResourceClaimStatuses { + if status.ResourceClaimName != nil { + c.queueAdd(fmt.Sprintf("%s/%s", pod.Namespace, *status.ResourceClaimName)) + } + } +} + func (c *Controller) enqueueResourceClaim(rc *resourcev1beta1.ResourceClaim) { key, err := keyFunc(rc) if err != nil { @@ -203,28 +223,46 @@ func (c *Controller) sync(key string) error { log := c.log.With("key", key) log.Info("syncing resource claim") - rc, err := c.getResourceClaim(key) + rc, err := c.getMyResourceClaim(key) if err != nil { return err } if rc == nil { return nil } - if !rc.GetDeletionTimestamp().IsZero() { - return c.handleDelete(rc) - } + + resourceClaimDeleting := !rc.GetDeletionTimestamp().IsZero() pod, err := c.getReservedFor(rc) if err != nil { return err } - if pod == nil { + + podExist := pod != nil + + if !podExist && !resourceClaimDeleting { c.log.Info("no reserved pod found for resource claim, re-enqueue after 10s") c.queueAfterAdd(key, time.Second*10) return nil } - onMyNode := c.podOnMyNode(pod) + onMyNode := podExist && c.podOnMyNode(pod) + + if onMyNode && c.podFinished(pod) { + log.Info("Pod finished, detach all usb devices for this pod", + slog.String("podName", pod.Name), + slog.String("podNamespace", pod.Namespace), + ) + return c.handleClientPodFinished(pod) + } + + if resourceClaimDeleting { + log.Info("ResourceClaim is deleting, unbind all usb devices for this resource claim") + return c.handleServerDeleteResourceClaim(rc) + } + + // pod exists here + log = log.With(slog.String("podName", pod.Name), slog.String("podNamespace", pod.Namespace)) myAllocationDevices, otherAllocationDevices, err := c.getAllocationDevices(rc) if err != nil { @@ -242,7 +280,7 @@ func (c *Controller) sync(key string) error { } case shouldAttach: log.Info("attaching usb to my node") - if err = c.handleClient(rc, otherAllocationDevices); err != nil { + if err = c.handleClient(rc, otherAllocationDevices, pod); err != nil { return fmt.Errorf("failed to handle client: %w", err) } @@ -251,76 +289,62 @@ func (c *Controller) sync(key string) error { return nil } -// TODO: handle detach too -func (c *Controller) handleDelete(rc *resourcev1beta1.ResourceClaim) error { - unbound, err := c.allUnBound(rc) - if err != nil { - return err - } - if unbound { - return c.removeFinalizer(rc) - } +func (c *Controller) podFinished(pod *corev1.Pod) bool { + return !pod.GetDeletionTimestamp().IsZero() || pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed +} - myAllocationDevices, _, err := c.getAllocationDevices(rc) +// handle on client node, should detach all usb for this pod +func (c *Controller) handleClientPodFinished(pod *corev1.Pod) error { + err := c.recordManager.Refresh() if err != nil { - return err - } - - myAllocationDevicesByName := make(map[string]resourcev1beta1.Device) - for _, device := range myAllocationDevices { - myAllocationDevicesByName[device.Name] = device + return fmt.Errorf("failed to Refresh record: %w", err) } - shouldUpdate := false - - for i := range rc.Status.Devices { - allocatedDeviceStatus := &rc.Status.Devices[i] + ports := make(map[int]struct{}) - usbGatewayStatus, err := vdraapi.FromData(allocatedDeviceStatus.Data) - if err != nil { - return err - } - if usbGatewayStatus == nil { - continue - } - if !usbGatewayStatus.Bound { - continue - } - - device, ok := myAllocationDevicesByName[allocatedDeviceStatus.Device] - if !ok { - continue + for _, entry := range c.recordManager.GetEntries() { + if entry.PodUID == pod.UID { + if _, ok := ports[entry.Port]; ok { + continue + } + if err = c.usbIP.Detach(entry.Port); err != nil { + return fmt.Errorf("failed to detach usb: %w", err) + } + ports[entry.Port] = struct{}{} } + } - busID := "" - if attr, ok := device.Basic.Attributes["busID"]; ok && attr.StringValue != nil { - busID = *attr.StringValue - } else { - continue - } + return c.removeFinalizerForPod(pod) +} - // TODO: device can be added to other resource claims. Not supported yet. - c.log.Info("unbinding usb") - if err = c.usbIP.Unbind(busID); err != nil { - return fmt.Errorf("failed to unbind usb: %w", err) - } +// handle on server node, should unbind usb +func (c *Controller) handleServerDeleteResourceClaim(rc *resourcev1beta1.ResourceClaim) error { + infos, err := c.usbIP.GetBindInfo() + if err != nil { + return fmt.Errorf("failed to get used info: %w", err) + } - usbGatewayStatus.Bound = false - allocatedDeviceStatus.Data, err = vdraapi.ToData(usbGatewayStatus) + for _, deviceStatus := range rc.Status.Devices { + usbGatewayStatus, err := vdraapi.FromData(deviceStatus.Data) if err != nil { return err } - shouldUpdate = true - } - if shouldUpdate { - _, err = c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).UpdateStatus(context.Background(), rc, metav1.UpdateOptions{}) - if err != nil { - return fmt.Errorf("failed to update resource claim status: %w", err) + busID := usbGatewayStatus.BusID + + for _, info := range infos { + if info.BusID == busID { + if info.Bound { + err = c.usbIP.Unbind(busID) + if err != nil { + return fmt.Errorf("failed to unbind usb: %w", err) + } + } + } } } - return nil + return c.removeFinalizerForResourceClaim(rc) } func (c *Controller) allUnBound(rc *resourcev1beta1.ResourceClaim) (bool, error) { @@ -340,35 +364,32 @@ func (c *Controller) allUnBound(rc *resourcev1beta1.ResourceClaim) (bool, error) return true, nil } -func (c *Controller) addFinalizer(rc *resourcev1beta1.ResourceClaim) error { - var newFinalizers []string - for _, fin := range rc.GetFinalizers() { - if fin == finalizer { - return nil - } - newFinalizers = append(newFinalizers, fin) +func (c *Controller) addFinalizerForResourceClaim(rc *resourcev1beta1.ResourceClaim) (err error) { + if addFinalizer(rc) { + _, err = c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).Update(context.Background(), rc, metav1.UpdateOptions{}) } - newFinalizers = append(newFinalizers, finalizer) - rc.SetFinalizers(newFinalizers) - _, err := c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).Update(context.Background(), rc, metav1.UpdateOptions{}) - return err + return } -func (c *Controller) removeFinalizer(rc *resourcev1beta1.ResourceClaim) error { - var newFinalizers []string - for _, fin := range rc.GetFinalizers() { - if fin == finalizer { - continue - } - newFinalizers = append(newFinalizers, fin) +func (c *Controller) removeFinalizerForResourceClaim(rc *resourcev1beta1.ResourceClaim) (err error) { + if removeFinalizer(rc) { + _, err = c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).Update(context.Background(), rc, metav1.UpdateOptions{}) } - if len(newFinalizers) == len(rc.GetFinalizers()) { - return nil + return +} + +func (c *Controller) addFinalizerForPod(pod *corev1.Pod) (err error) { + if addFinalizer(pod) { + _, err = c.client.CoreV1().Pods(pod.Namespace).Update(context.Background(), pod, metav1.UpdateOptions{}) } + return +} - rc.SetFinalizers(newFinalizers) - _, err := c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).Update(context.Background(), rc, metav1.UpdateOptions{}) - return err +func (c *Controller) removeFinalizerForPod(pod *corev1.Pod) (err error) { + if removeFinalizer(pod) { + _, err = c.client.CoreV1().Pods(pod.Namespace).Update(context.Background(), pod, metav1.UpdateOptions{}) + } + return } func (c *Controller) handleServer(rc *resourcev1beta1.ResourceClaim, myAllocationDevices []resourcev1beta1.Device) error { @@ -462,7 +483,7 @@ func (c *Controller) handleServer(rc *resourcev1beta1.ResourceClaim, myAllocatio } if shouldUpdate { - err := c.addFinalizer(rc) + err := c.addFinalizerForResourceClaim(rc) if err != nil { return err } @@ -475,7 +496,7 @@ func (c *Controller) handleServer(rc *resourcev1beta1.ResourceClaim, myAllocatio return nil } -func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAllocationDevices []resourcev1beta1.Device) error { +func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAllocationDevices []resourcev1beta1.Device, pod *corev1.Pod) error { indexAllocDevice := make(map[string]int) for i, allocDeviceStatus := range rc.Status.Devices { indexAllocDevice[allocDeviceStatus.Device] = i @@ -517,7 +538,7 @@ func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAlloca return fmt.Errorf("failed to Refresh record: %w", err) } - if err = c.attach(busID, usbGatewayStatus); err != nil { + if err = c.attach(busID, usbGatewayStatus, rc.UID, pod.UID); err != nil { return fmt.Errorf("failed to attach usb: %w", err) } @@ -529,7 +550,12 @@ func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAlloca } if shouldUpdate { - _, err := c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).UpdateStatus(context.Background(), rc, metav1.UpdateOptions{}) + err := c.addFinalizerForPod(pod) + if err != nil { + return fmt.Errorf("failed to add finalizer for pod: %s/%s: %w", pod.Namespace, pod.Name, err) + } + + _, err = c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).UpdateStatus(context.Background(), rc, metav1.UpdateOptions{}) if err != nil { return fmt.Errorf("failed to update resource claim status: %w", err) } @@ -538,7 +564,7 @@ func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAlloca return nil } -func (c *Controller) attach(busID string, usbGatewayStatus *vdraapi.USBGatewayStatus) error { +func (c *Controller) attach(busID string, usbGatewayStatus *vdraapi.USBGatewayStatus, claimUID, podUID types.UID) error { entries := c.recordManager.GetEntries() for _, entry := range entries { if entry.BusID == busID { @@ -547,6 +573,7 @@ func (c *Controller) attach(busID string, usbGatewayStatus *vdraapi.USBGatewaySt return err } } + // already attached return nil } } @@ -584,11 +611,13 @@ func (c *Controller) attach(busID string, usbGatewayStatus *vdraapi.USBGatewaySt } entry := Entry{ - Port: rhport, - RemotePort: usbGatewayStatus.RemotePort, - RemoteIP: usbGatewayStatus.RemoteIP, - RemoteBusID: busID, - BusID: usedInfo.LocalBusID, + Port: rhport, + RemotePort: usbGatewayStatus.RemotePort, + RemoteIP: usbGatewayStatus.RemoteIP, + RemoteBusID: busID, + BusID: usedInfo.LocalBusID, + ResourceClaimUID: claimUID, + PodUID: podUID, } if err = c.recordManager.AddEntry(entry); err != nil { @@ -608,7 +637,7 @@ func (c *Controller) detach(port int) error { return nil } -func (c *Controller) getResourceClaim(key string) (*resourcev1beta1.ResourceClaim, error) { +func (c *Controller) getMyResourceClaim(key string) (*resourcev1beta1.ResourceClaim, error) { obj, exists, err := c.resourceClaimIndexer.GetByKey(key) if err != nil && !k8serrors.IsNotFound(err) { return nil, fmt.Errorf("failed to get resourceclaim: %w", err) @@ -622,7 +651,31 @@ func (c *Controller) getResourceClaim(key string) (*resourcev1beta1.ResourceClai return nil, fmt.Errorf("unexpected type of resourceclaim: %T", obj) } - return rc.DeepCopy(), nil + if c.isMyResourceClaim(rc) { + return rc.DeepCopy(), nil + } + + return nil, nil +} + +func (c *Controller) isMyResourceClaim(rc *resourcev1beta1.ResourceClaim) bool { + if rc == nil { + return false + } + if slices.Contains(rc.GetFinalizers(), finalizer) { + return true + } + if rc.Status.Allocation == nil { + return false + } + for _, status := range rc.Status.Allocation.Devices.Results { + // now, driver virtualization-dra supports only usb, but we can add more devices later + // so we need to check if the device is usb + if status.Driver == common.VirtualizationDraPluginName && strings.HasPrefix(status.Device, "usb") { + return true + } + } + return false } func (c *Controller) getPod(name, namespace string) (*corev1.Pod, error) { @@ -717,12 +770,12 @@ func (c *Controller) getAllocationDevices(rc *resourcev1beta1.ResourceClaim) ([] var otherDevices []resourcev1beta1.Device for pool, allocResultsByDevice := range allocResultsByPool { - slices, ok := byPoolSlices[pool] + resourceSlices, ok := byPoolSlices[pool] if !ok { return nil, nil, fmt.Errorf("no resource slices found for pool %s", pool) } - for _, slice := range slices { + for _, slice := range resourceSlices { for _, device := range slice.Spec.Devices { allocResult, ok := allocResultsByDevice[device.Name] if !ok { @@ -741,3 +794,32 @@ func (c *Controller) getAllocationDevices(rc *resourcev1beta1.ResourceClaim) ([] return myDevices, otherDevices, nil } + +func addFinalizer(obj metav1.Object) bool { + var newFinalizers []string + for _, fin := range obj.GetFinalizers() { + if fin == finalizer { + return false + } + newFinalizers = append(newFinalizers, fin) + } + newFinalizers = append(newFinalizers, finalizer) + obj.SetFinalizers(newFinalizers) + return true +} + +func removeFinalizer(obj metav1.Object) bool { + var newFinalizers []string + for _, fin := range obj.GetFinalizers() { + if fin == finalizer { + continue + } + newFinalizers = append(newFinalizers, fin) + } + if len(newFinalizers) == len(obj.GetFinalizers()) { + return false + } + + obj.SetFinalizers(newFinalizers) + return true +} diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go index e75262faa0..bf81716002 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go @@ -24,6 +24,8 @@ import ( "slices" "sync" + "k8s.io/apimachinery/pkg/types" + "github.com/deckhouse/virtualization-dra/internal/usbip" ) @@ -34,11 +36,13 @@ type record struct { } type Entry struct { - Port int `json:"port"` - RemotePort int `json:"remotePort" json:"remotePort"` - RemoteIP string `json:"remoteIP" json:"remoteIP"` - RemoteBusID string `json:"remoteBusID" json:"remoteBusID"` - BusID string `json:"busID" json:"busID"` + Port int `json:"port"` + RemotePort int `json:"remotePort"` + RemoteIP string `json:"remoteIP"` + RemoteBusID string `json:"remoteBusID"` + BusID string `json:"busID"` + ResourceClaimUID types.UID `json:"resourceClaimUID"` + PodUID types.UID `json:"podUID"` } func (e Entry) Validate() error { @@ -57,6 +61,12 @@ func (e Entry) Validate() error { if e.BusID == "" { return fmt.Errorf("busID is required") } + if e.ResourceClaimUID == "" { + return fmt.Errorf("resourceClaimUID is required") + } + if e.PodUID == "" { + return fmt.Errorf("podUID is required") + } return nil } From 00f5e39fa6ffbe7bd8c257296938b6f423640aac Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Thu, 15 Jan 2026 22:11:29 +0300 Subject: [PATCH 29/29] add info subcommands Signed-off-by: Yaroslav Borbat --- .../cmd/go-usbip/app/app.go | 45 ++++++++++++- .../cmd/go-usbip/app/attach-info.go | 12 +--- .../cmd/go-usbip/app/bind-info.go | 12 +--- .../cmd/go-usbip/app/info.go | 65 +++++++++++++++++++ .../internal/usbip/binder.go | 44 ++++++++----- images/virtualization-dra/pkg/usb/usb.go | 6 +- 6 files changed, 142 insertions(+), 42 deletions(-) create mode 100644 images/virtualization-dra/cmd/go-usbip/app/info.go diff --git a/images/virtualization-dra/cmd/go-usbip/app/app.go b/images/virtualization-dra/cmd/go-usbip/app/app.go index 9fec58bb34..c5604c78b7 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/app.go +++ b/images/virtualization-dra/cmd/go-usbip/app/app.go @@ -16,7 +16,14 @@ limitations under the License. package app -import "github.com/spf13/cobra" +import ( + "encoding/json" + "fmt" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "sigs.k8s.io/yaml" +) const long = ` _ _ @@ -24,7 +31,7 @@ const long = ` / _' |/ _ \ _____| | | / __| '_ \| | '_ \ | (_| | (_) |_____| |_| \__ \ |_) | | |_) | \__, | \___/ \__,_|___/_.__/|_| .__/ -|___/ |_| +|___/ |_| go-usbip is a implementation of USBIP server and client. ` @@ -46,7 +53,41 @@ func NewUSBIPCommand() *cobra.Command { NewDetachCommand(), NewAttachInfoCommand(), NewBindInfoCommand(), + NewInfoCommand(), ) + printer.AddFlags(cmd.PersistentFlags()) + return cmd } + +var printer = &printOptions{} + +type printOptions struct { + output string +} + +func (o *printOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVarP(&o.output, "output", "o", "json", "Output format") +} + +func (o *printOptions) PrintObject(cmd *cobra.Command, data interface{}) error { + switch o.output { + case "json": + b, err := json.MarshalIndent(data, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal json: %w", err) + } + cmd.Println(string(b)) + return nil + case "yaml": + b, err := yaml.Marshal(data) + if err != nil { + return fmt.Errorf("failed to marshal yaml: %w", err) + } + cmd.Println(string(b)) + return nil + default: + return fmt.Errorf("unsupported format %q. Supported formats: [json, yaml]", o.output) + } +} diff --git a/images/virtualization-dra/cmd/go-usbip/app/attach-info.go b/images/virtualization-dra/cmd/go-usbip/app/attach-info.go index a76bddbc93..0953f1e490 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/attach-info.go +++ b/images/virtualization-dra/cmd/go-usbip/app/attach-info.go @@ -17,9 +17,6 @@ limitations under the License. package app import ( - "encoding/json" - "fmt" - "github.com/spf13/cobra" "github.com/deckhouse/virtualization-dra/internal/usbip" @@ -51,12 +48,5 @@ func (o *attachInfoOptions) Run(cmd *cobra.Command, _ []string) error { return err } - bytes, err := json.Marshal(infos) - if err != nil { - return fmt.Errorf("failed to marshal json: %w", err) - } - - cmd.Println(string(bytes)) - - return nil + return printer.PrintObject(cmd, infos) } diff --git a/images/virtualization-dra/cmd/go-usbip/app/bind-info.go b/images/virtualization-dra/cmd/go-usbip/app/bind-info.go index 5f0814f180..b446ccf5b2 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/bind-info.go +++ b/images/virtualization-dra/cmd/go-usbip/app/bind-info.go @@ -17,9 +17,6 @@ limitations under the License. package app import ( - "encoding/json" - "fmt" - "github.com/spf13/cobra" "github.com/deckhouse/virtualization-dra/internal/usbip" @@ -51,12 +48,5 @@ func (o *bindInfoOptions) Run(cmd *cobra.Command, _ []string) error { return err } - bytes, err := json.Marshal(infos) - if err != nil { - return fmt.Errorf("failed to marshal json: %w", err) - } - - cmd.Println(string(bytes)) - - return nil + return printer.PrintObject(cmd, infos) } diff --git a/images/virtualization-dra/cmd/go-usbip/app/info.go b/images/virtualization-dra/cmd/go-usbip/app/info.go new file mode 100644 index 0000000000..434e03823e --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/app/info.go @@ -0,0 +1,65 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "cmp" + "slices" + + "github.com/spf13/cobra" + + "github.com/deckhouse/virtualization-dra/pkg/usb" +) + +func NewInfoCommand() *cobra.Command { + o := &infoOptions{} + cmd := &cobra.Command{ + Use: "info", + Short: "Get info", + Example: o.Usage(), + RunE: o.Run, + } + + return cmd +} + +type infoOptions struct{} + +func (o *infoOptions) Usage() string { + return ` # Get info + $ go-usbip info +` +} + +func (o *infoOptions) Run(cmd *cobra.Command, _ []string) error { + discoverDevices, err := usb.DefaultDiscoverPluggedUSBDevices() + if err != nil { + return err + } + + devices := make([]*usb.Device, 0, len(discoverDevices)) + + for _, device := range discoverDevices { + devices = append(devices, device) + } + + slices.SortFunc(devices, func(a, b *usb.Device) int { + return cmp.Compare(a.Path, b.Path) + }) + + return printer.PrintObject(cmd, devices) +} diff --git a/images/virtualization-dra/internal/usbip/binder.go b/images/virtualization-dra/internal/usbip/binder.go index 647f1fc6f0..aac13bc8d2 100644 --- a/images/virtualization-dra/internal/usbip/binder.go +++ b/images/virtualization-dra/internal/usbip/binder.go @@ -17,6 +17,7 @@ limitations under the License. package usbip import ( + "bufio" "fmt" "os" "path/filepath" @@ -68,7 +69,7 @@ func (b *usbBinder) Unbind(busID string) error { return fmt.Errorf("device with bus ID %s does not exist: %w", busID, err) } - if b.isBound(devInfo) { + if !b.isBound(devInfo) { return fmt.Errorf("device %s is not bound to %s driver", devInfo.BusID, usbipHostDriverName) } @@ -87,7 +88,6 @@ func (b *usbBinder) Unbind(busID string) error { } return nil - // return b.storeBind(busID, false) } func (b *usbBinder) IsBound(busID string) (bool, error) { @@ -145,23 +145,37 @@ func (b *usbBinder) getUSBDeviceInfo(busID string) (*usbDeviceInfo, error) { } bDevClassPath := filepath.Join(path, "bDeviceClass") - if data, err := os.ReadFile(bDevClassPath); err == nil { - info.IsHub = strings.TrimSpace(string(data)) == "09" // 09 = USB Hub class + data, err := os.ReadFile(bDevClassPath) + if err != nil { + return nil, fmt.Errorf("failed to read %s: %w", bDevClassPath, err) } + info.IsHub = strings.TrimSpace(string(data)) == "09" // 09 = USB Hub class - driverLink := filepath.Join(path, "driver") - if link, err := os.Readlink(driverLink); err == nil { - info.Driver = filepath.Base(link) + ueventPath := filepath.Join(path, "uevent") + ueventFile, err := os.Open(ueventPath) + if err != nil { + return nil, fmt.Errorf("unable to open the file %s: %w", ueventPath, err) } + defer ueventFile.Close() + scanner := bufio.NewScanner(ueventFile) - ueventPath := filepath.Join(path, "uevent") - if data, err := os.ReadFile(ueventPath); err == nil { - lines := strings.Split(string(data), "\n") - for _, line := range lines { - if strings.HasPrefix(line, "DEVNAME=") { - info.DevPath = filepath.Join("/dev", strings.TrimPrefix(line, "DEVNAME=")) - break - } + count := 0 + for scanner.Scan() { + line := scanner.Text() + values := strings.Split(line, "=") + if len(values) != 2 { + continue + } + switch values[0] { + case "DEVNAME": + info.DevPath = filepath.Join("/dev", values[1]) + count++ + case "DRIVER": + info.Driver = values[1] + count++ + } + if count == 2 { + break } } diff --git a/images/virtualization-dra/pkg/usb/usb.go b/images/virtualization-dra/pkg/usb/usb.go index ae220ea329..fc6c474ab6 100644 --- a/images/virtualization-dra/pkg/usb/usb.go +++ b/images/virtualization-dra/pkg/usb/usb.go @@ -161,9 +161,10 @@ func parseSysUeventFile(path string, device *Device) error { // TYPE=0/0/0 // BUSNUM=003 // DEVNUM=002 - file, err := os.Open(filepath.Join(path, "uevent")) + ueventPath := filepath.Join(path, "uevent") + file, err := os.Open(ueventPath) if err != nil { - return fmt.Errorf("unable to open the file %s: %w", path, err) + return fmt.Errorf("unable to open the file %s: %w", ueventPath, err) } defer file.Close() @@ -252,7 +253,6 @@ func parseSysUeventFile(path string, device *Device) error { } device.DeviceNumber = uint32(val) default: - slog.Info("Skipping unhandled line", slog.String("line", line)) } } return nil