From 6b5de5e9f30f4b4bca332fd1af8cd5d5dd44da01 Mon Sep 17 00:00:00 2001 From: Sergey Zhuravlev Date: Fri, 20 Feb 2026 18:27:00 +0500 Subject: [PATCH 01/13] Using full path when running real QEMU binary in a chroot (#16) Co-authored-by: Sergey Zhuravlev --- contrib/qemu.wrapper.debian | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/qemu.wrapper.debian b/contrib/qemu.wrapper.debian index 3629124..8e7514a 100644 --- a/contrib/qemu.wrapper.debian +++ b/contrib/qemu.wrapper.debian @@ -55,7 +55,7 @@ if [[ "$QEMU_ROOTDIR" != "/" ]] ; then fi # run chrooted QEMU binary - exec chroot "$QEMU_ROOTDIR" sh -c "cd $PWD && exec /usr/bin/qemu-system-x86_64 ${ARGS}" + exec chroot "$QEMU_ROOTDIR" /bin/sh -c "cd $PWD && exec /usr/bin/qemu-system-x86_64 ${ARGS}" fi UNSHARED=1 exec unshare -m "$0" ${ARGS} From 37fb6006c4541881762ae29ddb21214a98ec0a27 Mon Sep 17 00:00:00 2001 From: Sergey Zhuravlev Date: Fri, 20 Feb 2026 18:27:14 +0500 Subject: [PATCH 02/13] Troubleshooting in network SchemeProperties with reflect (#19) Co-authored-by: Sergey Zhuravlev --- server/network/scheme_properties.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/server/network/scheme_properties.go b/server/network/scheme_properties.go index 37ad680..7c072e9 100644 --- a/server/network/scheme_properties.go +++ b/server/network/scheme_properties.go @@ -120,7 +120,7 @@ func (p *SchemeProperties) ValueAs(key string, target interface{}) error { return p.valueAs(key, target) } -func (p *SchemeProperties) valueAs(key string, target interface{}) error { +func (p *SchemeProperties) valueAs(key string, target interface{}) (err error) { if target == nil { return fmt.Errorf("target must be a non-nil pointer") } @@ -162,12 +162,17 @@ func (p *SchemeProperties) valueAs(key string, target interface{}) error { valueRV := reflect.ValueOf(value) - // Is value from attrs can be assigned to the target ? - if !valueRV.Type().AssignableTo(targetElem.Type()) { - return fmt.Errorf("type mismatch: value type = %s, target type = %s", valueRV.Type(), targetElem.Type()) + // Is value from attrs can be converted to the target ? + if !valueRV.Type().ConvertibleTo(targetElem.Type()) { + return fmt.Errorf("type mismatch: key = %s, value type = %s, target type = %s", key, valueRV.Type(), targetElem.Type()) } - targetElem.Set(valueRV) + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("type mismatch: key = %s, %v", key, r) + } + }() + targetElem.Set(valueRV.Convert(targetElem.Type())) return nil } @@ -353,6 +358,10 @@ func GetNetworkSchemes(vmname string, ifnames ...string) ([]*SchemeProperties, e return nil, err } } else { + if os.IsNotExist(err) { + // no one found, no problem + return nil, nil + } return nil, err } From 85574b5a044a469c4b25b67ab23f694ae9c0d1d7 Mon Sep 17 00:00:00 2001 From: Sergey Zhuravlev Date: Fri, 20 Feb 2026 18:27:25 +0500 Subject: [PATCH 03/13] Fixed help line in vnetctl "update-conf" command (#18) Co-authored-by: Sergey Zhuravlev --- cmd/vnetctl/commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/vnetctl/commands.go b/cmd/vnetctl/commands.go index ffaf1c0..9aecb7c 100644 --- a/cmd/vnetctl/commands.go +++ b/cmd/vnetctl/commands.go @@ -37,7 +37,7 @@ var CommandCreateConf = &cli.Command{ var CommandUpdateConf = &cli.Command{ Name: "update-conf", - Usage: "create a new network configuration", + Usage: "update an existing network configuration", ArgsUsage: "VMNAME IFNAME", HideHelp: true, Category: "Configuration", From 7ff80c4ba029f9f930fbe5832ab0e2b278cccbd6 Mon Sep 17 00:00:00 2001 From: Sergey Zhuravlev Date: Fri, 20 Feb 2026 18:27:36 +0500 Subject: [PATCH 04/13] Removed debug messages (#17) Co-authored-by: Sergey Zhuravlev --- internal/hostnet/vrouter.go | 4 ++-- server/network/configuration.go | 23 ----------------------- 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/internal/hostnet/vrouter.go b/internal/hostnet/vrouter.go index 8014da6..d1ab5cd 100644 --- a/internal/hostnet/vrouter.go +++ b/internal/hostnet/vrouter.go @@ -224,11 +224,11 @@ func RouterConfigureAddrs(linkname string, addrs []string, gateway4, gateway6 st for _, addr := range addrs { if err := routerAddRoute(link, addr, "main"); err != nil { - fmt.Printf("DEBUG ConfigureRouterAddrs(): addRoute err (type = %T): %+v\n", err, err) + //fmt.Printf("DEBUG ConfigureRouterAddrs(): addRoute err (type = %T): %+v\n", err, err) return err } if err := routerAddRule(link, addr, "main"); err != nil { - fmt.Printf("DEBUG ConfigureRouterAddrs(): addRule err (type = %T): %+v\n", err, err) + //fmt.Printf("DEBUG ConfigureRouterAddrs(): addRule err (type = %T): %+v\n", err, err) return err } } diff --git a/server/network/configuration.go b/server/network/configuration.go index 77620ce..e8b5c00 100644 --- a/server/network/configuration.go +++ b/server/network/configuration.go @@ -2,7 +2,6 @@ package network import ( "context" - "encoding/json" "errors" "fmt" "io/fs" @@ -34,16 +33,6 @@ func (s *Server) CreateConf(ctx context.Context, vmname, ifname string, opts Net } } - /* - TODO: - убрать - if b, err := json.MarshalIndent(opts, "", " "); err == nil { - fmt.Printf("DEBUG: CreateConf: vmname = %s, ifname = %s, opts = %s\n", vmname, ifname, string(b)) - } else { - fmt.Printf("DEBUG: CreateConf: error = %s\n", err.Error()) - } - */ - err := s.TaskRunFunc(ctx, server.BlockAnyOperations(vmname, ifname+"/hostnet"), true, nil, func(l *log.Entry) error { schemes, err := GetNetworkSchemes(vmname) if err != nil { @@ -58,12 +47,6 @@ func (s *Server) CreateConf(ctx context.Context, vmname, ifname string, opts Net schemes = append(schemes, opts.Properties()) - if b, err := json.MarshalIndent(schemes, "", " "); err == nil { - fmt.Printf("DEBUG: CreateConf: vmname = %s, ifname = %s, schemes = %s\n", vmname, ifname, string(b)) - } else { - fmt.Printf("DEBUG: CreateConf: schemes error = %s\n", err.Error()) - } - if err := WriteNetworkSchemes(vmname, schemes...); err != nil { return err } @@ -120,12 +103,6 @@ func (s *Server) UpdateConf(ctx context.Context, vmname, ifname string, apply bo case SchemeUpdate_GATEWAY4, SchemeUpdate_GATEWAY6: scheme.Set(p.String(), update.Value) } - - if b, err := json.MarshalIndent(schemes, "", " "); err == nil { - fmt.Printf("DEBUG: UpdateConf: vmname = %s, ifname = %s, schemes = %s\n", vmname, ifname, string(b)) - } else { - fmt.Printf("DEBUG: UpdateConf: schemes error = %s\n", err.Error()) - } } if err := WriteNetworkSchemes(vmname, schemes...); err != nil { From 6fd70ddb142821426f276f938a962219244d9c7a Mon Sep 17 00:00:00 2001 From: Sergey Zhuravlev Date: Sun, 22 Feb 2026 18:16:17 +0500 Subject: [PATCH 05/13] Added command 'vmm info' -- details in human-readable format (#20) Co-authored-by: Sergey Zhuravlev --- client/configuration.go | 218 ++++++++++++++++++++++++++++++ cmd/vmm/commands/configuration.go | 16 ++- cmd/vmm/main.go | 1 + 3 files changed, 234 insertions(+), 1 deletion(-) diff --git a/client/configuration.go b/client/configuration.go index 133845c..3b645f8 100644 --- a/client/configuration.go +++ b/client/configuration.go @@ -11,6 +11,7 @@ import ( "github.com/0xef53/kvmrun/client/flag_types" "github.com/0xef53/kvmrun/kvmrun" + "github.com/0xef53/kvmrun/kvmrun/backend/block" pb_machines "github.com/0xef53/kvmrun/api/services/machines/v2" pb_types "github.com/0xef53/kvmrun/api/types/v2" @@ -145,6 +146,223 @@ func MachineInspect(ctx context.Context, vmname string, c *cli.Command, grpcClie return nil } +func MachineInfo(ctx context.Context, vmname string, c *cli.Command, grpcClient *grpc_interfaces.Kvmrun) error { + req := pb_machines.GetRequest{ + Name: vmname, + } + + resp, err := grpcClient.Machines().Get(ctx, &req) + if err != nil { + return err + } + + appendLine := func(s string, a ...interface{}) string { + switch len(a) { + case 0: + s += "\n" + case 1: + s += fmt.Sprintf("%*s : \n", 20, a[0]) + case 2: + var format string + switch a[1].(type) { + case int, int32, int64, uint, uint32, uint64: + format = "%*s : %d\n" + case string: + format = "%*s : %s\n" + default: + format = "%*s : %q\n" + } + s += fmt.Sprintf(format, 20, a[0], a[1]) + } + return s + } + + bootDevice := func(opts *pb_types.MachineOpts) string { + var bootdev string = "default" + var bootidx uint32 = ^uint32(0) + + for _, d := range opts.Cdrom { + if d.Bootindex > 0 && d.Bootindex < bootidx { + bootdev = d.Name + bootidx = d.Bootindex + } + } + + for _, d := range opts.Storage { + if d.Bootindex > 0 && d.Bootindex < bootidx { + bootdev = d.Path + bootidx = d.Bootindex + } + } + + return bootdev + } + + printBrief := func(m *pb_types.Machine) { + var opts *pb_types.MachineOpts + + if m.Runtime != nil { + opts = m.Runtime + } else { + opts = m.Config + } + + var s string + + // Header + if m.Runtime != nil { + s += fmt.Sprintf("* %s (state: %s, pid: %d", m.Name, m.State, m.PID) + } else { + s += fmt.Sprintf("* %s (state: %s", m.Name, m.State) + } + + if opts.VsockDevice != nil { + s += fmt.Sprintf(", cid: %d)", opts.VsockDevice.ContextID) + } else { + s += ")" + } + + s = appendLine(s) + + // Machine type + var machineType string + + if m.Runtime != nil { + machineType = opts.MachineType + } else { + machineType = "default" + } + + s = appendLine(s, "Machine type", machineType) + s = appendLine(s) + + // Processor + if opts.CPU != nil { + s = appendLine(s, "Processor") + s = appendLine(s, "Model", opts.CPU.Model) + s = appendLine(s, "Actual", opts.CPU.Actual) + s = appendLine(s, "Total", opts.CPU.Total) + + if c.Bool("verbose") { + s = appendLine(s, "Sockets", opts.CPU.Sockets) + } + + s = appendLine(s) + } + + // Memory + if opts.Memory != nil { + s = appendLine(s, "Memory") + s = appendLine(s, "Actual", fmt.Sprintf("%d MiB", opts.Memory.Actual)) + s = appendLine(s, "Total", fmt.Sprintf("%d MiB", opts.Memory.Total)) + s = appendLine(s) + } + + // Firmware + if opts.Firmware != nil { + s = appendLine(s, "Firmware") + s = appendLine(s, "Image", opts.Firmware.Image) + s = appendLine(s, "Flash", opts.Firmware.Flash) + s = appendLine(s) + } + + s = appendLine(s, "Boot device", bootDevice(opts)) + + if opts.CloudInitDrive != nil { + s = appendLine(s, "CloudInit", opts.CloudInitDrive.Path) + } + + s = appendLine(s) + + // Cdrom + if count := len(opts.Cdrom); count > 0 { + s = appendLine(s, "Cdroms", count) + + for _, d := range opts.Cdrom { + s = appendLine(s, d.Name, fmt.Sprintf("%s, %s", d.Driver, d.Media)) + } + + s = appendLine(s) + } + + // Storage + if count := len(opts.Storage); count > 0 { + s = appendLine(s, "Storage", count) + + for _, d := range opts.Storage { + // ignore all errors -- it's OK in this case + size, _ := block.GetSize64(d.Path) + + s = appendLine(s, filepath.Base(d.Path), fmt.Sprintf("%.2f GiB, %s", float64(size/(1<<30)), d.Driver)) + + if c.Bool("verbose") { + s = appendLine(s, "Path", d.Path) + s = appendLine(s, "IopsRd", d.IopsRd) + s = appendLine(s, "IopsWr", d.IopsWr) + s = appendLine(s) + } + } + + s = appendLine(s) + } + + // Network + if count := len(opts.Network); count > 0 { + s = appendLine(s, "Network", count) + + for _, nc := range opts.Network { + s = appendLine(s, nc.Ifname, fmt.Sprintf("%s, %s (queue = %d)", nc.HwAddr, nc.Driver, nc.Queues)) + + if c.Bool("verbose") { + s = appendLine(s, "Ifup", nc.Ifup) + s = appendLine(s, "Ifdown", nc.Ifdown) + s = appendLine(s) + } + } + + s = appendLine(s) + } + + // Input devices + if count := len(opts.Inputs); count > 0 { + s = appendLine(s, "Input devices", count) + + for _, d := range opts.Inputs { + s = appendLine(s, "", d.Type) + } + + s = appendLine(s) + } + + // Host PCI devices + if count := len(opts.HostPCI); count > 0 { + s = appendLine(s, "Host devices", count) + + for _, d := range opts.HostPCI { + s = appendLine(s, "", d.PCIAddr) + } + + s = appendLine(s) + } + + // External kernel + if opts.Kernel != nil { + s = appendLine(s, "External kernel") + s = appendLine(s, "Image", opts.Kernel.Image) + s = appendLine(s, "Cmdline", opts.Kernel.Cmdline) + s = appendLine(s, "Initrd", opts.Kernel.Initrd) + s = appendLine(s, "Modules", opts.Kernel.Modiso) + s = appendLine(s) + } + + fmt.Printf("%s", s) + } + + printBrief(resp.Machine) + + return nil +} + func MachineListEvents(ctx context.Context, vmname string, c *cli.Command, grpcClient *grpc_interfaces.Kvmrun) error { req := pb_machines.GetEventsRequest{ Name: vmname, diff --git a/cmd/vmm/commands/configuration.go b/cmd/vmm/commands/configuration.go index c882636..1f84a44 100644 --- a/cmd/vmm/commands/configuration.go +++ b/cmd/vmm/commands/configuration.go @@ -41,9 +41,23 @@ var CommandRemoveConf = &cli.Command{ }, } +var CommandInfo = &cli.Command{ + Name: "info", + Usage: "print a virtual machine details in human-readable format", + ArgsUsage: "VMNAME", + HideHelp: true, + Category: "Configuration", + Flags: []cli.Flag{ + &cli.BoolFlag{Name: "verbose", Aliases: []string{"v"}, Usage: "enable verbose output"}, + }, + Action: func(ctx context.Context, c *cli.Command) error { + return grpc_client.CommandGRPC(ctx, c, client.MachineInfo) + }, +} + var CommandInspect = &cli.Command{ Name: "inspect", - Usage: "print a virtual machine details", + Usage: "print low-level information about a virtual machine in JSON", ArgsUsage: "VMNAME", HideHelp: true, Category: "Configuration", diff --git a/cmd/vmm/main.go b/cmd/vmm/main.go index e04131b..b110220 100644 --- a/cmd/vmm/main.go +++ b/cmd/vmm/main.go @@ -55,6 +55,7 @@ func main() { commands.CommandCreateConf, commands.CommandRemoveConf, commands.CommandPrintList, + commands.CommandInfo, commands.CommandInspect, commands.MemoryCommands, commands.CPUCommands, From 15d5b4a176e152ad7d57cd071dfae8f70e60745c Mon Sep 17 00:00:00 2001 From: Sergey Zhuravlev Date: Sun, 22 Feb 2026 18:19:39 +0500 Subject: [PATCH 06/13] Version 2.0.2 --- kvmrun/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kvmrun/version.go b/kvmrun/version.go index b77e32f..d92ed36 100644 --- a/kvmrun/version.go +++ b/kvmrun/version.go @@ -4,4 +4,4 @@ import ( "github.com/0xef53/kvmrun/internal/version" ) -var Version = version.Version{Major: 2, Minor: 0, Micro: 1} +var Version = version.Version{Major: 2, Minor: 0, Micro: 2} From 83eaf07e6afd7787fe50e542d38bb6b654b65b2b Mon Sep 17 00:00:00 2001 From: Sergey Zhuravlev Date: Tue, 24 Feb 2026 16:44:59 +0500 Subject: [PATCH 07/13] Small fix: the name of the field with IP addresses (ips => addrs) in config_network (#22) Co-authored-by: Sergey Zhuravlev --- server/cloudinit/cloudinit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/cloudinit/cloudinit.go b/server/cloudinit/cloudinit.go index 6e1e871..062d07e 100644 --- a/server/cloudinit/cloudinit.go +++ b/server/cloudinit/cloudinit.go @@ -160,7 +160,7 @@ func buildEthernetsConfig(vmdir string) (map[string]cloudinit.EthernetConfig, er ID string `json:"id"` Name string `json:"ifname"` Scheme string `json:"scheme"` - Addrs []string `json:"ips"` + Addrs []string `json:"addrs"` Gateway4 string `json:"gateway4"` Gateway6 string `json:"gateway6"` From 80f21befe0cfbb99f5b1ab5a200f01a7d9040a14 Mon Sep 17 00:00:00 2001 From: Sergey Zhuravlev Date: Thu, 26 Feb 2026 19:59:04 +0500 Subject: [PATCH 08/13] Fix network configuring (#23) * Incorrect name resolution from the machine's home dir * Fixed GetRouteTableIndex() function The following files are now scanned: * /etc/iproute2/rt_tables * /usr/share/iproute2/rt_tables * /etc/iproute2/rt_tables.d/* --------- Co-authored-by: Sergey Zhuravlev --- client/ifupdown/ifupdown.go | 9 ++-- internal/utils/network.go | 77 ++++++++++++++++++++++++++--------- kvmrun/instance_properties.go | 2 +- 3 files changed, 64 insertions(+), 24 deletions(-) diff --git a/client/ifupdown/ifupdown.go b/client/ifupdown/ifupdown.go index 3c10f24..7537aca 100644 --- a/client/ifupdown/ifupdown.go +++ b/client/ifupdown/ifupdown.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "path/filepath" "github.com/0xef53/kvmrun/kvmrun" @@ -27,11 +28,11 @@ func InterfaceUp(ctx context.Context, ifname string, secondStage bool) error { var vmname string if cwd, err := os.Getwd(); err == nil { - if err := kvmrun.ValidateMachineName(cwd); err != nil { + if err := kvmrun.ValidateMachineName(filepath.Base(cwd)); err != nil { return err } - vmname = cwd + vmname = filepath.Base(cwd) } else { return fmt.Errorf("cannot determine machine name: %w", err) } @@ -67,11 +68,11 @@ func InterfaceDown(ctx context.Context, ifname string) error { var vmname string if cwd, err := os.Getwd(); err == nil { - if err := kvmrun.ValidateMachineName(cwd); err != nil { + if err := kvmrun.ValidateMachineName(filepath.Base(cwd)); err != nil { return err } - vmname = cwd + vmname = filepath.Base(cwd) } else { return fmt.Errorf("cannot determine machine name: %w", err) } diff --git a/internal/utils/network.go b/internal/utils/network.go index f55e732..1117040 100644 --- a/internal/utils/network.go +++ b/internal/utils/network.go @@ -2,9 +2,11 @@ package utils import ( "bufio" + "errors" "fmt" "net" "os" + "path/filepath" "strconv" "strings" @@ -24,34 +26,71 @@ func ParseIPNet(s string) (*net.IPNet, error) { } func GetRouteTableIndex(table string) (int, error) { - fd, err := os.Open("/etc/iproute2/rt_tables") - if err != nil { - return -1, err - } - defer fd.Close() + var errTableNotFound = errors.New("table not found") + + findIn := func(fname string) (int, error) { + fd, err := os.Open(fname) + if err != nil { + return -1, err + } + defer fd.Close() + + scanner := bufio.NewScanner(fd) - scanner := bufio.NewScanner(fd) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if strings.HasPrefix(line, "#") { + continue + } + + ff := strings.Fields(line) + + if len(ff) == 2 && strings.ToLower(ff[1]) == table { + if v, err := strconv.Atoi(ff[0]); err == nil { + return v, nil + } else { + return -1, err + } + } + } - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if strings.HasPrefix(line, "#") { - continue + if err := scanner.Err(); err != nil { + return -1, err } - ff := strings.Fields(line) + return -1, errTableNotFound + } + + possiblePlaces := []string{ + "/etc/iproute2/rt_tables", + "/usr/share/iproute2/rt_tables", + } - if len(ff) == 2 && strings.ToLower(ff[1]) == table { - if v, err := strconv.Atoi(ff[0]); err == nil { - return v, nil - } else { - return -1, err + // Also look in /etc/iproute2/rt_tables.d/* + if ff, err := os.ReadDir("/etc/iproute2/rt_tables.d"); err == nil { + for _, f := range ff { + fmt.Println(f.Name()) + if f.Type().IsRegular() { + possiblePlaces = append(possiblePlaces, filepath.Join("/etc/iproute2/rt_tables.d", f.Name())) } } + } else { + if !os.IsNotExist(err) { + return -1, err + } } - if err := scanner.Err(); err != nil { - return -1, err + for _, p := range possiblePlaces { + idx, err := findIn(p) + if err != nil { + if os.IsNotExist(err) || err == errTableNotFound { + continue + } + return -1, err + } + + return idx, nil } - return -1, fmt.Errorf("table not found: %s", table) + return -1, fmt.Errorf("%w: %s", errTableNotFound, table) } diff --git a/kvmrun/instance_properties.go b/kvmrun/instance_properties.go index c8e58cc..04cf4e9 100644 --- a/kvmrun/instance_properties.go +++ b/kvmrun/instance_properties.go @@ -15,7 +15,7 @@ func ValidateMachineName(name string) error { return nil } - return fmt.Errorf("invalid machine name: only [0-9A-Za-z_] are allowed, min length is 3 and max length is 16") + return fmt.Errorf("invalid machine name '%s': only [0-9A-Za-z_] are allowed, min length is 3 and max length is 16", name) } type InstanceProperties struct { From 4516b6cce8864c121605345914a23873b9bce797 Mon Sep 17 00:00:00 2001 From: Sergey Zhuravlev Date: Fri, 27 Feb 2026 15:39:29 +0500 Subject: [PATCH 09/13] Fixed work with the net_cls cgroup v1 controller (#24) Co-authored-by: Sergey Zhuravlev --- contrib/qemu.wrapper.debian | 8 ++++---- internal/utils/network.go | 1 - server/network/hostnet_configure.go | 13 +++++++++++++ server/network/hostnet_deconfigure.go | 16 ++++++++++++++++ 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/contrib/qemu.wrapper.debian b/contrib/qemu.wrapper.debian index 8e7514a..4058eef 100644 --- a/contrib/qemu.wrapper.debian +++ b/contrib/qemu.wrapper.debian @@ -30,10 +30,6 @@ if [[ "$QEMU_ROOTDIR" != "/" ]] ; then mount --bind "/sys/fs/cgroup" "${QEMU_ROOTDIR}/sys/fs/cgroup" fi - if mountpoint -q --nofollow "/sys/fs/cgroup/net_cls" ; then - mount --bind "/sys/fs/cgroup/net_cls" "${QEMU_ROOTDIR}/sys/fs/cgroup/net_cls" - fi - # kvmrun dirs install -d "${QEMU_ROOTDIR}/etc/kvmrun" install -d "${QEMU_ROOTDIR}/usr/lib/kvmrun" @@ -54,6 +50,10 @@ if [[ "$QEMU_ROOTDIR" != "/" ]] ; then /usr/lib/kvmrun/delegate-cgroup-v1-controller "$SYSTEMD_UNITNAME" "net_cls" fi + if mountpoint -q --nofollow "/sys/fs/cgroup/net_cls" ; then + mount --bind --make-shared "/sys/fs/cgroup/net_cls" "${QEMU_ROOTDIR}/sys/fs/cgroup/net_cls" + fi + # run chrooted QEMU binary exec chroot "$QEMU_ROOTDIR" /bin/sh -c "cd $PWD && exec /usr/bin/qemu-system-x86_64 ${ARGS}" fi diff --git a/internal/utils/network.go b/internal/utils/network.go index 1117040..167786a 100644 --- a/internal/utils/network.go +++ b/internal/utils/network.go @@ -69,7 +69,6 @@ func GetRouteTableIndex(table string) (int, error) { // Also look in /etc/iproute2/rt_tables.d/* if ff, err := os.ReadDir("/etc/iproute2/rt_tables.d"); err == nil { for _, f := range ff { - fmt.Println(f.Name()) if f.Type().IsRegular() { possiblePlaces = append(possiblePlaces, filepath.Join("/etc/iproute2/rt_tables.d", f.Name())) } diff --git a/server/network/hostnet_configure.go b/server/network/hostnet_configure.go index 0e7e3f6..33c7e5a 100644 --- a/server/network/hostnet_configure.go +++ b/server/network/hostnet_configure.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" + cg "github.com/0xef53/kvmrun/internal/cgroups" "github.com/0xef53/kvmrun/internal/hostnet" "github.com/0xef53/kvmrun/kvmrun" "github.com/0xef53/kvmrun/server" @@ -105,6 +106,18 @@ func (s *Server) ConfigureHostNetwork(ctx context.Context, vmname, ifname string return nil } + if cgroups, err := cg.GetProcessGroups(int(routerAttrs.ProcessID)); err == nil { + if g, ok := cgroups["net_cls"]; ok { + fname := filepath.Join(kvmrun.CHROOTDIR, vmname, "run/cgroups.net_cls.path") + + if err := os.WriteFile(fname, []byte(g.Path()), 0644); err != nil { + log.WithField("ifname", ifname).Warnf("Non-fatal error: %s", err) + } + } + } else { + log.WithField("ifname", ifname).Warnf("Non-fatal error: %s", err) + } + return err } case Scheme_BRIDGE: diff --git a/server/network/hostnet_deconfigure.go b/server/network/hostnet_deconfigure.go index 44f10a1..2325619 100644 --- a/server/network/hostnet_deconfigure.go +++ b/server/network/hostnet_deconfigure.go @@ -3,6 +3,8 @@ package network import ( "context" "fmt" + "os" + "path/filepath" "strings" "github.com/0xef53/kvmrun/internal/hostnet" @@ -60,6 +62,20 @@ func (s *Server) DeconfigureHostNetwork(ctx context.Context, vmname, ifname stri return err } + // Remove net_cls controller for this virt.machine + if b, err := os.ReadFile(filepath.Join(kvmrun.CHROOTDIR, vmname, "run/cgroups.net_cls.path")); err == nil { + dirname := string(b) + + // Just a fast check + if strings.HasSuffix(dirname, fmt.Sprintf("kvmrun@%s.service", vmname)) { + if err := os.RemoveAll(dirname); err == nil { + log.WithField("ifname", ifname).Infof("Removed: %s", dirname) + } else { + log.WithField("ifname", ifname).Warnf("Non-fatal error: %s", err) + } + } + } + return hostnet.RouterDeconfigure(ifname, attrs.BindInterface) case Scheme_BRIDGE: attrs, err := scheme.ExtractAttrs_Bridge() From 86aebb5fa20a087355431c7eed6e01f645c8f64c Mon Sep 17 00:00:00 2001 From: Sergey Zhuravlev Date: Mon, 16 Mar 2026 14:27:57 +0500 Subject: [PATCH 10/13] Replace outbound traffic shaping from cgroups net_cls to Traffic Control IP filters (#25) * Replace outbound traffic shaping from cgroups net_cls to Traffic Control IP filters --------- Co-authored-by: Sergey Zhuravlev --- api/services/cloudinit/v2/cloudinit.pb.go | 92 ++-- api/services/cloudinit/v2/cloudinit.proto | 6 +- contrib/qemu.wrapper.debian | 17 - internal/appconf/appconf.go | 43 +- internal/hostnet/bridge_port.go | 20 +- internal/hostnet/link.go | 63 ++- internal/hostnet/vrouter.go | 527 +++++++++++++--------- internal/hostnet/vrouter_tc.go | 344 ++++++++++++++ internal/hostnet/vxlan_port.go | 20 +- internal/utils/network.go | 29 ++ kvmrun/kvmrun.go | 1 + server/network/configuration.go | 66 ++- server/network/hostnet_configure.go | 61 +-- server/network/hostnet_deconfigure.go | 16 - server/network/scheme_properties.go | 28 ++ 15 files changed, 949 insertions(+), 384 deletions(-) create mode 100644 internal/hostnet/vrouter_tc.go diff --git a/api/services/cloudinit/v2/cloudinit.pb.go b/api/services/cloudinit/v2/cloudinit.pb.go index 35d709e..f5eeef2 100644 --- a/api/services/cloudinit/v2/cloudinit.pb.go +++ b/api/services/cloudinit/v2/cloudinit.pb.go @@ -8,6 +8,7 @@ package cloudinit import ( context "context" + _ "github.com/0xef53/go-grpc/options" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" @@ -216,49 +217,54 @@ var file_services_cloudinit_v2_cloudinit_proto_rawDesc = []byte{ 0x69, 0x6e, 0x69, 0x74, 0x2f, 0x76, 0x32, 0x2f, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x20, 0x6b, 0x76, 0x6d, 0x72, 0x75, 0x6e, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x63, 0x6c, 0x6f, - 0x75, 0x64, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x76, 0x32, 0x22, 0x94, 0x03, 0x0a, 0x11, 0x42, 0x75, - 0x69, 0x6c, 0x64, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x21, 0x0a, 0x0c, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x4e, 0x61, - 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x65, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x20, - 0x0a, 0x0b, 0x73, 0x75, 0x62, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x66, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x75, 0x62, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, - 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x67, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, - 0x0a, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x68, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, - 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x2b, 0x0a, 0x11, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, - 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x5f, 0x7a, 0x6f, 0x6e, 0x65, 0x18, 0x69, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x10, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x5a, - 0x6f, 0x6e, 0x65, 0x12, 0x1b, 0x0a, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0xc9, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x17, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0xca, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x1b, 0x0a, 0x08, 0x74, 0x69, 0x6d, - 0x65, 0x7a, 0x6f, 0x6e, 0x65, 0x18, 0xcb, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x69, - 0x6d, 0x65, 0x7a, 0x6f, 0x6e, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, - 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0xaa, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, - 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x20, 0x0a, 0x0b, - 0x75, 0x73, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0xab, 0x02, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x0a, 0x75, 0x73, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x20, - 0x0a, 0x0b, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0xad, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x46, 0x69, 0x6c, 0x65, - 0x22, 0x35, 0x0a, 0x12, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, - 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6f, 0x75, 0x74, - 0x70, 0x75, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x32, 0x8b, 0x01, 0x0a, 0x10, 0x43, 0x6c, 0x6f, 0x75, - 0x64, 0x49, 0x6e, 0x69, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x77, 0x0a, 0x0a, - 0x42, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x33, 0x2e, 0x6b, 0x76, 0x6d, - 0x72, 0x75, 0x6e, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x75, - 0x69, 0x6c, 0x64, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x34, 0x2e, 0x6b, 0x76, 0x6d, 0x72, 0x75, 0x6e, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x69, 0x6e, 0x69, 0x74, 0x2e, - 0x76, 0x32, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3e, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x30, 0x78, 0x65, 0x66, 0x35, 0x33, 0x2f, 0x6b, 0x76, 0x6d, 0x72, 0x75, - 0x6e, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x69, 0x6e, 0x69, 0x74, 0x2f, 0x76, 0x32, 0x3b, 0x63, 0x6c, 0x6f, 0x75, - 0x64, 0x69, 0x6e, 0x69, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x75, 0x64, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x76, 0x32, 0x1a, 0x35, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x30, 0x78, 0x65, 0x66, 0x35, 0x33, 0x2f, 0x67, 0x6f, 0x2d, + 0x67, 0x72, 0x70, 0x63, 0x2f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2f, 0x66, 0x69, 0x65, + 0x6c, 0x64, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x22, 0xa4, 0x03, 0x0a, 0x11, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, + 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6d, 0x61, + 0x63, 0x68, 0x69, 0x6e, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x6c, 0x61, + 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x65, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6c, 0x61, + 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x20, 0x0a, 0x0b, 0x73, 0x75, 0x62, 0x70, 0x6c, 0x61, 0x74, + 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x66, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x75, 0x62, 0x70, + 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x6f, 0x75, 0x64, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x67, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x6f, 0x75, + 0x64, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, + 0x68, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x2b, 0x0a, + 0x11, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x5f, 0x7a, 0x6f, + 0x6e, 0x65, 0x18, 0x69, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, + 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x5a, 0x6f, 0x6e, 0x65, 0x12, 0x1b, 0x0a, 0x08, 0x68, 0x6f, + 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0xc9, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x68, + 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x17, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, + 0x6e, 0x18, 0xca, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, + 0x12, 0x1b, 0x0a, 0x08, 0x74, 0x69, 0x6d, 0x65, 0x7a, 0x6f, 0x6e, 0x65, 0x18, 0xcb, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x69, 0x6d, 0x65, 0x7a, 0x6f, 0x6e, 0x65, 0x12, 0x2c, 0x0a, + 0x0d, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0xaa, + 0x02, 0x20, 0x01, 0x28, 0x0c, 0x42, 0x06, 0xca, 0xed, 0x1a, 0x02, 0x08, 0x01, 0x52, 0x0c, 0x76, + 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x28, 0x0a, 0x0b, 0x75, + 0x73, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0xab, 0x02, 0x20, 0x01, 0x28, + 0x0c, 0x42, 0x06, 0xca, 0xed, 0x1a, 0x02, 0x08, 0x01, 0x52, 0x0a, 0x75, 0x73, 0x65, 0x72, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, + 0x66, 0x69, 0x6c, 0x65, 0x18, 0xad, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6f, 0x75, 0x74, + 0x70, 0x75, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x22, 0x35, 0x0a, 0x12, 0x42, 0x75, 0x69, 0x6c, 0x64, + 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, + 0x0b, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0a, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x32, 0x8b, + 0x01, 0x0a, 0x10, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x49, 0x6e, 0x69, 0x74, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x12, 0x77, 0x0a, 0x0a, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x6d, 0x61, 0x67, + 0x65, 0x12, 0x33, 0x2e, 0x6b, 0x76, 0x6d, 0x72, 0x75, 0x6e, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x69, 0x6e, 0x69, + 0x74, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x34, 0x2e, 0x6b, 0x76, 0x6d, 0x72, 0x75, 0x6e, 0x2e, + 0x61, 0x70, 0x69, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x63, 0x6c, 0x6f, + 0x75, 0x64, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x49, + 0x6d, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3e, 0x5a, 0x3c, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x30, 0x78, 0x65, 0x66, 0x35, + 0x33, 0x2f, 0x6b, 0x76, 0x6d, 0x72, 0x75, 0x6e, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x69, 0x6e, 0x69, 0x74, 0x2f, + 0x76, 0x32, 0x3b, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x69, 0x6e, 0x69, 0x74, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/api/services/cloudinit/v2/cloudinit.proto b/api/services/cloudinit/v2/cloudinit.proto index 12c4550..831679b 100644 --- a/api/services/cloudinit/v2/cloudinit.proto +++ b/api/services/cloudinit/v2/cloudinit.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package kvmrun.api.services.cloudinit.v2; +import "github.com/0xef53/go-grpc/options/field_options.proto"; + option go_package = "github.com/0xef53/kvmrun/api/services/cloudinit/v2;cloudinit"; service CloudInitService { @@ -24,8 +26,8 @@ message BuildImageRequest { string domain = 202; string timezone = 203; - bytes vendor_config = 298; - bytes user_config = 299; + bytes vendor_config = 298 [(grpc.options.v1.log_formatting).display = Hide]; + bytes user_config = 299 [(grpc.options.v1.log_formatting).display = Hide]; string output_file = 301; } diff --git a/contrib/qemu.wrapper.debian b/contrib/qemu.wrapper.debian index 4058eef..5b3999c 100644 --- a/contrib/qemu.wrapper.debian +++ b/contrib/qemu.wrapper.debian @@ -41,19 +41,6 @@ if [[ "$QEMU_ROOTDIR" != "/" ]] ; then mount --bind "/var/lib/kvmrun" "${QEMU_ROOTDIR}/var/lib/kvmrun" mount --bind "/usr/share/kvmrun" "${QEMU_ROOTDIR}/usr/share/kvmrun" - # The net_cls controller is used by the "routed scheme" network configuration - # to limit an outgoing traffic of virtual machines. - # This script is used here because systemd does not manage controllers - # that will not be part of cgroup v2 in the future. - - if [[ -n "${SYSTEMD_UNITNAME:-}" ]] ; then - /usr/lib/kvmrun/delegate-cgroup-v1-controller "$SYSTEMD_UNITNAME" "net_cls" - fi - - if mountpoint -q --nofollow "/sys/fs/cgroup/net_cls" ; then - mount --bind --make-shared "/sys/fs/cgroup/net_cls" "${QEMU_ROOTDIR}/sys/fs/cgroup/net_cls" - fi - # run chrooted QEMU binary exec chroot "$QEMU_ROOTDIR" /bin/sh -c "cd $PWD && exec /usr/bin/qemu-system-x86_64 ${ARGS}" fi @@ -61,8 +48,4 @@ if [[ "$QEMU_ROOTDIR" != "/" ]] ; then UNSHARED=1 exec unshare -m "$0" ${ARGS} fi -if [[ -n "${SYSTEMD_UNITNAME:-}" ]] ; then - /usr/lib/kvmrun/delegate-cgroup-v1-controller "$SYSTEMD_UNITNAME" "net_cls" -fi - exec /usr/bin/qemu-system-x86_64 "$@" diff --git a/internal/appconf/appconf.go b/internal/appconf/appconf.go index 2ca97dc..a8734ce 100644 --- a/internal/appconf/appconf.go +++ b/internal/appconf/appconf.go @@ -4,10 +4,13 @@ import ( "crypto/tls" "crypto/x509" "fmt" + "net" "os" "path/filepath" "strings" + "github.com/0xef53/kvmrun/internal/utils" + grpcserver "github.com/0xef53/go-grpc/server" "gopkg.in/gcfg.v1" @@ -18,10 +21,46 @@ type KvmrunConfig struct { CertDir string `gcfg:"cert-dir"` } +type VnetConfig struct { + UnmanagedNets UnmanagedNets `gcfg:"unmanaged"` +} + +type UnmanagedNets struct { + subnets []*net.IPNet +} + +func (v *UnmanagedNets) UnmarshalText(b []byte) error { + s := strings.TrimSpace(string(b)) + + ipnet, err := utils.ParseIPNet(s) + if err != nil { + return err + } + + if v.subnets == nil { + v.subnets = make([]*net.IPNet, 0, 4) + } + + v.subnets = append(v.subnets, ipnet) + + return nil +} + +func (v *UnmanagedNets) Contains(ip net.IP) bool { + for _, subnet := range v.subnets { + if subnet.Contains(ip) { + return true + } + } + + return false +} + // Config represents the Kvmrun configuration type Config struct { - Kvmrun KvmrunConfig `gcfg:"common"` - Server grpcserver.Config `gcfg:"server"` + Kvmrun KvmrunConfig `gcfg:"common"` + Server grpcserver.Config `gcfg:"server"` + VirtNet VnetConfig `gcfg:"virtual-net"` TLSConfig *tls.Config `gcfg:"-"` diff --git a/internal/hostnet/bridge_port.go b/internal/hostnet/bridge_port.go index 422a724..390c270 100644 --- a/internal/hostnet/bridge_port.go +++ b/internal/hostnet/bridge_port.go @@ -27,25 +27,25 @@ func BridgePortConfigure(linkname string, attrs *BridgePortAttrs, secondStage bo case netlink.LinkNotFoundError: return fmt.Errorf("bridge does not exist: %s", attrs.BridgeName) default: - return fmt.Errorf("netlink: %s", err) + return fmt.Errorf("netlink: %w", err) } link, err := netlink.LinkByName(linkname) if err != nil { - return fmt.Errorf("netlink: %s", err) + return fmt.Errorf("netlink: %w", err) } if err := netlink.LinkSetMaster(link, brLink); err != nil { - return fmt.Errorf("netlink: %s", err) + return fmt.Errorf("netlink: %w", err) } if err := netlink.LinkSetUp(link); err != nil { - return fmt.Errorf("netlink: %s", err) + return fmt.Errorf("netlink: %w", err) } if attrs.MTU >= 68 { if err := netlink.LinkSetMTU(link, int(attrs.MTU)); err != nil { - return fmt.Errorf("netlink: %s: %s", link.Attrs().Name, err) + return fmt.Errorf("netlink: %s: %w", link.Attrs().Name, err) } } @@ -59,11 +59,11 @@ func BridgePortDeconfigure(linkname string, brname string) error { // link already removed, so do nothing return nil } - return fmt.Errorf("netlink: %s", err) + return fmt.Errorf("netlink: %w", err) } if err := netlink.LinkSetNoMaster(link); err != nil { - return fmt.Errorf("netlink: %s", err) + return fmt.Errorf("netlink: %w", err) } return nil @@ -87,14 +87,14 @@ func CreateBridgeIfNotExist(linkname string) (netlink.Link, error) { link = &netlink.Bridge{LinkAttrs: attrs} if err := netlink.LinkAdd(link); err != nil { - return nil, fmt.Errorf("netlink: %s", err) + return nil, fmt.Errorf("netlink: %w", err) } default: - return nil, fmt.Errorf("netlink: %s", err) + return nil, fmt.Errorf("netlink: %w", err) } if err := netlink.LinkSetUp(link); err != nil { - return nil, fmt.Errorf("netlink: %s", err) + return nil, fmt.Errorf("netlink: %w", err) } return link, nil diff --git a/internal/hostnet/link.go b/internal/hostnet/link.go index 16d1d18..352e46f 100644 --- a/internal/hostnet/link.go +++ b/internal/hostnet/link.go @@ -2,8 +2,14 @@ package hostnet import ( "crypto/md5" + "encoding/json" + "errors" "fmt" "math/big" + "os" + "path/filepath" + + "github.com/0xef53/kvmrun/kvmrun" "github.com/vishvananda/netlink" ) @@ -30,7 +36,9 @@ func RemoveLinkIfExist(linkname string) error { return nil } -func GetLinkID(linkname string, linkindex int) uint16 { +type LinkID uint16 + +func GetLinkID(linkname string, linkindex int) LinkID { h := md5.New() fmt.Fprintf(h, "%s:%d", linkname, linkindex) @@ -41,5 +49,56 @@ func GetLinkID(linkname string, linkindex int) uint16 { // Should be a number between 200 and 65000 x := big.NewInt(0).Mod(bi, big.NewInt(64800)).Int64() + 200 - return uint16(x) + return LinkID(x) +} + +func (v LinkID) ClassID() uint32 { + return netlink.MakeHandle(1, uint16(v)) +} + +func (v LinkID) String() string { + return fmt.Sprintf("1:0x%x", uint16(v)) +} + +func ensureLink(link netlink.Link) { + if link == nil { + panic("link is not specified") + } +} + +var ErrLinkNotFound = errors.New("link not found") + +func LinkFromDumpFile(linkname string) (netlink.Link, error) { + link := netlink.GenericLink{ + LinkAttrs: netlink.NewLinkAttrs(), + LinkType: "generic", + } + + b, err := os.ReadFile(filepath.Join(kvmrun.NETWORKDIR, linkname+".link")) + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("%w: %w", ErrLinkNotFound, err) + } + + return nil, err + } + + if err := json.Unmarshal(b, &link.LinkAttrs); err != nil { + return nil, err + } + + return &link, nil +} + +func LinkWriteDumpFile(link netlink.Link) error { + if err := os.MkdirAll(kvmrun.NETWORKDIR, 0755); err != nil { + return err + } + + b, err := json.MarshalIndent(link.Attrs(), "", " ") + if err != nil { + return err + } + + return os.WriteFile(filepath.Join(kvmrun.NETWORKDIR, link.Attrs().Name+".link"), b, 0644) } diff --git a/internal/hostnet/vrouter.go b/internal/hostnet/vrouter.go index d1ab5cd..61f5b87 100644 --- a/internal/hostnet/vrouter.go +++ b/internal/hostnet/vrouter.go @@ -2,8 +2,6 @@ package hostnet import ( "context" - "encoding/json" - "errors" "fmt" "net" "net/netip" @@ -12,251 +10,311 @@ import ( "path/filepath" "strings" - cg "github.com/0xef53/kvmrun/internal/cgroups" "github.com/0xef53/kvmrun/internal/garp" "github.com/0xef53/kvmrun/internal/ipmath" "github.com/0xef53/kvmrun/internal/utils" + "github.com/0xef53/kvmrun/kvmrun" "github.com/vishvananda/netlink" ) -var ( - ErrCgroupBinding = errors.New("failed to configure cgroup") -) - type VirtualRouterAttrs struct { - Addrs []string - MTU uint32 - BindInterface string - Gateway4 string - Gateway6 string - InLimit uint32 - OutLimit uint32 - - ProcessID uint32 + Addrs []string + UnmanagedAddrs []string + MTU uint32 + BindIface string + Gateway4 string + Gateway6 string + InLimit uint32 + OutLimit uint32 } func RouterConfigure(linkname string, attrs *VirtualRouterAttrs, secondStage bool) error { if secondStage { - return RouterConfigureAddrs(linkname, attrs.Addrs, attrs.Gateway4, attrs.Gateway6) + if err := RouterConfigureAddrs(attrs.BindIface, linkname, false, attrs.Addrs...); err != nil { + return nil + } + + if err := RouterConfigureAddrs(attrs.BindIface, linkname, true, attrs.UnmanagedAddrs...); err != nil { + return nil + } + + // Send Gratuitous ARP for all router gateways + return RouterAnnounceGateways(linkname, append(attrs.Addrs, attrs.UnmanagedAddrs...), attrs.Gateway4) } return RouterConfigureInterface(linkname, attrs) } +func RouterDeconfigure(linkname, tcBindIface string) error { + // Read the attributes from a saved dump file, since we cannot know for sure + // that the physical link still exists (custom ifup/ifdown scripts may have + // already deleted it). + // If no dump file, ErrLinkNotFound will be returned. + link, err := LinkFromDumpFile(linkname) + if err != nil { + return err + } + defer os.Remove(filepath.Join(kvmrun.NETWORKDIR, linkname+".link")) + + // Remove all rules including IPv4/IPv6 blackhole + routerRemoveRules(link) + + // Remove all routes and GW addresses + routerRemoveRoutes(link) + + // Remove QoS configuration for incoming traffic + routerSetInboundLimits(link, 0) + + // Remove QoS configuration for outgoing traffic + if len(tcBindIface) > 0 { + routerSetOutboundLimits(GetLinkID(linkname, link.Attrs().Index), 0, tcBindIface) + } + + return nil +} + func RouterConfigureInterface(linkname string, attrs *VirtualRouterAttrs) error { - if attrs.OutLimit > 0 && len(attrs.BindInterface) == 0 { + if attrs.OutLimit > 0 && len(attrs.BindIface) == 0 { return fmt.Errorf("can not setup outbound limit: bind_interface is not set") } link, err := netlink.LinkByName(linkname) if err != nil { - return fmt.Errorf("netlink: %s", err) + return fmt.Errorf("netlink: %w", err) + } + + // Save the dump with the network interface attributes. + // It will be needed in the RouterDeconfigure() function, + // which may no longer exist by the time it's called. + if err := LinkWriteDumpFile(link); err != nil { + return fmt.Errorf("cannot write link dump file: %w", err) } linkID := GetLinkID(linkname, link.Attrs().Index) if err := netlink.LinkSetUp(link); err != nil { - return fmt.Errorf("netlink: %s", err) + return fmt.Errorf("netlink: %w", err) } if attrs.MTU >= 68 { if err := netlink.LinkSetMTU(link, int(attrs.MTU)); err != nil { - return fmt.Errorf("netlink: %s: %s", linkname, err) + return fmt.Errorf("netlink: %s: %w", linkname, err) } } - if err := routerCreateBlackholeRules(linkname); err != nil { + // IPv4 && IPv6 blackhole rules + if err := routerAddBlackholeRules(link); err != nil { return err } - if err := routerSetInboundLimits(linkname, attrs.InLimit); err != nil { + if err := routerSetInboundLimits(link, attrs.InLimit); err != nil { return err } - if len(attrs.BindInterface) > 0 { - if err := routerSetOutboundLimits(linkname, linkID, attrs.ProcessID, attrs.OutLimit, attrs.BindInterface); err != nil { + if len(attrs.BindIface) > 0 { + if err := routerSetOutboundLimits(linkID, attrs.OutLimit, attrs.BindIface); err != nil { return err } + } + + return nil +} + +func RouterConfigureAddrs(tcBindIface, linkname string, unmanaged bool, addrs ...string) error { + link, err := netlink.LinkByName(linkname) + if err != nil { + return fmt.Errorf("netlink: %w", err) + } - if b, err := json.MarshalIndent(link.Attrs(), "", " "); err == nil { - if err := os.WriteFile(filepath.Join("/run/kvm-network", linkname), b, 0644); err != nil { + linkID := GetLinkID(linkname, link.Attrs().Index) + + for _, addr := range addrs { + if err := routerAddRoute(link, addr, "main"); err != nil { + return fmt.Errorf("failed to create route: %w", err) + } + + if err := routerAddRule(link, addr, "main"); err != nil { + return fmt.Errorf("failed to create rule: %w", err) + } + + if len(tcBindIface) > 0 && !unmanaged { + if err := tcAddFilter(tcBindIface, linkID.ClassID(), addr, ADDR_DIRECTION_SRC); err != nil { return err } - } else { - return err } } return nil } -func routerCreateBlackholeRules(linkname string) error { - /* - TODO: should be rewritten using the "netlink" library +func RouterDeconfigureAddrs(tcBindIface, linkname string, unmanaged bool, addrs ...string) error { + link, err := netlink.LinkByName(linkname) + if err != nil { + return fmt.Errorf("netlink: %w", err) + } - See https://github.com/vishvananda/netlink/issues/838 for details - */ + linkID := GetLinkID(linkname, link.Attrs().Index) - for _, f := range []string{"inet", "inet6"} { - args := []string{"-family", f, "rule", "add", "iif", linkname, "blackhole"} + for _, addr := range addrs { + if err := routerRemoveRoutes(link, addr); err != nil { + return fmt.Errorf("failed to remove route: %w", err) + } - if out, err := exec.Command("ip", args...).CombinedOutput(); err != nil { - return fmt.Errorf("failed to create blackhole rule for %s (family = %s): %s", linkname, f, strings.TrimSpace(string(out))) + if err := routerRemoveRules(link, addr); err != nil { + return fmt.Errorf("failed to remove rule: %w", err) + } + + if len(tcBindIface) > 0 && !unmanaged { + if err := tcRemoveFilters(tcBindIface, linkID.ClassID(), addr); err != nil { + return err + } } } return nil } -func RouterSetInboundLimits(linkname string, rate uint32) error { - return routerSetInboundLimits(linkname, rate) -} +func RouterAnnounceGateways(linkname string, addrs []string, gateway4 string) error { + link, err := netlink.LinkByName(linkname) + if err != nil { + return fmt.Errorf("netlink: %w", err) + } -func routerSetInboundLimits(linkname string, rate uint32) error { - /* - TODO: should be rewritten using the "netlink" library - */ + gws := make(map[string]struct{}) - // We don't care about possible errors - exec.Command("tc", "qdisc", "del", "dev", linkname, "root").Run() + for _, addr := range addrs { + ip, err := utils.ParseIPNet(addr) + if err != nil { + return err + } - if rate == 0 { - return nil - } + if ip.IP.To4() == nil { + // Only IPv4 addrs are supported + continue + } + + var gw string - // Make new configuration - qdiscArgs := []string{"qdisc", "replace", "dev", linkname, "root", "handle", "1", "htb", "default", "1"} + maskOnes, _ := ip.Mask.Size() + + if maskOnes <= 30 { + lastIP, _ := ipmath.GetLastIPv4(ip) - if out, err := exec.Command("tc", qdiscArgs...).CombinedOutput(); err != nil { - if exitCode, ok := utils.CommandExitCode(err); !ok || exitCode != 2 { - return fmt.Errorf("failed to create qdisc rule for %s (%s): %s", linkname, err, strings.TrimSpace(string(out))) + gw = netip.MustParseAddr(lastIP.String()).Prev().String() + } else { + gw = gateway4 } - } - classArgs := []string{"class", "replace", "dev", linkname, "parent", "1:", "classid", "1:1", "htb", "rate", fmt.Sprintf("%dmbit", rate)} + if _, ok := gws[gw]; !ok && len(gw) > 0 { + gws[gw] = struct{}{} - if out, err := exec.Command("tc", classArgs...).CombinedOutput(); err != nil { - return fmt.Errorf("failed to create class rule for %s (%s): %s", linkname, err, strings.TrimSpace(string(out))) + go garp.Send(context.Background(), link.Attrs().Name, gw, 10, 1) + } } return nil } -func RouterSetOutboundLimits(linkname string, rate uint32, bindInterface string, pid uint32) error { +// +// QoS functions +// + +func RouterSetInboundLimits(linkname string, rateMbit uint32) error { link, err := netlink.LinkByName(linkname) if err != nil { return fmt.Errorf("netlink: %w", err) } - linkID := GetLinkID(linkname, link.Attrs().Index) - - if _, err := netlink.LinkByName(bindInterface); err != nil { - return fmt.Errorf("netlink: %w", err) - } - - return routerSetOutboundLimits(linkname, linkID, pid, rate, bindInterface) + return routerSetInboundLimits(link, rateMbit) } -func routerSetOutboundLimits(linkname string, linkID uint16, pid, rate uint32, bindInterface string) error { - /* - TODO: should be rewritten using the "netlink" library - */ +func routerSetInboundLimits(link netlink.Link, rateMbit uint32) error { + ensureLink(link) + + linkname := link.Attrs().Name // We don't care about possible errors - exec.Command("tc", "class", "del", "dev", bindInterface, "classid", fmt.Sprintf("1:0x%x", linkID)).Run() + tcRemoveQdisc(linkname) - if rate == 0 { + if rateMbit == 0 { return nil } - // Make new configuration - qdiscArgs := []string{"qdisc", "add", "dev", bindInterface, "root", "handle", "1", "htb"} - - // Try to add htb discipline to the root of bindInterface. - // If discipline is exist, the return code will be 2. - if out, err := exec.Command("tc", qdiscArgs...).CombinedOutput(); err != nil { - if exitCode, ok := utils.CommandExitCode(err); !ok || exitCode != 2 { - return fmt.Errorf("failed to create qdisc rule for %s (%s): %s", bindInterface, err, strings.TrimSpace(string(out))) - } + // Make a new qdisc on interface if not exists + if err := tcCreateQdisc(linkname); err != nil { + return err } - filterArgs := []string{"filter", "replace", "dev", bindInterface, "parent", "1:", "protocol", "all", "prio", "10", "handle", "1:", "cgroup"} - - if out, err := exec.Command("tc", filterArgs...).CombinedOutput(); err != nil { - return fmt.Errorf("failed to create filter rule for %s (%s): %s", bindInterface, err, strings.TrimSpace(string(out))) + if err := tcCreateClass(linkname, netlink.MakeHandle(1, 1), rateMbit); err != nil { + return err } - classArgs := []string{"class", "add", "dev", bindInterface, "parent", "1:", "classid", fmt.Sprintf("1:0x%x", linkID), "htb", "rate", fmt.Sprintf("%dmbit", rate)} + return nil +} - if out, err := exec.Command("tc", classArgs...).CombinedOutput(); err != nil { - return fmt.Errorf("failed to create class rule for %s (linkname = %s, classid = 1:0x%x) (%s): %s", bindInterface, linkname, linkID, err, strings.TrimSpace(string(out))) +func RouterSetOutboundLimits(linkname string, rateMbit uint32, tcBindIface string) error { + link, err := netlink.LinkByName(linkname) + if err != nil { + return fmt.Errorf("netlink: %w", err) } - // Try to set net_cls.classid for the virt.machine process - err := func() error { - if pid > 0 { - cgmgr, err := cg.LoadManager(int(pid)) - if err != nil { - return err - } + linkID := GetLinkID(linkname, link.Attrs().Index) - if err := cgmgr.SetNetClassID(int64(65536 + int(linkID))); err != nil { - return err - } - } + if _, err := netlink.LinkByName(tcBindIface); err != nil { + return fmt.Errorf("netlink: %w", err) + } - return nil - }() + return routerSetOutboundLimits(linkID, rateMbit, tcBindIface) +} - if err != nil { - return fmt.Errorf("%w: %s", ErrCgroupBinding, err) +func routerSetOutboundLimits(linkID LinkID, rateMbit uint32, tcBindIface string) error { + if rateMbit == 0 { // means that the configuration needs to be completely removed + // filters must be removed first in this case + tcRemoveFilters(tcBindIface, linkID.ClassID()) } - return nil -} + // We don't care about possible errors + tcRemoveClass(tcBindIface, linkID.ClassID()) -func RouterConfigureAddrs(linkname string, addrs []string, gateway4, gateway6 string) error { - link, err := netlink.LinkByName(linkname) - if err != nil { - return fmt.Errorf("netlink: %s", err) + if rateMbit == 0 { + return nil } - for _, addr := range addrs { - if err := routerAddRoute(link, addr, "main"); err != nil { - //fmt.Printf("DEBUG ConfigureRouterAddrs(): addRoute err (type = %T): %+v\n", err, err) - return err - } - if err := routerAddRule(link, addr, "main"); err != nil { - //fmt.Printf("DEBUG ConfigureRouterAddrs(): addRule err (type = %T): %+v\n", err, err) - return err - } + // Make a new qdisc on interface if not exists + if err := tcCreateQdisc(tcBindIface); err != nil { + return err } - // Send Gratuitous ARP for all router gateways - if err := routerAnnounceGateways(link, addrs, gateway4); err != nil { + if err := tcCreateClass(tcBindIface, linkID.ClassID(), rateMbit); err != nil { return err } return nil } +// +// ip-route / ip-addr functions +// + func routerAddRoute(link netlink.Link, addr, table string) error { + ensureLink(link) + tableNum, err := utils.GetRouteTableIndex(table) if err != nil { return err } - ip, err := utils.ParseIPNet(addr) + ipnet, err := utils.ParseIPNet(addr) if err != nil { return err } - maskOnes, maskBits := ip.Mask.Size() + maskOnes, maskBits := ipnet.Mask.Size() - if ip.IP.To4() != nil { + if ipnet.IP.To4() != nil { if maskOnes <= 30 { - lastIP, _ := ipmath.GetLastIPv4(ip) + lastIP, _ := ipmath.GetLastIPv4(ipnet) gwAddr := netlink.Addr{ IPNet: &net.IPNet{ @@ -266,12 +324,12 @@ func routerAddRoute(link netlink.Link, addr, table string) error { } if err := netlink.AddrAdd(link, &gwAddr); err != nil { - return err + return fmt.Errorf("netlink: unable to add gateway address: %w", err) } } } else { if maskOnes <= 64 { - lastIP, _ := ipmath.GetLastIPv6(ip) + lastIP, _ := ipmath.GetLastIPv6(ipnet) lastAddr := netlink.Addr{ IPNet: &net.IPNet{ @@ -281,14 +339,14 @@ func routerAddRoute(link netlink.Link, addr, table string) error { } if err := netlink.AddrAdd(link, &lastAddr); err != nil { - return err + return fmt.Errorf("netlink: unable to add gateway address: %w", err) } } else { - return fmt.Errorf("too small ipv6 netmask") + return fmt.Errorf("too small IPv6 netmask") } } - _, dst, err := net.ParseCIDR(addr) + _, dst, err := utils.ParseCIDR(addr) if err != nil { return err } @@ -301,13 +359,76 @@ func routerAddRoute(link netlink.Link, addr, table string) error { } if err := netlink.RouteReplace(&r); err != nil { - return fmt.Errorf("netlink: %s", err) + return fmt.Errorf("netlink: %w", err) } return nil } +func routerRemoveRoutes(link netlink.Link, addrs ...string) error { + ensureLink(link) + + routes, err := netlink.RouteList(link, netlink.FAMILY_ALL) + if err != nil { + return fmt.Errorf("netlink: %w", err) + } + + // We should also delete gateway addresses from the link + ifacesAddrs, err := netlink.AddrList(link, netlink.FAMILY_ALL) + if err != nil { + return fmt.Errorf("netlink: %w", err) + } + + var candidates []netlink.Route + + if len(addrs) == 0 { + candidates = routes + } else { + dstPrefixes := make(map[string]struct{}) + + // Normalize specified addr list + for _, addr := range addrs { + _, dst, err := utils.ParseCIDR(addr) + if err != nil { + return err + } + + dstPrefixes[dst.String()] = struct{}{} + } + + candidates = make([]netlink.Route, 0, len(addrs)) + + for _, route := range routes { + if _, ok := dstPrefixes[route.Dst.String()]; ok { + candidates = append(candidates, route) + } + } + } + + for _, route := range candidates { + if route.Dst.IP.IsLinkLocalUnicast() || route.Dst.IP.IsLinkLocalMulticast() { + continue + } + + for _, addr := range ifacesAddrs { + if route.Dst.Contains(addr.IP) { + netlink.AddrDel(link, &addr) + } + } + + netlink.RouteDel(&route) + } + + return nil +} + +// +// ip-rule functions +// + func routerAddRule(link netlink.Link, addr, table string) error { + ensureLink(link) + tableNum, err := utils.GetRouteTableIndex(table) if err != nil { return err @@ -331,124 +452,100 @@ func routerAddRule(link netlink.Link, addr, table string) error { } if err := netlink.RuleAdd(rule); err != nil { - return err + return fmt.Errorf("netlink: %w", err) } return nil } -func routerAnnounceGateways(link netlink.Link, addrs []string, gateway4 string) error { - gws := make(map[string]struct{}) +func routerRemoveRules(link netlink.Link, prefixes ...string) error { + ensureLink(link) - for _, addr := range addrs { - ip, err := utils.ParseIPNet(addr) - if err != nil { - return err - } - - if ip.IP.To4() == nil { - // Only IPv4 addrs are supported - continue - } - - var gw string + linkname := link.Attrs().Name - maskOnes, _ := ip.Mask.Size() + /* + TODO: see https://github.com/vishvananda/netlink/issues/838 for details. + */ + rules, err := netlink.RuleList(netlink.FAMILY_ALL) + if err != nil { + return fmt.Errorf("netlink: %w", err) + } - if maskOnes <= 30 { - lastIP, _ := ipmath.GetLastIPv4(ip) + var candidates []netlink.Rule - gw = netip.MustParseAddr(lastIP.String()).Prev().String() - } else { - gw = gateway4 - } + if len(prefixes) == 0 { + candidates = rules + } else { + normalized := make(map[string]struct{}) - if _, ok := gws[gw]; !ok && len(gw) > 0 { - gws[gw] = struct{}{} + // Normalize specified prefixes list + for _, addr := range prefixes { + ipnet, err := utils.ParseIPNet(addr) + if err != nil { + return err + } - go garp.Send(context.Background(), link.Attrs().Name, gw, 10, 1) + normalized[ipnet.String()] = struct{}{} } - } - return nil -} + candidates = make([]netlink.Rule, 0, len(prefixes)) -func RouterDeconfigure(linkname, bindInterface string) error { - // Remove all rules including blackhole - if rules, err := netlink.RuleList(netlink.FAMILY_ALL); err == nil { for _, rule := range rules { - if rule.IifName == linkname { - netlink.RuleDel(&rule) + if _, ok := normalized[rule.Src.String()]; ok { + candidates = append(candidates, rule) } } } - routerRemoveBlackholeRulesV6(linkname) - - // Remove all routes and addresses - if link, err := netlink.LinkByName(linkname); err == nil { - if addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL); err == nil { - for _, addr := range addrs { - if addr.IP.IsLinkLocalUnicast() || addr.IP.IsLinkLocalMulticast() { - continue - } - netlink.AddrDel(link, &addr) - } - } - if routes, err := netlink.RouteList(link, netlink.FAMILY_ALL); err == nil { - for _, route := range routes { - if route.Dst.IP.IsLinkLocalUnicast() || route.Dst.IP.IsLinkLocalMulticast() { - continue - } - netlink.RouteDel(&route) - } + for _, rule := range candidates { + if rule.IifName == linkname { + netlink.RuleDel(&rule) } } - // Remove all TC rules - routerSetInboundLimits(linkname, 0) - - if len(bindInterface) > 0 { - attrs := struct { - Index int `json:"index"` - }{} - - if b, err := os.ReadFile(filepath.Join("/run/kvm-network", linkname)); err == nil { - if err := json.Unmarshal(b, &attrs); err != nil { - return err - } - } else { - if os.IsNotExist(err) { - return nil + if len(prefixes) == 0 { + // Remove all blackhole rules from link + for _, f := range []string{"inet", "inet6"} { + args := []string{"-family", f, "rule", "del", "from", "all", "iif", linkname, "blackhole"} + + for { + if out, err := exec.Command("ip", args...).CombinedOutput(); err != nil { + if exitCode, ok := utils.CommandExitCode(err); ok && exitCode == 2 { + break + } + return fmt.Errorf( + "failed to remove blackhole rule for %s (%w): %s", + linkname, + err, + strings.TrimSpace(string(out)), + ) + } } - return err } - - linkID := GetLinkID(linkname, attrs.Index) - - routerSetOutboundLimits(linkname, linkID, 0, 0, bindInterface) - - os.Remove(filepath.Join("/run/kvm-network", linkname)) } return nil } -func routerRemoveBlackholeRulesV6(linkname string) error { - /* - TODO: should be rewritten using the "netlink" library +func routerAddBlackholeRules(link netlink.Link) error { + ensureLink(link) - See https://github.com/vishvananda/netlink/issues/838 for details + /* + TODO: see https://github.com/vishvananda/netlink/issues/838 for details. */ - args := []string{"-family", "inet6", "rule", "del", "from", "all", "iif", linkname, "blackhole"} + linkname := link.Attrs().Name + + for _, f := range []string{"inet", "inet6"} { + args := []string{"-family", f, "rule", "add", "iif", linkname, "blackhole"} - for { if out, err := exec.Command("ip", args...).CombinedOutput(); err != nil { - if exitCode, ok := utils.CommandExitCode(err); ok && exitCode == 2 { - break - } - return fmt.Errorf("failed to remove IPv6 blackhole rule for %s (%s): %s", linkname, err, strings.TrimSpace(string(out))) + return fmt.Errorf( + "failed to create blackhole rule for %s (family = %s): %s", + linkname, + f, + strings.TrimSpace(string(out)), + ) } } diff --git a/internal/hostnet/vrouter_tc.go b/internal/hostnet/vrouter_tc.go new file mode 100644 index 0000000..c08d77f --- /dev/null +++ b/internal/hostnet/vrouter_tc.go @@ -0,0 +1,344 @@ +package hostnet + +/* + TODO: probably all the TC functions should be rewritten using the "netlink" library +*/ + +import ( + "fmt" + "net" + "os/exec" + "strings" + + "github.com/0xef53/kvmrun/internal/utils" + "golang.org/x/sys/unix" + + "github.com/vishvananda/netlink" +) + +func tcCreateQdisc(bindIface string) error { + args := []string{ + "qdisc", + "replace", + "dev", bindIface, + "root", + "handle", "1", + "htb", + "default", "1", + } + + // Try to add htb discipline to the root of bindIface. + // If discipline is exist, the return code will be 2. + out, err := exec.Command("tc", args...).CombinedOutput() + if err != nil { + if exitCode, ok := utils.CommandExitCode(err); !ok || exitCode != 2 { + return fmt.Errorf( + "failed to create qdisc on %s (%w): %s", + bindIface, + err, + strings.TrimSpace(string(out)), + ) + } + } + + return nil +} + +func tcRemoveQdisc(bindIface string) error { + args := []string{ + "qdisc", + "del", + "dev", bindIface, + "root", + } + + out, err := exec.Command("tc", args...).CombinedOutput() + if err != nil { + return fmt.Errorf( + "failed to remove qdisc on %s (%w): %s", + bindIface, + err, + strings.TrimSpace(string(out)), + ) + } + + return nil +} + +func tcCreateClass(bindIface string, classID, rateMbit uint32) error { + major, minor := netlink.MajorMinor(classID) + + strClassID := fmt.Sprintf("%x:%x", major, minor) + + args := []string{ + "class", + "replace", + "dev", bindIface, + "parent", "1:", + "classid", strClassID, + "htb", + "rate", fmt.Sprintf("%dmbit", rateMbit), + } + + out, err := exec.Command("tc", args...).CombinedOutput() + if err != nil { + return fmt.Errorf( + "failed to create class on %s (class_id = %s) (%w): %s", + bindIface, + strClassID, + err, + strings.TrimSpace(string(out)), + ) + } + + return nil +} + +func tcRemoveClass(bindIface string, classID uint32) error { + major, minor := netlink.MajorMinor(classID) + + strClassID := fmt.Sprintf("%x:%x", major, minor) + + args := []string{ + "class", + "del", + "dev", bindIface, + "classid", strClassID, + } + + if out, err := exec.Command("tc", args...).CombinedOutput(); err != nil { + return fmt.Errorf( + "failed to delete class on %s (class_id = %s) (%s): %s", + bindIface, + strClassID, + err, + strings.TrimSpace(string(out)), + ) + } + + return nil +} + +type AddrDirection uint8 + +const ( + ADDR_DIRECTION_SRC AddrDirection = iota + ADDR_DIRECTION_DST +) + +func (d AddrDirection) String() string { + switch d { + case ADDR_DIRECTION_SRC: + return "src" + case ADDR_DIRECTION_DST: + return "dst" + } + + return "UNKNOWN" +} + +func tcAddFilter(bindIface string, classID uint32, prefix string, direction AddrDirection) error { + ipnet, err := utils.ParseIPNet(prefix) + if err != nil { + return err + } + + var proto, selProto string + + if ipnet.IP.To4() != nil { + proto = "ip" + selProto = "ip" + } else { + proto = "ipv6" + selProto = "ip6" + } + + flowID := fmt.Sprintf("%d:0x%x", int(classID/65536), classID%65536) + + args := []string{ + "filter", + "add", + "dev", bindIface, + "parent", "1:", + "protocol", proto, + "u32", + "match", selProto, direction.String(), ipnet.String(), "flowid", flowID, + } + + if out, err := exec.Command("tc", args...).CombinedOutput(); err != nil { + return fmt.Errorf( + "failed to create filter on %s (addr = %s, class_id = %s) (%s): %s", + bindIface, + ipnet.String(), + flowID, + err, + strings.TrimSpace(string(out)), + ) + } + + return nil +} + +func tcRemoveFilters(bindIface string, classID uint32, prefixes ...string) error { + if classID == 0 { + return nil + } + + bindLink, err := netlink.LinkByName(bindIface) + if err != nil { + return fmt.Errorf("netlink: %s", err) + } + + filters, err := netlink.FilterList(bindLink, 0) + if err != nil { + return fmt.Errorf("filter list failed: %w", err) + } + + removeByPrio := func(prio uint16) error { + strClassID := fmt.Sprintf("%d:%x", int(classID/65536), classID%65536) + + args := []string{ + "filter", + "del", + "dev", bindIface, + "pref", fmt.Sprintf("%d", prio), + } + + if out, err := exec.Command("tc", args...).CombinedOutput(); err != nil { + return fmt.Errorf( + "failed to remove filter on %s (prio = %d, class_id = %s) (%w): %s", + bindIface, + prio, + strClassID, + err, + strings.TrimSpace(string(out)), + ) + } + + return nil + } + + candidates := make(map[uint16]netlink.Filter) + + for _, filter := range filters { + // We are only interested in u32 filters with specified classID ... + if v, ok := filter.(*netlink.U32); ok && v.Sel != nil && v.ClassId == classID { + // ... and IPv4/IPv6 addresses in selectors + if v.Protocol == unix.ETH_P_IP || v.Protocol == unix.ETH_P_IPV6 { + candidates[v.Priority] = filter + } + } + } + + if len(prefixes) > 0 { // remove filters only for the specified prefixes + normalized := make(map[string]struct{}) + + for _, s := range prefixes { + _, ipnet, err := utils.ParseCIDR(s) + if err != nil { + return err + } + + normalized[ipnet.String()] = struct{}{} + } + + for _, filter := range candidates { + u32 := filter.(*netlink.U32) + + info, err := tcParseU32Sel(u32.Protocol, u32.Sel) + if err != nil { + return err + } + + if _, ok := normalized[info.Addr.String()]; ok { + if err := removeByPrio(u32.Priority); err != nil { + return err + } + } + } + } else { // remove all filters with this class ID + for _, filter := range candidates { + u32 := filter.(*netlink.U32) + + if err := removeByPrio(u32.Priority); err != nil { + return err + } + } + } + + return nil +} + +type U32Info struct { + Addr net.IPNet + AddrType AddrDirection +} + +func tcParseU32Sel(protocol uint16, sel *netlink.TcU32Sel) (*U32Info, error) { + if count := len(sel.Keys); !(count == 1 || count == 2) { + return nil, fmt.Errorf("unsupported length of u32 selector keys") + } + + info := U32Info{} + + switch protocol { + case unix.ETH_P_IP: + switch sel.Keys[0].Off { + case 12: + info.AddrType = ADDR_DIRECTION_SRC + case 16: + info.AddrType = ADDR_DIRECTION_DST + default: + return nil, fmt.Errorf("unknown IPv4 key offset: %d", sel.Keys[0].Off) + } + + info.Addr = net.IPNet{ + IP: utils.IntToIPv4(sel.Keys[0].Val), + Mask: net.IPMask(utils.IntToIPv4(sel.Keys[0].Mask)), + } + + return &info, nil + case unix.ETH_P_IPV6: + if len(sel.Keys) < 2 { + return nil, fmt.Errorf("insufficient keys for IPv6 address") + } + + switch sel.Keys[0].Off { + case 8: + info.AddrType = ADDR_DIRECTION_SRC + case 24: + info.AddrType = ADDR_DIRECTION_DST + default: + return nil, fmt.Errorf("unknown IPv6 key offset: %d", sel.Keys[0].Off) + } + + info.Addr = net.IPNet{ + IP: make(net.IP, net.IPv6len), + Mask: make(net.IPMask, net.IPv6len), + } + + // Each key contains 4 bytes, put them to the IP + for idx := 0; idx < 2; idx++ { + val := sel.Keys[idx].Val + + info.Addr.IP[idx*4] = byte(val >> 24) + info.Addr.IP[idx*4+1] = byte(val >> 16) + info.Addr.IP[idx*4+2] = byte(val >> 8) + info.Addr.IP[idx*4+3] = byte(val) + } + + // Do the same with the mask + for idx := 0; idx < 2; idx++ { + val := sel.Keys[idx].Mask + + info.Addr.Mask[idx*4] = byte(val >> 24) + info.Addr.Mask[idx*4+1] = byte(val >> 16) + info.Addr.Mask[idx*4+2] = byte(val >> 8) + info.Addr.Mask[idx*4+3] = byte(val) + } + + return &info, nil + } + + return nil, fmt.Errorf("unsupported protocol of filter") +} diff --git a/internal/hostnet/vxlan_port.go b/internal/hostnet/vxlan_port.go index 461f5ea..4e5fdb2 100644 --- a/internal/hostnet/vxlan_port.go +++ b/internal/hostnet/vxlan_port.go @@ -33,26 +33,26 @@ func VxlanPortConfigure(linkname string, attrs *VxlanPortAttrs, secondStage bool } if err := netlink.LinkSetMaster(vxLink, brLink); err != nil { - return fmt.Errorf("netlink: %s", err) + return fmt.Errorf("netlink: %w", err) } link, err := netlink.LinkByName(linkname) if err != nil { - return fmt.Errorf("netlink: %s", err) + return fmt.Errorf("netlink: %w", err) } if err := netlink.LinkSetMaster(link, brLink); err != nil { - return fmt.Errorf("netlink: %s", err) + return fmt.Errorf("netlink: %w", err) } if err := netlink.LinkSetUp(link); err != nil { - return fmt.Errorf("netlink: %s", err) + return fmt.Errorf("netlink: %w", err) } if attrs.MTU >= 68 { for _, l := range []netlink.Link{brLink, vxLink, link} { if err := netlink.LinkSetMTU(l, int(attrs.MTU)); err != nil { - return fmt.Errorf("netlink: %s: %s", l.Attrs().Name, err) + return fmt.Errorf("netlink: %s: %w", l.Attrs().Name, err) } } } @@ -79,7 +79,7 @@ func VxlanPortDeconfigure(linkname string, vni uint32) error { links, err := netlink.LinkList() if err != nil { - return fmt.Errorf("netlink: %s", err) + return fmt.Errorf("netlink: %w", err) } for _, link := range links { @@ -96,7 +96,7 @@ func VxlanPortDeconfigure(linkname string, vni uint32) error { } if err := netlink.LinkDel(brLink); err != nil { - return fmt.Errorf("netlink: %s", err) + return fmt.Errorf("netlink: %w", err) } return nil @@ -128,14 +128,14 @@ func CreateVxlanIfNotExist(linkname string, vni uint32, srcIP net.IP) (netlink.L } if err := netlink.LinkAdd(link); err != nil { - return nil, fmt.Errorf("netlink: %s", err) + return nil, fmt.Errorf("netlink: %w", err) } default: - return nil, fmt.Errorf("netlink: %s", err) + return nil, fmt.Errorf("netlink: %w", err) } if err := netlink.LinkSetUp(link); err != nil { - return nil, fmt.Errorf("netlink: %s", err) + return nil, fmt.Errorf("netlink: %w", err) } return link, nil diff --git a/internal/utils/network.go b/internal/utils/network.go index 167786a..1fd4cfe 100644 --- a/internal/utils/network.go +++ b/internal/utils/network.go @@ -2,6 +2,7 @@ package utils import ( "bufio" + "encoding/binary" "errors" "fmt" "net" @@ -25,6 +26,18 @@ func ParseIPNet(s string) (*net.IPNet, error) { return netlink.ParseIPNet(s) } +func ParseCIDR(s string) (net.IP, *net.IPNet, error) { + if !strings.Contains(s, "/") { + if net.ParseIP(s).To4() != nil { + s += "/32" + } else { + s += "/128" + } + } + + return net.ParseCIDR(s) +} + func GetRouteTableIndex(table string) (int, error) { var errTableNotFound = errors.New("table not found") @@ -93,3 +106,19 @@ func GetRouteTableIndex(table string) (int, error) { return -1, fmt.Errorf("%w: %s", errTableNotFound, table) } + +func IntToIPv4(n uint32) net.IP { + ip := make(net.IP, net.IPv4len) + + binary.BigEndian.PutUint32(ip, n) + + return ip +} + +func IPv4ToInt(ip net.IP) (uint32, error) { + if ip.To4() == nil { + return 0, fmt.Errorf("not an IPv4 address: %s", ip.String()) + } + + return binary.BigEndian.Uint32(ip.To4()), nil +} diff --git a/kvmrun/kvmrun.go b/kvmrun/kvmrun.go index 00d3966..5308d0c 100644 --- a/kvmrun/kvmrun.go +++ b/kvmrun/kvmrun.go @@ -12,6 +12,7 @@ const ( VMNETINIT = "/usr/lib/kvmrun/netinit" QMPMONDIR = "/var/run/kvm-monitor" + NETWORKDIR = "/var/run/kvm-network" CHROOTDIR = "/var/lib/kvmrun/chroot" KERNELSDIR = "/var/lib/kvmrun/kernels" MODULESDIR = "/var/lib/kvmrun/modules" diff --git a/server/network/configuration.go b/server/network/configuration.go index e8b5c00..7a1f379 100644 --- a/server/network/configuration.go +++ b/server/network/configuration.go @@ -2,13 +2,8 @@ package network import ( "context" - "errors" "fmt" - "io/fs" - "os" - "path/filepath" "slices" - "strconv" "strings" "github.com/0xef53/kvmrun/internal/hostnet" @@ -121,28 +116,51 @@ func (s *Server) UpdateConf(ctx context.Context, vmname, ifname string, apply bo case SchemeUpdate_IN_LIMIT: err = hostnet.RouterSetInboundLimits(ifname, attrs.InLimit) case SchemeUpdate_OUT_LIMIT: - // PID is needed to configure net_cls.classid for use in traffic control rules - pid, _err := func() (uint32, error) { - b, err := os.ReadFile(filepath.Join(kvmrun.CHROOTDIR, vmname, "pid")) - if err != nil { - return 0, err - } - - v, err := strconv.ParseUint(string(b), 10, 32) - if err != nil { - return 0, err + err = hostnet.RouterSetOutboundLimits(ifname, attrs.OutLimit, attrs.BindInterface) + case SchemeUpdate_ADDRS: + err = func() error { + if addrUpdates, ok := update.Value.([]*AddrUpdate); ok { + toAppend, toRemove, err := SplitAddrUpdate(addrUpdates...) + if err != nil { + return fmt.Errorf("cannot parse list of IPs updates: %w", err) + } + + for _, ipnet := range toAppend { + var unmanaged bool + + // Skip QoS configuring for prefixes from unmanaged networks + if s.AppConf.VirtNet.UnmanagedNets.Contains(ipnet.IP) { + unmanaged = true + + l.Infof("Skip QoS configuring for unmanaged %s", ipnet.String()) + } + + err := hostnet.RouterConfigureAddrs(attrs.BindInterface, ifname, unmanaged, ipnet.String()) + if err != nil { + return err + } + } + + for _, ipnet := range toRemove { + var unmanaged bool + + // Skip QoS deconfiguring for prefixes from unmanaged networks + if s.AppConf.VirtNet.UnmanagedNets.Contains(ipnet.IP) { + unmanaged = true + + l.Infof("Skip QoS deconfiguring for unmanaged %s", ipnet.String()) + } + + err := hostnet.RouterDeconfigureAddrs(attrs.BindInterface, ifname, unmanaged, ipnet.String()) + if err != nil { + return err + } + } } - return uint32(v), nil + // Send Gratuitous ARP for all router gateways + return hostnet.RouterAnnounceGateways(attrs.BindInterface, attrs.Addrs, attrs.Gateway4) }() - - if _err != nil { - err = hostnet.RouterSetOutboundLimits(ifname, attrs.OutLimit, attrs.BindInterface, pid) - } else if !errors.Is(_err, fs.ErrNotExist) { - return _err - } - case SchemeUpdate_ADDRS: - err = hostnet.RouterConfigureAddrs(ifname, attrs.Addrs, attrs.Gateway4, attrs.Gateway6) } if err != nil { diff --git a/server/network/hostnet_configure.go b/server/network/hostnet_configure.go index 33c7e5a..f6e1dd7 100644 --- a/server/network/hostnet_configure.go +++ b/server/network/hostnet_configure.go @@ -2,17 +2,12 @@ package network import ( "context" - "errors" "fmt" - "io/fs" "net" - "os" - "path/filepath" - "strconv" "strings" - cg "github.com/0xef53/kvmrun/internal/cgroups" "github.com/0xef53/kvmrun/internal/hostnet" + "github.com/0xef53/kvmrun/internal/utils" "github.com/0xef53/kvmrun/kvmrun" "github.com/0xef53/kvmrun/server" @@ -74,51 +69,31 @@ func (s *Server) ConfigureHostNetwork(ctx context.Context, vmname, ifname string } routerAttrs := hostnet.VirtualRouterAttrs{ - BindInterface: attrs.BindInterface, - MTU: attrs.MTU, - Addrs: attrs.Addrs, - Gateway4: attrs.Gateway4, - Gateway6: attrs.Gateway6, - InLimit: attrs.InLimit, - OutLimit: attrs.OutLimit, - } - - // PID is needed to configure net_cls.classid for use in traffic control rules - if b, err := os.ReadFile(filepath.Join(kvmrun.CHROOTDIR, vmname, "pid")); err == nil { - if v, err := strconv.ParseUint(string(b), 10, 32); err == nil { - routerAttrs.ProcessID = uint32(v) - } else { - return err - } - } else { - if errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("%w: %s", kvmrun.ErrNotRunning, vmname) - } - return err + BindIface: attrs.BindInterface, + MTU: attrs.MTU, + Gateway4: attrs.Gateway4, + Gateway6: attrs.Gateway6, + InLimit: attrs.InLimit, + OutLimit: attrs.OutLimit, } - configureFn = func(secondStage bool) error { - err = hostnet.RouterConfigure(ifname, &routerAttrs, secondStage) - - if err != nil && errors.Is(err, hostnet.ErrCgroupBinding) { - log.WithField("ifname", ifname).Warnf("Non-fatal error: %s", err) - - return nil + for _, addr := range attrs.Addrs { + ipnet, err := utils.ParseIPNet(addr) + if err != nil { + return err } - if cgroups, err := cg.GetProcessGroups(int(routerAttrs.ProcessID)); err == nil { - if g, ok := cgroups["net_cls"]; ok { - fname := filepath.Join(kvmrun.CHROOTDIR, vmname, "run/cgroups.net_cls.path") + if s.AppConf.VirtNet.UnmanagedNets.Contains(ipnet.IP) { + routerAttrs.UnmanagedAddrs = append(routerAttrs.UnmanagedAddrs, addr) - if err := os.WriteFile(fname, []byte(g.Path()), 0644); err != nil { - log.WithField("ifname", ifname).Warnf("Non-fatal error: %s", err) - } - } + l.Infof("Skip QoS configuring for unmanaged %s", ipnet.String()) } else { - log.WithField("ifname", ifname).Warnf("Non-fatal error: %s", err) + routerAttrs.Addrs = append(routerAttrs.Addrs, addr) } + } - return err + configureFn = func(secondStage bool) error { + return hostnet.RouterConfigure(ifname, &routerAttrs, secondStage) } case Scheme_BRIDGE: attrs, err := scheme.ExtractAttrs_Bridge() diff --git a/server/network/hostnet_deconfigure.go b/server/network/hostnet_deconfigure.go index 2325619..44f10a1 100644 --- a/server/network/hostnet_deconfigure.go +++ b/server/network/hostnet_deconfigure.go @@ -3,8 +3,6 @@ package network import ( "context" "fmt" - "os" - "path/filepath" "strings" "github.com/0xef53/kvmrun/internal/hostnet" @@ -62,20 +60,6 @@ func (s *Server) DeconfigureHostNetwork(ctx context.Context, vmname, ifname stri return err } - // Remove net_cls controller for this virt.machine - if b, err := os.ReadFile(filepath.Join(kvmrun.CHROOTDIR, vmname, "run/cgroups.net_cls.path")); err == nil { - dirname := string(b) - - // Just a fast check - if strings.HasSuffix(dirname, fmt.Sprintf("kvmrun@%s.service", vmname)) { - if err := os.RemoveAll(dirname); err == nil { - log.WithField("ifname", ifname).Infof("Removed: %s", dirname) - } else { - log.WithField("ifname", ifname).Warnf("Non-fatal error: %s", err) - } - } - } - return hostnet.RouterDeconfigure(ifname, attrs.BindInterface) case Scheme_BRIDGE: attrs, err := scheme.ExtractAttrs_Bridge() diff --git a/server/network/scheme_properties.go b/server/network/scheme_properties.go index 7c072e9..0df5663 100644 --- a/server/network/scheme_properties.go +++ b/server/network/scheme_properties.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "net" "os" "path/filepath" "reflect" @@ -172,6 +173,7 @@ func (p *SchemeProperties) valueAs(key string, target interface{}) (err error) { err = fmt.Errorf("type mismatch: key = %s, %v", key, r) } }() + targetElem.Set(valueRV.Convert(targetElem.Type())) return nil @@ -444,3 +446,29 @@ type AddrUpdate struct { Action AddrUpdateAction Prefix string } + +// returns two lists: to appand and to remove +func SplitAddrUpdate(updates ...*AddrUpdate) ([]*net.IPNet, []*net.IPNet, error) { + if len(updates) == 0 { + return nil, nil, nil + } + + toAppend := make([]*net.IPNet, 0, 8) + toRemove := make([]*net.IPNet, 0, 8) + + for _, update := range updates { + ipnet, err := utils.ParseIPNet(update.Prefix) + if err != nil { + return nil, nil, fmt.Errorf("invalid IP address: %s", update.Prefix) + } + + switch update.Action { + case AddrUpdate_APPEND: + toAppend = append(toAppend, ipnet) + case AddrUpdate_REMOVE: + toRemove = append(toRemove, ipnet) + } + } + + return toAppend, toRemove, nil +} From 8bfcc47536f325e4d5f9fd6eea4a569a48257954 Mon Sep 17 00:00:00 2001 From: Sergey Zhuravlev Date: Thu, 19 Mar 2026 01:00:30 +0500 Subject: [PATCH 11/13] Reduced handle_id range for TC classes --- internal/hostnet/link.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/hostnet/link.go b/internal/hostnet/link.go index 352e46f..488f0a7 100644 --- a/internal/hostnet/link.go +++ b/internal/hostnet/link.go @@ -46,8 +46,8 @@ func GetLinkID(linkname string, linkindex int) LinkID { bi := big.NewInt(0) bi.SetBytes(h.Sum(nil)) - // Should be a number between 200 and 65000 - x := big.NewInt(0).Mod(bi, big.NewInt(64800)).Int64() + 200 + // Should be a number between 200 (0xc8) and 52000 (0xcb20) + x := big.NewInt(0).Mod(bi, big.NewInt(51800)).Int64() + 200 return LinkID(x) } From d8469fd6111b40444e5c37e40048d09933d54d42 Mon Sep 17 00:00:00 2001 From: Sergey Zhuravlev Date: Wed, 8 Apr 2026 22:22:05 +0500 Subject: [PATCH 12/13] Wrap the main context of the migration task in context.WithoutCancel when RemoveAfter == true --- server/machine/migration.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/machine/migration.go b/server/machine/migration.go index 742e075..4b70c9c 100644 --- a/server/machine/migration.go +++ b/server/machine/migration.go @@ -358,7 +358,7 @@ func (t *MachineMigrationTask) OnSuccess() error { var resp *pb_machines.GetResponse err := t.KvmrunGRPC(t.dstServer, func(client *grpc_interfaces.Kvmrun) (err error) { - resp, err = client.Machines().Get(t.Ctx(), &pb_machines.GetRequest{Name: t.vmname}) + resp, err = client.Machines().Get(context.WithoutCancel(t.Ctx()), &pb_machines.GetRequest{Name: t.vmname}) return err }) From c7dd68e1620fdf005a39e5b9c1f1dbcbf2d7c173 Mon Sep 17 00:00:00 2001 From: Sergey Zhuravlev Date: Thu, 9 Apr 2026 12:26:25 +0500 Subject: [PATCH 13/13] Fix "Run 'systemctl daemon-reload'" error after installing/updating deb-package --- debian/postinst | 4 ++-- debian/postrm | 2 ++ debian/rules | 6 ++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/debian/postinst b/debian/postinst index 0334440..6f94143 100644 --- a/debian/postinst +++ b/debian/postinst @@ -16,8 +16,6 @@ set -e case "$1" in configure) - install -d '/var/lib/supervise' - if [ -f '/usr/share/kvmrun/tls/CA.crt' ]; then echo 'Found an existing CA.crt. Skipping certificates generation.' else @@ -27,6 +25,8 @@ case "$1" in ln -fs '/usr/bin/vmm' '/usr/bin/kvmhelper' invoke-rc.d rsyslog restart + + systemctl --system daemon-reload >/dev/null || true ;; abort-upgrade|abort-remove|abort-deconfigure) diff --git a/debian/postrm b/debian/postrm index f47c8e8..84f570e 100644 --- a/debian/postrm +++ b/debian/postrm @@ -20,6 +20,8 @@ case "$1" in rm -f '/usr/sbin/kvmhelper' invoke-rc.d rsyslog restart + + systemctl --system daemon-reload >/dev/null || true ;; *) diff --git a/debian/rules b/debian/rules index 59085dc..6f3b1c5 100755 --- a/debian/rules +++ b/debian/rules @@ -28,6 +28,12 @@ build: override_dh_installsystemd: dh_installsystemd -p kvmrun --name=kvmrund --no-restart-on-upgrade + +override_dh_systemd_enable: + echo "Not running dh_systemd_enable" + +override_dh_systemd_start: + echo "Not running dh_systemd_start" clean: true