[lxc-devel] [lxd/master] Device Infiniband
tomponline on Github
lxc-bot at linuxcontainers.org
Thu Aug 1 15:49:35 UTC 2019
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 438 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20190801/eb1b23db/attachment-0001.bin>
-------------- next part --------------
From cb3c7cec885d2523cea029f7d607dbfd93b03a5f Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 31 Jul 2019 10:58:11 +0100
Subject: [PATCH 01/42] shared/container: Adds IsUnixUserID and IsOctalFileMode
functions
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
shared/container.go | 26 ++++++++++++++++++++++++++
1 file changed, 26 insertions(+)
diff --git a/shared/container.go b/shared/container.go
index 563897207d..7abf044398 100644
--- a/shared/container.go
+++ b/shared/container.go
@@ -115,6 +115,32 @@ func IsNotEmpty(value string) error {
return nil
}
+func IsUnixUserID(value string) error {
+ if value == "" {
+ return nil
+ }
+
+ _, err := strconv.ParseUint(value, 10, 32)
+ if err != nil {
+ return fmt.Errorf("Invalid value for a UNIX ID")
+ }
+
+ return nil
+}
+
+func IsOctalFileMode(value string) error {
+ if value == "" {
+ return nil
+ }
+
+ _, err := strconv.ParseUint(value, 8, 32)
+ if err != nil {
+ return fmt.Errorf("Invalid value for an octal file mode")
+ }
+
+ return nil
+}
+
// IsRootDiskDevice returns true if the given device representation is
// configured as root disk for a container. It typically get passed a specific
// entry of api.Container.Devices.
From 86bae4535a1817bedfae68282a6e78bb142294a0 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 30 Jul 2019 14:51:56 +0100
Subject: [PATCH 02/42] device/instance/id: Adds DevicesPath() function
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device_instance_id.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/lxd/device/device_instance_id.go b/lxd/device/device_instance_id.go
index 8d7ca4cbbf..38aae3aae9 100644
--- a/lxd/device/device_instance_id.go
+++ b/lxd/device/device_instance_id.go
@@ -11,6 +11,7 @@ type InstanceIdentifier interface {
Name() string
Type() string
Project() string
+ DevicesPath() string
ExpandedConfig() map[string]string
ExpandedDevices() config.Devices
}
From c207948f9e789e65a6a5307448628849083b2c2c Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 30 Jul 2019 17:05:54 +0100
Subject: [PATCH 03/42] device/runconfig: Adds PostStartHooks and simplifies
NetworkInterface
Removes unused fields.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device_runconfig.go | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/lxd/device/device_runconfig.go b/lxd/device/device_runconfig.go
index b3e8eed2b5..831dcef688 100644
--- a/lxd/device/device_runconfig.go
+++ b/lxd/device/device_runconfig.go
@@ -8,7 +8,6 @@ type RunConfigItem struct {
// RunConfig represents LXD defined run-time config used for device setup.
type RunConfig struct {
- NetworkInterfaces [][]RunConfigItem
- Mounts []map[string]string
- Cgroups []map[string]string
+ NetworkInterface []RunConfigItem
+ PostStartHooks []func() error
}
From bd1748965af20e2117e470e8b3ffa4ea5918ee76 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 30 Jul 2019 17:05:25 +0100
Subject: [PATCH 04/42] device/nic: Updates nic devices to use new RunConfig
format
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/nic_bridged.go | 4 ++--
lxd/device/nic_ipvlan.go | 2 +-
lxd/device/nic_macvlan.go | 4 ++--
lxd/device/nic_p2p.go | 4 ++--
lxd/device/nic_physical.go | 4 ++--
lxd/device/nic_sriov.go | 4 ++--
6 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/lxd/device/nic_bridged.go b/lxd/device/nic_bridged.go
index 043801e989..f249f73901 100644
--- a/lxd/device/nic_bridged.go
+++ b/lxd/device/nic_bridged.go
@@ -169,12 +169,12 @@ func (d *nicBridged) Start() (*RunConfig, error) {
}
runConf := RunConfig{}
- runConf.NetworkInterfaces = [][]RunConfigItem{{
+ runConf.NetworkInterface = []RunConfigItem{
{Key: "name", Value: d.config["name"]},
{Key: "type", Value: "phys"},
{Key: "flags", Value: "up"},
{Key: "link", Value: peerName},
- }}
+ }
return &runConf, nil
}
diff --git a/lxd/device/nic_ipvlan.go b/lxd/device/nic_ipvlan.go
index 7a2dc0dd57..c380d549d3 100644
--- a/lxd/device/nic_ipvlan.go
+++ b/lxd/device/nic_ipvlan.go
@@ -173,7 +173,7 @@ func (d *nicIPVLAN) Start() (*RunConfig, error) {
nic = append(nic, RunConfigItem{Key: "ipv6.gateway", Value: "dev"})
}
- runConf.NetworkInterfaces = append(runConf.NetworkInterfaces, nic)
+ runConf.NetworkInterface = nic
return &runConf, nil
}
diff --git a/lxd/device/nic_macvlan.go b/lxd/device/nic_macvlan.go
index 84cec51bd1..1aabbeb660 100644
--- a/lxd/device/nic_macvlan.go
+++ b/lxd/device/nic_macvlan.go
@@ -99,12 +99,12 @@ func (d *nicMACVLAN) Start() (*RunConfig, error) {
}
runConf := RunConfig{}
- runConf.NetworkInterfaces = [][]RunConfigItem{{
+ runConf.NetworkInterface = []RunConfigItem{
{Key: "name", Value: d.config["name"]},
{Key: "type", Value: "phys"},
{Key: "flags", Value: "up"},
{Key: "link", Value: saveData["host_name"]},
- }}
+ }
return &runConf, nil
}
diff --git a/lxd/device/nic_p2p.go b/lxd/device/nic_p2p.go
index 2cc7960476..ea6909c816 100644
--- a/lxd/device/nic_p2p.go
+++ b/lxd/device/nic_p2p.go
@@ -84,12 +84,12 @@ func (d *nicP2P) Start() (*RunConfig, error) {
}
runConf := RunConfig{}
- runConf.NetworkInterfaces = [][]RunConfigItem{{
+ runConf.NetworkInterface = []RunConfigItem{
{Key: "name", Value: d.config["name"]},
{Key: "type", Value: "phys"},
{Key: "flags", Value: "up"},
{Key: "link", Value: peerName},
- }}
+ }
return &runConf, nil
}
diff --git a/lxd/device/nic_physical.go b/lxd/device/nic_physical.go
index ebb0354a7a..5faa6a2ec4 100644
--- a/lxd/device/nic_physical.go
+++ b/lxd/device/nic_physical.go
@@ -105,12 +105,12 @@ func (d *nicPhysical) Start() (*RunConfig, error) {
}
runConf := RunConfig{}
- runConf.NetworkInterfaces = [][]RunConfigItem{{
+ runConf.NetworkInterface = []RunConfigItem{
{Key: "name", Value: d.config["name"]},
{Key: "type", Value: "phys"},
{Key: "flags", Value: "up"},
{Key: "link", Value: saveData["host_name"]},
- }}
+ }
return &runConf, nil
}
diff --git a/lxd/device/nic_sriov.go b/lxd/device/nic_sriov.go
index 5eeed41fec..ebad700af6 100644
--- a/lxd/device/nic_sriov.go
+++ b/lxd/device/nic_sriov.go
@@ -131,12 +131,12 @@ func (d *nicSRIOV) Start() (*RunConfig, error) {
}
runConf := RunConfig{}
- runConf.NetworkInterfaces = [][]RunConfigItem{{
+ runConf.NetworkInterface = []RunConfigItem{
{Key: "name", Value: d.config["name"]},
{Key: "type", Value: "phys"},
{Key: "flags", Value: "up"},
{Key: "link", Value: saveData["host_name"]},
- }}
+ }
return &runConf, nil
}
From 66c3d28804f5fa2bdd6f39c6f78cf1b85cd96257 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 30 Jul 2019 14:52:19 +0100
Subject: [PATCH 05/42] container: Adds DevicesPath() function to interface
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/container.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/lxd/container.go b/lxd/container.go
index e892d4fda0..0c9c1b24f6 100644
--- a/lxd/container.go
+++ b/lxd/container.go
@@ -629,6 +629,7 @@ type container interface {
LogFilePath() string
ConsoleBufferLogPath() string
LogPath() string
+ DevicesPath() string
// Storage
StoragePool() (string, error)
From 1f71fbe1991cf7a129ae7cd166b095598467cc8c Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 29 Jul 2019 12:20:16 +0100
Subject: [PATCH 06/42] device/utils/instance: Adds access to
InstanceLoadByProjectAndName function
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device_utils_instance.go | 3 +++
1 file changed, 3 insertions(+)
diff --git a/lxd/device/device_utils_instance.go b/lxd/device/device_utils_instance.go
index 52226afb4f..f96c217564 100644
--- a/lxd/device/device_utils_instance.go
+++ b/lxd/device/device_utils_instance.go
@@ -6,3 +6,6 @@ import (
// InstanceLoadNodeAll returns all local instance configs.
var InstanceLoadNodeAll func(s *state.State) ([]InstanceIdentifier, error)
+
+// InstanceLoadByProjectAndName returns instance config by project and name.
+var InstanceLoadByProjectAndName func(s *state.State, project, name string) (InstanceIdentifier, error)
From af163f1823b4e4bf4accba0d73d06e93f3785947 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 29 Jul 2019 12:21:00 +0100
Subject: [PATCH 07/42] container: Adds access to containerLoadByProjectAndName
function from device package
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/container.go | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/lxd/container.go b/lxd/container.go
index 0c9c1b24f6..8d929eaac2 100644
--- a/lxd/container.go
+++ b/lxd/container.go
@@ -49,6 +49,17 @@ func init() {
return identifiers, nil
}
+
+ // Expose containerLoadByProjectAndName to the device package converting the response to an InstanceIdentifier.
+ // This is because container types are defined in the main package and are not importable.
+ device.InstanceLoadByProjectAndName = func(s *state.State, project, name string) (device.InstanceIdentifier, error) {
+ container, err := containerLoadByProjectAndName(s, project, name)
+ if err != nil {
+ return nil, err
+ }
+
+ return device.InstanceIdentifier(container), nil
+ }
}
// Helper functions
From c46539f63f257f81d663434335e13296f91be475 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 29 Jul 2019 12:19:29 +0100
Subject: [PATCH 08/42] device: Adds device name to device structure
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device.go | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
diff --git a/lxd/device/device.go b/lxd/device/device.go
index 7290051334..7bd4199dbb 100644
--- a/lxd/device/device.go
+++ b/lxd/device/device.go
@@ -60,7 +60,7 @@ type device interface {
Device
// init stores the InstanceIdentifier, daemon State and Config into device and performs any setup.
- init(InstanceIdentifier, *state.State, config.Device, VolatileGetter, VolatileSetter)
+ init(InstanceIdentifier, *state.State, string, config.Device, VolatileGetter, VolatileSetter)
// validateConfig checks Config stored by init() is valid for the instance type.
validateConfig() error
@@ -69,18 +69,20 @@ type device interface {
// deviceCommon represents the common struct for all devices.
type deviceCommon struct {
instance InstanceIdentifier
+ name string
config map[string]string
state *state.State
volatileGet func() map[string]string
volatileSet func(map[string]string) error
}
-// init stores the InstanceIdentifier, daemon state and Config into device. It can also be provided
-// with volatile get and set functions for the device to allow persistent data to be accessed.
-// This is implemented as part of deviceCommon so that the majority of devices don't need to
-// implement it and can just embed deviceCommon.
-func (d *deviceCommon) init(instance InstanceIdentifier, state *state.State, conf config.Device, volatileGet VolatileGetter, volatileSet VolatileSetter) {
+// init stores the InstanceIdentifier, daemon state, device name and config into device.
+// It also needs to be provided with volatile get and set functions for the device to allow
+// persistent data to be accessed. This is implemented as part of deviceCommon so that the majority
+// of devices don't need to implement it and can just embed deviceCommon.
+func (d *deviceCommon) init(instance InstanceIdentifier, state *state.State, name string, conf config.Device, volatileGet VolatileGetter, volatileSet VolatileSetter) {
d.instance = instance
+ d.name = name
d.config = conf
d.state = state
d.volatileGet = volatileGet
@@ -109,7 +111,7 @@ func (d *deviceCommon) Remove() error {
}
// New instantiates a new device struct, validates the supplied config and sets it into the device.
-func New(instance InstanceIdentifier, state *state.State, conf config.Device, volatileGet VolatileGetter, volatileSet VolatileSetter) (Device, error) {
+func New(instance InstanceIdentifier, state *state.State, name string, conf config.Device, volatileGet VolatileGetter, volatileSet VolatileSetter) (Device, error) {
devFunc := devTypes[conf["type"]]
// Check if top-level type is recognised, if it is known type it will return a function.
@@ -124,7 +126,7 @@ func New(instance InstanceIdentifier, state *state.State, conf config.Device, vo
}
// Init the device and run validation of supplied config.
- dev.init(instance, state, conf, volatileGet, volatileSet)
+ dev.init(instance, state, name, conf, volatileGet, volatileSet)
err := dev.validateConfig()
if err != nil {
return nil, err
From 6df7baac1f4126ebbd638ff1e1fbcb25dd4cc781 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 29 Jul 2019 12:21:42 +0100
Subject: [PATCH 09/42] container: Updates use of device.New() with device name
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/container.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lxd/container.go b/lxd/container.go
index 8d929eaac2..02cf62c655 100644
--- a/lxd/container.go
+++ b/lxd/container.go
@@ -355,7 +355,7 @@ func containerValidDevices(state *state.State, cluster *db.Cluster, devices conf
if m["type"] == "nic" {
// Validate config using device interface.
- _, err := device.New(&containerLXC{}, state, config.Device(m), nil, nil)
+ _, err := device.New(&containerLXC{}, state, name, config.Device(m), nil, nil)
if err != nil {
return err
}
From 752bd1bedcd19cf7c8fd8c511463374fddd5014d Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 29 Jul 2019 12:22:30 +0100
Subject: [PATCH 10/42] container/lxc: Updates use of device.New with device
name
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/container_lxc.go | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index 309a2b9600..e74bd48eaa 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -1845,7 +1845,7 @@ func (c *containerLXC) deviceLoad(deviceName string, rawConfig map[string]string
}
}
- d, err := device.New(c, c.state, configCopy, c.deviceVolatileGetFunc(deviceName), c.deviceVolatileSetFunc(deviceName))
+ d, err := device.New(c, c.state, deviceName, configCopy, c.deviceVolatileGetFunc(deviceName), c.deviceVolatileSetFunc(deviceName))
if err != nil {
return nil, nil, err
}
@@ -4383,7 +4383,7 @@ func (c *containerLXC) Update(args db.ContainerArgs, userRequested bool) error {
return []string{} // Device types aren't the same, so this cannot be an update.
}
- d, err := device.New(c, c.state, config.Device(newDevice), nil, nil)
+ d, err := device.New(c, c.state, "", config.Device(newDevice), nil, nil)
if err != device.ErrUnsupportedDevType {
if err != nil {
return []string{} // Couldn't create Device, so this cannot be an update.
From 0585c58787845fc427ac0337d4af6ba12855aea6 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 29 Jul 2019 12:24:16 +0100
Subject: [PATCH 11/42] network/utils: Removes networkValidAddress
Moved to device_utils_network.go
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/networks_utils.go | 13 -------------
1 file changed, 13 deletions(-)
diff --git a/lxd/networks_utils.go b/lxd/networks_utils.go
index 085ac97464..299d65c8cd 100644
--- a/lxd/networks_utils.go
+++ b/lxd/networks_utils.go
@@ -496,19 +496,6 @@ func networkValidAddressCIDRV4(value string) error {
return nil
}
-func networkValidAddress(value string) error {
- if value == "" {
- return nil
- }
-
- ip := net.ParseIP(value)
- if ip == nil {
- return fmt.Errorf("Not an IP address: %s", value)
- }
-
- return nil
-}
-
func networkAddressForSubnet(subnet *net.IPNet) (net.IP, string, error) {
ifaces, err := net.Interfaces()
if err != nil {
From 3261b87e2cc01215ce9ca6b0718b4382f0803e66 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 29 Jul 2019 12:25:52 +0100
Subject: [PATCH 12/42] device/utils/network: Moves proxy related network
functions into device package
Updates comments on some address related functions
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device_utils_network.go | 48 +++++++++++++++++++++++++++---
1 file changed, 44 insertions(+), 4 deletions(-)
diff --git a/lxd/device/device_utils_network.go b/lxd/device/device_utils_network.go
index 7c18ac9297..b6a2b5f928 100644
--- a/lxd/device/device_utils_network.go
+++ b/lxd/device/device_utils_network.go
@@ -501,7 +501,21 @@ func networkSetVethLimits(m config.Device) error {
return nil
}
-// NetworkValidAddressV4 validates an IPv4 addresss string.
+// NetworkValidAddress validates an IP address string. If string is empty, returns valid.
+func NetworkValidAddress(value string) error {
+ if value == "" {
+ return nil
+ }
+
+ ip := net.ParseIP(value)
+ if ip == nil {
+ return fmt.Errorf("Not an IP address: %s", value)
+ }
+
+ return nil
+}
+
+// NetworkValidAddressV4 validates an IPv4 addresss string. If string is empty, returns valid.
func NetworkValidAddressV4(value string) error {
if value == "" {
return nil
@@ -515,7 +529,7 @@ func NetworkValidAddressV4(value string) error {
return nil
}
-// NetworkValidAddressV6 validates an IPv6 addresss string.
+// NetworkValidAddressV6 validates an IPv6 addresss string. If string is empty, returns valid.
func NetworkValidAddressV6(value string) error {
if value == "" {
return nil
@@ -553,7 +567,7 @@ func NetworkValidAddressV6List(value string) error {
return nil
}
-// NetworkValidNetworkV4 validates an IPv4 CIDR string.
+// NetworkValidNetworkV4 validates an IPv4 CIDR string. If string is empty, returns valid.
func NetworkValidNetworkV4(value string) error {
if value == "" {
return nil
@@ -575,7 +589,7 @@ func NetworkValidNetworkV4(value string) error {
return nil
}
-// NetworkValidNetworkV6 validates an IPv6 CIDR string.
+// NetworkValidNetworkV6 validates an IPv6 CIDR string. If string is empty, returns valid.
func NetworkValidNetworkV6(value string) error {
if value == "" {
return nil
@@ -660,3 +674,29 @@ func NetworkSRIOVGetFreeVFInterface(reservedDevices map[string]struct{}, vfListP
return "", nil
}
+
+// networkParsePortRange validates a port range in the form n-n.
+func networkParsePortRange(r string) (int64, int64, error) {
+ entries := strings.Split(r, "-")
+ if len(entries) > 2 {
+ return -1, -1, fmt.Errorf("Invalid port range %s", r)
+ }
+
+ base, err := strconv.ParseInt(entries[0], 10, 64)
+ if err != nil {
+ return -1, -1, err
+ }
+
+ size := int64(1)
+ if len(entries) > 1 {
+ size, err = strconv.ParseInt(entries[1], 10, 64)
+ if err != nil {
+ return -1, -1, err
+ }
+
+ size -= base
+ size++
+ }
+
+ return base, size, nil
+}
From fbe5699fab15bacc20c154a8f09158a26c1236f0 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 29 Jul 2019 12:26:59 +0100
Subject: [PATCH 13/42] networks/config: Updates references to
NetworkValidAddress
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/networks_config.go | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lxd/networks_config.go b/lxd/networks_config.go
index bdf416498c..0babb960e9 100644
--- a/lxd/networks_config.go
+++ b/lxd/networks_config.go
@@ -48,10 +48,10 @@ var networkConfigKeys = map[string]func(value string) error{
"tunnel.TARGET.protocol": func(value string) error {
return shared.IsOneOf(value, []string{"gre", "vxlan"})
},
- "tunnel.TARGET.local": networkValidAddress,
- "tunnel.TARGET.remote": networkValidAddress,
+ "tunnel.TARGET.local": device.NetworkValidAddress,
+ "tunnel.TARGET.remote": device.NetworkValidAddress,
"tunnel.TARGET.port": networkValidPort,
- "tunnel.TARGET.group": networkValidAddress,
+ "tunnel.TARGET.group": device.NetworkValidAddress,
"tunnel.TARGET.id": shared.IsInt64,
"tunnel.TARGET.interface": networkValidName,
"tunnel.TARGET.ttl": shared.IsUint8,
From f09fdb192e37fb322c2bba9d7789ae1dc74b9ffa Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 29 Jul 2019 12:27:35 +0100
Subject: [PATCH 14/42] device/utils/proxy: Adds proxy specific shared
functions
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device_utils_proxy.go | 73 ++++++++++++++++++++++++++++++++
1 file changed, 73 insertions(+)
create mode 100644 lxd/device/device_utils_proxy.go
diff --git a/lxd/device/device_utils_proxy.go b/lxd/device/device_utils_proxy.go
new file mode 100644
index 0000000000..691e4cbe53
--- /dev/null
+++ b/lxd/device/device_utils_proxy.go
@@ -0,0 +1,73 @@
+package device
+
+import (
+ "fmt"
+ "net"
+ "strings"
+
+ "github.com/lxc/lxd/shared"
+)
+
+// ProxyAddress represents a proxy address configuration.
+type ProxyAddress struct {
+ ConnType string
+ Addr []string
+ Abstract bool
+}
+
+// ProxyParseAddr validates a proxy address and parses it into its constituent parts.
+func ProxyParseAddr(addr string) (*ProxyAddress, error) {
+ // Split into <protocol> and <address>.
+ fields := strings.SplitN(addr, ":", 2)
+
+ if !shared.StringInSlice(fields[0], []string{"tcp", "udp", "unix"}) {
+ return nil, fmt.Errorf("Unknown connection type '%s'", fields[0])
+ }
+
+ newProxyAddr := &ProxyAddress{
+ ConnType: fields[0],
+ Abstract: strings.HasPrefix(fields[1], "@"),
+ }
+
+ // unix addresses cannot have ports.
+ if newProxyAddr.ConnType == "unix" {
+ newProxyAddr.Addr = []string{fields[1]}
+ return newProxyAddr, nil
+ }
+
+ // Split <address> into <address> and <ports>.
+ address, port, err := net.SplitHostPort(fields[1])
+ if err != nil {
+ return nil, err
+ }
+
+ // Validate that it's a valid address.
+ if shared.StringInSlice(newProxyAddr.ConnType, []string{"udp", "tcp"}) {
+ err := NetworkValidAddress(address)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ // Split <ports> into individual ports and port ranges.
+ ports := strings.SplitN(port, ",", -1)
+ for _, p := range ports {
+ portFirst, portRange, err := networkParsePortRange(p)
+ if err != nil {
+ return nil, err
+ }
+
+ for i := int64(0); i < portRange; i++ {
+ var newAddr string
+ if strings.Contains(address, ":") {
+ // IPv6 addresses need to be enclosed in square brackets.
+ newAddr = fmt.Sprintf("[%s]:%d", address, portFirst+i)
+ } else {
+ newAddr = fmt.Sprintf("%s:%d", address, portFirst+i)
+ }
+ newProxyAddr.Addr = append(newProxyAddr.Addr, newAddr)
+ }
+ }
+
+ return newProxyAddr, nil
+}
From df4256cbb9ec0b4ac1b74a086b5db5c9989dcaba Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 29 Jul 2019 12:28:11 +0100
Subject: [PATCH 15/42] main/forkproxy: Updates references to shared types in
device package
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/main_forkproxy.go | 158 ++++++++++--------------------------------
1 file changed, 36 insertions(+), 122 deletions(-)
diff --git a/lxd/main_forkproxy.go b/lxd/main_forkproxy.go
index e08e4d41ae..3ead12b1b7 100644
--- a/lxd/main_forkproxy.go
+++ b/lxd/main_forkproxy.go
@@ -15,6 +15,7 @@ import (
"github.com/spf13/cobra"
"golang.org/x/sys/unix"
+ "github.com/lxc/lxd/lxd/device"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/netutils"
)
@@ -304,12 +305,6 @@ type cmdForkproxy struct {
global *cmdGlobal
}
-type proxyAddress struct {
- connType string
- addr []string
- abstract bool
-}
-
// UDP session tracking (map "client tuple" to udp session)
var udpSessions = map[string]*udpSession{}
var udpSessionsLock sync.Mutex
@@ -349,16 +344,16 @@ func rearmUDPFd(epFd C.int, connFd C.int) {
}
}
-func listenerInstance(epFd C.int, lAddr *proxyAddress, cAddr *proxyAddress, connFd C.int, lStruct *lStruct, proxy bool) error {
- if lAddr.connType == "udp" {
+func listenerInstance(epFd C.int, lAddr *device.ProxyAddress, cAddr *device.ProxyAddress, connFd C.int, lStruct *lStruct, proxy bool) error {
+ if lAddr.ConnType == "udp" {
// This only handles udp <-> udp. The C constructor will have
// verified this before.
go func() {
// single or multiple port -> single port
- connectAddr := cAddr.addr[0]
- if len(cAddr.addr) > 1 {
+ connectAddr := cAddr.Addr[0]
+ if len(cAddr.Addr) > 1 {
// multiple port -> multiple port
- connectAddr = cAddr.addr[(*lStruct).lAddrIndex]
+ connectAddr = cAddr.Addr[(*lStruct).lAddrIndex]
}
srcConn, err := net.FileConn((*lStruct).f)
@@ -368,7 +363,7 @@ func listenerInstance(epFd C.int, lAddr *proxyAddress, cAddr *proxyAddress, conn
return
}
- dstConn, err := net.Dial(cAddr.connType, connectAddr)
+ dstConn, err := net.Dial(cAddr.ConnType, connectAddr)
if err != nil {
fmt.Printf("Error: Failed to connect to target: %v\n", err)
rearmUDPFd(epFd, connFd)
@@ -391,20 +386,20 @@ func listenerInstance(epFd C.int, lAddr *proxyAddress, cAddr *proxyAddress, conn
}
// single or multiple port -> single port
- connectAddr := cAddr.addr[0]
- if lAddr.connType != "unix" && cAddr.connType != "unix" && len(cAddr.addr) > 1 {
+ connectAddr := cAddr.Addr[0]
+ if lAddr.ConnType != "unix" && cAddr.ConnType != "unix" && len(cAddr.Addr) > 1 {
// multiple port -> multiple port
- connectAddr = cAddr.addr[(*lStruct).lAddrIndex]
+ connectAddr = cAddr.Addr[(*lStruct).lAddrIndex]
}
- dstConn, err := net.Dial(cAddr.connType, connectAddr)
+ dstConn, err := net.Dial(cAddr.ConnType, connectAddr)
if err != nil {
srcConn.Close()
fmt.Printf("Error: Failed to connect to target: %v\n", err)
return err
}
- if proxy && cAddr.connType == "tcp" {
- if lAddr.connType == "unix" {
+ if proxy && cAddr.ConnType == "tcp" {
+ if lAddr.ConnType == "unix" {
dstConn.Write([]byte(fmt.Sprintf("PROXY UNKNOWN\r\n")))
} else {
cHost, cPort, err := net.SplitHostPort(srcConn.RemoteAddr().String())
@@ -429,7 +424,7 @@ func listenerInstance(epFd C.int, lAddr *proxyAddress, cAddr *proxyAddress, conn
}
}
- if cAddr.connType == "unix" && lAddr.connType == "unix" {
+ if cAddr.ConnType == "unix" && lAddr.ConnType == "unix" {
// Handle OOB if both src and dst are using unix sockets
go unixRelay(srcConn, dstConn)
} else {
@@ -470,23 +465,23 @@ func (c *cmdForkproxy) Run(cmd *cobra.Command, args []string) error {
}
listenAddr := args[1]
- lAddr, err := proxyParseAddr(listenAddr)
+ lAddr, err := device.ProxyParseAddr(listenAddr)
if err != nil {
return err
}
connectAddr := args[3]
- cAddr, err := proxyParseAddr(connectAddr)
+ cAddr, err := device.ProxyParseAddr(connectAddr)
if err != nil {
return err
}
- if (lAddr.connType == "udp" || lAddr.connType == "tcp") && cAddr.connType == "udp" || cAddr.connType == "tcp" {
+ if (lAddr.ConnType == "udp" || lAddr.ConnType == "tcp") && cAddr.ConnType == "udp" || cAddr.ConnType == "tcp" {
err := fmt.Errorf("Invalid port range")
- if len(lAddr.addr) > 1 && len(cAddr.addr) > 1 && (len(cAddr.addr) != len(lAddr.addr)) {
+ if len(lAddr.Addr) > 1 && len(cAddr.Addr) > 1 && (len(cAddr.Addr) != len(lAddr.Addr)) {
fmt.Println(err)
return err
- } else if len(lAddr.addr) == 1 && len(cAddr.addr) > 1 {
+ } else if len(lAddr.Addr) == 1 && len(cAddr.Addr) > 1 {
fmt.Println(err)
return err
}
@@ -495,15 +490,15 @@ func (c *cmdForkproxy) Run(cmd *cobra.Command, args []string) error {
if C.whoami == C.FORKPROXY_CHILD {
defer unix.Close(forkproxyUDSSockFDNum)
- if lAddr.connType == "unix" && !lAddr.abstract {
- err := os.Remove(lAddr.addr[0])
+ if lAddr.ConnType == "unix" && !lAddr.Abstract {
+ err := os.Remove(lAddr.Addr[0])
if err != nil && !os.IsNotExist(err) {
return err
}
}
- for _, addr := range lAddr.addr {
- file, err := getListenerFile(lAddr.connType, addr)
+ for _, addr := range lAddr.Addr {
+ file, err := getListenerFile(lAddr.ConnType, addr)
if err != nil {
return err
}
@@ -520,27 +515,27 @@ func (c *cmdForkproxy) Run(cmd *cobra.Command, args []string) error {
file.Close()
}
- if lAddr.connType == "unix" && !lAddr.abstract {
+ if lAddr.ConnType == "unix" && !lAddr.Abstract {
var err error
- listenAddrGid := -1
+ listenAddrGID := -1
if args[6] != "" {
- listenAddrGid, err = strconv.Atoi(args[6])
+ listenAddrGID, err = strconv.Atoi(args[6])
if err != nil {
return err
}
}
- listenAddrUid := -1
+ listenAddrUID := -1
if args[7] != "" {
- listenAddrUid, err = strconv.Atoi(args[7])
+ listenAddrUID, err = strconv.Atoi(args[7])
if err != nil {
return err
}
}
- if listenAddrGid != -1 || listenAddrUid != -1 {
- err = os.Chown(lAddr.addr[0], listenAddrUid, listenAddrGid)
+ if listenAddrGID != -1 || listenAddrUID != -1 {
+ err = os.Chown(lAddr.Addr[0], listenAddrUID, listenAddrGID)
if err != nil {
return err
}
@@ -554,7 +549,7 @@ func (c *cmdForkproxy) Run(cmd *cobra.Command, args []string) error {
}
listenAddrMode = os.FileMode(tmp)
- err = os.Chmod(lAddr.addr[0], listenAddrMode)
+ err = os.Chmod(lAddr.Addr[0], listenAddrMode)
if err != nil {
return err
}
@@ -565,7 +560,7 @@ func (c *cmdForkproxy) Run(cmd *cobra.Command, args []string) error {
}
files := []*os.File{}
- for range lAddr.addr {
+ for range lAddr.Addr {
rAgain:
f, err := netutils.AbstractUnixReceiveFd(forkproxyUDSSockFDNum)
if err != nil {
@@ -584,8 +579,8 @@ func (c *cmdForkproxy) Run(cmd *cobra.Command, args []string) error {
var listenerMap map[int]*lStruct
- isUDPListener := lAddr.connType == "udp"
- listenerMap = make(map[int]*lStruct, len(lAddr.addr))
+ isUDPListener := lAddr.ConnType == "udp"
+ listenerMap = make(map[int]*lStruct, len(lAddr.Addr))
if isUDPListener {
for i, f := range files {
listenerMap[int(f.Fd())] = &lStruct{
@@ -635,8 +630,8 @@ func (c *cmdForkproxy) Run(cmd *cobra.Command, args []string) error {
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, unix.SIGTERM)
- if lAddr.connType == "unix" && !lAddr.abstract {
- defer os.Remove(lAddr.addr[0])
+ if lAddr.ConnType == "unix" && !lAddr.Abstract {
+ defer os.Remove(lAddr.Addr[0])
}
epFd := C.epoll_create1(C.EPOLL_CLOEXEC)
@@ -1034,84 +1029,3 @@ func getListenerFile(protocol string, addr string) (*os.File, error) {
return file, nil
}
-
-func parsePortRange(r string) (int64, int64, error) {
- entries := strings.Split(r, "-")
- if len(entries) > 2 {
- return -1, -1, fmt.Errorf("Invalid port range %s", r)
- }
-
- base, err := strconv.ParseInt(entries[0], 10, 64)
- if err != nil {
- return -1, -1, err
- }
-
- size := int64(1)
- if len(entries) > 1 {
- size, err = strconv.ParseInt(entries[1], 10, 64)
- if err != nil {
- return -1, -1, err
- }
-
- size -= base
- size += 1
- }
-
- return base, size, nil
-}
-
-func proxyParseAddr(addr string) (*proxyAddress, error) {
- // Split into <protocol> and <address>
- fields := strings.SplitN(addr, ":", 2)
-
- if !shared.StringInSlice(fields[0], []string{"tcp", "udp", "unix"}) {
- return nil, fmt.Errorf("Unknown connection type '%s'", fields[0])
- }
-
- newProxyAddr := &proxyAddress{
- connType: fields[0],
- abstract: strings.HasPrefix(fields[1], "@"),
- }
-
- // unix addresses cannot have ports
- if newProxyAddr.connType == "unix" {
- newProxyAddr.addr = []string{fields[1]}
- return newProxyAddr, nil
- }
-
- // Split <address> into <address> and <ports>
- address, port, err := net.SplitHostPort(fields[1])
- if err != nil {
- return nil, err
- }
-
- // Validate that it's a valid address
- if shared.StringInSlice(newProxyAddr.connType, []string{"udp", "tcp"}) {
- err := networkValidAddress(address)
- if err != nil {
- return nil, err
- }
- }
-
- // Split <ports> into individual ports and port ranges
- ports := strings.SplitN(port, ",", -1)
- for _, p := range ports {
- portFirst, portRange, err := parsePortRange(p)
- if err != nil {
- return nil, err
- }
-
- for i := int64(0); i < portRange; i++ {
- var newAddr string
- if strings.Contains(address, ":") {
- // IPv6 addresses need to be enclosed in square brackets
- newAddr = fmt.Sprintf("[%s]:%d", address, portFirst+i)
- } else {
- newAddr = fmt.Sprintf("%s:%d", address, portFirst+i)
- }
- newProxyAddr.addr = append(newProxyAddr.addr, newAddr)
- }
- }
-
- return newProxyAddr, nil
-}
From 8c0c8de98fc663cf601c0745eeac191df4f79ef1 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 29 Jul 2019 12:29:41 +0100
Subject: [PATCH 16/42] container: Updates references to proxy functions and
vars moved to device package
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/container.go | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/lxd/container.go b/lxd/container.go
index 02cf62c655..0f50837e38 100644
--- a/lxd/container.go
+++ b/lxd/container.go
@@ -483,17 +483,17 @@ func containerValidDevices(state *state.State, cluster *db.Cluster, devices conf
return fmt.Errorf("Proxy device entry is missing the required \"connect\" property")
}
- listenAddr, err := proxyParseAddr(m["listen"])
+ listenAddr, err := device.ProxyParseAddr(m["listen"])
if err != nil {
return err
}
- connectAddr, err := proxyParseAddr(m["connect"])
+ connectAddr, err := device.ProxyParseAddr(m["connect"])
if err != nil {
return err
}
- if len(connectAddr.addr) > len(listenAddr.addr) {
+ if len(connectAddr.Addr) > len(listenAddr.Addr) {
// Cannot support single port -> multiple port
return fmt.Errorf("Cannot map a single port to multiple ports")
}
@@ -513,10 +513,10 @@ func containerValidDevices(state *state.State, cluster *db.Cluster, devices conf
}
// Support TCP <-> TCP and UDP <-> UDP
- if listenAddr.connType == "unix" || connectAddr.connType == "unix" ||
- listenAddr.connType != connectAddr.connType {
+ if listenAddr.ConnType == "unix" || connectAddr.ConnType == "unix" ||
+ listenAddr.ConnType != connectAddr.ConnType {
return fmt.Errorf("Proxying %s <-> %s is not supported when using NAT",
- listenAddr.connType, connectAddr.connType)
+ listenAddr.ConnType, connectAddr.ConnType)
}
}
From d9409d2987610ff16ebb07c5895d507406eda869 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 29 Jul 2019 14:24:59 +0100
Subject: [PATCH 17/42] proxy/device/utils: Removed unused code
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/proxy_device_utils.go | 80 ---------------------------------------
1 file changed, 80 deletions(-)
diff --git a/lxd/proxy_device_utils.go b/lxd/proxy_device_utils.go
index f8d9508b96..a96e7a0ab6 100644
--- a/lxd/proxy_device_utils.go
+++ b/lxd/proxy_device_utils.go
@@ -13,86 +13,6 @@ import (
"github.com/lxc/lxd/shared"
)
-type proxyProcInfo struct {
- listenPid string
- connectPid string
- connectAddr string
- listenAddr string
- listenAddrGid string
- listenAddrUid string
- listenAddrMode string
- securityUid string
- securityGid string
- proxyProtocol string
-}
-
-func parseAddr(addr string) (string, string) {
- fields := strings.SplitN(addr, ":", 2)
- return fields[0], fields[1]
-}
-
-func rewriteHostAddr(addr string) string {
- proto, addr := parseAddr(addr)
- if proto == "unix" && !strings.HasPrefix(addr, "@") {
- // Unix non-abstract sockets need to be addressed to the host
- // filesystem, not be scoped inside the LXD snap.
- addr = shared.HostPath(addr)
- }
- return fmt.Sprintf("%s:%s", proto, addr)
-}
-
-func setupProxyProcInfo(c container, device map[string]string) (*proxyProcInfo, error) {
- pid := c.InitPID()
- containerPid := strconv.Itoa(int(pid))
- lxdPid := strconv.Itoa(os.Getpid())
-
- connectAddr := device["connect"]
- listenAddr := device["listen"]
-
- if proto, _ := parseAddr(connectAddr); !shared.StringInSlice(proto, []string{"tcp", "udp", "unix"}) {
- return nil, fmt.Errorf("Proxy device doesn't support the connection type: %s", proto)
- }
- if proto, _ := parseAddr(listenAddr); !shared.StringInSlice(proto, []string{"tcp", "udp", "unix"}) {
- return nil, fmt.Errorf("Proxy device doesn't support the listener type: %s", proto)
- }
-
- var listenPid string
- var connectPid string
-
- bindVal, exists := device["bind"]
- if !exists {
- bindVal = "host"
- }
-
- switch bindVal {
- case "host":
- listenPid = lxdPid
- connectPid = containerPid
- listenAddr = rewriteHostAddr(listenAddr)
- case "container":
- listenPid = containerPid
- connectPid = lxdPid
- connectAddr = rewriteHostAddr(connectAddr)
- default:
- return nil, fmt.Errorf("Invalid binding side given. Must be \"host\" or \"container\"")
- }
-
- p := &proxyProcInfo{
- listenPid: listenPid,
- connectPid: connectPid,
- connectAddr: connectAddr,
- listenAddr: listenAddr,
- listenAddrGid: device["gid"],
- listenAddrUid: device["uid"],
- listenAddrMode: device["mode"],
- securityGid: device["security.gid"],
- securityUid: device["security.uid"],
- proxyProtocol: device["proxy_protocol"],
- }
-
- return p, nil
-}
-
func killProxyProc(pidPath string) error {
// Get the contents of the pid file
contents, err := ioutil.ReadFile(pidPath)
From 79bfe669961e17f9313f9089b8da72431d2f17ad Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 29 Jul 2019 14:26:04 +0100
Subject: [PATCH 18/42] device/instance/id: Adds LogPath() to instance
identifier interface
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device_instance_id.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/lxd/device/device_instance_id.go b/lxd/device/device_instance_id.go
index 38aae3aae9..8fc68732df 100644
--- a/lxd/device/device_instance_id.go
+++ b/lxd/device/device_instance_id.go
@@ -12,6 +12,7 @@ type InstanceIdentifier interface {
Type() string
Project() string
DevicesPath() string
+ LogPath() string
ExpandedConfig() map[string]string
ExpandedDevices() config.Devices
}
From 8b9e62288004273996c3d7aef6a37d8b566241aa Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 29 Jul 2019 15:18:04 +0100
Subject: [PATCH 19/42] device: Hooks up proxy device
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device.go | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lxd/device/device.go b/lxd/device/device.go
index 7bd4199dbb..4281bdcd3e 100644
--- a/lxd/device/device.go
+++ b/lxd/device/device.go
@@ -9,7 +9,8 @@ import (
// devTypes defines supported top-level device type creation functions.
var devTypes = map[string]func(config.Device) device{
- "nic": nicLoadByType,
+ "nic": nicLoadByType,
+ "proxy": func(c config.Device) device { return &proxy{} },
}
// VolatileSetter is a function that accepts one or more key/value strings to save into the LXD
From 0e15b01b8fd692a679c22b2c38306ca8f3976801 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 31 Jul 2019 13:46:17 +0100
Subject: [PATCH 20/42] main/forkproxy: Fixes crash when listener cannot be
setup
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/main_forkproxy.go | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/lxd/main_forkproxy.go b/lxd/main_forkproxy.go
index 3ead12b1b7..5903503af9 100644
--- a/lxd/main_forkproxy.go
+++ b/lxd/main_forkproxy.go
@@ -573,6 +573,13 @@ func (c *cmdForkproxy) Run(cmd *cobra.Command, args []string) error {
unix.Close(forkproxyUDSSockFDNum)
return err
}
+
+ if f == nil {
+ fmt.Printf("Failed to receive fd from listener process")
+ unix.Close(forkproxyUDSSockFDNum)
+ return err
+ }
+
files = append(files, f)
}
unix.Close(forkproxyUDSSockFDNum)
From 646ae09f32898f99b90bfa29b08ec755ca3d5451 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 31 Jul 2019 13:48:01 +0100
Subject: [PATCH 21/42] main/forkproxy: Reworks log messages to better define
the different types
Adds "Status: Started" log output when started OK so can be detected from LXD.
Status: daemon status start/stop message.
Error: daemon setup problem.
Warning: connection specific problem, non fatal.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/main_forkproxy.go | 34 ++++++++++++++++++----------------
1 file changed, 18 insertions(+), 16 deletions(-)
diff --git a/lxd/main_forkproxy.go b/lxd/main_forkproxy.go
index 5903503af9..1801182d73 100644
--- a/lxd/main_forkproxy.go
+++ b/lxd/main_forkproxy.go
@@ -358,14 +358,14 @@ func listenerInstance(epFd C.int, lAddr *device.ProxyAddress, cAddr *device.Prox
srcConn, err := net.FileConn((*lStruct).f)
if err != nil {
- fmt.Printf("Failed to re-assemble listener: %s", err)
+ fmt.Printf("Warning: Failed to re-assemble listener: %s\n", err)
rearmUDPFd(epFd, connFd)
return
}
dstConn, err := net.Dial(cAddr.ConnType, connectAddr)
if err != nil {
- fmt.Printf("Error: Failed to connect to target: %v\n", err)
+ fmt.Printf("Warning: Failed to connect to target: %v\n", err)
rearmUDPFd(epFd, connFd)
return
}
@@ -381,7 +381,7 @@ func listenerInstance(epFd C.int, lAddr *device.ProxyAddress, cAddr *device.Prox
listener := (*lStruct).lConn
srcConn, err := (*listener).Accept()
if err != nil {
- fmt.Printf("Error: Failed to accept new connection: %v\n", err)
+ fmt.Printf("Warning: Failed to accept new connection: %v\n", err)
return err
}
@@ -394,7 +394,7 @@ func listenerInstance(epFd C.int, lAddr *device.ProxyAddress, cAddr *device.Prox
dstConn, err := net.Dial(cAddr.ConnType, connectAddr)
if err != nil {
srcConn.Close()
- fmt.Printf("Error: Failed to connect to target: %v\n", err)
+ fmt.Printf("Warning: Failed to connect to target: %v\n", err)
return err
}
@@ -569,13 +569,13 @@ func (c *cmdForkproxy) Run(cmd *cobra.Command, args []string) error {
goto rAgain
}
- fmt.Printf("Failed to receive fd from listener process: %v\n", err)
+ fmt.Printf("Error: Failed to receive fd from listener process: %v\n", err)
unix.Close(forkproxyUDSSockFDNum)
return err
}
if f == nil {
- fmt.Printf("Failed to receive fd from listener process")
+ fmt.Printf("Error: Failed to receive fd from listener process\n")
unix.Close(forkproxyUDSSockFDNum)
return err
}
@@ -599,7 +599,7 @@ func (c *cmdForkproxy) Run(cmd *cobra.Command, args []string) error {
for i, f := range files {
listener, err := net.FileListener(f)
if err != nil {
- fmt.Printf("Failed to re-assemble listener: %v", err)
+ fmt.Printf("Error: Failed to re-assemble listener: %v\n", err)
return err
}
listenerMap[int(f.Fd())] = &lStruct{
@@ -650,7 +650,6 @@ func (c *cmdForkproxy) Run(cmd *cobra.Command, args []string) error {
self := unix.Getpid()
go func() {
<-sigs
-
for _, f := range files {
C.epoll_ctl(epFd, C.EPOLL_CTL_DEL, C.int(f.Fd()), nil)
f.Close()
@@ -678,16 +677,19 @@ func (c *cmdForkproxy) Run(cmd *cobra.Command, args []string) error {
*(*C.int)(unsafe.Pointer(&ev.data)) = C.int(f.Fd())
ret := C.epoll_ctl(epFd, C.EPOLL_CTL_ADD, C.int(f.Fd()), &ev)
if ret < 0 {
- return fmt.Errorf("Failed to add listener fd to epoll instance")
+ return fmt.Errorf("Error: Failed to add listener fd to epoll instance")
}
}
+ // This line is used by LXD to check forkproxy has started OK.
+ fmt.Println("Status: Started")
+
for {
var events [10]C.struct_epoll_event
nfds := C.lxc_epoll_wait_nointr(epFd, &events[0], 10, -1)
if nfds < 0 {
- fmt.Printf("Failed to wait on epoll instance\n")
+ fmt.Printf("Error: Failed to wait on epoll instance\n")
break
}
@@ -700,12 +702,12 @@ func (c *cmdForkproxy) Run(cmd *cobra.Command, args []string) error {
err := listenerInstance(epFd, lAddr, cAddr, curFd, srcConn, args[11] == "true")
if err != nil {
- fmt.Printf("Failed to prepare new listener instance: %s", err)
+ fmt.Printf("Warning: Failed to prepare new listener instance: %s\n", err)
}
}
}
- fmt.Printf("Stopping proxy\n")
+ fmt.Printf("Status: Stopping proxy\n")
return nil
}
@@ -849,12 +851,12 @@ func genericRelay(dst net.Conn, src net.Conn, timeout bool) {
select {
case errSnd := <-chSend:
if errSnd != nil {
- fmt.Printf("Error while sending data: %v\n", errSnd)
+ fmt.Printf("Warning: Error while sending data: %v\n", errSnd)
}
case errRcv := <-chRecv:
if errRcv != nil {
- fmt.Printf("Error while reading data: %v\n", errRcv)
+ fmt.Printf("Warning: Error while reading data: %v\n", errRcv)
}
}
@@ -942,12 +944,12 @@ func unixRelay(dst io.ReadWriteCloser, src io.ReadWriteCloser) {
select {
case errSnd := <-chSend:
if errSnd != nil {
- fmt.Printf("Error while sending data: %v\n", errSnd)
+ fmt.Printf("Warning: Error while sending data: %v\n", errSnd)
}
case errRcv := <-chRecv:
if errRcv != nil {
- fmt.Printf("Error while reading data: %v\n", errRcv)
+ fmt.Printf("Warning: Error while reading data: %v\n", errRcv)
}
}
From 94458c355da43ca000a7d5ae08d058bef46ce654 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 29 Jul 2019 12:29:05 +0100
Subject: [PATCH 22/42] device/proxy: Adds proxy device implementation
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/proxy.go | 469 ++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 469 insertions(+)
create mode 100644 lxd/device/proxy.go
diff --git a/lxd/device/proxy.go b/lxd/device/proxy.go
new file mode 100644
index 0000000000..dbb3ad7fed
--- /dev/null
+++ b/lxd/device/proxy.go
@@ -0,0 +1,469 @@
+package device
+
+import (
+ "bufio"
+ "bytes"
+ "fmt"
+ "io/ioutil"
+ "net"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "time"
+
+ "golang.org/x/sys/unix"
+ "gopkg.in/lxc/go-lxc.v2"
+
+ "github.com/lxc/lxd/lxd/device/config"
+ "github.com/lxc/lxd/lxd/instance"
+ "github.com/lxc/lxd/lxd/iptables"
+ "github.com/lxc/lxd/lxd/project"
+ "github.com/lxc/lxd/shared"
+)
+
+type proxy struct {
+ deviceCommon
+}
+
+type proxyProcInfo struct {
+ listenPid string
+ connectPid string
+ connectAddr string
+ listenAddr string
+ listenAddrGID string
+ listenAddrUID string
+ listenAddrMode string
+ securityUID string
+ securityGID string
+ proxyProtocol string
+}
+
+func (d *proxy) CanHotPlug() (bool, []string) {
+ return true, []string{}
+}
+
+// validateConfig checks the supplied config for correctness.
+func (d *proxy) validateConfig() error {
+ if d.instance.Type() != instance.TypeContainer {
+ return ErrUnsupportedDevType
+ }
+
+ validateAddr := func(input string) error {
+ _, err := ProxyParseAddr(input)
+ return err
+ }
+
+ // Supported bind types are: "host" or "guest" (and "container", a legacy option equivalent to "guest").
+ // If an empty value is supplied the default behavior is to assume "host" bind mode.
+ validateBind := func(input string) error {
+ if !shared.StringInSlice(d.config["bind"], []string{"", "host", "guest", "container"}) {
+ return fmt.Errorf("Invalid binding side given. Must be \"host\" or \"guest\"")
+ }
+
+ return nil
+ }
+
+ rules := map[string]func(string) error{
+ "listen": validateAddr,
+ "connect": validateAddr,
+ "bind": validateBind,
+ "mode": shared.IsOctalFileMode,
+ "nat": shared.IsBool,
+ "gid": shared.IsUnixUserID,
+ "uid": shared.IsUnixUserID,
+ "security.uid": shared.IsUnixUserID,
+ "security.gid": shared.IsUnixUserID,
+ "proxy_protocol": shared.IsBool,
+ }
+
+ err := config.ValidateDevice(rules, d.config)
+ if err != nil {
+ return err
+ }
+
+ listenAddr, err := ProxyParseAddr(d.config["listen"])
+ if err != nil {
+ return err
+ }
+
+ connectAddr, err := ProxyParseAddr(d.config["connect"])
+ if err != nil {
+ return err
+ }
+
+ if len(connectAddr.Addr) > len(listenAddr.Addr) {
+ // Cannot support single port -> multiple port
+ return fmt.Errorf("Cannot map a single port to multiple ports")
+ }
+
+ if shared.IsTrue(d.config["proxy_protocol"]) && !strings.HasPrefix(d.config["connect"], "tcp") {
+ return fmt.Errorf("The PROXY header can only be sent to tcp servers")
+ }
+
+ if (!strings.HasPrefix(d.config["listen"], "unix:") || strings.HasPrefix(d.config["listen"], "unix:@")) &&
+ (d.config["uid"] != "" || d.config["gid"] != "" || d.config["mode"] != "") {
+ return fmt.Errorf("Only proxy devices for non-abstract unix sockets can carry uid, gid, or mode properties")
+ }
+
+ if shared.IsTrue(d.config["nat"]) {
+ if d.config["bind"] != "" && d.config["bind"] != "host" {
+ return fmt.Errorf("Only host-bound proxies can use NAT")
+ }
+
+ // Support TCP <-> TCP and UDP <-> UDP
+ if listenAddr.ConnType == "unix" || connectAddr.ConnType == "unix" ||
+ listenAddr.ConnType != connectAddr.ConnType {
+ return fmt.Errorf("Proxying %s <-> %s is not supported when using NAT",
+ listenAddr.ConnType, connectAddr.ConnType)
+ }
+ }
+
+ return nil
+}
+
+// validateEnvironment checks the runtime environment for correctness.
+func (d *proxy) validateEnvironment() error {
+ if d.name == "" {
+ return fmt.Errorf("Device name cannot be empty")
+ }
+
+ return nil
+}
+
+// Start is run when the device is added to the container.
+func (d *proxy) Start() (*RunConfig, error) {
+ err := d.validateEnvironment()
+ if err != nil {
+ return nil, err
+ }
+
+ // Proxy devices have to be setup once the container is running.
+ runConf := RunConfig{}
+ runConf.PostStartHooks = []func() error{
+ func() error {
+ if shared.IsTrue(d.config["nat"]) {
+ return d.setupNAT()
+ }
+
+ proxyValues, err := d.setupProxyProcInfo()
+ if err != nil {
+ return err
+ }
+
+ devFileName := fmt.Sprintf("proxy.%s", d.name)
+ pidPath := filepath.Join(d.instance.DevicesPath(), devFileName)
+ logFileName := fmt.Sprintf("proxy.%s.log", d.name)
+ logPath := filepath.Join(d.instance.LogPath(), logFileName)
+
+ _, err = shared.RunCommand(
+ d.state.OS.ExecPath,
+ "forkproxy",
+ proxyValues.listenPid,
+ proxyValues.listenAddr,
+ proxyValues.connectPid,
+ proxyValues.connectAddr,
+ logPath,
+ pidPath,
+ proxyValues.listenAddrGID,
+ proxyValues.listenAddrUID,
+ proxyValues.listenAddrMode,
+ proxyValues.securityGID,
+ proxyValues.securityUID,
+ proxyValues.proxyProtocol,
+ )
+ if err != nil {
+ return fmt.Errorf("Error occurred when starting proxy device: %s", err)
+ }
+
+ // Poll log file a few times until we see "Started" to indicate successful start.
+ for i := 0; i < 10; i++ {
+ started, err := d.checkProcStarted(logPath)
+
+ if err != nil {
+ return fmt.Errorf("Error occurred when starting proxy device: %s", err)
+ }
+
+ if started {
+ return nil
+ }
+
+ time.Sleep(time.Second)
+ }
+
+ return fmt.Errorf("Error occurred when starting proxy device, please look in %s", logPath)
+ },
+ }
+
+ return &runConf, nil
+}
+
+// checkProcStarted checks for the "Started" line in the log file. Returns true if found, false
+// if not, and error if any other error occurs.
+func (d *proxy) checkProcStarted(logPath string) (bool, error) {
+ file, err := os.Open(logPath)
+ if err != nil {
+ return false, err
+ }
+ defer file.Close()
+
+ scanner := bufio.NewScanner(file)
+ for scanner.Scan() {
+ line := strings.TrimSpace(scanner.Text())
+
+ if line == "Status: Started" {
+ return true, nil
+ }
+
+ if strings.HasPrefix(line, "Error:") {
+ return false, fmt.Errorf("%s", line)
+ }
+ }
+ if err := scanner.Err(); err != nil {
+ return false, err
+ }
+
+ return false, nil
+}
+
+// Stop is run when the device is removed from the container.
+func (d *proxy) Stop() error {
+ // Remove possible iptables entries
+ iptables.ContainerClear("ipv4", fmt.Sprintf("%s (%s)", d.instance.Name(), d.name), "nat")
+ iptables.ContainerClear("ipv6", fmt.Sprintf("%s (%s)", d.instance.Name(), d.name), "nat")
+
+ devFileName := fmt.Sprintf("proxy.%s", d.name)
+ devPath := filepath.Join(d.instance.DevicesPath(), devFileName)
+
+ if !shared.PathExists(devPath) {
+ // There's no proxy process if NAT is enabled
+ return nil
+ }
+
+ err := d.killProxyProc(devPath)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (d *proxy) setupNAT() error {
+ listenAddr, err := ProxyParseAddr(d.config["listen"])
+ if err != nil {
+ return err
+ }
+
+ connectAddr, err := ProxyParseAddr(d.config["connect"])
+ if err != nil {
+ return err
+ }
+
+ address, _, err := net.SplitHostPort(connectAddr.Addr[0])
+ if err != nil {
+ return err
+ }
+
+ var IPv4Addr string
+ var IPv6Addr string
+
+ instanceConfig, err := InstanceLoadByProjectAndName(d.state, d.instance.Project(), d.instance.Name())
+ if err != nil {
+ return err
+ }
+
+ for _, devConfig := range instanceConfig.ExpandedDevices() {
+ if devConfig["type"] != "nic" || (devConfig["type"] == "nic" && devConfig["nictype"] != "bridged") {
+ continue
+ }
+
+ // Check whether the NIC has a static IP.
+ ip := devConfig["ipv4.address"]
+ // Ensure that the provided IP address matches the container's IP
+ // address otherwise we could mess with other containers.
+ if ip != "" && IPv4Addr == "" && (address == ip || address == "0.0.0.0") {
+ IPv4Addr = ip
+ }
+
+ ip = devConfig["ipv6.address"]
+ if ip != "" && IPv6Addr == "" && (address == ip || address == "::") {
+ IPv6Addr = ip
+ }
+ }
+
+ if IPv4Addr == "" && IPv6Addr == "" {
+ return fmt.Errorf("NIC IP doesn't match proxy target IP")
+ }
+
+ iptablesComment := fmt.Sprintf("%s (%s)", d.instance.Name(), d.name)
+
+ revert := true
+ defer func() {
+ if revert {
+ if IPv4Addr != "" {
+ iptables.ContainerClear("ipv4", iptablesComment, "nat")
+ }
+
+ if IPv6Addr != "" {
+ iptables.ContainerClear("ipv6", iptablesComment, "nat")
+ }
+ }
+ }()
+
+ for i, lAddr := range listenAddr.Addr {
+ address, port, err := net.SplitHostPort(lAddr)
+ if err != nil {
+ return err
+ }
+ var cPort string
+ if len(connectAddr.Addr) == 1 {
+ _, cPort, _ = net.SplitHostPort(connectAddr.Addr[0])
+ } else {
+ _, cPort, _ = net.SplitHostPort(connectAddr.Addr[i])
+ }
+
+ if IPv4Addr != "" {
+ // outbound <-> container
+ err := iptables.ContainerPrepend("ipv4", iptablesComment, "nat",
+ "PREROUTING", "-p", listenAddr.ConnType, "--destination",
+ address, "--dport", port, "-j", "DNAT",
+ "--to-destination", fmt.Sprintf("%s:%s", IPv4Addr, cPort))
+ if err != nil {
+ return err
+ }
+
+ // host <-> container
+ err = iptables.ContainerPrepend("ipv4", iptablesComment, "nat",
+ "OUTPUT", "-p", listenAddr.ConnType, "--destination",
+ address, "--dport", port, "-j", "DNAT",
+ "--to-destination", fmt.Sprintf("%s:%s", IPv4Addr, cPort))
+ if err != nil {
+ return err
+ }
+ }
+
+ if IPv6Addr != "" {
+ // outbound <-> container
+ err := iptables.ContainerPrepend("ipv6", iptablesComment, "nat",
+ "PREROUTING", "-p", listenAddr.ConnType, "--destination",
+ address, "--dport", port, "-j", "DNAT",
+ "--to-destination", fmt.Sprintf("[%s]:%s", IPv6Addr, cPort))
+ if err != nil {
+ return err
+ }
+
+ // host <-> container
+ err = iptables.ContainerPrepend("ipv6", iptablesComment, "nat",
+ "OUTPUT", "-p", listenAddr.ConnType, "--destination",
+ address, "--dport", port, "-j", "DNAT",
+ "--to-destination", fmt.Sprintf("[%s]:%s", IPv6Addr, cPort))
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ revert = false
+ return nil
+}
+
+func (d *proxy) rewriteHostAddr(addr string) string {
+ fields := strings.SplitN(addr, ":", 2)
+ proto := fields[0]
+ addr = fields[1]
+ if proto == "unix" && !strings.HasPrefix(addr, "@") {
+ // Unix non-abstract sockets need to be addressed to the host
+ // filesystem, not be scoped inside the LXD snap.
+ addr = shared.HostPath(addr)
+ }
+ return fmt.Sprintf("%s:%s", proto, addr)
+}
+
+func (d *proxy) setupProxyProcInfo() (*proxyProcInfo, error) {
+ cname := project.Prefix(d.instance.Project(), d.instance.Name())
+ cc, err := lxc.NewContainer(cname, d.state.OS.LxcPath)
+ if err != nil {
+ return nil, err
+ }
+ defer cc.Release()
+
+ containerPid := strconv.Itoa(cc.InitPid())
+ lxdPid := strconv.Itoa(os.Getpid())
+
+ var listenPid, connectPid string
+
+ connectAddr := d.config["connect"]
+ listenAddr := d.config["listen"]
+
+ switch d.config["bind"] {
+ case "host", "":
+ listenPid = lxdPid
+ connectPid = containerPid
+ listenAddr = d.rewriteHostAddr(listenAddr)
+ case "guest", "container":
+ listenPid = containerPid
+ connectPid = lxdPid
+ connectAddr = d.rewriteHostAddr(connectAddr)
+ default:
+ return nil, fmt.Errorf("Invalid binding side given. Must be \"host\" or \"guest\"")
+ }
+
+ p := &proxyProcInfo{
+ listenPid: listenPid,
+ connectPid: connectPid,
+ connectAddr: connectAddr,
+ listenAddr: listenAddr,
+ listenAddrGID: d.config["gid"],
+ listenAddrUID: d.config["uid"],
+ listenAddrMode: d.config["mode"],
+ securityGID: d.config["security.gid"],
+ securityUID: d.config["security.uid"],
+ proxyProtocol: d.config["proxy_protocol"],
+ }
+
+ return p, nil
+}
+
+func (d *proxy) killProxyProc(pidPath string) error {
+ // Get the contents of the pid file
+ contents, err := ioutil.ReadFile(pidPath)
+ if err != nil {
+ return err
+ }
+ pidString := strings.TrimSpace(string(contents))
+
+ // Check if the process still exists
+ if !shared.PathExists(fmt.Sprintf("/proc/%s", pidString)) {
+ os.Remove(pidPath)
+ return nil
+ }
+
+ // Check if it's forkdns
+ cmdArgs, err := ioutil.ReadFile(fmt.Sprintf("/proc/%s/cmdline", pidString))
+ if err != nil {
+ os.Remove(pidPath)
+ return nil
+ }
+
+ cmdFields := strings.Split(string(bytes.TrimRight(cmdArgs, string("\x00"))), string(byte(0)))
+ if len(cmdFields) < 5 || cmdFields[1] != "forkproxy" {
+ os.Remove(pidPath)
+ return nil
+ }
+
+ // Parse the pid
+ pidInt, err := strconv.Atoi(pidString)
+ if err != nil {
+ return err
+ }
+
+ // Actually kill the process
+ err = unix.Kill(pidInt, unix.SIGKILL)
+ if err != nil {
+ return err
+ }
+
+ // Cleanup
+ os.Remove(pidPath)
+ return nil
+}
From 5e92433c6e0f01ad3f8e48cbbc6c6933e5623f5d Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 29 Jul 2019 15:17:34 +0100
Subject: [PATCH 23/42] test: Renames the proxy device tests to fit with other
device tests
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/proxy_device_utils.go | 58 -------------------
test/main.sh | 2 +-
.../{proxy.sh => container_devices_proxy.sh} | 30 +++++-----
3 files changed, 16 insertions(+), 74 deletions(-)
delete mode 100644 lxd/proxy_device_utils.go
rename test/suites/{proxy.sh => container_devices_proxy.sh} (97%)
diff --git a/lxd/proxy_device_utils.go b/lxd/proxy_device_utils.go
deleted file mode 100644
index a96e7a0ab6..0000000000
--- a/lxd/proxy_device_utils.go
+++ /dev/null
@@ -1,58 +0,0 @@
-package main
-
-import (
- "bytes"
- "fmt"
- "io/ioutil"
- "os"
- "strconv"
- "strings"
-
- "golang.org/x/sys/unix"
-
- "github.com/lxc/lxd/shared"
-)
-
-func killProxyProc(pidPath string) error {
- // Get the contents of the pid file
- contents, err := ioutil.ReadFile(pidPath)
- if err != nil {
- return err
- }
- pidString := strings.TrimSpace(string(contents))
-
- // Check if the process still exists
- if !shared.PathExists(fmt.Sprintf("/proc/%s", pidString)) {
- os.Remove(pidPath)
- return nil
- }
-
- // Check if it's forkdns
- cmdArgs, err := ioutil.ReadFile(fmt.Sprintf("/proc/%s/cmdline", pidString))
- if err != nil {
- os.Remove(pidPath)
- return nil
- }
-
- cmdFields := strings.Split(string(bytes.TrimRight(cmdArgs, string("\x00"))), string(byte(0)))
- if len(cmdFields) < 5 || cmdFields[1] != "forkproxy" {
- os.Remove(pidPath)
- return nil
- }
-
- // Parse the pid
- pidInt, err := strconv.Atoi(pidString)
- if err != nil {
- return err
- }
-
- // Actually kill the process
- err = unix.Kill(pidInt, unix.SIGKILL)
- if err != nil {
- return err
- }
-
- // Cleanup
- os.Remove(pidPath)
- return nil
-}
diff --git a/test/main.sh b/test/main.sh
index 6e0b6369f9..66ff5401e5 100755
--- a/test/main.sh
+++ b/test/main.sh
@@ -238,7 +238,7 @@ run_test test_kernel_limits "kernel limits"
run_test test_macaroon_auth "macaroon authentication"
run_test test_console "console"
run_test test_query "query"
-run_test test_proxy_device "proxy device"
+run_test test_container_devices_proxy "container devices - proxy"
run_test test_storage_local_volume_handling "storage local volume handling"
run_test test_backup_import "backup import"
run_test test_backup_export "backup export"
diff --git a/test/suites/proxy.sh b/test/suites/container_devices_proxy.sh
similarity index 97%
rename from test/suites/proxy.sh
rename to test/suites/container_devices_proxy.sh
index 3ec9e270bc..436199c008 100644
--- a/test/suites/proxy.sh
+++ b/test/suites/container_devices_proxy.sh
@@ -1,14 +1,14 @@
-test_proxy_device() {
- test_proxy_device_tcp
- test_proxy_device_tcp_unix
- test_proxy_device_tcp_udp
- test_proxy_device_udp
- test_proxy_device_unix
- test_proxy_device_unix_udp
- test_proxy_device_unix_tcp
+test_container_devices_proxy() {
+ container_devices_proxy_tcp
+ container_devices_proxy_tcp_unix
+ container_devices_proxy_tcp_udp
+ container_devices_proxy_udp
+ container_devices_proxy_unix
+ container_devices_proxy_unix_udp
+ container_devices_proxy_unix_tcp
}
-test_proxy_device_tcp() {
+container_devices_proxy_tcp() {
echo "====> Testing tcp proxying"
ensure_import_testimage
ensure_has_localhost_remote "${LXD_ADDR}"
@@ -152,7 +152,7 @@ test_proxy_device_tcp() {
lxc network delete lxdt$$
}
-test_proxy_device_unix() {
+container_devices_proxy_unix() {
echo "====> Testing unix proxying"
ensure_import_testimage
ensure_has_localhost_remote "${LXD_ADDR}"
@@ -235,7 +235,7 @@ test_proxy_device_unix() {
lxc delete -f proxyTester
}
-test_proxy_device_tcp_unix() {
+container_devices_proxy_tcp_unix() {
echo "====> Testing tcp to unix proxying"
ensure_import_testimage
ensure_has_localhost_remote "${LXD_ADDR}"
@@ -312,7 +312,7 @@ test_proxy_device_tcp_unix() {
lxc delete -f proxyTester
}
-test_proxy_device_unix_tcp() {
+container_devices_proxy_unix_tcp() {
echo "====> Testing unix to tcp proxying"
ensure_import_testimage
ensure_has_localhost_remote "${LXD_ADDR}"
@@ -380,7 +380,7 @@ test_proxy_device_unix_tcp() {
lxc delete -f proxyTester
}
-test_proxy_device_udp() {
+container_devices_proxy_udp() {
echo "====> Testing udp proxying"
ensure_import_testimage
ensure_has_localhost_remote "${LXD_ADDR}"
@@ -442,7 +442,7 @@ test_proxy_device_udp() {
lxc delete -f proxyTester
}
-test_proxy_device_unix_udp() {
+container_devices_proxy_unix_udp() {
echo "====> Testing unix to udp proxying"
ensure_import_testimage
ensure_has_localhost_remote "${LXD_ADDR}"
@@ -510,7 +510,7 @@ test_proxy_device_unix_udp() {
lxc delete -f proxyTester
}
-test_proxy_device_tcp_udp() {
+container_devices_proxy_tcp_udp() {
echo "====> Testing tcp to udp proxying"
ensure_import_testimage
ensure_has_localhost_remote "${LXD_ADDR}"
From 7f6888d49a5e0388699800911fee5bd7a3cda280 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 29 Jul 2019 12:30:12 +0100
Subject: [PATCH 24/42] container/lxc: Updates references for proxy to use
device package
Adds postStartHooks to startCommon.
Reduces hard coded lists of device package support checks and removes unused/moved code.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/container_lxc.go | 605 ++++++++++++-------------------------------
1 file changed, 169 insertions(+), 436 deletions(-)
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index e74bd48eaa..26e60b9fbc 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -32,7 +32,6 @@ import (
"github.com/lxc/lxd/lxd/device"
"github.com/lxc/lxd/lxd/device/config"
"github.com/lxc/lxd/lxd/instance"
- "github.com/lxc/lxd/lxd/iptables"
"github.com/lxc/lxd/lxd/maas"
"github.com/lxc/lxd/lxd/project"
"github.com/lxc/lxd/lxd/state"
@@ -506,14 +505,10 @@ func containerLXCCreate(s *state.State, args db.ContainerArgs) (container, error
return nil, err
}
- // Add NIC devices to container.
+ // Add devices to container.
for k, m := range c.expandedDevices {
- if m["type"] != "nic" {
- continue
- }
-
err = c.deviceAdd(k, m)
- if err != nil {
+ if err != nil && err != device.ErrUnsupportedDevType {
c.Delete()
return nil, err
}
@@ -1826,6 +1821,21 @@ func (c *containerLXC) initLXC(config bool) error {
return nil
}
+// runPostStartHooks executes the callback functions after container has started.
+func (c *containerLXC) runPostStartHooks(hooks []func() error) error {
+ // Run any post start hooks.
+ if len(hooks) > 0 {
+ for _, hook := range hooks {
+ err := hook()
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+}
+
// deviceLoad instantiates and validates a new device and returns it along with enriched config.
func (c *containerLXC) deviceLoad(deviceName string, rawConfig map[string]string) (device.Device, map[string]string, error) {
var configCopy config.Device
@@ -1839,9 +1849,9 @@ func (c *containerLXC) deviceLoad(deviceName string, rawConfig map[string]string
}
} else {
// Othewise copy the config so it cannot be modified by device.
- err := shared.DeepCopy(&rawConfig, &configCopy)
- if err != nil {
- return nil, nil, err
+ configCopy = make(map[string]string)
+ for k, v := range rawConfig {
+ configCopy[k] = v
}
}
@@ -1879,34 +1889,52 @@ func (c *containerLXC) deviceStart(deviceName string, rawConfig map[string]strin
return nil, err
}
- // Ensure nic and infiniband devices are attached if running.
- if isRunning && shared.StringInSlice(rawConfig["type"], []string{"nic", "infiniband"}) {
- devName := ""
- for _, dev := range runConfig.NetworkInterfaces[0] {
- if dev.Key == "link" {
- devName = dev.Value
- break
+ // If container is running and runConfig returned, then live attach device.
+ if isRunning && runConfig != nil {
+ // If network interface settings returned, then attach NIC to container.
+ if len(runConfig.NetworkInterface) > 0 {
+ err = c.deviceAttachNIC(configCopy, runConfig)
+ if err != nil {
+ return nil, err
}
}
- if devName == "" {
- return nil, fmt.Errorf("Device didn't provide a link property to use")
- }
-
- // Load the go-lxc struct.
- err = c.initLXC(false)
+ err = c.runPostStartHooks(runConfig.PostStartHooks)
if err != nil {
return nil, err
}
+ }
- // Add the interface to the container.
- err = c.c.AttachInterface(devName, configCopy["name"])
- if err != nil {
- return nil, fmt.Errorf("Failed to attach interface: %s to %s: %s", devName, configCopy["name"], err)
+ return runConfig, nil
+}
+
+// deviceAttachNIC live attaches a NIC device to a container.
+func (c *containerLXC) deviceAttachNIC(configCopy map[string]string, runConfig *device.RunConfig) error {
+ devName := ""
+ for _, dev := range runConfig.NetworkInterface {
+ if dev.Key == "link" {
+ devName = dev.Value
+ break
}
}
- return runConfig, nil
+ if devName == "" {
+ return fmt.Errorf("Device didn't provide a link property to use")
+ }
+
+ // Load the go-lxc struct.
+ err := c.initLXC(false)
+ if err != nil {
+ return err
+ }
+
+ // Add the interface to the container.
+ err = c.c.AttachInterface(devName, configCopy["name"])
+ if err != nil {
+ return fmt.Errorf("Failed to attach interface: %s to %s: %s", devName, configCopy["name"], err)
+ }
+
+ return nil
}
// deviceUpdate loads a new device and calls its Update() function.
@@ -2164,18 +2192,19 @@ func UnshiftBtrfsRootfs(path string, diskIdmap *idmap.IdmapSet) error {
}
// Start functions
-func (c *containerLXC) startCommon() (string, error) {
+func (c *containerLXC) startCommon() (string, []func() error, error) {
var ourStart bool
+ postStartHooks := []func() error{}
// Load the go-lxc struct
err := c.initLXC(true)
if err != nil {
- return "", errors.Wrap(err, "Load go-lxc struct")
+ return "", postStartHooks, errors.Wrap(err, "Load go-lxc struct")
}
// Check that we're not already running
if c.IsRunning() {
- return "", fmt.Errorf("The container is already running")
+ return "", postStartHooks, fmt.Errorf("The container is already running")
}
// Sanity checks for devices
@@ -2188,19 +2217,7 @@ func (c *containerLXC) startCommon() (string, error) {
// So do only check for the existence of m["source"]
// when m["pool"] is empty.
if m["pool"] == "" && m["source"] != "" && !shared.IsTrue(m["optional"]) && !shared.PathExists(shared.HostPath(m["source"])) {
- return "", fmt.Errorf("Missing source '%s' for disk '%s'", m["source"], name)
- }
- case "nic":
- if m["parent"] != "" && !shared.PathExists(fmt.Sprintf("/sys/class/net/%s", m["parent"])) {
- return "", fmt.Errorf("Missing parent '%s' for nic '%s'", m["parent"], name)
- }
-
- if shared.IsTrue(m["security.ipv6_filtering"]) {
- // Check br_netfilter is loaded and enabled for IPv6.
- sysctlVal, err := device.NetworkSysctlGet("bridge/bridge-nf-call-ip6tables")
- if err != nil || sysctlVal != "1\n" {
- return "", errors.Wrapf(err, "security.ipv6_filtering requires br_netfilter and sysctl net.bridge.bridge-nf-call-ip6tables=1")
- }
+ return "", postStartHooks, fmt.Errorf("Missing source '%s' for disk '%s'", m["source"], name)
}
case "unix-char", "unix-block":
srcPath, exist := m["source"]
@@ -2212,10 +2229,10 @@ func (c *containerLXC) startCommon() (string, error) {
err = deviceInotifyAddClosestLivingAncestor(c.state, filepath.Dir(srcPath))
if err != nil {
logger.Errorf("Failed to add \"%s\" to inotify targets", srcPath)
- return "", fmt.Errorf("Failed to setup inotify watch for '%s': %v", srcPath, err)
+ return "", postStartHooks, fmt.Errorf("Failed to setup inotify watch for '%s': %v", srcPath, err)
}
} else if srcPath != "" && m["major"] == "" && m["minor"] == "" && !shared.PathExists(srcPath) {
- return "", fmt.Errorf("Missing source '%s' for device '%s'", srcPath, name)
+ return "", postStartHooks, fmt.Errorf("Missing source '%s' for device '%s'", srcPath, name)
}
}
}
@@ -2227,7 +2244,7 @@ func (c *containerLXC) startCommon() (string, error) {
module = strings.TrimPrefix(module, " ")
err := util.LoadModule(module)
if err != nil {
- return "", fmt.Errorf("Failed to load kernel module '%s': %s", module, err)
+ return "", postStartHooks, fmt.Errorf("Failed to load kernel module '%s': %s", module, err)
}
}
}
@@ -2236,22 +2253,22 @@ func (c *containerLXC) startCommon() (string, error) {
if ok {
err := c.initStorage()
if err != nil {
- return "", errors.Wrap(err, "Initialize storage")
+ return "", postStartHooks, errors.Wrap(err, "Initialize storage")
}
size, err := units.ParseByteSizeString(newSize)
if err != nil {
- return "", err
+ return "", postStartHooks, err
}
err = c.storage.StorageEntitySetQuota(storagePoolVolumeTypeContainer, size, c)
if err != nil {
- return "", errors.Wrap(err, "Set storage quota")
+ return "", postStartHooks, errors.Wrap(err, "Set storage quota")
}
// Remove the volatile key from the DB
err = c.state.Cluster.ContainerConfigRemove(c.id, "volatile.apply_quota")
if err != nil {
- return "", errors.Wrap(err, "Remove volatile.apply_quota config key")
+ return "", postStartHooks, errors.Wrap(err, "Remove volatile.apply_quota config key")
}
// Remove the volatile key from the in-memory configs
@@ -2262,17 +2279,17 @@ func (c *containerLXC) startCommon() (string, error) {
/* Deal with idmap changes */
nextIdmap, err := c.NextIdmap()
if err != nil {
- return "", errors.Wrap(err, "Set ID map")
+ return "", postStartHooks, errors.Wrap(err, "Set ID map")
}
diskIdmap, err := c.DiskIdmap()
if err != nil {
- return "", errors.Wrap(err, "Set last ID map")
+ return "", postStartHooks, errors.Wrap(err, "Set last ID map")
}
if !nextIdmap.Equals(diskIdmap) && !(diskIdmap == nil && c.state.OS.Shiftfs) {
if shared.IsTrue(c.expandedConfig["security.protection.shift"]) {
- return "", fmt.Errorf("Container is protected against filesystem shifting")
+ return "", postStartHooks, fmt.Errorf("Container is protected against filesystem shifting")
}
logger.Debugf("Container idmap changed, remapping")
@@ -2280,7 +2297,7 @@ func (c *containerLXC) startCommon() (string, error) {
ourStart, err = c.StorageStart()
if err != nil {
- return "", errors.Wrap(err, "Storage start")
+ return "", postStartHooks, errors.Wrap(err, "Storage start")
}
if diskIdmap != nil {
@@ -2295,7 +2312,7 @@ func (c *containerLXC) startCommon() (string, error) {
if ourStart {
c.StorageStop()
}
- return "", err
+ return "", postStartHooks, err
}
}
@@ -2311,7 +2328,7 @@ func (c *containerLXC) startCommon() (string, error) {
if ourStart {
c.StorageStop()
}
- return "", err
+ return "", postStartHooks, err
}
}
@@ -2319,14 +2336,14 @@ func (c *containerLXC) startCommon() (string, error) {
if nextIdmap != nil && !c.state.OS.Shiftfs {
idmapBytes, err := json.Marshal(nextIdmap.Idmap)
if err != nil {
- return "", err
+ return "", postStartHooks, err
}
jsonDiskIdmap = string(idmapBytes)
}
err = c.VolatileSet(map[string]string{"volatile.last_state.idmap": jsonDiskIdmap})
if err != nil {
- return "", errors.Wrapf(err, "Set volatile.last_state.idmap config key on container %q (id %d)", c.name, c.id)
+ return "", postStartHooks, errors.Wrapf(err, "Set volatile.last_state.idmap config key on container %q (id %d)", c.name, c.id)
}
c.updateProgress("")
@@ -2338,26 +2355,25 @@ func (c *containerLXC) startCommon() (string, error) {
} else {
idmapBytes, err = json.Marshal(nextIdmap.Idmap)
if err != nil {
- return "", err
+ return "", postStartHooks, err
}
}
if c.localConfig["volatile.idmap.current"] != string(idmapBytes) {
err = c.VolatileSet(map[string]string{"volatile.idmap.current": string(idmapBytes)})
if err != nil {
- return "", errors.Wrapf(err, "Set volatile.idmap.current config key on container %q (id %d)", c.name, c.id)
+ return "", postStartHooks, errors.Wrapf(err, "Set volatile.idmap.current config key on container %q (id %d)", c.name, c.id)
}
}
// Generate the Seccomp profile
if err := SeccompCreateProfile(c); err != nil {
- return "", err
+ return "", postStartHooks, err
}
// Cleanup any existing leftover devices
c.removeUnixDevices()
c.removeDiskDevices()
- c.removeProxyDevices()
var usbs []usbDevice
diskDevices := map[string]config.Device{}
@@ -2372,7 +2388,7 @@ func (c *containerLXC) startCommon() (string, error) {
if err != nil {
// Deal with device hotplug
if m["required"] == "" || shared.IsTrue(m["required"]) {
- return "", err
+ return "", postStartHooks, err
}
srcPath := m["source"]
@@ -2384,7 +2400,7 @@ func (c *containerLXC) startCommon() (string, error) {
err = deviceInotifyAddClosestLivingAncestor(c.state, srcPath)
if err != nil {
logger.Errorf("Failed to add \"%s\" to inotify targets", srcPath)
- return "", err
+ return "", postStartHooks, err
}
continue
}
@@ -2394,12 +2410,12 @@ func (c *containerLXC) startCommon() (string, error) {
dType, dMajor, dMinor, err := deviceGetAttributes(devPath)
if err != nil {
if m["required"] == "" || shared.IsTrue(m["required"]) {
- return "", err
+ return "", postStartHooks, err
}
} else {
err = lxcSetConfigItem(c.c, "lxc.cgroup.devices.allow", fmt.Sprintf("%s %d:%d rwm", dType, dMajor, dMinor))
if err != nil {
- return "", fmt.Errorf("Failed to add cgroup rule for device")
+ return "", postStartHooks, fmt.Errorf("Failed to add cgroup rule for device")
}
}
}
@@ -2407,7 +2423,7 @@ func (c *containerLXC) startCommon() (string, error) {
if usbs == nil {
usbs, err = deviceLoadUsb()
if err != nil {
- return "", err
+ return "", postStartHooks, err
}
}
@@ -2418,14 +2434,14 @@ func (c *containerLXC) startCommon() (string, error) {
err := c.setupUnixDevice(fmt.Sprintf("unix.%s", k), m, usb.major, usb.minor, usb.path, shared.IsTrue(m["required"]), false)
if err != nil {
- return "", err
+ return "", postStartHooks, err
}
}
} else if m["type"] == "gpu" {
allGpus := deviceWantsAllGPUs(m)
gpus, nvidiaDevices, err := deviceLoadGpu(allGpus)
if err != nil {
- return "", err
+ return "", postStartHooks, err
}
sawNvidia := false
@@ -2442,7 +2458,7 @@ func (c *containerLXC) startCommon() (string, error) {
err := c.setupUnixDevice(fmt.Sprintf("unix.%s", k), m, gpu.major, gpu.minor, gpu.path, true, false)
if err != nil {
- return "", err
+ return "", postStartHooks, err
}
if !gpu.isNvidia {
@@ -2452,12 +2468,12 @@ func (c *containerLXC) startCommon() (string, error) {
if gpu.nvidia.path != "" {
err = c.setupUnixDevice(fmt.Sprintf("unix.%s", k), m, gpu.nvidia.major, gpu.nvidia.minor, gpu.nvidia.path, true, false)
if err != nil {
- return "", err
+ return "", postStartHooks, err
}
} else if !allGpus {
errMsg := fmt.Errorf("Failed to detect correct \"/dev/nvidia\" path")
logger.Errorf("%s", errMsg)
- return "", errMsg
+ return "", postStartHooks, errMsg
}
sawNvidia = true
@@ -2472,7 +2488,7 @@ func (c *containerLXC) startCommon() (string, error) {
}
err := c.setupUnixDevice(fmt.Sprintf("unix.%s", k), m, gpu.major, gpu.minor, gpu.path, true, false)
if err != nil {
- return "", err
+ return "", postStartHooks, err
}
}
}
@@ -2480,47 +2496,52 @@ func (c *containerLXC) startCommon() (string, error) {
if !found {
msg := "Failed to detect requested GPU device"
logger.Error(msg)
- return "", fmt.Errorf(msg)
+ return "", postStartHooks, fmt.Errorf(msg)
}
} else if m["type"] == "disk" {
if m["path"] != "/" {
diskDevices[k] = m
}
- } else if m["type"] == "nic" || m["type"] == "infiniband" {
- networkKeyPrefix := "lxc.net"
- if !util.RuntimeLiblxcVersionAtLeast(2, 1, 0) {
- networkKeyPrefix = "lxc.network"
- }
-
- m, err := c.fillNetworkDevice(k, m)
+ } else if m["type"] == "infiniband" {
+ // Start infiniband device.
+ // Increment nicID so that LXC network index is unique per device.
+ nicID++
+ err := c.startInfiniband(nicID, k, m)
if err != nil {
- return "", err
+ return "", postStartHooks, err
}
-
+ } else {
// Use new Device interface if supported.
- nicID++
runConfig, err := c.deviceStart(k, m, false)
if err != device.ErrUnsupportedDevType {
if err != nil {
- return "", err
+ return "", postStartHooks, err
}
- for _, dev := range runConfig.NetworkInterfaces[0] {
- err = lxcSetConfigItem(c.c, fmt.Sprintf("%s.%d.%s", networkKeyPrefix, nicID, dev.Key), dev.Value)
- if err != nil {
- return "", err
+ // Pass any network setup config into LXC.
+ if len(runConfig.NetworkInterface) > 0 {
+ // Increment nicID so that LXC network index is unique per device.
+ nicID++
+
+ networkKeyPrefix := "lxc.net"
+ if !util.RuntimeLiblxcVersionAtLeast(2, 1, 0) {
+ networkKeyPrefix = "lxc.network"
}
- }
- continue
- }
+ for _, dev := range runConfig.NetworkInterface {
+ err = lxcSetConfigItem(c.c, fmt.Sprintf("%s.%d.%s", networkKeyPrefix, nicID, dev.Key), dev.Value)
+ if err != nil {
+ return "", postStartHooks, err
+ }
+ }
+ }
- // Start infiniband device.
- if m["type"] == "infiniband" {
- err := c.startInfiniband(nicID, k, m)
- if err != nil {
- return "", err
+ // Add any post start hooks.
+ if len(runConfig.PostStartHooks) > 0 {
+ postStartHooks = append(postStartHooks, runConfig.PostStartHooks...)
}
+
+ continue
}
}
}
@@ -2530,23 +2551,23 @@ func (c *containerLXC) startCommon() (string, error) {
return err
})
if err != nil {
- return "", err
+ return "", postStartHooks, err
}
// Create any missing directory
err = os.MkdirAll(c.LogPath(), 0700)
if err != nil {
- return "", err
+ return "", postStartHooks, err
}
err = os.MkdirAll(c.DevicesPath(), 0711)
if err != nil {
- return "", err
+ return "", postStartHooks, err
}
err = os.MkdirAll(c.ShmountsPath(), 0711)
if err != nil {
- return "", err
+ return "", postStartHooks, err
}
// Rotate the log file
@@ -2555,14 +2576,14 @@ func (c *containerLXC) startCommon() (string, error) {
os.Remove(logfile + ".old")
err := os.Rename(logfile, logfile+".old")
if err != nil {
- return "", err
+ return "", postStartHooks, err
}
}
// Storage is guaranteed to be mountable now.
ourStart, err = c.StorageStart()
if err != nil {
- return "", err
+ return "", postStartHooks, err
}
// Generate the LXC config
@@ -2570,7 +2591,7 @@ func (c *containerLXC) startCommon() (string, error) {
err = c.c.SaveConfigFile(configPath)
if err != nil {
os.Remove(configPath)
- return "", err
+ return "", postStartHooks, err
}
// Undo liblxc modifying container directory ownership
@@ -2579,7 +2600,7 @@ func (c *containerLXC) startCommon() (string, error) {
if ourStart {
c.StorageStop()
}
- return "", err
+ return "", postStartHooks, err
}
// Set right permission to allow traversal
@@ -2595,7 +2616,7 @@ func (c *containerLXC) startCommon() (string, error) {
if ourStart {
c.StorageStop()
}
- return "", err
+ return "", postStartHooks, err
}
// Update the backup.yaml file
@@ -2604,7 +2625,7 @@ func (c *containerLXC) startCommon() (string, error) {
if ourStart {
c.StorageStop()
}
- return "", err
+ return "", postStartHooks, err
}
// If starting stateless, wipe state
@@ -2615,7 +2636,7 @@ func (c *containerLXC) startCommon() (string, error) {
// Unmount any previously mounted shiftfs
unix.Unmount(c.RootfsPath(), unix.MNT_DETACH)
- return configPath, nil
+ return configPath, postStartHooks, nil
}
// detachInterfaceRename enters the container's network namespace and moves the named interface
@@ -2658,7 +2679,7 @@ func (c *containerLXC) Start(stateful bool) error {
}
// Run the shared start code
- configPath, err := c.startCommon()
+ configPath, postStartHooks, err := c.startCommon()
if err != nil {
return errors.Wrap(err, "Common start logic")
}
@@ -2710,10 +2731,11 @@ func (c *containerLXC) Start(stateful bool) error {
return errors.Wrap(err, "Start container")
}
- // Start proxy devices
- err = c.restartProxyDevices()
+ // Run any post start hooks.
+ err = c.runPostStartHooks(postStartHooks)
if err != nil {
- // Attempt to stop the container
+ // Attempt to stop container.
+ op.Done(err)
c.Stop(false)
return err
}
@@ -2777,10 +2799,11 @@ func (c *containerLXC) Start(stateful bool) error {
return err
}
- // Start proxy devices
- err = c.restartProxyDevices()
+ // Run any post start hooks.
+ err = c.runPostStartHooks(postStartHooks)
if err != nil {
- // Attempt to stop the container
+ // Attempt to stop container.
+ op.Done(err)
c.Stop(false)
return err
}
@@ -2858,18 +2881,6 @@ func (c *containerLXC) OnStart() error {
}(c)
}
- // Apply network limits
- for _, name := range c.expandedDevices.DeviceNames() {
- m := c.expandedDevices[name]
- if m["type"] != "nic" && m["type"] != "infiniband" {
- continue
- }
-
- if m["limits.max"] == "" && m["limits.ingress"] == "" && m["limits.egress"] == "" {
- continue
- }
- }
-
// Database updates
err = c.state.Cluster.Transaction(func(tx *db.ClusterTx) error {
// Record current state
@@ -3080,8 +3091,8 @@ func (c *containerLXC) OnStopNS(target string, netns string) error {
return fmt.Errorf("Invalid stop target: %s", target)
}
- // Clean up networking devices
- c.cleanupNetworkDevices(netns)
+ // Clean up devices.
+ c.cleanupDevices(netns)
return nil
}
@@ -3104,14 +3115,8 @@ func (c *containerLXC) OnStop(target string) error {
// Make sure we can't call go-lxc functions by mistake
c.fromHook = true
- // Kill all proxy devices, must happen before StorageStop
- err := c.removeProxyDevices()
- if err != nil {
- return fmt.Errorf("Unable to remove proxy devices: %v", err)
- }
-
// Stop the storage for this container
- _, err = c.StorageStop()
+ _, err := c.StorageStop()
if err != nil {
if op != nil {
op.Done(err)
@@ -3189,25 +3194,26 @@ func (c *containerLXC) OnStop(target string) error {
return nil
}
-// cleanupNetworkDevices performs any needed network device cleanup steps when container is stopped.
-func (c *containerLXC) cleanupNetworkDevices(netns string) {
+// cleanupDevices performs any needed device cleanup steps when container is stopped.
+func (c *containerLXC) cleanupDevices(netns string) {
for _, k := range c.expandedDevices.DeviceNames() {
m := c.expandedDevices[k]
- // Use the device interface if device supports it.
- if m["type"] == "nic" {
- err := c.deviceStop(k, m, netns)
- if err != nil {
- logger.Errorf("Failed to stop device: %v", err)
- }
- }
- // Clean up volatile host_name.
+ // Clean up infiniband volatile data.
if m["type"] == "infiniband" {
err := c.clearInfinibandVolatile(k)
if err != nil {
logger.Errorf("Failed to stop infiniband device: %v", err)
}
}
+
+ // Use the device interface if device supports it.
+ err := c.deviceStop(k, m, netns)
+ if err == device.ErrUnsupportedDevType {
+ continue
+ } else if err != nil {
+ logger.Errorf("Failed to stop device: %v", err)
+ }
}
}
@@ -3705,7 +3711,6 @@ func (c *containerLXC) cleanup() {
// Unmount any leftovers
c.removeUnixDevices()
c.removeDiskDevices()
- c.removeProxyDevices()
// Remove the security profiles
AADeleteProfile(c)
@@ -3811,14 +3816,10 @@ func (c *containerLXC) Delete() error {
return err
}
- // Remove NIC devices from container.
+ // Remove devices from container.
for k, m := range c.expandedDevices {
- if m["type"] != "nic" {
- continue
- }
-
err = c.deviceRemove(k, m)
- if err != nil {
+ if err != nil && err != device.ErrUnsupportedDevType {
return err
}
}
@@ -4936,11 +4937,6 @@ func (c *containerLXC) Update(args db.ContainerArgs, userRequested bool) error {
}
}
}
- } else if m["type"] == "proxy" {
- err = c.removeProxyDevice(k)
- if err != nil {
- return err
- }
}
}
@@ -5047,11 +5043,6 @@ func (c *containerLXC) Update(args db.ContainerArgs, userRequested bool) error {
logger.Error(msg)
return fmt.Errorf(msg)
}
- } else if m["type"] == "proxy" {
- err = c.insertProxyDevice(k, m)
- if err != nil {
- return err
- }
}
}
@@ -5061,14 +5052,9 @@ func (c *containerLXC) Update(args db.ContainerArgs, userRequested bool) error {
}
updateDiskLimit := false
- for k, m := range updateDevices {
+ for _, m := range updateDevices {
if m["type"] == "disk" {
updateDiskLimit = true
- } else if m["type"] == "proxy" {
- err = c.updateProxyDevice(k, m)
- if err != nil {
- return err
- }
}
}
@@ -5282,48 +5268,40 @@ func (c *containerLXC) updateDevices(removeDevices map[string]config.Device, add
isRunning := c.IsRunning()
for k, m := range removeDevices {
- if m["type"] != "nic" {
- continue
- }
-
if isRunning {
err := c.deviceStop(k, m, "")
- if err != nil {
+ if err == device.ErrUnsupportedDevType {
+ continue // No point in trying to remove device below.
+ } else if err != nil {
return err
}
}
err := c.deviceRemove(k, m)
- if err != nil {
+ if err != nil && err != device.ErrUnsupportedDevType {
return err
}
}
for k, m := range addDevices {
- if m["type"] != "nic" {
- continue
- }
-
err := c.deviceAdd(k, m)
- if err != nil {
+ if err == device.ErrUnsupportedDevType {
+ continue // No point in trying to start device below.
+ } else if err != nil {
return err
}
if isRunning {
_, err := c.deviceStart(k, m, isRunning)
- if err != nil {
+ if err != nil && err != device.ErrUnsupportedDevType {
return err
}
}
}
for k, m := range updateDevices {
- if m["type"] != "nic" {
- continue
- }
-
err := c.deviceUpdate(k, m, oldExpandedDevices[k], isRunning)
- if err != nil {
+ if err != nil && err != device.ErrUnsupportedDevType {
return err
}
}
@@ -5675,7 +5653,7 @@ func (c *containerLXC) Migrate(args *CriuMigrationArgs) error {
*/
if args.cmd == lxc.MIGRATE_RESTORE {
// Run the shared start
- _, err := c.startCommon()
+ _, postStartHooks, err := c.startCommon()
if err != nil {
return err
}
@@ -5732,10 +5710,10 @@ func (c *containerLXC) Migrate(args *CriuMigrationArgs) error {
fmt.Sprintf("%v", preservesInodes))
if migrateErr == nil {
- // Start proxy devices
- err = c.restartProxyDevices()
+ // Run any post start hooks.
+ err := c.runPostStartHooks(postStartHooks)
if err != nil {
- // Attempt to stop the container
+ // Attempt to stop container.
c.Stop(false)
return err
}
@@ -7220,251 +7198,6 @@ func (c *containerLXC) removeUnixDevices() error {
return nil
}
-func (c *containerLXC) insertProxyDevice(devName string, m config.Device) error {
- if !c.IsRunning() {
- return fmt.Errorf("Can't add proxy device to stopped container")
- }
-
- if shared.IsTrue(m["nat"]) {
- return c.doNat(devName, m)
- }
-
- proxyValues, err := setupProxyProcInfo(c, m)
- if err != nil {
- return err
- }
-
- devFileName := fmt.Sprintf("proxy.%s", devName)
- pidPath := filepath.Join(c.DevicesPath(), devFileName)
- logFileName := fmt.Sprintf("proxy.%s.log", devName)
- logPath := filepath.Join(c.LogPath(), logFileName)
-
- _, err = shared.RunCommand(
- c.state.OS.ExecPath,
- "forkproxy",
- proxyValues.listenPid,
- proxyValues.listenAddr,
- proxyValues.connectPid,
- proxyValues.connectAddr,
- logPath,
- pidPath,
- proxyValues.listenAddrGid,
- proxyValues.listenAddrUid,
- proxyValues.listenAddrMode,
- proxyValues.securityGid,
- proxyValues.securityUid,
- proxyValues.proxyProtocol,
- )
- if err != nil {
- return fmt.Errorf("Error occurred when starting proxy device: %s", err)
- }
-
- return nil
-}
-
-func (c *containerLXC) doNat(proxy string, device config.Device) error {
- listenAddr, err := proxyParseAddr(device["listen"])
- if err != nil {
- return err
- }
-
- connectAddr, err := proxyParseAddr(device["connect"])
- if err != nil {
- return err
- }
-
- address, _, err := net.SplitHostPort(connectAddr.addr[0])
- if err != nil {
- return err
- }
-
- var IPv4Addr string
- var IPv6Addr string
-
- for _, name := range c.expandedDevices.DeviceNames() {
- m := c.expandedDevices[name]
- if m["type"] != "nic" || (m["type"] == "nic" && m["nictype"] != "bridged") {
- continue
- }
-
- // Check whether the NIC has a static IP
- ip := m["ipv4.address"]
- // Ensure that the provided IP address matches the container's IP
- // address otherwise we could mess with other containers.
- if ip != "" && IPv4Addr == "" && (address == ip || address == "0.0.0.0") {
- IPv4Addr = ip
- }
-
- ip = m["ipv6.address"]
- if ip != "" && IPv6Addr == "" && (address == ip || address == "::") {
- IPv6Addr = ip
- }
- }
-
- if IPv4Addr == "" && IPv6Addr == "" {
- return fmt.Errorf("NIC IP doesn't match proxy target IP")
- }
-
- iptablesComment := fmt.Sprintf("%s (%s)", c.Name(), proxy)
-
- revert := true
- defer func() {
- if revert {
- if IPv4Addr != "" {
- iptables.ContainerClear("ipv4", iptablesComment, "nat")
- }
-
- if IPv6Addr != "" {
- iptables.ContainerClear("ipv6", iptablesComment, "nat")
- }
- }
- }()
-
- for i, lAddr := range listenAddr.addr {
- address, port, err := net.SplitHostPort(lAddr)
- if err != nil {
- return err
- }
- var cPort string
- if len(connectAddr.addr) == 1 {
- _, cPort, _ = net.SplitHostPort(connectAddr.addr[0])
- } else {
- _, cPort, _ = net.SplitHostPort(connectAddr.addr[i])
- }
-
- if IPv4Addr != "" {
- // outbound <-> container
- err := iptables.ContainerPrepend("ipv4", iptablesComment, "nat",
- "PREROUTING", "-p", listenAddr.connType, "--destination",
- address, "--dport", port, "-j", "DNAT",
- "--to-destination", fmt.Sprintf("%s:%s", IPv4Addr, cPort))
- if err != nil {
- return err
- }
-
- // host <-> container
- err = iptables.ContainerPrepend("ipv4", iptablesComment, "nat",
- "OUTPUT", "-p", listenAddr.connType, "--destination",
- address, "--dport", port, "-j", "DNAT",
- "--to-destination", fmt.Sprintf("%s:%s", IPv4Addr, cPort))
- if err != nil {
- return err
- }
- }
-
- if IPv6Addr != "" {
- // outbound <-> container
- err := iptables.ContainerPrepend("ipv6", iptablesComment, "nat",
- "PREROUTING", "-p", listenAddr.connType, "--destination",
- address, "--dport", port, "-j", "DNAT",
- "--to-destination", fmt.Sprintf("[%s]:%s", IPv6Addr, cPort))
- if err != nil {
- return err
- }
-
- // host <-> container
- err = iptables.ContainerPrepend("ipv6", iptablesComment, "nat",
- "OUTPUT", "-p", listenAddr.connType, "--destination",
- address, "--dport", port, "-j", "DNAT",
- "--to-destination", fmt.Sprintf("[%s]:%s", IPv6Addr, cPort))
- if err != nil {
- return err
- }
- }
- }
-
- revert = false
- logger.Info(fmt.Sprintf("Using NAT for proxy device '%s'", proxy))
- return nil
-}
-
-func (c *containerLXC) removeProxyDevice(devName string) error {
- if !c.IsRunning() {
- return fmt.Errorf("Can't remove proxy device from stopped container")
- }
-
- // Remove possible iptables entries
- iptables.ContainerClear("ipv4", fmt.Sprintf("%s (%s)", c.Name(), devName), "nat")
- iptables.ContainerClear("ipv6", fmt.Sprintf("%s (%s)", c.Name(), devName), "nat")
-
- devFileName := fmt.Sprintf("proxy.%s", devName)
- devPath := filepath.Join(c.DevicesPath(), devFileName)
-
- if !shared.PathExists(devPath) {
- // There's no proxy process if NAT is enabled
- return nil
- }
-
- err := killProxyProc(devPath)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (c *containerLXC) removeProxyDevices() error {
- // Remove possible iptables entries
- iptables.ContainerClear("ipv4", fmt.Sprintf("%s", c.Name()), "nat")
- iptables.ContainerClear("ipv6", fmt.Sprintf("%s", c.Name()), "nat")
-
- // Check that we actually have devices to remove
- if !shared.PathExists(c.DevicesPath()) {
- return nil
- }
-
- // Load the directory listing
- devFiles, err := ioutil.ReadDir(c.DevicesPath())
- if err != nil {
- return err
- }
-
- for _, f := range devFiles {
- // Skip non-proxy devices
- if !strings.HasPrefix(f.Name(), "proxy.") {
- continue
- }
-
- // Kill the process
- devicePath := filepath.Join(c.DevicesPath(), f.Name())
- err = killProxyProc(devicePath)
- if err != nil {
- logger.Error("Failed removing proxy device", log.Ctx{"err": err, "path": devicePath})
- }
- }
-
- return nil
-}
-
-func (c *containerLXC) updateProxyDevice(devName string, m config.Device) error {
- if !c.IsRunning() {
- return fmt.Errorf("Can't update proxy device in stopped container")
- }
-
- devFileName := fmt.Sprintf("proxy.%s", devName)
- pidPath := filepath.Join(c.DevicesPath(), devFileName)
- err := killProxyProc(pidPath)
- if err != nil {
- return fmt.Errorf("Error occurred when removing old proxy device: %v", err)
- }
-
- return c.insertProxyDevice(devName, m)
-}
-
-func (c *containerLXC) restartProxyDevices() error {
- for _, name := range c.expandedDevices.DeviceNames() {
- m := c.expandedDevices[name]
- if m["type"] == "proxy" {
- err := c.insertProxyDevice(name, m)
- if err != nil {
- return fmt.Errorf("Error when starting proxy device '%s' for container %s: %v\n", name, c.name, err)
- }
- }
- }
-
- return nil
-}
-
// fillNetworkDevice takes a nic or infiniband device type and enriches it with automatically
// generated name and hwaddr properties if these are missing from the device.
func (c *containerLXC) fillNetworkDevice(name string, m config.Device) (config.Device, error) {
From 682251777843b6692555118e182ad31abe33cc1a Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Mon, 29 Jul 2019 17:06:31 +0100
Subject: [PATCH 25/42] test: Updates forkproxy tests
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/main_forkproxy_test.go | 94 +++++++++++++-------------
test/suites/container_devices_proxy.sh | 2 +-
2 files changed, 49 insertions(+), 47 deletions(-)
diff --git a/lxd/main_forkproxy_test.go b/lxd/main_forkproxy_test.go
index fd3bb9027e..76ec9685df 100644
--- a/lxd/main_forkproxy_test.go
+++ b/lxd/main_forkproxy_test.go
@@ -5,59 +5,61 @@ import (
"testing"
"github.com/stretchr/testify/require"
+
+ "github.com/lxc/lxd/lxd/device"
)
func TestParseAddr(t *testing.T) {
tests := []struct {
name string
address string
- expected *proxyAddress
+ expected *device.ProxyAddress
shouldFail bool
}{
// Port testing
{
"Single port",
"tcp:127.0.0.1:2000",
- &proxyAddress{
- "tcp",
- []string{"127.0.0.1:2000"},
- false,
+ &device.ProxyAddress{
+ ConnType: "tcp",
+ Addr: []string{"127.0.0.1:2000"},
+ Abstract: false,
},
false,
},
{
"Multiple ports",
"tcp:127.0.0.1:2000,2002",
- &proxyAddress{
- "tcp",
- []string{
+ &device.ProxyAddress{
+ ConnType: "tcp",
+ Addr: []string{
"127.0.0.1:2000",
"127.0.0.1:2002",
},
- false,
+ Abstract: false,
},
false,
},
{
"Port range",
"tcp:127.0.0.1:2000-2002",
- &proxyAddress{
- "tcp",
- []string{
+ &device.ProxyAddress{
+ ConnType: "tcp",
+ Addr: []string{
"127.0.0.1:2000",
"127.0.0.1:2001",
"127.0.0.1:2002",
},
- false,
+ Abstract: false,
},
false,
},
{
"Mixed ports and port ranges",
"tcp:127.0.0.1:2000,2002,3000-3003,4000-4003",
- &proxyAddress{
- "tcp",
- []string{
+ &device.ProxyAddress{
+ ConnType: "tcp",
+ Addr: []string{
"127.0.0.1:2000",
"127.0.0.1:2002",
"127.0.0.1:3000",
@@ -69,7 +71,7 @@ func TestParseAddr(t *testing.T) {
"127.0.0.1:4002",
"127.0.0.1:4003",
},
- false,
+ Abstract: false,
},
false,
},
@@ -77,30 +79,30 @@ func TestParseAddr(t *testing.T) {
{
"UDP",
"udp:127.0.0.1:2000",
- &proxyAddress{
- "udp",
- []string{"127.0.0.1:2000"},
- false,
+ &device.ProxyAddress{
+ ConnType: "udp",
+ Addr: []string{"127.0.0.1:2000"},
+ Abstract: false,
},
false,
},
{
"Unix socket",
"unix:/foobar",
- &proxyAddress{
- "unix",
- []string{"/foobar"},
- false,
+ &device.ProxyAddress{
+ ConnType: "unix",
+ Addr: []string{"/foobar"},
+ Abstract: false,
},
false,
},
{
"Abstract unix socket",
"unix:@/foobar",
- &proxyAddress{
- "unix",
- []string{"@/foobar"},
- true,
+ &device.ProxyAddress{
+ ConnType: "unix",
+ Addr: []string{"@/foobar"},
+ Abstract: true,
},
false,
},
@@ -114,40 +116,40 @@ func TestParseAddr(t *testing.T) {
{
"Valid IPv6 address (1)",
"tcp:[fd39:2561:7238:91b5:0:0:0:0]:2000",
- &proxyAddress{
- "tcp",
- []string{"[fd39:2561:7238:91b5:0:0:0:0]:2000"},
- false,
+ &device.ProxyAddress{
+ ConnType: "tcp",
+ Addr: []string{"[fd39:2561:7238:91b5:0:0:0:0]:2000"},
+ Abstract: false,
},
false,
},
{
"Valid IPv6 address (2)",
"tcp:[fd39:2561:7238:91b5::0]:2000",
- &proxyAddress{
- "tcp",
- []string{"[fd39:2561:7238:91b5::0]:2000"},
- false,
+ &device.ProxyAddress{
+ ConnType: "tcp",
+ Addr: []string{"[fd39:2561:7238:91b5::0]:2000"},
+ Abstract: false,
},
false,
},
{
"Valid IPv6 address (3)",
"tcp:[::1]:2000",
- &proxyAddress{
- "tcp",
- []string{"[::1]:2000"},
- false,
+ &device.ProxyAddress{
+ ConnType: "tcp",
+ Addr: []string{"[::1]:2000"},
+ Abstract: false,
},
false,
},
{
"Valid IPv6 address (4)",
"tcp:[::]:2000",
- &proxyAddress{
- "tcp",
- []string{"[::]:2000"},
- false,
+ &device.ProxyAddress{
+ ConnType: "tcp",
+ Addr: []string{"[::]:2000"},
+ Abstract: false,
},
false,
},
@@ -167,7 +169,7 @@ func TestParseAddr(t *testing.T) {
for i, tt := range tests {
log.Printf("Running test #%d: %s", i, tt.name)
- addr, err := proxyParseAddr(tt.address)
+ addr, err := device.ProxyParseAddr(tt.address)
if tt.shouldFail {
require.Error(t, err)
require.Nil(t, addr)
diff --git a/test/suites/container_devices_proxy.sh b/test/suites/container_devices_proxy.sh
index 436199c008..d3397baa43 100644
--- a/test/suites/container_devices_proxy.sh
+++ b/test/suites/container_devices_proxy.sh
@@ -163,7 +163,7 @@ container_devices_proxy_unix() {
lxc launch testimage proxyTester
# Initial test
- lxc config device add proxyTester proxyDev proxy "listen=unix:${HOST_SOCK}" connect=unix:/tmp/"lxdtest-$(basename "${LXD_DIR}").sock" bind=host
+ lxc config device add proxyTester proxyDev proxy "listen=unix:${HOST_SOCK}" uid=1234 gid=1234 security.uid=1234 security.gid=1234 connect=unix:/tmp/"lxdtest-$(basename "${LXD_DIR}").sock" bind=host
(
PID="$(lxc query /1.0/containers/proxyTester/state | jq .pid)"
cd "/proc/${PID}/root/tmp/" || exit
From 9666e93e895af76cf1f3008fba8b51e8f8decc96 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Tue, 30 Jul 2019 18:08:35 +0100
Subject: [PATCH 26/42] container: Removes proxy validation
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/container.go | 90 ++++++------------------------------------------
1 file changed, 11 insertions(+), 79 deletions(-)
diff --git a/lxd/container.go b/lxd/container.go
index 0f50837e38..69ae62350f 100644
--- a/lxd/container.go
+++ b/lxd/container.go
@@ -228,31 +228,6 @@ func containerValidDeviceConfigKey(t, k string) bool {
default:
return false
}
- case "proxy":
- switch k {
- case "bind":
- return true
- case "connect":
- return true
- case "gid":
- return true
- case "listen":
- return true
- case "mode":
- return true
- case "proxy_protocol":
- return true
- case "nat":
- return true
- case "security.gid":
- return true
- case "security.uid":
- return true
- case "uid":
- return true
- default:
- return false
- }
case "none":
return false
default:
@@ -347,19 +322,22 @@ func containerValidDevices(state *state.State, cluster *db.Cluster, devices conf
return fmt.Errorf("Invalid device type for device '%s'", name)
}
+ // Validate config using device interface.
+ _, err := device.New(&containerLXC{}, state, name, config.Device(m), nil, nil)
+ if err != device.ErrUnsupportedDevType {
+ if err != nil {
+ return err
+ }
+ continue
+ }
+
for k := range m {
- if m["type"] != "nic" && !containerValidDeviceConfigKey(m["type"], k) {
+ if !containerValidDeviceConfigKey(m["type"], k) {
return fmt.Errorf("Invalid device configuration key for %s: %s", m["type"], k)
}
}
- if m["type"] == "nic" {
- // Validate config using device interface.
- _, err := device.New(&containerLXC{}, state, name, config.Device(m), nil, nil)
- if err != nil {
- return err
- }
- } else if m["type"] == "infiniband" {
+ if m["type"] == "infiniband" {
if m["nictype"] == "" {
return fmt.Errorf("Missing nic type")
}
@@ -474,52 +452,6 @@ func containerValidDevices(state *state.State, cluster *db.Cluster, devices conf
if m["id"] != "" && (m["pci"] != "" || m["productid"] != "" || m["vendorid"] != "") {
return fmt.Errorf("Cannot use pci, productid or vendorid when id is set")
}
- } else if m["type"] == "proxy" {
- if m["listen"] == "" {
- return fmt.Errorf("Proxy device entry is missing the required \"listen\" property")
- }
-
- if m["connect"] == "" {
- return fmt.Errorf("Proxy device entry is missing the required \"connect\" property")
- }
-
- listenAddr, err := device.ProxyParseAddr(m["listen"])
- if err != nil {
- return err
- }
-
- connectAddr, err := device.ProxyParseAddr(m["connect"])
- if err != nil {
- return err
- }
-
- if len(connectAddr.Addr) > len(listenAddr.Addr) {
- // Cannot support single port -> multiple port
- return fmt.Errorf("Cannot map a single port to multiple ports")
- }
-
- if shared.IsTrue(m["proxy_protocol"]) && !strings.HasPrefix(m["connect"], "tcp") {
- return fmt.Errorf("The PROXY header can only be sent to tcp servers")
- }
-
- if (!strings.HasPrefix(m["listen"], "unix:") || strings.HasPrefix(m["listen"], "unix:@")) &&
- (m["uid"] != "" || m["gid"] != "" || m["mode"] != "") {
- return fmt.Errorf("Only proxy devices for non-abstract unix sockets can carry uid, gid, or mode properties")
- }
-
- if shared.IsTrue(m["nat"]) {
- if m["bind"] != "" && m["bind"] != "host" {
- return fmt.Errorf("Only host-bound proxies can use NAT")
- }
-
- // Support TCP <-> TCP and UDP <-> UDP
- if listenAddr.ConnType == "unix" || connectAddr.ConnType == "unix" ||
- listenAddr.ConnType != connectAddr.ConnType {
- return fmt.Errorf("Proxying %s <-> %s is not supported when using NAT",
- listenAddr.ConnType, connectAddr.ConnType)
- }
- }
-
} else if m["type"] == "none" {
continue
} else {
From 2a9d1941dac0934e8abca83510156766b7a73f38 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 1 Aug 2019 09:57:44 +0100
Subject: [PATCH 27/42] test: Adds nic bridged filtering tests for when DHCP is
disabled
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
...container_devices_nic_bridged_filtering.sh | 57 ++++++++++++++++++-
1 file changed, 55 insertions(+), 2 deletions(-)
diff --git a/test/suites/container_devices_nic_bridged_filtering.sh b/test/suites/container_devices_nic_bridged_filtering.sh
index ac10a28b82..a94c43b009 100644
--- a/test/suites/container_devices_nic_bridged_filtering.sh
+++ b/test/suites/container_devices_nic_bridged_filtering.sh
@@ -221,13 +221,13 @@ test_container_devices_nic_bridged_filtering() {
# Check IPv6 filter is present in ebtables.
if ! ebtables -L --Lmac2 --Lx | grep -e "2001:db8::2" ; then
- echo "IPv6 filter not applied as part of ipv6_filtering in ebtables"
+ echo "IPv6 ebtables filter not applied as part of ipv6_filtering in ebtables"
false
fi
# Check IPv6 filter is present in ip6tables.
if ! ip6tables -S -w -t filter | grep -e "20010db8000000000000000000000002" ; then
- echo "IPv6 filter not applied as part of ipv6_filtering in ip6tables"
+ echo "IPv6 ip6tables filter not applied as part of ipv6_filtering in ip6tables"
false
fi
@@ -314,6 +314,59 @@ test_container_devices_nic_bridged_filtering() {
lxc delete -f "${ctPrefix}A"
lxc delete -f "${ctPrefix}B"
+
+ # Check filtering works with non-DHCP statically defined IPs.
+ lxc network set "${brName}" ipv4.dhcp false
+ lxc network set "${brName}" ipv6.dhcp false
+ lxc network set "${brName}" ipv6.dhcp.stateful false
+ lxc init testimage "${ctPrefix}A" -p "${ctPrefix}"
+ lxc config device add "${ctPrefix}A" eth0 nic nictype=nic name=eth0 nictype=bridged parent="${brName}" ipv4.address=192.0.2.2 ipv6.address=2001:db8::2 security.ipv4_filtering=true security.ipv6_filtering=true
+ lxc start "${ctPrefix}A"
+
+ # Check MAC filter is present in ebtables.
+ ctAHost=$(lxc config get "${ctPrefix}A" volatile.eth0.host_name)
+ ctAMAC=$(lxc config get "${ctPrefix}A" volatile.eth0.hwaddr)
+ if ! ebtables -L --Lmac2 --Lx | grep -e "-s ! ${ctAMAC} -i ${ctAHost} -j DROP" ; then
+ echo "MAC ebtables filter not applied as part of ipv6_filtering in ebtables"
+ false
+ fi
+
+ # Check MAC filter is present in ip6tables.
+ macHex=$(echo "${ctAMAC}" |sed "s/://g")
+ if ! ip6tables -S -w -t filter | grep -e "${macHex}" ; then
+ echo "MAC ip6tables filter not applied as part of ipv6_filtering in ip6tables"
+ false
+ fi
+
+ # Check IPv4 filter is present in ebtables.
+ if ! ebtables -L --Lmac2 --Lx | grep -e "192.0.2.2" ; then
+ echo "IPv4 filter not applied as part of ipv4_filtering in ebtables"
+ false
+ fi
+
+ # Check IPv6 filter is present in ebtables.
+ if ! ebtables -L --Lmac2 --Lx | grep -e "2001:db8::2" ; then
+ echo "IPv6 filter not applied as part of ipv6_filtering in ebtables"
+ false
+ fi
+
+ # Check IPv6 filter is present in ip6tables.
+ if ! ip6tables -S -w -t filter | grep -e "20010db8000000000000000000000002" ; then
+ echo "IPv6 filter not applied as part of ipv6_filtering in ip6tables"
+ false
+ fi
+
+ # Check that you cannot remove static IPs with filtering enabled and DHCP disabled.
+ if lxc config device unset "${ctPrefix}A" eth0 ipv4.address ; then
+ echo "Shouldn't be able to unset IPv4 address with ipv4_filtering enabled and DHCPv4 disabled"
+ fi
+
+ if lxc config device unset "${ctPrefix}A" eth0 ipv6.address ; then
+ echo "Shouldn't be able to unset IPv6 address with ipv4_filtering enabled and DHCPv6 disabled"
+ fi
+
+ # Cleanup.
+ lxc delete -f "${ctPrefix}A"
lxc network delete "${brName}"
lxc profile delete "${ctPrefix}"
}
From a796c3180e40ae40680c5c0cde2de07912a860b3 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 1 Aug 2019 09:56:03 +0100
Subject: [PATCH 28/42] device/nic/bridged: Adds checks for DHCP being enabled
if no static IP set
Because IP filtering depends on dnsmasq's DHCP for dynamic IP allocation, it must be enabled in order to use IP filtering when no static IP is set in the device's config.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/nic_bridged.go | 45 +++++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 16 deletions(-)
diff --git a/lxd/device/nic_bridged.go b/lxd/device/nic_bridged.go
index f249f73901..831b0b5d82 100644
--- a/lxd/device/nic_bridged.go
+++ b/lxd/device/nic_bridged.go
@@ -559,6 +559,9 @@ func (d *nicBridged) setFilters() (err error) {
// Retrieve existing IPs, or allocate new ones if needed.
IPv4, IPv6, err := d.allocateFilterIPs()
+ if err != nil {
+ return err
+ }
// If anything goes wrong, clean up so we don't leave orphaned rules.
defer func() {
@@ -598,7 +601,7 @@ func (d *nicBridged) allocateFilterIPs() (net.IP, net.IP, error) {
if d.config["ipv4.address"] != "" {
IPv4 = net.ParseIP(d.config["ipv4.address"])
if IPv4 == nil {
- return IPv4, IPv6, fmt.Errorf("Invalid static IPv4 address %s", d.config["ipv4.address"])
+ return nil, nil, fmt.Errorf("Invalid static IPv4 address %s", d.config["ipv4.address"])
}
}
@@ -606,31 +609,41 @@ func (d *nicBridged) allocateFilterIPs() (net.IP, net.IP, error) {
if d.config["ipv6.address"] != "" {
IPv6 = net.ParseIP(d.config["ipv6.address"])
if IPv6 == nil {
- return IPv4, IPv6, fmt.Errorf("Invalid static IPv6 address %s", d.config["ipv6.address"])
+ return nil, nil, fmt.Errorf("Invalid static IPv6 address %s", d.config["ipv6.address"])
}
}
+ _, dbInfo, err := d.state.Cluster.NetworkGet(d.config["parent"])
+ if err != nil {
+ return nil, nil, err
+ }
+
+ netConfig := dbInfo.Config
+
+ // Check DHCPv4 is enabled on parent if dynamic IPv4 allocation is needed.
+ if shared.IsTrue(d.config["security.ipv4_filtering"]) && IPv4 == nil && netConfig["ipv4.dhcp"] != "" && !shared.IsTrue(netConfig["ipv4.dhcp"]) {
+ return nil, nil, fmt.Errorf("Cannot use security.ipv4_filtering as DHCPv4 is disabled on parent %s and no static IPv4 address set", d.config["parent"])
+ }
+
+ // Check DHCPv6 is enabled on parent if dynamic IPv6 allocation is needed.
+ if shared.IsTrue(d.config["security.ipv6_filtering"]) && IPv6 == nil && netConfig["ipv6.dhcp"] != "" && !shared.IsTrue(netConfig["ipv6.dhcp"]) {
+ return nil, nil, fmt.Errorf("Cannot use security.ipv6_filtering as DHCPv6 is disabled on parent %s and no static IPv6 address set", d.config["parent"])
+ }
+
dnsmasq.ConfigMutex.Lock()
defer dnsmasq.ConfigMutex.Unlock()
// Read current static IP allocation configured from dnsmasq host config (if exists).
curIPv4, curIPv6, err := d.getDHCPStaticIPs(d.config["parent"], d.instance.Name())
if err != nil && !os.IsNotExist(err) {
- return IPv4, IPv6, err
- }
-
- _, dbInfo, err := d.state.Cluster.NetworkGet(d.config["parent"])
- if err != nil {
- return IPv4, IPv6, err
+ return nil, nil, err
}
- netConfig := dbInfo.Config
-
// If no static IPv4, then check if there is a valid static DHCP IPv4 address defined.
if IPv4 == nil && curIPv4.IP != nil {
_, subnet, err := net.ParseCIDR(netConfig["ipv4.address"])
if err != nil {
- return IPv4, IPv6, err
+ return nil, nil, err
}
// Check the existing static DHCP IP is still valid in the subnet & ranges, if not
@@ -661,14 +674,14 @@ func (d *nicBridged) allocateFilterIPs() (net.IP, net.IP, error) {
// Get existing allocations in network.
IPv4Allocs, IPv6Allocs, err := d.getDHCPAllocatedIPs(d.config["parent"])
if err != nil {
- return IPv4, IPv6, err
+ return nil, nil, err
}
// Allocate a new IPv4 address is IPv4 filtering enabled.
if IPv4 == nil && shared.IsTrue(d.config["security.ipv4_filtering"]) {
IPv4, err = d.getDHCPFreeIPv4(IPv4Allocs, netConfig, d.instance.Name(), d.config["hwaddr"])
if err != nil {
- return IPv4, IPv6, err
+ return nil, nil, err
}
}
@@ -676,7 +689,7 @@ func (d *nicBridged) allocateFilterIPs() (net.IP, net.IP, error) {
if IPv6 == nil && shared.IsTrue(d.config["security.ipv6_filtering"]) {
IPv6, err = d.getDHCPFreeIPv6(IPv6Allocs, netConfig, d.instance.Name(), d.config["hwaddr"])
if err != nil {
- return IPv4, IPv6, err
+ return nil, nil, err
}
}
}
@@ -695,12 +708,12 @@ func (d *nicBridged) allocateFilterIPs() (net.IP, net.IP, error) {
err = dnsmasq.UpdateStaticEntry(d.config["parent"], d.instance.Project(), d.instance.Name(), netConfig, d.config["hwaddr"], IPv4Str, IPv6Str)
if err != nil {
- return IPv4, IPv6, err
+ return nil, nil, err
}
err = dnsmasq.Kill(d.config["parent"], true)
if err != nil {
- return IPv4, IPv6, err
+ return nil, nil, err
}
}
From 015cbada88e6fbef679749d83a7df728eb72f665 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 1 Aug 2019 10:59:54 +0100
Subject: [PATCH 29/42] container/lxc/infiniband: Moves fillNetworkDevice into
startInfiniband
Part of moving device specific setup out of container_lxc.go
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/container_lxc_infiniband.go | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/lxd/container_lxc_infiniband.go b/lxd/container_lxc_infiniband.go
index 908038aa3b..f7b4310870 100644
--- a/lxd/container_lxc_infiniband.go
+++ b/lxd/container_lxc_infiniband.go
@@ -134,6 +134,11 @@ func (c *containerLXC) getInfinibandReserved(m config.Device) (map[string]struct
}
func (c *containerLXC) startInfiniband(networkidx int, deviceName string, m config.Device) error {
+ m, err := c.fillNetworkDevice(deviceName, m)
+ if err != nil {
+ return err
+ }
+
infiniband, err := deviceLoadInfiniband()
if err != nil {
return err
From 3f4a4dfbcef4fabe79ede662e35338f6061169ea Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 1 Aug 2019 11:21:37 +0100
Subject: [PATCH 30/42] device/nic: Updates comments to remove "container"
references
Also clarifies comments on Start() and Stop() functions.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/nic_bridged.go | 8 ++++----
lxd/device/nic_ipvlan.go | 4 ++--
lxd/device/nic_macvlan.go | 4 ++--
lxd/device/nic_p2p.go | 4 ++--
lxd/device/nic_physical.go | 6 +++---
lxd/device/nic_sriov.go | 12 ++++++------
6 files changed, 19 insertions(+), 19 deletions(-)
diff --git a/lxd/device/nic_bridged.go b/lxd/device/nic_bridged.go
index 831b0b5d82..72a2886980 100644
--- a/lxd/device/nic_bridged.go
+++ b/lxd/device/nic_bridged.go
@@ -116,7 +116,7 @@ func (d *nicBridged) Add() error {
return nil
}
-// Start is run when the device is added to the instance and instance is starting or running.
+// Start is run when the device is added to a running instance or instance is starting up.
func (d *nicBridged) Start() (*RunConfig, error) {
err := d.validateEnvironment()
if err != nil {
@@ -237,7 +237,7 @@ func (d *nicBridged) Update(oldConfig config.Device, isRunning bool) error {
return nil
}
-// Stop is run when the device is removed from the instance.
+// Stop is run when the device or instance is stopped.
func (d *nicBridged) Stop() error {
defer d.volatileSet(map[string]string{
"host_name": "",
@@ -267,7 +267,7 @@ func (d *nicBridged) Stop() error {
return nil
}
-// Remove is run when the instance is deleted.
+// Remove is run when the device is removed from the instance or the instance is deleted.
func (d *nicBridged) Remove() error {
err := d.networkClearLease(d.instance.Name(), d.config["parent"], d.config["hwaddr"], clearLeaseAll)
if err != nil {
@@ -428,7 +428,7 @@ func (d *nicBridged) removeFilters(m config.Device) error {
return nil
}
-// getDHCPStaticContainerIPs retrieves the dnsmasq statically allocated IPs for a instance.
+// getDHCPStaticIPs retrieves the dnsmasq statically allocated IPs for a instance.
// Returns IPv4 and IPv6 dhcpAllocation structs respectively.
func (d *nicBridged) getDHCPStaticIPs(network string, instanceName string) (dhcpAllocation, dhcpAllocation, error) {
var IPv4, IPv6 dhcpAllocation
diff --git a/lxd/device/nic_ipvlan.go b/lxd/device/nic_ipvlan.go
index c380d549d3..8ea90a2e85 100644
--- a/lxd/device/nic_ipvlan.go
+++ b/lxd/device/nic_ipvlan.go
@@ -107,7 +107,7 @@ func (d *nicIPVLAN) validateEnvironment() error {
return nil
}
-// Start is run when the device is added to the container.
+// Start is run when the instance is starting up (IPVLAN doesn't support hot plugging).
func (d *nicIPVLAN) Start() (*RunConfig, error) {
err := d.validateEnvironment()
if err != nil {
@@ -208,7 +208,7 @@ func (d *nicIPVLAN) setupParentSysctls(parentName string) error {
return nil
}
-// Stop is run when the device is removed from the container.
+// Stop is run when the device or instance is stopped.
func (d *nicIPVLAN) Stop() error {
defer d.volatileSet(map[string]string{
"last_state.created": "",
diff --git a/lxd/device/nic_macvlan.go b/lxd/device/nic_macvlan.go
index 1aabbeb660..90bc69514d 100644
--- a/lxd/device/nic_macvlan.go
+++ b/lxd/device/nic_macvlan.go
@@ -41,7 +41,7 @@ func (d *nicMACVLAN) validateEnvironment() error {
return nil
}
-// Start is run when the device is added to the container.
+// Start is run when the device is added to a running instance or instance is starting up.
func (d *nicMACVLAN) Start() (*RunConfig, error) {
err := d.validateEnvironment()
if err != nil {
@@ -109,7 +109,7 @@ func (d *nicMACVLAN) Start() (*RunConfig, error) {
return &runConf, nil
}
-// Stop is run when the device is removed from the container.
+// Stop is run when the device or instance is stopped.
func (d *nicMACVLAN) Stop() error {
defer d.volatileSet(map[string]string{
"host_name": "",
diff --git a/lxd/device/nic_p2p.go b/lxd/device/nic_p2p.go
index ea6909c816..fcdccdf8d3 100644
--- a/lxd/device/nic_p2p.go
+++ b/lxd/device/nic_p2p.go
@@ -52,7 +52,7 @@ func (d *nicP2P) CanHotPlug() (bool, []string) {
return true, []string{"limits.ingress", "limits.egress", "limits.max", "ipv4.routes", "ipv6.routes"}
}
-// Start is run when the device is added to the container.
+// Start is run when the device is added to a running instance or instance is starting up.
func (d *nicP2P) Start() (*RunConfig, error) {
err := d.validateEnvironment()
if err != nil {
@@ -116,7 +116,7 @@ func (d *nicP2P) Update(oldConfig config.Device, isRunning bool) error {
return nil
}
-// Stop is run when the device is removed from the container.
+// Stop is run when the device or instance is stopped.
func (d *nicP2P) Stop() error {
defer d.volatileSet(map[string]string{
"host_name": "",
diff --git a/lxd/device/nic_physical.go b/lxd/device/nic_physical.go
index 5faa6a2ec4..fbed8544e9 100644
--- a/lxd/device/nic_physical.go
+++ b/lxd/device/nic_physical.go
@@ -48,7 +48,7 @@ func (d *nicPhysical) validateEnvironment() error {
return nil
}
-// Start is run when the device is added to the container.
+// Start is run when the device is added to a running instance or instance is starting up.
func (d *nicPhysical) Start() (*RunConfig, error) {
err := d.validateEnvironment()
if err != nil {
@@ -75,7 +75,7 @@ func (d *nicPhysical) Start() (*RunConfig, error) {
}()
// If we didn't create the device we should track various properties so we can
- // restore them when the container is stopped or the device is detached.
+ // restore them when the instance is stopped or the device is detached.
if createdDev == false {
err = networkSnapshotPhysicalNic(saveData["host_name"], saveData)
if err != nil {
@@ -115,7 +115,7 @@ func (d *nicPhysical) Start() (*RunConfig, error) {
return &runConf, nil
}
-// Stop is run when the device is removed from the container.
+// Stop is run when the device or instance is stopped.
func (d *nicPhysical) Stop() error {
defer d.volatileSet(map[string]string{
"host_name": "",
diff --git a/lxd/device/nic_sriov.go b/lxd/device/nic_sriov.go
index ebad700af6..f66625c28f 100644
--- a/lxd/device/nic_sriov.go
+++ b/lxd/device/nic_sriov.go
@@ -58,7 +58,7 @@ func (d *nicSRIOV) validateEnvironment() error {
return nil
}
-// Start is run when the device is added to the container.
+// Start is run when the device is added to a running instance or instance is starting up.
func (d *nicSRIOV) Start() (*RunConfig, error) {
err := d.validateEnvironment()
if err != nil {
@@ -141,7 +141,7 @@ func (d *nicSRIOV) Start() (*RunConfig, error) {
return &runConf, nil
}
-// Stop is run when the device is removed from the container.
+// Stop is run when the device or instance is stopped.
func (d *nicSRIOV) Stop() error {
defer d.volatileSet(map[string]string{
"host_name": "",
@@ -297,7 +297,7 @@ func (d *nicSRIOV) setupSriovParent(vfDevice string, vfID int, volatile map[stri
volatile["last_state.vf.vlan"] = fmt.Sprintf("%d", vfInfo.vlan)
volatile["last_state.vf.spoofcheck"] = fmt.Sprintf("%t", vfInfo.spoofcheck)
- // Record the host interface we represents the VF device which we will move into container.
+ // Record the host interface we represents the VF device which we will move into instance.
volatile["host_name"] = vfDevice
volatile["last_state.created"] = "false" // Indicates don't delete device at stop time.
@@ -366,7 +366,7 @@ func (d *nicSRIOV) setupSriovParent(vfDevice string, vfID int, volatile map[stri
return err
}
- // Ensure spoof checking is disabled if not enabled in container.
+ // Ensure spoof checking is disabled if not enabled in instance.
_, err = shared.RunCommand("ip", "link", "set", "dev", d.config["parent"], "vf", volatile["last_state.vf.id"], "spoofchk", "off")
if err != nil {
return err
@@ -499,7 +499,7 @@ func (d *nicSRIOV) networkDeviceBindWait(devName string) error {
return fmt.Errorf("Bind of interface \"%s\" took too long", devName)
}
-// restoreSriovParent restores SR-IOV parent device settings when removed from a container using the
+// restoreSriovParent restores SR-IOV parent device settings when removed from an instance using the
// volatile data that was stored when the device was first added with setupSriovParent().
func (d *nicSRIOV) restoreSriovParent(volatile map[string]string) error {
// Nothing to do if we don't know the original device name or the VF ID.
@@ -566,7 +566,7 @@ func (d *nicSRIOV) restoreSriovParent(volatile map[string]string) error {
return err
}
- // Wait for VF driver to be reloaded, this will remove the VF interface from the container
+ // Wait for VF driver to be reloaded, this will remove the VF interface from the instance
// and it will re-appear on the host. Unfortunately the time between sending the bind event
// to the nic and it actually appearing on the host is non-zero, so we need to watch and wait,
// otherwise next step of restoring MAC and MTU settings in restorePhysicalNic will fail.
From ae37230152133bd8ea1afdc1c29fa89402823bf6 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 1 Aug 2019 16:37:45 +0100
Subject: [PATCH 31/42] device/infiniband/physical: Adds infiniband physical
device implementation
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/infiniband_physical.go | 114 ++++++++++++++++++++++++++++++
1 file changed, 114 insertions(+)
create mode 100644 lxd/device/infiniband_physical.go
diff --git a/lxd/device/infiniband_physical.go b/lxd/device/infiniband_physical.go
new file mode 100644
index 0000000000..6380505705
--- /dev/null
+++ b/lxd/device/infiniband_physical.go
@@ -0,0 +1,114 @@
+package device
+
+import (
+ "fmt"
+
+ "github.com/lxc/lxd/lxd/device/config"
+ "github.com/lxc/lxd/lxd/instance"
+ "github.com/lxc/lxd/shared"
+)
+
+type infinibandPhysical struct {
+ deviceCommon
+}
+
+// validateConfig checks the supplied config for correctness.
+func (d *infinibandPhysical) validateConfig() error {
+ if d.instance.Type() != instance.TypeContainer {
+ return ErrUnsupportedDevType
+ }
+
+ requiredFields := []string{"parent"}
+ optionalFields := []string{
+ "name",
+ "mtu",
+ "hwaddr",
+ }
+ err := config.ValidateDevice(nicValidationRules(requiredFields, optionalFields), d.config)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// validateEnvironment checks the runtime environment for correctness.
+func (d *infinibandPhysical) validateEnvironment() error {
+ if d.config["name"] == "" {
+ return fmt.Errorf("Requires name property to start")
+ }
+
+ if !shared.PathExists(fmt.Sprintf("/sys/class/net/%s", d.config["parent"])) {
+ return fmt.Errorf("Parent device '%s' doesn't exist", d.config["parent"])
+ }
+
+ return nil
+}
+
+// Start is run when the device is added to a running instance or instance is starting up.
+func (d *infinibandPhysical) Start() (*RunConfig, error) {
+ err := d.validateEnvironment()
+ if err != nil {
+ return nil, err
+ }
+
+ saveData := make(map[string]string)
+
+ devices, err := infinibandLoadDevices()
+ if err != nil {
+ return nil, err
+ }
+
+ saveData["host_name"] = d.config["parent"]
+ ifDev, ok := devices[saveData["host_name"]]
+ if !ok {
+ return nil, fmt.Errorf("Specified infiniband device \"%s\" not found", saveData["host_name"])
+ }
+
+ idmapSet, err := d.instance.CurrentIdmap()
+ if err != nil {
+ return nil, err
+ }
+
+ runConf := RunConfig{}
+
+ err = infinibandAddDevices(d.state, idmapSet, d.instance.DevicesPath(), d.name, &ifDev, &runConf)
+ if err != nil {
+ return nil, err
+ }
+
+ // Set the MAC address.
+ if d.config["hwaddr"] != "" {
+ _, err := shared.RunCommand("ip", "link", "set", "dev", saveData["host_name"], "address", d.config["hwaddr"])
+ if err != nil {
+ return nil, fmt.Errorf("Failed to set the MAC address: %s", err)
+ }
+ }
+
+ // Set the MTU.
+ if d.config["mtu"] != "" {
+ _, err := shared.RunCommand("ip", "link", "set", "dev", saveData["host_name"], "mtu", d.config["mtu"])
+ if err != nil {
+ return nil, fmt.Errorf("Failed to set the MTU: %s", err)
+ }
+ }
+
+ err = d.volatileSet(saveData)
+ if err != nil {
+ return nil, err
+ }
+
+ runConf.NetworkInterface = []RunConfigItem{
+ {Key: "name", Value: d.config["name"]},
+ {Key: "type", Value: "phys"},
+ {Key: "flags", Value: "up"},
+ {Key: "link", Value: saveData["host_name"]},
+ }
+
+ return &runConf, nil
+}
+
+// Stop is run when the device or instance is stopped.
+func (d *infinibandPhysical) Stop() error {
+ return nil
+}
From c62ebd92d0e5ffd20b2b9cf2f89d0f561a0cc969 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 1 Aug 2019 16:39:03 +0100
Subject: [PATCH 32/42] device/utils/disk: Adds disk management utils file
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device_utils_disk.go | 103 ++++++++++++++++++++++++++++++++
1 file changed, 103 insertions(+)
create mode 100644 lxd/device/device_utils_disk.go
diff --git a/lxd/device/device_utils_disk.go b/lxd/device/device_utils_disk.go
new file mode 100644
index 0000000000..ddb9cc89d3
--- /dev/null
+++ b/lxd/device/device_utils_disk.go
@@ -0,0 +1,103 @@
+package device
+
+import (
+ "fmt"
+ "strings"
+
+ "golang.org/x/sys/unix"
+
+ "github.com/lxc/lxd/shared"
+)
+
+func BlockFsDetect(dev string) (string, error) {
+ out, err := shared.RunCommand("blkid", "-s", "TYPE", "-o", "value", dev)
+ if err != nil {
+ return "", err
+ }
+
+ return strings.TrimSpace(out), nil
+}
+
+func IsBlockdev(path string) bool {
+ // Get a stat struct from the provided path
+ stat := unix.Stat_t{}
+ err := unix.Stat(path, &stat)
+ if err != nil {
+ return false
+ }
+
+ // Check if it's a block device
+ if stat.Mode&unix.S_IFMT == unix.S_IFBLK {
+ return true
+ }
+
+ // Not a device
+ return false
+}
+
+func DiskMount(srcPath string, dstPath string, readonly bool, recursive bool, propagation string) error {
+ var err error
+
+ // Prepare the mount flags
+ flags := 0
+ if readonly {
+ flags |= unix.MS_RDONLY
+ }
+
+ // Detect the filesystem
+ fstype := "none"
+ if IsBlockdev(srcPath) {
+ fstype, err = BlockFsDetect(srcPath)
+ if err != nil {
+ return err
+ }
+ } else {
+ flags |= unix.MS_BIND
+ if propagation != "" {
+ switch propagation {
+ case "private":
+ flags |= unix.MS_PRIVATE
+ case "shared":
+ flags |= unix.MS_SHARED
+ case "slave":
+ flags |= unix.MS_SLAVE
+ case "unbindable":
+ flags |= unix.MS_UNBINDABLE
+ case "rprivate":
+ flags |= unix.MS_PRIVATE | unix.MS_REC
+ case "rshared":
+ flags |= unix.MS_SHARED | unix.MS_REC
+ case "rslave":
+ flags |= unix.MS_SLAVE | unix.MS_REC
+ case "runbindable":
+ flags |= unix.MS_UNBINDABLE | unix.MS_REC
+ default:
+ return fmt.Errorf("Invalid propagation mode '%s'", propagation)
+ }
+ }
+
+ if recursive {
+ flags |= unix.MS_REC
+ }
+ }
+
+ // Mount the filesystem
+ if err = unix.Mount(srcPath, dstPath, fstype, uintptr(flags), ""); err != nil {
+ return fmt.Errorf("Unable to mount %s at %s: %s", srcPath, dstPath, err)
+ }
+
+ // Remount bind mounts in readonly mode if requested
+ if readonly == true && flags&unix.MS_BIND == unix.MS_BIND {
+ flags = unix.MS_RDONLY | unix.MS_BIND | unix.MS_REMOUNT
+ if err = unix.Mount("", dstPath, fstype, uintptr(flags), ""); err != nil {
+ return fmt.Errorf("Unable to mount %s in readonly mode: %s", dstPath, err)
+ }
+ }
+
+ flags = unix.MS_REC | unix.MS_SLAVE
+ if err = unix.Mount("", dstPath, "", uintptr(flags), ""); err != nil {
+ return fmt.Errorf("unable to make mount %s private: %s", dstPath, err)
+ }
+
+ return nil
+}
From c90ba7c426a84b045ab41264be7abba4901e9e78 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 1 Aug 2019 16:39:21 +0100
Subject: [PATCH 33/42] device/utils/infiniband: Adds infiniband utils file
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device_utils_infiniband.go | 406 ++++++++++++++++++++++++++
1 file changed, 406 insertions(+)
create mode 100644 lxd/device/device_utils_infiniband.go
diff --git a/lxd/device/device_utils_infiniband.go b/lxd/device/device_utils_infiniband.go
new file mode 100644
index 0000000000..41c4c9f02e
--- /dev/null
+++ b/lxd/device/device_utils_infiniband.go
@@ -0,0 +1,406 @@
+package device
+
+import (
+ "bytes"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+
+ "github.com/lxc/lxd/lxd/device/config"
+ "github.com/lxc/lxd/lxd/state"
+ "github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/idmap"
+)
+
+const SCIB string = "/sys/class/infiniband"
+const SCNET string = "/sys/class/net"
+
+type IBF struct {
+ // port the function belongs to
+ Port int64
+
+ // name of the {physical,virtual} function
+ Fun string
+
+ // whether this is a physical (true) or virtual (false) function
+ PF bool
+
+ // device of the function
+ Device string
+
+ // uverb device of the function
+ PerPortDevices []string
+ PerFunDevices []string
+}
+
+func infinibandLoadDevices() (map[string]IBF, error) {
+ // check if there are any infiniband devices
+ fscib, err := os.Open(SCIB)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, os.ErrNotExist
+ }
+ return nil, err
+ }
+ defer fscib.Close()
+
+ // eg.g. mlx_i for i = 0, 1, ..., n
+ IBDevNames, err := fscib.Readdirnames(-1)
+ if err != nil {
+ return nil, err
+ }
+
+ if len(IBDevNames) == 0 {
+ return nil, os.ErrNotExist
+ }
+
+ // retrieve all network device names
+ fscnet, err := os.Open(SCNET)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, os.ErrNotExist
+ }
+ return nil, err
+ }
+ defer fscnet.Close()
+
+ // retrieve all network devices
+ NetDevNames, err := fscnet.Readdirnames(-1)
+ if err != nil {
+ return nil, err
+ }
+
+ if len(NetDevNames) == 0 {
+ return nil, os.ErrNotExist
+ }
+
+ UseableDevices := make(map[string]IBF)
+ for _, IBDevName := range IBDevNames {
+ IBDevResourceFile := fmt.Sprintf("/sys/class/infiniband/%s/device/resource", IBDevName)
+ IBDevResourceBuf, err := ioutil.ReadFile(IBDevResourceFile)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, NetDevName := range NetDevNames {
+ NetDevResourceFile := fmt.Sprintf("/sys/class/net/%s/device/resource", NetDevName)
+ NetDevResourceBuf, err := ioutil.ReadFile(NetDevResourceFile)
+ if err != nil {
+ if os.IsNotExist(err) {
+ continue
+ }
+ return nil, err
+ }
+
+ // If the device and the VF have the same address space
+ // they belong together.
+ if bytes.Compare(IBDevResourceBuf, NetDevResourceBuf) != 0 {
+ continue
+ }
+
+ // Now let's find the ports.
+ IBDevID := fmt.Sprintf("/sys/class/net/%s/dev_id", NetDevName)
+ IBDevPort := fmt.Sprintf("/sys/class/net/%s/dev_port", NetDevName)
+ DevIDBuf, err := ioutil.ReadFile(IBDevID)
+ if err != nil {
+ if os.IsNotExist(err) {
+ continue
+ }
+ return nil, err
+ }
+
+ DevIDString := strings.TrimSpace(string(DevIDBuf))
+ DevIDPort, err := strconv.ParseInt(DevIDString, 0, 64)
+ if err != nil {
+ return nil, err
+ }
+
+ DevPort := int64(0)
+ DevPortBuf, err := ioutil.ReadFile(IBDevPort)
+ if err != nil {
+ if !os.IsNotExist(err) {
+ return nil, err
+ }
+ } else {
+ DevPortString := strings.TrimSpace(string(DevPortBuf))
+ DevPort, err = strconv.ParseInt(DevPortString, 0, 64)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ Port := DevIDPort
+ if DevPort > DevIDPort {
+ Port = DevPort
+ }
+ Port++
+
+ NewIBF := IBF{
+ Port: Port,
+ Fun: IBDevName,
+ Device: NetDevName,
+ }
+
+ // identify the /dev/infiniband/uverb<idx> device
+ tmp := []string{}
+ IBUverb := fmt.Sprintf("/sys/class/net/%s/device/infiniband_verbs", NetDevName)
+ fuverb, err := os.Open(IBUverb)
+ if err != nil {
+ if !os.IsNotExist(err) {
+ return nil, err
+ }
+ } else {
+ defer fuverb.Close()
+
+ // optional: retrieve all network devices
+ tmp, err = fuverb.Readdirnames(-1)
+ if err != nil {
+ return nil, err
+ }
+
+ if len(tmp) == 0 {
+ return nil, os.ErrNotExist
+ }
+ }
+ for _, v := range tmp {
+ if strings.Index(v, "-") != -1 {
+ return nil, fmt.Errorf("Infiniband character device \"%s\" contains \"-\". Cannot guarantee unique encoding", v)
+ }
+ NewIBF.PerPortDevices = append(NewIBF.PerPortDevices, v)
+ }
+
+ // identify the /dev/infiniband/ucm<idx> device
+ tmp = []string{}
+ IBcm := fmt.Sprintf("/sys/class/net/%s/device/infiniband_ucm", NetDevName)
+ fcm, err := os.Open(IBcm)
+ if err != nil {
+ if !os.IsNotExist(err) {
+ return nil, err
+ }
+ } else {
+ defer fcm.Close()
+
+ // optional: retrieve all network devices
+ tmp, err = fcm.Readdirnames(-1)
+ if err != nil {
+ return nil, err
+ }
+
+ if len(tmp) == 0 {
+ return nil, os.ErrNotExist
+ }
+ }
+ for _, v := range tmp {
+ if strings.Index(v, "-") != -1 {
+ return nil, fmt.Errorf("Infiniband character device \"%s\" contains \"-\". Cannot guarantee unique encoding", v)
+ }
+ devPath := fmt.Sprintf("/dev/infiniband/%s", v)
+ NewIBF.PerPortDevices = append(NewIBF.PerPortDevices, devPath)
+ }
+
+ // identify the /dev/infiniband/{issm,umad}<idx> devices
+ IBmad := fmt.Sprintf("/sys/class/net/%s/device/infiniband_mad", NetDevName)
+ ents, err := ioutil.ReadDir(IBmad)
+ if err != nil {
+ if !os.IsNotExist(err) {
+ return nil, err
+ }
+ } else {
+ for _, ent := range ents {
+ IBmadPort := fmt.Sprintf("%s/%s/port", IBmad, ent.Name())
+ portBuf, err := ioutil.ReadFile(IBmadPort)
+ if err != nil {
+ if !os.IsNotExist(err) {
+ return nil, err
+ }
+ continue
+ }
+
+ portStr := strings.TrimSpace(string(portBuf))
+ PortMad, err := strconv.ParseInt(portStr, 0, 64)
+ if err != nil {
+ return nil, err
+ }
+
+ if PortMad != NewIBF.Port {
+ continue
+ }
+
+ if strings.Index(ent.Name(), "-") != -1 {
+ return nil, fmt.Errorf("Infiniband character device \"%s\" contains \"-\". Cannot guarantee unique encoding", ent.Name())
+ }
+
+ NewIBF.PerFunDevices = append(NewIBF.PerFunDevices, ent.Name())
+ }
+ }
+
+ // figure out whether this is a physical function
+ IBPF := fmt.Sprintf("/sys/class/net/%s/device/physfn", NetDevName)
+ NewIBF.PF = !shared.PathExists(IBPF)
+
+ UseableDevices[NetDevName] = NewIBF
+ }
+ }
+
+ // check whether the device is an infiniband device
+ return UseableDevices, nil
+}
+
+func infinibandAddDevices(s *state.State, idmapSet *idmap.IdmapSet, devicesPath string, deviceName string, ifDev *IBF, runConf *RunConfig) error {
+ // load all devices
+ dents, err := ioutil.ReadDir(devicesPath)
+ if err != nil {
+ if !os.IsNotExist(err) {
+ return err
+ }
+ }
+
+ err = infinibandAddDevicesPerPort(s, idmapSet, devicesPath, deviceName, ifDev, dents, runConf)
+ if err != nil {
+ return err
+ }
+
+ err = infinibandAddDevicesPerFun(s, idmapSet, devicesPath, deviceName, ifDev, runConf)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func infinibandAddDevicesPerPort(s *state.State, idmapSet *idmap.IdmapSet, devicesPath string, deviceName string, ifDev *IBF, devices []os.FileInfo, runConf *RunConfig) error {
+ for _, unixCharDev := range ifDev.PerPortDevices {
+ destPath := fmt.Sprintf("/dev/infiniband/%s", unixCharDev)
+ relDestPath := destPath[1:]
+ devPrefix := fmt.Sprintf("infiniband.unix.%s", deviceName)
+
+ // Unix device
+ dummyDevice := config.Device{
+ "source": destPath,
+ }
+
+ deviceExists := false
+ // only handle infiniband.unix.<device-name>.
+ prefix := fmt.Sprintf("infiniband.unix.")
+ for _, ent := range devices {
+
+ // skip non infiniband.unix.<device-name> devices
+ devName := ent.Name()
+ if !strings.HasPrefix(devName, prefix) {
+ continue
+ }
+
+ // extract the path inside the container
+ idx := strings.LastIndex(devName, ".")
+ if idx == -1 {
+ return fmt.Errorf("Invalid infiniband device name \"%s\"", devName)
+ }
+ rPath := devName[idx+1:]
+ rPath = strings.Replace(rPath, "-", "/", -1)
+ if rPath != relDestPath {
+ continue
+ }
+
+ deviceExists = true
+ break
+ }
+
+ // move this logic into the container live attach section.
+ /*if inject && !deviceExists {
+ err := c.insertUnixDevice(devPrefix, dummyDevice, false)
+ if err != nil {
+ return err
+ }
+ continue
+ }*/
+
+ paths, err := UnixCreateDevice(s, idmapSet, devicesPath, devPrefix, dummyDevice, false)
+ if err != nil {
+ return err
+ }
+ devPath := paths[0]
+
+ if deviceExists {
+ continue
+ }
+
+ // Instruct liblxc to perform the mount.
+ runConf.Mounts = append(runConf.Mounts, RunConfigItem{
+ Key: "lxc.mount.entry",
+ Value: fmt.Sprintf("%s %s none bind,create=file 0 0",
+ shared.EscapePathFstab(devPath),
+ shared.EscapePathFstab(relDestPath)),
+ })
+
+ // move this logic into the container cgroup attach section.
+ //if c.isCurrentlyPrivileged() && !c.state.OS.RunningInUserNS && c.state.OS.CGroupDevicesController {
+ // Add the new device cgroup rule
+ dType, dMajor, dMinor, err := UnixGetDeviceAttributes(devPath)
+ if err != nil {
+ return err
+ }
+
+ // Instruct liblxc to setup the cgroup rule.
+ runConf.CGroups = append(runConf.CGroups, RunConfigItem{
+ Key: "lxc.cgroup.devices.allow",
+ Value: fmt.Sprintf("%s %d:%d rwm", dType, dMajor, dMinor),
+ })
+ }
+
+ return nil
+}
+
+func infinibandAddDevicesPerFun(s *state.State, idmapSet *idmap.IdmapSet, devicesPath string, deviceName string, ifDev *IBF, runConf *RunConfig) error {
+ for _, unixCharDev := range ifDev.PerFunDevices {
+ destPath := fmt.Sprintf("/dev/infiniband/%s", unixCharDev)
+ uniqueDevPrefix := fmt.Sprintf("infiniband.unix.%s", deviceName)
+ relativeDestPath := fmt.Sprintf("dev/infiniband/%s", unixCharDev)
+ uniqueDevName := fmt.Sprintf("%s.%s", uniqueDevPrefix, strings.Replace(relativeDestPath, "/", "-", -1))
+ hostDevPath := filepath.Join(devicesPath, uniqueDevName)
+
+ dummyDevice := config.Device{
+ "source": destPath,
+ }
+
+ // move this logic into the container live attach section.
+ /*if inject {
+ err := c.insertUnixDevice(uniqueDevPrefix, dummyDevice, false)
+ if err != nil {
+ return err
+ }
+ continue
+ }*/
+
+ // Instruct liblxc to perform the mount.
+ runConf.Mounts = append(runConf.Mounts, RunConfigItem{
+ Key: "lxc.mount.entry",
+ Value: fmt.Sprintf("%s %s none bind,create=file 0 0", hostDevPath, relativeDestPath),
+ })
+
+ paths, err := UnixCreateDevice(s, idmapSet, devicesPath, uniqueDevPrefix, dummyDevice, false)
+ if err != nil {
+ return err
+ }
+ devPath := paths[0]
+
+ // move this logic into the container cgroup attach section.
+ //if c.isCurrentlyPrivileged() && !s.OS.RunningInUserNS && s.OS.CGroupDevicesController {
+
+ // Add the new device cgroup rule
+ dType, dMajor, dMinor, err := UnixGetDeviceAttributes(devPath)
+ if err != nil {
+ return err
+ }
+
+ // Instruct liblxc to setup the cgroup rule.
+ runConf.CGroups = append(runConf.CGroups, RunConfigItem{
+ Key: "lxc.cgroup.devices.allow",
+ Value: fmt.Sprintf("%s %d:%d rwm", dType, dMajor, dMinor),
+ })
+ }
+
+ return nil
+}
From 5523d4bc7356e11673d47dc6e0c2ff5a1dc2c1c0 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 1 Aug 2019 16:39:36 +0100
Subject: [PATCH 34/42] devices/utils/unix: Adds unix device utils file
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device_utils_unix.go | 194 ++++++++++++++++++++++++++++++++
1 file changed, 194 insertions(+)
create mode 100644 lxd/device/device_utils_unix.go
diff --git a/lxd/device/device_utils_unix.go b/lxd/device/device_utils_unix.go
new file mode 100644
index 0000000000..3650fab0f8
--- /dev/null
+++ b/lxd/device/device_utils_unix.go
@@ -0,0 +1,194 @@
+package device
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+
+ "golang.org/x/sys/unix"
+
+ "github.com/lxc/lxd/lxd/device/config"
+ "github.com/lxc/lxd/lxd/state"
+ "github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/idmap"
+ "github.com/lxc/lxd/shared/logger"
+)
+
+func UnixGetDeviceAttributes(path string) (string, int, int, error) {
+ // Get a stat struct from the provided path
+ stat := unix.Stat_t{}
+ err := unix.Stat(path, &stat)
+ if err != nil {
+ return "", 0, 0, err
+ }
+
+ // Check what kind of file it is
+ dType := ""
+ if stat.Mode&unix.S_IFMT == unix.S_IFBLK {
+ dType = "b"
+ } else if stat.Mode&unix.S_IFMT == unix.S_IFCHR {
+ dType = "c"
+ } else {
+ return "", 0, 0, fmt.Errorf("Not a device")
+ }
+
+ // Return the device information
+ major := shared.Major(stat.Rdev)
+ minor := shared.Minor(stat.Rdev)
+ return dType, major, minor, nil
+}
+
+func unixGetDeviceModeOct(strmode string) (int, error) {
+ // Default mode
+ if strmode == "" {
+ return 0600, nil
+ }
+
+ // Converted mode
+ i, err := strconv.ParseInt(strmode, 8, 32)
+ if err != nil {
+ return 0, fmt.Errorf("Bad device mode: %s", strmode)
+ }
+
+ return int(i), nil
+}
+
+// UnixCreateDevice Unix devices handling.
+func UnixCreateDevice(s *state.State, idmapSet *idmap.IdmapSet, devicesPath string, prefix string, m config.Device, defaultMode bool) ([]string, error) {
+ var err error
+ var major, minor int
+
+ // Extra checks for nesting
+ if s.OS.RunningInUserNS {
+ for key, value := range m {
+ if shared.StringInSlice(key, []string{"major", "minor", "mode", "uid", "gid"}) && value != "" {
+ return nil, fmt.Errorf("The \"%s\" property may not be set when adding a device to a nested container", key)
+ }
+ }
+ }
+
+ srcPath := m["source"]
+ if srcPath == "" {
+ srcPath = m["path"]
+ }
+ srcPath = shared.HostPath(srcPath)
+
+ // Get the major/minor of the device we want to create
+ if m["major"] == "" && m["minor"] == "" {
+ // If no major and minor are set, use those from the device on the host
+ _, major, minor, err = UnixGetDeviceAttributes(srcPath)
+ if err != nil {
+ return nil, fmt.Errorf("Failed to get device attributes for %s: %s", m["path"], err)
+ }
+ } else if m["major"] == "" || m["minor"] == "" {
+ return nil, fmt.Errorf("Both major and minor must be supplied for device: %s", m["path"])
+ } else {
+ major, err = strconv.Atoi(m["major"])
+ if err != nil {
+ return nil, fmt.Errorf("Bad major %s in device %s", m["major"], m["path"])
+ }
+
+ minor, err = strconv.Atoi(m["minor"])
+ if err != nil {
+ return nil, fmt.Errorf("Bad minor %s in device %s", m["minor"], m["path"])
+ }
+ }
+
+ // Get the device mode
+ mode := os.FileMode(0660)
+ if m["mode"] != "" {
+ tmp, err := unixGetDeviceModeOct(m["mode"])
+ if err != nil {
+ return nil, fmt.Errorf("Bad mode %s in device %s", m["mode"], m["path"])
+ }
+ mode = os.FileMode(tmp)
+ } else if !defaultMode {
+ mode, err = shared.GetPathMode(srcPath)
+ if err != nil {
+ errno, isErrno := shared.GetErrno(err)
+ if !isErrno || errno != unix.ENOENT {
+ return nil, fmt.Errorf("Failed to retrieve mode of device %s: %s", m["path"], err)
+ }
+ mode = os.FileMode(0660)
+ }
+ }
+
+ if m["type"] == "unix-block" {
+ mode |= unix.S_IFBLK
+ } else {
+ mode |= unix.S_IFCHR
+ }
+
+ // Get the device owner
+ uid := 0
+ gid := 0
+
+ if m["uid"] != "" {
+ uid, err = strconv.Atoi(m["uid"])
+ if err != nil {
+ return nil, fmt.Errorf("Invalid uid %s in device %s", m["uid"], m["path"])
+ }
+ }
+
+ if m["gid"] != "" {
+ gid, err = strconv.Atoi(m["gid"])
+ if err != nil {
+ return nil, fmt.Errorf("Invalid gid %s in device %s", m["gid"], m["path"])
+ }
+ }
+
+ // Create the devices directory if missing
+ if !shared.PathExists(devicesPath) {
+ os.Mkdir(devicesPath, 0711)
+ if err != nil {
+ return nil, fmt.Errorf("Failed to create devices path: %s", err)
+ }
+ }
+
+ destPath := m["path"]
+ if destPath == "" {
+ destPath = m["source"]
+ }
+ relativeDestPath := strings.TrimPrefix(destPath, "/")
+ devName := fmt.Sprintf("%s.%s", strings.Replace(prefix, "/", "-", -1), strings.Replace(relativeDestPath, "/", "-", -1))
+ devPath := filepath.Join(devicesPath, devName)
+
+ // Create the new entry
+ if !s.OS.RunningInUserNS {
+ encoded_device_number := (minor & 0xff) | (major << 8) | ((minor & ^0xff) << 12)
+ if err := unix.Mknod(devPath, uint32(mode), encoded_device_number); err != nil {
+ return nil, fmt.Errorf("Failed to create device %s for %s: %s", devPath, m["path"], err)
+ }
+
+ if err := os.Chown(devPath, uid, gid); err != nil {
+ return nil, fmt.Errorf("Failed to chown device %s: %s", devPath, err)
+ }
+
+ // Needed as mknod respects the umask
+ if err := os.Chmod(devPath, mode); err != nil {
+ return nil, fmt.Errorf("Failed to chmod device %s: %s", devPath, err)
+ }
+
+ if idmapSet != nil {
+ if err := idmapSet.ShiftFile(devPath); err != nil {
+ // uidshift failing is weird, but not a big problem. Log and proceed
+ logger.Debugf("Failed to uidshift device %s: %s\n", m["path"], err)
+ }
+ }
+ } else {
+ f, err := os.Create(devPath)
+ if err != nil {
+ return nil, err
+ }
+ f.Close()
+
+ err = DiskMount(srcPath, devPath, false, false, "")
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ return []string{devPath, relativeDestPath}, nil
+}
From 37cda1878cd8a8e3d6666bc5b2b29098b78e5552 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 1 Aug 2019 16:40:54 +0100
Subject: [PATCH 35/42] container: Updates references to deviceGetAttributes to
device package
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/container.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lxd/container.go b/lxd/container.go
index 69ae62350f..9e68bd3eb8 100644
--- a/lxd/container.go
+++ b/lxd/container.go
@@ -425,7 +425,7 @@ func containerValidDevices(state *state.State, cluster *db.Cluster, devices conf
return fmt.Errorf("The device path doesn't exist on the host and major/minor wasn't specified")
}
- dType, _, _, err := deviceGetAttributes(srcPath)
+ dType, _, _, err := device.UnixGetDeviceAttributes(srcPath)
if err != nil {
return err
}
From 785ba283fb7e958522b79b501e9e3db41893258a Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 1 Aug 2019 16:41:21 +0100
Subject: [PATCH 36/42] util: Removes BlockFsDetect as moved into device
package
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
shared/util.go | 9 ---------
1 file changed, 9 deletions(-)
diff --git a/shared/util.go b/shared/util.go
index da6bd619cd..4ee9b179fa 100644
--- a/shared/util.go
+++ b/shared/util.go
@@ -587,15 +587,6 @@ func IsBlockdevPath(pathName string) bool {
return ((fm&os.ModeDevice != 0) && (fm&os.ModeCharDevice == 0))
}
-func BlockFsDetect(dev string) (string, error) {
- out, err := RunCommand("blkid", "-s", "TYPE", "-o", "value", dev)
- if err != nil {
- return "", err
- }
-
- return strings.TrimSpace(out), nil
-}
-
// DeepCopy copies src to dest by using encoding/gob so its not that fast.
func DeepCopy(src, dest interface{}) error {
buff := new(bytes.Buffer)
From fa6754f99ec4191de2a75f5b9416216de09fd417 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 1 Aug 2019 16:44:08 +0100
Subject: [PATCH 37/42] device: Moves device related functions into device
package
And updates references.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/devices.go | 388 ++-----------------------------------------------
1 file changed, 16 insertions(+), 372 deletions(-)
diff --git a/lxd/devices.go b/lxd/devices.go
index b5da1fa6e2..b0d987e68e 100644
--- a/lxd/devices.go
+++ b/lxd/devices.go
@@ -22,6 +22,7 @@ import (
"github.com/jaypipes/pcidb"
"golang.org/x/sys/unix"
+ "github.com/lxc/lxd/lxd/device"
"github.com/lxc/lxd/lxd/state"
"github.com/lxc/lxd/lxd/sys"
"github.com/lxc/lxd/lxd/util"
@@ -1099,62 +1100,6 @@ func deviceTaskSchedulerTrigger(srcType string, srcName string, srcStatus string
}
}
-func deviceIsBlockdev(path string) bool {
- // Get a stat struct from the provided path
- stat := unix.Stat_t{}
- err := unix.Stat(path, &stat)
- if err != nil {
- return false
- }
-
- // Check if it's a block device
- if stat.Mode&unix.S_IFMT == unix.S_IFBLK {
- return true
- }
-
- // Not a device
- return false
-}
-
-func deviceModeOct(strmode string) (int, error) {
- // Default mode
- if strmode == "" {
- return 0600, nil
- }
-
- // Converted mode
- i, err := strconv.ParseInt(strmode, 8, 32)
- if err != nil {
- return 0, fmt.Errorf("Bad device mode: %s", strmode)
- }
-
- return int(i), nil
-}
-
-func deviceGetAttributes(path string) (string, int, int, error) {
- // Get a stat struct from the provided path
- stat := unix.Stat_t{}
- err := unix.Stat(path, &stat)
- if err != nil {
- return "", 0, 0, err
- }
-
- // Check what kind of file it is
- dType := ""
- if stat.Mode&unix.S_IFMT == unix.S_IFBLK {
- dType = "b"
- } else if stat.Mode&unix.S_IFMT == unix.S_IFCHR {
- dType = "c"
- } else {
- return "", 0, 0, fmt.Errorf("Not a device")
- }
-
- // Return the device information
- major := shared.Major(stat.Rdev)
- minor := shared.Minor(stat.Rdev)
- return dType, major, minor, nil
-}
-
func deviceNextInterfaceHWAddr() (string, error) {
// Generate a new random MAC address using the usual prefix
ret := bytes.Buffer{}
@@ -1173,73 +1118,6 @@ func deviceNextInterfaceHWAddr() (string, error) {
return ret.String(), nil
}
-func deviceMountDisk(srcPath string, dstPath string, readonly bool, recursive bool, propagation string) error {
- var err error
-
- // Prepare the mount flags
- flags := 0
- if readonly {
- flags |= unix.MS_RDONLY
- }
-
- // Detect the filesystem
- fstype := "none"
- if deviceIsBlockdev(srcPath) {
- fstype, err = shared.BlockFsDetect(srcPath)
- if err != nil {
- return err
- }
- } else {
- flags |= unix.MS_BIND
- if propagation != "" {
- switch propagation {
- case "private":
- flags |= unix.MS_PRIVATE
- case "shared":
- flags |= unix.MS_SHARED
- case "slave":
- flags |= unix.MS_SLAVE
- case "unbindable":
- flags |= unix.MS_UNBINDABLE
- case "rprivate":
- flags |= unix.MS_PRIVATE | unix.MS_REC
- case "rshared":
- flags |= unix.MS_SHARED | unix.MS_REC
- case "rslave":
- flags |= unix.MS_SLAVE | unix.MS_REC
- case "runbindable":
- flags |= unix.MS_UNBINDABLE | unix.MS_REC
- default:
- return fmt.Errorf("Invalid propagation mode '%s'", propagation)
- }
- }
-
- if recursive {
- flags |= unix.MS_REC
- }
- }
-
- // Mount the filesystem
- if err = unix.Mount(srcPath, dstPath, fstype, uintptr(flags), ""); err != nil {
- return fmt.Errorf("Unable to mount %s at %s: %s", srcPath, dstPath, err)
- }
-
- // Remount bind mounts in readonly mode if requested
- if readonly == true && flags&unix.MS_BIND == unix.MS_BIND {
- flags = unix.MS_RDONLY | unix.MS_BIND | unix.MS_REMOUNT
- if err = unix.Mount("", dstPath, fstype, uintptr(flags), ""); err != nil {
- return fmt.Errorf("Unable to mount %s in readonly mode: %s", dstPath, err)
- }
- }
-
- flags = unix.MS_REC | unix.MS_SLAVE
- if err = unix.Mount("", dstPath, "", uintptr(flags), ""); err != nil {
- return fmt.Errorf("unable to make mount %s private: %s", dstPath, err)
- }
-
- return nil
-}
-
func deviceParseCPU(cpuAllowance string, cpuPriority string) (string, string, string, error) {
var err error
@@ -1304,7 +1182,7 @@ func deviceParseCPU(cpuAllowance string, cpuPriority string) (string, string, st
func deviceGetParentBlocks(path string) ([]string, error) {
var devices []string
- var device []string
+ var dev []string
// Expand the mount path
absPath, err := filepath.Abs(path)
@@ -1341,16 +1219,16 @@ func deviceGetParentBlocks(path string) ([]string, error) {
match = rows[4]
// Go backward to avoid problems with optional fields
- device = []string{rows[2], rows[len(rows)-2]}
+ dev = []string{rows[2], rows[len(rows)-2]}
}
- if device == nil {
+ if dev == nil {
return nil, fmt.Errorf("Couldn't find a match /proc/self/mountinfo entry")
}
// Handle the most simple case
- if !strings.HasPrefix(device[0], "0:") {
- return []string{device[0]}, nil
+ if !strings.HasPrefix(dev[0], "0:") {
+ return []string{dev[0]}, nil
}
// Deal with per-filesystem oddities. We don't care about failures here
@@ -1359,11 +1237,11 @@ func deviceGetParentBlocks(path string) ([]string, error) {
if fs == "zfs" && shared.PathExists("/dev/zfs") {
// Accessible zfs filesystems
- poolName := strings.Split(device[1], "/")[0]
+ poolName := strings.Split(dev[1], "/")[0]
output, err := shared.RunCommand("zpool", "status", "-P", "-L", poolName)
if err != nil {
- return nil, fmt.Errorf("Failed to query zfs filesystem information for %s: %s", device[1], output)
+ return nil, fmt.Errorf("Failed to query zfs filesystem information for %s: %s", dev[1], output)
}
header := true
@@ -1401,7 +1279,7 @@ func deviceGetParentBlocks(path string) ([]string, error) {
}
if path != "" {
- _, major, minor, err := deviceGetAttributes(path)
+ _, major, minor, err := device.UnixGetDeviceAttributes(path)
if err != nil {
continue
}
@@ -1413,11 +1291,11 @@ func deviceGetParentBlocks(path string) ([]string, error) {
if len(devices) == 0 {
return nil, fmt.Errorf("Unable to find backing block for zfs pool: %s", poolName)
}
- } else if fs == "btrfs" && shared.PathExists(device[1]) {
+ } else if fs == "btrfs" && shared.PathExists(dev[1]) {
// Accessible btrfs filesystems
- output, err := shared.RunCommand("btrfs", "filesystem", "show", device[1])
+ output, err := shared.RunCommand("btrfs", "filesystem", "show", dev[1])
if err != nil {
- return nil, fmt.Errorf("Failed to query btrfs filesystem information for %s: %s", device[1], output)
+ return nil, fmt.Errorf("Failed to query btrfs filesystem information for %s: %s", dev[1], output)
}
for _, line := range strings.Split(output, "\n") {
@@ -1426,23 +1304,23 @@ func deviceGetParentBlocks(path string) ([]string, error) {
continue
}
- _, major, minor, err := deviceGetAttributes(fields[len(fields)-1])
+ _, major, minor, err := device.UnixGetDeviceAttributes(fields[len(fields)-1])
if err != nil {
return nil, err
}
devices = append(devices, fmt.Sprintf("%d:%d", major, minor))
}
- } else if shared.PathExists(device[1]) {
+ } else if shared.PathExists(dev[1]) {
// Anything else with a valid path
- _, major, minor, err := deviceGetAttributes(device[1])
+ _, major, minor, err := device.UnixGetDeviceAttributes(dev[1])
if err != nil {
return nil, err
}
devices = append(devices, fmt.Sprintf("%d:%d", major, minor))
} else {
- return nil, fmt.Errorf("Invalid block device: %s", device[1])
+ return nil, fmt.Errorf("Invalid block device: %s", dev[1])
}
return devices, nil
@@ -1563,240 +1441,6 @@ func deviceLoadUsb() ([]usbDevice, error) {
return result, nil
}
-const SCIB string = "/sys/class/infiniband"
-const SCNET string = "/sys/class/net"
-
-type IBF struct {
- // port the function belongs to
- Port int64
-
- // name of the {physical,virtual} function
- Fun string
-
- // whether this is a physical (true) or virtual (false) function
- PF bool
-
- // device of the function
- Device string
-
- // uverb device of the function
- PerPortDevices []string
- PerFunDevices []string
-}
-
-func deviceLoadInfiniband() (map[string]IBF, error) {
- // check if there are any infiniband devices
- fscib, err := os.Open(SCIB)
- if err != nil {
- if os.IsNotExist(err) {
- return nil, os.ErrNotExist
- }
- return nil, err
- }
- defer fscib.Close()
-
- // eg.g. mlx_i for i = 0, 1, ..., n
- IBDevNames, err := fscib.Readdirnames(-1)
- if err != nil {
- return nil, err
- }
-
- if len(IBDevNames) == 0 {
- return nil, os.ErrNotExist
- }
-
- // retrieve all network device names
- fscnet, err := os.Open(SCNET)
- if err != nil {
- if os.IsNotExist(err) {
- return nil, os.ErrNotExist
- }
- return nil, err
- }
- defer fscnet.Close()
-
- // retrieve all network devices
- NetDevNames, err := fscnet.Readdirnames(-1)
- if err != nil {
- return nil, err
- }
-
- if len(NetDevNames) == 0 {
- return nil, os.ErrNotExist
- }
-
- UseableDevices := make(map[string]IBF)
- for _, IBDevName := range IBDevNames {
- IBDevResourceFile := fmt.Sprintf("/sys/class/infiniband/%s/device/resource", IBDevName)
- IBDevResourceBuf, err := ioutil.ReadFile(IBDevResourceFile)
- if err != nil {
- return nil, err
- }
-
- for _, NetDevName := range NetDevNames {
- NetDevResourceFile := fmt.Sprintf("/sys/class/net/%s/device/resource", NetDevName)
- NetDevResourceBuf, err := ioutil.ReadFile(NetDevResourceFile)
- if err != nil {
- if os.IsNotExist(err) {
- continue
- }
- return nil, err
- }
-
- // If the device and the VF have the same address space
- // they belong together.
- if bytes.Compare(IBDevResourceBuf, NetDevResourceBuf) != 0 {
- continue
- }
-
- // Now let's find the ports.
- IBDevID := fmt.Sprintf("/sys/class/net/%s/dev_id", NetDevName)
- IBDevPort := fmt.Sprintf("/sys/class/net/%s/dev_port", NetDevName)
- DevIDBuf, err := ioutil.ReadFile(IBDevID)
- if err != nil {
- if os.IsNotExist(err) {
- continue
- }
- return nil, err
- }
-
- DevIDString := strings.TrimSpace(string(DevIDBuf))
- DevIDPort, err := strconv.ParseInt(DevIDString, 0, 64)
- if err != nil {
- return nil, err
- }
-
- DevPort := int64(0)
- DevPortBuf, err := ioutil.ReadFile(IBDevPort)
- if err != nil {
- if !os.IsNotExist(err) {
- return nil, err
- }
- } else {
- DevPortString := strings.TrimSpace(string(DevPortBuf))
- DevPort, err = strconv.ParseInt(DevPortString, 0, 64)
- if err != nil {
- return nil, err
- }
- }
-
- Port := DevIDPort
- if DevPort > DevIDPort {
- Port = DevPort
- }
- Port++
-
- NewIBF := IBF{
- Port: Port,
- Fun: IBDevName,
- Device: NetDevName,
- }
-
- // identify the /dev/infiniband/uverb<idx> device
- tmp := []string{}
- IBUverb := fmt.Sprintf("/sys/class/net/%s/device/infiniband_verbs", NetDevName)
- fuverb, err := os.Open(IBUverb)
- if err != nil {
- if !os.IsNotExist(err) {
- return nil, err
- }
- } else {
- defer fuverb.Close()
-
- // optional: retrieve all network devices
- tmp, err = fuverb.Readdirnames(-1)
- if err != nil {
- return nil, err
- }
-
- if len(tmp) == 0 {
- return nil, os.ErrNotExist
- }
- }
- for _, v := range tmp {
- if strings.Index(v, "-") != -1 {
- return nil, fmt.Errorf("Infiniband character device \"%s\" contains \"-\". Cannot guarantee unique encoding", v)
- }
- NewIBF.PerPortDevices = append(NewIBF.PerPortDevices, v)
- }
-
- // identify the /dev/infiniband/ucm<idx> device
- tmp = []string{}
- IBcm := fmt.Sprintf("/sys/class/net/%s/device/infiniband_ucm", NetDevName)
- fcm, err := os.Open(IBcm)
- if err != nil {
- if !os.IsNotExist(err) {
- return nil, err
- }
- } else {
- defer fcm.Close()
-
- // optional: retrieve all network devices
- tmp, err = fcm.Readdirnames(-1)
- if err != nil {
- return nil, err
- }
-
- if len(tmp) == 0 {
- return nil, os.ErrNotExist
- }
- }
- for _, v := range tmp {
- if strings.Index(v, "-") != -1 {
- return nil, fmt.Errorf("Infiniband character device \"%s\" contains \"-\". Cannot guarantee unique encoding", v)
- }
- devPath := fmt.Sprintf("/dev/infiniband/%s", v)
- NewIBF.PerPortDevices = append(NewIBF.PerPortDevices, devPath)
- }
-
- // identify the /dev/infiniband/{issm,umad}<idx> devices
- IBmad := fmt.Sprintf("/sys/class/net/%s/device/infiniband_mad", NetDevName)
- ents, err := ioutil.ReadDir(IBmad)
- if err != nil {
- if !os.IsNotExist(err) {
- return nil, err
- }
- } else {
- for _, ent := range ents {
- IBmadPort := fmt.Sprintf("%s/%s/port", IBmad, ent.Name())
- portBuf, err := ioutil.ReadFile(IBmadPort)
- if err != nil {
- if !os.IsNotExist(err) {
- return nil, err
- }
- continue
- }
-
- portStr := strings.TrimSpace(string(portBuf))
- PortMad, err := strconv.ParseInt(portStr, 0, 64)
- if err != nil {
- return nil, err
- }
-
- if PortMad != NewIBF.Port {
- continue
- }
-
- if strings.Index(ent.Name(), "-") != -1 {
- return nil, fmt.Errorf("Infiniband character device \"%s\" contains \"-\". Cannot guarantee unique encoding", ent.Name())
- }
-
- NewIBF.PerFunDevices = append(NewIBF.PerFunDevices, ent.Name())
- }
- }
-
- // figure out whether this is a physical function
- IBPF := fmt.Sprintf("/sys/class/net/%s/device/physfn", NetDevName)
- NewIBF.PF = !shared.PathExists(IBPF)
-
- UseableDevices[NetDevName] = NewIBF
- }
- }
-
- // check whether the device is an infiniband device
- return UseableDevices, nil
-}
-
func deviceInotifyInit(s *state.State) (int, error) {
s.OS.InotifyWatch.Lock()
defer s.OS.InotifyWatch.Unlock()
From 15f4a83f2e95e6099695e1c89fb2d09cb7d8d46c Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 1 Aug 2019 16:45:06 +0100
Subject: [PATCH 38/42] container/lxc: Updates infiniband to use device package
Removes any functions that have been moved to device package.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/container_lxc.go | 197 +++++++------------------------------------
1 file changed, 31 insertions(+), 166 deletions(-)
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index 26e60b9fbc..59f9718111 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -1689,7 +1689,7 @@ func (c *containerLXC) initLXC(config bool) error {
// mounting a directory.
isFile := false
if m["pool"] == "" {
- isFile = !shared.IsDir(srcPath) && !deviceIsBlockdev(srcPath)
+ isFile = !shared.IsDir(srcPath) && !device.IsBlockdev(srcPath)
}
// Deal with a rootfs
@@ -2137,7 +2137,12 @@ func (c *containerLXC) setupUnixDevice(prefix string, dev config.Device, major i
temp["minor"] = fmt.Sprintf("%d", minor)
temp["path"] = path
- paths, err := c.createUnixDevice(prefix, temp, defaultMode)
+ idmapSet, err := c.CurrentIdmap()
+ if err != nil {
+ return err
+ }
+
+ paths, err := device.UnixCreateDevice(c.state, idmapSet, c.DevicesPath(), prefix, temp, defaultMode)
if err != nil {
logger.Debug("Failed to create device", log.Ctx{"err": err, "device": prefix})
if createMustSucceed {
@@ -2383,8 +2388,13 @@ func (c *containerLXC) startCommon() (string, []func() error, error) {
for _, k := range c.expandedDevices.DeviceNames() {
m := c.expandedDevices[k]
if shared.StringInSlice(m["type"], []string{"unix-char", "unix-block"}) {
+ idmapSet, err := c.CurrentIdmap()
+ if err != nil {
+ return "", postStartHooks, err
+ }
+
// Unix device
- paths, err := c.createUnixDevice(fmt.Sprintf("unix.%s", k), m, true)
+ paths, err := device.UnixCreateDevice(c.state, idmapSet, c.DevicesPath(), fmt.Sprintf("unix.%s", k), m, true)
if err != nil {
// Deal with device hotplug
if m["required"] == "" || shared.IsTrue(m["required"]) {
@@ -2407,7 +2417,7 @@ func (c *containerLXC) startCommon() (string, []func() error, error) {
devPath := paths[0]
if c.isCurrentlyPrivileged() && !c.state.OS.RunningInUserNS && c.state.OS.CGroupDevicesController {
// Add the new device cgroup rule
- dType, dMajor, dMinor, err := deviceGetAttributes(devPath)
+ dType, dMajor, dMinor, err := device.UnixGetDeviceAttributes(devPath)
if err != nil {
if m["required"] == "" || shared.IsTrue(m["required"]) {
return "", postStartHooks, err
@@ -2502,14 +2512,6 @@ func (c *containerLXC) startCommon() (string, []func() error, error) {
if m["path"] != "/" {
diskDevices[k] = m
}
- } else if m["type"] == "infiniband" {
- // Start infiniband device.
- // Increment nicID so that LXC network index is unique per device.
- nicID++
- err := c.startInfiniband(nicID, k, m)
- if err != nil {
- return "", postStartHooks, err
- }
} else {
// Use new Device interface if supported.
runConfig, err := c.deviceStart(k, m, false)
@@ -4951,11 +4953,6 @@ func (c *containerLXC) Update(args db.ContainerArgs, userRequested bool) error {
}
} else if m["type"] == "disk" && m["path"] != "/" {
diskDevices[k] = m
- } else if m["type"] == "infiniband" {
- err := c.addInfinibandDevice(k, m)
- if err != nil {
- return err
- }
} else if m["type"] == "usb" {
if usbs == nil {
usbs, err = deviceLoadUsb()
@@ -6794,157 +6791,19 @@ func (c *containerLXC) deviceExistsInDevicesFolder(prefix string, path string) b
return shared.PathExists(devPath)
}
-// Unix devices handling
-func (c *containerLXC) createUnixDevice(prefix string, m config.Device, defaultMode bool) ([]string, error) {
- var err error
- var major, minor int
-
- // Extra checks for nesting
- if c.state.OS.RunningInUserNS {
- for key, value := range m {
- if shared.StringInSlice(key, []string{"major", "minor", "mode", "uid", "gid"}) && value != "" {
- return nil, fmt.Errorf("The \"%s\" property may not be set when adding a device to a nested container", key)
- }
- }
- }
-
- srcPath := m["source"]
- if srcPath == "" {
- srcPath = m["path"]
- }
- srcPath = shared.HostPath(srcPath)
-
- // Get the major/minor of the device we want to create
- if m["major"] == "" && m["minor"] == "" {
- // If no major and minor are set, use those from the device on the host
- _, major, minor, err = deviceGetAttributes(srcPath)
- if err != nil {
- return nil, fmt.Errorf("Failed to get device attributes for %s: %s", m["path"], err)
- }
- } else if m["major"] == "" || m["minor"] == "" {
- return nil, fmt.Errorf("Both major and minor must be supplied for device: %s", m["path"])
- } else {
- major, err = strconv.Atoi(m["major"])
- if err != nil {
- return nil, fmt.Errorf("Bad major %s in device %s", m["major"], m["path"])
- }
-
- minor, err = strconv.Atoi(m["minor"])
- if err != nil {
- return nil, fmt.Errorf("Bad minor %s in device %s", m["minor"], m["path"])
- }
- }
-
- // Get the device mode
- mode := os.FileMode(0660)
- if m["mode"] != "" {
- tmp, err := deviceModeOct(m["mode"])
- if err != nil {
- return nil, fmt.Errorf("Bad mode %s in device %s", m["mode"], m["path"])
- }
- mode = os.FileMode(tmp)
- } else if !defaultMode {
- mode, err = shared.GetPathMode(srcPath)
- if err != nil {
- errno, isErrno := shared.GetErrno(err)
- if !isErrno || errno != unix.ENOENT {
- return nil, fmt.Errorf("Failed to retrieve mode of device %s: %s", m["path"], err)
- }
- mode = os.FileMode(0660)
- }
- }
-
- if m["type"] == "unix-block" {
- mode |= unix.S_IFBLK
- } else {
- mode |= unix.S_IFCHR
- }
-
- // Get the device owner
- uid := 0
- gid := 0
-
- if m["uid"] != "" {
- uid, err = strconv.Atoi(m["uid"])
- if err != nil {
- return nil, fmt.Errorf("Invalid uid %s in device %s", m["uid"], m["path"])
- }
- }
-
- if m["gid"] != "" {
- gid, err = strconv.Atoi(m["gid"])
- if err != nil {
- return nil, fmt.Errorf("Invalid gid %s in device %s", m["gid"], m["path"])
- }
- }
-
- // Create the devices directory if missing
- if !shared.PathExists(c.DevicesPath()) {
- os.Mkdir(c.DevicesPath(), 0711)
- if err != nil {
- return nil, fmt.Errorf("Failed to create devices path: %s", err)
- }
- }
-
- destPath := m["path"]
- if destPath == "" {
- destPath = m["source"]
- }
- relativeDestPath := strings.TrimPrefix(destPath, "/")
- devName := fmt.Sprintf("%s.%s", strings.Replace(prefix, "/", "-", -1), strings.Replace(relativeDestPath, "/", "-", -1))
- devPath := filepath.Join(c.DevicesPath(), devName)
-
- // Create the new entry
- if !c.state.OS.RunningInUserNS {
- encoded_device_number := (minor & 0xff) | (major << 8) | ((minor & ^0xff) << 12)
- if err := unix.Mknod(devPath, uint32(mode), encoded_device_number); err != nil {
- return nil, fmt.Errorf("Failed to create device %s for %s: %s", devPath, m["path"], err)
- }
-
- if err := os.Chown(devPath, uid, gid); err != nil {
- return nil, fmt.Errorf("Failed to chown device %s: %s", devPath, err)
- }
-
- // Needed as mknod respects the umask
- if err := os.Chmod(devPath, mode); err != nil {
- return nil, fmt.Errorf("Failed to chmod device %s: %s", devPath, err)
- }
-
- idmapset, err := c.CurrentIdmap()
- if err != nil {
- return nil, err
- }
-
- if idmapset != nil {
- if err := idmapset.ShiftFile(devPath); err != nil {
- // uidshift failing is weird, but not a big problem. Log and proceed
- logger.Debugf("Failed to uidshift device %s: %s\n", m["path"], err)
- }
- }
- } else {
- f, err := os.Create(devPath)
- if err != nil {
- return nil, err
- }
- f.Close()
-
- err = deviceMountDisk(srcPath, devPath, false, false, "")
- if err != nil {
- return nil, err
- }
- }
-
- return []string{devPath, relativeDestPath}, nil
-}
-
func (c *containerLXC) insertUnixDevice(prefix string, m config.Device, defaultMode bool) error {
// Check that the container is running
if !c.IsRunning() {
return fmt.Errorf("Can't insert device into stopped container")
}
+ idmapSet, err := c.CurrentIdmap()
+ if err != nil {
+ return err
+ }
+
// Create the device on the host
- paths, err := c.createUnixDevice(prefix, m, defaultMode)
+ paths, err := device.UnixCreateDevice(c.state, idmapSet, c.DevicesPath(), prefix, m, defaultMode)
if err != nil {
return fmt.Errorf("Failed to setup device: %s", err)
}
@@ -6983,7 +6842,7 @@ func (c *containerLXC) insertUnixDevice(prefix string, m config.Device, defaultM
}
if dType == "" || dMajor < 0 || dMinor < 0 {
- dType, dMajor, dMinor, err = deviceGetAttributes(devPath)
+ dType, dMajor, dMinor, err = device.UnixGetDeviceAttributes(devPath)
if err != nil {
return err
}
@@ -7036,7 +6895,13 @@ func (c *containerLXC) InsertSeccompUnixDevice(prefix string, m config.Device, p
} else {
m["path"] = filepath.Join(rootPath, m["path"])
}
- paths, err := c.createUnixDevice(prefix, m, true)
+
+ idmapSet, err := c.CurrentIdmap()
+ if err != nil {
+ return err
+ }
+
+ paths, err := device.UnixCreateDevice(c.state, idmapSet, c.DevicesPath(), prefix, m, true)
if err != nil {
return fmt.Errorf("Failed to setup device: %s", err)
}
@@ -7104,7 +6969,7 @@ func (c *containerLXC) removeUnixDevice(prefix string, m config.Device, eject bo
devPath := filepath.Join(c.DevicesPath(), devName)
if dType == "" || dMajor < 0 || dMinor < 0 {
- dType, dMajor, dMinor, err = deviceGetAttributes(devPath)
+ dType, dMajor, dMinor, err = device.UnixGetDeviceAttributes(devPath)
if err != nil {
return err
}
@@ -7378,7 +7243,7 @@ func (c *containerLXC) createDiskDevice(name string, m config.Device) (string, e
isFile := false
if m["pool"] == "" {
- isFile = !shared.IsDir(srcPath) && !deviceIsBlockdev(srcPath)
+ isFile = !shared.IsDir(srcPath) && !device.IsBlockdev(srcPath)
} else {
// Deal with mounting storage volumes created via the storage
// api. Extract the name of the storage volume that we are
@@ -7483,7 +7348,7 @@ func (c *containerLXC) createDiskDevice(name string, m config.Device) (string, e
}
// Mount the fs
- err := deviceMountDisk(srcPath, devPath, isReadOnly, isRecursive, m["propagation"])
+ err := device.DiskMount(srcPath, devPath, isReadOnly, isRecursive, m["propagation"])
if err != nil {
return "", err
}
From 7ccd2c88400d58df8dd42fd7768e8d00fb58fb46 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 1 Aug 2019 16:45:51 +0100
Subject: [PATCH 39/42] container/lxc/infiniband: Removes old file
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/container_lxc_infiniband.go | 202 +-------------------------------
1 file changed, 2 insertions(+), 200 deletions(-)
diff --git a/lxd/container_lxc_infiniband.go b/lxd/container_lxc_infiniband.go
index f7b4310870..23d495defc 100644
--- a/lxd/container_lxc_infiniband.go
+++ b/lxd/container_lxc_infiniband.go
@@ -4,7 +4,6 @@ import (
"fmt"
"io/ioutil"
"os"
- "path/filepath"
"strconv"
"strings"
@@ -133,58 +132,6 @@ func (c *containerLXC) getInfinibandReserved(m config.Device) (map[string]struct
return reservedDevices, nil
}
-func (c *containerLXC) startInfiniband(networkidx int, deviceName string, m config.Device) error {
- m, err := c.fillNetworkDevice(deviceName, m)
- if err != nil {
- return err
- }
-
- infiniband, err := deviceLoadInfiniband()
- if err != nil {
- return err
- }
-
- reservedDevices, err := c.getInfinibandReserved(m)
- if err != nil {
- return err
- }
-
- m, err = c.fillInfinibandSriovNetworkDevice(deviceName, m, reservedDevices)
- if err != nil {
- return err
- }
-
- err = c.initLXCInfiniband(networkidx, m)
- if err != nil {
- return err
- }
-
- key := m["parent"]
- if m["nictype"] == "sriov" {
- key = m["host_name"]
- }
-
- ifDev, ok := infiniband[key]
- if !ok {
- return fmt.Errorf("Specified infiniband device \"%s\" not found", key)
- }
-
- err = c.addInfinibandDevices(deviceName, &ifDev, false)
- if err != nil {
- return err
- }
-
- // Important we save this to DB now so other devices starting next can see we reserved this
- // host_name device.
- configKey := fmt.Sprintf("volatile.%s.host_name", deviceName)
- err = c.VolatileSet(map[string]string{configKey: key})
- if err != nil {
- return err
- }
-
- return nil
-}
-
func (c *containerLXC) initLXCInfiniband(networkidx int, m config.Device) error {
networkKeyPrefix := "lxc.net"
if !util.RuntimeLiblxcVersionAtLeast(2, 1, 0) {
@@ -242,153 +189,7 @@ func (c *containerLXC) initLXCInfiniband(networkidx int, m config.Device) error
return nil
}
-func (c *containerLXC) addInfinibandDevicesPerPort(deviceName string, ifDev *IBF, devices []os.FileInfo, inject bool) error {
- for _, unixCharDev := range ifDev.PerPortDevices {
- destPath := fmt.Sprintf("/dev/infiniband/%s", unixCharDev)
- relDestPath := destPath[1:]
- devPrefix := fmt.Sprintf("infiniband.unix.%s", deviceName)
-
- // Unix device
- dummyDevice := config.Device{
- "source": destPath,
- }
-
- deviceExists := false
- // only handle infiniband.unix.<device-name>.
- prefix := fmt.Sprintf("infiniband.unix.")
- for _, ent := range devices {
-
- // skip non infiniband.unix.<device-name> devices
- devName := ent.Name()
- if !strings.HasPrefix(devName, prefix) {
- continue
- }
-
- // extract the path inside the container
- idx := strings.LastIndex(devName, ".")
- if idx == -1 {
- return fmt.Errorf("Invalid infiniband device name \"%s\"", devName)
- }
- rPath := devName[idx+1:]
- rPath = strings.Replace(rPath, "-", "/", -1)
- if rPath != relDestPath {
- continue
- }
-
- deviceExists = true
- break
- }
-
- if inject && !deviceExists {
- err := c.insertUnixDevice(devPrefix, dummyDevice, false)
- if err != nil {
- return err
- }
- continue
- }
-
- paths, err := c.createUnixDevice(devPrefix, dummyDevice, false)
- if err != nil {
- return err
- }
- devPath := paths[0]
-
- if deviceExists {
- continue
- }
-
- // inform liblxc about the mount
- err = lxcSetConfigItem(c.c, "lxc.mount.entry",
- fmt.Sprintf("%s %s none bind,create=file 0 0",
- shared.EscapePathFstab(devPath),
- shared.EscapePathFstab(relDestPath)))
- if err != nil {
- return err
- }
-
- if c.isCurrentlyPrivileged() && !c.state.OS.RunningInUserNS && c.state.OS.CGroupDevicesController {
- // Add the new device cgroup rule
- dType, dMajor, dMinor, err := deviceGetAttributes(devPath)
- if err != nil {
- return err
- }
-
- err = lxcSetConfigItem(c.c, "lxc.cgroup.devices.allow", fmt.Sprintf("%s %d:%d rwm", dType, dMajor, dMinor))
- if err != nil {
- return fmt.Errorf("Failed to add cgroup rule for device")
- }
- }
- }
-
- return nil
-}
-
-func (c *containerLXC) addInfinibandDevicesPerFun(deviceName string, ifDev *IBF, inject bool) error {
- for _, unixCharDev := range ifDev.PerFunDevices {
- destPath := fmt.Sprintf("/dev/infiniband/%s", unixCharDev)
- uniqueDevPrefix := fmt.Sprintf("infiniband.unix.%s", deviceName)
- relativeDestPath := fmt.Sprintf("dev/infiniband/%s", unixCharDev)
- uniqueDevName := fmt.Sprintf("%s.%s", uniqueDevPrefix, strings.Replace(relativeDestPath, "/", "-", -1))
- hostDevPath := filepath.Join(c.DevicesPath(), uniqueDevName)
-
- dummyDevice := config.Device{
- "source": destPath,
- }
-
- if inject {
- err := c.insertUnixDevice(uniqueDevPrefix, dummyDevice, false)
- if err != nil {
- return err
- }
- continue
- }
-
- // inform liblxc about the mount
- err := lxcSetConfigItem(c.c, "lxc.mount.entry", fmt.Sprintf("%s %s none bind,create=file 0 0", hostDevPath, relativeDestPath))
- if err != nil {
- return err
- }
-
- paths, err := c.createUnixDevice(uniqueDevPrefix, dummyDevice, false)
- if err != nil {
- return err
- }
- devPath := paths[0]
- if c.isCurrentlyPrivileged() && !c.state.OS.RunningInUserNS && c.state.OS.CGroupDevicesController {
- // Add the new device cgroup rule
- dType, dMajor, dMinor, err := deviceGetAttributes(devPath)
- if err != nil {
- return err
- }
-
- err = lxcSetConfigItem(c.c, "lxc.cgroup.devices.allow", fmt.Sprintf("%s %d:%d rwm", dType, dMajor, dMinor))
- if err != nil {
- return fmt.Errorf("Failed to add cgroup rule for device")
- }
- }
- }
-
- return nil
-}
-
-func (c *containerLXC) addInfinibandDevices(deviceName string, ifDev *IBF, inject bool) error {
- // load all devices
- dents, err := ioutil.ReadDir(c.DevicesPath())
- if err != nil {
- if !os.IsNotExist(err) {
- return err
- }
- }
-
- err = c.addInfinibandDevicesPerPort(deviceName, ifDev, dents, inject)
- if err != nil {
- return err
- }
-
- return c.addInfinibandDevicesPerFun(deviceName, ifDev, inject)
-}
-
-func (c *containerLXC) addInfinibandDevice(deviceName string, device config.Device) error {
+/*func (c *containerLXC) addInfinibandDevice(deviceName string, device config.Device) error {
device, err := c.fillNetworkDevice(deviceName, device)
if err != nil {
return err
@@ -443,6 +244,7 @@ func (c *containerLXC) addInfinibandDevice(deviceName string, device config.Devi
return nil
}
+*/
func (c *containerLXC) removeInfinibandDevice(deviceName string, device config.Device) error {
device, err := c.fillNetworkDevice(deviceName, device)
From 7d55a4dc5ae24f311fc190cd5162fdfb8c90752a Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 1 Aug 2019 16:46:20 +0100
Subject: [PATCH 40/42] device/instance/id: Adds CurrentIdmap function to
InstanceIdentifier interface
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device_instance_id.go | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lxd/device/device_instance_id.go b/lxd/device/device_instance_id.go
index 8fc68732df..1a9ae1c2a3 100644
--- a/lxd/device/device_instance_id.go
+++ b/lxd/device/device_instance_id.go
@@ -2,6 +2,7 @@ package device
import (
"github.com/lxc/lxd/lxd/device/config"
+ "github.com/lxc/lxd/shared/idmap"
)
// InstanceIdentifier is an interface that allows us to identify an Instance and its properties.
@@ -15,4 +16,5 @@ type InstanceIdentifier interface {
LogPath() string
ExpandedConfig() map[string]string
ExpandedDevices() config.Devices
+ CurrentIdmap() (*idmap.IdmapSet, error)
}
From 098a786cd2482b1b65e2b541f98c5ceea30f2020 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 1 Aug 2019 16:47:08 +0100
Subject: [PATCH 41/42] device/runconfig: Adds CGroups and Mounts slice to
RunConfig
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device_runconfig.go | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lxd/device/device_runconfig.go b/lxd/device/device_runconfig.go
index 831dcef688..c00330e731 100644
--- a/lxd/device/device_runconfig.go
+++ b/lxd/device/device_runconfig.go
@@ -9,5 +9,7 @@ type RunConfigItem struct {
// RunConfig represents LXD defined run-time config used for device setup.
type RunConfig struct {
NetworkInterface []RunConfigItem
+ CGroups []RunConfigItem
+ Mounts []RunConfigItem
PostStartHooks []func() error
}
From 6d6dbc96cd785285194632d6ec1e0e1b3edca44b Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 1 Aug 2019 16:47:34 +0100
Subject: [PATCH 42/42] device/utils/instance: Adds instanceGetReservedDevices
function
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device_utils_instance.go | 44 +++++++++++++++++++++++++++++
1 file changed, 44 insertions(+)
diff --git a/lxd/device/device_utils_instance.go b/lxd/device/device_utils_instance.go
index f96c217564..c7bb2d456f 100644
--- a/lxd/device/device_utils_instance.go
+++ b/lxd/device/device_utils_instance.go
@@ -1,6 +1,10 @@
package device
import (
+ "fmt"
+ "sync"
+
+ "github.com/lxc/lxd/lxd/device/config"
"github.com/lxc/lxd/lxd/state"
)
@@ -9,3 +13,43 @@ var InstanceLoadNodeAll func(s *state.State) ([]InstanceIdentifier, error)
// InstanceLoadByProjectAndName returns instance config by project and name.
var InstanceLoadByProjectAndName func(s *state.State, project, name string) (InstanceIdentifier, error)
+
+// reservedDevicesMutex used to coordinate access for checking reserved devices.
+var reservedDevicesMutex sync.Mutex
+
+// instanceGetReservedDevices returns a map of host device names that have been used by devices in
+// other instances on the local node. Used for selecting physical and SR-IOV VF devices.
+func instanceGetReservedDevices(s *state.State, m config.Device) (map[string]struct{}, error) {
+ reservedDevicesMutex.Lock()
+ defer reservedDevicesMutex.Unlock()
+
+ instances, err := InstanceLoadNodeAll(s)
+ if err != nil {
+ return nil, err
+ }
+
+ // Build a unique set of reserved host network devices we cannot use.
+ reservedDevices := map[string]struct{}{}
+ for _, instance := range instances {
+ devices := instance.ExpandedDevices()
+ config := instance.ExpandedConfig()
+ for devName, devConfig := range devices {
+ // Record all parent devices, these are not eligible for use as physical or
+ // SR-IOV parents for selecting VF devices.
+ parent := devConfig["parent"]
+ reservedDevices[parent] = struct{}{}
+
+ // If the device on another instance has the same device type as us, and has
+ // the same parent as us, and a non-empty host_name, then mark that
+ // host_name as reserved, as that device is using it as a SR-IOV VF.
+ if devConfig["type"] == m["type"] && parent == m["parent"] {
+ hostName := config[fmt.Sprintf("volatile.%s.host_name", devName)]
+ if hostName != "" {
+ reservedDevices[hostName] = struct{}{}
+ }
+ }
+ }
+ }
+
+ return reservedDevices, nil
+}
More information about the lxc-devel
mailing list