[lxc-devel] [lxd/master] move some functions to shared
monstermunchkin on Github
lxc-bot at linuxcontainers.org
Fri Jan 12 14:07:01 UTC 2018
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 395 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20180112/3465d9d9/attachment.bin>
-------------- next part --------------
From 37aeeaadad4599402ba68eb66aee9214453e8aa5 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Fri, 12 Jan 2018 12:27:57 +0100
Subject: [PATCH 1/2] lxd,shared: move archive functions to shared
This moves the Unpack() function, as well as some of its dependent code
to shared.
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/container_lxc.go | 6 +--
lxd/devices.go | 34 -------------
lxd/images.go | 122 +++--------------------------------------------
lxd/main_test.go | 2 +-
lxd/patches.go | 16 +++----
lxd/storage.go | 66 +++++++++++--------------
lxd/storage_btrfs.go | 4 +-
lxd/storage_ceph.go | 4 +-
lxd/storage_dir.go | 4 +-
lxd/storage_lvm.go | 4 +-
lxd/storage_lvm_utils.go | 2 +-
lxd/storage_migration.go | 2 +-
lxd/storage_mock.go | 3 +-
lxd/storage_shared.go | 4 +-
lxd/storage_zfs.go | 4 +-
shared/archive.go | 120 ++++++++++++++++++++++++++++++++++++++++++++++
shared/devices.go | 42 ++++++++++++++++
shared/storage.go | 13 +++++
18 files changed, 236 insertions(+), 216 deletions(-)
create mode 100644 shared/archive.go
create mode 100644 shared/devices.go
create mode 100644 shared/storage.go
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index f72a1aedc..97ce25f66 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -1168,7 +1168,7 @@ func (c *containerLXC) initLXC(config bool) error {
return err
}
- memoryTotal, err := deviceTotalMemory()
+ memoryTotal, err := shared.DeviceTotalMemory()
if err != nil {
return err
}
@@ -3846,7 +3846,7 @@ func (c *containerLXC) Update(args db.ContainerArgs, userRequested bool) error {
return err
}
- memoryTotal, err := deviceTotalMemory()
+ memoryTotal, err := shared.DeviceTotalMemory()
if err != nil {
return err
}
@@ -5807,7 +5807,7 @@ func (c *containerLXC) StorageStart() (bool, error) {
isOurOperation, err := c.StorageStartSensitive()
// Remove this as soon as zfs is fixed
- if c.storage.GetStorageType() == storageTypeZfs && err == syscall.EBUSY {
+ if c.storage.GetStorageType() == shared.StorageTypeZfs && err == syscall.EBUSY {
return isOurOperation, nil
}
diff --git a/lxd/devices.go b/lxd/devices.go
index f29b2df24..ec11a7de9 100644
--- a/lxd/devices.go
+++ b/lxd/devices.go
@@ -1067,40 +1067,6 @@ func deviceParseCPU(cpuAllowance string, cpuPriority string) (string, string, st
return fmt.Sprintf("%d", cpuShares), cpuCfsQuota, cpuCfsPeriod, nil
}
-func deviceTotalMemory() (int64, error) {
- // Open /proc/meminfo
- f, err := os.Open("/proc/meminfo")
- if err != nil {
- return -1, err
- }
- defer f.Close()
-
- // Read it line by line
- scan := bufio.NewScanner(f)
- for scan.Scan() {
- line := scan.Text()
-
- // We only care about MemTotal
- if !strings.HasPrefix(line, "MemTotal:") {
- continue
- }
-
- // Extract the before last (value) and last (unit) fields
- fields := strings.Split(line, " ")
- value := fields[len(fields)-2] + fields[len(fields)-1]
-
- // Feed the result to shared.ParseByteSizeString to get an int value
- valueBytes, err := shared.ParseByteSizeString(value)
- if err != nil {
- return -1, err
- }
-
- return valueBytes, nil
- }
-
- return -1, fmt.Errorf("Couldn't find MemTotal")
-}
-
func deviceGetParentBlocks(path string) ([]string, error) {
var devices []string
var device []string
diff --git a/lxd/images.go b/lxd/images.go
index 68fd51f6d..72d4a530b 100644
--- a/lxd/images.go
+++ b/lxd/images.go
@@ -17,7 +17,6 @@ import (
"strconv"
"strings"
"sync"
- "syscall"
"time"
"github.com/gorilla/mux"
@@ -47,117 +46,8 @@ import (
end for whichever finishes last. */
var imagePublishLock sync.Mutex
-func detectCompression(fname string) ([]string, string, error) {
- f, err := os.Open(fname)
- if err != nil {
- return []string{""}, "", err
- }
- defer f.Close()
-
- // read header parts to detect compression method
- // bz2 - 2 bytes, 'BZ' signature/magic number
- // gz - 2 bytes, 0x1f 0x8b
- // lzma - 6 bytes, { [0x000, 0xE0], '7', 'z', 'X', 'Z', 0x00 } -
- // xy - 6 bytes, header format { 0xFD, '7', 'z', 'X', 'Z', 0x00 }
- // tar - 263 bytes, trying to get ustar from 257 - 262
- header := make([]byte, 263)
- _, err = f.Read(header)
- if err != nil {
- return []string{""}, "", err
- }
-
- switch {
- case bytes.Equal(header[0:2], []byte{'B', 'Z'}):
- return []string{"-jxf"}, ".tar.bz2", nil
- case bytes.Equal(header[0:2], []byte{0x1f, 0x8b}):
- return []string{"-zxf"}, ".tar.gz", nil
- case (bytes.Equal(header[1:5], []byte{'7', 'z', 'X', 'Z'}) && header[0] == 0xFD):
- return []string{"-Jxf"}, ".tar.xz", nil
- case (bytes.Equal(header[1:5], []byte{'7', 'z', 'X', 'Z'}) && header[0] != 0xFD):
- return []string{"--lzma", "-xf"}, ".tar.lzma", nil
- case bytes.Equal(header[0:3], []byte{0x5d, 0x00, 0x00}):
- return []string{"--lzma", "-xf"}, ".tar.lzma", nil
- case bytes.Equal(header[257:262], []byte{'u', 's', 't', 'a', 'r'}):
- return []string{"-xf"}, ".tar", nil
- case bytes.Equal(header[0:4], []byte{'h', 's', 'q', 's'}):
- return []string{""}, ".squashfs", nil
- default:
- return []string{""}, "", fmt.Errorf("Unsupported compression")
- }
-
-}
-
-func unpack(file string, path string, sType storageType, runningInUserns bool) error {
- extractArgs, extension, err := detectCompression(file)
- if err != nil {
- return err
- }
-
- command := ""
- args := []string{}
- if strings.HasPrefix(extension, ".tar") {
- command = "tar"
- if runningInUserns {
- args = append(args, "--wildcards")
- args = append(args, "--exclude=dev/*")
- args = append(args, "--exclude=./dev/*")
- args = append(args, "--exclude=rootfs/dev/*")
- args = append(args, "--exclude=rootfs/./dev/*")
- }
- args = append(args, "-C", path, "--numeric-owner")
- args = append(args, extractArgs...)
- args = append(args, file)
- } else if strings.HasPrefix(extension, ".squashfs") {
- command = "unsquashfs"
- args = append(args, "-f", "-d", path, "-n")
-
- // Limit unsquashfs chunk size to 10% of memory and up to 256MB (default)
- // When running on a low memory system, also disable multi-processing
- mem, err := deviceTotalMemory()
- mem = mem / 1024 / 1024 / 10
- if err == nil && mem < 256 {
- args = append(args, "-da", fmt.Sprintf("%d", mem), "-fr", fmt.Sprintf("%d", mem), "-p", "1")
- }
-
- args = append(args, file)
- } else {
- return fmt.Errorf("Unsupported image format: %s", extension)
- }
-
- output, err := shared.RunCommand(command, args...)
- if err != nil {
- // Check if we ran out of space
- fs := syscall.Statfs_t{}
-
- err1 := syscall.Statfs(path, &fs)
- if err1 != nil {
- return err1
- }
-
- // Check if we're running out of space
- if int64(fs.Bfree) < int64(2*fs.Bsize) {
- if sType == storageTypeLvm {
- return fmt.Errorf("Unable to unpack image, run out of disk space (consider increasing your pool's volume.size).")
- } else {
- return fmt.Errorf("Unable to unpack image, run out of disk space.")
- }
- }
-
- co := output
- logger.Debugf("Unpacking failed")
- logger.Debugf(co)
-
- // Truncate the output to a single line for inclusion in the error
- // message. The first line isn't guaranteed to pinpoint the issue,
- // but it's better than nothing and better than a multi-line message.
- return fmt.Errorf("Unpack failed, %s. %s", err, strings.SplitN(co, "\n", 2)[0])
- }
-
- return nil
-}
-
-func unpackImage(imagefname string, destpath string, sType storageType, runningInUserns bool) error {
- err := unpack(imagefname, destpath, sType, runningInUserns)
+func unpackImage(imagefname string, destpath string, sType shared.StorageType, runningInUserns bool) error {
+ err := shared.Unpack(imagefname, destpath, sType, runningInUserns)
if err != nil {
return err
}
@@ -169,7 +59,7 @@ func unpackImage(imagefname string, destpath string, sType storageType, runningI
return fmt.Errorf("Error creating rootfs directory")
}
- err = unpack(imagefname+".rootfs", rootfsPath, sType, runningInUserns)
+ err = shared.Unpack(imagefname+".rootfs", rootfsPath, sType, runningInUserns)
if err != nil {
return err
}
@@ -771,7 +661,7 @@ func imagesPost(d *Daemon, r *http.Request) Response {
func getImageMetadata(fname string) (*api.ImageMetadata, error) {
metadataName := "metadata.yaml"
- compressionArgs, _, err := detectCompression(fname)
+ compressionArgs, _, err := shared.DetectCompression(fname)
if err != nil {
return nil, fmt.Errorf(
@@ -1589,7 +1479,7 @@ func imageExport(d *Daemon, r *http.Request) Response {
imagePath := shared.VarPath("images", imgInfo.Fingerprint)
rootfsPath := imagePath + ".rootfs"
- _, ext, err := detectCompression(imagePath)
+ _, ext, err := shared.DetectCompression(imagePath)
if err != nil {
ext = ""
}
@@ -1604,7 +1494,7 @@ func imageExport(d *Daemon, r *http.Request) Response {
// Recompute the extension for the root filesystem, it may use a different
// compression algorithm than the metadata.
- _, ext, err = detectCompression(rootfsPath)
+ _, ext, err = shared.DetectCompression(rootfsPath)
if err != nil {
ext = ""
}
diff --git a/lxd/main_test.go b/lxd/main_test.go
index 0f05ac700..4a76412a2 100644
--- a/lxd/main_test.go
+++ b/lxd/main_test.go
@@ -68,7 +68,7 @@ func (suite *lxdTestSuite) SetupTest() {
// the next function.
poolConfig := map[string]string{}
- mockStorage, _ := storageTypeToString(storageTypeMock)
+ mockStorage, _ := storageTypeToString(shared.StorageTypeMock)
// Create the database entry for the storage pool.
poolDescription := fmt.Sprintf("%s storage pool", lxdTestSuiteDefaultStoragePool)
_, err = dbStoragePoolCreateAndUpdateCache(suite.d.db, lxdTestSuiteDefaultStoragePool, poolDescription, mockStorage, poolConfig)
diff --git a/lxd/patches.go b/lxd/patches.go
index 79367d5a3..60ad7773f 100644
--- a/lxd/patches.go
+++ b/lxd/patches.go
@@ -166,16 +166,16 @@ func patchStorageApi(name string, d *Daemon) error {
lvmVgName := daemonConfig["storage.lvm_vg_name"].Get()
zfsPoolName := daemonConfig["storage.zfs_pool_name"].Get()
defaultPoolName := "default"
- preStorageApiStorageType := storageTypeDir
+ preStorageApiStorageType := shared.StorageTypeDir
if lvmVgName != "" {
- preStorageApiStorageType = storageTypeLvm
+ preStorageApiStorageType = shared.StorageTypeLvm
defaultPoolName = lvmVgName
} else if zfsPoolName != "" {
- preStorageApiStorageType = storageTypeZfs
+ preStorageApiStorageType = shared.StorageTypeZfs
defaultPoolName = zfsPoolName
} else if d.os.BackingFS == "btrfs" {
- preStorageApiStorageType = storageTypeBtrfs
+ preStorageApiStorageType = shared.StorageTypeBtrfs
} else {
// Dir storage pool.
}
@@ -229,13 +229,13 @@ func patchStorageApi(name string, d *Daemon) error {
// If any of these are actually called, there's no way back.
poolName := defaultPoolName
switch preStorageApiStorageType {
- case storageTypeBtrfs:
+ case shared.StorageTypeBtrfs:
err = upgradeFromStorageTypeBtrfs(name, d, defaultPoolName, defaultStorageTypeName, cRegular, cSnapshots, imgPublic, imgPrivate)
- case storageTypeDir:
+ case shared.StorageTypeDir:
err = upgradeFromStorageTypeDir(name, d, defaultPoolName, defaultStorageTypeName, cRegular, cSnapshots, imgPublic, imgPrivate)
- case storageTypeLvm:
+ case shared.StorageTypeLvm:
err = upgradeFromStorageTypeLvm(name, d, defaultPoolName, defaultStorageTypeName, cRegular, cSnapshots, imgPublic, imgPrivate)
- case storageTypeZfs:
+ case shared.StorageTypeZfs:
// The user is using a zfs dataset. This case needs to be
// handled with care:
diff --git a/lxd/storage.go b/lxd/storage.go
index 4a651d70e..99dacae79 100644
--- a/lxd/storage.go
+++ b/lxd/storage.go
@@ -75,53 +75,41 @@ func readStoragePoolDriversCache() map[string]string {
return drivers.(map[string]string)
}
-// storageType defines the type of a storage
-type storageType int
-
-const (
- storageTypeBtrfs storageType = iota
- storageTypeCeph
- storageTypeDir
- storageTypeLvm
- storageTypeMock
- storageTypeZfs
-)
-
var supportedStoragePoolDrivers = []string{"btrfs", "ceph", "dir", "lvm", "zfs"}
-func storageTypeToString(sType storageType) (string, error) {
+func storageTypeToString(sType shared.StorageType) (string, error) {
switch sType {
- case storageTypeBtrfs:
+ case shared.StorageTypeBtrfs:
return "btrfs", nil
- case storageTypeCeph:
+ case shared.StorageTypeCeph:
return "ceph", nil
- case storageTypeDir:
+ case shared.StorageTypeDir:
return "dir", nil
- case storageTypeLvm:
+ case shared.StorageTypeLvm:
return "lvm", nil
- case storageTypeMock:
+ case shared.StorageTypeMock:
return "mock", nil
- case storageTypeZfs:
+ case shared.StorageTypeZfs:
return "zfs", nil
}
return "", fmt.Errorf("invalid storage type")
}
-func storageStringToType(sName string) (storageType, error) {
+func storageStringToType(sName string) (shared.StorageType, error) {
switch sName {
case "btrfs":
- return storageTypeBtrfs, nil
+ return shared.StorageTypeBtrfs, nil
case "ceph":
- return storageTypeCeph, nil
+ return shared.StorageTypeCeph, nil
case "dir":
- return storageTypeDir, nil
+ return shared.StorageTypeDir, nil
case "lvm":
- return storageTypeLvm, nil
+ return shared.StorageTypeLvm, nil
case "mock":
- return storageTypeMock, nil
+ return shared.StorageTypeMock, nil
case "zfs":
- return storageTypeZfs, nil
+ return shared.StorageTypeZfs, nil
}
return -1, fmt.Errorf("invalid storage type name")
@@ -132,7 +120,7 @@ func storageStringToType(sName string) (storageType, error) {
type storage interface {
// Functions dealing with basic driver properties only.
StorageCoreInit() error
- GetStorageType() storageType
+ GetStorageType() shared.StorageType
GetStorageTypeName() string
GetStorageTypeVersion() string
@@ -234,42 +222,42 @@ func storageCoreInit(driver string) (storage, error) {
}
switch sType {
- case storageTypeBtrfs:
+ case shared.StorageTypeBtrfs:
btrfs := storageBtrfs{}
err = btrfs.StorageCoreInit()
if err != nil {
return nil, err
}
return &btrfs, nil
- case storageTypeDir:
+ case shared.StorageTypeDir:
dir := storageDir{}
err = dir.StorageCoreInit()
if err != nil {
return nil, err
}
return &dir, nil
- case storageTypeCeph:
+ case shared.StorageTypeCeph:
ceph := storageCeph{}
err = ceph.StorageCoreInit()
if err != nil {
return nil, err
}
return &ceph, nil
- case storageTypeLvm:
+ case shared.StorageTypeLvm:
lvm := storageLvm{}
err = lvm.StorageCoreInit()
if err != nil {
return nil, err
}
return &lvm, nil
- case storageTypeMock:
+ case shared.StorageTypeMock:
mock := storageMock{}
err = mock.StorageCoreInit()
if err != nil {
return nil, err
}
return &mock, nil
- case storageTypeZfs:
+ case shared.StorageTypeZfs:
zfs := storageZfs{}
err = zfs.StorageCoreInit()
if err != nil {
@@ -310,7 +298,7 @@ func storageInit(s *state.State, poolName string, volumeName string, volumeType
}
switch sType {
- case storageTypeBtrfs:
+ case shared.StorageTypeBtrfs:
btrfs := storageBtrfs{}
btrfs.poolID = poolID
btrfs.pool = pool
@@ -322,7 +310,7 @@ func storageInit(s *state.State, poolName string, volumeName string, volumeType
return nil, err
}
return &btrfs, nil
- case storageTypeDir:
+ case shared.StorageTypeDir:
dir := storageDir{}
dir.poolID = poolID
dir.pool = pool
@@ -334,7 +322,7 @@ func storageInit(s *state.State, poolName string, volumeName string, volumeType
return nil, err
}
return &dir, nil
- case storageTypeCeph:
+ case shared.StorageTypeCeph:
ceph := storageCeph{}
ceph.poolID = poolID
ceph.pool = pool
@@ -346,7 +334,7 @@ func storageInit(s *state.State, poolName string, volumeName string, volumeType
return nil, err
}
return &ceph, nil
- case storageTypeLvm:
+ case shared.StorageTypeLvm:
lvm := storageLvm{}
lvm.poolID = poolID
lvm.pool = pool
@@ -358,7 +346,7 @@ func storageInit(s *state.State, poolName string, volumeName string, volumeType
return nil, err
}
return &lvm, nil
- case storageTypeMock:
+ case shared.StorageTypeMock:
mock := storageMock{}
mock.poolID = poolID
mock.pool = pool
@@ -370,7 +358,7 @@ func storageInit(s *state.State, poolName string, volumeName string, volumeType
return nil, err
}
return &mock, nil
- case storageTypeZfs:
+ case shared.StorageTypeZfs:
zfs := storageZfs{}
zfs.poolID = poolID
zfs.pool = pool
diff --git a/lxd/storage_btrfs.go b/lxd/storage_btrfs.go
index 1218ee271..6cbe83532 100644
--- a/lxd/storage_btrfs.go
+++ b/lxd/storage_btrfs.go
@@ -61,7 +61,7 @@ func (s *storageBtrfs) getCustomSubvolumePath(poolName string) string {
}
func (s *storageBtrfs) StorageCoreInit() error {
- s.sType = storageTypeBtrfs
+ s.sType = shared.StorageTypeBtrfs
typeName, err := storageTypeToString(s.sType)
if err != nil {
return err
@@ -1433,7 +1433,7 @@ func (s *storageBtrfs) ImageCreate(fingerprint string) error {
// Unpack the image in imageMntPoint.
imagePath := shared.VarPath("images", fingerprint)
- err = unpackImage(imagePath, tmpImageSubvolumeName, storageTypeBtrfs, s.s.OS.RunningInUserNS)
+ err = unpackImage(imagePath, tmpImageSubvolumeName, shared.StorageTypeBtrfs, s.s.OS.RunningInUserNS)
if err != nil {
return err
}
diff --git a/lxd/storage_ceph.go b/lxd/storage_ceph.go
index 1af3cc410..5bc742f85 100644
--- a/lxd/storage_ceph.go
+++ b/lxd/storage_ceph.go
@@ -21,7 +21,7 @@ type storageCeph struct {
}
func (s *storageCeph) StorageCoreInit() error {
- s.sType = storageTypeCeph
+ s.sType = shared.StorageTypeCeph
typeName, err := storageTypeToString(s.sType)
if err != nil {
return err
@@ -2368,7 +2368,7 @@ func (s *storageCeph) ImageCreate(fingerprint string) error {
// rsync contents into image
imagePath := shared.VarPath("images", fingerprint)
- err = unpackImage(imagePath, imageMntPoint, storageTypeCeph, s.s.OS.RunningInUserNS)
+ err = unpackImage(imagePath, imageMntPoint, shared.StorageTypeCeph, s.s.OS.RunningInUserNS)
if err != nil {
logger.Errorf(`Failed to unpack image for RBD storage `+
`volume for image "%s" on storage pool "%s": %s`,
diff --git a/lxd/storage_dir.go b/lxd/storage_dir.go
index 78e562fa7..afbb324a4 100644
--- a/lxd/storage_dir.go
+++ b/lxd/storage_dir.go
@@ -21,7 +21,7 @@ type storageDir struct {
// Only initialize the minimal information we need about a given storage type.
func (s *storageDir) StorageCoreInit() error {
- s.sType = storageTypeDir
+ s.sType = shared.StorageTypeDir
typeName, err := storageTypeToString(s.sType)
if err != nil {
return err
@@ -509,7 +509,7 @@ func (s *storageDir) ContainerCreateFromImage(container container, imageFingerpr
}()
imagePath := shared.VarPath("images", imageFingerprint)
- err = unpackImage(imagePath, containerMntPoint, storageTypeDir, s.s.OS.RunningInUserNS)
+ err = unpackImage(imagePath, containerMntPoint, shared.StorageTypeDir, s.s.OS.RunningInUserNS)
if err != nil {
return err
}
diff --git a/lxd/storage_lvm.go b/lxd/storage_lvm.go
index 95c596b1c..8866b0d17 100644
--- a/lxd/storage_lvm.go
+++ b/lxd/storage_lvm.go
@@ -25,7 +25,7 @@ type storageLvm struct {
// Only initialize the minimal information we need about a given storage type.
func (s *storageLvm) StorageCoreInit() error {
- s.sType = storageTypeLvm
+ s.sType = shared.StorageTypeLvm
typeName, err := storageTypeToString(s.sType)
if err != nil {
return err
@@ -1627,7 +1627,7 @@ func (s *storageLvm) ImageCreate(fingerprint string) error {
}
imagePath := shared.VarPath("images", fingerprint)
- err = unpackImage(imagePath, imageMntPoint, storageTypeLvm, s.s.OS.RunningInUserNS)
+ err = unpackImage(imagePath, imageMntPoint, shared.StorageTypeLvm, s.s.OS.RunningInUserNS)
if err != nil {
return err
}
diff --git a/lxd/storage_lvm_utils.go b/lxd/storage_lvm_utils.go
index f9f3f1340..b71dcb1e7 100644
--- a/lxd/storage_lvm_utils.go
+++ b/lxd/storage_lvm_utils.go
@@ -461,7 +461,7 @@ func (s *storageLvm) containerCreateFromImageLv(c container, fp string) error {
imagePath := shared.VarPath("images", fp)
containerMntPoint := getContainerMountPoint(s.pool.Name, containerName)
- err = unpackImage(imagePath, containerMntPoint, storageTypeLvm, s.s.OS.RunningInUserNS)
+ err = unpackImage(imagePath, containerMntPoint, shared.StorageTypeLvm, s.s.OS.RunningInUserNS)
if err != nil {
logger.Errorf(`Failed to unpack image "%s" into non-thinpool `+
`LVM storage volume "%s" for container "%s" on `+
diff --git a/lxd/storage_migration.go b/lxd/storage_migration.go
index 76308e96d..57f122bb7 100644
--- a/lxd/storage_migration.go
+++ b/lxd/storage_migration.go
@@ -149,7 +149,7 @@ func rsyncMigrationSink(live bool, container container, snapshots []*Snapshot, c
return fmt.Errorf("the container's root device is missing the pool property")
}
- isDirBackend := container.Storage().GetStorageType() == storageTypeDir
+ isDirBackend := container.Storage().GetStorageType() == shared.StorageTypeDir
if isDirBackend {
if !containerOnly {
for _, snap := range snapshots {
diff --git a/lxd/storage_mock.go b/lxd/storage_mock.go
index aacf7cdc0..bef1f3dac 100644
--- a/lxd/storage_mock.go
+++ b/lxd/storage_mock.go
@@ -5,6 +5,7 @@ import (
"github.com/gorilla/websocket"
+ "github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
"github.com/lxc/lxd/shared/idmap"
"github.com/lxc/lxd/shared/logger"
@@ -15,7 +16,7 @@ type storageMock struct {
}
func (s *storageMock) StorageCoreInit() error {
- s.sType = storageTypeMock
+ s.sType = shared.StorageTypeMock
typeName, err := storageTypeToString(s.sType)
if err != nil {
return err
diff --git a/lxd/storage_shared.go b/lxd/storage_shared.go
index f2e1b692d..e4ab8c288 100644
--- a/lxd/storage_shared.go
+++ b/lxd/storage_shared.go
@@ -11,7 +11,7 @@ import (
)
type storageShared struct {
- sType storageType
+ sType shared.StorageType
sTypeName string
sTypeVersion string
@@ -24,7 +24,7 @@ type storageShared struct {
volume *api.StorageVolume
}
-func (s *storageShared) GetStorageType() storageType {
+func (s *storageShared) GetStorageType() shared.StorageType {
return s.sType
}
diff --git a/lxd/storage_zfs.go b/lxd/storage_zfs.go
index 30a616b93..137621ad6 100644
--- a/lxd/storage_zfs.go
+++ b/lxd/storage_zfs.go
@@ -40,7 +40,7 @@ func (s *storageZfs) getOnDiskPoolName() string {
// Only initialize the minimal information we need about a given storage type.
func (s *storageZfs) StorageCoreInit() error {
- s.sType = storageTypeZfs
+ s.sType = shared.StorageTypeZfs
typeName, err := storageTypeToString(s.sType)
if err != nil {
return err
@@ -1907,7 +1907,7 @@ func (s *storageZfs) ImageCreate(fingerprint string) error {
}
// Unpack the image into the temporary mountpoint.
- err = unpackImage(imagePath, tmpImageDir, storageTypeZfs, s.s.OS.RunningInUserNS)
+ err = unpackImage(imagePath, tmpImageDir, shared.StorageTypeZfs, s.s.OS.RunningInUserNS)
if err != nil {
return err
}
diff --git a/shared/archive.go b/shared/archive.go
new file mode 100644
index 000000000..cc576a727
--- /dev/null
+++ b/shared/archive.go
@@ -0,0 +1,120 @@
+package shared
+
+import (
+ "bytes"
+ "fmt"
+ "os"
+ "strings"
+ "syscall"
+
+ "github.com/lxc/lxd/shared/logger"
+)
+
+func DetectCompression(fname string) ([]string, string, error) {
+ f, err := os.Open(fname)
+ if err != nil {
+ return []string{""}, "", err
+ }
+ defer f.Close()
+
+ // read header parts to detect compression method
+ // bz2 - 2 bytes, 'BZ' signature/magic number
+ // gz - 2 bytes, 0x1f 0x8b
+ // lzma - 6 bytes, { [0x000, 0xE0], '7', 'z', 'X', 'Z', 0x00 } -
+ // xy - 6 bytes, header format { 0xFD, '7', 'z', 'X', 'Z', 0x00 }
+ // tar - 263 bytes, trying to get ustar from 257 - 262
+ header := make([]byte, 263)
+ _, err = f.Read(header)
+ if err != nil {
+ return []string{""}, "", err
+ }
+
+ switch {
+ case bytes.Equal(header[0:2], []byte{'B', 'Z'}):
+ return []string{"-jxf"}, ".tar.bz2", nil
+ case bytes.Equal(header[0:2], []byte{0x1f, 0x8b}):
+ return []string{"-zxf"}, ".tar.gz", nil
+ case (bytes.Equal(header[1:5], []byte{'7', 'z', 'X', 'Z'}) && header[0] == 0xFD):
+ return []string{"-Jxf"}, ".tar.xz", nil
+ case (bytes.Equal(header[1:5], []byte{'7', 'z', 'X', 'Z'}) && header[0] != 0xFD):
+ return []string{"--lzma", "-xf"}, ".tar.lzma", nil
+ case bytes.Equal(header[0:3], []byte{0x5d, 0x00, 0x00}):
+ return []string{"--lzma", "-xf"}, ".tar.lzma", nil
+ case bytes.Equal(header[257:262], []byte{'u', 's', 't', 'a', 'r'}):
+ return []string{"-xf"}, ".tar", nil
+ case bytes.Equal(header[0:4], []byte{'h', 's', 'q', 's'}):
+ return []string{""}, ".squashfs", nil
+ default:
+ return []string{""}, "", fmt.Errorf("Unsupported compression")
+ }
+
+}
+
+func Unpack(file string, path string, sType StorageType, runningInUserns bool) error {
+ extractArgs, extension, err := DetectCompression(file)
+ if err != nil {
+ return err
+ }
+
+ command := ""
+ args := []string{}
+ if strings.HasPrefix(extension, ".tar") {
+ command = "tar"
+ if runningInUserns {
+ args = append(args, "--wildcards")
+ args = append(args, "--exclude=dev/*")
+ args = append(args, "--exclude=./dev/*")
+ args = append(args, "--exclude=rootfs/dev/*")
+ args = append(args, "--exclude=rootfs/./dev/*")
+ }
+ args = append(args, "-C", path, "--numeric-owner")
+ args = append(args, extractArgs...)
+ args = append(args, file)
+ } else if strings.HasPrefix(extension, ".squashfs") {
+ command = "unsquashfs"
+ args = append(args, "-f", "-d", path, "-n")
+
+ // Limit unsquashfs chunk size to 10% of memory and up to 256MB (default)
+ // When running on a low memory system, also disable multi-processing
+ mem, err := DeviceTotalMemory()
+ mem = mem / 1024 / 1024 / 10
+ if err == nil && mem < 256 {
+ args = append(args, "-da", fmt.Sprintf("%d", mem), "-fr", fmt.Sprintf("%d", mem), "-p", "1")
+ }
+
+ args = append(args, file)
+ } else {
+ return fmt.Errorf("Unsupported image format: %s", extension)
+ }
+
+ output, err := RunCommand(command, args...)
+ if err != nil {
+ // Check if we ran out of space
+ fs := syscall.Statfs_t{}
+
+ err1 := syscall.Statfs(path, &fs)
+ if err1 != nil {
+ return err1
+ }
+
+ // Check if we're running out of space
+ if int64(fs.Bfree) < int64(2*fs.Bsize) {
+ if sType == StorageTypeLvm {
+ return fmt.Errorf("Unable to unpack image, run out of disk space (consider increasing your pool's volume.size).")
+ } else {
+ return fmt.Errorf("Unable to unpack image, run out of disk space.")
+ }
+ }
+
+ co := output
+ logger.Debugf("Unpacking failed")
+ logger.Debugf(co)
+
+ // Truncate the output to a single line for inclusion in the error
+ // message. The first line isn't guaranteed to pinpoint the issue,
+ // but it's better than nothing and better than a multi-line message.
+ return fmt.Errorf("Unpack failed, %s. %s", err, strings.SplitN(co, "\n", 2)[0])
+ }
+
+ return nil
+}
diff --git a/shared/devices.go b/shared/devices.go
new file mode 100644
index 000000000..df281970a
--- /dev/null
+++ b/shared/devices.go
@@ -0,0 +1,42 @@
+package shared
+
+import (
+ "bufio"
+ "fmt"
+ "os"
+ "strings"
+)
+
+func DeviceTotalMemory() (int64, error) {
+ // Open /proc/meminfo
+ f, err := os.Open("/proc/meminfo")
+ if err != nil {
+ return -1, err
+ }
+ defer f.Close()
+
+ // Read it line by line
+ scan := bufio.NewScanner(f)
+ for scan.Scan() {
+ line := scan.Text()
+
+ // We only care about MemTotal
+ if !strings.HasPrefix(line, "MemTotal:") {
+ continue
+ }
+
+ // Extract the before last (value) and last (unit) fields
+ fields := strings.Split(line, " ")
+ value := fields[len(fields)-2] + fields[len(fields)-1]
+
+ // Feed the result to shared.ParseByteSizeString to get an int value
+ valueBytes, err := ParseByteSizeString(value)
+ if err != nil {
+ return -1, err
+ }
+
+ return valueBytes, nil
+ }
+
+ return -1, fmt.Errorf("Couldn't find MemTotal")
+}
diff --git a/shared/storage.go b/shared/storage.go
new file mode 100644
index 000000000..d13209fd5
--- /dev/null
+++ b/shared/storage.go
@@ -0,0 +1,13 @@
+package shared
+
+// StorageType defines the type of a storage
+type StorageType int
+
+const (
+ StorageTypeBtrfs StorageType = iota
+ StorageTypeCeph
+ StorageTypeDir
+ StorageTypeLvm
+ StorageTypeMock
+ StorageTypeZfs
+)
From 1c8d4cc81b18c8d37dd14393851716c6a163e0c7 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Fri, 12 Jan 2018 14:37:51 +0100
Subject: [PATCH 2/2] *: move download function to shared
This moves the DownloadFileSha256 function and its dependent code to
shared.
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
client/interfaces.go | 20 ++------------
client/lxd_images.go | 4 +--
client/simplestreams_images.go | 4 +--
client/util.go | 62 ------------------------------------------
lxc/utils.go | 3 +-
lxd/daemon_images.go | 4 +--
shared/util.go | 62 ++++++++++++++++++++++++++++++++++++++++++
7 files changed, 73 insertions(+), 86 deletions(-)
diff --git a/client/interfaces.go b/client/interfaces.go
index b88fbef2d..6bfc245f0 100644
--- a/client/interfaces.go
+++ b/client/interfaces.go
@@ -6,6 +6,7 @@ import (
"github.com/gorilla/websocket"
+ "github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
"github.com/lxc/lxd/shared/cancel"
)
@@ -172,21 +173,6 @@ type ConnectionInfo struct {
Protocol string
}
-// The ProgressData struct represents new progress information on an operation
-type ProgressData struct {
- // Preferred string repreentation of progress (always set)
- Text string
-
- // Progress in percent
- Percentage int
-
- // Number of bytes transferred (for files)
- TransferredBytes int64
-
- // Total number of bytes (for files)
- TotalBytes int64
-}
-
// The ImageCreateArgs struct is used for direct image upload
type ImageCreateArgs struct {
// Reader for the meta file
@@ -202,7 +188,7 @@ type ImageCreateArgs struct {
RootfsName string
// Progress handler (called with upload progress)
- ProgressHandler func(progress ProgressData)
+ ProgressHandler func(progress shared.ProgressData)
}
// The ImageFileRequest struct is used for an image download request
@@ -214,7 +200,7 @@ type ImageFileRequest struct {
RootfsFile io.WriteSeeker
// Progress handler (called whenever some progress is made)
- ProgressHandler func(progress ProgressData)
+ ProgressHandler func(progress shared.ProgressData)
// A canceler that can be used to interrupt some part of the image download request
Canceler *cancel.Canceler
diff --git a/client/lxd_images.go b/client/lxd_images.go
index 8d0889640..385ad04de 100644
--- a/client/lxd_images.go
+++ b/client/lxd_images.go
@@ -145,7 +145,7 @@ func (r *ProtocolLXD) GetPrivateImageFile(fingerprint string, secret string, req
Tracker: &ioprogress.ProgressTracker{
Length: response.ContentLength,
Handler: func(percent int64, speed int64) {
- req.ProgressHandler(ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, shared.GetByteSizeString(speed, 2))})
+ req.ProgressHandler(shared.ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, shared.GetByteSizeString(speed, 2))})
},
},
}
@@ -363,7 +363,7 @@ func (r *ProtocolLXD) CreateImage(image api.ImagesPost, args *ImageCreateArgs) (
Tracker: &ioprogress.ProgressTracker{
Length: size,
Handler: func(percent int64, speed int64) {
- args.ProgressHandler(ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, shared.GetByteSizeString(speed, 2))})
+ args.ProgressHandler(shared.ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, shared.GetByteSizeString(speed, 2))})
},
},
}
diff --git a/client/simplestreams_images.go b/client/simplestreams_images.go
index 39a1f5e87..065bedabe 100644
--- a/client/simplestreams_images.go
+++ b/client/simplestreams_images.go
@@ -67,11 +67,11 @@ func (r *ProtocolSimpleStreams) GetImageFile(fingerprint string, req ImageFileRe
// Try over http
url := fmt.Sprintf("http://%s/%s", strings.TrimPrefix(r.httpHost, "https://"), path)
- size, err := downloadFileSha256(r.http, r.httpUserAgent, req.ProgressHandler, req.Canceler, filename, url, sha256, target)
+ size, err := shared.DownloadFileSha256(r.http, r.httpUserAgent, req.ProgressHandler, req.Canceler, filename, url, sha256, target)
if err != nil {
// Try over https
url = fmt.Sprintf("%s/%s", r.httpHost, path)
- size, err = downloadFileSha256(r.http, r.httpUserAgent, req.ProgressHandler, req.Canceler, filename, url, sha256, target)
+ size, err = shared.DownloadFileSha256(r.http, r.httpUserAgent, req.ProgressHandler, req.Canceler, filename, url, sha256, target)
if err != nil {
return -1, err
}
diff --git a/client/util.go b/client/util.go
index e041fd979..256a184f8 100644
--- a/client/util.go
+++ b/client/util.go
@@ -1,16 +1,12 @@
package lxd
import (
- "crypto/sha256"
- "fmt"
"io"
"net"
"net/http"
"net/url"
"github.com/lxc/lxd/shared"
- "github.com/lxc/lxd/shared/cancel"
- "github.com/lxc/lxd/shared/ioprogress"
)
func tlsHTTPClient(client *http.Client, tlsClientCert string, tlsClientKey string, tlsCA string, tlsServerCert string, insecureSkipVerify bool, proxy func(req *http.Request) (*url.URL, error)) (*http.Client, error) {
@@ -84,64 +80,6 @@ func unixHTTPClient(client *http.Client, path string) (*http.Client, error) {
return client, nil
}
-func downloadFileSha256(httpClient *http.Client, useragent string, progress func(progress ProgressData), canceler *cancel.Canceler, filename string, url string, hash string, target io.WriteSeeker) (int64, error) {
- // Always seek to the beginning
- target.Seek(0, 0)
-
- // Prepare the download request
- req, err := http.NewRequest("GET", url, nil)
- if err != nil {
- return -1, err
- }
-
- if useragent != "" {
- req.Header.Set("User-Agent", useragent)
- }
-
- // Perform the request
- r, doneCh, err := cancel.CancelableDownload(canceler, httpClient, req)
- if err != nil {
- return -1, err
- }
- defer r.Body.Close()
- defer close(doneCh)
-
- if r.StatusCode != http.StatusOK {
- return -1, fmt.Errorf("Unable to fetch %s: %s", url, r.Status)
- }
-
- // Handle the data
- body := r.Body
- if progress != nil {
- body = &ioprogress.ProgressReader{
- ReadCloser: r.Body,
- Tracker: &ioprogress.ProgressTracker{
- Length: r.ContentLength,
- Handler: func(percent int64, speed int64) {
- if filename != "" {
- progress(ProgressData{Text: fmt.Sprintf("%s: %d%% (%s/s)", filename, percent, shared.GetByteSizeString(speed, 2))})
- } else {
- progress(ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, shared.GetByteSizeString(speed, 2))})
- }
- },
- },
- }
- }
-
- sha256 := sha256.New()
- size, err := io.Copy(io.MultiWriter(target, sha256), body)
- if err != nil {
- return -1, err
- }
-
- result := fmt.Sprintf("%x", sha256.Sum(nil))
- if result != hash {
- return -1, fmt.Errorf("Hash mismatch for %s: %s != %s", url, result, hash)
- }
-
- return size, nil
-}
-
type nullReadWriteCloser int
func (nullReadWriteCloser) Close() error { return nil }
diff --git a/lxc/utils.go b/lxc/utils.go
index 0215bd394..358269841 100644
--- a/lxc/utils.go
+++ b/lxc/utils.go
@@ -13,6 +13,7 @@ import (
"time"
"github.com/lxc/lxd/client"
+ "github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
"github.com/lxc/lxd/shared/i18n"
)
@@ -119,7 +120,7 @@ func (p *ProgressRenderer) Warn(status string, timeout time.Duration) {
fmt.Print(msg)
}
-func (p *ProgressRenderer) UpdateProgress(progress lxd.ProgressData) {
+func (p *ProgressRenderer) UpdateProgress(progress shared.ProgressData) {
p.Update(progress.Text)
}
diff --git a/lxd/daemon_images.go b/lxd/daemon_images.go
index abc81a965..ca5d49f16 100644
--- a/lxd/daemon_images.go
+++ b/lxd/daemon_images.go
@@ -343,7 +343,7 @@ func (d *Daemon) ImageDownload(op *operation, server string, protocol string, ce
defer cleanup()
// Setup a progress handler
- progress := func(progress lxd.ProgressData) {
+ progress := func(progress shared.ProgressData) {
if op == nil {
return
}
@@ -462,7 +462,7 @@ func (d *Daemon) ImageDownload(op *operation, server string, protocol string, ce
Tracker: &ioprogress.ProgressTracker{
Length: raw.ContentLength,
Handler: func(percent int64, speed int64) {
- progress(lxd.ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, shared.GetByteSizeString(speed, 2))})
+ progress(shared.ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, shared.GetByteSizeString(speed, 2))})
},
},
}
diff --git a/shared/util.go b/shared/util.go
index e039905f0..f262c3aca 100644
--- a/shared/util.go
+++ b/shared/util.go
@@ -4,6 +4,7 @@ import (
"bufio"
"bytes"
"crypto/rand"
+ "crypto/sha256"
"encoding/gob"
"encoding/hex"
"encoding/json"
@@ -24,6 +25,9 @@ import (
"strings"
"time"
"unicode"
+
+ "github.com/lxc/lxd/shared/cancel"
+ "github.com/lxc/lxd/shared/ioprogress"
)
const SnapshotDelimiter = "/"
@@ -902,3 +906,61 @@ func EscapePathFstab(path string) string {
"\\", "\\\\")
return r.Replace(path)
}
+
+func DownloadFileSha256(httpClient *http.Client, useragent string, progress func(progress ProgressData), canceler *cancel.Canceler, filename string, url string, hash string, target io.WriteSeeker) (int64, error) {
+ // Always seek to the beginning
+ target.Seek(0, 0)
+
+ // Prepare the download request
+ req, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ return -1, err
+ }
+
+ if useragent != "" {
+ req.Header.Set("User-Agent", useragent)
+ }
+
+ // Perform the request
+ r, doneCh, err := cancel.CancelableDownload(canceler, httpClient, req)
+ if err != nil {
+ return -1, err
+ }
+ defer r.Body.Close()
+ defer close(doneCh)
+
+ if r.StatusCode != http.StatusOK {
+ return -1, fmt.Errorf("Unable to fetch %s: %s", url, r.Status)
+ }
+
+ // Handle the data
+ body := r.Body
+ if progress != nil {
+ body = &ioprogress.ProgressReader{
+ ReadCloser: r.Body,
+ Tracker: &ioprogress.ProgressTracker{
+ Length: r.ContentLength,
+ Handler: func(percent int64, speed int64) {
+ if filename != "" {
+ progress(ProgressData{Text: fmt.Sprintf("%s: %d%% (%s/s)", filename, percent, GetByteSizeString(speed, 2))})
+ } else {
+ progress(ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, GetByteSizeString(speed, 2))})
+ }
+ },
+ },
+ }
+ }
+
+ sha256 := sha256.New()
+ size, err := io.Copy(io.MultiWriter(target, sha256), body)
+ if err != nil {
+ return -1, err
+ }
+
+ result := fmt.Sprintf("%x", sha256.Sum(nil))
+ if result != hash {
+ return -1, fmt.Errorf("Hash mismatch for %s: %s != %s", url, result, hash)
+ }
+
+ return size, nil
+}
More information about the lxc-devel
mailing list