[lxc-devel] [lxd/master] Support squashfs as compression algorithm
ulziibuyan on Github
lxc-bot at linuxcontainers.org
Mon Oct 28 00:08:55 UTC 2019
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 324 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20191027/7234ac7f/attachment.bin>
-------------- next part --------------
From c3dc603edb8538ba309d620b644ebf3cecc4d3d1 Mon Sep 17 00:00:00 2001
From: Uzi Erdenebileg <lzijbuan at gmail.com>
Date: Sat, 26 Oct 2019 05:47:36 +0000
Subject: [PATCH 1/5] api: Add compression_squashfs extension
Signed-off-by: Uzi Erdenebileg <lzijbuan at gmail.com>
---
doc/api-extensions.md | 3 +++
shared/version/api.go | 1 +
2 files changed, 4 insertions(+)
diff --git a/doc/api-extensions.md b/doc/api-extensions.md
index e33d9ecced..1ffb73084e 100644
--- a/doc/api-extensions.md
+++ b/doc/api-extensions.md
@@ -866,3 +866,6 @@ Adds the `security.syscalls.intercept.mount`,
`security.syscalls.intercept.mount.shift` configuration keys to control whether
and how the mount system call will be interecepted by LXD and processed with
elevated permissions.
+
+## compression_squashfs
+Adds support for importing/exporting of images/backups using SquashFS file system format.
diff --git a/shared/version/api.go b/shared/version/api.go
index f6f64cb741..6498b1d156 100644
--- a/shared/version/api.go
+++ b/shared/version/api.go
@@ -173,6 +173,7 @@ var APIExtensions = []string{
"backup_compression_algorithm",
"ceph_data_pool_name",
"container_syscall_intercept_mount",
+ "compression_squashfs",
}
// APIExtensionsCount returns the number of available API extensions.
From be109f623f0b7db4a3537a1c5cbaafaa4adcc57d Mon Sep 17 00:00:00 2001
From: Uzi Erdenebileg <lzijbuan at gmail.com>
Date: Sat, 26 Oct 2019 05:58:46 +0000
Subject: [PATCH 2/5] lxd/cluster: Validate squashfs-tools-ng executables
Signed-off-by: Uzi Erdenebileg <lzijbuan at gmail.com>
---
lxd/cluster/config.go | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/lxd/cluster/config.go b/lxd/cluster/config.go
index dacd9fb186..2a7381664b 100644
--- a/lxd/cluster/config.go
+++ b/lxd/cluster/config.go
@@ -321,6 +321,11 @@ func validateCompression(value string) error {
return nil
}
+ // Going to look up tar2sqfs executable binary
+ if value == "squashfs" {
+ value = "tar2sqfs"
+ }
+
_, err := exec.LookPath(value)
return err
}
From f3ebcfbbe7cc0c51b068dc16d5b12fe31df27e1b Mon Sep 17 00:00:00 2001
From: Uzi Erdenebileg <lzijbuan at gmail.com>
Date: Sat, 26 Oct 2019 20:24:00 +0000
Subject: [PATCH 3/5] lxd: Modify compressFile() to support SquashFS
Signed-off-by: Uzi Erdenebileg <lzijbuan at gmail.com>
---
lxd/images.go | 47 +++++++++++++++++++++++++++++++++++++++--------
1 file changed, 39 insertions(+), 8 deletions(-)
diff --git a/lxd/images.go b/lxd/images.go
index 67b95e67b4..2880cec0c6 100644
--- a/lxd/images.go
+++ b/lxd/images.go
@@ -139,17 +139,48 @@ func unpackImage(imagefname string, destpath string, sType storageType, runningI
func compressFile(compress string, infile io.Reader, outfile io.Writer) error {
reproducible := []string{"gzip"}
+ var cmd *exec.Cmd
- args := []string{"-c"}
- if shared.StringInSlice(compress, reproducible) {
- args = append(args, "-n")
- }
+ if compress == "squashfs" {
+ // 'tar2sqfs' do not support writing to stdout. So write to a temporary
+ // file first and then replay the compressed content to outfile.
+ tempfile, err := ioutil.TempFile("", "lxd_compress_")
+ if err != nil {
+ return err
+ }
+ defer tempfile.Close()
+ defer os.Remove(tempfile.Name())
- cmd := exec.Command(compress, args...)
- cmd.Stdin = infile
- cmd.Stdout = outfile
+ // Prepare 'tar2sqfs' arguments
+ args := []string{"tar2sqfs", "--no-skip", "--force",
+ "--compressor", "xz", tempfile.Name()}
+ cmd = exec.Command(args[0], args[1:]...)
+ cmd.Stdin = infile
- return cmd.Run()
+ err = cmd.Run()
+ if err != nil {
+ return err
+ }
+ // Replay the result to outfile
+ tempfile.Seek(0, 0)
+ _, err = io.Copy(outfile, tempfile)
+ if err != nil {
+ return err
+ }
+
+ } else {
+ args := []string{"-c"}
+ if shared.StringInSlice(compress, reproducible) {
+ args = append(args, "-n")
+ }
+
+ cmd := exec.Command(compress, args...)
+ cmd.Stdin = infile
+ cmd.Stdout = outfile
+ cmd.Run()
+ }
+
+ return nil
}
/*
From fc7b1b3d9700440b05fae35782a82a051788af19 Mon Sep 17 00:00:00 2001
From: Uzi Erdenebileg <lzijbuan at gmail.com>
Date: Sun, 27 Oct 2019 21:44:08 +0000
Subject: [PATCH 4/5] lxd: Support SquashFS compressed backup imports
Signed-off-by: Uzi Erdenebileg <lzijbuan at gmail.com>
---
lxd/backup/backup.go | 23 +++++++++++++++++-
lxd/container.go | 53 +++++++++++++++++++++++++++++++++++++----
shared/archive_linux.go | 3 ++-
3 files changed, 72 insertions(+), 7 deletions(-)
diff --git a/lxd/backup/backup.go b/lxd/backup/backup.go
index 8f3fdc1067..79f6b8ab4b 100644
--- a/lxd/backup/backup.go
+++ b/lxd/backup/backup.go
@@ -4,6 +4,7 @@ import (
"archive/tar"
"fmt"
"io"
+ "io/ioutil"
"os"
"os/exec"
"strings"
@@ -47,7 +48,7 @@ func GetInfo(r io.ReadSeeker) (*Info, error) {
// Extract
r.Seek(0, 0)
- _, _, unpacker, err := shared.DetectCompressionFile(r)
+ _, algo, unpacker, err := shared.DetectCompressionFile(r)
if err != nil {
return nil, err
}
@@ -58,6 +59,26 @@ func GetInfo(r io.ReadSeeker) (*Info, error) {
}
if len(unpacker) > 0 {
+ if algo == ".squashfs" {
+ // 'sqfs2tar' tool does not support reading from stdin. So
+ // create a temporary file to write the compressed data and
+ // pass it to the tool as program argument
+ tempfile, err := ioutil.TempFile("", "lxd_decompress_")
+ if err != nil {
+ return nil, err
+ }
+ defer os.Remove(tempfile.Name())
+
+ // Write compressed data
+ _, err = io.Copy(tempfile, r)
+ if err != nil {
+ return nil, err
+ }
+
+ tempfile.Close()
+ // Prepare to pass the temporary file as program argument
+ unpacker = append(unpacker, tempfile.Name())
+ }
cmd := exec.Command(unpacker[0], unpacker[1:]...)
cmd.Stdin = r
diff --git a/lxd/container.go b/lxd/container.go
index f4cfe3fe44..ac6948190e 100644
--- a/lxd/container.go
+++ b/lxd/container.go
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"io"
+ "io/ioutil"
"os"
"os/exec"
"path/filepath"
@@ -316,16 +317,58 @@ func containerCreateFromBackup(s *state.State, info backup.Info, data io.ReadSee
}
// Find the compression algorithm
- tarArgs, _, _, err := shared.DetectCompressionFile(data)
+ tarArgs, algo, decomArgs, err := shared.DetectCompressionFile(data)
if err != nil {
return nil, err
}
data.Seek(0, 0)
- // Unpack tarball
- err = pool.ContainerBackupLoad(info, data, tarArgs)
- if err != nil {
- return nil, err
+ if algo == ".squashfs" {
+ // Create a temporary file. 'sqfs2tar' tool do not support reading
+ // from stdin. So write the compressed data to the temporary file
+ // and pass it as program argument
+ tempfile, err := ioutil.TempFile("", "lxd_decompress_")
+ if err != nil {
+ return nil, err
+ }
+ defer os.Remove(tempfile.Name())
+
+ // Write the compressed data
+ _, err = io.Copy(tempfile, data)
+ if err != nil {
+ return nil, err
+ }
+
+ tempfile.Close()
+ // Prepare to pass the temporary file as program argument
+ decomArgs := append(decomArgs, tempfile.Name())
+
+ // Create another temporary file to write the decompressed data
+ tarData, err := ioutil.TempFile("", "lxd_decompress_")
+ if err != nil {
+ return nil, err
+ }
+ defer os.Remove(tarData.Name())
+ // Decompress to tarData temporary file
+ err = shared.RunCommandWithFds(nil, tarData,
+ decomArgs[0], decomArgs[1:]...)
+ if err != nil {
+ return nil, err
+ }
+ tarData.Seek(0, 0)
+
+ // Unpack tarball from decompressed temporary file
+ err = pool.ContainerBackupLoad(info, tarData, tarArgs)
+ if err != nil {
+ return nil, err
+ }
+
+ } else {
+ // Unpack tarball
+ err = pool.ContainerBackupLoad(info, data, tarArgs)
+ if err != nil {
+ return nil, err
+ }
}
if fixBackupFile || customPool {
diff --git a/shared/archive_linux.go b/shared/archive_linux.go
index 1e659d63ba..a77a09b70a 100644
--- a/shared/archive_linux.go
+++ b/shared/archive_linux.go
@@ -50,7 +50,8 @@ func DetectCompressionFile(f io.ReadSeeker) ([]string, string, []string, error)
case bytes.Equal(header[257:262], []byte{'u', 's', 't', 'a', 'r'}):
return []string{"-xf"}, ".tar", []string{}, nil
case bytes.Equal(header[0:4], []byte{'h', 's', 'q', 's'}):
- return []string{""}, ".squashfs", nil, nil
+ return []string{"-xf"}, ".squashfs",
+ []string{"sqfs2tar", "--no-skip"}, nil
default:
return nil, "", nil, fmt.Errorf("Unsupported compression")
}
From b5f7dcf964fd586993c5a92e2eaab9117df21faa Mon Sep 17 00:00:00 2001
From: Uzi Erdenebileg <lzijbuan at gmail.com>
Date: Sun, 27 Oct 2019 23:53:02 +0000
Subject: [PATCH 5/5] lxd: Add SquashFS compressed image publish/export support
Signed-off-by: Uzi Erdenebileg <lzijbuan at gmail.com>
---
lxd/images.go | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/lxd/images.go b/lxd/images.go
index 2880cec0c6..362ee28551 100644
--- a/lxd/images.go
+++ b/lxd/images.go
@@ -152,7 +152,7 @@ func compressFile(compress string, infile io.Reader, outfile io.Writer) error {
defer os.Remove(tempfile.Name())
// Prepare 'tar2sqfs' arguments
- args := []string{"tar2sqfs", "--no-skip", "--force",
+ args := []string{"tar2sqfs", "--force",
"--compressor", "xz", tempfile.Name()}
cmd = exec.Command(args[0], args[1:]...)
cmd.Stdin = infile
@@ -839,7 +839,7 @@ func getImageMetadata(fname string) (*api.ImageMetadata, string, error) {
defer r.Close()
// Decompress if needed
- _, _, unpacker, err := shared.DetectCompressionFile(r)
+ _, algo, unpacker, err := shared.DetectCompressionFile(r)
if err != nil {
return nil, "unknown", err
}
@@ -851,6 +851,9 @@ func getImageMetadata(fname string) (*api.ImageMetadata, string, error) {
// Open the tarball
if len(unpacker) > 0 {
+ if algo == ".squashfs" {
+ unpacker = append(unpacker, fname)
+ }
cmd := exec.Command(unpacker[0], unpacker[1:]...)
cmd.Stdin = r
More information about the lxc-devel
mailing list