[lxc-devel] [distrobuilder/master] Use proper source directory
monstermunchkin on Github
lxc-bot at linuxcontainers.org
Wed Mar 7 10:05:47 UTC 2018
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 441 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20180307/f03c5eab/attachment.bin>
-------------- next part --------------
From 17b0573f1ef971b29a227cd0e280aae41049fe05 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Tue, 6 Mar 2018 15:24:31 +0100
Subject: [PATCH 1/7] distrobuilder: Do not use "rootfs" directory with
build-dir sub-command
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
distrobuilder/main.go | 27 ++++++++++++---------------
1 file changed, 12 insertions(+), 15 deletions(-)
diff --git a/distrobuilder/main.go b/distrobuilder/main.go
index b3e020b..6adef8c 100644
--- a/distrobuilder/main.go
+++ b/distrobuilder/main.go
@@ -67,6 +67,7 @@ type cmdGlobal struct {
definition *shared.Definition
sourceDir string
targetDir string
+ rootfsDir string
}
func main() {
@@ -131,10 +132,10 @@ func (c *cmdGlobal) preRunBuild(cmd *cobra.Command, args []string) error {
}
}
- // Special case: Make cache directory the target directory. This will
- // prevent us from having to move ${cacheDir}/rootfs to ${target}/rootfs.
if cmd.CalledAs() == "build-dir" {
- c.flagCacheDir = c.targetDir
+ c.rootfsDir = c.targetDir
+ } else {
+ c.rootfsDir = filepath.Join(c.flagCacheDir, "rootfs")
}
var err error
@@ -151,10 +152,12 @@ func (c *cmdGlobal) preRunBuild(cmd *cobra.Command, args []string) error {
return err
}
- // Create cache directory
- err = os.MkdirAll(c.flagCacheDir, 0755)
- if err != nil {
- return err
+ // Create cache directory if we also plan on creating LXC or LXD images
+ if cmd.CalledAs() != "build-dir" {
+ err = os.MkdirAll(c.flagCacheDir, 0755)
+ if err != nil {
+ return err
+ }
}
// Get the downloader to use for this image
@@ -164,13 +167,13 @@ func (c *cmdGlobal) preRunBuild(cmd *cobra.Command, args []string) error {
}
// Download the root filesystem
- err = downloader.Run(c.definition.Source, c.definition.Image.Release, arch, c.flagCacheDir)
+ err = downloader.Run(c.definition.Source, c.definition.Image.Release, arch, c.rootfsDir)
if err != nil {
return fmt.Errorf("Error while downloading source: %s", err)
}
// Setup the mounts and chroot into the rootfs
- exitChroot, err := setupChroot(filepath.Join(c.flagCacheDir, "rootfs"))
+ exitChroot, err := setupChroot(c.rootfsDir)
if err != nil {
return fmt.Errorf("Failed to setup chroot: %s", err)
}
@@ -204,12 +207,6 @@ func (c *cmdGlobal) preRunPack(cmd *cobra.Command, args []string) error {
return err
}
- // Create cache directory
- err = os.MkdirAll(c.flagCacheDir, 0755)
- if err != nil {
- return err
- }
-
return nil
}
From cff68bec6177839e9c7a83cbbbce219b0ef7e98a Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Tue, 6 Mar 2018 16:48:28 +0100
Subject: [PATCH 2/7] sources: Handle rootfs properly
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
sources/alpine-http.go | 9 ++-------
sources/archlinux-http.go | 23 +++++++++++++----------
sources/centos-http.go | 21 ++++++++++++---------
sources/debootstrap.go | 6 +++---
sources/ubuntu-http.go | 4 ++--
5 files changed, 32 insertions(+), 31 deletions(-)
diff --git a/sources/alpine-http.go b/sources/alpine-http.go
index 17ffb4b..c5bf779 100644
--- a/sources/alpine-http.go
+++ b/sources/alpine-http.go
@@ -21,7 +21,7 @@ func NewAlpineLinuxHTTP() *AlpineLinuxHTTP {
}
// Run downloads an Alpine Linux mini root filesystem.
-func (s *AlpineLinuxHTTP) Run(source shared.DefinitionSource, release, arch, cacheDir string) error {
+func (s *AlpineLinuxHTTP) Run(source shared.DefinitionSource, release, arch, rootfsDir string) error {
fname := fmt.Sprintf("alpine-minirootfs-%s-%s.tar.gz", release, arch)
tarball := fmt.Sprintf("%s/v%s/releases/%s/%s", source.URL,
strings.Join(strings.Split(release, ".")[0:2], "."), arch, fname)
@@ -44,13 +44,8 @@ func (s *AlpineLinuxHTTP) Run(source shared.DefinitionSource, release, arch, cac
return errors.New("Failed to verify tarball")
}
- err = os.MkdirAll(filepath.Join(cacheDir, "rootfs"), 0755)
- if err != nil {
- return err
- }
-
// Unpack
- err = lxd.Unpack(filepath.Join(os.TempDir(), fname), filepath.Join(cacheDir, "rootfs"), false, false)
+ err = lxd.Unpack(filepath.Join(os.TempDir(), fname), rootfsDir, false, false)
if err != nil {
return err
}
diff --git a/sources/archlinux-http.go b/sources/archlinux-http.go
index b6b491d..efa22b5 100644
--- a/sources/archlinux-http.go
+++ b/sources/archlinux-http.go
@@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"os"
+ "path"
"path/filepath"
lxd "github.com/lxc/lxd/shared"
@@ -20,7 +21,7 @@ func NewArchLinuxHTTP() *ArchLinuxHTTP {
}
// Run downloads an Arch Linux tarball.
-func (s *ArchLinuxHTTP) Run(source shared.DefinitionSource, release, arch, cacheDir string) error {
+func (s *ArchLinuxHTTP) Run(source shared.DefinitionSource, release, arch, rootfsDir string) error {
fname := fmt.Sprintf("archlinux-bootstrap-%s-x86_64.tar.gz", release)
tarball := fmt.Sprintf("%s/%s/%s", source.URL, release, fname)
@@ -43,23 +44,25 @@ func (s *ArchLinuxHTTP) Run(source shared.DefinitionSource, release, arch, cache
return errors.New("Failed to verify tarball")
}
- err = os.MkdirAll(cacheDir, 0755)
+ // Unpack
+ err = lxd.Unpack(filepath.Join(os.TempDir(), fname), rootfsDir, false, false)
if err != nil {
return err
}
- // Unpack
- err = lxd.Unpack(filepath.Join(os.TempDir(), fname), cacheDir, false, false)
+ // Move everything inside 'root.x86_64' (which was is the tarball) to its
+ // parent directory
+ files, err := filepath.Glob(fmt.Sprintf("%s/*", filepath.Join(rootfsDir, "root.x86_64")))
if err != nil {
return err
}
- os.RemoveAll(filepath.Join(cacheDir, "rootfs"))
-
- err = os.Rename(filepath.Join(cacheDir, "root.x86_64"), filepath.Join(cacheDir, "rootfs"))
- if err != nil {
- return err
+ for _, file := range files {
+ err = os.Rename(file, filepath.Join(rootfsDir, path.Base(file)))
+ if err != nil {
+ return err
+ }
}
- return nil
+ return os.RemoveAll(filepath.Join(rootfsDir, "root.x86_64"))
}
diff --git a/sources/centos-http.go b/sources/centos-http.go
index e65c008..244e56d 100644
--- a/sources/centos-http.go
+++ b/sources/centos-http.go
@@ -17,8 +17,7 @@ import (
// CentOSHTTP represents the CentOS HTTP downloader.
type CentOSHTTP struct {
- fname string
- cacheDir string
+ fname string
}
// NewCentOSHTTP creates a new CentOSHTTP instance.
@@ -27,8 +26,7 @@ func NewCentOSHTTP() *CentOSHTTP {
}
// Run downloads the tarball and unpacks it.
-func (s *CentOSHTTP) Run(source shared.DefinitionSource, release, arch, cacheDir string) error {
- s.cacheDir = cacheDir
+func (s *CentOSHTTP) Run(source shared.DefinitionSource, release, arch, rootfsDir string) error {
baseURL := fmt.Sprintf("%s/%s/isos/%s/", source.URL, strings.Split(release, ".")[0], arch)
s.fname = getRelease(source.URL, release, source.Variant, arch)
@@ -51,16 +49,14 @@ func (s *CentOSHTTP) Run(source shared.DefinitionSource, release, arch, cacheDir
return fmt.Errorf("Error downloading CentOS image: %s", err)
}
- return s.unpack(filepath.Join(os.TempDir(), s.fname), cacheDir)
+ return s.unpack(filepath.Join(os.TempDir(), s.fname), rootfsDir)
}
-func (s CentOSHTTP) unpack(filePath, cacheDir string) error {
+func (s CentOSHTTP) unpack(filePath, rootfsDir string) error {
isoDir := filepath.Join(os.TempDir(), "distrobuilder", "iso")
squashfsDir := filepath.Join(os.TempDir(), "distrobuilder", "squashfs")
tempRootDir := filepath.Join(os.TempDir(), "distrobuilder", "rootfs")
- os.RemoveAll(filepath.Join(cacheDir, "rootfs"))
-
os.MkdirAll(isoDir, 0755)
os.MkdirAll(tempRootDir, 0755)
defer os.RemoveAll(filepath.Join(os.TempDir(), "distrobuilder"))
@@ -84,7 +80,14 @@ func (s CentOSHTTP) unpack(filePath, cacheDir string) error {
}
defer syscall.Unmount(tempRootDir, 0)
- err = shared.RunCommand("rsync", "-qa", tempRootDir, cacheDir)
+ // Remove rootfsDir otherwise rsync will copy the content into the directory
+ // itself
+ err = os.RemoveAll(rootfsDir)
+ if err != nil {
+ return err
+ }
+
+ err = shared.RunCommand("rsync", "-qa", tempRootDir+"/", rootfsDir)
if err != nil {
return err
}
diff --git a/sources/debootstrap.go b/sources/debootstrap.go
index 8ff74fa..da98d55 100644
--- a/sources/debootstrap.go
+++ b/sources/debootstrap.go
@@ -17,10 +17,10 @@ func NewDebootstrap() *Debootstrap {
}
// Run runs debootstrap.
-func (s *Debootstrap) Run(source shared.DefinitionSource, release, arch, cacheDir string) error {
+func (s *Debootstrap) Run(source shared.DefinitionSource, release, arch, rootfsDir string) error {
var args []string
- os.RemoveAll(filepath.Join(cacheDir, "rootfs"))
+ os.RemoveAll(rootfsDir)
if source.Variant != "" {
args = append(args, "--variant", source.Variant)
@@ -40,7 +40,7 @@ func (s *Debootstrap) Run(source shared.DefinitionSource, release, arch, cacheDi
args = append(args, "--keyring", keyring)
}
- args = append(args, release, filepath.Join(cacheDir, "rootfs"))
+ args = append(args, release, rootfsDir)
if source.URL != "" {
args = append(args, source.URL)
diff --git a/sources/ubuntu-http.go b/sources/ubuntu-http.go
index fcc5d4b..ef61a00 100644
--- a/sources/ubuntu-http.go
+++ b/sources/ubuntu-http.go
@@ -26,7 +26,7 @@ func NewUbuntuHTTP() *UbuntuHTTP {
}
// Run downloads the tarball and unpacks it.
-func (s *UbuntuHTTP) Run(source shared.DefinitionSource, release, arch, cacheDir string) error {
+func (s *UbuntuHTTP) Run(source shared.DefinitionSource, release, arch, rootfsDir string) error {
baseURL := fmt.Sprintf("%s/releases/%s/release/", source.URL, release)
if strings.ContainsAny(release, "0123456789") {
@@ -61,7 +61,7 @@ func (s *UbuntuHTTP) Run(source shared.DefinitionSource, release, arch, cacheDir
return fmt.Errorf("Error downloading Ubuntu image: %s", err)
}
- return s.unpack(filepath.Join(os.TempDir(), s.fname), filepath.Join(cacheDir, "rootfs"))
+ return s.unpack(filepath.Join(os.TempDir(), s.fname), rootfsDir)
}
func (s UbuntuHTTP) unpack(filePath, rootDir string) error {
From 5164681635d95793b2b48e57044d9f55b3033102 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Wed, 7 Mar 2018 09:37:35 +0100
Subject: [PATCH 3/7] generators: Add sourceDir to generators
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
distrobuilder/main_lxc.go | 5 +++--
distrobuilder/main_lxd.go | 3 ++-
generators/generators.go | 12 ++++++------
generators/hostname.go | 10 ++++------
generators/hosts.go | 12 +++++-------
5 files changed, 20 insertions(+), 22 deletions(-)
diff --git a/distrobuilder/main_lxc.go b/distrobuilder/main_lxc.go
index 40a1cd9..8690ef7 100644
--- a/distrobuilder/main_lxc.go
+++ b/distrobuilder/main_lxc.go
@@ -58,7 +58,8 @@ func (c *cmdLXC) run(cmd *cobra.Command, args []string) error {
continue
}
- err := generator.CreateLXCData(c.global.flagCacheDir, file.Path, img)
+ err := generator.CreateLXCData(c.global.flagCacheDir, c.global.sourceDir,
+ file.Path, img)
if err != nil {
continue
}
@@ -70,7 +71,7 @@ func (c *cmdLXC) run(cmd *cobra.Command, args []string) error {
}
// Clean up the chroot by restoring the orginal files.
- err = generators.RestoreFiles(c.global.flagCacheDir)
+ err = generators.RestoreFiles(c.global.flagCacheDir, c.global.sourceDir)
if err != nil {
return fmt.Errorf("Failed to restore cached files: %s", err)
}
diff --git a/distrobuilder/main_lxd.go b/distrobuilder/main_lxd.go
index d109a1c..d695b9a 100644
--- a/distrobuilder/main_lxd.go
+++ b/distrobuilder/main_lxd.go
@@ -76,7 +76,8 @@ func (c *cmdLXD) run(cmd *cobra.Command, args []string) error {
return fmt.Errorf("Unknown generator '%s'", file.Generator)
}
- err := generator.CreateLXDData(c.global.sourceDir, file.Path, img)
+ err := generator.CreateLXDData(c.global.flagCacheDir, c.global.sourceDir,
+ file.Path, img)
if err != nil {
return fmt.Errorf("Failed to create LXD data: %s", err)
}
diff --git a/generators/generators.go b/generators/generators.go
index 19c55dd..c126f14 100644
--- a/generators/generators.go
+++ b/generators/generators.go
@@ -13,8 +13,8 @@ import (
// Generator interface.
type Generator interface {
- CreateLXCData(string, string, *image.LXCImage) error
- CreateLXDData(string, string, *image.LXDImage) error
+ CreateLXCData(string, string, string, *image.LXCImage) error
+ CreateLXDData(string, string, string, *image.LXDImage) error
}
// Get returns a Generator.
@@ -30,19 +30,19 @@ func Get(generator string) Generator {
}
// StoreFile caches a file which can be restored with the RestoreFiles function.
-func StoreFile(cacheDir, path string) error {
+func StoreFile(cacheDir, sourceDir, path string) error {
// create temporary directory containing old files
err := os.MkdirAll(filepath.Join(cacheDir, "tmp", p.Dir(path)), 0755)
if err != nil {
return err
}
- return shared.FileCopy(filepath.Join(cacheDir, "rootfs", path),
+ return shared.FileCopy(filepath.Join(sourceDir, path),
filepath.Join(cacheDir, "tmp", path))
}
// RestoreFiles restores original files which were cached by StoreFile.
-func RestoreFiles(cacheDir string) error {
+func RestoreFiles(cacheDir, sourceDir string) error {
f := func(path string, info os.FileInfo, err error) error {
if info.IsDir() {
// We don't care about directories. They should be present so there's
@@ -50,7 +50,7 @@ func RestoreFiles(cacheDir string) error {
return nil
}
- return os.Rename(path, filepath.Join(cacheDir, "rootfs",
+ return shared.FileCopy(path, filepath.Join(sourceDir,
strings.TrimPrefix(path, filepath.Join(cacheDir, "tmp"))))
}
diff --git a/generators/hostname.go b/generators/hostname.go
index f0b1206..ff47a00 100644
--- a/generators/hostname.go
+++ b/generators/hostname.go
@@ -14,17 +14,15 @@ import (
type HostnameGenerator struct{}
// CreateLXCData creates a hostname template.
-func (g HostnameGenerator) CreateLXCData(cacheDir, path string, img *image.LXCImage) error {
- rootfs := filepath.Join(cacheDir, "rootfs")
-
+func (g HostnameGenerator) CreateLXCData(cacheDir, sourceDir, path string, img *image.LXCImage) error {
// Store original file
- err := StoreFile(cacheDir, path)
+ err := StoreFile(cacheDir, sourceDir, path)
if err != nil {
return err
}
// Create new hostname file
- file, err := os.Create(filepath.Join(rootfs, path))
+ file, err := os.Create(filepath.Join(sourceDir, path))
if err != nil {
return err
}
@@ -41,7 +39,7 @@ func (g HostnameGenerator) CreateLXCData(cacheDir, path string, img *image.LXCIm
}
// CreateLXDData creates a hostname template.
-func (g HostnameGenerator) CreateLXDData(cacheDir, path string, img *image.LXDImage) error {
+func (g HostnameGenerator) CreateLXDData(cacheDir, sourceDir, path string, img *image.LXDImage) error {
templateDir := filepath.Join(cacheDir, "templates")
err := os.MkdirAll(templateDir, 0755)
diff --git a/generators/hosts.go b/generators/hosts.go
index 489f0f5..177ced8 100644
--- a/generators/hosts.go
+++ b/generators/hosts.go
@@ -13,16 +13,14 @@ import (
type HostsGenerator struct{}
// CreateLXCData creates a LXC specific entry in the hosts file.
-func (g HostsGenerator) CreateLXCData(cacheDir, path string, img *image.LXCImage) error {
- rootfs := filepath.Join(cacheDir, "rootfs")
-
+func (g HostsGenerator) CreateLXCData(cacheDir, sourceDir, path string, img *image.LXCImage) error {
// Store original file
- err := StoreFile(cacheDir, path)
+ err := StoreFile(cacheDir, sourceDir, path)
if err != nil {
return err
}
- file, err := os.OpenFile(filepath.Join(rootfs, path),
+ file, err := os.OpenFile(filepath.Join(sourceDir, path),
os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return err
@@ -37,7 +35,7 @@ func (g HostsGenerator) CreateLXCData(cacheDir, path string, img *image.LXCImage
}
// CreateLXDData creates a hosts template.
-func (g HostsGenerator) CreateLXDData(cacheDir, path string, img *image.LXDImage) error {
+func (g HostsGenerator) CreateLXDData(cacheDir, sourceDir, path string, img *image.LXDImage) error {
templateDir := filepath.Join(cacheDir, "templates")
// Create templates path
@@ -53,7 +51,7 @@ func (g HostsGenerator) CreateLXDData(cacheDir, path string, img *image.LXDImage
}
defer file.Close()
- hostsFile, err := os.Open(filepath.Join(cacheDir, "rootfs", path))
+ hostsFile, err := os.Open(filepath.Join(sourceDir, path))
if err != nil {
return err
}
From b759ae35ceef52fa9d5f1ead7be5819f185181ff Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Wed, 7 Mar 2018 09:39:35 +0100
Subject: [PATCH 4/7] image: Build correct images
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
image/lxc.go | 9 +++++----
image/lxd.go | 8 ++++----
2 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/image/lxc.go b/image/lxc.go
index 7a71cb6..009d3e2 100644
--- a/image/lxc.go
+++ b/image/lxc.go
@@ -73,7 +73,8 @@ func (l *LXCImage) Build() error {
return err
}
- err = shared.Pack(filepath.Join(l.targetDir, "rootfs.tar.xz"), l.sourceDir, "rootfs")
+ err = shared.Pack(filepath.Join(l.targetDir, "rootfs.tar.xz"), l.sourceDir,
+ "--transform", "s,^./,rootfs/,", ".")
if err != nil {
return err
}
@@ -107,8 +108,8 @@ func (l *LXCImage) createMetadata() error {
var excludesUser string
- if lxd.PathExists(filepath.Join(l.sourceDir, "rootfs", "dev")) {
- err := filepath.Walk(filepath.Join(l.sourceDir, "rootfs", "dev"),
+ if lxd.PathExists(filepath.Join(l.sourceDir, "dev")) {
+ err := filepath.Walk(filepath.Join(l.sourceDir, "dev"),
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
@@ -116,7 +117,7 @@ func (l *LXCImage) createMetadata() error {
if info.Mode()&os.ModeDevice != 0 {
excludesUser += fmt.Sprintf("%s\n",
- strings.TrimPrefix(path, filepath.Join(l.sourceDir, "rootfs")))
+ strings.TrimPrefix(path, l.sourceDir))
}
return nil
diff --git a/image/lxd.go b/image/lxd.go
index 260b9ea..77589dd 100644
--- a/image/lxd.go
+++ b/image/lxd.go
@@ -47,7 +47,7 @@ func (l *LXDImage) Build(unified bool) error {
return nil
}
- file, err := os.Create(filepath.Join(l.sourceDir, "metadata.yaml"))
+ file, err := os.Create(filepath.Join(l.cacheDir, "metadata.yaml"))
if err != nil {
return err
}
@@ -66,7 +66,7 @@ func (l *LXDImage) Build(unified bool) error {
paths := []string{"metadata.yaml"}
// Only include templates directory in the tarball if it's present.
- info, err := os.Stat(filepath.Join(l.sourceDir, "templates"))
+ info, err := os.Stat(filepath.Join(l.cacheDir, "templates"))
if err == nil && info.IsDir() {
paths = append(paths, "templates")
}
@@ -94,14 +94,14 @@ func (l *LXDImage) Build(unified bool) error {
}
} else {
// Create rootfs as squashfs.
- err = shared.RunCommand("mksquashfs", filepath.Join(l.sourceDir, "rootfs"),
+ err = shared.RunCommand("mksquashfs", l.sourceDir,
filepath.Join(l.targetDir, "rootfs.squashfs"), "-noappend")
if err != nil {
return err
}
// Create metadata tarball.
- err = shared.Pack(filepath.Join(l.targetDir, "lxd.tar.xz"), l.sourceDir, paths...)
+ err = shared.Pack(filepath.Join(l.targetDir, "lxd.tar.xz"), l.cacheDir, paths...)
if err != nil {
return err
}
From a5efaa581bcac0023865042b65f5a71218a9f01d Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Wed, 7 Mar 2018 09:40:20 +0100
Subject: [PATCH 5/7] distrobuilder: Get absolute source directory
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
distrobuilder/main.go | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/distrobuilder/main.go b/distrobuilder/main.go
index 6adef8c..f469978 100644
--- a/distrobuilder/main.go
+++ b/distrobuilder/main.go
@@ -194,7 +194,11 @@ func (c *cmdGlobal) preRunBuild(cmd *cobra.Command, args []string) error {
func (c *cmdGlobal) preRunPack(cmd *cobra.Command, args []string) error {
var err error
- c.sourceDir = args[1]
+ // resolve path
+ c.sourceDir, err = filepath.Abs(args[1])
+ if err != nil {
+ return err
+ }
c.targetDir = "."
if len(args) == 3 {
From f70b4107b09ee6262d8197dad577265aa0892378 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Wed, 7 Mar 2018 10:44:33 +0100
Subject: [PATCH 6/7] *: Fix tarballs and add compression
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
distrobuilder/main_lxd.go | 2 +-
image/lxc.go | 4 ++--
image/lxd.go | 19 ++++++++++++++-----
shared/util.go | 36 +++++++++++++++++++++++++++++++++---
4 files changed, 50 insertions(+), 11 deletions(-)
diff --git a/distrobuilder/main_lxd.go b/distrobuilder/main_lxd.go
index d695b9a..144abc6 100644
--- a/distrobuilder/main_lxd.go
+++ b/distrobuilder/main_lxd.go
@@ -83,7 +83,7 @@ func (c *cmdLXD) run(cmd *cobra.Command, args []string) error {
}
}
- err := img.Build(c.flagType == "unified")
+ err := img.Build(c.flagType == "unified", c.flagCompression)
if err != nil {
return fmt.Errorf("Failed to create LXD image: %s", err)
}
diff --git a/image/lxc.go b/image/lxc.go
index 009d3e2..24d1eff 100644
--- a/image/lxc.go
+++ b/image/lxc.go
@@ -73,7 +73,7 @@ func (l *LXCImage) Build() error {
return err
}
- err = shared.Pack(filepath.Join(l.targetDir, "rootfs.tar.xz"), l.sourceDir,
+ err = shared.Pack(filepath.Join(l.targetDir, "rootfs.tar"), "xz", l.sourceDir,
"--transform", "s,^./,rootfs/,", ".")
if err != nil {
return err
@@ -143,7 +143,7 @@ func (l *LXCImage) packMetadata() error {
files = append(files, "templates")
}
- err := shared.Pack(filepath.Join(l.targetDir, "meta.tar.xz"),
+ err := shared.Pack(filepath.Join(l.targetDir, "meta.tar"), "xz",
filepath.Join(l.cacheDir, "metadata"), files...)
if err != nil {
return fmt.Errorf("Failed to create metadata: %s", err)
diff --git a/image/lxd.go b/image/lxd.go
index 77589dd..b92b74e 100644
--- a/image/lxd.go
+++ b/image/lxd.go
@@ -41,7 +41,7 @@ func NewLXDImage(sourceDir, targetDir, cacheDir string,
}
// Build creates a LXD image.
-func (l *LXDImage) Build(unified bool) error {
+func (l *LXDImage) Build(unified bool, compression string) error {
err := l.createMetadata()
if err != nil {
return nil
@@ -86,12 +86,20 @@ func (l *LXDImage) Build(unified bool) error {
fname = "lxd"
}
- paths = append(paths, "rootfs")
- err = shared.Pack(filepath.Join(l.targetDir, fmt.Sprintf("%s.tar.xz", fname)),
- l.sourceDir, paths...)
+ // Add the rootfs to the tarball, prefix all files with "rootfs"
+ err = shared.Pack(filepath.Join(l.targetDir, fmt.Sprintf("%s.tar", fname)),
+ "", l.sourceDir, "--transform", "s,^./,rootfs/,", ".")
if err != nil {
return err
}
+
+ // Add the metadata to the tarball which is located in the cache directory
+ err = shared.PackUpdate(filepath.Join(l.targetDir, fmt.Sprintf("%s.tar", fname)),
+ compression, l.cacheDir, paths...)
+ if err != nil {
+ return err
+ }
+
} else {
// Create rootfs as squashfs.
err = shared.RunCommand("mksquashfs", l.sourceDir,
@@ -101,7 +109,8 @@ func (l *LXDImage) Build(unified bool) error {
}
// Create metadata tarball.
- err = shared.Pack(filepath.Join(l.targetDir, "lxd.tar.xz"), l.cacheDir, paths...)
+ err = shared.Pack(filepath.Join(l.targetDir, "lxd.tar"), compression,
+ l.cacheDir, paths...)
if err != nil {
return err
}
diff --git a/shared/util.go b/shared/util.go
index 598104d..51fc1ed 100644
--- a/shared/util.go
+++ b/shared/util.go
@@ -120,9 +120,39 @@ func CreateGPGKeyring(keyserver string, keys []string) (string, error) {
return filepath.Join(gpgDir, "distrobuilder.gpg"), nil
}
-// Pack creates an xz-compressed tarball.
-func Pack(filename, path string, args ...string) error {
- return RunCommand("tar", append([]string{"-cJf", filename, "-C", path}, args...)...)
+// Pack creates an uncompressed tarball.
+func Pack(filename, compression, path string, args ...string) error {
+ err := RunCommand("tar", append([]string{"-cf", filename, "-C", path}, args...)...)
+ if err != nil {
+ return err
+ }
+
+ return compressTarball(filename, compression)
+}
+
+// PackUpdate updates an existing tarball.
+func PackUpdate(filename, compression, path string, args ...string) error {
+ err := RunCommand("tar", append([]string{"-uf", filename, "-C", path}, args...)...)
+ if err != nil {
+ return err
+ }
+
+ return compressTarball(filename, compression)
+}
+
+// compressTarball compresses a tarball, or not.
+func compressTarball(filename, compression string) error {
+ switch compression {
+ case "lzop":
+ // lzo does not remove the uncompressed file per default
+ defer os.Remove(filename)
+ fallthrough
+ case "bzip2", "xz", "lzip", "lzma", "gzip":
+ return RunCommand(compression, "-f", filename)
+ }
+
+ // Do not compress
+ return nil
}
//GetExpiryDate returns an expiry date based on the creationDate and format.
From 3520763f54f3fafddae400d7b47521d734d0ff9b Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Wed, 7 Mar 2018 11:01:36 +0100
Subject: [PATCH 7/7] Update tests
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
generators/generators_test.go | 7 ++++---
generators/hostname_test.go | 8 +++++---
generators/hosts_test.go | 8 +++++---
image/lxd_test.go | 6 +++---
4 files changed, 17 insertions(+), 12 deletions(-)
diff --git a/generators/generators_test.go b/generators/generators_test.go
index df3a545..63a9dd6 100644
--- a/generators/generators_test.go
+++ b/generators/generators_test.go
@@ -40,6 +40,7 @@ func TestGet(t *testing.T) {
func TestRestoreFiles(t *testing.T) {
cacheDir := filepath.Join(os.TempDir(), "distrobuilder-test")
+ rootfsDir := filepath.Join(cacheDir, "rootfs")
setup(t, cacheDir)
defer teardown(cacheDir)
@@ -60,7 +61,7 @@ func TestRestoreFiles(t *testing.T) {
t.Fatalf("Failed to chmod cache directory: %s", err)
}
- err = StoreFile(cacheDir, filepath.Join("/testdir1", "testfile1"))
+ err = StoreFile(cacheDir, cacheDir, filepath.Join("/testdir1", "testfile1"))
if err == nil {
t.Fatal("Expected failure")
}
@@ -71,7 +72,7 @@ func TestRestoreFiles(t *testing.T) {
t.Fatalf("Failed to chmod cache directory: %s", err)
}
- err = StoreFile(cacheDir, filepath.Join("/testdir1", "testfile1"))
+ err = StoreFile(cacheDir, rootfsDir, filepath.Join("/testdir1", "testfile1"))
if err != nil {
t.Fatalf("Failed to store file: %s", err)
}
@@ -83,7 +84,7 @@ func TestRestoreFiles(t *testing.T) {
createTestFile(t, filepath.Join(cacheDir, "rootfs", "testdir1", "testfile1"),
"modified file")
- err = RestoreFiles(cacheDir)
+ err = RestoreFiles(cacheDir, rootfsDir)
if err != nil {
t.Fatalf("Failed to restore file: %s", err)
}
diff --git a/generators/hostname_test.go b/generators/hostname_test.go
index b2c00ed..efc8131 100644
--- a/generators/hostname_test.go
+++ b/generators/hostname_test.go
@@ -11,6 +11,7 @@ import (
func TestHostnameGeneratorCreateLXCData(t *testing.T) {
cacheDir := filepath.Join(os.TempDir(), "distrobuilder-test")
+ rootfsDir := filepath.Join(cacheDir, "rootfs")
setup(t, cacheDir)
defer teardown(cacheDir)
@@ -34,7 +35,7 @@ func TestHostnameGeneratorCreateLXCData(t *testing.T) {
createTestFile(t, filepath.Join(cacheDir, "rootfs", "etc", "hostname"), "hostname")
- err = generator.CreateLXCData(cacheDir, "/etc/hostname", image)
+ err = generator.CreateLXCData(cacheDir, rootfsDir, "/etc/hostname", image)
if err != nil {
t.Fatalf("Unexpected error: %s", err)
}
@@ -42,7 +43,7 @@ func TestHostnameGeneratorCreateLXCData(t *testing.T) {
validateTestFile(t, filepath.Join(cacheDir, "tmp", "etc", "hostname"), "hostname")
validateTestFile(t, filepath.Join(cacheDir, "rootfs", "etc", "hostname"), "LXC_NAME\n")
- err = RestoreFiles(cacheDir)
+ err = RestoreFiles(cacheDir, rootfsDir)
if err != nil {
t.Fatalf("Failed to restore files: %s", err)
}
@@ -52,6 +53,7 @@ func TestHostnameGeneratorCreateLXCData(t *testing.T) {
func TestHostnameGeneratorCreateLXDData(t *testing.T) {
cacheDir := filepath.Join(os.TempDir(), "distrobuilder-test")
+ rootfsDir := filepath.Join(cacheDir, "rootfs")
setup(t, cacheDir)
defer teardown(cacheDir)
@@ -73,7 +75,7 @@ func TestHostnameGeneratorCreateLXDData(t *testing.T) {
t.Fatalf("Unexpected error: %s", err)
}
- err = generator.CreateLXDData(cacheDir, "/etc/hostname", image)
+ err = generator.CreateLXDData(cacheDir, rootfsDir, "/etc/hostname", image)
if err != nil {
t.Fatalf("Unexpected error: %s", err)
}
diff --git a/generators/hosts_test.go b/generators/hosts_test.go
index f29da90..b19133e 100644
--- a/generators/hosts_test.go
+++ b/generators/hosts_test.go
@@ -11,6 +11,7 @@ import (
func TestHostsGeneratorCreateLXCData(t *testing.T) {
cacheDir := filepath.Join(os.TempDir(), "distrobuilder-test")
+ rootfsDir := filepath.Join(cacheDir, "rootfs")
setup(t, cacheDir)
defer teardown(cacheDir)
@@ -35,7 +36,7 @@ func TestHostsGeneratorCreateLXCData(t *testing.T) {
createTestFile(t, filepath.Join(cacheDir, "rootfs", "etc", "hosts"),
"127.0.0.1\tlocalhost\n")
- err = generator.CreateLXCData(cacheDir, "/etc/hosts", image)
+ err = generator.CreateLXCData(cacheDir, rootfsDir, "/etc/hosts", image)
if err != nil {
t.Fatalf("Unexpected error: %s", err)
}
@@ -45,7 +46,7 @@ func TestHostsGeneratorCreateLXCData(t *testing.T) {
validateTestFile(t, filepath.Join(cacheDir, "rootfs", "etc", "hosts"),
"127.0.0.1\tlocalhost\n127.0.0.1\tLXC_NAME\n")
- err = RestoreFiles(cacheDir)
+ err = RestoreFiles(cacheDir, rootfsDir)
if err != nil {
t.Fatalf("Failed to restore files: %s", err)
}
@@ -56,6 +57,7 @@ func TestHostsGeneratorCreateLXCData(t *testing.T) {
func TestHostsGeneratorCreateLXDData(t *testing.T) {
cacheDir := filepath.Join(os.TempDir(), "distrobuilder-test")
+ rootfsDir := filepath.Join(cacheDir, "rootfs")
setup(t, cacheDir)
defer teardown(cacheDir)
@@ -80,7 +82,7 @@ func TestHostsGeneratorCreateLXDData(t *testing.T) {
createTestFile(t, filepath.Join(cacheDir, "rootfs", "etc", "hosts"),
"127.0.0.1\tlocalhost\n")
- err = generator.CreateLXDData(cacheDir, "/etc/hosts", image)
+ err = generator.CreateLXDData(cacheDir, rootfsDir, "/etc/hosts", image)
if err != nil {
t.Fatalf("Unexpected error: %s", err)
}
diff --git a/image/lxd_test.go b/image/lxd_test.go
index e957b99..04f6caa 100644
--- a/image/lxd_test.go
+++ b/image/lxd_test.go
@@ -70,7 +70,7 @@ func TestLXDBuild(t *testing.T) {
func testLXDBuildSplitImage(t *testing.T, image *LXDImage) {
// Create split tarball and squashfs.
- err := image.Build(false)
+ err := image.Build(false, "xz")
if err != nil {
t.Fatalf("Unexpected error: %s", err)
}
@@ -90,7 +90,7 @@ func testLXDBuildSplitImage(t *testing.T, image *LXDImage) {
func testLXDBuildUnifiedImage(t *testing.T, image *LXDImage) {
// Create unified tarball with custom name.
- err := image.Build(true)
+ err := image.Build(true, "xz")
if err != nil {
t.Fatalf("Unexpected error: %s", err)
}
@@ -102,7 +102,7 @@ func testLXDBuildUnifiedImage(t *testing.T, image *LXDImage) {
// Create unified tarball with default name.
image.definition.Name = ""
- err = image.Build(true)
+ err = image.Build(true, "xz")
if err != nil {
t.Fatalf("Unexpected error: %s", err)
}
More information about the lxc-devel
mailing list