[lxc-devel] [distrobuilder/master] Restructure definition

monstermunchkin on Github lxc-bot at linuxcontainers.org
Mon Oct 14 16:45:15 UTC 2019


A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 398 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20191014/e16ae5c4/attachment.bin>
-------------- next part --------------
From 55c499a0bda1fd202163be79f8d8f1e445442c7b Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Mon, 14 Oct 2019 16:24:46 +0200
Subject: [PATCH 1/6] Move EarlyPackages to Package sets

Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
 shared/definition.go   | 29 ++++++++++++++++++++++++++++-
 sources/debootstrap.go | 11 +++++++++--
 2 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/shared/definition.go b/shared/definition.go
index 5699c20..25390df 100644
--- a/shared/definition.go
+++ b/shared/definition.go
@@ -25,6 +25,7 @@ type DefinitionPackagesSet struct {
 	DefinitionFilter `yaml:",inline"`
 	Packages         []string `yaml:"packages"`
 	Action           string   `yaml:"action"`
+	Early            bool     `yaml:"early,omitempty"`
 }
 
 // A DefinitionPackagesRepository contains data of a specific repository
@@ -90,7 +91,6 @@ type DefinitionSource struct {
 	SameAs           string   `yaml:"same_as,omitempty"`
 	AptSources       string   `yaml:"apt_sources,omitempty"`
 	SkipVerification bool     `yaml:"skip_verification,omitempty"`
-	EarlyPackages    []string `yaml:"early_packages,omitempty"`
 }
 
 // A DefinitionTargetLXCConfig represents the config part of the metadata.
@@ -440,6 +440,33 @@ func (d *Definition) GetRunnableActions(trigger string) []DefinitionAction {
 	return out
 }
 
+// GetEarlyPackages returns a list of packages which are to be installed or removed earlier than the actual package handling.
+func (d *Definition) GetEarlyPackages(action string) []string {
+	var out []string
+
+	for _, set := range d.Packages.Sets {
+		if set.Action != action || !set.Early {
+			continue
+		}
+
+		if len(set.Releases) > 0 && !shared.StringInSlice(d.Image.Release, set.Releases) {
+			continue
+		}
+
+		if len(set.Architectures) > 0 && !shared.StringInSlice(d.Image.ArchitectureMapped, set.Architectures) {
+			continue
+		}
+
+		if len(set.Variants) > 0 && !shared.StringInSlice(d.Image.Variant, set.Variants) {
+			continue
+		}
+
+		out = append(out, set.Packages...)
+	}
+
+	return out
+}
+
 func (d *Definition) getMappedArchitecture() (string, error) {
 	var arch string
 
diff --git a/sources/debootstrap.go b/sources/debootstrap.go
index 8e880c9..acc71c6 100644
--- a/sources/debootstrap.go
+++ b/sources/debootstrap.go
@@ -38,8 +38,15 @@ func (s *Debootstrap) Run(definition shared.Definition, rootfsDir string) error
 		args = append(args, "--no-check-gpg")
 	}
 
-	if len(definition.Source.EarlyPackages) > 0 {
-		args = append(args, fmt.Sprintf("--include=%s", strings.Join(definition.Source.EarlyPackages, ",")))
+	earlyPackagesInstall := definition.GetEarlyPackages("install")
+	earlyPackagesRemove := definition.GetEarlyPackages("remove")
+
+	if len(earlyPackagesInstall) > 0 {
+		args = append(args, fmt.Sprintf("--include=%s", strings.Join(earlyPackagesInstall, ",")))
+	}
+
+	if len(earlyPackagesRemove) > 0 {
+		args = append(args, fmt.Sprintf("--exclude=%s", strings.Join(earlyPackagesRemove, ",")))
 	}
 
 	if len(definition.Source.Keys) > 0 {

From 13ba2fc9be2f77db463f0743d91d9dcbf977a15a Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Mon, 14 Oct 2019 18:22:50 +0200
Subject: [PATCH 2/6] distrobuilder: Run template on repo.URL

Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
 distrobuilder/chroot.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/distrobuilder/chroot.go b/distrobuilder/chroot.go
index f24adfc..43912db 100644
--- a/distrobuilder/chroot.go
+++ b/distrobuilder/chroot.go
@@ -42,6 +42,12 @@ func managePackages(def shared.DefinitionPackages, actions []shared.DefinitionAc
 				continue
 			}
 
+			// Run template on repo.URL
+			repo.URL, err = shared.RenderTemplate(repo.URL, def)
+			if err != nil {
+				return err
+			}
+
 			err = manager.RepoHandler(repo)
 			if err != nil {
 				return fmt.Errorf("Error for repository %s: %s", repo.Name, err)

From e3e5bf551bd55c46b7750529c02d4327bda33df3 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Mon, 14 Oct 2019 18:21:18 +0200
Subject: [PATCH 3/6] managers/apt: Add RepoHandler

Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
 managers/apt.go | 77 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 77 insertions(+)

diff --git a/managers/apt.go b/managers/apt.go
index 22d7659..e7106e7 100644
--- a/managers/apt.go
+++ b/managers/apt.go
@@ -1,5 +1,17 @@
 package managers
 
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	lxd "github.com/lxc/lxd/shared"
+
+	"github.com/lxc/distrobuilder/shared"
+)
+
 // NewApt creates a new Manager instance.
 func NewApt() *Manager {
 	return &Manager{
@@ -30,5 +42,70 @@ func NewApt() *Manager {
 				"dist-upgrade",
 			},
 		},
+		RepoHandler: func(repoAction shared.DefinitionPackagesRepository) error {
+			var targetFile string
+
+			if repoAction.Name == "sources.list" {
+				targetFile = filepath.Join("/etc/apt", repoAction.Name)
+			} else {
+				targetFile = filepath.Join("/etc/apt/sources.list.d", repoAction.Name)
+
+				if !strings.HasSuffix(targetFile, ".list") {
+					targetFile = fmt.Sprintf("%s.list", targetFile)
+				}
+
+			}
+
+			if !lxd.PathExists(filepath.Dir(targetFile)) {
+				err := os.MkdirAll(filepath.Dir(targetFile), 0755)
+				if err != nil {
+					return err
+				}
+			}
+
+			f, err := os.OpenFile(targetFile, os.O_CREATE|os.O_RDWR, 0644)
+			if err != nil {
+				return err
+			}
+			defer f.Close()
+
+			content, err := ioutil.ReadAll(f)
+			if err != nil {
+				return err
+			}
+
+			// Truncate file if it's not generated by distrobuilder
+			if !strings.HasPrefix(string(content), "# Generated by distrobuilder\n") {
+				err = f.Truncate(0)
+				if err != nil {
+					return err
+				}
+
+				_, err = f.Seek(0, 0)
+				if err != nil {
+					return err
+				}
+
+				_, err = f.WriteString("# Generated by distrobuilder\n")
+				if err != nil {
+					return err
+				}
+			}
+
+			_, err = f.WriteString(repoAction.URL)
+			if err != nil {
+				return err
+			}
+
+			// Append final new line if missing
+			if !strings.HasSuffix(repoAction.URL, "\n") {
+				_, err = f.WriteString("\n")
+				if err != nil {
+					return err
+				}
+			}
+
+			return nil
+		},
 	}
 }

From 90e58e2359b364309cd31016d5210a23335dec3b Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Mon, 14 Oct 2019 18:22:07 +0200
Subject: [PATCH 4/6] sources: Stop using definition.Source.AptSources

Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
 sources/debootstrap.go | 29 +----------------------------
 sources/ubuntu-http.go | 29 +----------------------------
 2 files changed, 2 insertions(+), 56 deletions(-)

diff --git a/sources/debootstrap.go b/sources/debootstrap.go
index acc71c6..acf48f3 100644
--- a/sources/debootstrap.go
+++ b/sources/debootstrap.go
@@ -84,32 +84,5 @@ func (s *Debootstrap) Run(definition shared.Definition, rootfsDir string) error
 		defer os.Remove(scriptPath)
 	}
 
-	err := shared.RunCommand("debootstrap", args...)
-	if err != nil {
-		return err
-	}
-
-	if definition.Source.AptSources != "" {
-		// Run the template
-		out, err := shared.RenderTemplate(definition.Source.AptSources, definition)
-		if err != nil {
-			return err
-		}
-
-		// Append final new line if missing
-		if !strings.HasSuffix(out, "\n") {
-			out += "\n"
-		}
-
-		// Replace content of sources.list with the templated content.
-		file, err := os.Create(filepath.Join(rootfsDir, "etc", "apt", "sources.list"))
-		if err != nil {
-			return err
-		}
-		defer file.Close()
-
-		file.WriteString(out)
-	}
-
-	return nil
+	return shared.RunCommand("debootstrap", args...)
 }
diff --git a/sources/ubuntu-http.go b/sources/ubuntu-http.go
index 2b6ff6d..b82042a 100644
--- a/sources/ubuntu-http.go
+++ b/sources/ubuntu-http.go
@@ -48,34 +48,7 @@ func (s *UbuntuHTTP) Run(definition shared.Definition, rootfsDir string) error {
 }
 
 func (s *UbuntuHTTP) runDefaultVariant(definition shared.Definition, rootfsDir string) error {
-	err := s.unpack(filepath.Join(s.fpath, s.fname), rootfsDir)
-	if err != nil {
-		return err
-	}
-
-	if definition.Source.AptSources != "" {
-		// Run the template
-		out, err := shared.RenderTemplate(definition.Source.AptSources, definition)
-		if err != nil {
-			return err
-		}
-
-		// Append final new line if missing
-		if !strings.HasSuffix(out, "\n") {
-			out += "\n"
-		}
-
-		// Replace content of sources.list with the templated content.
-		file, err := os.Create(filepath.Join(rootfsDir, "etc", "apt", "sources.list"))
-		if err != nil {
-			return err
-		}
-		defer file.Close()
-
-		file.WriteString(out)
-	}
-
-	return nil
+	return s.unpack(filepath.Join(s.fpath, s.fname), rootfsDir)
 }
 
 func (s *UbuntuHTTP) runCoreVariant(definition shared.Definition, rootfsDir string) error {

From 5c4abe45a865d5671845de585b97b361bb2daa9e Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Mon, 14 Oct 2019 18:33:01 +0200
Subject: [PATCH 5/6] shared: Remove AptSources

Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
 shared/definition.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/shared/definition.go b/shared/definition.go
index 25390df..9d3180a 100644
--- a/shared/definition.go
+++ b/shared/definition.go
@@ -89,7 +89,6 @@ type DefinitionSource struct {
 	Variant          string   `yaml:"variant,omitempty"`
 	Suite            string   `yaml:"suite,omitempty"`
 	SameAs           string   `yaml:"same_as,omitempty"`
-	AptSources       string   `yaml:"apt_sources,omitempty"`
 	SkipVerification bool     `yaml:"skip_verification,omitempty"`
 }
 

From db981259c73a908f6e2c19d28d454025adbace88 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Mon, 14 Oct 2019 18:42:27 +0200
Subject: [PATCH 6/6] doc/examples: Update schema.yaml

Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
 doc/examples/scheme.yaml | 42 +++++++++++++++++++++++++++-------------
 1 file changed, 29 insertions(+), 13 deletions(-)

diff --git a/doc/examples/scheme.yaml b/doc/examples/scheme.yaml
index df114ce..ab54080 100644
--- a/doc/examples/scheme.yaml
+++ b/doc/examples/scheme.yaml
@@ -13,18 +13,14 @@ image:
 
 source:
   downloader: ubuntu-http
-  URL: http://archive.ubuntu.com
+  url: http://archive.ubuntu.com
   keys:
     - 0xdeadbeaf
   keyserver: http://keyserver.ubuntu.com
   variant: default
   suite: suite
   same_as: xenial
-  apt_sources: |-
-     deb http://archive.ubuntu.com/ubuntu {{ image.release }}-updates main restricted universe multiverse
   skip_verification: false
-  early_packages:
-    - gnupg
 
 targets:
   lxc:
@@ -118,7 +114,12 @@ packages:
   cleanup: false
   sets:
     - packages:
-       - vim
+        - gnupg
+      action: install
+      early: true
+
+    - packages:
+        - vim
       action: install
       releases:
         - a
@@ -130,11 +131,12 @@ packages:
 
     - packages:
         - grub
-      actionn: remove
+      action: remove
 
   repositories:
     - name: reponame
-      url: http://example.com
+      url: |-
+        deb http://archive.ubuntu.com/ubuntu {{ image.release }}-updates main restricted universe multiverse
       type: type
       key: 0xdeadbeaf
       releases:
@@ -145,16 +147,30 @@ packages:
       variants:
         - default
 
-    - packages:
-        - grub
-      actionn: remove
-
 actions:
+  - trigger: post-unpack
+    action: |-
+      #!/bin/sh
+
+      do something after the rootfs has been unpacked
+
+  - trigger: post-files
+    action: |-
+      #!/bin/sh
+
+      do something after the files section has been processed
+
+  - trigger: post-update
+    action: |-
+      #!/bin/sh
+
+      do something after packages have been processed
+
   - trigger: post-packages
     action: |-
       #!/bin/sh
 
-      echo do something
+      do something after the packages section has been processed
 
     releases:
       - a


More information about the lxc-devel mailing list