[lxc-devel] [lxd-pkg-snap/latest-edge] [RFC] build lxd binaries without using go plugin's build step

cjp256 on Github lxc-bot at linuxcontainers.org
Fri May 29 02:25:07 UTC 2020


A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 1204 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200528/8ed86208/attachment.bin>
-------------- next part --------------
From ea9d9a3b05aadcd670084ca6150b378e69a5f12f Mon Sep 17 00:00:00 2001
From: Chris Patterson <chris.patterson at canonical.com>
Date: Thu, 28 May 2020 22:07:28 -0400
Subject: [PATCH] build lxd binaries without using go plugin's build step

Perform the go build commands performed by the go plugin in
`override-build`.  The `override-build` is already very heavily
customized and there is little need to rely on the plugin anymore.

This patch simply adds a few extra build commands with the
correctly exported CGO_LDFLAGS to find the library dependencies
that are being built/staged.

Remove the custom go plugin, and rely on the `go` plugin again
but only using its `pull` functionality.

Signed-off-by: Chris Patterson <chris.patterson at canonical.com>
---
 snap/plugins/nogetgo.py | 339 ----------------------------------------
 snapcraft.yaml          |  13 +-
 2 files changed, 9 insertions(+), 343 deletions(-)
 delete mode 100644 snap/plugins/nogetgo.py

diff --git a/snap/plugins/nogetgo.py b/snap/plugins/nogetgo.py
deleted file mode 100644
index 7005580..0000000
--- a/snap/plugins/nogetgo.py
+++ /dev/null
@@ -1,339 +0,0 @@
-# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
-#
-# Copyright (C) 2015-2019 Canonical Ltd
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-"""The go plugin can be used for go projects using `go get`.
-
-This plugin uses the common plugin keywords as well as those for "sources".
-For more information check the 'plugins' topic for the former and the
-'sources' topic for the latter.
-
-Additionally, this plugin uses the following plugin-specific keywords:
-
-    - go-channel:
-      (string, default: latest/stable)
-      The Snap Store channel to install go from. If set to an empty string,
-      go will be installed using the system's traditional package manager.
-
-    - go-packages:
-      (list of strings)
-      Go packages to fetch, these must be a "main" package. Dependencies
-      are pulled in automatically by `go get`.
-      Packages that are not "main" will not cause an error, but would
-      not be useful either.
-      If the package is a part of the go-importpath the local package
-      corresponding to those sources will be used.
-
-    - go-importpath:
-      (string)
-      This entry tells the checked out `source` to live within a certain path
-      within `GOPATH`.
-      This entry is not required if `source` uses `go.mod`.
-
-    - go-buildtags:
-      (list of strings)
-      Tags to use during the go build. Default is not to use any build tags.
-"""
-
-import logging
-import os
-import re
-import shutil
-from glob import iglob
-from pkg_resources import parse_version
-from typing import Any, Dict, List, Optional, TYPE_CHECKING
-
-import snapcraft
-from snapcraft import common
-from snapcraft.internal import elf, errors
-
-if TYPE_CHECKING:
-    from snapcraft.project import Project
-
-
-logger = logging.getLogger(__name__)
-_GO_MOD_REQUIRED_GO_VERSION = "1.13"
-
-
-class GoModRequiredVersionError(errors.SnapcraftException):
-    def __init__(self, *, go_version: str) -> None:
-        self._go_version = go_version
-
-    def get_brief(self) -> str:
-        return "Use of go.mod requires Go {!r} or greater, found: {!r}".format(
-            _GO_MOD_REQUIRED_GO_VERSION, self._go_version
-        )
-
-    def get_resolution(self) -> str:
-        return "Set go-channel to a newer version of Go and try again."
-
-
-def _get_cgo_ldflags(library_paths: List[str]) -> str:
-    cgo_ldflags: List[str] = list()
-
-    existing_cgo_ldflags = os.getenv("CGO_LDFLAGS")
-    if existing_cgo_ldflags:
-        cgo_ldflags.append(existing_cgo_ldflags)
-
-    flags = common.combine_paths(library_paths, "-L", " ")
-    if flags:
-        cgo_ldflags.append(flags)
-
-    ldflags = os.getenv("LDFLAGS")
-    if ldflags:
-        cgo_ldflags.append(ldflags)
-
-    return " ".join(cgo_ldflags)
-
-
-class GoPlugin(snapcraft.BasePlugin):
-    @classmethod
-    def schema(cls) -> Dict[str, Any]:
-        schema = super().schema()
-        schema["properties"]["go-channel"] = {
-            "type": "string",
-            "default": "latest/stable",
-        }
-        schema["properties"]["go-packages"] = {
-            "type": "array",
-            "minitems": 1,
-            "uniqueItems": True,
-            "items": {"type": "string"},
-            "default": [],
-        }
-        schema["properties"]["go-importpath"] = {"type": "string", "default": ""}
-        schema["properties"]["go-buildtags"] = {
-            "type": "array",
-            "minitems": 1,
-            "uniqueItems": True,
-            "items": {"type": "string"},
-            "default": [],
-        }
-        schema["anyOf"] = [{"required": ["source"]}, {"required": ["go-packages"]}]
-
-        return schema
-
-    @classmethod
-    def get_build_properties(cls) -> List[str]:
-        # Inform Snapcraft of the properties associated with building. If these
-        # change in the YAML Snapcraft will consider the build step dirty.
-        return ["go-packages", "go-buildtags", "go-channel"]
-
-    @classmethod
-    def get_pull_properties(cls) -> List[str]:
-        # Inform Snapcraft of the properties associated with pulling. If these
-        # change in the YAML Snapcraft will consider the pull step dirty.
-        return ["go-packages", "go-channel"]
-
-    def __init__(self, name: str, options, project: "Project") -> None:
-        super().__init__(name, options, project)
-
-        self._setup_base_tools(options.go_channel, project.info.get_build_base())
-        self._is_classic = project.info.confinement == "classic"
-
-        self._install_bin_dir = os.path.join(self.installdir, "bin")
-
-        self._gopath = os.path.join(self.partdir, "go")
-        self._gopath_src = os.path.join(self._gopath, "src")
-        self._gopath_bin = os.path.join(self._gopath, "bin")
-        self._gopath_pkg = os.path.join(self._gopath, "pkg")
-
-        self._version_regex = re.compile(r"^go version go(.*) .*$")
-        self._go_version: Optional[str] = None
-
-    def _setup_base_tools(self, go_channel: str, base: Optional[str]) -> None:
-        if go_channel:
-            self.build_snaps.append("go/{}".format(go_channel))
-        elif base in ("core", "core16", "core18"):
-            self.build_packages.append("golang-go")
-        else:
-            raise errors.PluginBaseError(part_name=self.name, base=base)
-
-    def _is_using_go_mod(self, cwd: str) -> bool:
-        if not os.path.exists(os.path.join(cwd, "go.mod")):
-            return False
-
-        if self._go_version is None:
-            go_version_cmd_output: str = self._run_output(["go", "version"])
-            version_match = self._version_regex.match(go_version_cmd_output)
-
-            if version_match is None:
-                raise RuntimeError(
-                    "Unable to parse go version output: {!r}".format(
-                        go_version_cmd_output
-                    )
-                )
-
-            self._go_version = version_match.group(1)
-
-        if parse_version(self._go_version) < parse_version(_GO_MOD_REQUIRED_GO_VERSION):
-            raise GoModRequiredVersionError(go_version=self._go_version)
-
-        return True
-
-    def _pull_go_mod(self) -> None:
-        self._run(["go", "mod", "download"], cwd=self.sourcedir)
-
-    def _pull_go_packages(self) -> None:
-        os.makedirs(self._gopath_src, exist_ok=True)
-
-        # use -d to only download (build will happen later)
-        # use -t to also get the test-deps
-        # since we are not using -u the sources will stick to the
-        # original checkout.
-        if any(iglob("{}/**/*.go".format(self.sourcedir), recursive=True)):
-            go_package = self._get_local_go_package()
-            go_package_path = os.path.join(self._gopath_src, go_package)
-            if os.path.islink(go_package_path):
-                os.unlink(go_package_path)
-            os.makedirs(os.path.dirname(go_package_path), exist_ok=True)
-            os.symlink(self.sourcedir, go_package_path)
-
-    def pull(self) -> None:
-        super().pull()
-
-        if self._is_using_go_mod(cwd=self.sourcedir):
-            return self._pull_go_mod()
-        else:
-            self._pull_go_packages()
-
-    def clean_pull(self) -> None:
-        super().clean_pull()
-
-        # Remove the gopath (if present)
-        if os.path.exists(self._gopath):
-            shutil.rmtree(self._gopath)
-
-    def _get_local_go_package(self) -> str:
-        if self.options.go_importpath:
-            go_package = self.options.go_importpath
-        else:
-            logger.warning(
-                "Please consider setting `go-importpath` for the {!r} "
-                "part".format(self.name)
-            )
-            go_package = os.path.basename(os.path.abspath(self.options.source))
-        return go_package
-
-    def _get_local_main_packages(self) -> List[str]:
-        search_path = "./{}/...".format(self._get_local_go_package())
-        packages = self._run_output(
-            ["go", "list", "-f", "{{.ImportPath}} {{.Name}}", search_path]
-        )
-        packages_split = [p.split() for p in packages.splitlines()]
-        main_packages = [p[0] for p in packages_split if p[1] == "main"]
-        return main_packages
-
-    def _build(self, *, package: str = "") -> None:
-        build_cmd = ["go", "build"]
-
-        if self.options.go_buildtags:
-            build_cmd.extend(["-tags={}".format(",".join(self.options.go_buildtags))])
-
-        relink_cmd = build_cmd + ["-ldflags", "-linkmode=external"]
-
-        if self._is_using_go_mod(self.builddir) and not package:
-            work_dir = self.builddir
-            build_cmd.extend(["-o", self._install_bin_dir, "./..."])
-            relink_cmd.extend(["-o", self._install_bin_dir, "./..."])
-        else:
-            work_dir = self._install_bin_dir
-            build_cmd.append(package)
-            relink_cmd.append(package)
-
-        pre_build_files = os.listdir(self._install_bin_dir)
-        self._run(build_cmd, cwd=work_dir)
-        post_build_files = os.listdir(self._install_bin_dir)
-
-        new_files = set(post_build_files) - set(pre_build_files)
-        if len(new_files) != 1:
-            raise RuntimeError(f"Expected one binary to be built, found: {new_files!r}")
-        binary_path = os.path.join(self._install_bin_dir, new_files.pop())
-
-        # Relink with system linker if executable is dynamic in order to be
-        # able to set rpath later on. This workaround can be removed after
-        # https://github.com/NixOS/patchelf/issues/146 is fixed.
-        if self._is_classic and elf.ElfFile(path=binary_path).is_dynamic:
-            self._run(relink_cmd, cwd=work_dir)
-
-    def _build_go_packages(self) -> None:
-        if self.options.go_packages:
-            packages = self.options.go_packages
-        else:
-            packages = self._get_local_main_packages()
-
-        for package in packages:
-            self._build(package=package)
-
-    def build(self) -> None:
-        super().build()
-
-        # Ensure install directory exists.
-        os.makedirs(self._install_bin_dir, exist_ok=True)
-
-        if self._is_using_go_mod(cwd=self.builddir):
-            self._build()
-        else:
-            self._build_go_packages()
-
-    def clean_build(self) -> None:
-        super().clean_build()
-
-        if os.path.isdir(self._gopath_bin):
-            shutil.rmtree(self._gopath_bin)
-
-        if os.path.isdir(self._gopath_pkg):
-            shutil.rmtree(self._gopath_pkg)
-
-    def _run(self, cmd: List[str], cwd: str = None, **kwargs) -> None:
-        env = self._build_environment()
-
-        if cwd is None:
-            cwd = self._gopath_src
-
-        return self.run(cmd, cwd=cwd, env=env, **kwargs)
-
-    def _run_output(self, cmd: List[str], **kwargs) -> str:
-        env = self._build_environment()
-        return self.run_output(cmd, cwd=self._gopath_src, env=env, **kwargs)
-
-    def _build_environment(self) -> Dict[str, str]:
-        env = os.environ.copy()
-        env["GOPATH"] = self._gopath
-        env["GOBIN"] = self._gopath_bin
-
-        library_paths: List[str] = []
-        for root in [self.installdir, self.project.stage_dir]:
-            library_paths.extend(
-                common.get_library_paths(root, self.project.arch_triplet)
-            )
-
-        cgo_ldflags = _get_cgo_ldflags(library_paths)
-        if cgo_ldflags:
-            env["CGO_LDFLAGS"] = cgo_ldflags
-
-        if self.project.is_cross_compiling:
-            env["CC"] = "{}-gcc".format(self.project.arch_triplet)
-            env["CXX"] = "{}-g++".format(self.project.arch_triplet)
-            env["CGO_ENABLED"] = "1"
-            # See https://golang.org/doc/install/source#environment
-            go_archs = {"armhf": "arm", "i386": "386", "ppc64el": "ppc64le"}
-            env["GOARCH"] = go_archs.get(self.project.deb_arch, self.project.deb_arch)
-            if self.project.deb_arch == "armhf":
-                env["GOARM"] = "7"
-        return env
-
-    def enable_cross_compilation(self) -> None:
-        pass
diff --git a/snapcraft.yaml b/snapcraft.yaml
index f8a1823..fdea1d3 100644
--- a/snapcraft.yaml
+++ b/snapcraft.yaml
@@ -875,7 +875,7 @@ parts:
       - squashfs-tools
       - usbutils
       - xdelta3
-    plugin: nogetgo
+    plugin: go
     go-buildtags:
       - libsqlite3
     go-importpath: github.com/lxc/lxd
@@ -901,9 +901,14 @@ parts:
       sed -i "/^\/\*/a #cgo CFLAGS: -I${SNAPCRAFT_STAGE}/include/" ../go/src/github.com/mattn/go-sqlite3/sqlite3.go
 
       # Run the standard build
-      set +ex
-      snapcraftctl build
-      set -ex
+      mkdir -p ${SNAPCRAFT_PART_INSTALL}/bin
+      cd ${SNAPCRAFT_PART_INSTALL}/bin
+      export CGO_LDFLAGS="-L $SNAPCRAFT_STAGE/lib"
+      go build -tags=libsqlite3 github.com/lxc/lxd/lxc
+      go build -tags=libsqlite3 github.com/lxc/lxd/lxc-to-lxd
+      go build -tags=libsqlite3 github.com/lxc/lxd/lxd
+      go build -tags=libsqlite3 github.com/lxc/lxd/lxd-benchmark
+      cd ${SNAPCRAFT_PART_SRC}
 
       # Build the agent (special build tags)
       CGO_ENABLED=0 go build -o "${SNAPCRAFT_PART_INSTALL}/bin/lxd-agent" -tags=agent,netgo github.com/lxc/lxd/lxd-agent


More information about the lxc-devel mailing list