[lxc-devel] [lxd/master] Add performance tests for regression testing
albertodonato on Github
lxc-bot at linuxcontainers.org
Thu Sep 7 09:15:52 UTC 2017
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 649 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20170907/62a357f3/attachment.bin>
-------------- next part --------------
From 2e0c0e191da1f1c8229906ed38ff73e04edf308d Mon Sep 17 00:00:00 2001
From: Alberto Donato <alberto.donato at canonical.com>
Date: Wed, 6 Sep 2017 18:08:36 +0200
Subject: [PATCH 1/4] benchmark: add csv reporting
Signed-off-by: Alberto Donato <alberto.donato at canonical.com>
---
lxd-benchmark/benchmark/report.go | 95 +++++++++++++++++++++++++++++++++++++++
lxd-benchmark/main.go | 54 ++++++++++++++++++----
2 files changed, 140 insertions(+), 9 deletions(-)
create mode 100644 lxd-benchmark/benchmark/report.go
diff --git a/lxd-benchmark/benchmark/report.go b/lxd-benchmark/benchmark/report.go
new file mode 100644
index 000000000..60ab02f6d
--- /dev/null
+++ b/lxd-benchmark/benchmark/report.go
@@ -0,0 +1,95 @@
+package benchmark
+
+import (
+ "encoding/csv"
+ "fmt"
+ "io"
+ "os"
+ "time"
+)
+
+// Subset of JMeter CSV log format that are required by Jenkins performance
+// plugin
+// (see http://jmeter.apache.org/usermanual/listeners.html#csvlogformat)
+var csvFields = []string{
+ "timeStamp", // in milliseconds since 1/1/1970
+ "elapsed", // in milliseconds
+ "label",
+ "responseCode",
+ "success", // "true" or "false"
+}
+
+// CSVReport reads/writes a CSV report file.
+type CSVReport struct {
+ Filename string
+
+ records [][]string
+}
+
+// Load reads current content of the filename and loads records.
+func (r *CSVReport) Load() error {
+ file, err := os.Open(r.Filename)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+
+ reader := csv.NewReader(file)
+ for line := 1; err != io.EOF; line++ {
+ record, err := reader.Read()
+ if err == io.EOF {
+ break
+ } else if err != nil {
+ return err
+ }
+
+ err = r.addRecord(record)
+ if err != nil {
+ return err
+ }
+ }
+ logf("Loaded report file %s", r.Filename)
+ return nil
+}
+
+// Write writes current records to file.
+func (r *CSVReport) Write() error {
+ file, err := os.OpenFile(r.Filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0640)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+
+ writer := csv.NewWriter(file)
+ err = writer.WriteAll(r.records)
+ if err != nil {
+ return err
+ }
+
+ logf("Written report file %s", r.Filename)
+ return nil
+}
+
+// AddRecord adds a record to the report.
+func (r *CSVReport) AddRecord(label string, elapsed time.Duration) error {
+ if len(r.records) == 0 {
+ r.addRecord(csvFields)
+ }
+
+ record := []string{
+ fmt.Sprintf("%d", time.Now().UnixNano()/int64(time.Millisecond)), // timestamp
+ fmt.Sprintf("%d", elapsed/time.Millisecond),
+ label,
+ "", // responseCode is not used
+ "true", // success"
+ }
+ return r.addRecord(record)
+}
+
+func (r *CSVReport) addRecord(record []string) error {
+ if len(record) != len(csvFields) {
+ return fmt.Errorf("Invalid number of fields : %q", record)
+ }
+ r.records = append(r.records, record)
+ return nil
+}
diff --git a/lxd-benchmark/main.go b/lxd-benchmark/main.go
index f9c28d791..76bc9e2fd 100644
--- a/lxd-benchmark/main.go
+++ b/lxd-benchmark/main.go
@@ -3,6 +3,7 @@ package main
import (
"fmt"
"os"
+ "time"
"github.com/lxc/lxd/client"
"github.com/lxc/lxd/lxd-benchmark/benchmark"
@@ -17,6 +18,8 @@ var argImage = gnuflag.String("image", "ubuntu:", "Image to use for the test")
var argPrivileged = gnuflag.Bool("privileged", false, "Use privileged containers")
var argStart = gnuflag.Bool("start", true, "Start the container after creation")
var argFreeze = gnuflag.Bool("freeze", false, "Freeze the container right after start")
+var argReportFile = gnuflag.String("report-file", "", "A CSV file to write test file to. If the file is present, it will be appended to.")
+var argReportLabel = gnuflag.String("report-label", "", "A label for the report entry. By default, the action is used.")
func main() {
err := run(os.Args)
@@ -66,32 +69,65 @@ func run(args []string) error {
benchmark.PrintServerInfo(c)
- switch os.Args[1] {
+ var report *benchmark.CSVReport
+ if *argReportFile != "" {
+ report = &benchmark.CSVReport{Filename: *argReportFile}
+ if shared.PathExists(*argReportFile) {
+ err := report.Load()
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ action := os.Args[1]
+ var duration time.Duration
+ switch action {
case "spawn":
- _, err = benchmark.SpawnContainers(c, *argCount, *argParallel, *argImage, *argPrivileged, *argStart, *argFreeze)
- return err
+ duration, err = benchmark.SpawnContainers(
+ c, *argCount, *argParallel, *argImage, *argPrivileged, *argStart, *argFreeze)
+ if err != nil {
+ return err
+ }
case "start":
containers, err := benchmark.GetContainers(c)
if err != nil {
return err
}
- _, err = benchmark.StartContainers(c, containers, *argParallel)
- return err
+ duration, err = benchmark.StartContainers(c, containers, *argParallel)
+ if err != nil {
+ return err
+ }
case "stop":
containers, err := benchmark.GetContainers(c)
if err != nil {
return err
}
- _, err = benchmark.StopContainers(c, containers, *argParallel)
- return err
+ duration, err = benchmark.StopContainers(c, containers, *argParallel)
+ if err != nil {
+ return err
+ }
case "delete":
containers, err := benchmark.GetContainers(c)
if err != nil {
return err
}
- _, err = benchmark.DeleteContainers(c, containers, *argParallel)
- return err
+ duration, err = benchmark.DeleteContainers(c, containers, *argParallel)
+ if err != nil {
+ return err
+ }
}
+ if report != nil {
+ label := action
+ if *argReportLabel != "" {
+ label = *argReportLabel
+ }
+ report.AddRecord(label, duration)
+ err := report.Write()
+ if err != nil {
+ return err
+ }
+ }
return nil
}
From 564795f4e8f62c3c3489c6bd5e00c235ca36008d Mon Sep 17 00:00:00 2001
From: Alberto Donato <alberto.donato at canonical.com>
Date: Thu, 7 Sep 2017 10:51:05 +0200
Subject: [PATCH 2/4] benchmark: fix ensureImage when a local alias is passed
Signed-off-by: Alberto Donato <alberto.donato at canonical.com>
---
lxd-benchmark/benchmark/benchmark.go | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/lxd-benchmark/benchmark/benchmark.go b/lxd-benchmark/benchmark/benchmark.go
index 1f4bf82e7..aca077cb4 100644
--- a/lxd-benchmark/benchmark/benchmark.go
+++ b/lxd-benchmark/benchmark/benchmark.go
@@ -275,12 +275,22 @@ func ensureImage(c lxd.ContainerServer, image string) (string, error) {
logf("Failed to import image: %s", err)
return "", err
}
- } else {
- logf("Found image in local store: %s", fingerprint)
}
} else {
+ alias, _, err := c.GetImageAlias(image)
+ if err == nil {
+ fingerprint = alias.Target
+ } else {
+ _, _, err = c.GetImage(image)
+ }
+
+ if err != nil {
+ logf("Image not found in local store: %s", image)
+ return "", err
+ }
fingerprint = image
- logf("Found image in local store: %s", fingerprint)
}
+
+ logf("Found image in local store: %s", fingerprint)
return fingerprint, nil
}
From ef08b624130621f4c38528b03dfa304614b7e15f Mon Sep 17 00:00:00 2001
From: Alberto Donato <alberto.donato at canonical.com>
Date: Thu, 7 Sep 2017 11:00:11 +0200
Subject: [PATCH 3/4] benchmark: more cleanups
Signed-off-by: Alberto Donato <alberto.donato at canonical.com>
---
lxd-benchmark/benchmark/benchmark.go | 58 ++++++++++--------------------------
lxd-benchmark/benchmark/operation.go | 18 +++++++++++
2 files changed, 34 insertions(+), 42 deletions(-)
diff --git a/lxd-benchmark/benchmark/benchmark.go b/lxd-benchmark/benchmark/benchmark.go
index aca077cb4..1d25968e7 100644
--- a/lxd-benchmark/benchmark/benchmark.go
+++ b/lxd-benchmark/benchmark/benchmark.go
@@ -55,23 +55,19 @@ func SpawnContainers(c lxd.ContainerServer, count int, parallel int, image strin
defer wg.Done()
name := getContainerName(count, index)
-
- err := createContainer(c, fingerprint, name, privileged)
- if err != nil {
+ if err := createContainer(c, fingerprint, name, privileged); err != nil {
logf("Failed to spawn container '%s': %s", name, err)
return
}
if start {
- err := startContainer(c, name)
- if err != nil {
+ if err := startContainer(c, name); err != nil {
logf("Failed to start container '%s': %s", name, err)
return
}
if freeze {
- err := freezeContainer(c, name)
- if err != nil {
+ if err := freezeContainer(c, name); err != nil {
logf("Failed to freeze container '%s': %s", name, err)
return
}
@@ -96,9 +92,7 @@ func CreateContainers(c lxd.ContainerServer, count int, parallel int, fingerprin
defer wg.Done()
name := getContainerName(count, index)
-
- err := createContainer(c, fingerprint, name, privileged)
- if err != nil {
+ if err := createContainer(c, fingerprint, name, privileged); err != nil {
logf("Failed to spawn container '%s': %s", name, err)
return
}
@@ -144,8 +138,7 @@ func StartContainers(c lxd.ContainerServer, containers []api.Container, parallel
container := containers[index]
if !container.IsActive() {
- err := startContainer(c, container.Name)
- if err != nil {
+ if err := startContainer(c, container.Name); err != nil {
logf("Failed to start container '%s': %s", container.Name, err)
return
}
@@ -173,8 +166,7 @@ func StopContainers(c lxd.ContainerServer, containers []api.Container, parallel
container := containers[index]
if container.IsActive() {
- err := stopContainer(c, container.Name)
- if err != nil {
+ if err := stopContainer(c, container.Name); err != nil {
logf("Failed to stop container '%s': %s", container.Name, err)
return
}
@@ -200,26 +192,17 @@ func DeleteContainers(c lxd.ContainerServer, containers []api.Container, paralle
deleteContainer := func(index int, wg *sync.WaitGroup) {
defer wg.Done()
- ct := containers[index]
-
- if ct.IsActive() {
- err := stopContainer(c, ct.Name)
- if err != nil {
- logf("Failed to stop container '%s': %s", ct.Name, err)
+ container := containers[index]
+ name := container.Name
+ if container.IsActive() {
+ if err := stopContainer(c, name); err != nil {
+ logf("Failed to stop container '%s': %s", name, err)
return
}
}
- // Delete
- op, err := c.DeleteContainer(ct.Name)
- if err != nil {
- logf("Failed to delete container: %s", ct.Name)
- return
- }
-
- err = op.Wait()
- if err != nil {
- logf("Failed to delete container: %s", ct.Name)
+ if err := deleteContainer(c, name); err != nil {
+ logf("Failed to delete container: %s", name)
return
}
}
@@ -250,13 +233,11 @@ func ensureImage(c lxd.ContainerServer, image string) (string, error) {
fingerprint = "default"
}
- alias, _, err := imageServer.GetImageAlias(fingerprint)
- if err == nil {
+ if alias, _, err := imageServer.GetImageAlias(fingerprint); err == nil {
fingerprint = alias.Target
}
- _, _, err = c.GetImage(fingerprint)
- if err != nil {
+ if _, _, err = c.GetImage(fingerprint); err != nil {
logf("Importing image into local store: %s", fingerprint)
image, _, err := imageServer.GetImage(fingerprint)
if err != nil {
@@ -264,14 +245,7 @@ func ensureImage(c lxd.ContainerServer, image string) (string, error) {
return "", err
}
- op, err := c.CopyImage(imageServer, *image, nil)
- if err != nil {
- logf("Failed to import image: %s", err)
- return "", err
- }
-
- err = op.Wait()
- if err != nil {
+ if err := copyImage(c, imageServer, *image); err != nil {
logf("Failed to import image: %s", err)
return "", err
}
diff --git a/lxd-benchmark/benchmark/operation.go b/lxd-benchmark/benchmark/operation.go
index e01e5aaf3..b4f4f3cea 100644
--- a/lxd-benchmark/benchmark/operation.go
+++ b/lxd-benchmark/benchmark/operation.go
@@ -58,3 +58,21 @@ func freezeContainer(c lxd.ContainerServer, name string) error {
return op.Wait()
}
+
+func deleteContainer(c lxd.ContainerServer, name string) error {
+ op, err := c.DeleteContainer(name)
+ if err != nil {
+ return err
+ }
+
+ return op.Wait()
+}
+
+func copyImage(c lxd.ContainerServer, s lxd.ImageServer, image api.Image) error {
+ op, err := c.CopyImage(s, image, nil)
+ if err != nil {
+ return err
+ }
+
+ return op.Wait()
+}
From d212d23031cdaaa2055a9db0838c5f94587644af Mon Sep 17 00:00:00 2001
From: Alberto Donato <alberto.donato at canonical.com>
Date: Thu, 17 Aug 2017 16:32:28 +0200
Subject: [PATCH 4/4] add performance regression tests
Signed-off-by: Alberto Donato <alberto.donato at canonical.com>
---
test/perf.sh | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 82 insertions(+)
create mode 100755 test/perf.sh
diff --git a/test/perf.sh b/test/perf.sh
new file mode 100755
index 000000000..ab9b76551
--- /dev/null
+++ b/test/perf.sh
@@ -0,0 +1,82 @@
+#!/bin/sh -eu
+#
+# Performance tests runner
+#
+
+[ -n "${GOPATH:-}" ] && export "PATH=${GOPATH}/bin:${PATH}"
+
+PERF_LOG_CSV="perf.csv"
+
+import_subdir_files() {
+ test "$1"
+ # shellcheck disable=SC2039
+ local file
+ for file in "$1"/*.sh; do
+ # shellcheck disable=SC1090
+ . "$file"
+ done
+}
+
+import_subdir_files includes
+
+log_message() {
+ echo "==>" "$@"
+}
+
+run_benchmark() {
+ # shellcheck disable=SC2039
+ local label description opts
+ label="$1"
+ description="$2"
+ shift 2
+
+ log_message "Benchmark start: $label - $description"
+ lxd_benchmark "$@" --report-file "$PERF_LOG_CSV" --report-label "$label"
+ log_message "Benchmark completed: $label"
+}
+
+lxd_benchmark() {
+ # shellcheck disable=SC2039
+ local opts
+ [ "${LXD_TEST_IMAGE:-}" ] && opts="--image $LXD_TEST_IMAGE" || opts=""
+ lxd-benchmark "$@" $opts
+}
+
+cleanup() {
+ if [ "$TEST_RESULT" != "success" ]; then
+ rm -f "$PERF_LOG_CSV"
+ fi
+ lxd_benchmark delete # ensure all test containers have been deleted
+ kill_lxd "$LXD_DIR"
+ cleanup_lxds "$TEST_DIR"
+ log_message "Performance tests result: $TEST_RESULT"
+}
+
+trap cleanup EXIT HUP INT TERM
+
+# Setup test directories
+TEST_DIR=$(mktemp -d -p "$(pwd)" tmp.XXX)
+LXD_DIR=$(mktemp -d -p "${TEST_DIR}" XXX)
+export LXD_DIR
+chmod +x "${TEST_DIR}" "${LXD_DIR}"
+
+if [ -z "${LXD_BACKEND:-}" ]; then
+ LXD_BACKEND="dir"
+fi
+
+import_storage_backends
+
+spawn_lxd "${LXD_DIR}" true
+
+# shellcheck disable=SC2034
+TEST_RESULT=failure
+
+run_benchmark "create-one" "create 1 container" spawn --count 1 --start=false
+run_benchmark "start-one" "start 1 container" start
+run_benchmark "stop-one" "stop 1 container" stop
+run_benchmark "delete-one" "delete 1 container" delete
+run_benchmark "create-128" "create 128 containers" spawn --count 128 --start=false
+run_benchmark "delete-128" "delete 128 containers" delete
+
+# shellcheck disable=SC2034
+TEST_RESULT=success
More information about the lxc-devel
mailing list