[lxc-devel] [lxd/master] Db logic cleanup part 3
freeekanayaka on Github
lxc-bot at linuxcontainers.org
Thu May 21 11:05:48 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200521/fb6db2c4/attachment.bin>
-------------- next part --------------
From 0563a86a15bff08550a5c0b237f1bb268c897bd2 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 21 May 2020 10:47:50 +0100
Subject: [PATCH 1/5] lxd/db: Use query.SelectString helper in GetLocalImages()
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/db/images.go | 19 +++----------------
lxd/images.go | 7 ++++++-
2 files changed, 9 insertions(+), 17 deletions(-)
diff --git a/lxd/db/images.go b/lxd/db/images.go
index 8845d9711d..f2c268affa 100644
--- a/lxd/db/images.go
+++ b/lxd/db/images.go
@@ -22,28 +22,15 @@ var ImageSourceProtocol = map[int]string{
2: "simplestreams",
}
-// GetLocalImages returns the names of all local images.
-func (c *Cluster) GetLocalImages() ([]string, error) {
+// GetLocalImagesFingerprints returns the fingerprints of all local images.
+func (c *ClusterTx) GetLocalImagesFingerprints() ([]string, error) {
q := `
SELECT images.fingerprint
FROM images_nodes
JOIN images ON images.id = images_nodes.image_id
WHERE node_id = ?
`
- var fp string
- inargs := []interface{}{c.nodeID}
- outfmt := []interface{}{fp}
- dbResults, err := queryScan(c, q, inargs, outfmt)
- if err != nil {
- return []string{}, err
- }
-
- results := []string{}
- for _, r := range dbResults {
- results = append(results, r[0].(string))
- }
-
- return results, nil
+ return query.SelectStrings(c.tx, q, c.nodeID)
}
// GetImages returns the names of all images (optionally only the public ones).
diff --git a/lxd/images.go b/lxd/images.go
index 08e2fe7f7e..20db22bbd9 100644
--- a/lxd/images.go
+++ b/lxd/images.go
@@ -1335,7 +1335,12 @@ func pruneExpiredImagesTask(d *Daemon) (task.Func, task.Schedule) {
func pruneLeftoverImages(d *Daemon) {
opRun := func(op *operations.Operation) error {
// Get all images
- images, err := d.cluster.GetLocalImages()
+ var images []string
+ err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
+ var err error
+ images, err = tx.GetLocalImagesFingerprints()
+ return err
+ })
if err != nil {
return errors.Wrap(err, "Unable to retrieve the list of images")
}
From b15f6be89b2476ada6ad157cbc43e327e26bb78a Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 21 May 2020 10:55:35 +0100
Subject: [PATCH 2/5] lxd/db: Use query.SelectString helper in
GetImagesFingerprints()
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/api_cluster_test.go | 2 +-
lxd/db/images.go | 44 +++++++++++++---------------------
lxd/images.go | 4 ++--
lxd/instance/instance_utils.go | 2 +-
lxd/patches.go | 8 +++----
lxd/storage_volumes.go | 2 +-
6 files changed, 26 insertions(+), 36 deletions(-)
diff --git a/lxd/api_cluster_test.go b/lxd/api_cluster_test.go
index 5eb9b12b99..fa280460dd 100644
--- a/lxd/api_cluster_test.go
+++ b/lxd/api_cluster_test.go
@@ -499,7 +499,7 @@ func TestCluster_LeaveForce(t *testing.T) {
// The image is gone, since the deleted node was the only one having a
// copy of it.
daemon = daemons[0]
- images, err := daemon.State().Cluster.GetImages("default", false)
+ images, err := daemon.State().Cluster.GetImagesFingerprints("default", false)
require.NoError(t, err)
assert.Equal(t, []string{}, images)
}
diff --git a/lxd/db/images.go b/lxd/db/images.go
index f2c268affa..c1e9b88c62 100644
--- a/lxd/db/images.go
+++ b/lxd/db/images.go
@@ -33,22 +33,8 @@ SELECT images.fingerprint
return query.SelectStrings(c.tx, q, c.nodeID)
}
-// GetImages returns the names of all images (optionally only the public ones).
-func (c *Cluster) GetImages(project string, public bool) ([]string, error) {
- err := c.Transaction(func(tx *ClusterTx) error {
- enabled, err := tx.ProjectHasImages(project)
- if err != nil {
- return errors.Wrap(err, "Check if project has images")
- }
- if !enabled {
- project = "default"
- }
- return nil
- })
- if err != nil {
- return nil, err
- }
-
+// GetImagesFingerprints returns the names of all images (optionally only the public ones).
+func (c *Cluster) GetImagesFingerprints(project string, public bool) ([]string, error) {
q := `
SELECT fingerprint
FROM images
@@ -59,20 +45,24 @@ SELECT fingerprint
q += " AND public=1"
}
- var fp string
- inargs := []interface{}{project}
- outfmt := []interface{}{fp}
- dbResults, err := queryScan(c, q, inargs, outfmt)
- if err != nil {
- return []string{}, err
- }
+ var fingerprints []string
- results := []string{}
- for _, r := range dbResults {
- results = append(results, r[0].(string))
+ err := c.Transaction(func(tx *ClusterTx) error {
+ enabled, err := tx.ProjectHasImages(project)
+ if err != nil {
+ return errors.Wrap(err, "Check if project has images")
+ }
+ if !enabled {
+ project = "default"
+ }
+ fingerprints, err = query.SelectStrings(tx.tx, q, project)
+ return err
+ })
+ if err != nil {
+ return nil, err
}
- return results, nil
+ return fingerprints, nil
}
// ExpiredImage used to store expired image info.
diff --git a/lxd/images.go b/lxd/images.go
index 20db22bbd9..17400c104a 100644
--- a/lxd/images.go
+++ b/lxd/images.go
@@ -986,7 +986,7 @@ func getImageMetadata(fname string) (*api.ImageMetadata, string, error) {
}
func doImagesGet(d *Daemon, recursion bool, project string, public bool, clauses []filter.Clause) (interface{}, error) {
- results, err := d.cluster.GetImages(project, public)
+ results, err := d.cluster.GetImagesFingerprints(project, public)
if err != nil {
return []string{}, err
}
@@ -1113,7 +1113,7 @@ func autoUpdateImages(ctx context.Context, d *Daemon) error {
}
func autoUpdateImagesInProject(ctx context.Context, d *Daemon, project string) error {
- images, err := d.cluster.GetImages(project, false)
+ images, err := d.cluster.GetImagesFingerprints(project, false)
if err != nil {
return errors.Wrap(err, "Unable to retrieve the list of images")
}
diff --git a/lxd/instance/instance_utils.go b/lxd/instance/instance_utils.go
index 7f2c0115c0..d564fa551f 100644
--- a/lxd/instance/instance_utils.go
+++ b/lxd/instance/instance_utils.go
@@ -666,7 +666,7 @@ func ResolveImage(s *state.State, project string, source api.InstanceSource) (st
return "", fmt.Errorf("Property match is only supported for local images")
}
- hashes, err := s.Cluster.GetImages(project, false)
+ hashes, err := s.Cluster.GetImagesFingerprints(project, false)
if err != nil {
return "", err
}
diff --git a/lxd/patches.go b/lxd/patches.go
index 34827c8df7..1a5067d9ae 100644
--- a/lxd/patches.go
+++ b/lxd/patches.go
@@ -386,13 +386,13 @@ func patchStorageApi(name string, d *Daemon) error {
}
// Get list of existing public images.
- imgPublic, err := d.cluster.GetImages("default", true)
+ imgPublic, err := d.cluster.GetImagesFingerprints("default", true)
if err != nil {
return err
}
// Get list of existing private images.
- imgPrivate, err := d.cluster.GetImages("default", false)
+ imgPrivate, err := d.cluster.GetImagesFingerprints("default", false)
if err != nil {
return err
}
@@ -2159,7 +2159,7 @@ INSERT INTO storage_pools_config(storage_pool_id, node_id, key, value)
}
func patchStorageApiDirCleanup(name string, d *Daemon) error {
- fingerprints, err := d.cluster.GetImages("default", false)
+ fingerprints, err := d.cluster.GetImagesFingerprints("default", false)
if err != nil {
return err
}
@@ -2741,7 +2741,7 @@ func patchStorageApiDirBindMount(name string, d *Daemon) error {
}
func patchFixUploadedAt(name string, d *Daemon) error {
- images, err := d.cluster.GetImages("default", false)
+ images, err := d.cluster.GetImagesFingerprints("default", false)
if err != nil {
return err
}
diff --git a/lxd/storage_volumes.go b/lxd/storage_volumes.go
index c3b046a698..f4c194fe2c 100644
--- a/lxd/storage_volumes.go
+++ b/lxd/storage_volumes.go
@@ -127,7 +127,7 @@ func storagePoolVolumesGet(d *Daemon, r *http.Request) response.Response {
return response.SmartError(err)
}
- projectImages, err := d.cluster.GetImages(projectName, false)
+ projectImages, err := d.cluster.GetImagesFingerprints(projectName, false)
if err != nil {
return response.SmartError(err)
}
From 19271bb47bb7a7dd0a18fd1c7af0b527d2fc4dc9 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 21 May 2020 11:22:18 +0100
Subject: [PATCH 3/5] shared/generate/db: Support int64 fields
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
shared/generate/db/mapping.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/shared/generate/db/mapping.go b/shared/generate/db/mapping.go
index 423d8e231b..37e57ad9b6 100644
--- a/shared/generate/db/mapping.go
+++ b/shared/generate/db/mapping.go
@@ -275,6 +275,7 @@ var columnarTypeNames = []string{
"bool",
"instancetype.Type",
"int",
+ "int64",
"string",
"time.Time",
}
From 68836640891e9bae11cc0a2a28d19278c84b1316 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 21 May 2020 11:30:32 +0100
Subject: [PATCH 4/5] lxd/db: Initial code generation for images (without
references)
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/db/images.go | 41 +++++++++
lxd/db/images.mapper.go | 159 ++++++++++++++++++++++++++++++++++
lxd/db/instances.go | 2 +-
shared/generate/db/mapping.go | 2 +
4 files changed, 203 insertions(+), 1 deletion(-)
create mode 100644 lxd/db/images.mapper.go
diff --git a/lxd/db/images.go b/lxd/db/images.go
index c1e9b88c62..c6dc65a2a5 100644
--- a/lxd/db/images.go
+++ b/lxd/db/images.go
@@ -15,6 +15,47 @@ import (
"github.com/lxc/lxd/shared/osarch"
)
+// Code generation directives.
+//
+//go:generate -command mapper lxd-generate db mapper -t images.mapper.go
+//go:generate mapper reset
+//
+//go:generate mapper stmt -p db -e image objects
+//go:generate mapper stmt -p db -e image objects-by-Project
+//go:generate mapper stmt -p db -e image objects-by-Project-and-Public
+//go:generate mapper stmt -p db -e image objects-by-Project-and-Fingerprint
+//go:generate mapper stmt -p db -e image objects-by-Fingerprint
+//go:generate mapper stmt -p db -e image objects-by-Cached
+//
+//go:generate mapper method -p db -e image List
+//go:generate mapper method -p db -e image Get
+
+// Image is a value object holding db-related details about an image.
+type Image struct {
+ ID int
+ Project string `db:"primary=yes&join=projects.name"`
+ Fingerprint string `db:"primary=yes&comparison=like"`
+ Type int
+ Filename string
+ Size int64
+ Public bool
+ Architecture int
+ CreationDate time.Time
+ ExpiryDate time.Time
+ UploadDate time.Time
+ Cached bool
+ LastUseDate time.Time
+ AutoUpdate int
+}
+
+// ImageFilter can be used to filter results yielded by GetImages.
+type ImageFilter struct {
+ Project string
+ Fingerprint string // Matched with LIKE
+ Public bool
+ Cached bool
+}
+
// ImageSourceProtocol maps image source protocol codes to human-readable names.
var ImageSourceProtocol = map[int]string{
0: "lxd",
diff --git a/lxd/db/images.mapper.go b/lxd/db/images.mapper.go
new file mode 100644
index 0000000000..8eb456837d
--- /dev/null
+++ b/lxd/db/images.mapper.go
@@ -0,0 +1,159 @@
+// +build linux,cgo,!agent
+
+package db
+
+// The code below was generated by lxd-generate - DO NOT EDIT!
+
+import (
+ "database/sql"
+ "fmt"
+ "github.com/lxc/lxd/lxd/db/cluster"
+ "github.com/lxc/lxd/lxd/db/query"
+ "github.com/lxc/lxd/shared/api"
+ "github.com/pkg/errors"
+)
+
+var _ = api.ServerEnvironment{}
+
+var imageObjects = cluster.RegisterStmt(`
+SELECT images.id, projects.name AS project, images.fingerprint, images.type, images.filename, images.size, images.public, images.architecture, images.creation_date, images.expiry_date, images.upload_date, images.cached, images.last_use_date, images.auto_update
+ FROM images JOIN projects ON images.project_id = projects.id
+ ORDER BY projects.id, images.fingerprint
+`)
+
+var imageObjectsByProject = cluster.RegisterStmt(`
+SELECT images.id, projects.name AS project, images.fingerprint, images.type, images.filename, images.size, images.public, images.architecture, images.creation_date, images.expiry_date, images.upload_date, images.cached, images.last_use_date, images.auto_update
+ FROM images JOIN projects ON images.project_id = projects.id
+ WHERE project = ? ORDER BY projects.id, images.fingerprint
+`)
+
+var imageObjectsByProjectAndPublic = cluster.RegisterStmt(`
+SELECT images.id, projects.name AS project, images.fingerprint, images.type, images.filename, images.size, images.public, images.architecture, images.creation_date, images.expiry_date, images.upload_date, images.cached, images.last_use_date, images.auto_update
+ FROM images JOIN projects ON images.project_id = projects.id
+ WHERE project = ? AND images.public = ? ORDER BY projects.id, images.fingerprint
+`)
+
+var imageObjectsByProjectAndFingerprint = cluster.RegisterStmt(`
+SELECT images.id, projects.name AS project, images.fingerprint, images.type, images.filename, images.size, images.public, images.architecture, images.creation_date, images.expiry_date, images.upload_date, images.cached, images.last_use_date, images.auto_update
+ FROM images JOIN projects ON images.project_id = projects.id
+ WHERE project = ? AND images.fingerprint LIKE ? ORDER BY projects.id, images.fingerprint
+`)
+
+var imageObjectsByFingerprint = cluster.RegisterStmt(`
+SELECT images.id, projects.name AS project, images.fingerprint, images.type, images.filename, images.size, images.public, images.architecture, images.creation_date, images.expiry_date, images.upload_date, images.cached, images.last_use_date, images.auto_update
+ FROM images JOIN projects ON images.project_id = projects.id
+ WHERE images.fingerprint LIKE ? ORDER BY projects.id, images.fingerprint
+`)
+
+var imageObjectsByCached = cluster.RegisterStmt(`
+SELECT images.id, projects.name AS project, images.fingerprint, images.type, images.filename, images.size, images.public, images.architecture, images.creation_date, images.expiry_date, images.upload_date, images.cached, images.last_use_date, images.auto_update
+ FROM images JOIN projects ON images.project_id = projects.id
+ WHERE images.cached = ? ORDER BY projects.id, images.fingerprint
+`)
+
+// GetImages returns all available images.
+func (c *ClusterTx) GetImages(filter ImageFilter) ([]Image, error) {
+ // Result slice.
+ objects := make([]Image, 0)
+
+ // Check which filter criteria are active.
+ criteria := map[string]interface{}{}
+ if filter.Project != "" {
+ criteria["Project"] = filter.Project
+ }
+ if filter.Fingerprint != "" {
+ criteria["Fingerprint"] = filter.Fingerprint
+ }
+ if filter.Public != false {
+ criteria["Public"] = filter.Public
+ }
+ if filter.Cached != false {
+ criteria["Cached"] = filter.Cached
+ }
+
+ // Pick the prepared statement and arguments to use based on active criteria.
+ var stmt *sql.Stmt
+ var args []interface{}
+
+ if criteria["Project"] != nil && criteria["Public"] != nil {
+ stmt = c.stmt(imageObjectsByProjectAndPublic)
+ args = []interface{}{
+ filter.Project,
+ filter.Public,
+ }
+ } else if criteria["Project"] != nil && criteria["Fingerprint"] != nil {
+ stmt = c.stmt(imageObjectsByProjectAndFingerprint)
+ args = []interface{}{
+ filter.Project,
+ filter.Fingerprint,
+ }
+ } else if criteria["Project"] != nil {
+ stmt = c.stmt(imageObjectsByProject)
+ args = []interface{}{
+ filter.Project,
+ }
+ } else if criteria["Fingerprint"] != nil {
+ stmt = c.stmt(imageObjectsByFingerprint)
+ args = []interface{}{
+ filter.Fingerprint,
+ }
+ } else if criteria["Cached"] != nil {
+ stmt = c.stmt(imageObjectsByCached)
+ args = []interface{}{
+ filter.Cached,
+ }
+ } else {
+ stmt = c.stmt(imageObjects)
+ args = []interface{}{}
+ }
+
+ // Dest function for scanning a row.
+ dest := func(i int) []interface{} {
+ objects = append(objects, Image{})
+ return []interface{}{
+ &objects[i].ID,
+ &objects[i].Project,
+ &objects[i].Fingerprint,
+ &objects[i].Type,
+ &objects[i].Filename,
+ &objects[i].Size,
+ &objects[i].Public,
+ &objects[i].Architecture,
+ &objects[i].CreationDate,
+ &objects[i].ExpiryDate,
+ &objects[i].UploadDate,
+ &objects[i].Cached,
+ &objects[i].LastUseDate,
+ &objects[i].AutoUpdate,
+ }
+ }
+
+ // Select.
+ err := query.SelectObjects(stmt, dest, args...)
+ if err != nil {
+ return nil, errors.Wrap(err, "Failed to fetch images")
+ }
+
+ return objects, nil
+}
+
+// GetImage returns the image with the given key.
+func (c *ClusterTx) GetImage(project string, fingerprint string) (*Image, error) {
+ filter := ImageFilter{}
+ filter.Project = project
+ filter.Fingerprint = fingerprint
+
+ objects, err := c.GetImages(filter)
+ if err != nil {
+ return nil, errors.Wrap(err, "Failed to fetch Image")
+ }
+
+ switch len(objects) {
+ case 0:
+ return nil, ErrNoSuchObject
+ case 1:
+ return &objects[0], nil
+ default:
+ return nil, fmt.Errorf("More than one image matches")
+ }
+}
diff --git a/lxd/db/instances.go b/lxd/db/instances.go
index 7e86fa4b16..16c4a91dfd 100644
--- a/lxd/db/instances.go
+++ b/lxd/db/instances.go
@@ -79,7 +79,7 @@ import (
//go:generate mapper method -p db -e instance Delete
//go:generate mapper method -p db -e instance Update struct=Instance
-// Instance is a value object holding db-related details about a container.
+// Instance is a value object holding db-related details about an instance.
type Instance struct {
ID int
Project string `db:"primary=yes&join=projects.name"`
diff --git a/shared/generate/db/mapping.go b/shared/generate/db/mapping.go
index 37e57ad9b6..3d7df29efc 100644
--- a/shared/generate/db/mapping.go
+++ b/shared/generate/db/mapping.go
@@ -201,6 +201,8 @@ func (f *Field) ZeroValue() string {
// FIXME: we use -1 since at the moment integer criteria are
// required to be positive.
return "-1"
+ case "bool":
+ return "false"
default:
panic("unsupported zero value")
}
From 8ce277990b34a8593e6f4c547a4994d84ead345a Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 21 May 2020 12:04:21 +0100
Subject: [PATCH 5/5] lxd/db: Use the generated GetImages code to implement
GetExpiredImages
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/db/images.go | 44 ++++++++++++++------------------------------
1 file changed, 14 insertions(+), 30 deletions(-)
diff --git a/lxd/db/images.go b/lxd/db/images.go
index c6dc65a2a5..1491cee0ee 100644
--- a/lxd/db/images.go
+++ b/lxd/db/images.go
@@ -114,41 +114,25 @@ type ExpiredImage struct {
// GetExpiredImages returns the names and project name of all images that have expired since the given time.
func (c *Cluster) GetExpiredImages(expiry int64) ([]ExpiredImage, error) {
- q := `
- SELECT
- fingerprint,
- last_use_date,
- upload_date,
- projects.name as projectName
- FROM images
- JOIN projects ON projects.id = images.project_id
- WHERE images.cached = 1`
-
- var fpStr string
- var useStr string
- var uploadStr string
- var projectName string
-
- inargs := []interface{}{}
- outfmt := []interface{}{fpStr, useStr, uploadStr, projectName}
- dbResults, err := queryScan(c, q, inargs, outfmt)
+ var images []Image
+ err := c.Transaction(func(tx *ClusterTx) error {
+ var err error
+ images, err = tx.GetImages(ImageFilter{Cached: true})
+ return err
+ })
if err != nil {
- return []ExpiredImage{}, err
+ return nil, err
}
results := []ExpiredImage{}
- for _, r := range dbResults {
+ for _, r := range images {
// Figure out the expiry
- timestamp := r[2]
- if r[1] != "" {
- timestamp = r[1]
+ timestamp := r.UploadDate
+ if !r.LastUseDate.IsZero() {
+ timestamp = r.LastUseDate
}
- var imageExpiry time.Time
- err = imageExpiry.UnmarshalText([]byte(timestamp.(string)))
- if err != nil {
- return []ExpiredImage{}, err
- }
+ imageExpiry := timestamp
imageExpiry = imageExpiry.Add(time.Duration(expiry*24) * time.Hour)
// Check if expired
@@ -157,8 +141,8 @@ func (c *Cluster) GetExpiredImages(expiry int64) ([]ExpiredImage, error) {
}
result := ExpiredImage{
- Fingerprint: r[0].(string),
- ProjectName: r[3].(string),
+ Fingerprint: r.Fingerprint,
+ ProjectName: r.Project,
}
results = append(results, result)
More information about the lxc-devel
mailing list