[lxc-devel] [lxd/master] Fix UsedBy on storage-pools

stgraber on Github lxc-bot at linuxcontainers.org
Thu Jun 18 03:04:46 UTC 2020


A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200617/e37363fd/attachment.bin>
-------------- next part --------------
From f9a3adc9c0cf21af50cb9fa4b4ce2cbcdeb1e6cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Wed, 17 Jun 2020 22:53:11 -0400
Subject: [PATCH 1/3] lxc: Don't over-escape URLs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxc/query.go | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/lxc/query.go b/lxc/query.go
index 084941e2a6..e4fae86dee 100644
--- a/lxc/query.go
+++ b/lxc/query.go
@@ -45,12 +45,16 @@ func (c *cmdQuery) Command() *cobra.Command {
 }
 
 func (c *cmdQuery) pretty(input interface{}) string {
-	pretty, err := json.MarshalIndent(input, "", "\t")
+	pretty := bytes.NewBufferString("")
+	enc := json.NewEncoder(pretty)
+	enc.SetEscapeHTML(false)
+	enc.SetIndent("", "\t")
+	err := enc.Encode(input)
 	if err != nil {
 		return fmt.Sprintf("%v", input)
 	}
 
-	return fmt.Sprintf("%s", pretty)
+	return fmt.Sprintf("%s", pretty.String())
 }
 
 func (c *cmdQuery) Run(cmd *cobra.Command, args []string) error {

From d86e304c11b3a74e864c1e192f2b667d96899249 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Wed, 17 Jun 2020 22:53:20 -0400
Subject: [PATCH 2/3] lxd: Don't over-escape URLs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxd/util/http.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lxd/util/http.go b/lxd/util/http.go
index 79dda3a3b8..3e26038429 100644
--- a/lxd/util/http.go
+++ b/lxd/util/http.go
@@ -36,7 +36,9 @@ func WriteJSON(w http.ResponseWriter, body interface{}, debug bool) error {
 		output = io.MultiWriter(w, captured)
 	}
 
-	err := json.NewEncoder(output).Encode(body)
+	enc := json.NewEncoder(output)
+	enc.SetEscapeHTML(false)
+	err := enc.Encode(body)
 
 	if captured != nil {
 		shared.DebugJson(captured)

From 4506b0d50b985a7eeebc57876fb662d5f55d286a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Wed, 17 Jun 2020 17:49:06 -0400
Subject: [PATCH 3/3] lxd/db/storage: Rework UsedBy for pools
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxd/db/storage_pools.go    | 135 +++++++++++++++++++++++++++++++++++++
 lxd/storage_pools.go       |  19 ++++--
 lxd/storage_pools_utils.go |  63 -----------------
 3 files changed, 148 insertions(+), 69 deletions(-)

diff --git a/lxd/db/storage_pools.go b/lxd/db/storage_pools.go
index f7c6d7a308..72d436d6f8 100644
--- a/lxd/db/storage_pools.go
+++ b/lxd/db/storage_pools.go
@@ -5,6 +5,7 @@ package db
 import (
 	"database/sql"
 	"fmt"
+	"sort"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -37,6 +38,140 @@ storage_pools_config JOIN storage_pools ON storage_pools.id=storage_pools_config
 	return pools, nil
 }
 
+// GetStoragePoolUsedBy looks up all users of a storage pool.
+func (c *ClusterTx) GetStoragePoolUsedBy(name string) ([]string, error) {
+	usedby := []string{}
+
+	// Get the pool ID.
+	id, err := c.GetStoragePoolID(name)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get the cluster nodes.
+	nodes, err := c.GetNodes()
+	if err != nil {
+		return nil, err
+	}
+	nodesName := map[int64]string{}
+
+	for _, node := range nodes {
+		nodesName[node.ID] = node.Name
+	}
+
+	// Get all the storage volumes on this node.
+	vols := []struct {
+		volName     string
+		volType     int64
+		projectName string
+		nodeID      int64
+	}{}
+	dest := func(i int) []interface{} {
+		vols = append(vols, struct {
+			volName     string
+			volType     int64
+			projectName string
+			nodeID      int64
+		}{})
+
+		return []interface{}{&vols[i].volName, &vols[i].volType, &vols[i].projectName, &vols[i].nodeID}
+	}
+
+	stmt, err := c.tx.Prepare("SELECT storage_volumes.name, storage_volumes.type, projects.name, storage_volumes.node_id FROM storage_volumes LEFT JOIN projects ON projects.id=storage_volumes.project_id WHERE storage_pool_id=? AND (node_id=? OR storage_volumes.type == 2) ORDER BY storage_volumes.type ASC, projects.name ASC, storage_volumes.name ASC, storage_volumes.node_id ASC")
+	if err != nil {
+		return nil, err
+	}
+
+	err = query.SelectObjects(stmt, dest, id, c.nodeID)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, r := range vols {
+		// Handle the containers.
+		if r.volType == StoragePoolVolumeTypeContainer {
+			if r.projectName == "default" {
+				usedby = append(usedby, fmt.Sprintf("/1.0/container/%s", r.volName))
+			} else {
+				usedby = append(usedby, fmt.Sprintf("/1.0/container/%s?project=%s", r.volName, r.projectName))
+			}
+		}
+
+		// Handle the images.
+		if r.volType == StoragePoolVolumeTypeImage {
+			// Get the projects using an image.
+			stmt := "SELECT projects.name FROM images LEFT JOIN projects ON projects.id=images.project_id WHERE fingerprint=?"
+			projects, err := query.SelectStrings(c.tx, stmt, r.volName)
+			if err != nil {
+				return nil, err
+			}
+
+			for _, project := range projects {
+				if project == "default" {
+					usedby = append(usedby, fmt.Sprintf("/1.0/images/%s", r.volName))
+				} else {
+					usedby = append(usedby, fmt.Sprintf("/1.0/images/%s?project=%s", r.volName, project))
+				}
+			}
+		}
+
+		// Handle the custom volumes.
+		if r.volType == StoragePoolVolumeTypeCustom {
+			if len(nodes) > 1 {
+				if r.projectName == "default" {
+					usedby = append(usedby, fmt.Sprintf("/1.0/storage-pools/%s/volumes/custom/%s?target=%s", name, r.volName, nodesName[r.nodeID]))
+				} else {
+					usedby = append(usedby, fmt.Sprintf("/1.0/storage-pools/%s/volumes/custom/%s?project=%s&target=%s", name, r.volName, r.projectName, nodesName[r.nodeID]))
+				}
+			} else {
+				if r.projectName == "default" {
+					usedby = append(usedby, fmt.Sprintf("/1.0/storage-pools/%s/volumes/custom/%s", name, r.volName))
+				} else {
+					usedby = append(usedby, fmt.Sprintf("/1.0/storage-pools/%s/volumes/custom/%s?project=%s", name, r.volName, r.projectName))
+				}
+			}
+		}
+
+		// Handle the virtual machines.
+		if r.volType == StoragePoolVolumeTypeVM {
+			if r.projectName == "default" {
+				usedby = append(usedby, fmt.Sprintf("/1.0/virtual-machine/%s", r.volName))
+			} else {
+				usedby = append(usedby, fmt.Sprintf("/1.0/virtual-machine/%s?project=%s", r.volName, r.projectName))
+			}
+		}
+	}
+
+	// Get all the profiles using the storage pool.
+	profiles, err := c.GetProfiles(ProfileFilter{})
+	if err != nil {
+		return nil, err
+	}
+
+	for _, profile := range profiles {
+		for _, v := range profile.Devices {
+			if v["type"] != "disk" {
+				continue
+			}
+
+			if v["pool"] != name {
+				continue
+			}
+		}
+
+		if profile.Project == "default" {
+			usedby = append(usedby, fmt.Sprintf("/1.0/profiles/%s", profile.Name))
+		} else {
+			usedby = append(usedby, fmt.Sprintf("/1.0/profiles/%s?project=%s", profile.Name, profile.Project))
+		}
+	}
+
+	// Sort the output.
+	sort.Strings(usedby)
+
+	return usedby, nil
+}
+
 // GetStoragePoolID returns the ID of the pool with the given name.
 func (c *ClusterTx) GetStoragePoolID(name string) (int64, error) {
 	stmt := "SELECT id FROM storage_pools WHERE name=?"
diff --git a/lxd/storage_pools.go b/lxd/storage_pools.go
index 489d9d8f03..d6636f4558 100644
--- a/lxd/storage_pools.go
+++ b/lxd/storage_pools.go
@@ -56,13 +56,16 @@ func storagePoolsGet(d *Daemon, r *http.Request) response.Response {
 		if !recursion {
 			resultString = append(resultString, fmt.Sprintf("/%s/storage-pools/%s", version.APIVersion, pool))
 		} else {
-			plID, pl, err := d.cluster.GetStoragePool(pool)
+			_, pl, err := d.cluster.GetStoragePool(pool)
 			if err != nil {
 				continue
 			}
-
 			// Get all users of the storage pool.
-			poolUsedBy, err := storagePoolUsedByGet(d.State(), projectParam(r), plID, pool)
+			poolUsedBy := []string{}
+			err = d.cluster.Transaction(func(tx *db.ClusterTx) error {
+				poolUsedBy, err = tx.GetStoragePoolUsedBy(pool)
+				return err
+			})
 			if err != nil {
 				return response.SmartError(err)
 			}
@@ -308,14 +311,18 @@ func storagePoolGet(d *Daemon, r *http.Request) response.Response {
 	poolName := mux.Vars(r)["name"]
 
 	// Get the existing storage pool.
-	poolID, pool, err := d.cluster.GetStoragePool(poolName)
+	_, pool, err := d.cluster.GetStoragePool(poolName)
 	if err != nil {
 		return response.SmartError(err)
 	}
 
 	// Get all users of the storage pool.
-	poolUsedBy, err := storagePoolUsedByGet(d.State(), projectParam(r), poolID, poolName)
-	if err != nil && err != db.ErrNoSuchObject {
+	poolUsedBy := []string{}
+	err = d.cluster.Transaction(func(tx *db.ClusterTx) error {
+		poolUsedBy, err = tx.GetStoragePoolUsedBy(poolName)
+		return err
+	})
+	if err != nil {
 		return response.SmartError(err)
 	}
 	pool.UsedBy = poolUsedBy
diff --git a/lxd/storage_pools_utils.go b/lxd/storage_pools_utils.go
index 2594ccb543..7d7087e962 100644
--- a/lxd/storage_pools_utils.go
+++ b/lxd/storage_pools_utils.go
@@ -2,14 +2,12 @@ package main
 
 import (
 	"fmt"
-	"strings"
 
 	"github.com/lxc/lxd/lxd/db"
 	"github.com/lxc/lxd/lxd/state"
 	storagePools "github.com/lxc/lxd/lxd/storage"
 	"github.com/lxc/lxd/shared"
 	"github.com/lxc/lxd/shared/api"
-	"github.com/lxc/lxd/shared/version"
 )
 
 func storagePoolUpdate(state *state.State, name, newDescription string, newConfig map[string]string, withDB bool) error {
@@ -21,67 +19,6 @@ func storagePoolUpdate(state *state.State, name, newDescription string, newConfi
 	return pool.Update(!withDB, newDescription, newConfig, nil)
 }
 
-// Report all LXD objects that are currently using the given storage pool.
-// Volumes of type "custom" are not reported.
-// /1.0/containers/alp1
-// /1.0/containers/alp1/snapshots/snap0
-// /1.0/images/cedce20b5b236f1071134beba7a5fd2aa923fda49eea4c66454dd559a5d6e906
-// /1.0/profiles/default
-func storagePoolUsedByGet(state *state.State, project string, poolID int64, poolName string) ([]string, error) {
-	// Retrieve all non-custom volumes that exist on this storage pool.
-	volumes, err := state.Cluster.GetLocalStoragePoolVolumes(project, poolID, []int{db.StoragePoolVolumeTypeContainer, db.StoragePoolVolumeTypeImage, db.StoragePoolVolumeTypeCustom, db.StoragePoolVolumeTypeVM})
-	if err != nil && err != db.ErrNoSuchObject {
-		return []string{}, err
-	}
-
-	// Retrieve all profiles that exist on this storage pool.
-	profiles, err := profilesUsingPoolGetNames(state.Cluster, project, poolName)
-
-	if err != nil {
-		return []string{}, err
-	}
-
-	slicelen := len(volumes) + len(profiles)
-	if slicelen == 0 {
-		return []string{}, nil
-	}
-
-	// Save some allocation cycles by preallocating the correct len.
-	poolUsedBy := make([]string, slicelen)
-	for i := 0; i < len(volumes); i++ {
-		apiEndpoint, _ := storagePoolVolumeTypeNameToAPIEndpoint(volumes[i].Type)
-		switch apiEndpoint {
-		case storagePoolVolumeAPIEndpointContainers:
-			if strings.Index(volumes[i].Name, shared.SnapshotDelimiter) > 0 {
-				parentName, snapOnlyName, _ := shared.InstanceGetParentAndSnapshotName(volumes[i].Name)
-				poolUsedBy[i] = fmt.Sprintf("/%s/containers/%s/snapshots/%s", version.APIVersion, parentName, snapOnlyName)
-			} else {
-				poolUsedBy[i] = fmt.Sprintf("/%s/containers/%s", version.APIVersion, volumes[i].Name)
-			}
-		case storagePoolVolumeAPIEndpointVMs:
-			if strings.Index(volumes[i].Name, shared.SnapshotDelimiter) > 0 {
-				parentName, snapOnlyName, _ := shared.InstanceGetParentAndSnapshotName(volumes[i].Name)
-				poolUsedBy[i] = fmt.Sprintf("/%s/virtual-machines/%s/snapshots/%s", version.APIVersion, parentName, snapOnlyName)
-			} else {
-				poolUsedBy[i] = fmt.Sprintf("/%s/virtual-machines/%s", version.APIVersion, volumes[i].Name)
-			}
-		case storagePoolVolumeAPIEndpointImages:
-			poolUsedBy[i] = fmt.Sprintf("/%s/images/%s", version.APIVersion, volumes[i].Name)
-		case storagePoolVolumeAPIEndpointCustom:
-			poolUsedBy[i] = fmt.Sprintf("/%s/storage-pools/%s/volumes/%s/%s", version.APIVersion, poolName, volumes[i].Type, volumes[i].Name)
-		default:
-			// If that happens the db is busted, so report an error.
-			return []string{}, fmt.Errorf("invalid storage type for storage volume \"%s\"", volumes[i].Name)
-		}
-	}
-
-	for i := 0; i < len(profiles); i++ {
-		poolUsedBy[i+len(volumes)] = fmt.Sprintf("/%s/profiles/%s", version.APIVersion, profiles[i])
-	}
-
-	return poolUsedBy, err
-}
-
 func profilesUsingPoolGetNames(db *db.Cluster, project string, poolName string) ([]string, error) {
 	usedBy := []string{}
 


More information about the lxc-devel mailing list