[lxc-devel] [lxd/master] Implement clustered DNS for FAN networking
stgraber on Github
lxc-bot at linuxcontainers.org
Sat Aug 11 00:10:17 UTC 2018
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20180811/df2de11a/attachment.bin>
-------------- next part --------------
From 1ce8b1dc3b582d7f5302a0188d301334659a8923 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 10 Aug 2018 00:14:17 -0400
Subject: [PATCH 1/5] lxd/networks: Drop unused db property
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/networks.go | 1 -
1 file changed, 1 deletion(-)
diff --git a/lxd/networks.go b/lxd/networks.go
index 3c1ed5bd3b..8921fef896 100644
--- a/lxd/networks.go
+++ b/lxd/networks.go
@@ -839,7 +839,6 @@ func networkStateGet(d *Daemon, r *http.Request) Response {
type network struct {
// Properties
- db *db.Node
state *state.State
id int64
name string
From 35bb9480d2931a4a41ab62d5b333da52cbd08460 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 10 Aug 2018 19:52:34 -0400
Subject: [PATCH 2/5] lxd/networks/state: Skip non-existing interfaces
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/networks.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lxd/networks.go b/lxd/networks.go
index 8921fef896..9598628186 100644
--- a/lxd/networks.go
+++ b/lxd/networks.go
@@ -830,7 +830,7 @@ func networkStateGet(d *Daemon, r *http.Request) Response {
_, dbInfo, _ := d.cluster.NetworkGet(name)
// Sanity check
- if osInfo == nil && dbInfo == nil {
+ if osInfo == nil || dbInfo == nil {
return NotFound(fmt.Errorf("Interface '%s' not found", name))
}
From a1d41274a797b9f1b05583756fa6c82456386fca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 10 Aug 2018 19:51:57 -0400
Subject: [PATCH 3/5] lxd: Add endpoints to state struct
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/daemon.go | 2 +-
lxd/state/state.go | 21 ++++++++++++---------
lxd/state/testing.go | 2 +-
3 files changed, 14 insertions(+), 11 deletions(-)
diff --git a/lxd/daemon.go b/lxd/daemon.go
index f8703fe4bc..5d91c3069b 100644
--- a/lxd/daemon.go
+++ b/lxd/daemon.go
@@ -217,7 +217,7 @@ func isJSONRequest(r *http.Request) bool {
// State creates a new State instance liked to our internal db and os.
func (d *Daemon) State() *state.State {
- return state.NewState(d.db, d.cluster, d.maas, d.os)
+ return state.NewState(d.db, d.cluster, d.maas, d.os, d.endpoints)
}
// UnixSocket returns the full path to the unix.socket file that this daemon is
diff --git a/lxd/state/state.go b/lxd/state/state.go
index 10bd8fbf87..48ae76e10e 100644
--- a/lxd/state/state.go
+++ b/lxd/state/state.go
@@ -2,6 +2,7 @@ package state
import (
"github.com/lxc/lxd/lxd/db"
+ "github.com/lxc/lxd/lxd/endpoints"
"github.com/lxc/lxd/lxd/maas"
"github.com/lxc/lxd/lxd/sys"
)
@@ -10,19 +11,21 @@ import (
// and the operating system. It's typically used by model entities such as
// containers, volumes, etc. in order to perform changes.
type State struct {
- Node *db.Node
- Cluster *db.Cluster
- MAAS *maas.Controller
- OS *sys.OS
+ Node *db.Node
+ Cluster *db.Cluster
+ MAAS *maas.Controller
+ OS *sys.OS
+ Endpoints *endpoints.Endpoints
}
// NewState returns a new State object with the given database and operating
// system components.
-func NewState(node *db.Node, cluster *db.Cluster, maas *maas.Controller, os *sys.OS) *State {
+func NewState(node *db.Node, cluster *db.Cluster, maas *maas.Controller, os *sys.OS, endpoints *endpoints.Endpoints) *State {
return &State{
- Node: node,
- Cluster: cluster,
- MAAS: maas,
- OS: os,
+ Node: node,
+ Cluster: cluster,
+ MAAS: maas,
+ OS: os,
+ Endpoints: endpoints,
}
}
diff --git a/lxd/state/testing.go b/lxd/state/testing.go
index 27d3ac86a0..f4733458de 100644
--- a/lxd/state/testing.go
+++ b/lxd/state/testing.go
@@ -23,7 +23,7 @@ func NewTestState(t *testing.T) (*State, func()) {
osCleanup()
}
- state := NewState(node, cluster, nil, os)
+ state := NewState(node, cluster, nil, os, nil)
return state, cleanup
}
From 2df5c1d593c3dbd5341ecfae5f52dda4c916cbe2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 10 Aug 2018 15:54:53 -0400
Subject: [PATCH 4/5] lxd: Add dns forwarder
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/main.go | 4 ++
lxd/main_forkdns.go | 103 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 107 insertions(+)
create mode 100644 lxd/main_forkdns.go
diff --git a/lxd/main.go b/lxd/main.go
index 06ac4157ed..e4ccec708c 100644
--- a/lxd/main.go
+++ b/lxd/main.go
@@ -92,6 +92,10 @@ func main() {
forkconsoleCmd := cmdForkconsole{global: &globalCmd}
app.AddCommand(forkconsoleCmd.Command())
+ // forkdns sub-command
+ forkDNSCmd := cmdForkDNS{global: &globalCmd}
+ app.AddCommand(forkDNSCmd.Command())
+
// forkexec sub-command
forkexecCmd := cmdForkexec{global: &globalCmd}
app.AddCommand(forkexecCmd.Command())
diff --git a/lxd/main_forkdns.go b/lxd/main_forkdns.go
new file mode 100644
index 0000000000..0f52164238
--- /dev/null
+++ b/lxd/main_forkdns.go
@@ -0,0 +1,103 @@
+package main
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/miekg/dns"
+ "github.com/spf13/cobra"
+)
+
+type cmdForkDNS struct {
+ global *cmdGlobal
+}
+
+type dnsHandler struct {
+ domain string
+ servers []string
+}
+
+func (h *dnsHandler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
+ msg := dns.Msg{}
+ msg.SetReply(r)
+
+ // We only support single questions for now
+ if len(r.Question) != 1 {
+ msg.SetRcode(r, dns.RcodeNameError)
+ w.WriteMsg(&msg)
+ return
+ }
+
+ // Rewrite the question to the internal domain
+ origName := r.Question[0].Name
+ newName := origName
+ if strings.HasSuffix(r.Question[0].Name, fmt.Sprintf(".%s.", h.domain)) {
+ newName = fmt.Sprintf("%s.__internal.", strings.SplitN(r.Question[0].Name, fmt.Sprintf(".%s.", h.domain), 2)[0])
+ }
+
+ // Query all the servers
+ for _, server := range h.servers {
+ // Send the request to the backend server
+ r.Question[0].Name = newName
+ resp, err := dns.Exchange(r, fmt.Sprintf("%s:53", server))
+ r.Question[0].Name = origName
+ if err != nil || len(resp.Answer) == 0 {
+ // Error or empty response, try the next one
+ continue
+ }
+
+ // Send back the answer
+ msg.Answer = resp.Answer
+ w.WriteMsg(&msg)
+ return
+ }
+
+ // Fallback to NXDOMAIN
+ msg.SetRcode(r, dns.RcodeNameError)
+ w.WriteMsg(&msg)
+}
+
+func (c *cmdForkDNS) Command() *cobra.Command {
+ // Main subcommand
+ cmd := &cobra.Command{}
+ cmd.Use = "forkdns <listen address> <domain> <servers...>"
+ cmd.Short = "Internal DNS proxy for clustering"
+ cmd.Long = `Description:
+ Spawns a tiny DNS server which forwards to all upstream servers until
+ one returns a valid record.
+`
+ cmd.RunE = c.Run
+ cmd.Hidden = true
+
+ return cmd
+}
+
+func (c *cmdForkDNS) Run(cmd *cobra.Command, args []string) error {
+ // Sanity checks
+ if len(args) < 3 {
+ cmd.Help()
+
+ if len(args) == 0 {
+ return nil
+ }
+
+ return fmt.Errorf("Missing required arguments")
+ }
+
+ srv := &dns.Server{
+ Addr: args[0],
+ Net: "udp",
+ }
+
+ srv.Handler = &dnsHandler{
+ domain: args[1],
+ servers: args[2:],
+ }
+
+ err := srv.ListenAndServe()
+ if err != nil {
+ return fmt.Errorf("Failed to set udp listener: %v\n", err)
+ }
+
+ return nil
+}
From ba6ff6f6e296bf5755629445ac3f9a1480e34432 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 10 Aug 2018 19:53:15 -0400
Subject: [PATCH 5/5] lxd/networks: Add support for FAN clustered DNS
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Closes #4788
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/networks.go | 115 ++++++++++++++++++++++++++++++++++++++++--
lxd/networks_utils.go | 58 +++++++++++++++++++++
2 files changed, 170 insertions(+), 3 deletions(-)
diff --git a/lxd/networks.go b/lxd/networks.go
index 9598628186..1484fe5554 100644
--- a/lxd/networks.go
+++ b/lxd/networks.go
@@ -12,6 +12,7 @@ import (
"strconv"
"strings"
"sync"
+ "time"
"github.com/gorilla/mux"
log "github.com/lxc/lxd/shared/log15"
@@ -20,6 +21,7 @@ import (
lxd "github.com/lxc/lxd/client"
"github.com/lxc/lxd/lxd/cluster"
"github.com/lxc/lxd/lxd/db"
+ "github.com/lxc/lxd/lxd/node"
"github.com/lxc/lxd/lxd/state"
"github.com/lxc/lxd/lxd/util"
"github.com/lxc/lxd/shared"
@@ -1420,6 +1422,8 @@ func (n *network) Start() error {
}
// Configure the fan
+ dnsClustered := false
+ dnsClusteredAddress := ""
if n.config["bridge.mode"] == "fan" {
tunName := fmt.Sprintf("%s-fan", n.name)
@@ -1554,6 +1558,14 @@ func (n *network) Start() error {
if err != nil {
return err
}
+
+ // Setup clustered DNS
+ dnsClustered, err = cluster.Enabled(n.state.Node)
+ if err != nil {
+ return err
+ }
+
+ dnsClusteredAddress = strings.Split(fanAddress, "/")[0]
}
// Configure tunnels
@@ -1641,12 +1653,17 @@ func (n *network) Start() error {
}
}
- // Kill any existing dnsmasq daemon for this network
+ // Kill any existing dnsmasq and forkdns daemon for this network
err = networkKillDnsmasq(n.name, false)
if err != nil {
return err
}
+ err = networkKillForkDNS(n.name)
+ if err != nil {
+ return err
+ }
+
// Configure dnsmasq
if n.config["bridge.mode"] == "fan" || !shared.StringInSlice(n.config["ipv4.address"], []string{"", "none"}) || !shared.StringInSlice(n.config["ipv6.address"], []string{"", "none"}) {
// Setup the dnsmasq domain
@@ -1656,7 +1673,13 @@ func (n *network) Start() error {
}
if n.config["dns.mode"] != "none" {
- dnsmasqCmd = append(dnsmasqCmd, []string{"-s", dnsDomain, "-S", fmt.Sprintf("/%s/", dnsDomain)}...)
+ if dnsClustered {
+ dnsmasqCmd = append(dnsmasqCmd, []string{"-s", "__internal", "-S", "/__internal/"}...)
+ dnsmasqCmd = append(dnsmasqCmd, []string{"-S", fmt.Sprintf("/%s/%s#1053", dnsDomain, dnsClusteredAddress)}...)
+ dnsmasqCmd = append(dnsmasqCmd, fmt.Sprintf("--dhcp-option=15,%s", dnsDomain))
+ } else {
+ dnsmasqCmd = append(dnsmasqCmd, []string{"-s", dnsDomain, "-S", fmt.Sprintf("/%s/", dnsDomain)}...)
+ }
}
// Create a config file to contain additional config (and to prevent dnsmasq from reading /etc/dnsmasq.conf)
@@ -1702,6 +1725,11 @@ func (n *network) Start() error {
if err != nil {
return err
}
+
+ // Spawn DNS forwarder if needed (backgrounded to avoid deadlocks during cluster boot)
+ if dnsClustered {
+ go n.spawnForkDNS(dnsClusteredAddress)
+ }
}
return nil
@@ -1751,12 +1779,17 @@ func (n *network) Stop() error {
return err
}
- // Kill any existing dnsmasq daemon for this network
+ // Kill any existing dnsmasq and forkdns daemon for this network
err = networkKillDnsmasq(n.name, false)
if err != nil {
return err
}
+ err = networkKillForkDNS(n.name)
+ if err != nil {
+ return err
+ }
+
// Get a list of interfaces
ifaces, err := net.Interfaces()
if err != nil {
@@ -1897,3 +1930,79 @@ func (n *network) Update(newNetwork api.NetworkPut) error {
return nil
}
+
+func (n *network) spawnForkDNS(listenAddress string) error {
+ // Get the list of nodes
+ nodes, err := cluster.List(n.state)
+ if err != nil {
+ logger.Errorf("Failed to start forkdns for network '%s': %v", n.name, err)
+ return err
+ }
+
+ localAddress, err := node.HTTPSAddress(n.state.Node)
+ if err != nil {
+ logger.Errorf("Failed to start forkdns for network '%s': %v", n.name, err)
+ return err
+ }
+
+ // Grab the network address from the various nodes
+ addresses := []string{listenAddress}
+
+ cert := n.state.Endpoints.NetworkCert()
+ for _, node := range nodes {
+ address := strings.TrimPrefix(node.URL, "https://")
+ if address == localAddress {
+ continue
+ }
+
+ again:
+ client, err := cluster.Connect(address, cert, false)
+ if err != nil {
+ time.Sleep(30 * time.Second)
+ goto again
+ }
+
+ state, err := client.GetNetworkState(n.name)
+ if err != nil {
+ time.Sleep(30 * time.Second)
+ goto again
+ }
+
+ for _, addr := range state.Addresses {
+ if addr.Family != "inet" || addr.Scope != "global" {
+ continue
+ }
+
+ addresses = append(addresses, addr.Address)
+ break
+ }
+ }
+
+ // Setup the dnsmasq domain
+ dnsDomain := n.config["dns.domain"]
+ if dnsDomain == "" {
+ dnsDomain = "lxd"
+ }
+
+ // Spawn the daemon
+ cmd := exec.Cmd{}
+ cmd.Path = n.state.OS.ExecPath
+ cmd.Args = []string{n.state.OS.ExecPath, "forkdns", fmt.Sprintf("%s:1053", listenAddress), dnsDomain}
+ cmd.Args = append(cmd.Args, addresses...)
+
+ err = cmd.Start()
+ if err != nil {
+ logger.Errorf("Failed to start forkdns for network '%s': %v", n.name, err)
+ return err
+ }
+
+ // Write the PID file
+ pidPath := shared.VarPath("networks", n.name, "forkdns.pid")
+ err = ioutil.WriteFile(pidPath, []byte(fmt.Sprintf("%d\n", cmd.Process.Pid)), 0600)
+ if err != nil {
+ logger.Errorf("Failed to start forkdns for network '%s': %v", n.name, err)
+ return err
+ }
+
+ return nil
+}
diff --git a/lxd/networks_utils.go b/lxd/networks_utils.go
index 684614bbf0..e2ee93bb17 100644
--- a/lxd/networks_utils.go
+++ b/lxd/networks_utils.go
@@ -2,6 +2,7 @@ package main
import (
"bufio"
+ "bytes"
"encoding/binary"
"encoding/hex"
"fmt"
@@ -659,6 +660,63 @@ func networkFanAddress(underlay *net.IPNet, overlay *net.IPNet) (string, string,
return fmt.Sprintf("%s/%d", ipBytes.String(), overlaySize), dev, ipStr, err
}
+func networkKillForkDNS(name string) error {
+ // Check if we have a running dnsmasq at all
+ pidPath := shared.VarPath("networks", name, "forkdns.pid")
+ if !shared.PathExists(pidPath) {
+ return nil
+ }
+
+ // Grab the PID
+ content, err := ioutil.ReadFile(pidPath)
+ if err != nil {
+ return err
+ }
+ pid := strings.TrimSpace(string(content))
+
+ // Check for empty string
+ if pid == "" {
+ os.Remove(pidPath)
+ return nil
+ }
+
+ // Check if the process still exists
+ if !shared.PathExists(fmt.Sprintf("/proc/%s", pid)) {
+ os.Remove(pidPath)
+ return nil
+ }
+
+ // Check if it's dnsmasq
+ cmdArgs, err := ioutil.ReadFile(fmt.Sprintf("/proc/%s/cmdline", pid))
+ if err != nil {
+ cmdArgs = []byte{}
+ }
+
+ cmdFields := strings.Split(string(bytes.TrimRight(cmdArgs, string("\x00"))), string(byte(0)))
+
+ // Deal with deleted paths
+ if len(cmdFields) < 5 || cmdFields[1] != "forkdns" {
+ os.Remove(pidPath)
+ return nil
+ }
+
+ // Parse the pid
+ pidInt, err := strconv.Atoi(pid)
+ if err != nil {
+ return err
+ }
+
+ // Actually kill the process
+ err = syscall.Kill(pidInt, syscall.SIGKILL)
+ if err != nil {
+ return err
+ }
+
+ // Cleanup
+ os.Remove(pidPath)
+ return nil
+}
+
func networkKillDnsmasq(name string, reload bool) error {
// Check if we have a running dnsmasq at all
pidPath := shared.VarPath("networks", name, "dnsmasq.pid")
More information about the lxc-devel
mailing list