[lxc-devel] [lxd/master] Plug new dqlite backend
freeekanayaka on Github
lxc-bot at linuxcontainers.org
Sun Jun 23 18:48:20 UTC 2019
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20190623/6fee76a6/attachment-0001.bin>
-------------- next part --------------
From 5108cdd1eb6de69f0a0c7f052cf44942a61757ae Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Wed, 24 Apr 2019 15:09:22 +0200
Subject: [PATCH 01/39] Convert lxd/db tests to the new go-dqlite API
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/db/migration_test.go | 1 +
lxd/db/testing.go | 55 ++++++++++++++++++++++++----------------
2 files changed, 34 insertions(+), 22 deletions(-)
diff --git a/lxd/db/migration_test.go b/lxd/db/migration_test.go
index 1456440d63..316dd4f320 100644
--- a/lxd/db/migration_test.go
+++ b/lxd/db/migration_test.go
@@ -19,6 +19,7 @@ func TestLoadPreClusteringData(t *testing.T) {
// config
assert.Equal(t, []string{"id", "key", "value"}, dump.Schema["config"])
assert.Len(t, dump.Data["config"], 3)
+ assert.Equal(t, "1.2.3.4:666", dump.Data["config"][0][2])
rows := []interface{}{int64(1), "core.https_address", "1.2.3.4:666"}
assert.Equal(t, rows, dump.Data["config"][0])
rows = []interface{}{int64(2), "core.trust_password", "sekret"}
diff --git a/lxd/db/testing.go b/lxd/db/testing.go
index b1410f4820..f265db8d82 100644
--- a/lxd/db/testing.go
+++ b/lxd/db/testing.go
@@ -9,9 +9,8 @@ import (
"testing"
"time"
- "github.com/CanonicalLtd/go-dqlite"
- "github.com/CanonicalLtd/raft-test"
- "github.com/hashicorp/raft"
+ dqlite "github.com/CanonicalLtd/go-dqlite"
+ "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -107,39 +106,51 @@ func newDqliteServer(t *testing.T) (*dqlite.DatabaseServerStore, func()) {
address := listener.Addr().String()
- store, err := dqlite.DefaultServerStore(":memory:")
- require.NoError(t, err)
- require.NoError(t, store.Set(context.Background(), []dqlite.ServerInfo{{Address: address}}))
-
- id := fmt.Sprintf("%d", dqliteSerial)
- dqliteSerial++
- registry := dqlite.NewRegistry(id)
+ dir, dirCleanup := newDir(t)
- fsm := dqlite.NewFSM(registry)
+ info := dqlite.ServerInfo{ID: uint64(1), Address: listener.Addr().String()}
+ server, err := dqlite.NewServer(info, dir)
- r, raftCleanup := rafttest.Server(t, fsm, rafttest.Transport(func(i int) raft.Transport {
- require.Equal(t, i, 0)
- address := raft.ServerAddress(listener.Addr().String())
- _, transport := raft.NewInmemTransport(address)
- return transport
- }))
-
- log := newLogFunc(t)
+ err = server.Bootstrap([]dqlite.ServerInfo{info})
+ require.NoError(t, err)
- server, err := dqlite.NewServer(
- r, registry, listener, dqlite.WithServerLogFunc(log))
+ err = server.Start(listener)
require.NoError(t, err)
cleanup := func() {
require.NoError(t, server.Close())
- raftCleanup()
+ dirCleanup()
}
+ store, err := dqlite.DefaultServerStore(":memory:")
+ require.NoError(t, err)
+ ctx := context.Background()
+ require.NoError(t, store.Set(ctx, []dqlite.ServerInfo{{Address: address}}))
+
return store, cleanup
}
var dqliteSerial = 0
+// Return a new temporary directory.
+func newDir(t *testing.T) (string, func()) {
+ t.Helper()
+
+ dir, err := ioutil.TempDir("", "dqlite-replication-test-")
+ assert.NoError(t, err)
+
+ cleanup := func() {
+ _, err := os.Stat(dir)
+ if err != nil {
+ assert.True(t, os.IsNotExist(err))
+ } else {
+ assert.NoError(t, os.RemoveAll(dir))
+ }
+ }
+
+ return dir, cleanup
+}
+
func newLogFunc(t *testing.T) dqlite.LogFunc {
return func(l dqlite.LogLevel, format string, a ...interface{}) {
format = fmt.Sprintf("%s: %s", l.String(), format)
From 6e76ebf85f9c82e08ef9d8c64398769349751714 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 09:30:50 +0200
Subject: [PATCH 02/39] Drop hashicorp/raft setup code
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/raft.go | 141 ++++----------------------------------------
1 file changed, 11 insertions(+), 130 deletions(-)
diff --git a/lxd/cluster/raft.go b/lxd/cluster/raft.go
index 41f5c3b7d5..a77882b31c 100644
--- a/lxd/cluster/raft.go
+++ b/lxd/cluster/raft.go
@@ -3,7 +3,6 @@ package cluster
import (
"bytes"
"fmt"
- "io/ioutil"
"log"
"math"
"net"
@@ -14,13 +13,12 @@ import (
"strings"
"time"
- "github.com/CanonicalLtd/go-dqlite"
- "github.com/CanonicalLtd/raft-http"
- "github.com/CanonicalLtd/raft-membership"
- "github.com/boltdb/bolt"
- "github.com/hashicorp/go-hclog"
+ dqlite "github.com/CanonicalLtd/go-dqlite"
+ rafthttp "github.com/CanonicalLtd/raft-http"
+ raftmembership "github.com/CanonicalLtd/raft-membership"
+ hclog "github.com/hashicorp/go-hclog"
"github.com/hashicorp/raft"
- "github.com/hashicorp/raft-boltdb"
+ raftboltdb "github.com/hashicorp/raft-boltdb"
"github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/lxd/node"
"github.com/lxc/lxd/shared"
@@ -80,11 +78,11 @@ func newRaft(database *db.Node, cert *shared.CertInfo, latency float64) (*raftIn
// A LXD-specific wrapper around raft.Raft, which also holds a reference to its
// network transport and dqlite FSM.
type raftInstance struct {
+ info dqlite.ServerInfo
layer *rafthttp.Layer // HTTP-based raft transport layer
handler http.HandlerFunc // Handles join/leave/connect requests
membershipChanger func(*raft.Raft) // Forwards to raft membership requests from handler
logs *raftboltdb.BoltStore // Raft logs store, needs to be closed upon shutdown
- registry *dqlite.Registry // The dqlite Registry linked to the FSM and the Driver
fsm raft.FSM // The dqlite FSM linked to the raft instance
raft *raft.Raft // The actual raft instance
}
@@ -92,48 +90,11 @@ type raftInstance struct {
// Create a new raftFactory, instantiating all needed raft dependencies.
func raftInstanceInit(
db *db.Node, node *db.RaftNode, cert *shared.CertInfo, latency float64) (*raftInstance, error) {
- // FIXME: should be a parameter
- timeout := 5 * time.Second
- raftLogger := raftLogger()
-
- // Raft config.
- config := raftConfig(latency)
- config.Logger = raftLogger
- config.LocalID = raft.ServerID(strconv.Itoa(int(node.ID)))
-
- // Raft transport
- var handler *rafthttp.Handler
- var membershipChanger func(*raft.Raft)
- var layer *rafthttp.Layer
- var transport raft.Transport
addr := node.Address
if addr == "" {
- // This should normally be used only for testing as it can
- // cause split-brian, but since we are not exposing raft to the
- // network at all it's safe to do so. When this node gets
- // exposed to the network and assigned an address, we need to
- // restart raft anyways.
- config.StartAsLeader = true
- transport = raftMemoryTransport()
- } else {
- dial, err := raftDial(cert)
- if err != nil {
- return nil, err
- }
-
- transport, handler, layer, err = raftNetworkTransport(db, addr, log.New(&raftLogWriter{}, "", 0), timeout, dial)
- if err != nil {
- return nil, err
- }
- membershipChanger = func(raft *raft.Raft) {
- raftmembership.HandleChangeRequests(raft, handler.Requests())
- }
- }
-
- err := raft.ValidateConfig(config)
- if err != nil {
- return nil, errors.Wrap(err, "invalid raft configuration")
+ // This is a standalone node not exposed to the network.
+ addr = "1"
}
// Rename legacy data directory if needed.
@@ -158,71 +119,15 @@ func raftInstanceInit(
}
}
- // Raft logs store
- logs, err := raftboltdb.New(raftboltdb.Options{
- Path: filepath.Join(dir, "logs.db"),
- BoltOptions: &bolt.Options{Timeout: timeout},
- })
- if err != nil {
- return nil, errors.Wrap(err, "failed to create bolt store for raft logs")
- }
-
- // Raft snapshot store (don't log snapshots since we take them frequently)
- snaps, err := raft.NewFileSnapshotStoreWithLogger(dir, 2, log.New(ioutil.Discard, "", 0))
- if err != nil {
- return nil, errors.Wrap(err, "failed to create file snapshot store")
- }
-
- // If we are the initial node, we use the last index persisted in the
- // logs store and other checks to determine if we have ever
- // bootstrapped the cluster, and if not we do so (see raft.HasExistingState).
- if node.ID == 1 {
- err := raftMaybeBootstrap(config, logs, snaps, transport)
- if err != nil {
- return nil, errors.Wrap(err, "failed to boostrap cluster")
- }
- }
-
- // The dqlite registry and FSM.
- registry := dqlite.NewRegistry(strconv.Itoa(serial))
- serial++
- fsm := dqlite.NewFSM(registry)
-
- // The actual raft instance.
- raft, err := raft.NewRaft(config, fsm, logs, logs, snaps, transport)
- if err != nil {
- logs.Close()
- return nil, errors.Wrap(err, "failed to start raft")
- }
-
- if membershipChanger != nil {
- // Process Raft connections over HTTP. This goroutine will
- // terminate when instance.handler.Close() is called, which
- // happens indirectly when the raft instance is shutdown in
- // instance.Shutdown(), and the associated transport is closed.
- go membershipChanger(raft)
- }
-
- instance := &raftInstance{
- layer: layer,
- handler: raftHandler(cert, handler),
- membershipChanger: membershipChanger,
- logs: logs,
- registry: registry,
- fsm: fsm,
- raft: raft,
- }
+ instance := &raftInstance{}
+ instance.info.ID = uint64(node.ID)
+ instance.info.Address = addr
return instance, nil
}
var serial = 99
-// Registry returns the dqlite Registry associated with the raft instance.
-func (i *raftInstance) Registry() *dqlite.Registry {
- return i.registry
-}
-
// FSM returns the dqlite FSM associated with the raft instance.
func (i *raftInstance) FSM() raft.FSM {
return i.fsm
@@ -396,30 +301,6 @@ func raftConfig(latency float64) *raft.Config {
return config
}
-// Helper to bootstrap the raft cluster if needed.
-func raftMaybeBootstrap(
- conf *raft.Config,
- logs *raftboltdb.BoltStore,
- snaps raft.SnapshotStore,
- trans raft.Transport) error {
- // First check if we were already bootstrapped.
- hasExistingState, err := raft.HasExistingState(logs, logs, snaps)
- if err != nil {
- return errors.Wrap(err, "failed to check if raft has existing state")
- }
- if hasExistingState {
- return nil
- }
- server := raft.Server{
- ID: conf.LocalID,
- Address: trans.LocalAddr(),
- }
- configuration := raft.Configuration{
- Servers: []raft.Server{server},
- }
- return raft.BootstrapCluster(conf, logs, logs, snaps, trans, configuration)
-}
-
func raftHandler(info *shared.CertInfo, handler *rafthttp.Handler) http.HandlerFunc {
if handler == nil {
return nil
From 924c4d13344ec0fd0e25c5bab6e1a1fae8aef935 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 09:31:45 +0200
Subject: [PATCH 03/39] Improve gateway standalone test
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway_test.go | 19 +++++++++++++++----
1 file changed, 15 insertions(+), 4 deletions(-)
diff --git a/lxd/cluster/gateway_test.go b/lxd/cluster/gateway_test.go
index 076d2ad160..364da1f73d 100644
--- a/lxd/cluster/gateway_test.go
+++ b/lxd/cluster/gateway_test.go
@@ -10,7 +10,7 @@ import (
"path/filepath"
"testing"
- "github.com/CanonicalLtd/go-dqlite"
+ dqlite "github.com/CanonicalLtd/go-dqlite"
"github.com/hashicorp/raft"
"github.com/lxc/lxd/lxd/cluster"
"github.com/lxc/lxd/lxd/db"
@@ -46,16 +46,27 @@ func TestGateway_Single(t *testing.T) {
}
dial := gateway.DialFunc()
- conn, err := dial(context.Background(), "")
+ netConn, err := dial(context.Background(), "")
assert.NoError(t, err)
- assert.NotNil(t, conn)
+ assert.NotNil(t, netConn)
+ require.NoError(t, netConn.Close())
leader, err := gateway.LeaderAddress()
assert.Equal(t, "", leader)
assert.EqualError(t, err, "Node is not clustered")
+
+ driver, err := dqlite.NewDriver(
+ gateway.ServerStore(),
+ dqlite.WithDialFunc(gateway.DialFunc()))
+ require.NoError(t, err)
+
+ conn, err := driver.Open("test.db")
+ require.NoError(t, err)
+
+ require.NoError(t, conn.Close())
}
-// If there's a network address configured, we expose the gRPC endpoint with
+// If there's a network address configured, we expose the dqlite endpoint with
// an HTTP handler.
func TestGateway_SingleWithNetworkAddress(t *testing.T) {
db, cleanup := db.NewTestNode(t)
From 80a2a44b3edf461c698178755fb2b817fd77c249 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 09:32:09 +0200
Subject: [PATCH 04/39] No need to shutdown hashicorp/raft instance
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway.go | 12 +-----------
1 file changed, 1 insertion(+), 11 deletions(-)
diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index 865b4b0ebc..438c50eba4 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -14,7 +14,7 @@ import (
"sync"
"time"
- "github.com/CanonicalLtd/go-dqlite"
+ dqlite "github.com/CanonicalLtd/go-dqlite"
"github.com/hashicorp/raft"
"github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/lxd/util"
@@ -349,16 +349,6 @@ func (g *Gateway) Kill() {
func (g *Gateway) Shutdown() error {
logger.Debugf("Stop database gateway")
- g.lock.RLock()
- if g.raft != nil {
- err := g.raft.Shutdown()
- if err != nil {
- g.lock.RUnlock()
- return errors.Wrap(err, "Failed to shutdown raft")
- }
- }
- g.lock.RUnlock()
-
if g.server != nil {
g.Sync()
g.server.Close()
From d34b10663d3362feca7b90882d03d18b6b065c97 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 09:38:09 +0200
Subject: [PATCH 05/39] Bootstrap dqlite for new servers
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway.go | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index 438c50eba4..9b3df765fa 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -531,6 +531,20 @@ func (g *Gateway) init() error {
return errors.Wrap(err, "Failed to create dqlite server")
}
+ if raft.info.Address == "1" {
+ // Bootstrap the node. This is a no-op if we are
+ // already bootstrapped..
+ err := server.Bootstrap([]dqlite.ServerInfo{raft.info})
+ if err != nil && err != dqlite.ErrServerCantBootstrap {
+ return errors.Wrap(err, "Failed to bootstrap dqlite server")
+ }
+ }
+
+ err = server.Start(listener)
+ if err != nil {
+ return errors.Wrap(err, "Failed to start dqlite server")
+ }
+
g.lock.Lock()
g.server = server
g.raft = raft
From 906ba809e15ce1e6c4e32db62aa0f2ef63d7931f Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 09:38:48 +0200
Subject: [PATCH 06/39] Instantiate dqlite
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway.go | 18 ++++++++++++------
1 file changed, 12 insertions(+), 6 deletions(-)
diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index 9b3df765fa..fc8171c299 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -511,21 +511,27 @@ func (g *Gateway) init() error {
if err != nil {
return errors.Wrap(err, "Failed to allocate loopback port")
}
+ options := []dqlite.ServerOption{
+ dqlite.WithServerLogFunc(DqliteLog),
+ }
- if raft.HandlerFunc() == nil {
+ if raft.info.Address == "1" {
+ if raft.info.ID != 1 {
+ panic("unexpected server ID")
+ }
g.memoryDial = dqliteMemoryDial(listener)
g.store.inMemory = dqlite.NewInmemServerStore()
- g.store.Set(context.Background(), []dqlite.ServerInfo{{Address: "0"}})
+ g.store.Set(context.Background(), []dqlite.ServerInfo{raft.info})
} else {
go runDqliteProxy(listener, g.acceptCh)
g.store.inMemory = nil
}
- provider := &raftAddressProvider{db: g.db}
+ dir := filepath.Join(g.db.Dir(), "global")
server, err := dqlite.NewServer(
- raft.Raft(), raft.Registry(), listener,
- dqlite.WithServerAddressProvider(provider),
- dqlite.WithServerLogFunc(DqliteLog),
+ raft.info,
+ dir,
+ options...,
)
if err != nil {
return errors.Wrap(err, "Failed to create dqlite server")
From 4ba345904cdd5050a32ec33b2a2313912b5ee8d8 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 09:39:04 +0200
Subject: [PATCH 07/39] Fix comment
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway.go | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index fc8171c299..56e34fbd84 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -504,8 +504,8 @@ func (g *Gateway) init() error {
}
// If the resulting raft instance is not nil, it means that this node
- // should serve as database node, so create a dqlite driver to be
- // exposed it over gRPC.
+ // should serve as database node, so create a dqlite driver possibly
+ // exposing it over the network.
if raft != nil {
listener, err := net.Listen("unix", "")
if err != nil {
From c3e0620d12b0b329b63f18d307438193956fa9c8 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 09:50:57 +0200
Subject: [PATCH 08/39] Set max open conns before running schema upgrades
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/db/db.go | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/lxd/db/db.go b/lxd/db/db.go
index 2a19ec6f24..8fcd4890d3 100644
--- a/lxd/db/db.go
+++ b/lxd/db/db.go
@@ -6,7 +6,7 @@ import (
"sync"
"time"
- "github.com/CanonicalLtd/go-dqlite"
+ dqlite "github.com/CanonicalLtd/go-dqlite"
"github.com/pkg/errors"
"github.com/lxc/lxd/lxd/db/cluster"
@@ -164,6 +164,9 @@ func OpenCluster(name string, store dqlite.ServerStore, address, dir string, tim
return nil, errors.Wrap(err, "failed to open database")
}
+ db.SetMaxOpenConns(1)
+ db.SetMaxIdleConns(1)
+
// Test that the cluster database is operational. We wait up to the
// given timeout , in case there's no quorum of nodes online yet.
timer := time.After(timeout)
@@ -210,9 +213,6 @@ func OpenCluster(name string, store dqlite.ServerStore, address, dir string, tim
return nil, errors.Wrap(err, "failed to ensure schema")
}
- db.SetMaxOpenConns(1)
- db.SetMaxIdleConns(1)
-
if !nodesVersionsMatch {
cluster := &Cluster{
db: db,
From 3355bc0225b5ada17fa4e90249a1fab47410de15 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 09:52:31 +0200
Subject: [PATCH 09/39] Update docstring
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway.go | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index 56e34fbd84..79bb09f3d5 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -30,11 +30,11 @@ import (
// When a new gateway is created, the node-level database is queried to check
// what kind of role this node plays and if it's exposed over the network. It
// will initialize internal data structures accordingly, for example starting a
-// dqlite driver if this node is a database node.
+// local dqlite server if this node is a database node.
//
// After creation, the Daemon is expected to expose whatever http handlers the
-// HandlerFuncs method returns and to access the dqlite cluster using the gRPC
-// dialer returned by the Dialer method.
+// HandlerFuncs method returns and to access the dqlite cluster using the
+// dialer returned by the DialFunc method.
func NewGateway(db *db.Node, cert *shared.CertInfo, options ...Option) (*Gateway, error) {
ctx, cancel := context.WithCancel(context.Background())
From d80335336fd53ca12198275c75ff754ba4aa59ad Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 09:53:43 +0200
Subject: [PATCH 10/39] Conditionally check leadership in dqlite dial function
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway.go | 68 ++++++++++++++++++++++--------------------
1 file changed, 35 insertions(+), 33 deletions(-)
diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index 79bb09f3d5..248b9760b8 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -643,52 +643,54 @@ func (g *Gateway) cachedRaftNodes() ([]string, error) {
return addresses, nil
}
-func dqliteNetworkDial(ctx context.Context, addr string, g *Gateway) (net.Conn, error) {
+func dqliteNetworkDial(ctx context.Context, addr string, g *Gateway, checkLeader bool) (net.Conn, error) {
config, err := tlsClientConfig(g.cert)
if err != nil {
return nil, err
}
- // Make a probe HEAD request to check if the target node is the leader.
path := fmt.Sprintf("https://%s%s", addr, databaseEndpoint)
- request, err := http.NewRequest("HEAD", path, nil)
- if err != nil {
- return nil, err
- }
- request = request.WithContext(ctx)
- client := &http.Client{Transport: &http.Transport{TLSClientConfig: config}}
- response, err := client.Do(request)
- if err != nil {
- return nil, err
- }
+ if checkLeader {
+ // Make a probe HEAD request to check if the target node is the leader.
+ request, err := http.NewRequest("HEAD", path, nil)
+ if err != nil {
+ return nil, err
+ }
+ request = request.WithContext(ctx)
+ client := &http.Client{Transport: &http.Transport{TLSClientConfig: config}}
+ response, err := client.Do(request)
+ if err != nil {
+ return nil, err
+ }
- // If the remote server has detected that we are out of date, let's
- // trigger an upgrade.
- if response.StatusCode == http.StatusUpgradeRequired {
- g.lock.Lock()
- defer g.lock.Unlock()
- if !g.upgradeTriggered {
- err = triggerUpdate()
- if err == nil {
- g.upgradeTriggered = true
+ // If the remote server has detected that we are out of date, let's
+ // trigger an upgrade.
+ if response.StatusCode == http.StatusUpgradeRequired {
+ g.lock.Lock()
+ defer g.lock.Unlock()
+ if !g.upgradeTriggered {
+ err = triggerUpdate()
+ if err == nil {
+ g.upgradeTriggered = true
+ }
}
+ return nil, fmt.Errorf("Upgrade needed")
}
- return nil, fmt.Errorf("Upgrade needed")
- }
- // If the endpoint does not exists, it means that the target node is
- // running version 1 of dqlite protocol. In that case we simply behave
- // as the node was at an older LXD version.
- if response.StatusCode == http.StatusNotFound {
- return nil, db.ErrSomeNodesAreBehind
- }
+ // If the endpoint does not exists, it means that the target node is
+ // running version 1 of dqlite protocol. In that case we simply behave
+ // as the node was at an older LXD version.
+ if response.StatusCode == http.StatusNotFound {
+ return nil, db.ErrSomeNodesAreBehind
+ }
- if response.StatusCode != http.StatusOK {
- return nil, fmt.Errorf(response.Status)
+ if response.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf(response.Status)
+ }
}
// Establish the connection
- request = &http.Request{
+ request := &http.Request{
Method: "POST",
Proto: "HTTP/1.1",
ProtoMajor: 1,
@@ -717,7 +719,7 @@ func dqliteNetworkDial(ctx context.Context, addr string, g *Gateway) (net.Conn,
return nil, errors.Wrap(err, "Sending HTTP request failed")
}
- response, err = http.ReadResponse(bufio.NewReader(conn), request)
+ response, err := http.ReadResponse(bufio.NewReader(conn), request)
if err != nil {
return nil, errors.Wrap(err, "Failed to read response")
}
From 0a1cf26f3ac09588713e064eec2222c197ada605 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 09:54:11 +0200
Subject: [PATCH 11/39] Copy network data between TLS Go conn and Unix socket
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway.go | 36 +++++++++++++++++++++++++++++++++++-
1 file changed, 35 insertions(+), 1 deletion(-)
diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index 248b9760b8..50aba6f5ec 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -730,7 +730,41 @@ func dqliteNetworkDial(ctx context.Context, addr string, g *Gateway, checkLeader
return nil, fmt.Errorf("Missing or unexpected Upgrade header in response")
}
- return conn, err
+ listener, err := net.Listen("unix", "")
+ if err != nil {
+ return nil, errors.Wrap(err, "Failed to create unix listener")
+ }
+
+ goUnix, err := net.Dial("unix", listener.Addr().String())
+ if err != nil {
+ return nil, errors.Wrap(err, "Failed to connect to unix listener")
+ }
+
+ cUnix, err := listener.Accept()
+ if err != nil {
+ return nil, errors.Wrap(err, "Failed to connect to unix listener")
+ }
+
+ listener.Close()
+
+ go func() {
+ _, err := io.Copy(eagain.Writer{Writer: goUnix}, eagain.Reader{Reader: conn})
+ if err != nil {
+ logger.Warnf("Error during dqlite proxy copy: %v", err)
+ }
+ conn.Close()
+ }()
+
+ go func() {
+ _, err := io.Copy(eagain.Writer{Writer: conn}, eagain.Reader{Reader: goUnix})
+ if err != nil {
+ logger.Warnf("Error during dqlite proxy copy: %v", err)
+ }
+
+ goUnix.Close()
+ }()
+
+ return cUnix, nil
}
// Create a dial function that connects to the given listener.
From e1d9e00fe4372856fc3b5ac54026982bf1ebc471 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 09:55:26 +0200
Subject: [PATCH 12/39] Add Gateway.isLeader() function
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway.go | 8 ++++++++
lxd/cluster/gateway_export_test.go | 5 +++++
2 files changed, 13 insertions(+)
diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index 50aba6f5ec..cbf3a6ad29 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -588,6 +588,14 @@ func (g *Gateway) waitLeadership() error {
return fmt.Errorf("RAFT node did not self-elect within %s", time.Duration(n)*sleep)
}
+func (g *Gateway) isLeader() bool {
+ if g.server == nil {
+ return false
+ }
+ info := g.server.Leader()
+ return info != nil && info.ID == g.raft.info.ID
+}
+
// Return information about the LXD nodes that a currently part of the raft
// cluster, as configured in the raft log. It returns an error if this node is
// not the leader.
diff --git a/lxd/cluster/gateway_export_test.go b/lxd/cluster/gateway_export_test.go
index 6592158dbf..a5089643aa 100644
--- a/lxd/cluster/gateway_export_test.go
+++ b/lxd/cluster/gateway_export_test.go
@@ -11,6 +11,11 @@ func (g *Gateway) Raft() *raft.Raft {
return g.raft.raft
}
+// IsLeader returns true if this node is the leader.
+func (g *Gateway) IsLeader() bool {
+ return g.isLeader()
+}
+
// Cert returns the gateway's internal TLS certificate information.
func (g *Gateway) Cert() *shared.CertInfo {
return g.cert
From 05eb4d2b64a5d807a6c4493c959624fddc73420b Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 09:56:37 +0200
Subject: [PATCH 13/39] Wire isLeader()
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway.go | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index cbf3a6ad29..248153b22c 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -187,11 +187,12 @@ func (g *Gateway) HandlerFuncs() map[string]http.HandlerFunc {
return
}
- // Before actually establishing the gRPC SQL connection, our
- // dialer probes the node to see if it's currently the leader
+ // Before actually establishing the connection, our dialer
+ // probes the node to see if it's currently the leader
// (otherwise it tries with another node or retry later).
if r.Method == "HEAD" {
- if g.raft.Raft().State() != raft.Leader {
+ info := g.server.Leader()
+ if info == nil || info.ID != g.raft.info.ID {
http.Error(w, "503 not leader", http.StatusServiceUnavailable)
return
}
@@ -421,11 +422,11 @@ func (g *Gateway) LeaderAddress() (string, error) {
// If this is a raft node, return the address of the current leader, or
// wait a bit until one is elected.
- if g.raft != nil {
+ if g.server != nil {
for ctx.Err() == nil {
- address := string(g.raft.Raft().Leader())
- if address != "" {
- return address, nil
+ info := g.server.Leader()
+ if info != nil {
+ return info.Address, nil
}
time.Sleep(time.Second)
}
@@ -577,7 +578,7 @@ func (g *Gateway) waitLeadership() error {
sleep := 250 * time.Millisecond
for i := 0; i < n; i++ {
g.lock.RLock()
- if g.raft.raft.State() == raft.Leader {
+ if g.isLeader() {
g.lock.RUnlock()
return nil
}
From ad609bfee85242fa418fdd3d9ba902ea9c78fc00 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 09:56:59 +0200
Subject: [PATCH 14/39] Check TLS cert in raft connection handler
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway.go | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index 248153b22c..139d690125 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -268,6 +268,11 @@ func (g *Gateway) HandlerFuncs() map[string]http.HandlerFunc {
return
}
+ if !tlsCheckCert(r, g.cert) {
+ http.Error(w, "403 invalid client certificate", http.StatusForbidden)
+ return
+ }
+
// If this node is not clustered return a 404.
if g.raft.HandlerFunc() == nil {
http.NotFound(w, r)
From 9028ab4dd7be3597578b0d6b7145716f6896c248 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 09:57:26 +0200
Subject: [PATCH 15/39] Use ID instead of address to detect initial node
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway.go | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index 139d690125..5fbc71bfba 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -543,9 +543,9 @@ func (g *Gateway) init() error {
return errors.Wrap(err, "Failed to create dqlite server")
}
- if raft.info.Address == "1" {
+ if raft.info.ID == 1 {
// Bootstrap the node. This is a no-op if we are
- // already bootstrapped..
+ // already bootstrapped.
err := server.Bootstrap([]dqlite.ServerInfo{raft.info})
if err != nil && err != dqlite.ErrServerCantBootstrap {
return errors.Wrap(err, "Failed to bootstrap dqlite server")
From 68df753506fc2cd2b45ace7e18438d7217a5c803 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 09:58:40 +0200
Subject: [PATCH 16/39] Get information about current servers from dqlite
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway.go | 12 ++++--------
1 file changed, 4 insertions(+), 8 deletions(-)
diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index 5fbc71bfba..12ed0ee538 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -612,14 +612,14 @@ func (g *Gateway) currentRaftNodes() ([]db.RaftNode, error) {
if g.raft == nil {
return nil, raft.ErrNotLeader
}
- servers, err := g.raft.Servers()
+ servers, err := g.server.Cluster()
if err != nil {
return nil, err
}
provider := raftAddressProvider{db: g.db}
nodes := make([]db.RaftNode, len(servers))
for i, server := range servers {
- address, err := provider.ServerAddr(server.ID)
+ address, err := provider.ServerAddr(raft.ServerID(fmt.Sprintf("%d", server.ID)))
if err != nil {
if err != db.ErrNoSuchObject {
return nil, errors.Wrap(err, "Failed to fetch raft server address")
@@ -627,13 +627,9 @@ func (g *Gateway) currentRaftNodes() ([]db.RaftNode, error) {
// Use the initial address as fallback. This is an edge
// case that happens when a new leader is elected and
// its raft_nodes table is not fully up-to-date yet.
- address = server.Address
+ address = raft.ServerAddress(server.Address)
}
- id, err := strconv.Atoi(string(server.ID))
- if err != nil {
- return nil, errors.Wrap(err, "Non-numeric server ID")
- }
- nodes[i].ID = int64(id)
+ nodes[i].ID = int64(server.ID)
nodes[i].Address = string(address)
}
return nodes, nil
From f7ab0739fe5d99c7217772ce87b98f2dfc31fc4a Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 09:59:14 +0200
Subject: [PATCH 17/39] Retry copy-related errors
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index 12ed0ee538..589e5ff8c7 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -808,7 +808,7 @@ func runDqliteProxy(listener net.Listener, acceptCh chan net.Conn) {
src := <-acceptCh
dst, err := net.Dial("unix", listener.Addr().String())
if err != nil {
- panic(err)
+ continue
}
go func() {
From 966b396d398b50a321c018dc5b4c0530d2a93d53 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 10:00:18 +0200
Subject: [PATCH 18/39] Better formatting
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway_test.go | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/lxd/cluster/gateway_test.go b/lxd/cluster/gateway_test.go
index 364da1f73d..5174b76409 100644
--- a/lxd/cluster/gateway_test.go
+++ b/lxd/cluster/gateway_test.go
@@ -57,7 +57,8 @@ func TestGateway_Single(t *testing.T) {
driver, err := dqlite.NewDriver(
gateway.ServerStore(),
- dqlite.WithDialFunc(gateway.DialFunc()))
+ dqlite.WithDialFunc(gateway.DialFunc()),
+ )
require.NoError(t, err)
conn, err := driver.Open("test.db")
@@ -87,7 +88,10 @@ func TestGateway_SingleWithNetworkAddress(t *testing.T) {
mux.HandleFunc(path, handler)
}
- driver, err := dqlite.NewDriver(store, dqlite.WithDialFunc(gateway.DialFunc()))
+ driver, err := dqlite.NewDriver(
+ gateway.ServerStore(),
+ dqlite.WithDialFunc(gateway.DialFunc()),
+ )
require.NoError(t, err)
conn, err := driver.Open("test.db")
From 152021bc4bef5cb97df70c4645cfcbc02966b680 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 10:01:00 +0200
Subject: [PATCH 19/39] Custom dqlite dial function
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway.go | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index 589e5ff8c7..cb2bbfaecf 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -323,7 +323,14 @@ func (g *Gateway) DialFunc() dqlite.DialFunc {
return g.memoryDial(ctx, address)
}
- return dqliteNetworkDial(ctx, address, g)
+ return dqliteNetworkDial(ctx, address, g, true)
+ }
+}
+
+// Dial function for establishing raft connections.
+func (g *Gateway) raftDial() dqlite.DialFunc {
+ return func(ctx context.Context, address string) (net.Conn, error) {
+ return dqliteNetworkDial(ctx, address, g.cert, false)
}
}
@@ -531,6 +538,7 @@ func (g *Gateway) init() error {
} else {
go runDqliteProxy(listener, g.acceptCh)
g.store.inMemory = nil
+ options = append(options, dqlite.WithServerDialFunc(g.raftDial()))
}
dir := filepath.Join(g.db.Dir(), "global")
From d6f6eed7646c107f216464a15d29ab4fae7e31f9 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 10:02:03 +0200
Subject: [PATCH 20/39] Update comment
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/membership.go | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lxd/cluster/membership.go b/lxd/cluster/membership.go
index de49f7c45b..be4c70882e 100644
--- a/lxd/cluster/membership.go
+++ b/lxd/cluster/membership.go
@@ -131,8 +131,8 @@ func Bootstrap(state *state.State, gateway *Gateway, name string) error {
// Make sure we can actually connect to the cluster database through
// the network endpoint. This also releases the previously acquired
// lock and makes the Go SQL pooling system invalidate the old
- // connection, so new queries will be executed over the new gRPC
- // network connection.
+ // connection, so new queries will be executed over the new network
+ // connection.
err = state.Cluster.ExitExclusive(func(tx *db.ClusterTx) error {
_, err := tx.Nodes()
return err
From 963eca91b0a58ea1b7bfb01e38c1be79eca26ac6 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 10:02:26 +0200
Subject: [PATCH 21/39] Use dqlite's join primitive
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/membership.go | 20 +++++++++++++-------
1 file changed, 13 insertions(+), 7 deletions(-)
diff --git a/lxd/cluster/membership.go b/lxd/cluster/membership.go
index be4c70882e..973c1e0c39 100644
--- a/lxd/cluster/membership.go
+++ b/lxd/cluster/membership.go
@@ -306,20 +306,24 @@ func Join(state *state.State, gateway *Gateway, cert *shared.CertInfo, name stri
// If we are listed among the database nodes, join the raft cluster.
id := ""
- target := ""
+ var target *db.RaftNode
for _, node := range nodes {
if node.Address == address {
id = strconv.Itoa(int(node.ID))
} else {
- target = node.Address
+ target = &node
}
}
if id != "" {
+ if target == nil {
+ panic("no other node found")
+ }
logger.Info(
"Joining dqlite raft cluster",
- log15.Ctx{"id": id, "address": address, "target": target})
- changer := gateway.raft.MembershipChanger()
- err := changer.Join(raft.ServerID(id), raft.ServerAddress(target), 5*time.Second)
+ log15.Ctx{"id": id, "address": address, "target": target.Address})
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ err := gateway.server.Join(ctx, gateway.ServerStore(), gateway.raftDial())
if err != nil {
return err
}
@@ -593,8 +597,10 @@ func Promote(state *state.State, gateway *Gateway, nodes []db.RaftNode) error {
logger.Info(
"Joining dqlite raft cluster",
log15.Ctx{"id": id, "address": address, "target": target})
- changer := gateway.raft.MembershipChanger()
- err = changer.Join(raft.ServerID(id), raft.ServerAddress(target), 5*time.Second)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ err = gateway.server.Join(ctx, gateway.ServerStore(), gateway.raftDial())
if err != nil {
return err
}
From bb0725e32f535ce4b90b3ba504fb0fdd3308c6e6 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 10:02:44 +0200
Subject: [PATCH 22/39] Use dqlite leave primitive
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/membership.go | 17 ++++++-----------
1 file changed, 6 insertions(+), 11 deletions(-)
diff --git a/lxd/cluster/membership.go b/lxd/cluster/membership.go
index 973c1e0c39..970b1d0edc 100644
--- a/lxd/cluster/membership.go
+++ b/lxd/cluster/membership.go
@@ -8,9 +8,7 @@ import (
"strconv"
"time"
- rafthttp "github.com/CanonicalLtd/raft-http"
- raftmembership "github.com/CanonicalLtd/raft-membership"
- "github.com/hashicorp/raft"
+ dqlite "github.com/CanonicalLtd/go-dqlite"
"github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/lxd/db/cluster"
"github.com/lxc/lxd/lxd/node"
@@ -21,6 +19,7 @@ import (
"github.com/lxc/lxd/shared/log15"
"github.com/lxc/lxd/shared/logger"
"github.com/pkg/errors"
+ "golang.org/x/net/context"
)
// Bootstrap turns a non-clustered LXD instance into the first (and leader)
@@ -680,19 +679,15 @@ func Leave(state *state.State, gateway *Gateway, name string, force bool) (strin
return address, nil
}
- id := strconv.Itoa(int(raftNodes[raftNodeRemoveIndex].ID))
+ id := uint64(raftNodes[raftNodeRemoveIndex].ID)
// Get the address of another database node,
target := raftNodes[(raftNodeRemoveIndex+1)%len(raftNodes)].Address
logger.Info(
"Remove node from dqlite raft cluster",
log15.Ctx{"id": id, "address": address, "target": target})
- dial, err := raftDial(gateway.cert)
- if err != nil {
- return "", err
- }
- err = rafthttp.ChangeMembership(
- raftmembership.LeaveRequest, raftEndpoint, dial,
- raft.ServerID(id), address, target, 5*time.Second)
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ err = dqlite.Leave(ctx, uint64(id), gateway.ServerStore(), gateway.raftDial())
if err != nil {
return "", err
}
From 98d7dd3cef257d5b2ac7aa8d21ef0860b0c8e15f Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 10:01:11 +0200
Subject: [PATCH 23/39] Update docstring
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway.go | 1 -
lxd/cluster/gateway_test.go | 2 +-
2 files changed, 1 insertion(+), 2 deletions(-)
diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index cb2bbfaecf..aaa63671b8 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -10,7 +10,6 @@ import (
"net/url"
"os"
"path/filepath"
- "strconv"
"sync"
"time"
diff --git a/lxd/cluster/gateway_test.go b/lxd/cluster/gateway_test.go
index 5174b76409..fdeecbb710 100644
--- a/lxd/cluster/gateway_test.go
+++ b/lxd/cluster/gateway_test.go
@@ -140,7 +140,7 @@ func TestGateway_NetworkAuth(t *testing.T) {
}
-// RaftNodes returns an error if the underlying raft instance is not the leader.
+// RaftNodes returns all nodes of the cluster.
func TestGateway_RaftNodesNotLeader(t *testing.T) {
db, cleanup := db.NewTestNode(t)
defer cleanup()
From bfa108612f6dafb77398fa52716e21184a8a5c42 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 10:01:41 +0200
Subject: [PATCH 24/39] Update unit tests
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway_test.go | 12 +++--
lxd/cluster/heartbeat_test.go | 4 +-
lxd/cluster/membership_test.go | 11 ++--
lxd/cluster/raft_test.go | 99 +---------------------------------
4 files changed, 13 insertions(+), 113 deletions(-)
diff --git a/lxd/cluster/gateway_test.go b/lxd/cluster/gateway_test.go
index fdeecbb710..d151dd0809 100644
--- a/lxd/cluster/gateway_test.go
+++ b/lxd/cluster/gateway_test.go
@@ -11,7 +11,6 @@ import (
"testing"
dqlite "github.com/CanonicalLtd/go-dqlite"
- "github.com/hashicorp/raft"
"github.com/lxc/lxd/lxd/cluster"
"github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/shared"
@@ -79,7 +78,7 @@ func TestGateway_SingleWithNetworkAddress(t *testing.T) {
defer server.Close()
address := server.Listener.Addr().String()
- store := setRaftRole(t, db, address)
+ setRaftRole(t, db, address)
gateway := newGateway(t, db, cert)
defer gateway.Shutdown()
@@ -156,9 +155,12 @@ func TestGateway_RaftNodesNotLeader(t *testing.T) {
gateway := newGateway(t, db, cert)
defer gateway.Shutdown()
- // Get the node immediately, before the election has took place.
- _, err := gateway.RaftNodes()
- assert.Equal(t, raft.ErrNotLeader, err)
+ nodes, err := gateway.RaftNodes()
+ require.NoError(t, err)
+
+ assert.Len(t, nodes, 1)
+ assert.Equal(t, nodes[0].ID, int64(1))
+ assert.Equal(t, nodes[0].Address, address)
}
// Create a new test Gateway with the given parameters, and ensure no error
diff --git a/lxd/cluster/heartbeat_test.go b/lxd/cluster/heartbeat_test.go
index a99ccf942a..7b5f5b793e 100644
--- a/lxd/cluster/heartbeat_test.go
+++ b/lxd/cluster/heartbeat_test.go
@@ -6,7 +6,7 @@ import (
"testing"
"time"
- "github.com/CanonicalLtd/go-dqlite"
+ dqlite "github.com/CanonicalLtd/go-dqlite"
"github.com/hashicorp/raft"
"github.com/lxc/lxd/lxd/cluster"
"github.com/lxc/lxd/lxd/db"
@@ -160,7 +160,7 @@ func (f *heartbeatFixture) Leader() *cluster.Gateway {
for {
for _, gateway := range f.gateways {
- if gateway.Raft().State() == raft.Leader {
+ if gateway.IsLeader() {
return gateway
}
}
diff --git a/lxd/cluster/membership_test.go b/lxd/cluster/membership_test.go
index 384d85b699..c08c19a7e2 100644
--- a/lxd/cluster/membership_test.go
+++ b/lxd/cluster/membership_test.go
@@ -8,7 +8,7 @@ import (
"testing"
"time"
- "github.com/CanonicalLtd/go-dqlite"
+ dqlite "github.com/CanonicalLtd/go-dqlite"
"github.com/lxc/lxd/lxd/cluster"
"github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/lxd/state"
@@ -352,14 +352,9 @@ func TestJoin(t *testing.T) {
require.NoError(t, err)
// The node has gone from the raft cluster.
- raft := targetGateway.Raft()
- future := raft.GetConfiguration()
- require.NoError(t, future.Error())
- assert.Len(t, future.Configuration().Servers, 1)
-
- count, err = cluster.Count(state)
+ members, err := targetGateway.RaftNodes()
require.NoError(t, err)
- assert.Equal(t, 1, count)
+ assert.Len(t, members, 1)
}
func FLAKY_TestPromote(t *testing.T) {
diff --git a/lxd/cluster/raft_test.go b/lxd/cluster/raft_test.go
index 56cae84936..0952831791 100644
--- a/lxd/cluster/raft_test.go
+++ b/lxd/cluster/raft_test.go
@@ -3,114 +3,17 @@ package cluster_test
import (
"net/http"
"net/http/httptest"
- "strconv"
"testing"
- "time"
- "github.com/CanonicalLtd/go-dqlite"
- "github.com/CanonicalLtd/raft-test"
- "github.com/hashicorp/raft"
+ dqlite "github.com/CanonicalLtd/go-dqlite"
"github.com/lxc/lxd/lxd/cluster"
"github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/lxd/util"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/logging"
- "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
-// By default a node starts in single mode.
-func TestRaftFactory_Single(t *testing.T) {
- db, cleanup := db.NewTestNode(t)
- defer cleanup()
-
- cert := shared.TestingKeyPair()
-
- instance := newRaft(t, db, cert)
- defer instance.Shutdown()
-
- rafttest.WaitLeader(t, instance.Raft(), time.Second)
- assert.Equal(t, raft.Leader, instance.Raft().State())
-}
-
-// If there's a network address configured, but we are the only raft node in
-// the factory starts raft in single mode.
-func TestRaftFactory_SingleWithNetworkAddress(t *testing.T) {
- db, cleanup := db.NewTestNode(t)
- defer cleanup()
-
- cert := shared.TestingKeyPair()
-
- setRaftRole(t, db, "1.2.3.4:666")
-
- instance := newRaft(t, db, cert)
- defer instance.Shutdown()
-
- rafttest.WaitLeader(t, instance.Raft(), time.Second)
- assert.Equal(t, raft.Leader, instance.Raft().State())
-}
-
-// When the factory is started the first time on a non-clustered node, it will
-// use the memory transport and the raft node will not have a real network
-// address. The in-memory address gets saved in the first log committed in the
-// store as the address of the server with ID "1". If the LXD instance is then
-// reconfigured to enable clustering, we now use a real network transport and
-// setup a ServerAddressProvider that will override the initial in-memory
-// address of node "1" with its real network address, as configured in the
-// raft_nodes table.
-func TestRaftFactory_TransitionToClusteredMode(t *testing.T) {
- db, cleanup := db.NewTestNode(t)
- defer cleanup()
-
- cert := shared.TestingKeyPair()
-
- instance := newRaft(t, db, cert)
- instance.Shutdown()
-
- setRaftRole(t, db, "1.2.3.4:666")
-
- instance = newRaft(t, db, cert)
- defer instance.Shutdown()
-
- rafttest.WaitLeader(t, instance.Raft(), time.Second)
- assert.Equal(t, raft.Leader, instance.Raft().State())
-}
-
-// If there is more than one node, the raft object is created with
-// cluster-compatible parameters..
-func TestRaftFactory_MultiNode(t *testing.T) {
- cert := shared.TestingKeyPair()
-
- leader := ""
- for i := 0; i < 2; i++ {
- db, cleanup := db.NewTestNode(t)
- defer cleanup()
-
- mux := http.NewServeMux()
- server := newServer(cert, mux)
- defer server.Close()
-
- address := server.Listener.Addr().String()
- setRaftRole(t, db, address)
-
- instance := newRaft(t, db, cert)
- defer instance.Shutdown()
- if i == 0 {
- leader = address
- rafttest.WaitLeader(t, instance.Raft(), time.Second)
- }
-
- mux.HandleFunc("/internal/raft", instance.HandlerFunc())
-
- if i > 0 {
- id := raft.ServerID(strconv.Itoa(i + 1))
- target := raft.ServerAddress(leader)
- err := instance.MembershipChanger().Join(id, target, 5*time.Second)
- require.NoError(t, err)
- }
- }
-}
-
// Create a new test RaftInstance.
func newRaft(t *testing.T, db *db.Node, cert *shared.CertInfo) *cluster.RaftInstance {
logging.Testing(t)
From 815aaad00672ac0bbe97e0baceb47ed4326b923a Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 10:51:19 +0200
Subject: [PATCH 25/39] Fix import
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/membership.go | 1 -
1 file changed, 1 deletion(-)
diff --git a/lxd/cluster/membership.go b/lxd/cluster/membership.go
index 970b1d0edc..9458682d69 100644
--- a/lxd/cluster/membership.go
+++ b/lxd/cluster/membership.go
@@ -19,7 +19,6 @@ import (
"github.com/lxc/lxd/shared/log15"
"github.com/lxc/lxd/shared/logger"
"github.com/pkg/errors"
- "golang.org/x/net/context"
)
// Bootstrap turns a non-clustered LXD instance into the first (and leader)
From 29b7b2f816727be3584206e670832745a82118cd Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Sun, 12 May 2019 11:59:46 +0200
Subject: [PATCH 26/39] Extract container list logic from containersShutdown
into containersOnDisk
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/containers.go | 60 ++++++++++++++++++++++++++++++++---------------
1 file changed, 41 insertions(+), 19 deletions(-)
diff --git a/lxd/containers.go b/lxd/containers.go
index 4f541ba0dd..5fa73a126b 100644
--- a/lxd/containers.go
+++ b/lxd/containers.go
@@ -204,6 +204,34 @@ func (slice containerStopList) Swap(i, j int) {
slice[i], slice[j] = slice[j], slice[i]
}
+// Return the names of all local containers, grouped by project. The
+// information is obtained by reading the data directory.
+func containersOnDisk() (map[string][]string, error) {
+ containers := map[string][]string{}
+
+ files, err := ioutil.ReadDir(shared.VarPath("containers"))
+ if err != nil {
+ return nil, err
+ }
+
+ for _, file := range files {
+ name := file.Name()
+ project := "default"
+ if strings.Contains(name, "_") {
+ fields := strings.Split(file.Name(), "_")
+ project = fields[0]
+ name = fields[1]
+ }
+ names, ok := containers[project]
+ if !ok {
+ names = []string{}
+ }
+ containers[project] = append(names, name)
+ }
+
+ return containers, nil
+}
+
func containersShutdown(s *state.State) error {
var wg sync.WaitGroup
@@ -217,30 +245,24 @@ func containersShutdown(s *state.State) error {
containers = []container{}
// List all containers on disk
- files, err := ioutil.ReadDir(shared.VarPath("containers"))
+ cnames, err := containersOnDisk()
if err != nil {
return err
}
- for _, file := range files {
- project := "default"
- name := file.Name()
- if strings.Contains(name, "_") {
- fields := strings.Split(file.Name(), "_")
- project = fields[0]
- name = fields[1]
+ for project, names := range cnames {
+ for _, name := range names {
+ c, err := containerLXCLoad(s, db.ContainerArgs{
+ Project: project,
+ Name: name,
+ Config: make(map[string]string),
+ }, nil)
+ if err != nil {
+ return err
+ }
+
+ containers = append(containers, c)
}
-
- c, err := containerLXCLoad(s, db.ContainerArgs{
- Project: project,
- Name: name,
- Config: make(map[string]string),
- }, nil)
- if err != nil {
- return err
- }
-
- containers = append(containers, c)
}
}
From 27c94bd55a0715b6e1c91dbea867bb92ee5a3b80 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Sun, 12 May 2019 12:00:25 +0200
Subject: [PATCH 27/39] Fix lint
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/containers.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lxd/containers.go b/lxd/containers.go
index 5fa73a126b..8893308994 100644
--- a/lxd/containers.go
+++ b/lxd/containers.go
@@ -276,7 +276,7 @@ func containersShutdown(s *state.State) error {
}
}
- var lastPriority int = 0
+ var lastPriority int
if len(containers) != 0 {
lastPriority, _ = strconv.Atoi(containers[0].ExpandedConfig()["boot.stop.priority"])
From a6d91dbcd61eae63741da15b54aabd1e8030ea84 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Sun, 12 May 2019 12:02:09 +0200
Subject: [PATCH 28/39] Don't use the db in legacy patch 12
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/patches.go | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/lxd/patches.go b/lxd/patches.go
index 8d24632089..52cf3d3093 100644
--- a/lxd/patches.go
+++ b/lxd/patches.go
@@ -3464,7 +3464,7 @@ var legacyPatches = map[int](func(d *Daemon) error){
30: patchUpdateFromV29,
31: patchUpdateFromV30,
}
-var legacyPatchesNeedingDB = []int{11, 12, 16} // Legacy patches doing DB work
+var legacyPatchesNeedingDB = []int{11, 16} // Legacy patches doing DB work
func patchUpdateFromV10(d *Daemon) error {
if shared.PathExists(shared.VarPath("lxc")) {
@@ -3483,13 +3483,15 @@ func patchUpdateFromV10(d *Daemon) error {
}
func patchUpdateFromV11(d *Daemon) error {
- cNames, err := d.cluster.LegacyContainersList(db.CTypeSnapshot)
+ containers, err := containersOnDisk()
if err != nil {
return err
}
errors := 0
+ cNames := containers["default"]
+
for _, cName := range cNames {
snapParentName, snapOnlyName, _ := containerGetParentAndSnapshotName(cName)
oldPath := shared.VarPath("containers", snapParentName, "snapshots", snapOnlyName)
From aed112ae485309d5091bdd78898ea9c530320c58 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 09:50:14 +0200
Subject: [PATCH 29/39] Only use the schema db transaction in legacy patches
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/daemon.go | 33 ++++++++++++++++++---------------
lxd/db/db.go | 8 ++++----
lxd/db/legacy.go | 22 +++++-----------------
lxd/patches.go | 40 ++++++++++++----------------------------
4 files changed, 39 insertions(+), 64 deletions(-)
diff --git a/lxd/daemon.go b/lxd/daemon.go
index a01acc3381..066a39e4d4 100644
--- a/lxd/daemon.go
+++ b/lxd/daemon.go
@@ -16,12 +16,13 @@ import (
"time"
"github.com/CanonicalLtd/candidclient"
- "github.com/CanonicalLtd/go-dqlite"
+ dqlite "github.com/CanonicalLtd/go-dqlite"
"github.com/gorilla/mux"
"github.com/pkg/errors"
"golang.org/x/net/context"
"golang.org/x/sys/unix"
"gopkg.in/lxc/go-lxc.v2"
+
"gopkg.in/macaroon-bakery.v2/bakery"
"gopkg.in/macaroon-bakery.v2/bakery/checkers"
"gopkg.in/macaroon-bakery.v2/bakery/identchecker"
@@ -766,6 +767,20 @@ func (d *Daemon) init() error {
}
}
+ // This logic used to belong to patchUpdateFromV10, but has been moved
+ // here because it needs database access.
+ if shared.PathExists(shared.VarPath("lxc")) {
+ err := os.Rename(shared.VarPath("lxc"), shared.VarPath("containers"))
+ if err != nil {
+ return err
+ }
+
+ logger.Debugf("Restarting all the containers following directory rename")
+ s := d.State()
+ containersShutdown(s)
+ containersRestart(s)
+ }
+
// Setup the user-agent
if clustered {
version.UserAgentFeatures([]string{"cluster"})
@@ -1303,23 +1318,11 @@ func initializeDbObject(d *Daemon) (*db.Dump, error) {
legacy := map[int]*db.LegacyPatch{}
for i, patch := range legacyPatches {
legacy[i] = &db.LegacyPatch{
- Hook: func(node *sql.DB) error {
- // FIXME: Use the low-level *node* SQL db as backend for both the
- // db.Node and db.Cluster objects, since at this point we
- // haven't migrated the data to the cluster database yet.
- cluster := d.cluster
- defer func() {
- d.cluster = cluster
- }()
- d.db = db.ForLegacyPatches(node)
- d.cluster = db.ForLocalInspection(node)
- return patch(d)
+ Hook: func(tx *sql.Tx) error {
+ return patch(tx)
},
}
}
- for _, i := range legacyPatchesNeedingDB {
- legacy[i].NeedsDB = true
- }
// Hook to run when the local database is created from scratch. It will
// create the default profile and mark all patches as applied.
diff --git a/lxd/db/db.go b/lxd/db/db.go
index 8fcd4890d3..704ed7f704 100644
--- a/lxd/db/db.go
+++ b/lxd/db/db.go
@@ -58,7 +58,10 @@ func OpenNode(dir string, fresh func(*Node) error, legacyPatches map[int]*Legacy
return nil, nil, err
}
- legacyHook := legacyPatchHook(db, legacyPatches)
+ db.SetMaxOpenConns(1)
+ db.SetMaxIdleConns(1)
+
+ legacyHook := legacyPatchHook(legacyPatches)
hook := func(version int, tx *sql.Tx) error {
if version == node.UpdateFromPreClustering {
logger.Debug("Loading pre-clustering sqlite data")
@@ -89,9 +92,6 @@ func OpenNode(dir string, fresh func(*Node) error, legacyPatches map[int]*Legacy
}
}
- db.SetMaxOpenConns(1)
- db.SetMaxIdleConns(1)
-
return node, dump, nil
}
diff --git a/lxd/db/legacy.go b/lxd/db/legacy.go
index f1cc8427b2..87523d1351 100644
--- a/lxd/db/legacy.go
+++ b/lxd/db/legacy.go
@@ -11,33 +11,21 @@ import (
// could do non-db work and depend on functionality external to the db
// package. See UpdatesApplyAll below.
type LegacyPatch struct {
- NeedsDB bool // Whether the patch does any DB-related work
- Hook func(*sql.DB) error // The actual patch logic
+ Hook func(*sql.Tx) error // The actual patch logic
}
-func legacyPatchHook(db *sql.DB, patches map[int]*LegacyPatch) schema.Hook {
+func legacyPatchHook(patches map[int]*LegacyPatch) schema.Hook {
return func(version int, tx *sql.Tx) error {
patch, ok := patches[version]
if !ok {
return nil
}
- // FIXME We need to commit the transaction before the
- // hook and then open it again afterwards because this
- // legacy patch pokes with the database and would fail
- // with a lock error otherwise.
- if patch.NeedsDB {
- _, err := tx.Exec("COMMIT")
- if err != nil {
- return err
- }
- }
- err := patch.Hook(db)
+
+ err := patch.Hook(tx)
if err != nil {
return err
}
- if patch.NeedsDB {
- _, err = tx.Exec("BEGIN")
- }
+
return err
}
}
diff --git a/lxd/patches.go b/lxd/patches.go
index 52cf3d3093..28d210db34 100644
--- a/lxd/patches.go
+++ b/lxd/patches.go
@@ -1,6 +1,7 @@
package main
import (
+ "database/sql"
"fmt"
"io/ioutil"
stdlog "log"
@@ -3457,32 +3458,20 @@ func patchStorageApiUpdateContainerSnapshots(name string, d *Daemon) error {
//
// NOTE: don't add any legacy patch here, instead use the patches
// mechanism above.
-var legacyPatches = map[int](func(d *Daemon) error){
+var legacyPatches = map[int](func(tx *sql.Tx) error){
11: patchUpdateFromV10,
12: patchUpdateFromV11,
16: patchUpdateFromV15,
30: patchUpdateFromV29,
31: patchUpdateFromV30,
}
-var legacyPatchesNeedingDB = []int{11, 16} // Legacy patches doing DB work
-
-func patchUpdateFromV10(d *Daemon) error {
- if shared.PathExists(shared.VarPath("lxc")) {
- err := os.Rename(shared.VarPath("lxc"), shared.VarPath("containers"))
- if err != nil {
- return err
- }
-
- logger.Debugf("Restarting all the containers following directory rename")
- s := d.State()
- containersShutdown(s)
- containersRestart(s)
- }
+func patchUpdateFromV10(_ *sql.Tx) error {
+ // Logic was moved to Daemon.init().
return nil
}
-func patchUpdateFromV11(d *Daemon) error {
+func patchUpdateFromV11(_ *sql.Tx) error {
containers, err := containersOnDisk()
if err != nil {
return err
@@ -3552,27 +3541,22 @@ func patchUpdateFromV11(d *Daemon) error {
return nil
}
-func patchUpdateFromV15(d *Daemon) error {
+func patchUpdateFromV15(tx *sql.Tx) error {
// munge all LVM-backed containers' LV names to match what is
// required for snapshot support
- cNames, err := d.cluster.LegacyContainersList(db.CTypeRegular)
+ containers, err := containersOnDisk()
if err != nil {
return err
}
+ cNames := containers["default"]
vgName := ""
- err = d.db.Transaction(func(tx *db.NodeTx) error {
- config, err := tx.Config()
- if err != nil {
- return err
- }
- vgName = config["storage.lvm_vg_name"]
- return nil
- })
+ config, err := query.SelectConfig(tx, "config", "")
if err != nil {
return err
}
+ vgName = config["storage.lvm_vg_name"]
for _, cName := range cNames {
var lvLinkPath string
@@ -3613,7 +3597,7 @@ func patchUpdateFromV15(d *Daemon) error {
return nil
}
-func patchUpdateFromV29(d *Daemon) error {
+func patchUpdateFromV29(_ *sql.Tx) error {
if shared.PathExists(shared.VarPath("zfs.img")) {
err := os.Chmod(shared.VarPath("zfs.img"), 0600)
if err != nil {
@@ -3624,7 +3608,7 @@ func patchUpdateFromV29(d *Daemon) error {
return nil
}
-func patchUpdateFromV30(d *Daemon) error {
+func patchUpdateFromV30(_ *sql.Tx) error {
entries, err := ioutil.ReadDir(shared.VarPath("containers"))
if err != nil {
/* If the directory didn't exist before, the user had never
From b9359bf18d944d573086ae1baa321bb7b0e5e592 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Sun, 12 May 2019 17:00:42 +0200
Subject: [PATCH 30/39] Turn patchShrinkLogsDBFile into a no-op
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/daemon.go | 33 +++-----------
lxd/patches.go | 116 ++-----------------------------------------------
2 files changed, 9 insertions(+), 140 deletions(-)
diff --git a/lxd/daemon.go b/lxd/daemon.go
index 066a39e4d4..fc7782da36 100644
--- a/lxd/daemon.go
+++ b/lxd/daemon.go
@@ -604,34 +604,6 @@ func (d *Daemon) init() error {
return err
}
- clustered, err := cluster.Enabled(d.db)
- if err != nil {
- return err
- }
-
- // If not already applied, run the daemon patch that shrinks the boltdb
- // file. We can't run this daemon patch later on along with the other
- // ones because it needs to run before we open the cluster database.
- appliedPatches, err := d.db.Patches()
- if err != nil {
- return errors.Wrap(err, "Fetch applied daemon patches")
- }
- if !shared.StringInSlice("shrink_logs_db_file", appliedPatches) {
- if !clustered {
- // We actually run the patch only if this lxd daemon is
- // not clustered.
- err := patchShrinkLogsDBFile("", d)
- if err != nil {
- return errors.Wrap(err, "Shrink logs.db file")
- }
- }
-
- err = d.db.PatchesMarkApplied("shrink_logs_db_file")
- if err != nil {
- return err
- }
- }
-
/* Setup dqlite */
clusterLogLevel := "ERROR"
if shared.StringInSlice("dqlite", trace) {
@@ -690,6 +662,11 @@ func (d *Daemon) init() error {
return err
}
+ clustered, err := cluster.Enabled(d.db)
+ if err != nil {
+ return err
+ }
+
/* Open the cluster database */
for {
logger.Info("Initializing global database")
diff --git a/lxd/patches.go b/lxd/patches.go
index 28d210db34..f6023eab95 100644
--- a/lxd/patches.go
+++ b/lxd/patches.go
@@ -4,19 +4,11 @@ import (
"database/sql"
"fmt"
"io/ioutil"
- stdlog "log"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
- "time"
-
- "github.com/boltdb/bolt"
- "github.com/hashicorp/raft"
- "github.com/hashicorp/raft-boltdb"
- "github.com/pkg/errors"
- "golang.org/x/sys/unix"
"github.com/lxc/lxd/lxd/cluster"
"github.com/lxc/lxd/lxd/db"
@@ -24,6 +16,8 @@ import (
"github.com/lxc/lxd/shared"
log "github.com/lxc/lxd/shared/log15"
"github.com/lxc/lxd/shared/logger"
+ "github.com/pkg/errors"
+ "golang.org/x/sys/unix"
)
/* Patches are one-time actions that are sometimes needed to update
@@ -241,111 +235,9 @@ func patchNetworkPermissions(name string, d *Daemon) error {
return nil
}
-// Shrink a database/global/logs.db that grew unwildly due to a bug in the 3.6
-// release.
+// This patch used to shrink the database/global/logs.db file, but it's not
+// needed anymore since dqlite 1.0.
func patchShrinkLogsDBFile(name string, d *Daemon) error {
- dir := filepath.Join(d.os.VarDir, "database", "global")
- info, err := os.Stat(filepath.Join(dir, "logs.db"))
- if err != nil {
- if os.IsNotExist(err) {
- // The boltdb file is not there at all, nothing to do.
- return nil
- }
- return errors.Wrap(err, "Get the size of the boltdb database")
- }
-
- if info.Size() < 1024*1024*100 {
- // Only try to shrink databases bigger than 100 Megabytes.
- return nil
- }
-
- snaps, err := raft.NewFileSnapshotStoreWithLogger(
- dir, 2, stdlog.New(ioutil.Discard, "", 0))
- if err != nil {
- return errors.Wrap(err, "Open snapshots")
- }
-
- metas, err := snaps.List()
- if err != nil {
- return errors.Wrap(err, "Fetch snapshots")
- }
-
- if len(metas) == 0 {
- // No snapshot is available, we can't shrink. This should never
- // happen, in practice.
- logger.Warnf("Can't shrink boltdb store, no raft snapshot is available")
- return nil
- }
-
- meta := metas[0] // The most recent snapshot.
-
- // Copy all log entries from the current boltdb file into a new one,
- // which will be smaller since it excludes all truncated entries that
- pathCur := filepath.Join(dir, "logs.db")
- // got allocated before the latest snapshot.
- logsCur, err := raftboltdb.New(raftboltdb.Options{
- Path: pathCur,
- BoltOptions: &bolt.Options{
- Timeout: 10 * time.Second,
- ReadOnly: true,
- },
- })
- if err != nil {
- return errors.Wrap(err, "Open current boltdb store")
- }
- defer logsCur.Close()
-
- pathNew := filepath.Join(dir, "logs.db.new")
- logsNew, err := raftboltdb.New(raftboltdb.Options{
- Path: pathNew,
- BoltOptions: &bolt.Options{Timeout: 10 * time.Second},
- })
- if err != nil {
- return errors.Wrap(err, "Open new boltdb store")
- }
- defer logsNew.Close()
-
- lastIndex, err := logsCur.LastIndex()
- if err != nil {
- return errors.Wrap(err, "Get most recent raft index")
- }
-
- for index := meta.Index; index <= lastIndex; index++ {
- log := &raft.Log{}
-
- err := logsCur.GetLog(index, log)
- if err != nil {
- return errors.Wrapf(err, "Get raft entry at index %d", index)
- }
-
- err = logsNew.StoreLog(log)
- if err != nil {
- return errors.Wrapf(err, "Store raft entry at index %d", index)
- }
- }
-
- term, err := logsCur.GetUint64([]byte("CurrentTerm"))
- if err != nil {
- return errors.Wrap(err, "Get current term")
- }
- err = logsNew.SetUint64([]byte("CurrentTerm"), term)
- if err != nil {
- return errors.Wrap(err, "Store current term")
- }
-
- logsCur.Close()
- logsNew.Close()
-
- err = os.Remove(pathCur)
- if err != nil {
- return errors.Wrap(err, "Remove current boltdb store")
- }
-
- err = os.Rename(pathNew, pathCur)
- if err != nil {
- return errors.Wrap(err, "Rename new boltdb store")
- }
-
return nil
}
From 6174f01bac2e9395a815e44a0d3039953d6d75fd Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Sun, 12 May 2019 17:59:34 +0200
Subject: [PATCH 31/39] Perform data migration to dqlite 1.0 format
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway.go | 15 +-
lxd/cluster/migrate.go | 303 ++++++++++++++++++
lxd/cluster/migrate_test.go | 77 +++++
lxd/cluster/testdata/pre10/db.bin | Bin 0 -> 4096 bytes
lxd/cluster/testdata/pre10/db.bin-wal | Bin 0 -> 1005312 bytes
lxd/cluster/testdata/pre10/logs.db | Bin 0 -> 1245184 bytes
.../snapshots/1-62-1557738365979/meta.json | 1 +
.../snapshots/1-62-1557738365979/state.bin | Bin 0 -> 1009439 bytes
lxd/daemon.go | 1 +
9 files changed, 396 insertions(+), 1 deletion(-)
create mode 100644 lxd/cluster/migrate.go
create mode 100644 lxd/cluster/migrate_test.go
create mode 100644 lxd/cluster/testdata/pre10/db.bin
create mode 100644 lxd/cluster/testdata/pre10/db.bin-wal
create mode 100644 lxd/cluster/testdata/pre10/logs.db
create mode 100644 lxd/cluster/testdata/pre10/snapshots/1-62-1557738365979/meta.json
create mode 100644 lxd/cluster/testdata/pre10/snapshots/1-62-1557738365979/state.bin
diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index aaa63671b8..419a8b42c8 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -515,6 +515,20 @@ func (g *Gateway) init() error {
return errors.Wrap(err, "Failed to create raft factory")
}
+ dir := filepath.Join(g.db.Dir(), "global")
+ if shared.PathExists(filepath.Join(dir, "logs.db")) {
+ err := shared.DirCopy(dir, dir+".bak")
+ if err != nil {
+ return errors.Wrap(err, "Failed to backup global database")
+ }
+ err = MigrateToDqlite10(dir)
+ if err != nil {
+ return errors.Wrap(err, "Failed to migrate to dqlite 1.0")
+ }
+ os.Remove(filepath.Join(dir, "logs.db"))
+ os.RemoveAll(filepath.Join(dir, "snapshots"))
+ }
+
// If the resulting raft instance is not nil, it means that this node
// should serve as database node, so create a dqlite driver possibly
// exposing it over the network.
@@ -540,7 +554,6 @@ func (g *Gateway) init() error {
options = append(options, dqlite.WithServerDialFunc(g.raftDial()))
}
- dir := filepath.Join(g.db.Dir(), "global")
server, err := dqlite.NewServer(
raft.info,
dir,
diff --git a/lxd/cluster/migrate.go b/lxd/cluster/migrate.go
new file mode 100644
index 0000000000..5b39987798
--- /dev/null
+++ b/lxd/cluster/migrate.go
@@ -0,0 +1,303 @@
+package cluster
+
+import (
+ "bytes"
+ "encoding/binary"
+ "fmt"
+ "io/ioutil"
+ "path/filepath"
+ "strconv"
+
+ "github.com/boltdb/bolt"
+ "github.com/hashicorp/go-msgpack/codec"
+ "github.com/hashicorp/raft"
+ raftboltdb "github.com/hashicorp/raft-boltdb"
+ "github.com/lxc/lxd/shared"
+ "github.com/pkg/errors"
+)
+
+// MigrateToDqlite10 migrates the on-disk global database format to dqlite 1.0.
+func MigrateToDqlite10(dir string) error {
+ var conf raft.Configuration // Last committed configuration
+ var snapshotIndex uint64
+
+ if !shared.PathExists(filepath.Join(dir, "db.bin")) {
+ return fmt.Errorf("No database dump found")
+ }
+
+ if !shared.PathExists(filepath.Join(dir, "logs.db")) {
+ return fmt.Errorf("No raft logs")
+ }
+
+ logs, snaps, err := openLegacyRaftStore(dir)
+ if err != nil {
+ return err
+ }
+ defer logs.Close()
+
+ metas, err := snaps.List()
+ if err != nil {
+ return errors.Wrapf(err, "List snapshots")
+ }
+
+ if len(metas) > 0 {
+ meta := metas[0]
+ conf = meta.Configuration
+ snapshotIndex = meta.Index
+ }
+
+ lastIndex, err := logs.LastIndex()
+ if err != nil {
+ return errors.Wrapf(err, "Find last log")
+ }
+
+ // Scan through the log for any configuration change entries.
+ for index := snapshotIndex + 1; index <= lastIndex; index++ {
+ var entry raft.Log
+ err := logs.GetLog(index, &entry)
+ if err != nil {
+ return errors.Wrapf(err, "Get log at %d", index)
+ }
+ if entry.Type == raft.LogConfiguration {
+ conf = decodeLegacyRaftConfiguration(entry.Data)
+ }
+ }
+
+ err = writeRaftMetadata(dir)
+ if err != nil {
+ return errors.Wrap(err, "Write raft metadata")
+ }
+
+ err = writeRaftSnapshotMetadata(dir, lastIndex, conf)
+ if err != nil {
+ return errors.Wrap(err, "Write raft snapshot metadata")
+ }
+
+ err = writeRaftSnapshot(dir, lastIndex)
+ if err != nil {
+ return errors.Wrap(err, "Write dqlite snapshot")
+ }
+
+ return nil
+}
+
+// Open the hashicorp/raft store in the given dir.
+func openLegacyRaftStore(dir string) (*raftboltdb.BoltStore, raft.SnapshotStore, error) {
+ options := raftboltdb.Options{
+ Path: filepath.Join(dir, "logs.db"),
+ BoltOptions: &bolt.Options{ReadOnly: true},
+ }
+ logs, err := raftboltdb.New(options)
+ if err != nil {
+ return nil, nil, errors.Wrap(err, "Open boltdb file")
+ }
+
+ snaps, err := raft.NewFileSnapshotStore(dir, 1, ioutil.Discard)
+ if err != nil {
+ return nil, nil, errors.Wrap(err, "Open snapshot store")
+ }
+
+ return logs, snaps, nil
+}
+
+// Decode reverses the encode operation on a byte slice input.
+//
+// This code is taken from the hashicorp/raft package.
+func decodeLegacyRaftConfiguration(buf []byte) raft.Configuration {
+ var configuration raft.Configuration
+ r := bytes.NewBuffer(buf)
+ hd := codec.MsgpackHandle{}
+ dec := codec.NewDecoder(r, &hd)
+ err := dec.Decode(&configuration)
+ if err != nil {
+ panic(fmt.Errorf("Failed to decode configuration: %v", err))
+ }
+ return configuration
+}
+
+// Write the metadata1 and metadata2 files of dqlite 1.0 on-disk format.
+func writeRaftMetadata(dir string) error {
+ buf := encodeRaftMetadata(1)
+ err := ioutil.WriteFile(filepath.Join(dir, "metadata1"), buf, 0600)
+ if err != nil {
+ return err
+ }
+ buf = encodeRaftMetadata(2)
+ err = ioutil.WriteFile(filepath.Join(dir, "metadata2"), buf, 0600)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+// Encode a single raft metadata file.
+func encodeRaftMetadata(version uint64) []byte {
+ // Format, version, term, vote.
+ buf := make([]byte, 8*4)
+
+ binary.LittleEndian.PutUint64(buf[0:], raftDiskFormat)
+ binary.LittleEndian.PutUint64(buf[8:], version)
+ binary.LittleEndian.PutUint64(buf[16:], 1)
+ binary.LittleEndian.PutUint64(buf[24:], 0)
+
+ return buf
+}
+
+// Write the raft snapshot metadata file.
+func writeRaftSnapshotMetadata(dir string, index uint64, conf raft.Configuration) error {
+ // Format, CRC, configuration index, configuration length.
+ header := make([]byte, 8*4)
+
+ // Encoded configuration.
+ data := encodeRaftConfiguration(conf)
+
+ binary.LittleEndian.PutUint64(header[0:], raftDiskFormat)
+ binary.LittleEndian.PutUint64(header[16:], 1)
+ binary.LittleEndian.PutUint64(header[24:], uint64(len(data)))
+
+ filename := fmt.Sprintf("snapshot-1-%d-1.meta", index)
+
+ buf := header
+ buf = append(buf, data...)
+
+ crc := crcSum(buf[16:], 0)
+ binary.LittleEndian.PutUint64(buf[8:], uint64(crc))
+ err := ioutil.WriteFile(filepath.Join(dir, filename), buf, 0600)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func encodeRaftConfiguration(conf raft.Configuration) []byte {
+ buf := make([]byte, 1+8)
+
+ // Format.
+ buf[0] = raftDiskFormat
+
+ // Number of servers.
+ binary.LittleEndian.PutUint64(buf[1:], uint64(len(conf.Servers)))
+
+ for _, server := range conf.Servers {
+ id, err := strconv.Atoi(string(server.ID))
+ if err != nil {
+ panic("non numberic server ID")
+ }
+ enc := make([]byte, 8+(len(server.Address)+1)+1)
+ binary.LittleEndian.PutUint64(enc, uint64(id))
+ for i := range server.Address {
+ enc[8+i] = server.Address[i]
+ }
+ enc[8+len(server.Address)] = 0 // Null terminated string.
+ enc[8+len(server.Address)+1] = 1 // Voting flag.
+ buf = append(buf, enc...)
+ }
+
+ return buf
+}
+
+// Write the dqlite snapshot file.
+func writeRaftSnapshot(dir string, index uint64) error {
+ filename := fmt.Sprintf("snapshot-1-%d-1", index)
+
+ db, err := ioutil.ReadFile(filepath.Join(dir, "db.bin"))
+ if err != nil {
+ return err
+ }
+
+ wal, err := ioutil.ReadFile(filepath.Join(dir, "db.bin-wal"))
+ if err != nil {
+ return err
+ }
+
+ buf := encodeRaftSnapshot(db, wal)
+
+ err = ioutil.WriteFile(filepath.Join(dir, filename), buf, 0600)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func encodeRaftSnapshot(db, wal []byte) []byte {
+ header := make([]byte, 8*2)
+
+ binary.LittleEndian.PutUint64(header[0:], 1) // Format version.
+ binary.LittleEndian.PutUint64(header[8:], 1) // N of databases
+
+ // Database filename, main size and WAL size.
+ data := make([]byte, 8+8+8)
+ copy(data, []byte("db.bin"))
+ binary.LittleEndian.PutUint64(data[8:], uint64(len(db)))
+ binary.LittleEndian.PutUint64(data[16:], uint64(len(wal)))
+
+ buf := header
+ buf = append(buf, data...)
+ buf = append(buf, db...)
+ buf = append(buf, wal...)
+
+ return buf
+}
+
+// Copied from the raft C library
+const raftDiskFormat = 1
+
+var crcTable = []uint{
+ 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, 0x130476dc, 0x17c56b6b,
+ 0x1a864db2, 0x1e475005, 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61,
+ 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd, 0x4c11db70, 0x48d0c6c7,
+ 0x4593e01e, 0x4152fda9, 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75,
+ 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, 0x791d4014, 0x7ddc5da3,
+ 0x709f7b7a, 0x745e66cd, 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039,
+ 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5, 0xbe2b5b58, 0xbaea46ef,
+ 0xb7a96036, 0xb3687d81, 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d,
+ 0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, 0xc7361b4c, 0xc3f706fb,
+ 0xceb42022, 0xca753d95, 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1,
+ 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, 0x34867077, 0x30476dc0,
+ 0x3d044b19, 0x39c556ae, 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072,
+ 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, 0x018aeb13, 0x054bf6a4,
+ 0x0808d07d, 0x0cc9cdca, 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde,
+ 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02, 0x5e9f46bf, 0x5a5e5b08,
+ 0x571d7dd1, 0x53dc6066, 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba,
+ 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, 0xbfa1b04b, 0xbb60adfc,
+ 0xb6238b25, 0xb2e29692, 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6,
+ 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a, 0xe0b41de7, 0xe4750050,
+ 0xe9362689, 0xedf73b3e, 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2,
+ 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, 0xd5b88683, 0xd1799b34,
+ 0xdc3abded, 0xd8fba05a, 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637,
+ 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb, 0x4f040d56, 0x4bc510e1,
+ 0x46863638, 0x42472b8f, 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53,
+ 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, 0x36194d42, 0x32d850f5,
+ 0x3f9b762c, 0x3b5a6b9b, 0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff,
+ 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623, 0xf12f560e, 0xf5ee4bb9,
+ 0xf8ad6d60, 0xfc6c70d7, 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b,
+ 0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, 0xc423cd6a, 0xc0e2d0dd,
+ 0xcda1f604, 0xc960ebb3, 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7,
+ 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, 0x9b3660c6, 0x9ff77d71,
+ 0x92b45ba8, 0x9675461f, 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3,
+ 0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, 0x4e8ee645, 0x4a4ffbf2,
+ 0x470cdd2b, 0x43cdc09c, 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8,
+ 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24, 0x119b4be9, 0x155a565e,
+ 0x18197087, 0x1cd86d30, 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec,
+ 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, 0x2497d08d, 0x2056cd3a,
+ 0x2d15ebe3, 0x29d4f654, 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0,
+ 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c, 0xe3a1cbc1, 0xe760d676,
+ 0xea23f0af, 0xeee2ed18, 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4,
+ 0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, 0x9abc8bd5, 0x9e7d9662,
+ 0x933eb0bb, 0x97ffad0c, 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668,
+ 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4,
+}
+
+func crcSum(buf []byte, init uint) uint {
+ cursor := buf
+ crc := init
+
+ for i := 0; i < len(buf); i++ {
+ crc = (crc << 8) ^ crcTable[((crc>>24)^uint(cursor[0]))&255]
+ cursor = cursor[1:]
+ }
+
+ return crc
+}
diff --git a/lxd/cluster/migrate_test.go b/lxd/cluster/migrate_test.go
new file mode 100644
index 0000000000..fe3ff4983e
--- /dev/null
+++ b/lxd/cluster/migrate_test.go
@@ -0,0 +1,77 @@
+package cluster_test
+
+import (
+ "database/sql/driver"
+ "io/ioutil"
+ "net"
+ "os"
+ "testing"
+
+ dqlite "github.com/CanonicalLtd/go-dqlite"
+ "github.com/lxc/lxd/lxd/cluster"
+ "github.com/lxc/lxd/shared"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "golang.org/x/net/context"
+)
+
+// Test migrating legacy db data to the dqlite 1.0 format.
+func TestMigrateToDqlite10(t *testing.T) {
+ dir, cleanup := newLegacyRaftDir(t)
+ defer cleanup()
+
+ err := cluster.MigrateToDqlite10(dir)
+ assert.NoError(t, err)
+
+ listener, err := net.Listen("tcp", "127.0.0.1:0")
+ address := listener.Addr().String()
+ require.NoError(t, err)
+ info := dqlite.ServerInfo{ID: uint64(1), Address: address}
+ server, err := dqlite.NewServer(info, dir)
+ require.NoError(t, err)
+ defer server.Close()
+
+ err = server.Bootstrap([]dqlite.ServerInfo{info})
+ assert.EqualError(t, err, dqlite.ErrServerCantBootstrap.Error())
+
+ err = server.Start(listener)
+ require.NoError(t, err)
+
+ store, err := dqlite.DefaultServerStore(":memory:")
+ require.NoError(t, err)
+
+ require.NoError(t, store.Set(context.Background(), []dqlite.ServerInfo{info}))
+
+ drv, err := dqlite.NewDriver(store)
+ require.NoError(t, err)
+
+ conn, err := drv.Open("db.bin")
+ require.NoError(t, err)
+ defer conn.Close()
+
+ queryer := conn.(driver.Queryer)
+ rows, err := queryer.Query("SELECT name FROM containers", nil)
+ require.NoError(t, err)
+
+ values := make([]driver.Value, 1)
+ require.NoError(t, rows.Next(values))
+
+ assert.Equal(t, values[0], "c1")
+}
+
+// Return a new temporary directory with legacy raft data.
+func newLegacyRaftDir(t *testing.T) (string, func()) {
+ t.Helper()
+
+ dir, err := ioutil.TempDir("", "lxd-cluster-test-")
+ assert.NoError(t, err)
+
+ err = shared.DirCopy("testdata/pre10", dir)
+ assert.NoError(t, err)
+
+ cleanup := func() {
+ os.RemoveAll(dir)
+ }
+
+ return dir, cleanup
+}
diff --git a/lxd/cluster/testdata/pre10/db.bin b/lxd/cluster/testdata/pre10/db.bin
new file mode 100644
index 0000000000000000000000000000000000000000..86bbea7e842930b044bdf727ae0ede0f0825834d
GIT binary patch
literal 4096
zcmWFz^vNtqRY=P(%1ta$FlG>7U}9o$P*7lCU|@t|AVoG{WY9BF;00+HAlr;ljiVtj
n8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*O6ovo*=#~a$
literal 0
HcmV?d00001
diff --git a/lxd/cluster/testdata/pre10/db.bin-wal b/lxd/cluster/testdata/pre10/db.bin-wal
new file mode 100644
index 0000000000000000000000000000000000000000..1c8eb8a3433e16423a1853171c6820184fa059ce
GIT binary patch
literal 1005312
zcmeF)349yHq4;sh at +CX=CWK&;5Kw>+JGN}CZiPZ{93><q4smQC1uE9k$`RO>k>rrT
zD>Mg`($|(AuZ2PnUenT+HuU7tmhyUh9xbIO&_YiN<pF&hEiIH%=)eCNtyY$7IdMV`
z+xaQ6Bkj)4&U|O~&}3#da82b7i}Q)&EtZ)U`Py@}<(~U%)?9z%)7!uOv$?T0vB+Au
z{IB16`HMfUeCpwi8`dT=x+9)U4QLsMM{HeIW;x5 at 5Mj&2&l<T+?$s*(EjK at if3xuw
zd at l92%+l(0RERa@%5!r4a^*GUZ_10xb9Lv~UzxGB>T&y*Dz&PQmfu+RQu#U7wzAul
zzn6+;(k$DNoK at AZY?*b}Kt}8B*Q3e7jFuSGQ|U-d-;#*x>4;bqPxOtuVpd02xV<~<
z=x$%WHtZNTn4_`AAzq1?V|8bDcvZN|F_cOSXsK<E%k*uI_MYzbt2;$Z>%yJgj?VSs
zr)TZjmh2|^D03-_6vR;+QpVIBzF>3&TeSXRJ-fn)aOL{0 at ak2aj&s7B9E~MnZ*p{n
zSBAU7ogLwgB{Jk_OvIWT>pLAQ!fV4K;~niAJK9%-jktR{S8wPEmyEVW=CrA%X~Qv<
z4f-;xB{3M&x27-e7qvB_4QG<s&r-W|0YubsR~@sXTd8VLRqL*e;}qi9O_hBV+rcQW
zaSLit8!!rFR8eQP4e9y%VV12UrpLA6{*1%5*W#0fQnK9kv%C_q7KcoqD5;K_l?~^s
z`%zLQifHD}&ibl3fk5?j%eEx+D+&riycNe)B&EkoF3gqBd8@<c7goZNtsIR*spO at 4
zG-I#~CS!W;ok+d$PE<JKP4*CIa<p%hN7P19Z#%lh at t55?-70fC5=Y_Zk%Scl8d;b-
zHMHh;Gj>&W$JxW!v9fFZy0JMi);Sz&#PMISRvup?v9%^*XN-I at cbi`#&Im=(m^qV?
znD2|DXw5Aiy`@A<%gwTHMr|XPi4Arvol(`$)@I#x%IJelB&hgIX)LH4?@%*Vk3G`P
z?OMIAy=znU1U0cU<G4rK8#^T)%wx(p`L)+oHuSVj;E*zGicWgQqWYbkHQAHhb*o1E
z_iEA0hKJH3oOue(EvX%^1aj*SX%S>kcIUNsb)4Pa)#nns=UqA6X=B$9)Xm?rL9
zR)}k9_v&@I%Za{qD3RJ$YE3?qk>p_iwtb#7lS7%rK;lY07D;E4DXni(on~t%c$&?X
zR#BzMEt|3H>guY7g$u3k7+ov!P-%NnEjW2(#mQsKf1)=hxpTr8|B*9ssXG!a72PZ@
z`_asBYHUFmo0hy38msi7&H8|z()!2jRos%SZ0}jyZC)x%SFL>g8y!5I5j8MAyzhAX
zwR9#joYqTKOmPB at rV^sMB?k-cxkk>nV-lUq)VMX>yrmk6B6oo{HY~cE%N0k_oj~?f
zzSo<8q9{goQxr>XaeADHYMMOu%7#@7CvavfxJjtlKEJZ6VeVY(j#=ip%(#Gv-&IAm
zRa927Y-~N9=%r$U*8$VoI9WGWRW>Y}yO%79BxMgq at o)9^Ys#w{7A>;wY&Wu^r!t9n
zA}VTBI$H3)qBvUxYYsef at x)-Co)U-bV5Zd3wwFU`yc^Mi6qkz8WGdLPxLpiXR5qNm
zXk4y>tO~S(PnGA|DjS-ct(&rNxo-q%{qkXbP<;O~z95(%%ZvC%t!&KR$%&v%+dHgA
zz0!<0qvDLArL(`wietzw8=;rq(d5hgk~Y!$wMBemlt+BQrsn;-ahMJd$;(?TBEFNG
zm$>|S$OvJg=Asgy&1HM5e<C~LTeSa8;@2sv_;u__;@9k1;+OoMpKmYt%`ew{@BYw}
zt~#;Is=RCwe|aH*00IagfB*srAb<b at 2q1vKbP}*z%PrRO`ip*9Un#O>lUXzS5xn&y
z-&=O_d!PGGjaX(?UbBe5ybwSD0R#|0009ILKmY**5I|u136$Hc_Ii=6>=*#EU0}(}
z?f0Ga?A_1OUtszbI5|Q90R#|0009ILKmY**5I`U+V7J=l+Kq7%;<Fs)pYJbl(jR~G
zg?0CSc8hEm$W2Kg-|<2K0R#|0009ILKmY**5I_Kd=_w$`66D(jZfjrPeU;LCj~qup
zyMRRCg#ZEwAb<b at 2q1s}0tg_0z_b>qlDRX-5j+}r%SV6wyNU<sFEFjkoXjDB00Iag
zfB*srAb<b at 2q0hzRLeJJyTF^ixnak~pAPBt7chDF7XbtiKmY**5I_I{1Q0*~foUyZ
zm$@_B1-^IYxzD|R*UgX8Utn67IhjKM0R#|0009ILKmY**5J12b$aW1d+XXIL+4IvC
z>2pq_zktcZzX%|J00IagfB*srAb<b at 2uy2%TA4euUEu4P&&>OB_3q!&Utn67IhjKM
z0R#|0009ILKmY**5J12bm?7Vo?E+sp_GkAld2-D%`U{vm{EGkr2q1s}0tg_000Iag
zfWWjC$aW1d+XWtc>ZMQrdfBt{=r1s>%bd(1fB*srAb<b at 2q1s}0tg^r3e?LtX1l<U
zW5*Bg+r0eG^cOIB_!j{L5I_I{1Q0*~0R#|00D);Opvc^r?E-hN2_5&yude$E{RO6V
znUgsL5I_I{1Q0*~0R#|0009I{ftm7+*)H(+&dw{ped0~q=r3UM at Gk-gAb<b at 2q1s}
z0tg_000PrmV3y3C*)Fgr{m_$_{44k-`U_0!GADBgAb<b at 2q1s}0tg_000Ib at 0>{WV
zX1hR*<>a&HpZ&vm^cOIB_!j{L5I_I{1Q0*~0R#|00D);OFk9x%Y!|qpYsI_5yOSI1
zl)qbMS$9~J3gv<NFV}6YzjDUs>&~&iGGlAi<MuCAYE>UCztI*dd&#=P+9rY??Jq&F
zQD0`YBnD&p*7W85iHsi6hBL|R=SVa;n9&l0dMX`>>01&}JuMf-6MYeN+*QZyNY1Ki
zShmc%YapZb_Uq#Y9CyX6j;?Tfci7S0zI<)iF>WwNV~s<+5;4c>&hGH4aF=5!l^D=c
z+Z>na+Z^pZ-RoC(ikQ}gJG&j7>%~vc+O;j&P4ZFZauq3vqd26DsXKhZ=m at rG{lj{8
zg%RP(^<ClBt2!O$gf}@FOT^yf=nAh4cZEAU!W&Cu$kCXHH96LII#z_&hDF9Z+BbHz
zuLv7)_jIn_&=W2hZHvrlQ%zIH%*uxI)%_^M5~VkDN4HYdpsLng8^<ZOe7^FPLD@$^
z9gGqhw`c~n0iy^;6>etRke;s*W@$QNdR!as&p2FrEihROB}-~QODGX*ame(Evf0^L
zUo|HXsJ?F5mV|yqfuVRS52TbHFS#&Bd*15s`GqxRWGhGGP%3$;9?cjz8cfFY+&i&j
z<DIA-#+z(?ZF014l+}NusAL`8;$X>coo<y09f_mx^GL!90*x%poeNs?yBWJGyW{LJ
z=UCabe%;ud80#F4HR50`SSt^Pk=R-ju`@<KnY+y|5r=P4G-l3ZB<B0#C|YxiM{g++
z({i)yn^D`yWnzOJOJ`Iyw6$4xoih435eX6{nopK_uB;pHC^5p1JxI>&TD`8lYg6`Y
zGO^RxxChA_JAWO_!@)RHwbxZP^t4UjOl8;<U1p3$^*cLjvgfJmR*m-W)uNXT52XtV
zAh)D;yb{Q*Kcq#FJx`t2-qmq-dsm~|=W7~uzKW)FEt5zNMq*k<cdQT>zV6lQa_4 at 1
z>rf)Kt<;))CL_tg{%!j_-zA4KiGjqGdMuL8BvV at 7WI983)K2i~l`E~HN|9SOW7pNy
zRSgRlTHi6cR^*}5_M%#F^2myl$Cm#@?<8{PgfadjXW~+~4_Yd^S=?wvGsCH|1z~Jj
z at +xMm(uX$d1A0p9AG23+|FE*XXKlB6l`CDf^7U_Y at N`Dh!1(aK<L%ednaFTjFI6$c
z2{f8Yi0YOcEVw}$Ip2;+bS_il)^zj!XC#W;wbj_L=+-M&97PxL?5TXO*YBbzMs`ya
zOKx#`oQP_gJod_lRSPF at W-GY&s at XojvZ`V3T<eZm=DEzcfQa8!MYUB_R<UesJ)P*K
zVuIHJ)7m&$H&<0QEStNREQus#4 at U8C_4aGZs~Q$9vhHj*vZALliFhI^YE(K}@V=rr
zTLo(lJah5HV4t26hwNac)X}z=LutHw$buA?iqT{$*s-|N3sh7#oU>?Lu7a!zw1Q8S
z=h`Y8nwzbgvT(WYU}^pGVSP}1;WECGnIFrG_(rX4%-+d~piSF5tVX at kj5wp>jG?8o
zzsrha$Sxb9mtUyl%lt+)(fYMTd_|N;e8HyXt+#QQ4iCx8TP&hw@|U>$dB_N1qUNF!
zpv`4_tA9pzBC7GPO?;8B&V89bR{m?Y3v9n=?N$Hyvi9D(s<5Trvc at uFdDVYat*cm7
z`TuN9Wj9-&RQl>)n(-eq=2+I$K3dydTUVPMaFC0s<UczbPONHZYO-FN$l{8xoPFYZ
zvDTl^(t5i1(}_jcCl{|QaoL$D_jvU-yMyt+%^u*n*({lnQ{(t&U^4|pCr=~a<D5E6
zoW at 6XR20oV&%xW>bE_H>3$5GxvdtGGSNO<Cj1g at x7Rg;FjQ8T}k09GR5_)c~)K+({
z2wza}HDn^O6n}Bae at 7ELF!Gt(bMAxuinzV*_^O5tBGJBmC3;LkkDq8lb6f5+aWXx-
z<~LN$IsNqNw{IBrg;f~F6aAv97k_0fT%1eK=r65>LGmOc9wQjWyMy^{)!cRV3vB5x
zqGphy?~o(4p%|_BJE5!*S(Da$cu~?t5t$)mH0Jy9BFU{U5rzDopZ#Jgf at I$pNAX6!
z&l;&+-?c(Cpe^4ts=6DQ(}srnw?&evn4a3{om17|@mQ}@jLUp7-#}<)E-#VJ>l{~f
z8Y~fD<N{<~cqV!So{vnR<*!=Bm!1*TaRL-%Z%l~M4M*m=uyDUEX2FUyG#OR6=$5iD
zY|#lVzaqW22Ed}YMlw?rlZjtCo|>lcag`05JmcP*j4j2YFIi^n4K?pHF0z}Ct!fB~
zQ<iP?Da$;u7M)M$jCYzU2wi-XIT~}Tny%e2yJ}7-RDDDHsI!sT9$kDgDp)$sxu_t-
z1WrXpXroN!eWdYD+#VnNGnKq|augnZxs;Y}dNWs!B+A at g6Q}5?KGF)EZ>=@k&GM;V
zyf{}|qse`4lszXKnHHPLbH?cQyj+Tm)5+TK%5KN{o^EkY8P&2k>OT^S%YD?!g0rd-
zyEwoK8v;!P^OQSkSH*D`pE1on73`uY7E$H*_ne#EyR5hSYC*JEtm_W)|G9`LZ`hCj
za|xWhx34O>+th4UB5L8Mleb$PRSm0}tlLkLM}G0At+|l7JK at 5+C2=|^Ug;=0B^0k4
zwV^D<;sE*MR-SK9KdG`|ebabnw&D%*naL&bq~-;C`-YL*I47`R!!eZ&2f5gGU35bB
zd)hl at 7krg4c6&n6H8c0s;SGFcDEO{g>MO0h(2w`6A}tyY$6OeVbHUJXZ+{{>X65LX
zr3o~6m2Rst_A-7m=<rZ~Qi~P0Lyh`g8P&vh+_Cotqm#I|HZA!dq7ma8>^`%aJ8_JE
zJ7V5Jj09|P6ckRW8;Fshqc8f#IcmJC{^j$<f5eK*v at -fKJrbcvXTdk0`9+82$n8mq
zLvmzg(c!rLxOr6#=Zk%|?`5BbU%`s at KW_s2AGLAuNxm>GM%^)QB*a(wIr8hg*)A~S
z{hhbG?VJ;S%{YRRjT|gQ009ILKmY**5I_I{1Q0-Ax(OU7Gi0_4v|Df7 at RPN#yoGrO
zr&}?T9|RCU009ILKmY**5I_I{1WF1t$fahxz}vp_;)YdkIpqZU3zW2CAp!^>fB*sr
zAb<b at 2q1s}0 at F?4c$p!yU10DQ>utUded{l at 3rx3SCO-%ufB*srAb<b at 2q1s}0tl28
zm at Ai>?E*{R`TyF_{L>?6(JoNZiiHRufB*srAb<b at 2q1s}0tieufq61RX1l;!$`-13
z&H3I_v<pnPVkSQbAb<b at 2q1s}0tg_000Ib<6qqlUn(YD~TiJg3V}H7L1?>VQtyqWv
z0tg_000IagfB*srAb`Mh6F5O;$ZQw5?4~PjU-rkH&+<Hi=~m3-2LS{SKmY**5I_I{
z1Q0*~fsz6X<WjR;;AfUQ-hO}8Cl=6OprjQG5kLR|1Q0*~0R#|0009ILm~H|m$_$z9
z0<Akw`etqS?>|j{f$3Jv<Ocx+5I_I{1Q0*~0R#|00D+PMC&{H|yMXK5TmET3t+t;2
z0wt|jhyVfzAb<b at 2q1s}0tg_0z;qLE$PAh70x!RB#k$|$^N!EaE->ASnfxGt00Iag
zfB*srAb<b at 2p~{WV4+-UwhKJ5=Z1~1y{%?8;|NMxu at C_S5I_I{1Q0*~0R#|00D<Wy
zaI(yh*)H(eZ~frA&;Q30*V12Lx)n3|K>z^+5I_I{1Q0*~0R#|0prpVla;e!a at XFFJ
zMDJbylY3=<fn%(%Tdc3w at 2<bB-ag~EwePOEr{<m2uT|eu)m1g8;)@m6Ra|KMrEPcF
z+pG&MPgK0Wc0<Y7kKn?xU6%9CXs}umgE4(;`ttrnMvrL2nPm2J#A302bVKuaYll+F
zc%olVN21BWc%m<&j#}W`(d?;eXl}Ods>^7-{rafYqds~&y29<<VMllS^0i^dsFjY!
z8i#l#Vvg0F-QiW?F2}iDtJk%6ZE~Cw-sEWS>0ZCOQ*6I3+}Z8uTrYll)~;>IZjy^A
zQm|=pfXno4j_&XUqqp0l^$+XW6-GQgovSzWgdL5!__kXeRSm0}tlLk at N+kxgJ^`Wi
zC$zNqR|L9Lk7mS9#ow*PD;>FfuI^kBzQ9quZq$a3^_`;vWcQzox at pHkwW?vk0_(0h
zMZ6}Ry9o4TbMeUsZaJt87$l>NV|qH8N(^NZ$-x4#5rll970VZp=Cah{kh^Fy#8<BG
z3a?((nJr<-Zf4uj6<!(c3U_vdH_B=v7pEPKiCB~5S`l6w7L}- at ePc)aif~Pn$5q*|
zYT<Y_I60)JG at 0ylL>+mr?zpC{s$t<m>$R)1up?_m-noi#M^?N6-r-?UNlR44!DLLA
z^|!>T%(fwYjIKaf+1|6Z+u_O|0y#S~XmOR6`z>Xj+iz1LxLh`#_CYMRrq;@a at lQ+H
zL`2-#f7+Uy`wpsRgSPI_TB;fv8m!kY%$93*VfL4^s5G;S_Ii{UIi2X?kS~wLi;WGn
zSS+Qd)1?YD9o?)CXl6N$I?lAAL`2`3(FfDA%8nCgv#zBwy}FihtPrQu?$ztU`TckF
zbcy4pJ0cfu>~3Fou92J}eK3|7>>Cqbe!E;fGLP4SW7J%f%TZHJ)9KBX4eOi6JKzTO
z%oWMhW$8%PIHDGRZrRbdq^hB*$$G8bD4?Ry#bKL^i>xT*UdwK;SBu;}#=ly~bdSH*
z<QR!YoY9OeN*-WFszrRo$5<{i<DAq=9A at S|CVHgSG at a2@*)aZ<Jx^_3*+(r{va_x+
zdu6|NLvcYF;YWpTDqU1Y*uz&=hTDE#?n{(pi$laQ$%?aWab-itf_)x!`Qxo|`})(e
z_YONQFDgK@<QA7KvNupx`*r7#&S;r^U)J&mo4jSHY1*-NQB}j@#nxTtXLFRhu}UY0
zQ&GL>!)Zl%D_V8n8O%mA{)r*2r?%*+QYU<I2a`!gll^1PzD9M8>Zwd3o`_1264(Fi
z4QHtpMK at 1o;)>2PM#jfE#|X}*Q>z-<+N`^*g`6U_;+83s(9=1$kp+uNv5u at hWV9u3
zf%fZIf5TRnoKo2km}|YJ at T-8xdG4l5)Qa3!fKzisMYsON0Y`56i*EG;Cs#I{vuM2I
zJ69IDYii-AlXq-dSk<s-k#*N_VKHVy7KU3`s_?R*4q3V7u36*Md-LGVCQ=CQ^*U+>
znt1&_DA!Q&wf$82)!l3txafgTT{t6|ZlS-x5p2+4*9ahh00IagfB*srAb<b at 2pm3v
zMRHeWyTIGN`pO at h9{h8Z`2`MN0k9_o5I_I{1Q0*~0R#|0009J!pulNz-)6gj$MNB(
z|5W$ld9({0!4hHD2q1s}0tg_000IagfB*sr96o`?a#v=%K;xC=zS&*ZewB8C!&d<8
z2>}EUKmY**5I_I{1Q0*~fg>o;DEDo)3oQKUd+Q$j!1>FVU*HIq2)jlA0R#|0009IL
zKmY**5J2GY2{g%Fne75EoO$~#-~UBsIsF9=UjeWu1Q0*~0R#|0009ILKmY**j-bF2
zxo@*w;GPd`ntj)0fBP2w1&&~euxkVmKmY**5I_I{1Q0*~0R#@8K(pMH*)H&nU;NK)
z$;iA_^cOgM1;CyVKmY**5I_I{1Q0*~0R#{@f&wja-)6f&fB9Rs)nE9bx%3w}f+fPP
z5kLR|1Q0*~0R#|0009ILID7(5xhu0>;Ey+~{@y(ozFtm$fx}k-><Iw`5I_I{1Q0*~
z0R#|00D&VY&?@(BwhMe at Y59Mz`Qm~b=r3>tON3n`fB*srAb<b at 2q1s}0tg^*_ypSI
zuFQ6UTWcPC`VYrdRMReS_zHkMA%Fk^2q1s}0tg_000Iaga0CTha^GgVz|S9lTEA=0
zy$a(9j$nzfYXlHL009ILKmY**5I_I{1P-5oDtBeJ3%tDi%>!$e-t;*A1rA>UuqOl%
zKmY**5I_I{1Q0*~0R)bqfLrd{Y!~>ybKcr~?!U^{(qG^RmI%8>009ILKmY**5I_I{
z1Q0;r at CkV2uFQ4;&;Q)<(Au at HKS+Op!&d<82>}EUKmY**5I_I{1Q0*~fg>p3mHRf^
z1=Jrr{@z)a)NH4}z!5AFc8vf62q1s}0tg_000IagfWYAs at X1}7?E;JDEV=o-J*Up6
zzrf)u0QQ6c0tg_000IagfB*srAb`LT6!6P^o9zO-uUkF$v+mcH$^HVh<-0AFv&#Qa
zzPtW|b?fS8)PA~lQ}th}-(I=g_Db0wEFZDF+4AP<6RS_$-?Udevu?4~k{FEXTho{K
zCo+0O8_p!NpCjo^GNtwDk)dR=KOIqvKDJvd6`z+o+tsnWYL3rWeZ6x_Lcd}tmAq7s
zX3~-2v>uD}Zi}S!c&TL_UE%ibu;aYd;qx7();k&pwE?}Q#vxu?wEkhe$<e-1eA*aZ
z8}8`NZ_+vvYABT$%)}cPx2dhJwrFxNqa_CQRJv_ZdRcA<XDmuDHewuET at -<l^z5Rh
z?4DP4tzYLD3G7(2esyPl-;VY9_mM>G3^Nz4iI}6ObM^YpiKjL(Ae3cFbICJ4x5g8L
zeR?XJuY%+lD-TstL at wfq{%m5#vY9I<fMy1rXgYVyYOk8(@l@|zS5%9HeLT at uREb6w
zm8?S}s|%~pW%{<Ub!o4aDOZh+=vwpJo24Adt;p8RY+Z3=E6b?$#+vb}O<L5Ie7!Ma
z%*7z9)LE4c8=A*EI7jU%qK;ayY)8wos)puf>n<gu_4ezdR*(94W)8EvefipOzKZA4
z<Y>(1Lsab5o!#M8;V#FyU8~o%cWrW<6W-)#@97qY#TnjxBsN>EauH?|Te!ftquFC5
zw$4m!;cAiC!jIn3i7i|?k;IDa*M&Q~9i8jNPtV%5M%~EmMabv#SsXyr0!MfFg3;S$
z>p*se5sx_fH}r%ZjU(~3IK-w+#@dzZyTYqib!O=bf;2h0!YjjF;m(fmM(Ll;Qez_4
zBu_Id#92?AEj!vbcC at bu*EF at cD;qi%?4 at QHAhJqi-+6W|f0J>NZq6NEMu6gz^q6I%
z4zMxnvssj<dn0Ef=i-T;v5nLfolf51Nn9M3xf6dxWIlT-8ht)DQL=S<Ux#Kk&#k$r
zM9mo4sCZA=aM_g(Szskr%1C6W+_l1pxO9Z2;*qC{QnBWj6(!EhySQkMN=ufc)DA|x
ziuP2H_r&Q{4W}=#Zm-JT9hi*fc4GhH?^bj1>AA|Yx^qSNf_#M at 6*zYZ&06FulZd5h
zS8S<KUz0`kHKuP#MD=v;dXU{LQn0LKeJxm at -^J+qYJ at SCX0{FG?mO}){le<J_lt&6
zsv`%2ynS(u<UW^{q6%*u17<c_M|N*)8%=8BavasoQ$^YgX<je#sa(7`cZS*LCfWQL
zxfL79%3xl*ib#!GvNpW3+p)f<Tb%XHgQX~xd5tjchDxo<*L)-UqI~5=uPCIb-NxRE
zc2QV8a+k{Dt#Z+b7>l>gR`T4|#yOy1p9M8~U8u4_JAI$$QL_q(V@=+T<WH-kiel*w
zEhvtV)2+KU6<@@Pw;!GN9AGF{d-GS|qO6+A6SxwKc=mE}%a_5yy~gFE-RGq)AF|{X
z&m?t>cj;ResBGBioN$qiDXZX)mHw&*r_;J?S;;~gT{?li7Rw>L*^noZQg<8j1Tm)U
z95Fqv4fkgpF5|E+bqgXRJb-(UaZ6%HpD%mMzH3|2IXZWc7~lJhIy|zd<QaNo^|&YK
z{7LS>PsRCyHFj5YOJA7G*6c2f^GWe((b%x~Oq1Pm-1Ag!S8wz*CB9_`<X3I8UEp`l
z=U%z{lYj1I9D!8fg#ZEwAb<b at 2q1s}0tg_0z_b?#${d>Q0=GSV%QZi|{F at fW5ls7X
zCxZwefB*srAb<b at 2q1s}0tiTfko;w~3&g(tk>5V}lNWBMzkpQXg#ZEwAb<b at 2q1s}
z0tg_0z_b at wDsyPI3wS=XH2j{o{^Vx*3rzcRCxZwefB*srAb<b at 2q1s}0tiTf)8#L-
zU7+%K&lx9vX2V};7mzBv5I_I{1Q0*~0R#|0009ILnDzp1k~uWn1vY)-l4s`)y`z$G
z1k=9U$shs<Ab<b at 2q1s}0tg_000L6r4Ef7!7kKN8EjvFrx9VZ~3rH1S2q1s}0tg_0
z00IagfB*srOnZSdWe&}Df%nY3PCe<y%4g{>Fzw5o3?hI40tg_000IagfB*srARq;n
z$zNu>z?b@?P43Sr at 1nneRN;jH0tg_000IagfB*srAb`NM7dT7i&}<h-zjM)tPQCSi
zucyDjv at druhyVfzAb<b at 2q1s}0tg_0fD~w#zsz=llYai(oiiT4?=AEfkSe?oKmY**
z5I_I{1Q0*~0R#}3_5#ag4$XFfkG5aZv-$J)?5eXbu*|glpGE0bPN{pT?yECytGUpA
ziG5Mk!s>@B)>Zzh>_Zjxmj5eTGxc?LTzpPdLnvh3)sxYB`}K4tnbP|7$d+XP at PM9<
zM3aN at L|^Hpt2?^F?cHHVcl+|SVMpmOj>Z~?cqL+v)t%kpRpBnjxm~N*wRdfDoD<&U
zXz%G>zq(UIvo74(?dV)DetOofZOLv at 9ABg$hT?FS>DwIL;R{A at zD4UF*0U>&=z2O=
zZ|Dg-8plN2;t(-38G%-=?+UM8)tOD(*pN+*uJFomSGcnyym54P9gT at tlVg3SV?}sv
zSeSIQZ|rDa5w2-EXKiIeBxJQD24nix^yU4Dj2_X3Gs*1dNO5XR?uc4?`I=ps)m076
z&DM9ERI0E>ZM3SWcupR*_T&REnL%y9kR7MQhLXwt at yly48PmsKo7py`=c`I_LC5rT
zG?f_2B$9&#<(eMUhSHmpnK2uQl33Z^v$oseGRk%+mAq7sX2#$3=o*sCMvEht;ugn9
z6k?%3XmQBo?yc_R;wV{j<ht}Y^`=C1%%!3zD!FVwq8LekQ7pN|<3&`{bjF&>h7HXV
zt;3@$@7X(RR#i24Jl1PZFRHcK{714`FzaT)qHs~|EfL_r>u+IX<5!2A_n3NB`gD>j
z=zL|Fq?MwkDSUQi!zRzZ%Cev!MovZr3s>%Rg{vBTKI?V15 at lL^(J(eMmUZl<RE-da
zxmb<TJiu$n=tAG;r6jjO(a~2}8^=2Y55no7rfJp6%7zPl6DxV6EJhc-v2=yF>8X}C
zKA{zr4eJ(9EW-RrAzK^Tw{PvJYFND3y0dL;^(g{5Yy1jx;5B8u3s@|b*rKOO-A#;h
z?a64Fu~)0n7asGvBKO)<)3mE&dDR at BuljoDmV|zVdG3u2r}bE*cUvT-$4f2CF`TzL
ze7<=ZN*8Y9Xv|)avUg6|Tc0LJ`$q9;qqrUE=+1A_IudFql^D##8yB~!t*$mvvKcKg
zsHf6xi_*(-2{>a>da+USMphR^kiDR$vx}OtdtTYKew||^uw#w5yUXue-W}y)7H8cv
z%v`pLJ3B$NzH{QKO$>-zRGHFT@{G@|@x)-Cp33H{AUVd$LzNVfi+DoZHkV{GS55%U
z3_8(t?wHkHHOJ$r-np)*78yrqQ6(B#RI(0 at tS+oV;)H0Fd|_SMYh^O(Lhk!QYkqsP
zl+6{{__B4yk*zGF)*EZat2Sv-SMv47j4>C3tWsy~?KWl9o+9d~1<Q7{EX&?p?^3eo
zq*1F!eLS=1yfkX%fuEUj5siOWGQlfV(HEJKNQ*8!W}^3c3u>OcM&d9oay3nDXI3_J
zEEw<XZ$>`Ce8%>sGpZUEEU at laROD-hd{ar$1E(DKBwzAtgqhFWNm^XpI^-qJY!~?D
zZPu$Q|6O}0;|Qb*F9Z-k009ILKmY**5I_I{1g5=!Z2r!-3v at O$zw?^6FIm7if at xpw
zWDo%a5I_I{1Q0*~0R#|000AiwmcPvY0>KkzsO@)KtLZNwRd^wQ00IagfB*srAb<b@
z2p}-+1y;%&n(YFcXU<#rtDArIWBLnB`*J6P2q1s}0tg_000IagfB*srNP$)Im)S0G
z`}MP`DsJ(-mHq-!g%<({Ab<b at 2q1s}0tg_000Prq;B1*gvt8iv*PA~S{f7Do{RO6d
zxsyQz5I_I{1Q0*~0R#|0009J~z-sx+Y!~?M?#NsJ^E01WLVp3N!V3We5I_I{1Q0*~
z0R#|00D);Qutwx?joB`cDh~{N;q5nWrN6+mFLyGC00IagfB*srAb<b at 2q1uf6eyR!
z%yxm9WvAct*~ponmHh?EmHRBp%gQs#L*flD1Q0*~0R#|0009ILKmY**5SXL_wdJ-2
z_MucVp6J)pvu(C!tN3uK9?hg9(d1w}(N|VmW`xKtv|DX+?dj-deLxc#a>`7Z?E)9v
zVR`8Frth6A+Xbx3!xr(E7Xk<%fB*srAb<b at 2q1s}0tieYfhAVEWlr6EYni8dFgd8Z
zTC;!ZE!VnTYRKvGIaQCt<zMRdFIBxFTdgu{X1l=sKV0{pA3SgCOL826Rr$L`{N;rJ
z0tg_000IagfB*srAb<b@(@dbsHpea-4MeuuWY)}ff$LwbzT)CvpYx#6F7UD#M=;GY
zPIeGL009ILKmY**5I_I{1Q0md1!`>b?YUkAg2F91%yxmwf1Z8a*Dq<Y8RG~ZvnY=p
z?O5a=0R#|0009ILKmY**5I_KdDJyW2wZbyrV~^=^ZMZ+P!uZFr_JS3T+-w9QXC9e5
zvt8ifw|?yEGuA(Op3z_6C5!UXl+8N{M*sl?5I_I{1Q0*~0R#|00D%$$wdMBtW>bM!
zeYy-{whO%P<v-Ot(EWV1(Jt`1MR~nM1S~=T0R#|0009ILKmY**5I_KdDKB8N+3g||
zZ;}}?+Xarl?e#7H_`ql4#yEoCTa at 3+wY(5O009ILKmY**5I_I{1Q0-ADhixvEf)g`
zN{=Iu&ny&qIz#5mY!`Su_|7{%xw+?G%sV(0OPe$!fB*srAb<b at 2q1s}0tg^50RcJh
zV6Le^gg#SlVYUm*xb^w%k5&HfM%iDWO!<LDc}o1^g#ZEwAb<b at 2q1s}0tg_000NU-
zU|v~;#ojh2u4|d$l%8(QO)Qwsq=xn5#q@*r=8<)Y0j*EZt`V79CVtkK?E-zDX?=1@
z`ioxKU!Y9+xkY(S{NjZG0tg_000IagfB*srAb<b at Q%B&qa(kO~OanpL>~edvJ-^s!
z8xWZ)ml-qL1<q`3Z2GTPX1`DN7pPGF%c8ujJfl3M+$Y}hLI42-5I_I{1Q0*~0R#|0
z0D;LNP*-7FXipC5DJ_#o4yJ3%Z42yX2Z7l(TeDSskkbdGBhlnwJkeKHTV{mFF0@;1
zbM5KqW_>^t*>TA%ne75!{oR$`PrUzMUo++%e9)piI5{#*DiA;b0R#|0009ILKmY**
z5I|re0xi}`OOIl;c%spe&*SfPx?R0Kr#G&-oxxt!?+j`&S3uQzy}fRqx>&@hil6*A
zf?IyF<l6Mg?>8Is4nAs89-YWO34;Iv2q1s}0tg_000IagfB*uMNno+9$3Bsk0g>jS
zaRghxSJyT8`_2ku9Kl^9;|T7WOt~X12q1s}0tg_000IagfB*sr9L)mtVi<vSWEer&
z!5c at g`}mvmXTKf2+ISwpZZVEP*?lxilXnCVKmY**5I_I{1Q0*~0R)b;Kuei1G9c)7
zyIfvh(AgW+bf?!J^*KYm!Jt#`je0{`Fy!^BntIwB8%OZDXS>f$c3$K#o=0$}7)PMo
zd8G5t{t-X`0R#|0009ILKmY**5I`U+&}i$iPNaFD>_Ep6yteXf at A!@Lw)+ at IK#fBH
z0R#|0009ILKmY**5I|tE3XB>@U^{r at 2(HwAJnQj`o_w=x7qBT?vf~KEKfDk?009IL
zKmY**5I_I{1Q0;rpb50tjFADZkVp5s{eGu6=#4qOs^9DE4eCB;uj&c*YQ9)3=#Hu_
zhjSc3e5h8R^F-f^avXt8xz3_o$4G*MR_sKC00IagfB*srAb<b at 2q1t!NdY-LU?Ob<
zwnILSAav?!-|??**~vJ9k_E*=1Q0*~0R#|0009ILKmY**CV{}HaRlWDZydot1~)DF
z)?(#7vR$BDIXgFwK)m6F00IagfB*srAb<b at 2q1s}0^<s_l$#?1;=zzF6nFaEUXRnO
z1w2m8<%>H#e$^Ln#Z~dtfS9^u@{S|;zpsAb8`al5ey$uxP_A4o#}Qo2NP=;TofQZm
zfB*srAb<b at 2q1s}0tk#RU<?n~cSAw><QYfM)HB%l%C2LsW*ouz#mX845I_I{1Q0*~
z0R#|0009J!tiY&o1QiEw9Kr3GYhF0%_O2~PyTH8sI09wfk<BrCM*sl?5I_I{1Q0*~
z0R#|0;P48xR2U-zydKT(RpT+IFA&$9UXRb`45_Ni8S?tQL4V9I9vl!*o2Sb-f at j}+
z;$I&6=A!%MID!hrYmOuEDt8`UefEX`0tg_000IagfB*srAb`MR6_CCDC(>F_F$KpF
zv at iYCYY#*&`!eGQCTr~@T?in600IagfB*srAb<b at 2#gv>P<imi5!iOU&vpNMU8l%)
zfy(;l3dRwL54;dS009ILKmY**5I_I{1Q3|Q0xgxfX9oCVzIfd2ar%9#&*_b-F=xmZ
z3^_Hg?hfifUvH?_r#2p);|Odoem?!VXHK-raRil0twpKLJ&}Ns1XH-?lXL_SKmY**
z5I_I{1Q0*~fe8p?d;L$K$)IwYjU#YLn>|-s?zz8a&Gq-!uYA*2ltt}z%7c~)*<WC{
zvPC&tnOFZD{RJjahX{fI0tg_000IagfB*srATWsqj*k8Uq5xWZ6syG(jfQ+4f3MT+
z>J_~r;+oqT>{b2FpcZolRIS(B>-MROMQBz0tTEdKHa at uSrkmdKzaOYk9u@Nr=KBkX
z-U7@#IEkw^Nkaeu1Q0*~0R#|0009ILc;f=os=t8XXq-sPfCyLAU!eJ~|8 at Pb|Frz7
zPT6fKm;D9uy#?qm at W!hcF(7~d0tg_000IagfB*sr97%zx+FwAJT_k$|1l?|z%j*j|
zd!w4}^!lUXiHW_zpi}RSdP7<;<n^kWdYTCR2Koy;->zn6zVM^1HKKb!IsFBWWOZfV
z2q1s}0tg_000IagfB*uAS70ji7Z4&FCel1mCITK%e}Tu|yD9RU|J8q4r);s<WPgF&
z+ycxmaCj>tdqV&L1Q0*~0R#|0009ILIQRmSr at w&6+YPb at fGgzD{cgYC=?!{gPOs|s
zI(vh<&)KVbg1wqA77MzgYKsVc*!l~6!9BC$H^*%Fc8zkK#b)#uxLEOuKg=(1 at GB%S
zB7gt_2q1s}0tg_000Ib1CV?Z^UqCQiIgvI3n+SMl`wLujwcS&`;)9RYDQ8>CWPgF|
zlL%yQ0p=H&OqG?iAb<b at 2q1s}0tg_000Ic?zrdmFFCbELx6uP29t`<Hai`Di^*Fs+
zz~j_hzPQulSA79jTotnb#MC7s^d#*s at X|~F`I}vz%zUdxx!6)>^cRr51?VrZ|J98x
z5I_I{1Q0*~0R#|0009K1mcYU9FCg;&30G*}4F%;Q;3VlUu<6~mp8B`%%=(k;FJM*X
z(O+O{)nQVL00IagfB*srAb<b at 2q3T*0r6EgJ5#^QAM?fIZjaONQ+-ZvRE;@9zF^3y
zd3ASC5BhpTy*{<ErJ~XzdjNPnn%}F&V at _Wnt~tFPpLhy at s=AyZuiqQ=$NXX<fPmU8
zb~`Ql3!L at bw;o+|zB?-W3s at DeCEHhk{sMccRjfh)0R#|0009ILKmY**5SS(c;+t^O
zM4AjLWygYvv=&qdn5ouZ;Hr;2c-vP_`rIXTvQTGQ=r1r$DmU3c009ILKmY**5I_I{
z1P~D4ardjgfDn7?=<6 at A`PQ#pF!#6Heq19fjBFP$W*1<70S+Jp5I_I{1Q0*~0R#|0
z0D<W)Fya0J0^*0$sK0=m-N3TvYRf(M*Q~kzCoSK7%y*0RYS~|)Qn|yTysSK<Jfz&G
z+$BEnLI42-5I_I{1Q0*~0R#|00D&VVV6U{zwMUbK at kC#Jg>9iVIi#nwOd>g$uB<J$
zEwB%zlJP{po}O*9HCx4pOZ8|b9Wgd2t1UA^WEa}4BBpe7vp%4S%v8w?ne74}vOQXN
z;o6ICk?jIj<#!hGmlpyEAb<b at 2q1s}0tg_000Ib1Ndbqo+|r>W2DCms-Kq^|l9Az|
zn3mBaiNTDX+M at NlM7FBM&l<B`VCRzWKl`zdf2&N6Bd{uuTf|>p2q1s}0tg_000Iag
zfB*srATU)0PPBE{_tG#RvQ;#W;QZU1AO7XLtmn!80 at cd)qHzSu&0+yB1Q0*~0R#|0
z009ILKmY**5IA at O$5q=r_H-th691<k8A>Mm(~-esOi$0OvMnxNeyGL~oFy}3_80iX
zYu~)^;}2}V$QVcPf<<}Z;ANGV5I_I{1Q0*~0R#|0009ILK;TFU%(PZo=FhIP$Mm>1
z+ at FahQX)0&GDT*)z-{jdwft_VZ;>&M;APQY;7F#IeItMX0tg_000IagfB*srAb`L@
z7pSq#H=71T++i8N*)Gt0Oy=GHvvuJjqrbpEEXqF)`re5U0R#|0009ILKmY**5I_I{
z1dg~sxtNpCZWqZ|DSq<v4%*)L^RkPzyG}Lw3%q7gUOVCmB>@N^fB*srAb<b at 2q1s}
z0tg^*@CC}t?b&vLqIn1Z_UorJRn`k5vcG^`**s?6L9vJz0tg_000IagfB*srAb<b@
z2<)dogWcvUo<=YdO%BEreMf%Y!48=fa~#2=e|+<AescWT+l_Gq4_lOn_cNmeM*sl?
z5I_I{1Q0*~0R#|00D&Vautdx*Xt&Q at olXv?qIz3qV5qHsYpgBz&;w`op$BcbsRu<u
zSBRhdID*%nTyg7LR{1_|j3ao-qP%oOv%=01KmY**5I_I{1Q0*~0R#|0;2;asR)|&u
zyU}VOqAnUo;92gt at -vTqeunHXP^<V$j3ZDsip9JTKmY**5I_I{1Q0*~0R#{@iUsD^
z+D^9)rIPVPzn+f7^eu^~_}3U1P&T{9=CqGqIf=&+1Y|*);|P|0c<E#R`{nRVV;lkf
z1&-oMPSz1X009ILKmY**5I_I{1g5D#jkUsJpJPoWlbJG+#h}cl*)H%%<sV+V?ulRg
z+!#mjibZ*4nj(-b1Q0*~0R#|0009ILKmY**j$VQ4a{C;+Y#0y(A<1C23w+~_=l^rl
z<Da`+_7^Bqo)Gg3h+n)AKmY**5I_I{1Q0*~0R#|0V2TRNER($h=F8azb8`(E{RNaE
zEwi~zWNNAS$<I6J`tvgvJ?6fkP4*WkQ=S+71;j622q1s}0tg_000IagfB*srATad>
z?4p-|RW=ZmW%~=rh5?bOqIn13dCx_?7asflKIR>q`o&ES5I_I{1Q0*~0R#|0009IJ
zmB1vMcd)9q+_oS;)!}TLtyxZrc&Q%E<R?R{Ei*!77uu~h at jQa)W_>^tLN2LfjwATf
z%m;=Vm#>Q%^A0{}Q68*+&ir$zG>I1h1Q0*~0R#|0009ILKmdWmA)uUXY3)(07MDNf
zi^ts_r{AagoZhG!bB27ukW=&O?w}s@^@e(VYNJTeQ!=#IqxrpRJm&NT;+oUz@%fw~
zRdqQ-UcWc!kNJbWT0m_Uq3_Pd9f}7-zEIrhb9+5buNLq)HJ2~$^!Qa at z!g`6A#Y4w
zB0}F_g!brux8Lvd2E8$-SM__Hy+PgQ>{UG?T3;*{bVt<|5&9w-I_P%0TwY($*&Ee#
zr`I19Dd`OcoqBK78`4C~UR6_1YpJTV$k0)*D;A20)uBK at q}$&slkC==el4iRz5ci-
zrux;zB5qau<i`=b%Qy6+-&~g2ZOl9Ph(&os at n-+rsoW{T at j?Ir1Q0*~0R#|0009IL
zKw!EGC>L9rCemb3DFA*rk=B9=5%3eP2{aVQp8Qw3Li=tbu!(>hCel36XzQ^~q+y^+
zL|8PA;EtQ-^}TZMJMLy2fso;a00IagfB*srAb<b at 2q1t!Nr6c=j-dMBjU)I%y62d0
zTzcbmvR$A|*)7HqD7zU)P_h!R5CH at bKmY**5I_I{1Q0-AiU~~BaRe>Z#>fCoju!BH
zJWfw6*z5FqV<Bg+9`!r*Xw>cR4SC~1U;K at ZBlvtv`mKL^;pb7~c?5eb%AVXv0>%+c
zvD!{@5kLR|1Q0*~0R#|0009K{Q(y{=Bap)bCek`keZb=gl5e@<U%N6LI~hl?pS6eJ
z2q1s}0tg_000IagfB*tVk-#JyM_ at mA;|R|B?82*lx&Gv3vR%NYY{`xzP_`H&367!y
zCaVY_fB*srAb<b at 2q1s}0 at F}n@{A*Bu^S@;Jfi=<s*7hG#64<MJdYsib%u0T$mwyr
z{Gq6L;D8zks4a(c9KoNeo_O`|J5Ik`jw7%s*IJZob0Y~DM=%YK2(p9#0tg_000Iag
zfB*srAW$H1<i-)mUjGwmBCsF&aRg`n_SRp2=a#$Q!8n3~n!#HH5I_I{1Q0*~0R#|0
z009Jw1SZ)yf|`Rjj^G<NE?xcAuf6-DvR$BDbg+=mBT%*|XUmZUMFf06009ILKmY**
z5I_I{1Q0-AiU}ObaRe<j#>jw>J01#pW4be_`vOjHFcuf at +~Vm0zCg$u2>F7ZsHQHN
zyyFONAN)~Zb?}-0k>d!;m5by!f{U^v2^dE(#cDgrMF0T=5I_I{1Q0*~0R#}(Pl1Cz
zjzISMpGZ4F%_JH}aQB~n^ljTe-+Kn*2==q~5F7yn5I_I{1Q0*~0R#|0;3yK9Wa9{G
z58gO}o8Gqk^Vcr at z)xhmK!q|dKaN0|$2fwcsDhJK1Q0*~0R#|0009ILKww%5?B6(o
zmRe(EKtOlvK3~Y|jEX at 3;+X+HF-E}S3OL=G>JRANs2=oq)#m9kj=+BY|9$h0Prk59
zjw7g0JaQa?#~ev8Eh`0?LI42-5I_I{1Q0*~0R#|0zz~?|I0D)0e<F<qwNqpqflFq8
z&()TD?yp&Med=TPR;Cs-+v}7E#ry)Izd&}@L1iBO1vpF)KmY**5I_I{1Q0*~0R*PH
zz|qlPK$KEzk7Bh%y{=d&77RK=fq>@p`g>i at kk_p{{aR3sd;M`wO!cdaMQBz0tTEdK
z7To*aw#&Dc{i;TJM6?UgUtp@&dD4#n0tg_000IagfB*srATVKp>C|69WV&f04Fgpo
zU{Qa874JV!x$3+t{!ypww%BBU0r8AN*;|1A0u!!B1VjJ<1Q0*~0R#|0009ILn4|(z
zwZDMiyh!!{(7Yin==XS>o>)*k`Oq5+IeYb}->FBVZhvpc8xQ*8>S-eM8|W`^=8D~&
zo9|g)UZd<0?E>@{n50#iWFde60tg_000IagfB*sr9DIQ(&|g4gx@#h>1Jxqnf%O-7
z at iR4>=3KMj({;)gOPTC1kn1f#e}RKv;fN6d1Q0*~0R#|0009ILK;ZBSOrHJ%BHuU2
z9snNk|NN>Rk2&KWH7cHU5cN7kx+~=LxLy8G)F1V!fq>c~LLavN0$=Lgl=#3C>l$m6
zYel<&=r16jMsSg`M|niK^YE4ydqV&L1Q0*~0R#|0009ILK;W<n9J&4iBAZuDq=~>T
z0v_W20(ZW+qv6GyzSmkO8d0pWzd&|w0p=GttTmCHA%Fk^2q1s}0tg_000Iac5`jb6
zUqB at IZrKAM<c^1e-k9zT>b`)}8;r%pJGYn(z!wO410i3~6V=ouBJ?EfFOa(ZvKNMS
z{Q8GA%0;4Gfc^r9q#_b20tg_000IagfB*srAb`MR6gcSp1w<xyPo$loMg*KJ{RM71
zw(;`E=7(GAWC_i*&|g40 at In9q1Q0*~0R#|0009ILm=*&2*Iz(n`6<~0AfUT-pD*Nf
zM#X>ji#Y&%qA!5Q6>z#W)gREkQ9bDKs?8$wwCFGJtFzzw(nG(v{!-aik!=@XegP at L
z3jqWWKmY**5I_I{1Q0-A`U_07zkrbV(L@>xYDK`Q&|jcj7VVy^E%)4Cv*!9$m*4or
zYj4v(D*Fr6Dt?Rdvhs}bkaC}LmvV=4v$9>;tZWpEc_DxR0tg_000IagfB*srAaE25
z%&)bbZXHS`<B5Jf9f|2%5>fGQG&vYg^p(x7u{rIdS2oyfK6^ToOlf_3WGI>JH-jHn
zZS#y;F_?_$>6ulw#l_3*mA1KdV;}VuwuRQ at ke<>qiR55<Ms2xmL4KFBZMJ5axJ&hD
z#@tI;ZJ7}wyU=bGF{PuM^#M&3qDvN}*)FirdBeRI{@@*3WxIe?dC;OfXwE!X|C|iR
z3jqWWKmY**5I_I{1Q0*~fvGBRbc`ctnNevG;~(rkpB8j^V?L*N-l5;=jk;XUkWURc
z{i^C$<8HUs>k6sWqAXSMv&L)}_}+P+*}i`987t*D0;}?fMLeI7=M+xW3QxKbKmY**
z5I_I{1Q0*~0R;9(U^<N>XtecMC(<x5LjV<xBk=zBijHgE7y69sFHo=8#u-PTG%Lr7
zmAnu at 009ILKmY**5I_I{1Q3||0!qDYkv%aWI{2p}*)IHZ>TGS++yc=bUv$^cB=mHJ
zeTL|LZ!Dfx;|NZWMQHXHD1Yk7KYX*|j5itM2+px6=S=+qCI<*0fB*srAb<b at 2q1s}
z0tg%if$eUqV%c7|&{l0-wrpdy#cuZn;y$0x?bUU+OAp1pUNx>q1MyIAJk}d>x#Iy>
zFcx#Wd;RXX2oMjsV*#IE4d|h$H>i0ufB)84Yeq}8ZoRVOI}ew+U1Ah~%jZ<x4wq|b
z_HT3cw;Bq#oGw-T9kn9Ag~#FYFZGCx)YecS<nnsmZjZ68a9JR2%yxmZ2Y1|k<H!Dz
zFvbyl-J*Q`FvzhB1Q0*~0R#|0009ILKmY**5ZF(FaD{z)-5YLt5IbEccWkx`yfc2u
z5B}|@j!i~?fqz((f9z+n2#x>(2q1s}0tg_000IagfB*thK%m at OF1j>`gq|!DYqkrt
zJTPm|x*y$mz0qIbHH-4v6vzrmL;wK<5I_I{1Q0*~0R#|00D=7#C@;5X+XYUQ8=CC`
z^Ut~VvtQ3V(`&Q~ylzom-`{8nj{pJ)Ab<b at 2q1s}0tg_000L7%z-G&~3oH_8%|EB`
z-lu<b*7C1!{HYvAP@$wP%G1gN$_JI}#9LknAb<b at 2q1s}0tg_000Iag at WusZRa9E+
z)$`}1^nP7S>w2tDuP>L2X3b5e+xv$SgSy8iKcBK8rl+H+#84)Y95hxqWN3#r5c7K-
zjdEH+mrr#^RL at dRuxVbI+-Po7T2Cdk{^FGlR=I9bgO-YJPGt0GW;ms1V-eXYdQRbc
z at 9ufmyPo;%pJcm$Qvc#!pHnDS^Fjat1Q0*~0R#|0009ILKwxSK%v5ZP&8HHklf$W~
zezZTQ at HA0?`Thd`IQ`X!e_Qds*~Yws_gj?vr&jTjS_BY4009ILKmY**5I_I{1dd7p
zpS8+j-(amfWpgGolwR7_mKf0b^mJ=~VsLnCG&z{jL_dU7x;2^VE7P)_653?n17UT7
z_{p~m{H*=Hb?1Hak-Lp{fmIe|)lsQQrV&5 at 0R#|0009ILKmY**5ZE7q?S89b*<QEM
zR&8CjY@=Pw=;sT>eLkPttLtu;9*TRtYFv*7;-TJntT*Ix#{;flEarCi`rUC6ARcnZ
z0zSVQ&_hvgQ1fX1{;jdrjFxKMdS%CV9xjuk0-P?NQ*}FBuBF+(&Dq~-DByCsRPlGz
ziu at KHhs(dzBQ{c7LxGUX>vg+5#<myzvi^@ElSShQs<%J+{SQC;p&Mnpz%lh--p4qC
z`X}li5P^6hfB*srAb<b at 2q1s}0tg^*_ymqS#^$k){113!D3y#S`t|gjS++K7!E*67
zp6IKrn`v8E5S(!YnY7(%n`=);H|qnMC^q at 5!u<0H-u$&6E_nVUJAYvG7kIZtdH3Nf
zSoVYf0tg_000IagfB*srAb`NpEih!Qv)C6~tE)FwFRQaJOJ{}$2NQ#R&VgjVKGJ8P
zum?cMtGa at 2f363>xV;Blu56!zvE2%Uw|pdFzFlDBKX3WXP0hnM7|$tu!J at oybe9OZ
zM*sl?5I_I{1Q0*~0R#|0VEPEmDz`7T7qvEsyveQs`E~)N;xEa_8tvn9VP*ZlEXotg
zx0SamElPv<zzYEc5I_I{1Q0*~0R#|00D<Wxa9m}D_z(Iy**W at J3mzOeyFx5qFxOaa
zK0R=ge43zVd)<~~zm^dr7g~G8V+JGHFmH4=fuK$7uWf^|zx{eT;Ynp;%SBDbmZct1
zc!E^~P#cT at rJhq*FN<`~)s}niuUT{bSv8M+<B#uO at m1Mhpi;TcqWn?$mU6Rlo$`os
z`}8V$a)ST at 2q1s}0tg_000IagfWTxCI7a+GzSa2Vo@<<_tSh%K$Sq2yD~>O>d#oke
zBg&5}x3}ej6*MQ<M8|+dxy43XL751s8qa5JEo>YR0&}E-*)H&n%O1UD at y{HW8RH0E
zw<xbq79-Mu00IagfB*srAb<b at 2q1s}0tZvTX0yvV2al6+n(YEBU;MVZ={tK?8S at Ul
zYEfQ2n0q8H1Q0*~0R#|0009ILKmY**5SSDK6>@k%wq2k>rpIg-_}lN^*0lSYEABG-
z3%q7gUYiuDB at qZ9fB*srAb<b at 2q1s}0tg^*a0SZCjdp?KWwd6yKxpX?E@=JQGuQDv
zf`f}qYzQEL00IagfB*srAb<b at 2poBVlFuWUE7N7R3*38OMf3MNoc9{-0?%8N=Z}1<
zNdW=~Ab<b at 2q1s}0tg_000Iac1c75J>?NL4I8VlGwhL%0-x<B}u8^H^1P8&JC=fsZ
z0R#|0009ILKmY**5SY9I<~V}+B8h9vc7ea#@W9Sn-}jaG80`W)k6`i^A1Ols0R#|0
z009ILKmY**5J2FK3FMwfP%dLJ+Xd#o^aa;v+B!7(Jc8NvAG0VgE6*qoDfcONDR(G0
zE8CUL%0|VnG%Lp{w)z+ApQwMJ{>vg5F9Z-k009ILKmY**5I_I{1P+D3F|%#W_H-th
z()#qsmSq3%fSx|?7 at Nl)O%7(X#Gsx^M}|_#c%olV&zWUwvlc8DZ{vx+%DS1hg$2Q9
zDz?S;#DIt|9Z4sLQ&C;1w=FW43?^eDqB;>#E{KSHNKa)Fdb+|s!!|b;Zho!pbgQ}F
zNKD_7h>GZp9hS|mu{rIdS2oyfzI;Z8lF5EE_;J-DH)alv71>10tP<%gT5hit`#1Jc
zUtwEll?l at _iR55<Ms2xmL4KFBZMJ5axJ&hD#@tI;ZJ7}wyU=bGF{PuM^#M&3+A3La
zX1l<DtV{meWB<N0Y&@s%8H at 7Fp(s}3L;wK<5I_I{1Q0*~0R#|00D;3PaGcd{S!}mg
z*Hw$ghT*}%#9*IuAla{rG at LDd^5Y0Tyk|}DBR_rYDPtVLvlivq!<kC<g#ZEwAb<b@
z2q1s}0tg_000M_d;MgkhT*Ad;+Xafo5xoBJDW}}A>hiL>`Y(%d1lLVu9Km&rBRE7Q
zMzjbZfB*srAb<b at 2q1vKQ6tbGh6WT40Wf>vmz5evAXJVo7#OgRaRhaYBM_r0_BM{d
zE~}Y2j^Nb~^uGVo&)&Gn7)NlsMY;W`sf1({0R#|0009ILKmY**5I_KdBQKD$FSb@!
zZ>(NcXJ0nb<NthXon>T2-N*_t6rjxQQbSIcJGF8 at -GYpF|Z4TZd_E9mxnnqA_v
z2oP{NU8?x&$bJ@!j1?lovI!vHE-=%2*304gZ70ZuRrPBu$~lU&{;&1FtG}=Qlj0*U
z1Q0*~0R#|0009ILKmY**#u3<5S81_(w%3W-0mW>1{d#M!mewQL>HB<vxX<Tvdv)FI
z(nE2tSB>k at Ks?kNkM)LJ?s&izjK$pUUcWmo0>ne^Sit9319~Xx4Qd|Ef6|N!i+xd3
zHm=sRo=Rx_^1HChr at AAmXQ?N6T&)POc#aVurT6P%*Lti^pIsxCFPLjAPo~@Zhs0Ec
zLAzMkw!v5^rWuT;5<_zOK4YyzZs*VjVt%iqv3M^{Csd0q)dpirEfw9I$cVZ9hEsYr
z_;z=d6xgvWFOY~0XhW at o`qs>aSDm~%J)DT0yfl*<)?37fKKUW8^{2(Bvy<tJ{HVI*
ztL5a*wEQIAtkZ^4+F+k<u9B<Y+LDhew|OoyBj){!>ta!u^|C<C{sJF!tbg*mna3`Z
z^9xid{TAgX%H7Iml at BS`D*ejSG9)hq5I_I{1Q0*~0R#|0009ILI1qv3%k3U(zR_T$
z*`#h(xqXrG0EAqt$BbjjMel&3W{%pra{GeZqGY<}xN>{$nF*~0O&j)xa=V%fRM>7&
zUA*g&_KGTzkK<%s%yxle9(sAp(3_nRV;sS&7Uk6g$tl|)fB*srAb<b at 2q1s}0tg_0
zz@!$aC=-1j>~d~_2Jw at B9>Kqz;aGb6()VZk3s{wZSj1mm2q1s}0tg_000IagfB*sr
zATaF&D$4D4YnjMa(enrveD>3Gn!l}F#q$WJU4fG!1Q0*~0R#|0009ILKwuIJjQ2bO
z+Z%r#!5p4Pu#e{v6wNz$Tl0yXpIh-%i(FV;-)2#sP(H1^U%5(obN#>S*N6qY5I_I{
z1Q0*~0R#|0009J!W`VhLMDKHtk{={*j8vCH<z3s|$4<Rz_cqNwbdx0>BdGQ|S>h}a
zpwwiEGsW^TlO-x*;ekz-STD95cd|rLnTzHf{Ka$6?OA-wIW9TxV70Q*qCBJgO!=<z
z1?8h+=E03(B0^pWAb<b at 2q1s}0tg_000Iagus4BHbNtDclR2eU$cGgiTfAaqLc-Z&
zCL}y&?1Y4~iY6qSS$aZ3Wt<5KWt%|Jyo1;G_ssm_Wdn~g at 8I5wm~{vsfB*srAb<b@
z2q1s}0tifTfeM at GMlg2X!CRMKwPK}y+oxrJ0p=Z?;$=+o5kLR|1Q0*~0R#|0009IL
z7*`-W at 1RYrE}D1n*36x47u at 6jA@dH73&#os5I_I{1Q0*~0R#|00D*%pFy6d_<!^l6
z!4sKxaDV0<Jnhe)`0ppnpWh<K5!mbNEXpU98<fq;xyo^hwf^_@&0-NR1Q0*~0R#|0
z009ILKmdX1DsaMydz+$eyL-XZn^^Fq6GWEwHv8Y~`69rWS^nmUg$Fdt-*I!rmL(<=
zbQ~|1FPoL0u(4lDXCmp0meE_Yb2nai@!SRxqV$xGn~pnVvm=VCTr}_CeV4qtqvDC*
zzaqyG*cGou at rwBduTXAPbmeK~$I4pe&gokD$rl0$Ab<b at 2q1s}0tg_000I*e7-tT@
z6N{%0G#=F;=M`KqX8OPrislNOUwW><d1L1ooO@!qeVKLK#)accr9e(&*igJ;WM)G-
z=U~yigO at jd@ZY}nz9n}s at 8HDh0-+E<009ILKmY**5I_I{1Q3{#0u|+=d&1ax2bWa*
zqxut9ej>oUgHy6pNi+foAb<b at 2q1s}0tg_000I*f$j&<`=Nvpi{H)n?wdJ1sYt~#}
zbH~SC-L&}WON{;kODxKg2}VaS1Q0*~0R#|0009ILKmY**rnSKKjaJ37y>6kc+PZAn
zM*DiZ-4}@ad_K2V*WE5X6!&`7xE>9}L%s1>Z^-422VB8e%<b;=yW=82JmiiAe10{c
zhoat~=F$B9TVt&mE!Dd9%8u_mT;_JEA*ajdRNW4jYiagxbN06y3b>puRs0>bBEN;l
z;qou_h>g_NP$1;;dfjf1vF+F(?pBxEt9o31BY-HfqUR9=27mtN(=Ti~f#(rS>&i^#
z5I_I{1Q0*~0R#|00D(y+Fy8YBD&F|>2pV}F!TvmtpyJH(ue{QA#qF}cKuz5rEy{0{
zdzD>EQt7O}xqf^7Y4x>YAuj|FKmY**5I_I{1Q0*~fvG3pXx!(7_>0G%5dXA8HX;6T
zi^N__OyIxWed-i^`oSq8Gkbmd!O0 at Pn5Q2s6bldN=?5n`#FlD9VnFNDTeVblb0VWh
zGs7ue8;JS6C!ZwNo<27}O`$e4)W0o~(Fcb5#d8p%B2-2f6}jkn1n<A%q~-qC+gx&f
zff}X3qCB8{MfsF?QsD=bKPc-JzjBpW$O{1k5I_I{1Q0*~0R#|00D<W(u=jcY8;hs+
zH=aBoC;4AoykcbD|I@}kTVPRf*pbH<oI2+51*a4}zTo82k1tp_&SMfBCzRVyA9;pB
zapS;Aq8N*wNAScR+ry{6`{Q@>Jc8+6!O0y02q1s}0tg_000IagfWT2MP*EW|6O4Tx
z!NRi}x6Yfja5K*%ILhk>*+&2Y1Q0*~0R#|0009ILnC1f6=MhwhOuA%7&F2wZ at r@VW
zanhDw_R9VOmYEid@}M%$_*4I!T*M0j1Q0*~0R#|0009ILKmdWMDxjQgY3)(07MDNf
zi^ts_r{AagoZhG!bB27ukW=&O?w}s@^@e(VYNN>IQ!;cwck4c1$m@*yf*z+=_36%_
z#}#n8HPs)`y-_{r at v6-t^xZOa$Q=&_y)h9+_XV8ZU at R`)xqF>nUm)ZSh~0Rinkt|D
zcY_S=ar=8!Jsxw$J!(|M9rZdxx+~=LxLy8G)F1V!fq>c~LSH09Yu=C+^m{x`Pb?^6
z^u|KYUOnn}>d~l(C*+L>eR1_P5nATb?(=CumpA5fig^4YmoAscrFi6lUse5T-0jwS
zT_LqvgjU5*{yBw#U#w7nG57v$vcEvKUEmSrBJpRB@`!S$T*M0j1Q0*~0R#|0009IL
zKmdVhDxh3!X_`orfyn-kCem0?D+2DGNIQZ24t~`{nh5M7VAn)i2jtiNrinBR%n$*K
zo>O?%zh3{nUKO~X=M)MVUI-w700IagfB*srAb<b at he?3v6ed&RkphvSWU at aU8BE6X
z^vo*TVtYDMu-sm0n`@6I2gPFt>nm&vt;r!hrDYPy!E{}1xov at cD3y#S`t|f|o2}U@
zK3pmuO^}Wl8<f?S86mO@?N$*}I=WdO&_n?jjU#ySC%>$D*VD(WH0B+AkZ}Zu=?Gvq
z2q1s}0tg_000IagfB*uAUf}2$N6=DNX_4ayT%Kq^3`1}`{aQ$jXb1)4&R%~o>U0I8
z9!-v3P-8Atc2y`EM=+zk%V*j8o~3eyRe8iB#t$%#;Lul1_J9Ba2q1s}0tg_000Iag
zaO4H1(>Q`gTaR at j4Fj@UfE=BWpLg&N^A^o|<i!h at 8~p{oWKq6!<O_xrAb<b at 2q1s}
z0tg_000IagaMTE-?2E0{)f=mq)!CP&GsA;}iNQYSfXM!)+={i<I*Wal8Ky3(mbqPG
zIzgAusd^kP|5A@@sY`7Qg}kaO==OV>UE;F{5O6tNs`%^3ein<26(Zv76ocmT2yXq|
z=l7g{`>78Y{RLjND6bwh@?;bN1Q0*~0R#|0009ILKmdX1FK~>Qv_Q5tRF>Ok*`sm{
z!SOQFX1l;+&)xf;o4>dH650i(e^nrd2q1s}0tg_000IagfB*tVp}^R7fw at 8<|D3|+
z_x`ooaq*f at 8|?yrw<v!<3XRAz0tg_000IagfB*srAb<b@(^O!#e8_?5FHlusH~R~$
zl36wT3w-Ir(fJ=tE&qryzrbHD%3r4`1ld9W0R#|0009ILKmY**5J2GQ5;%Ub-CkW+
zZLhW3EwhZb*>MDC3x<4ufgMkJDtDIu^mj&ofxlUlza3pj<Q4%05I_I{1Q0*~0R#|0
z0D<W%Fvq^QTJ{&PSJ_2(fofxZ0hi3L*<awqf1L6B?B|!fBKr$iW?C%DgUUSPPyKUp
z5ibM~KmY**5I_I{1Q0*~0R*P1fO58_wMVg9T>h9Z9(Q}3exK at ddZTL08S(`~PR*;k
zgL=@{8|w9`jUty%$<P7at^0f-uQTckdYoR>r#pilSHS7kRDVGCM)jb_t2T?!cgxTr
zcRUpI#zYw17jSxmvAB5W?sa;7fsi*KcH at a^>Jkz91{vDp_V=oKJm!ph)ToF%>UD;6
zSIFscyZoW3Kk8Eh0kuVhzDS1Fydf><_jsJ1SWv|1jfI at OderaKqfrr0$Quv(;_7K4
zw9KV_B9AFlWw!u1_h7!iz-M3kRK-89S at UtZBHJ$Th;otmvqyPExl=CUg#ZEwAb<b@
z2q1s}0tg_0z%&(5F19pHq{%>J|3?#PET|O$cTc39Kz;|mY9dVpb`h{^BCP|}B4E=*
z8U|#yfO7GZ?=R4Lg|^mx;%)y{SN~<p?E330%FD_#%0tS1%3aDG%FW7lWwWwT at hi>B
zv5Kw!#rh}eAE>{s{$nx*UI-w700IagfB*srAb<b at 2pp(D!!b6GHJTjEXo*2Rm5vOh
zlJP{po~}4%wyjxod{3sdK0UG}**`p>r^QZ=x0eV!XO^w4FcJ|Yp6F|=n`v8E5PYU$
zTWn7Zi1^ZxbaFTq)s=eNB6G=LGNz~J)QO04K}1T1^i(Dx(qf-so0|(azt(oT)l6h0
zrf*3^MRdjv%VyWuoc7Tx8|*e;J|jcPWWO2wxN4hc)QW5(W>$%GnmNyIV6PPWH}+9q
zVOwaG3DYu(<Y2nKw%oQLzsuP+TeD2urFt}D?xn1@%m|TPXt#=(($UTOfF>}D#u1!)
z?oS7seVG%Cc?Tb49KnHBX|_WE0R#|0009ILKmY**5SVlVN5?pVmikJI9ChG}dpyyQ
zR}7B`1Y=IG+pjq_pD*UrW8OgAt$WqDC$26Qg;X?-;BB}5$8A6A_{w)=9|o)Ph((Ma
zU>w1utH~q?0R#|0009ILKmY**5J2D%2u!DO1dX;H>qHs`>V;#`ID${~eeTo0-}vrN
zF^=F6)G(q%009ILKmY**5I_I{1P~ZUfN=!INC{(r!G4b;P!8TWg75xq*(ZK<<LB1O
zaRg<`ZaI!%H_sy&r`oa at 0R#|0009ILKmY**5I|r`2~5>-1TBg&GC+%ZW5Kw$S3D0P
zq&dC4F1NGS6ACy(y{@>&>(xANkFK8f#>Nrs{^EyjUG=H&o+HN*lqq}UID$Pqk6=nw
za}tXH0tg_000IagfB*srAaGy;Q(zo{93C){)&b>!#}S<OAOGHV)rzg}WE{bPRU)=R
z009ILKmY**5I_I{1Q0k70mc!$p>YH=58gO}2T%A)>Kotxe2*MQU{kgj;|R9Mkp%V6
z9Z1Qt4FU)tfB*srAb<b at 2q1s}0+U5x@{A*B`Ty<ROK4nG9LMoV({vG?-b at x*6qi~}
zJ4f#0+!u5uMQRtmY8JvQJU4SMSXa6TLcmvvXqGO7ilCJQ5iwne;v!hV7g}8ygf1+o
z8?mURZi=83|7VgwT at -HTk(nF6gggjmW==l$Z06+mJ9lcU8Q>O?r{h?K^PyHci3*kK
zELDYZ<IoG;&=kJc+{qq+{`17CPfmWF*dD>Svt)Y&OVT7*lkaCo<vKDC1Q0*~0R#|0
z009ILKmY**1|TrJ9zmxbP-2T<YVbXRd(Y2p=+{?|(jyqa3gameKmY**5I_I{1Q0*~
z0R(O!K#!oO9zmmjJ%SnU%ggnJhi<n$g1r*3=-iKB!8t~c;08;VUn7730tg_000Iag
zfB*srR7PNsJ%VOqvl)<Qxo at NbpiH5q8KA>hWnq%3T$|KS{36PY>m4559>L<9r=NQ|
z{^p475$ttZ+dYC7J%Y+Cb0&%a0tg_000IagfB*srAkdva-+Kf$^<QS6pfN%{f=ee}
z{o%;UnJIb%-7O*>g#ZEwAb<b at 2q1s}0tg_`jQ~A at o_Yk+{p%5Y_rT)EKfb(v(e?-?
zoCEX-x>?RV1_1;RKmY**5I_I{1Q0-ABn57+N6?(!YzCx-ZtF1emCj8dU!qv0k<lvk
z(>P7jBuM?hJG5JR1fTk!H*WiB`Ks*^OgMq<5d at t(612D<!ALG at CWQb32q1s}0tg_0
z00Iag&@X|qJp!BhFSA)NU6CGvYXR4usjYuCdHkh4j~%`5wR^sOrS5FV`2vy`SaCY1
z4sz~bzm^%#g8%{uAb<b at 2q1s}0tg^5EP?+=UO;s3o^!@(VG#uLsZN!NleW at +WRwZR
zw#wT&F8o}3MNoKm%4knMk8ke_%wG8H%s=g4@=51{><dU<U^^{9USL?ume)l90R#|0
z009ILKmY**5ZD=k-INy)vDq>k1N&v*fARuXp8Ecg_pYrzTX$A!<2Em_l@=f`urrI0
zS3v*)1Q0*~0R#|0009IL7 at R;=^8zB$vIzh)uiHtXGbLvl7^O4US6PtCaR- at O1X`QG
z4|4CIjP4;X at W=YKFMh3Gd}7jBlYN1$yg<vbA7=-5Wq4);5I_I{1Q0*~0R#|0009Js
zE>MBIfEYboW_Q4mf!*Z=PS4C-IX7{~J9THFw#ViLjyVT5KlcAV^ee;!5I_I{1Q0*~
z0R#|0009IL=s{p~@&fWlEO!zBZV`Dpj#W4xYNeB?P^r#RRTwu8z0eI!;d at OPy<>TS
z)xSSDf4TOk<OP;wU!ap0*h~wM7wBQ- at lXU1KmY**5I_I{1Q0*~feHu=FE1cCi>AaD
z!ITUfa9-f~w;nwC;=3E~+q}S-Y^0DEsKCl)iU=Tp00IagfB*srAb<b at r37v+FCcG>
zJ3aj}w<PoqH5>bCHUW at lxo_lO{mK+t5&$}kRTd_h%C$-T#4n=UxZYuz?uh0EemnK{
z)eqme<k-BxnA56tP8Q&Nfl}*>K?op#00IagfB*srAb<b at l@=(Q7m#<%vt>35rtcb`
z8!NL<&=9s!$qVdz{K>!WKe(9I?WL4_0(pT-uW2Ta00IagfB*srAb<b at 2q18sz##Ji
z at -iRohP=QFw=UoEbAB$Ev~G4^VEc3d&KJ0DCGjHy2q1s}0tg_000IagfWR&k=zCs3
Kj8;ZHFYqrdB4h&q
literal 0
HcmV?d00001
diff --git a/lxd/cluster/testdata/pre10/logs.db b/lxd/cluster/testdata/pre10/logs.db
new file mode 100644
index 0000000000000000000000000000000000000000..452ee8a637916ca91945a0b5747e38f9f2bf4b89
GIT binary patch
literal 1245184
zcmeF)4R9RSVc_{5%-~}H0?iMRBa))jq(l)A(BSlZ4`qddz##>S1St at -tVpU+&vXws
zGB5+r3~17>Y)FdICf=&6T<zV)s`DnwE`7<ZYwhgj&g<IBUAb~qu6%6n!%oVH at 3QMS
zSB{f&a+h0sli04idp-T;69aG{QX~QXo8Y|Z?$@vT{ob4I>F)P>6vZ8UGW at G=ZTqVp
z=lynf!hoD!3tp3Nf*$wHtQ`OSfAw4c{d4n2f4vru5CH at bKmY**5I_I{1Q0*~0R$c#
z0eKp)bbJ2aFE_n*{(nx6fBO%={m5wIzxl$0lQXsi0R#|0009ILKmY**5I_I{1Zo1+
zd;MahNBpY|nt0>f_kUjecZQFN|IYY6XT%*mH*cLVE9SS9r>_6%1-o?FE|vfDw|{lY
zvPyQj{42qKHgn=vj9<I3lFOCM1zY+2Ou at 3Rcy7(xr6uutb=mHf>GZhieASK7(-fn2
zVWPH1LIe;%009ILKmY**5I_I{1Q7VZ0$#afr}O{X`F=>manAprYOKZEa^CK3I&ZHv
zB=EU0{>9+HV4szl%;XF1IliwcZX-S3A%Fk^2q1s}0tg_000IagfWU_+P&-}sD34dy
z@@;Y7-8t8<wFdYjIbOR?@W06MoZP10mg50A{#7~drlu%gk>iKu{9l*j6LS35<hUWn
z-<RVtIsREW?w8}gAjhAT<3A_IYYhsn$?+3%{uMd?k{n-_<FCu{OLBat9Dh at e&&lzR
z$?@Nn<LBl0D{}lrIsSjk@#p3EZ_4q69RD*p9+cyMD91<TxFSMl(B$}^xTDIWa=cHD
zZ<pf(GQJZ!<)zATCmm<}FJ=6{6QMJ{r#9c6UK^_wYmKkgy`#oGB$FAIp(?{2GTbG@
z-7>6IgS~QmzYGt^@Cg|nl;M*ytW}RAa(qmN<1*A`cvOb9>JpaYQ5nW%n3Q3yn#fTC
z1Q0*~0R#|0009ILK;YpOsGazAbN|2A9zbXXTJHY`n(pb>&MY5oiScMtjM^oKhZmik
zBY*$`2q1s}0tg_000IagaE}FQ_mwwz{vU2RyAL;=-Pg|lBP}r^O))tCzsF_F>JUHx
z0R#|0009ILKmY**5U4fy-mbV!wrjudmytAC8Co*5W%%1N{2dv7M}|+x^M9uuyE7b=
z;UO75DZ{5^cvyx{%kYQ{op$lgFebxiWH>Iv2^s1#oRr~F8M;gJiU0x#Ab<b at 2q1s}
z0tg_000NGH(>L51YWM%0(`QZmcW%;8i~r8J)7RY at -!A at Vi-lU(e=*iN^i*TJ)10_Y
z=W!?f>6KE+E>xVS|GP64&th?*OgjLnL4W`P2q1s}0tg_000IagfWU_>P<zmubN*ku
z7NCihI`0!fckSmz=!}2lygM#lce$<mzZHCJ-+ONk?A(4$(@u|ij>#wezu@)t51hXC
zrt{ReKJ^QV`a|`5>NnJ{sb3Z&3<w~A00IagfB*srAb<b at 2q1931pIy8y?*EBetu!3
z*L%=YT((PQC0{I*`-Xhp1ODYwF_&Mo%e%bZagP{z%g$EHX?KC1p&mCzb)w%RQYvRJ
z*-NH#{_j!$y(0b?5I_I{1Q0*~0R#|0009ILKwwh}Xda(3t>%}^1-m?Hu2hQYm1WDU
z*y((sVwWzPi@`yWFOT}Ciuhwd009ILKmY**5I_I{1Q0*~fz2xLxOdvW&IN$a&lD{C
z%C4qo|JO45i0nUbUfcJ%_dNr<+B*>R`ktRkojiT*eUH<IU_iaz(uY8OPfTDy009IL
zKmY**5I_I{1Q0*~fx9QLXTTfwmn+4RxX+(nE*2Nd=|a)6%iH_CM_Q+Uh<ynDT2cS{
z?q!LT5I_I{1Q0*~0R#|0009ILK;V7~Z1?mj`*#idEjwqfELN<1$?Ze%LvjEAer^}*
zMgRc>5I_I{1Q0*~0R#|00D-$MFzDT1y9D4qL-0q6`lGwP77|1N0R#|0009ILKmY**
z5I_Kd`!3)U&lmLj-TVJPR at 5Kg_YGnL5I_I{1Q0*~0R#|0009ILK;Z5R_<Vl9-+hMQ
zBORV0c=uOGiU=Tp00IagfB*srAb<b at 2y9V-oiblNN{`}`?==w)i2rftjTp{pt>a?S
z6Y}}}+Wvi?eSYBa_Q&`41dY?zo*!0kDPDD-qFz>CRQEamM!xT)$AAC=2q1s}0tg_0
z00IagfWT%IP+wFgFRC6T7_*|eTqvx^qDEAYWDQGCMiWWhjM$-sorq?VnW!;3G1R9x
zvEz2gjz*IaJsVAg^@tI*^+Y%r*F&Zei`$W`od`#a at rgk<b~2PpCL)%IV at Kn9Bw^*m
zyHG}tMB~XwT&yOXHH~K`{BG=UD3&qooTcZ&MpmSqjp#``nAF3eU at V!9WurzsZcGU8
z*PSgfBS|w63y1Zvl at KXLtfZc?voYPyW<@&5NG=i08BdGY)m&yHK`Ut`5_&QoH}yy?
z6V#KDkgdnegprHHa$(De8Ak at yFN*X3u%f=BhSk^AFRI^EzqDD)m+eLX0R#|0009IL
zKmY**5I~>{0rd at KteZ;%quz_2ZY~N8IT5?L9x&)c?B-H{U*ux0o2vj$%Yd<NE&%kq
zP4}PZ&~#sGX+H48TFnP8D?Lv00oizf<^x?+G@>AY00IagfB*srAb<b at 2t3dN4^Hy|
z!SorYK|nXx3_k4U1Mi6Qf6;uvX*@vlfd{(&vT+C?fB*srAb<b at 2q1s}0(V8={xu&E
zoB!!<E(LsO%?Gp&%?Iv^DTyF}00IagfB*srAb<b at 2q5tA2n1wSo#+4c$@iKFomT#d
zpgZBb2%E>n)PC83Kls?b_ud at nYj42s^*Ij!d~;aD>{b7bqW)0*p85^-YvL^f0tg_0
z00IagfB*srAb<b at 2s{`9!@b^v{^GJ-GAsFFp*-aC9`G-hin;uvUEbyOj(fz&TXwcm
zPG^gSTz;WvsK<>_o#^*?_xj7(OZJlKod0{&uPfq at 0RaRMKmY**5I_I{1Q0*~0R%RU
zz=TKsxIi|WjD}+wJrvA{pE<~xAw7{XVtT@~f^oylL^7eM_yGWsFOT|1iuhwd009IL
zKmY**5I_I{1Q0*~flVfG#Cy>n?&1o7`vU;|9eVH&wDce7U$_6jmmB>DzP!nx*jxk<
zKmY**5I_I{1Q0*~0R*<Jz=-HR;AwOp=()5013l_3(SJa^Fd%>c0tg_000IagfB*sr
zAb`Mq6PW06djupxp<pl)P3W1dY3q?#HmWBxiG*%vvXP{jNJb)tX*|90{sTSgmqh;o
z at xp)r0tg_000IagfB*srAb<b at 4}!p`_oAnp%K$wa>pw8iq5r^xkX^O`0R#|0009IL
zKmY**5I_KdO(XEQ+|mvH{-3|4|Na2|_cu*}v!MtefB*srAb<b at 2q1s}0tmPQZT<JX
zch-O3t6r}5-xmP`0tg_000IagfB*srAb<b at 2;4P+39s8DKbQ>Lu~00gM-mZBj~KCt
zo=MnIJ!6Cu88d2GiBQ&<_^|u$d)0Rp^<8 at K-?ic;Ap{UW009ILKmY**5I_I{1Ud>h
z-SWG+7U2Dm`tSQY^xq$B=|AA7|3Jr*V<G|wAb<b at 2q1s}0tg_000IxLKwJL--<|az
z at To7#{sZC-0|E#jfB*srAb<b at 2q1s}0tl=rFyX892*@Rp(PU1Kh9Y4-V#dR|8I0!i
zaLkCtgE>R|N}pvs^I-KK at TqS&{RiHl2f>=f%M1h%KmY**5I_I{1Q0*~0R+|-aJvQE
z{(^w-e)k_3?9hK;?QAgz0R#|0009ILKmY**5I_I{1Xc+gmQyzP`+q|%{r3myzrPBW
z2?!v700IagfB*srAb<b at 2t3#VZT<Iq at 2vlRuez_^e_y<1KmY**5I_I{1Q0*~0R#|0
z;2{#2=yiMKN5W<-V&p768qb+}Bpi+ENy7;0$w(}ch*>f5Lw#{$d^7jo?^PqU{`(R2
zOAk>IlT`!|KmY**5I_I{1Q0*~0R+|&aC+N!b5)>s)AZjT>d=3GTTB0eA^H!jQzMv#
z00IagfB*srAb<b at 2q1vKgDB9}f1vNq`VaJte7~vxfEZyw009ILKmY**5I_I{1Q0-A
z6AMiA$=~RUS<zfB6xL%=BdSNThNUN?iKK2u>`=l^L^H`u)EIqu`VaJ}Ly9^ie*usl
z1e>^qv*`#RfB*srAb<b at 2q1s}0$m7HTl;r$X`pZG^&i;Qq5nV^Vnjgz0R#|0009IL
zKmY**5I_Kdn!tp7<NW at gPrlbgI3WJVoi}1Qr?rlYNyD=LzP5kgXP+O~*4}@=Cup3$
z_WZE=b*0y7zJE)-tiGu38~HxX_Z?3L1Q0*~0R#|0009ILKmdWwEAVhM-xuXGc~SK!
z;cPY;4aYKiD3}qQ6mn)rPh^ako-nOo+%PkdOeks`8C3sB>1{OM7mfG%0f5b0!`X5K
z5I_I{1Q0*~0R#|00D&$9wp8<dk>Am7t^j;~reN7uhC4LhA8BbmFuYdtfm at 2tX+BVI
zJV5h-E~*+)5I_I{1Q0*~0R#|0009IZV1dord_a)C?lcHUghIh!B%07OS<}`dv20XN
zW)cbA&SWD=Gm(r$4AXd8#NJTzfiEdOnh!j{wU$jo009ILKmY**5I_I{1Q58(0-K at v
zfXMR8-CPFf5fL}gd|;$Q^MSjpOL7PxfB*srAb<b at 2q1s}0tjq%0YheRgU|m{TN?0>
ztkr=3vf_0b at XKfa@%+E7UYp1t0tg_000IagfB*srAh7iX9-IdJ!um5#ll)*ZY{x>e
zm>x+)EInexB6=oaNA-*mPGroeWhFvc!+HGwhuDDsUB&A*;D1Anh<`l)Z|i>$$RPp<
zAb<b at 2q1s}0tg`Ra0}eW2K)m4Q{7w(@QR2ZMgxAeLj(TpEzJkiwVDsSsPs6^2dY0D
z=rkVS`F{`hr-<AmfB*srAb<b at 2q1s}0-Imp!)!hvbiU#?2*@Rp(PU1Kh9Y4-V#dR|
z8I0!iaLkCtgE>PyrOz^+5wRbb<^yjiJ#O;>r||&I2R45-CkF^1fB*srAb<b at 2q1s}
z0_zLh{pJH=eg8a|y!{0MpNM#0n-6U7(0pKhEaoDB00IagfB*srAb<b at 2q1t!n?OvC
zZ}9nlJ6am>Z(q9szen9i1O7I1MiD>&0R#|0009ILKmY**Hm`vCqWjbK!I%}z<w9XS
z7B!-JBx_iDGMY&0X2cF9>_jw^%tVdRiQYcNX_6lao3V(Iv-D^@XX=q~G^!^JBd8}M
zu}C6j#l(a9;>Ng8*|ZJ#J!(X$Hr}TJ|K_dSY&il5Ab<b at 2q1s}0tg_0K(_+w8_HNW
zmj?Qrmj2yb73dXf-W(11cXVjLzq6(Jz>YPV4>-8n6`BupTgixr00IagfB*srAb<b@
z2q5sl3v8_UfXL2wAA;rsPN_NP|L)WOc>do5UyIp51Q0*~0R#|0009ILKmdU|6zID7
zfLPDJ+gi;Bc6Mk!aEF>KjsOA(Ab<b at 2q1s}0tg_0z{4i+tjw_U{J#PDUK647J`r>$
zoEM=p{*m+UxOm+y8}J7o+xOm^1CO*f;P?8bPn^E?=CFwAS1&2*57qCf-%!7%ep&s3
z`ks1SOkzL)0R#|0009ILKmY**5I_Kd4GIMO-l)G^DVEFyJH1>iE|$~TVj-7b*fZb_
zx6LRNExWwE-+QEWy1&o6*Y7Sg((66wDK6V3vyv|s%6&sV?*adEshG<z+T~qd at 3=>d
zyk%!AwYBsN^|&#r6a5~MQaO9cUNW8Ye~<bHiuhwd009ILKmY**5I_I{1Q0*~flVav
zjHgfW`}fY2iz}t9eYCQ)e01 at ObyQw`(5qJ;j#_rkTv at CPihOz0A1LCF0RaRMKmY**
z5I_I{1Q0*~0R*;?z)-KaI^cJ&4t##5VA)r8cj&>tNA at 2$ukHKX`<{W_?Hveueep!{
z*y(HUdz>}|Lu#x;9|HA)n9P6x0tg_000IagfB*srAb`L_EU<sb`>fMUzuIO$ZP}Oe
zS-aj`f7hT_uQ%wg&3v%?5YUF;A+F419RUOoKmY**5I_I{1Q0-AiwX>SdKLd}PpMd}
z^tgQp{#H@{c8g|~Y$1RE0tg_000IagfB*srAn@=C4EX%J{mumdr~QAA`W^8M0rA3s
z00IagfB*srAb<b at 2q1s}0-ID|dyjLsf4}qOz}j;ItM~uaWwUbWsB{0nNBt{t|6jZ?
zAb<b at 2q1s}0tg_000IagfWYP#@Qb_t9_K<pPxb!4a{<78hTxtK&k)@F*(C=EAb<b@
z2q1s}0tg_000IcyBY_igJv~Z~(kI_*A{-F^6G3;vc at Z{`i>Z6%^ZSF3?R)Rdfk)dP
z-|zJWjnmiO<k|f$Ag>4 at fB*srAb<b at 2q1s}0tl=Yc)*_B-#_H at 9;p9Zz%H+M-1%XF
zx9n`C{(}NTJ#LKZM8C%?e*Z6f$zC#@^M8-}bw&O9$oFgiR_B)~2q1s}0tg_000Iag
zfB*srY#ssiMP>4$>QRC*E1JuN!g?%fMD<A4u=Hd!k<`tI9ZJ}VXeOD78lxg#-*sX~
z!e%UD<SacJ&zX889F6Kp!wBlhNGy_wS+PXMj2q)3_E)NDCv%BpG?~+*p-5PdnDMY~
z2BSGW95bTvV9rP+BbM=ui2WHicG!-EVlh3Eh*)~Wh(+{F!j9 at 0BP^1QT2>;IH6}#t
z*PYmjP$(FTL=$=@Yub7wmK9r)NhEYTlZ_-zk#fW^ji)F2`xGa3HWIXwmYAK4$HjKX
zGR`K4Y&~Wsj9es^3tL9aIO0CL|C@^XO*K;e_a*g9Vtot<Ab<b at 2q1s}0tg_000IbX
zRRQ%4WvrV^15O+Kf7i`bfnE{up9j0RAm9 at bKNU>g{#t-nM0~lM%K)R^i=J*S0Q9?k
z_xE<_yT7lc|G-}Q4+wh(1Q0*~0R#|0009ILKmY**HX`tV^&c3xv;G4;>MhZKK)pr(
zfsNDw;v#?m0tg_000IagfB*srJg5Sjwg14xfZHR$bb1NI!eKpZB{F&>VkPyAosH>s
zHXDj%l95~@n)@LA2YS>mD(V+y4+8oRJgC)|?Lq(n1Q0*~0R#|0009ILxH|%yp#Ol=
zEufpL00SHBKd`Su|AD)MN-78-fB*srAb<b at 2q1s}0th at L0x!vYZt(a2_P6xk-$(!b
zhoriYO#~1?009ILKmY**5I_KdyC?8~_22j3S^s^ndb!$vU%l-1;J<rLq=Wzh2q1s}
z0tg_000IagfWW;McyRjfPx##)`C-xT-mt}Q3+BQ`R{Z{7Hlio(U{Vi<g0W;){J at _P
zj~f#ocK>~^`i`Q$BYW`EfB)WBOxB110tg_000IagfB*srAaE}Q?qC0Xr?-7Kmje7B
zR{#C|9s2J-*3y4qKm7;pWnE-l2q1s}0tg_000IagfB*tpLf`@GKQMS_{Re!aVTJSi
zf9hrRMW+YBmM9`JfdB#sAb<b at 2q1s}0tg`R&<cE*{Rbun-5vqSP%fE>Shk+9qj5cw
zuyW#ENc>t~G at gvalhH&tYZ}iySp5fl>g!Jbf!C`&2<SiX(0)G1I06VDfB*srAb<b@
z2q5r~2;BAl15RuIZmtIm-uM0kk9Fuj at Q{=`*+c*V1Q0*~0R#|0009ILK;Z5O%*kY&
z-~aQ;_nHU?#Q(VSMhxe))^RcEpzOb|?cev==LfX*{`);a<Mg%XheewPkGfA$_tpN<
zeBbe8KmY**5I_I{1Q0*~0R#}(yaF3*zCSVKHpq|LAv+pPM)a&`doLc*9~DjW!@;;7
zGL2Z=j%4jbIAV+om9Jb>JxVvX>y2kb?9Vu{yLslHF(G2V?!-1DNiz`(hxM?P5FG*{
zR#MN{*_du;v*Kq2l95~@nlqjju_v9_*+|ezT8V_7jK at tq63YbjWF%zkF*9N0BC%Z9
zGGfM&LD6*IqlOhVESm4j2K*ug1_Tg5009ILKmY**5I_I{1h%R`x6SvR#`fJ at 6c`dv
zx4O9=FeoB^x|>S at ei3o5o2vj$%Yd<NE&%j at ex_jAR}OY)zJI8t`M|+7nh%H-DccpA
z4>%4C2q1s}0tg_000IagfB*tpLSSRf2Sk>?>)eL#=1RdPZa(0YtaJWP^MNf<$H at c&
z2q1s}0tg_000IagfWU19x@|rn68imaE(&a-<^zX1G#|LF7ZDIZ009ILKmY**5I_I{
z1Q0-=S>P2p;ynM)E8lA(?DYJ<n7A1)hr=xm_ at 7$UfZzAD^ZY*%SsnQsMg5`rJ at p&v
z*VHeoUr^suudA2T3u;UqS07QmVm1Q;2q1s}0tg_000IagfB*uUOJKX|J>t(VnG1F~
zT`sPavbH+nJzSeoC|Y)T_ptY<M at CsL6_@Q&C2yB|{oA~IWxV}E-e)~xRxZCNlCkW|
z`K$=D#X>H>(6ejMtNYt$2K?Trzg#Jn#ER3)#o}Tu`kn!AxNSytBewU8?QB`V-zV1Z
zu4AOvd(h)-m|4je3*~J?KJS70DtCFk<IcvtWoIk3we$@2xG|~|{T`80IeW=oGM)2(
zkNP``_+vl-0R#|0009ILKmY**5I_Kd2VEfQ=~w(OdxoF9RH-bNj~zYgUci`K%okR!
zh^qz_GheVv<;h}cp~qZY79&S(Yr%G(|97}U1OBIF^MURA_WhzZaJaqsfakH(*EIL)
z!6`*O^`P^xeFz|c00IagfB*srAb<b at 2;6sp>oJe2TpvE at 9q=4Ke!=ha`=jw(G#U*>
zY&#URletL5$l2L=E}6+$nPf1Oiw6^y6$)iyp`3`2ONOj?G-kx at WHyp8!)9#piZxj=
zOOsbVHT_$E&=U$8Nj(_VjgS@$9;^P1S6_`}Jg5f^@z*w^zC>6H#*T%>LdIk=o(x7J
zp-|Xe_Vqs=`3v{z!P6bC9^7|RHUI$x5I_I{1Q0*~0R#|00D(;=@S5D-?aJQjyM8&I
zkzrPbmJDqf{<aKjP5Zwi$Di=HZ!{Sml;I&6J}JYeWO!JHPs{L#3`b=+Cc|fBI4;8p
z8R{~el;Kes%FQ7_009ILKmY**5I_I{1Q0*~0U_|ZRB)dE*DK#^B6Obrmk7EO&Wo`5
zd4OW-0oi~*_}ISp-W+(Wy#c at 17c@>^d(&ycKcvRicmSZAIIjpGfB*srAb<b at 2q1s}
z0th^u0uR^&0Eh1E0f4>gK1KbydP}{mzNqdS`ToOMrsNs{1Q0*~0R#|0009ILKwzr~
zs4ps$7gdiEj9JlKE)>>dQ6s8HvWBH6qlu($M(j|+PDC at wOw<^i80u4;*l{~#N2AGz
zo{c8Ldc=s at dLkT*>mk#K#qCJePJ|=I_{5+aI~mF)6A??qv7>Q4lCW~(T_~eRqVZ%T
zE>;uHn#MB|em8bF6w4TP&eC&XBP-I*M)af|OzPoKFqX{5vQZ-*Hzp<q+}LI$X(nRf
zupYJ&BE^W6)H8NArrX)9NGBP|C89aw>4|<fb~X~Ul2#(2C*yHbkHj)TJsAnvddy51
zxkxM*wv3o at WKa}tuNqd=chs=@y81=+o9dUgN_mqH1Q0*~0R#|0009ILKmY**ItZw5
zC}Z7R8W{Cn^mKDkV91Hs&GmpmCt^330{l+IZmt3hI1#(K0MPF~!tX$bNBBK1`wwXQ
z_kH&HfdlRR2YQ_D1J4huUsrri^8wj-faU`oDi)IvKmY**5I_I{1Q0*~0R%Rcz{Ame
zKom)}kwiCF4 at C0;^_$}SpXLLbsv@(o2q1s}0tg_000IagfB*t_BCwU34+w{`ZY}`G
z<^zv+Xg+W!lq`z?0tg_000IagfB*srAb`NbCs6yHKj-=X&fxrfo+keL#Q$mW-x+^J
zj&B!XwphrCpySX}jqT2I;tq-b&hG|Huaru5p)zlmmfV?&XR)|YuKm=YoJN2E0tg_0
z00IagfB*srAb<b at AC^E?CcItQUw!AC_dA1g-tG*qxq)KI(3atE%kXz(_#GMUaV{8$
zAt1v?Ww=*{`((IZhL6ecfD9j(;S(}EBEwM`j>+&D8IH?vLWa5wCuMk4hVEAJiU0x#
zAb<b at 2q1s}0tg_0z(Xp~%`*gzX9Uh?oZl<x=0SqSgoypR6T6$I35wqo7;%47Aesw@
zv&o3~sf2jK(j%dmshiQLrQ22{o(tI#BNxuOzbW7m4+apwCGa}GDe#b1TC$A*0tg_0
z00IagfB*srJX`|ZJ`_Nx{C-!zDIg-=>gIZY^SgeZ?&eZ}Uqqbi<|;t-HwC)605IbI
zra+X30ti|L1Q0*~0R#|0009ILKmY**x)i8&=6Cx4UzE#eB6QmSzx}@38{oeZp)>x%
zuZtIV=yiX;?|)N#@7w$S^s74ueC<EsciPkA_4W4k5BLX%whfP{+js2T^~ml$fk*f5
z+yB^s$Dhy+9(wYr!%rU>9eZYcLZ3VuG(zD>G!{=Jk3IX5=YHz=^HVQOpGciN_2SIw
z)7QSb^TNw#@)cXl6-!HIMGGt2m7X5uc})|W<2<<Fw7Xf=*GchO>&D<r=%xoik1`q2
zdYuK;?>o!9p#E6>PwL;O-yeR-|F_$&^#7^<U-y~)zvTOco*(#L@*M5?74`3&Xb<dg
zqqw6#aQwLE=2FGXEZX9!4HYwAuuJ8%Wna!`?Q+_AkVAf9%^5qU=TcMiDQ$l0g)=E_
z&1l-_peBaArOnLFr%t8jwB=HM$t+#fK5k#trY_E(o0%0UolVWoYqRIX|BGkNOjH-C
zCn>im-IPXaOedxJ)W_NrxNI)2*wq<sf+x?-rDjgeYA>Z;)kZs{KBmp3PNwEkv(u>y
z9Wtbi=B+X9+^lvYbtWY;K0S3|dg?^VP5a{P%*z*39h04Kaym9R_VUiYfPLJf<O`O4
zrTmFSQCrjIN~KsGceFbfK-ySy*3KLAYJb2mJU1__QHZM-Rc}*lccZ-4EU1FH<Q7O<
zQCF at m+x7ZUD_hO7bLProMGLN5d`_WsEVm7oSKgY?ob3}OHNCwr at QSg4lIl=I+uxoY
z>E9iX54?N)a^C)AQ$dKg;&YWQ*}0ArW%+z`CiO~lC2TCEjV_mpZ`s+3%Tg#>wtOeH
z-hC%3ocpHw5g5~^E;yg43!>gm&x_Arb?Ne?liNlb&Et&?Yl_sED0dB$_0_DNRb6rQ
zW2~K=J9l<<PTYB#c3OP?o8~&7uSRN<dF#2xSZ%fSDPoV%l1wdUP9nAUtw~JE$?Z!z
zh-GK7>YKJ at oM~c#8^^Zw2aX=~+<dbA!zMPUbx-L|7+&k6=FVRINjpC`b9QR()#?ta
zyFKHYpR_yMCEm@?l)Lkr8tw~R6kY9ZyYq8xT6TKwq>;C02dg`~cTctZXUy!!SC-2n
zUTqgDrwpxC0&@O`S_IXd-AAY9reB<z8x2LHV{JRVY{@n&`C=h$nc{oP32`i)pE)ZJ
zC-#-)eCcYZIrU7Yi-pCjx7}$Lmn-?D{HJUyU9J>M=E4KDn;q(MH!Dl4rBcXg+iw1Y
zf&Re3gPxynuN8HuvcIJk9BRxsw7UGe{c<Ar39I}Yd*V*tNX%09k~r*VD=Ved1>r8*
z at lfc_vX?K}OLoazT(ws5mE`2q#WVA at LuKcxRj+^T(aRN419L04Pj}HQSJEqGyHmv!
zJJ4(?FRELy(Da?FvA<ol(K1tO*7Vv}szws>0PQZ=@@-BQN6R;X>aKj<F99t{G*;7+
zikw_tBcZ{uu)i;G>R=apwx%x$gV*=>^#}Iu_1xG|+n2cq5b at gIQd>o371LJN({2wH
zT^<K&*6z-FyuUAST=d6Yb7w6!srq3Q;lTBu_4Nl1ANIUG<z~e$Rr0xfR at A6+w&{Iu
zYqpx^eDKWW@`VMvBtEi*N~cfTIv=IAzKAw$ai<h(Oie2mUl-%OeSwz_ugTSvRe{zt
z)_3097Z at M+yjq2mX9Q*Y6DxK>oPW6|2(?jP3t!vJ(dsuj5q0ePw8yPirkiF~>@m!8
z^|hxph3d2hy>pJ{47HQAZtK@&amMI;;+qz&eSdd9rYp<N;mt~mbMo3DuD%~~W7JS(
zQUY|mXTAC-vLnu_V;>POPa5Loktf8<t{ubT)uaAU5q}H_Ab<b at 2q1s}0tg_000Iag
zu$2V-9-rdzjlBNHBYjSn1CRP+Mf@=!fB*srAb<b at 2q1s}0tg_0z}6G+c|HCS?*DJS
zf+a@?Ab<b at 2q1s}0tg_000Ib91^gcGUccMF!8!jIo&FgRKmY**5I_I{1Q0*~0R#|0
zU<(U4?f*Id-@*k<)(}7d0R#|0009ILKmY**5D)^~|EC%tfB*srAb<b at 2q1s}0tg_m
z^#!>9zxB(Q93p at K0tg_000IagfB*srAi({9>Hq==Ab<b at 2q1s}0tg_000LWIfcyVj
zzl_Nt0tg_000IagfB*srAb<b at -2bNzAb<b at 2q1s}0tg_000Iagu=NGF|G)Lim>eR2
z00IagfB*srAb<b at 2q3`yf9e1N2q1s}0tg_000IagfB*tpUx54nTfdCSAp!^>fB*sr
zAb<b at 2q1s}0^I+n4j_O40tg_000IagfB*srAh7iXxc|TP%a|M at fB*srAb<b at 2q1s}
z0tg_${eS8J0tg_000IagfB*srAb<b at TVH_t|69L|$sqy=Ab<b at 2q1s}0tg_000P76
z-zhshHx#v3{k at Uj7``&{scrvq_$B|}ZoAU|r~ZH4XZHV+?-#tuo*#H_c#evw5Bnep
zF4)ICO1 at y(SIVDQ%vbERxl$=s$J5zjp<?C>cB!1U?92JAU3Mnr@(XEW%~?Bd6nFFo
zjvx2jT&kFvMSIPNYtGm)J(rrAPiga0FPuqfYev&X2Q at L|Ep29YK6NTJr!ANAOJ?b+
z_Hp~FHg$3S+{~;<>1=9tUYk89{$D(EW}>=CJxQ6XbW<9wF`bm=Qy*(j;Ig^6VpnIl
z37$MRmzp^>tG$$ZRU7S)`j|GCI+>bF%}%E-bjXl4nzzQZbF<os)R~mX`1I6;>8TSb
zH|>kFGcR9Ebxd}`$?4eO*!1?kz$?ZE3b8}!ZND+E_6H2ZbMwL)#a7Q(y)vk`DX6<q
zLTeUH!CZ2SpsjE#SC{R2ji{BTX4yG&WwD|K*DWxo7&?~J21_V!O=!;ciL!ZncBFrI
zJU;O5 at ymJplTC)=t at A-D*}0ArCGAINQm-`En8s4t=yIv}mYuD*IVuz_TfP%3cHfEG
z;l8QX*D-DCf>Zr3h)Om+FFq{QrOT7fhBne at 9&c<|Q>4a3xi6TkuV(eE>WZtMIql@!
zxwETt;?C2w)8fP0G}rksG*X+)ThBGdYOAeJ5g*@{WNJBc5~;m!O=40`ZeP+tEIW%;
z-?S~`OcM*-IJT`naP+9>=9BH86R|;}MC+SX+gA>+^(k at Vul|snpPM;5HTP<DZ_?dv
zY|RhJo$bHw=EuR^Q%w!`1uh=#Vo&ATv>ayKNh5F14p#S5 at 1AP+&zRYduPm3F3P4U7
zTB`)){13GVs{5&rPR&ifI5jsKibluU_E*`GZC3KdLfSGbwst}s_~vKM%6-3mWjSBE
z+G$Qblj&k%@#<~&cg5vOekuPc+e()!#ge)3AnhTihq^p^$<k`66mr_OoBv>-KXCA%
z=jYpNMIEZ_Z>a@`8Z!>9F8^-7NyvS|D*wiwxYO4Mvy{CgzG!7DE2Y&1;V#<oDCW+x
zmoM2%cF9~^wN~-{;pEiCGxN2hT<5A)uYc{)%N0=rb1Sz`chM|Y(ko at VQ^gcJ&}=C$
zs#~$p^aZK0zg at M_GE-~T^xF5IMiTPa>Mq#w)k_vf%R#)lD_{5c-I7FOH7%*g$>lW?
z8XODz`vRv9cClw``tCJ&eScqnVDDbfjUBapnR@^cul+5xRa90nZFN2E_E6E~aiC`H
z?ySfA`vS-Ju9GFPN!1Ud2nVkJtgk<C_^{{gDK{&2sglp-v!X_ovrX at NTeH<P=YwZ1
zmoF^XCGn9hR62dy*7+!{^&PTli#w%QV`^Hl_@)={?F+ngculURtO~TIvA*-(zQFjn
z=hZ5lJO?Y=pIEUA;)KgRC9944TKL*#j#j_PiKt`Or#)`HGTk(@Vvk{#tFJw+DO9I5
z=$#WPXQ-V~bz8qKi&I4B6W_FG?W?!@F<n`94sTZ4tke&2_5F|=qlPMz5}@Nf>(xIu
zJ88oWz2ZcEz&V*8?oTNr%4ucW3;qA5|7`E6zW>)d*7Kg{|4<i3ez5Jo+_qaeJ at iLI
z^FzZ!)s6z|o&LWac)UL_Hs*OJUquwBmJ8zi*Idk-WxL!u_IL~Gq1Krl4lmuZu2ny)
zD_Hxi^#ksMYbDUw1+KlPt!+WePSVZy8oS62yX(e^T9Uc#zV>=(Z+{?v&~tsEdYPhe
zBu_U|Oq&HOEsqE8dvTg0$gVVE at 0F!CGkYTSv8L0FZc=GIL8+gMi50l{lxvnF&Z*$_
z;Ya%eFN=+i-fp8;ZRn%jZm3-Hwj1Yc&&~aT{@u?$JMa%*ZaXm*hPnKrsOqh!rp=S(
z_OzdzHb<$GG&~v@?z_8rR$5zi^+dJv31=-z%ei8sHngI(p8Gj9qG~c(k1tBPC81gj
zCz;y&wGxr at J0#(p&sR at CMU?9M)+Fxayw%;>b8{!eg|Zi3ZL98X=FH{g#jELJ$+Amt
zM|SrI!eP(5s(Uyu)-MFsGFO)<*LC)^>;^kTXdGy22bgZZXx9 at HX!WC0>mjD0x<-Vi
z?5&E?eu1dAFKk}#M6F=OHjKGdxaF&3bKI64T75=&y$gLUX*Dv_l2Q%7b2@`#sXcvx
zSHo+52U%T;Ehkg8)B}V6$UT@|dZa&)6uT^M`!1`tV{O@=?p|v*)fBt+Q>Kl|S!3_K
zysLkAGCA;>skXgQ?YeX8&ZueX8vCNA7+vg&+}LiJI^P%Fcj9aC-QQC=-;uQDkDuJq
z&X?F~GaDOKTiz7A=&aqi+FZXXTf071-})m*WNpnBZ at W-+pX_E@EavPp+OONmEpm4!
zXHqBUwR0Ee#XhC&3SQd{gVu6yo7uEibyF7~u;vSXHH6wOw{2CeX}9h%Yinv+MN2B8
z%5QX^TV1<TZ*O;{P*FT*@AADBk>qDKaBn4kXuVSs`8Bngm9)`3cIdiC>kph7^IU(z
z`Q*2bU6C>6H{s at QOJaA>I#X-eCA7|KThJ-R)(G{_t+T&<_KCj0xv{nOY^@8{Gb1<g
ziSYyLok_ at rrNDugclHJDa<Scf{juu#*w623I+bu&`&i2{Q=U59!6`%2IccX;SLZ;#
z)>%baTp(O^V08Bd%PX11e0J5$_A5wTT-fS-P0C%%+Lt?5mKTer)q1_D?ffxoigVf3
z-woO~alLC-&W+Esd!~BZtjZn7+FwU%-yj+hC$y%*>GTDn5w-oG at 9v}4I_iI7zqsKl
z4%2G;VY-o^*v_UipZzT#%f{D}4j;+J%$ASi^*#Ig1Fwj6POY=f=2Nhi_3!Iq{cQ`k
z?&O=d#jQKFFA}u>U+<a*^AJD)0R#|0009ILKmY**5ZD?5JpXTNlq0!7009ILKmY**
z5I_I{1Q0-AJpu0jucyO21Q0*~0R#|0009ILKmY**wuS)b|68LR$pr!kAb<b at 2q1s}
z0tg_000Qd?aQ?rZ4)YK|009ILKmY**5I_I{1Q6I70-XPEjdCOx2q1s}0tg_000Iag
zfB*srtS7+v|9U#iLjVB;5I_I{1Q0*~0R#|0U~35Q{Qs>{j^qLX1Q0*~0R#|0009IL
zKmdXD1i1gdo(}U6KmY**5I_I{1Q0*~0R#}(8Uoz^-x}peE)YNf0R#|0009ILKmY**
z5Li!u`~U0dFb at F)5I_I{1Q0*~0R#|00D-L`!1 at 2yC`WRE00IagfB*srAb<b at 2q1vK
zdIGfnUr&d52q1s}0tg_000IagfB*srYz+bK|8I?QBo_!EfB*srAb<b at 2q1s}0tl=p
z;N1V;>G_G``N_zwk&ln~xBbb`e>V7=ga2sY#{<9FKi9vz_h0qC+xwdLkG;2gu6YhB
z-|79wLoctF-u;- at b5r@~a{-T%FIe`K at +TJa6+3ONREpK{w4!)^>E-dY<}R0tx%{GC
zPG^gSTz(;Kv`vWK7!UUc#>YK3hbv}g(QcdFHX50pOHIwEwE3wQ&ZM-qncC=}CWgGF
z&CJfHPNn9w^K&z2r{-SOUP`^HO<kNnH!~}iKbxAJ*JjU&{}<1knW!!z6G}HN+8W{G
z_El{@^|AKlE}M%hc6ElE&c)f8moKKYQJLO#kJcYJHRiegL{%!kWG)B at b1`q0MJOV@
zWoIj5rQ+{O>r74Nb7uBL>SJ2#ytW0kbF*y`s_U0YkKH(E^al<c at Z8+p!ds&YwSXR~
zO+NI&Eeqz7OVVa++2w30zg)=|3r%7TgnFU1$~Te9EKO+6D#l##ljr7AGpA;&CG4=*
zvaQXfPNwEkv(u>yPBn2Rm$lKnHRf=gNS#TEN;ExnVS4IBYH%zZ><gSaxK<4=F54y3
z+3a%KXuLOW{Or;Gz`=u_cV?=vjX8~X!B*VHj62|6SrL`ALscvkE!(NT9cERoF59bg
z1;WXxi)ZGwVErQ??P^h5tF*J;PUdpGW6j{sw6(PFVzG@)_6647Emb!n9jXRL$MoAr
zHL6i3Z<rJPfk43X&cSNARwq_p^p?`BPFnX<;^ws5k3+pYjvR3pG%c%Sm&=_Bw4A+U
zFPXJ+YWtj-%lWi at rD7M#PL*9F(k0t0RWi0&(N2in>HN&uRDJ!^7w5$1W<Kpqyf8m?
z_Po0}%XY!a7Zz5fS6@!nquS at S=`&iJBy%)2IQHy#U*O!>S|7NAUHN3O^zm}KYMeG&
z$0u$qJkuW-8}q#5cMGT`c5B at 6)*@?(xo+95d$f at 2So>(<Z1>u8YaESa#2(FEqT>h5
z-D;6u>t{@6W{sU%hmTop9o>FX2gjZp>kF)XWUo`#j_hp{o_TwCw0dNJ=jGOda^ttf
z9_w6GZrpoUR<7Fy5BD8PazYbnJYdCn^hjS|`oL{Jb at k8N==F0?SHC;l_(V$q)=KV3
z$0ECfvf9u$hjPWN-2Sju|FAh<83xC0oH^VdIC8{u^Ob6j<QJ=Qaix^CTSlI4$y>{;
z51zqlGHdS`%693pUFx*M7vEqi#cXkL)!x^wu35WO$>;J}2c*OCzxsu<(~OocPqmF}
z*~_>YUt=F5ILDsq4;($}x#?-<6k994GF9?+S-LeQ9PY&0nEj#AcKiyoq0jmqTfO;Y
zUm(8M^RvyT0wU+~OP8n>@)Y1HN!0SyzcpgxD}T#Z{rI82z)OeM`h3f>kjK>Ku|qdr
zJ=h;OeAshyrMVcZF`MHZ>{NKwSRY!s$z#?U^}hDuu5P3dT=zIyi`0GnzAML2?*HG9
zD;BI10R#|0009ILKmY**5I_Kddn-Wy|9k7r+7Lhh0R#|0009ILKmY**5V#)#od4es
zb=HXh0tg_000IagfB*srAb`NV72y2;-g>h(1Q0*~0R#|0009ILKmY**?uP*V|L=!7
z>qGzn1Q0*~0R#|0009ILK;YgAaR2|_db2hJ5I_I{1Q0*~0R#|0009K<hXD8g?}s|;
zL;wK<5I_I{1Q0*~0R#|0;NA*w|Nq{4vo-_}KmY**5I_I{1Q0*~0R--c0QdjzhdS#-
z009ILKmY**5I_I{1Q0;r-U at L4|K57DHUtnr009ILKmY**5I_I{1n!3b_y6yQI_pFL
z0R#|0009ILKmY**5J2GG3UK~^Z at pO?0tg_000IagfB*srAb<b at _d|g8|Mx?kbs~TO
z0tg_000IagfB*srAaHL5xc`4|y;&Oq2q1s}0tg_000IagfB*vbLxB7L_d}g^B7gt_
z2q1s}0tg_000IagaBl^;|9@}2SsMZfAb<b at 2q1s}0tg_000Q?zfcyXVL!EUZfB*sr
zAb<b at 2q1s}0tg^*Zw0vje{a268v+O*fB*srAb<b at 2q1s}0{26}x&J at pyQTE)@co(Z
z*2q5 at K0CZ^=+}l`9r%9+{$bw>-oNenGv%KuKdt=qz~cjtZ*=>5zjyYCN68l~`%3u}
zi}{M3HdiXe>Ug?bDVEFyJH1>iE|$|q%jlG%^!`g{WjCi^=-(ZU4*Z;cId6Y*xm0}1
z&Q{9lm9lN6Ggs3kJJ)I2^jvCcKBawhCiRNeX}&gEFqiC!K}`&o&BYaaOq;qO#xA7J
zq^9TVi%d3REtm3zN^bPXQDZWAG+QiG%zVKvm5&}SAD1h5?r`~tn_^>jO9JkuS0{~C
z*L-sB+*z#=Sv!4hX12a=?OgqRI&VE!%f)2g(k{-<oSW at _Yx7G&+1XOLdG7dRE?-!%
zOVxZeZH_zh!`c*)i(G!Ox-qNSYBRe)t3~a0J8$fm>fap>54?T0r4|YMTz;XY5;Z1u
ztV502%~j~*_SMyOY2C^stFfEhWPSNsDW~O(YTc~X6|GuX+UC1+)~YsTQCI5qrk0{i
z!KqTu_XS=aU+cr!wx+bvHsSb<iR1l&@o~>hwPI!#?Y7x%qdz4v=ciscld4y7xlP(=
zH6Nm4&&<xJPNn9w^K&z2r{-SOUP`^HO<kN9AB*QAx7pZgwUP-|H at 0~~^u~DD-PqyU
z#x~Cu8{0e at Y2Vo9ncZxxSpIBkc3zu3C;neNbH=S3axFr>p3l|@q84cLsgJcUSFHop
z8E!h_)Bp0tls4K(Z$cA`j=6JBo|{X}oSLoDHANZI=29n9bE(<s)CI at Cnx)aaHRkMQ
zPKdpp*jr9dU6`Iaks2I38tMy7A6TbmxFAlIsJ;u|eBmSRPI_E^yxa(_JLy%^+CE^b
z=2x at m?C#y1HTK2b?y=piYuTOL;Z9t9Eai?rEizv{6t(ZqYbez^eY=llHP4eWDN!>T
z3$?DP8m~H2a|*1(OeYa1mONIt33pDgQ##JBqEo8%X)PO9%ey#ewrxw5q|*u-UM*{C
z%6tCV{=l;bJlFfH-wtYwwdKV6#ov|M<Y#5&nVCJ2`dGa}w?&qR(5gkfGKo~iZd%9O
z`dVzMua<o|pS8>Kcu-v|-88LZeQlawUqyR;b>p~GD_580_m28bzqvZEf6#DCwedl4
zzP at OU+{<lgsqpS+pq7ov#_HW=vqe)Jj<a at cS5dCTtQ{}vTY2P&+{4^<k!pV2+=_*q
z%1}FYwUD~C<V@=1yms#5yx8m4J}fP<YR3ro+fb)j^_uTyUzD$N&})e)YPY+#mQ^%Y
z4|%9;T}mb+QfytiTFK?o?mnPtolQ0RY_cz4K6~5!Xsrr~&zkdXq`q5iD~e+`%!K%a
zJnOmnYU at F)b@}$ZOTeV8z4aq-OIB;jT^xx;I_n(V>Sb_uk8$;6Z+ob#$Lx5-tC4DJ
z9r}*P`vMpAu8V9{StV|qjP(a}-E;GJ$3ki^o%p)N@*#fNaCRb{zHK-=h*f2$S$57`
zS*&P5_ha4ZD~OZe2lx)MW=Y&wh*rO{- at MwgkCq=2_q at -o!;MKD_t1^mYwn=yJGl?O
zE3Ox;ySkRI^v#<&SzU#@KWW`9x(l}MX{t-Exu24&y3^eh?f*A=RfG5lAb<b at 2q1s}
z0tg_000IbXJ^|YQZ at w~PixEHo0R#|0009ILKmY**5ZI^y_y0F4MtlSiKmY**5I_I{
z1Q0*~0R%Rm0QdhlUzxGR2q1s}0tg_000IagfB*srY*c{r|BZ?f9{~gqKmY**5I_I{
z1Q0*~fz2mC`~S^XW^6G62q1s}0tg_000IagfB*s;72y8=M#YGa00IagfB*srAb<b@
z2q1vK<`dxl|K=+*wip2f5I_I{1Q0*~0R#|00D+APaQ}a!V#G%P0R#|0009ILKmY**
z5I|t_32^^^^OYG}i~s@%Ab<b at 2q1s}0tg_0z(xhQ|G!Z&;v;|n0tg_000IagfB*sr
zAh7uahW!VW?aE&&>b&~o at DGOnyKTQR_?rJs|Ka|F1Aowaw(na#zt}sX{AJJS&6l(G
zExz%_OZ|al(sT1-#mp?)<w~(+F4*bI#l at 8+yPVDz3%UG4=czN(bE&EMlr}&0!kLuT
zIgU0usEHwOX*09)sZ*&r?fl%#*{Qi#wU<(_YEu{I&&|w=WX`5$=e5~$;{U}nXC|tP
zw5FGCN})C0$L*`yeClKEi(fVuSM2HxH at S<mGcR9EX``!>ozO%IV{W9A=jKu~r)I0$
zwmRmRHkUe?noG at 2r!KT-R~yY+W7 at e{?L_KKN|;PfU6`Iaks2I(=}ccBo%AUAf at NPR
ze_}CTvD4;CrC1$Lw{A_1J8g8He)?u*rav$~?)mvAIu%yiLZ at 1a=TO_+Lm#|k3g(h4
zyGDsE7mJH)msg=^*=x_OTwS*7Ri(9{ExVj8<(Dh at Vxg&A%LQ|}e5qJjwU8)@lT#PZ
z%xginY?n*Lx9n_X?Nzte5Sfh$O>Xgo)<{B36bKWVv$^Zl9hpYQn&ZqXuTgJ0R7bfL
zElD}kHjqSP`&&|xlh;aUaO}C$eSw$9yRF0RmG{NB2T%0}!eP%l&$iUsYW~yJEY#{|
z)1*{O?d=fZgV*2Y#MZ73(tB0C>byIV1zoQ!4``(r980~}7kD*%yRvL5h{n#SY2wMZ
zgQ@;NH0pWR+o4Qb4;t=b?zHK3O4W^VkBikU%@24CX)pBK9!lf_EuX&T+PKz7 at Gk5Q
z2FFgF><hdW?XKkAvS=@Qcj^i86=J~o8j(EF7dU&Qy9Dbyg=%e>x_)K4KXBxT=k24b
zt4|Ba^J`a_4_;H&I)GWF{AIh;>D$B_$DWE=S$(u>Kk(F!E6!TS2FGqrztF!s8XfpK
z{c_&^WNqJ at UMbsFI&(E$vU8oLNrsQkq+Y2VhRV&$Xrt8wQuUit^{daAHg!RaT at YVK
zrswO6Og3UIm-2;5ZuH1eV={PDlx)S!7wl5`=;88lxdG1|E+27gUSoDk0 at VX*xjJdA
zy5^H}=gw-4$l7V~?XJFV=i8A?S?qP6tL1W1e6tfo=VrU#+WeCEit22s+&p)DGM6tb
z*rjT|nl{Is`C)B}$VDzMzBYGctIg~JtroT0?YyyLs(*JlJn;6}mRjU~LR%_PV^YUD
z)R^5|g~SfgE&1lUv~Fc`>w-LAn5-{fE9KgZYI at bWqE#zP+kAJ<TGgg3>Po%d)KZiw
zI92NT^}eRGttoA^O*npI;&}DT^-Z<9PimXpHu_U7`=z#-AAHXw6I%OQNtZ{emJ^vq
zqAdrW+D5PY6|}bVYNX*F<Oau%{#0LJ`oLOy|61Y==I5 at DJ=Y&NaKLlpaEos(=0`e;
zesIb)ck&%iBWn4SJ89bgw_hh<903FnKmY**5I_I{1Q0*~fvqP%`~R(1rsN0#1Q0*~
z0R#|0009ILKmdVu0q*~|i!hD=0tg_000IagfB*srAb`Nu6X5>;)+<wTga85vAb<b@
z2q1s}0tg_0K)V3<|Jy|vM*sl?5I_I{1Q0*~0R#|0VCxBR|9|V1DLFy_0R#|0009IL
zKmY**5I~?^fcyXLB8($|00IagfB*srAb<b at 2q3Wa1i1gd^~#hSA%Fk^2q1s}0tg_0
z00Iag&@S-#nSy0s>Af{?mzF%Y=C3Z>wHSRjZ=EnJ=C^+R>3#p~+dTul+plTb=~2(I
zJ}Wbs$roN2^68UDL*ZvX^4w1yKYi`nJx)skpZYaL{h|6j^&8?10|E#jfB*srAb<b@
z2q1s}0th^y0z*FU0snHTn9DEP<y~IyxJQh<WoIkpbhcQ?<rjK}dfXV*iGGiFufLqV
zWG|V{`M*c~14aBXAb<b at 2q1s}0tg_000IagfWRgac*f&bb`S6O^n?cr#eyB2to|EO
z-U$Vbq#lgwMpz5Rj)h{!jEK{dz at z@1BK{Z<KmY**5I_I{1Q0*~0R#|0V2cU#dw2Vt
ziv#ZY{|`m`|1Fk5vV#Bu2q1s}0tg_000IagfWX5pFzDUym#qTa_W%DuQUAlkjzazs
zKmY**5I_I{1Q0*~0R#}(v;t3fdX at cQzh&plmBq>lH_*;}?1Uzt7U16h|AC_ZVAJN2
z4MzY01Q0*~0R#|0009ILKmdUb0z*Fk{@Nu0_x%4Siu#id5t)Pl0tg_000IagfB*sr
zAb<b at n_j@{_50oS|KC>BZ*ThSkO2e`KmY**5I_I{1Q0*~0R$defyX^Q(E*@y`+w)R
z0D1o3Lt8b-I06VDfB*srAb<b at 2q1vKRu^!d|0gdAIQRd1)ZbOq?}`@&1Q0*~0R#|0
z009ILKmY**5O}}^_Vx5C{-e8dwpm#z+2u+3fWLC3v|>Lh9s}qfZ_LXtnG1Gxj&uLN
zNBu)Z{l0i%KmY**5I_I{1Q0*~0R#|00D;XTu*c^=>RELmpl6rQKklzjcCP_^ex_jA
zSNd+v+odJXt@*3V_U6eZTZ#Yz2q1s}0tg_000IagfWQY6_)WQ3kJ6(I$@iKF2gLv9
z-V`rlxcH_!{*m+5QRijs=B*QE#r)Qr2lxHcuX+YXw_nq=)1#haeO6{NlP|o`=kpJb
z><OH{_EpdD$ZsgSM&4D_AFAI|zoC9j{j&N6^*!~vdP%*Y#?*235!E~LHzVH}`Mr^M
zM}E1!2Hqfm00IagfB*srAb<b at 2q5qQ1Ohv~VNbSLsF?YJT`H$*ZN+<b?(&ZN%avkD
zv=2{TE-tPt*=3Rbqy7$&ckl2XZB9f)$>kS at hqrqVHbvjAdXM<sd(G)`aix^C)e-OE
z+LS`kvdg=NMM5%)*pg+tRLR at rUjH`lUKwxykoQ?nZ6ni`eL0`C%V~FoJ-Y_Iy1#v9
z!0(OLGqPMPF4m&&8SsYNW>hy~d%xJuTF$Er`1{2A-F1xgdJlS>4KpkGVxhck$mcy!
zU*#^Zcih=H=W%+qwe$@2xG|~|{T`80IeW=oGTrCx|Ei+?>IWz`7C`_31Q0*~0R#|0
z009ILKmdUcCQ$Mp at eB-H7&t!cKVGh^6bkvmg1+>MXIN>>7;elcF4{ezppn#rQQZh@
z!Pv2I at L158OeP~nFcFG{$Ae;AM2H9VpdtRW>bRKX&T!8E`$tYI>PxCV@*hY3{m9ow
zJ}*WY5I_I{1Q0*~0R#|0009ILSVQ2|;XcJ5zCJAO(u=#&i}qy3EZgboZS-h77mY?k
z5!((0?PM+zF>-b`o=awORwfw?<>JAFWraeSSSTkV<dPvP9*r4sJDH6n%&-}IVw-qe
z+~KimT9akFls6ZhbKYRo2&Ikiv2bF~kce<(w;Q2kFWO?&wzXjI8WhtH>~*IX%TtTX
zqJ2TaFD4#+*_|kE_h(D_W#=}!J6Cg-)669+7STpq*E06lfLPK9xJ#O)?4^7~++!C{
zx2s0K9_n`lZXB-*<gF!hd9q+%sl4{-Lo?--ymjc9^Z2_7F|y!{<jlpg7<;i;t~jGc
z&>1EU&6b at p@#d_#Trvv_c5Rk3`)4NVY01T9Vs6SCOY05id3*ipqN4uK>Q~f%q5h)!
zj=HG+`5HycWCRdE009ILKmY**5I_I{1Q58bz at t8Y*i*ka(6~G at yu;@|>~=ViS2wop
z^ohIuEtfNfhJF46a#FE8Sbe14s!JIDfX{EpNX^$L23l9$xGvG}w*UWoMg9BRZaxtZ
zKmY**5I_I{1Q0*~0R#|0V6zGIIzP|n_dEChJ?f7X at yCDw0tg_000IagfB*srAb<b@
zTTY<Y=l6Sh+~3<9>+pMf&&VGUIIr#d-20w^vGyMk==Hh3A at DxGA+Y7@1{p#C0R#|0
z009ILKmY**9%O;FenY@}=f5Gao8J%+zofA4ZwL&G997issK2KEWA)SOPmlb6Bd5g#
z1_Tg5009ILKmY**5I_Kdhgo3nZqf2QtkyfiyS>q!?(o6up+`2~BlKR~_2E7EaHpVJ
z_rZrdM1)QcKHM&*uX^yIDkgrg2Oo}zCD(lLq0@(8K)s---&6lr^?y`<UHv8T6u}GP
zp at IwuAb<b at 2q1s}0tg_000Iagu%1At=k+;PNOpIc;rzhBBds$U4;S3E>fwSrS3g{E
zN6W(nw|9QHpt{Dx1xI*>;Ce;LJOmIx009ILKmY**5I_I{1U9)quU9k+;2DCOycF4d
z1Q0*~0R#|0009ILKmY**ZX-~AhM?DdhTt<Do*{Uf#hHx&0tg_000IagfB*srAb<b@
zLg2Th!ggi6`pzfE2W0q~41Y(4-;v>8%JBb`;rC_uS2Fx-8U9}~9Pqer{4yMr;gAfs
z$#7VPBQjKFxLt-jWVlm?yJYx?40p?Lj|>Acd{l;eWw=j<`(^l;3=hcgaTz`#LrsPU
zWq3%2Ps;Eq86KA5(=t3FL&-pZ00IagfB*srAb<b at 2q1vK!z^$;=24aF!w0<sp5w<a
z_{9$gM&r3?G#ZN7b|`2kbCHOVv$OGBGLy41$zUiK4<;-t6w1UxIT0b33|aAL%!u2`
zY$RcZ&Di1<YqDaNCa-*I`nUd|CloZ2dN8URAuSj at R{a~Vz8cARP!Ag7uWd$siLe%o
z9Se(vjLBp?8H_|ip|HE`>wi4*7jDahr)in+FxOu4jsOA(Ab<b at 2q1s}0tjpZf!~$e
z*rW6)&QtcCK@<O-`~QiccoD;S5jx{PwOk!_UY@*p>x5Y`zZHCJ-+ONk9NK<O(@u|i
zj`dlY$xOcRg4Y)`PG5V|d2HX18dKCCiU;?9L;afiW%Uc{d+K%dl6ZK3tezNe5I_I{
z1Q0*~0R#|0009ILcnAgd4|$*UESHM8{Gwe>TlVFAR)pDNA(vn1*){0Z{p~XYes9!Y
zt`tk=f}LJ278h&L_Y8Q$Z8Hi*%Pw#4_a14T?(g&N^}FjB>GdA;6qoIiS;-d*<!wVg
z?}7R%cX_?z&c?lEXDhX}^bGa5F{%^&9+6Txd&yoh-N*KST~WWT?sNZ*)cOfLgyl+>
z5kLR|1Q0*~0R#|0009KHf`Iy>GI>$-Ru9IkXf78D>#?X2)gxKM(v#6dQa2-ZC}Ag}
znPetvjEWq8*NGjsLv}QpjOf{DBCJP at sI4c$!MGkWjab}{WbH&aVmLkXzv9GBhH}Y7
z#1e7rXk3pZtekik%IJ}3JQ<0L)r7OA at r;Q587Fo)6w4TP&eC&XBP-I*M)af|OzPoK
zFqX{5vQZ-*Hzq{v*PYmABxxpM;jkXI5+cQjmDDqKHm2LztVkyr$t9vW<LQZQeTs<f
zk4DWzFk(e at kxoqHG8h!O6iw`7h7mJzp^%vgCXE57L4im8rlNjReO>(fq6ohvCNUs@
z00IagfB*srAb<b at 2q3Uk1=Kf`v2HF6i2VP4Hx~tlM8sR&Tn}*0!9U&2r2xN(IM>Zp
zfPqo(MNc;u0JgcE<Dcx%IsPfxe_;90zF+#bXW+^94g~$a#||7k^wi<g*S_s>S`mz>
z-Zgp=sN?D*VkQFu2q1s}0tg_000IagfB*uUUqBu49`@&#MDzY~y4rw$_ptY<M@|r}
z at kL|(O5QH_`nQR;_wMAa){EdJMSW at W7bH1A009ILKmY**5I_I{1Q0;r9td0yH9uy5
zK-m<J*>`&p{98r+xA#DZRUm)>0tg_000IagfB*srAb`LI1ya5K>%({aOMdQC1b?Kc
zKic5xiH-mQ2q1s}0tg_000IagfB*uUK)~nmi6#o}{r?{;>W??UCbNkMAb<b at 2q1s}
z0tg_000Iaguu%b@&tE(L|B0ghWTT4{9{~gqKmY**5I_I{1Q0*~0R%RKfY<ByJMI5_
z)v}`gx%zwRKT+QmZy69k009ILKmY**5I_I{1Q0;r&INY#_9^~>{ku!{qHUIK+gh+k
ze9ojDdyD0%#pQg#4tt&PCttSga<-ITuH=gacZTM~*32a<7STpSK_eNA8lkiiJ{C at l
z?dx$C+B;UZOL=p#b!Nch%sU(~OW8~Lik+>jl<aCM?ym?w)!|nJ?>wWVfB*srAb<b@
z2q1s}0tg_0z(XPMhccHvN{`Yn-)kZq5dS}N-kmTnLT5a#x#Qwhzj^C~SuwwLUfcJ%
z_dNp>+plTb=~2(IJ}Wbs$roPe^$m~g37o$6zGrykH<Vo??<(pK)$gg_P`{>rS^a|g
zo_bxqq+U>C>bUxd>K*x;k?)NB-pIQnzg%AdZxBEL0R#|0009ILKmY**5cmKBft}v4
zCtEC3OmXYFR8B9Kin;uvUGCkv%RBBbSBfQb!A at T;F0L%uPV at 3d{T(9j-r+skoQQ~$
z%P))$Z}%Q-ioRX-9`WawM0(|Pxwulw+Ukh+aBWJVC~lbV9u^77C}K;N?NTK#w#C2A
zyI02BKjeMZQ`^Y2Wna!`MRM*6dv*<ab$|QJfZrRfXJolpT&zXkGvE!k&8Tj~_I|OQ
zwVYQM at b`)JyXzR~^&a#%8)iB;_RHIbeBJ}~Rqpb7$DNIPOWdTdt)*wE$Bj{)==X?}
z%GpcylIgVG9~}M*Mg8B^|3ST}7S-92_eQRdJUucbCNdy^00IagfB*srAb<b at 2y8w9
zZM0YMAKxL)p~X4lqCL52mMiIU#jMzqd27jBe(jCDM?{RUT8~kkJed&<%!98!EgFA^
zuMgMfIaAZsQ{-qo7mY?k5!((0?PM+zF>-b`o=awORwfw?<>JAFWraeSSSTkV<dPvP
z9*r4sJDH6n%&-~Tb6BkP$nI)dlT8iUuZNy;)_LQ2eVy+16znUN*FJq{ro5834jrqM
zR_qBevfzy5%*C=8d$CxqIHN|;872<RmYp&2=B&9~G7AfKZI(0pXC~@t$;D-2Zps@=
z>y0O#6qz|Z=4Pf-EBalBM1%u--3Ya|`iX;L;?bAgi5qIL|ClD0Gy?9Dt6JtCdP2;7
zc5fZfTwY$hny%PO%c2h0S<!yKVmnXoA5;U1=)3>ls{ggP>;GS<f2N)jcm6*uCNdy^
z00IagfB*srAb<b at 2q3Vv1s?bLk9+DT&h6JqMj!S0!;RUE3nfQdXEd&oJiXKBKjOA2
zs9oeZ+#0uWb>pcWKL24E*S&`D<gm|wKu#%^5AE^!W&4E5rmF=913teYBQ;+N&>r*o
zpKWYx>qUSkocsSC_3st&$AAC=2q1s}0tg_000IagfB*tpO`x||G!Cf!df$%}^+#JR
zgX9MR1Q0*~0R#|0009ILKmY**#BcWb{C-cb`}BUj!_)gG<ue4e{rf)q{D9v62*IA9
zar)ZxPV)g}yP~LHSNFO9M!pX&0tg_000IagfB*srAb<b at n^{18QJK7`dX!+yiso{m
zupWyVQ9Y71EIk=bBy}@lhZ1%onn`A&#;C~Ycb(XAJ7h<r$%vkfCc=8eh}wE09E|HB
z(}>0GNY+k-BgVLh{S_y6GL%auB9 at 3_N8@@VVdccTP)3hL<H<-|tR|c_jb}vc&p5He
zp;*SSbC#Y98(EQdHlio(U{Vi<g0W;amW>+mxG^DOzwX2~BS|w63y1Zvl at KXLtfZc?
zvoYPyW<@&5NG=i08BdGYPA>h?sF?^xtf(&1iHTeWgCdtkQjZx%%*cg8W+s?42AszS
zRL}pvslG1$eNp|U`Xy%)0|E#jfB*srAb<b at 2q1s}0$Wr-eM1 at R=F))3{_l5lQD8_!
zyw%P10OuV1)7 at MO@Qa9Z-CPB5eob(!n+pKj+~xz59hwh_Bp47t009ILKmY**5I_I{
z1Q0-=D}g_ev3isq#V6luB6QBs<IWo~oYPvz#U!Jp?S61o+x=57%C`F+^*<`&j{yM$
z5I_I{1Q0*~0R#|0009IxzrdqM{QiOA0soN4uk3K&W(}v^fJgmLiuhwd009ILKmY**
z5I_I{1Q0*~fh{4h+ka%hU#_h9`^8iLcMSNmhTC at E=+JgQ)Y5#wSk-)hNB?b!DnTX?
zKmY**5I_I{1Q0*~0R-9w9<E3K1;g36cyeG!kC{pF5W-|4r)Oe`tR75c!>04VLBk3f
zgFO1Ly_PVJ00IagfB*srAb<b at 2q3V<1-9Cw|GK#VFzhxT2z6*au*FN8>>+>v0tg_0
z00IagfB*srAn-s7Y_-c2{6fJ4UESC?1Q0*~0R#|0009ILKmdX61^!G%b)Ns<C*Nx#
zbe{kJk at N0^c at a9}vF+}-cpaA=_|I$mKKH(7;F<Ow{Jp;6kv)Oa*WULGkNk$RYvf%;
z{h|6j^&9Hf)Gw=FP~TIptC!RZYD^thA5py{e>3u(k>4A6cjTArE8q<R2q1s}0tg_0
z00IagfB*s?Kp?Qw8}?+2g^HOk*rjrMxm3*M7wxid=PvKKcxqj-WG>k0%f-c&CA%!r
zf7IV0^6nkpqs at tkD7pN?<HOs%2b-dASG`C4`6ZEFIbANUl(M!u;yqlOQYcz>dH1kL
zNJbG`vTT<sdAr=}-{##b<Lw{vKI^G%WZJSX=d&U?cZEH>2EDq!eP+P#jn*@=Tr4iu
zqVE~-hTCRTH)4Cg*v?wcs|)!1#QNQJjP!aBdYlb2EBRufylu$mJy2ieF0Xgo*|@jt
zY^AoAo}nH$Ms=d!BT_17FWF1xu+a363 at hsA)z64?{`2Y{)id(#k#RAJ0RaRMKmY**
z5I_I{1Q0-As|q~!c(39&0 at YLBNwbu_l&^?$$d!_9E?Ke2_0WMn#UH+LT%3oC^Ug(k
zGH)%J%aaBBO69drADStz<gG);Dy0>BLX0dpBRO-iEXH0emMhMv5p;%$L$hUPOuRX3
zE|<*0f?b>C%>J2)dRlUEnV6gM#?pG at iN{2i4v)E6D%+*Jxflu>$zaq7rH$~haAMbf
z5#hjIH$t&IwYV%UK_vExiAP^{CyFZq*;0PlIn#FMYPPjtYgH3%v^BM{J$uEHM|QhQ
zmh45_EZf!SkBaHXchu<?&2lAOu9y|6_}UwL10qIPt;cY2Gh+GRt9u;y>%;YV&eU{u
z9np9$8jXe`wjB!E$y_92<m_xbm(1j>OfneC#e)gU3WYMUP)<b1B|}y`8Z+W{G8;*l
zVKe4D#m}!s6g8s0qkdBToNB9ouKrKz8TCtBwUU!B1Q0*~0R#|0009ILKmY**x)TWa
z{D#{;pyeXT<B$6MVNd-Wy>W5mzz(1Puw1~s1o7Cg&woHpDVF!|@%d$ggUP0g4f}Tb
z{6}OYc_m@*<39iK#wyw`B0Sn at 3!G~Ofz}y~%Lo5|d-oF?$8{fYe0GPVb|iBj|5%bu
z+X!{ANQpbt{N0(M>{x9AR1pf)wW32%2Gs2AjI;r{OK>T|Z7vj0t3VFvu at MS1hbrj7
zfDFWdk)zv#FGenM0RcVw5V(y~)CU7`6QF&wvs6iXm5#)JEAqQs?Vs7-&U^FO(P7{I
ze(#-1f95ZUuM~OZf&c;tAb<b at 2q1s}0tg_000M_iU~oXD8)#bk`+q_Fw<51x5I_I{
z1Q0*~0R#|0009ILK;X~`3=U{oVIW(@Z>Gm8ewFSVf+G_Xe>Yj0**=9};pHpqlUeq_
zc}1K*bUYjo0tg_000IagfB*srAb<b at Pa$yoYC$NshtH^`!W(Z~)&5M=c3Qc=_;!7E
zHC&l}`>($BFaKIhry<Y{S2wLm!<g$_XF9iL;2FAM%Imfl9=*e!G^%s9e3LmFc!A+K
zmStz}{d$NsYgmqH8`Ufz8$GD>7(I9jBHn at k0tg_000IagfB*srAb`M875I<tfv4;L
z4R#+-%KLP^Ki|kUyeaR~=iX%YT;2xV-|mf<C+=P^`P+ZHR|gF9%KCNI{(A(*0|XF2
z009ILKmY**5I_KdJr{V~+JA#PU;A(R2(0}#wzIYW2E~LT9*BElO<Wce<<Iy0<E8-w
z5I_I{1Q0*~0R#|0009K{P(WN(X0M5YVpQvHl32E0bxl`yBD1ast{>>36I*`lyS1R^
zn$wlBAtjBL3+mM!w`%E59NO}x>gho>bo9WInesv4+ks`9=PRRGbX<$AFx_n^*WuG0
z$My7@>(}(y2?NWwl4=|p=1gTIiw=#XYC4{$yOArS{c55IPA$-r(D39=3^z<Hvr-Y=
z=*X%2iBr>Mo1yO13`?)sf$UDrNNmRmZOe|$vz2ld?IyMz1&%x{&#&u_RSosfb?bUu
zcf7=k<syN0Vos%V{|yRT5f6neSO2>&=2`V`4^KgMjQ|1&Ab<b at 2q1s}0tg^*@B-rd
z%DFs41JmlYLY_f^u{1Exc)(~Hm}e+pq!XB96hNebc?JN=S at nL<qk4a?TR$-Q(!@_L
zm4fZ{1I4s_;L@;opbVtx1Ks2S(gzOyJf;T-Ab<b at 2q1s}0tg_000KJ`I2!2#(x{Up
zk!SQkrVogR^83F`AJ|A9AbntGCmHWX009ILKmY**5I_I{1Q6(Vfy0zOAg#~k835>}
z56tyQAL#dwi~b>i00IagfB*srAb<b at 2q4g30{_+RPCEa;=sup5_vw89-}+GA$je`R
zxOF`MxoN7KzyIOX#6Pa9rL)@;_ at 5iljt`elo;v-)ixV$hSzlL&<(tOJ?<?X<@tOEU
zd?Y>)?~1$Pwzw&-imI3q&kMEuMfub6$1;cu0tg_000IagfB*srAb<b@`zLT}OtrPB
z)m#l5&3L7~u)NYr8jErJ<fwX~uxYz|m^5w-9f73)p4-_{00r at 5MP9iefB*srAb<b@
z2q1s}0tg_0z`h797lsvWs!%FjExj?Uz0qF1)oeDJH}s{}VqCNgGtdoJH|<HII%gYm
zhB+Gqj%oN-)t)ira~a?nx?#%eWaqiul)Yfc*LuE?E(K5!|Eb6;7X%PM009ILKmY**
z5I_I{1Q0kB0;2=kl(uD5A<h3ED*ulnJ{A8g{zg>9Y59Z;0tg_000IagfB*srAb<b@
zhfCnp(4hRm{bXY)yb;f?#EWs*j^p}`czjT9f91t&d#n9Ni_5as;Cll at ie}#)UTZCe
ztFq!?wkD$&Ix()7xUOqCacmi}Tqn>mlQ{B{pqA8Yfng<{;n(YyRjXQw3`hd2?zvUd
zi-XAVLp!Ycs%-DVpJ(mW<94*tSYB<knpwk>X_qF$rFzwwoKDjK4A-<4Onc7u&wZmP
z-}&mf?49j+r4cS}jd-~r1I*J|K)4d!Y^=u7>aCTyv%mEB{~>Wl5&teely(1i#6$5<
zhwCHeXb?aE0R#|0009ILKmY**5a<_yG5NcEA^V2j9fTMf9?)LtZfdm$Ul`ErLXUBW
zfl~w8g>KlUVFOkE(En<8b2cVWl!2!9^+A9v|Nr-j_<g at v&<_L<KmY**5I_I{1Q0*~
z0R#}(O#xNav^4*}Aih%Ml?wt0Ab<b at 2q1s}0tg_000IacHi5ykEI`w;-~WH7h~FKy
z1LSxRKmY**5I_I{1Q0*~0R#|0Kne^DJo^66{QsO02q1s}0tg_000IagfB*sr93FwM
z&;OtO{{I_A{O0f+B}ak)0tg_000IagfB*srAb`M;78o1Uwy*z3{{N9aBRF^j5I_I{
z1Q0*~0R#|00D;3Vuqpqa`Tr09$ITHUfB*srAb<b at 2q1s}0tg&cfpq@=pM9rUkKdlU
z_uY79sc`SRf4LlQXiWe0y>ExB;lCL#Pu#s;ni^T3oV+q!m>a6sW^0Y+AFBg at a{0>o
z^>qD!@$yd<@um1od?G#)ABcCwU2$976jw!6%!ucOTK=N^Y5C*w`!b9R0tg_000Iag
zfB*srAb<b at dm%73uFh!f)z(UQBVJf*E#6v++o#4<TZ>xF)v(cwSK159E3Krl7`IQ3
zsuv2Iw#$b}<HpeNh<avI_=r%aw8oO$Uwfh5y0sF;qO88Uv8CCn%N-5N9d$$G&X?nr
z)kfSN)Sgpc?8d!xLcLhnXm_C=uQejMyR5<D_;FR&w!e5<Q{6`gvfOGdZiJsI$)njg
z!t8}kCq|CRes0;W4axSiHp+wQnL^sJaJA8Dwx2sOpuX~`$#GSkNjvw0I9lCkrFf#4
zMRYc51-YkobTeKGv-JZ%Q^aR`@$s=w1Q0*~0R#|0009ILKmY**5ZF(FQw2?#(zMcW
zNe(L9YBn3q8~RdfG0x@>__ZQ_y`R0IEd&ri009ILKmY**5I_I{1Q6IGf#;9O^#Z51
zegDtu2c~<}5A0Dd*ewDGAb<b at 2q1s}0tg_000M_eppYF*Q7J0Ny2mvs?@RKZZ)6+Z
zl=tcLi<h$J^7g`(qW#&g7wsEnTC`t&Ul#4($y2m{hob#QxI91r0R#|0009ILKmY**
z5O^wq)3OeKbG`jW$@^fhqJ7!I3!AF&|DU4$=_63IKfKeT{RfJ8ASSZE^5;+W5%MMk
z5I_I{1Q0*~0R#|0009L0L_l0tX0M5YVpQvHl32E0bxl`yBD1ast{>>36I*`lyS1R^
zn$xn=ze=OM*os{@aP-LaZQU{5Sodwi)2+~~da)D5zU`PZGWr*3bYLZc at 6=^X?0UN6
z*Aw~3s_BmF1&$}1v7^vDFQb2yM%z}kX2wZfPi!-i`;Htvh>bwEEu$Jl)yOqH&#cJk
z`7}Cog3zzpwr<ybxkslS=(RYi>Twjw{RB?pyNP+WGCZWnXw9%APrlvKt6?B}68MQ;
ztNM{{_>mo^t(tYiJf0QpKUBm+F)#n_%lr3adt4Ae009ILKmY**5I_I{1Q0l^0^<A1
zxjaJy^7wz5XHejT47`_TJRtoJ{=+;&0h$baE6*rEX<EHj$TI*ioE7a~=uxz9bn6E$
zePiP9*9)Z!+Y1N=wBs+lcx8RPkj?x5b4C1Ic8?1J2q1s}0tg_000IagfB*sr?2o{T
zHdQE<u9n^y*53HP8Tw`0-|N2cr}8_yHo6fr9GOMSFazCib<>_Ss&lq6XPC1=;FyMQ
zRqYu=K9>QWp&O>WPIjKlP1y at s3c**3_-cO+k5&*s009ILKmY**5I_I{1Q0-A9|Xp_
z>;4T5Xro$W^8J4w%xMAv1Q0*~0R#|0009ILKmdXL7U=o?|F??x?S6NJ_7Fe-0R#|0
z009ILKmY**5I|tB1jeVb-~W#dYMtNzvnd3O9#aVHRS(!N0tg_000IagfB*srAb<b@
zhfknX$d0L~6qTXw;Z4f>lKkfz*@ide{pRO#tJ|Hw&v<#_?)8$hefmCiz%Z|@Umq^N
zuZ)-PDB?@;nfOF}Bt8)Dio4>rxGAoRs+bYa3$^@3`P1^p<vZn{W{q)+00IagfB*sr
zAb<b at 2q1vK(+Qj&Q|&_3YOaQjX1vl~SYBx*jm5Y<I5w`%$X}aVE8&fJVXd`zYbkEa
zCSK5b1fLvLFKph43`rU{riVw=Gn>LkggT`)mgN503+>jel_(Zv_0^3n%~m~bpB$Du
z>W0XkEXOOW4cQm%IrYVE+)F3ai-nC&F4W_-MkIHaHCP-!uIk$M7f)-d`{+QHTdl>7
z at KYt#-u6PL6C=lDKR1rN^9F55wx6|89#qd1(vF3zjaIW=J~5!a@~FviRh>yY_k%cE
z-DstFqL at W=HfjaAr*?ERUJA3x`yVLcftbkt%AY^oXUMw{KmY**5I_I{1Q0*~0R#}}
z4*_vmnY|_oN}fgj%xT%@U!~D`mijYiWb`l6=sfHFndfEnkJ9Koi~gAv89kpy=UMsB
zJX<LbDQUEu*me{+fgXB(U3aW%sE4jw*W<e5C06X1iJh2J+2nl>74cBa%fI{b{yo_q
z7X%PM009ILKmY**5I_I{1P-f!_`Y&3&(MH8{$J)96gVLR at 8uZ}NWX*sFwancCIjEf
zGYU|eR<9NE3;>j~;(fPA at qTqn{eVmTfW+s500IagfB*srAb<b at 2q18<0 at M#=6%tut
z!4uaHh at I6B6ve%?e&8PU0|)!RfYuQ}009ILKmY**5I_I{1a?Q at S*;(ah^!(YjGVfk
zI5pk3gHU&BhNai+z|(`8k=TwC+Lj$ZP5nSo+)wKV?o&UoJLegDK>z^+5I_I{1Q0*~
z0R#}}Yk_B=ejqIi$TJEcp0IwP+M|A;uRkj~hX4WyAb<b at 2q1s}0tg_0K)(sdx%@hp
zCtUx}+fu(@rGCHPK3DpK00IagfB*srAb<b at 2pmBH>i2h4zdy3G`h8WbW%c`OX$3#)
z{~f{qfE+Rc2q1s}0tg_000IagaHs_Or+&XO(yhohlB(%=p6*7jqdR^z(F3Oz=t*dJ
zuIU<Xm{?|Iuj}_!@lIO5|BkHSXZ^oJb;ffr2q1s}0tg_000IagfB*tL1P-o#zf+c<
zV<=!`uj==`9`*bFmihsY`hgy&44V)@009ILKmY**5I_I{1ol^e`hgwQ4~*`teqcZ*
zR&>_?TN9V5AK2eBpVkmS009ILKmY**5I_I{1a at CwFY5;?qgh2jT#KzxrvB?;;>d~s
z$My7@>(}(y2?NWwl4=|p=1hOp4-AO;jrxIkSx4~R?#r`B1Q0*~0R#|0009ILKmY**
z`bJ>a>j%<o|2*RXqX%9;;P<E>=$p at ljv#;l0tg_000IagfB*srAn+^*oO-nW-+)r;
zcKDLKPak`e@<v`-lUtwv!=<gye|dfDbNQBw-TMB?mnMF4sWi8}zQ1UgSJp2Li+hSH
zCKS<0?Pqxa>Ba>C1Q0*~0R#|0009ILK;Rh{c(U~V%2<}LA0$rQajTZ@#Gx&3s-7NH
zLq`v+zzc%Fw*$*I&+lb=zhs}kCJIUzIdwmAYPxJQ)Sa4P=`}l$-KiOg?Kq)r*|B+6
zM$e|vZerU}-~@W;`E}i~s-YgbZe5S-j+a=mV<vWDP8}Eb<@bME5w=_&V7~K5djB(i
zp3{8<5I_I{1Q0*~0R#|00D(U!kT<<QO~%hNC@{AB>HV_*Z{-;U5Hj#wo&kXJ&%V>F
z$8TTkk>3B>mh^#(k4qm|Q;KQ&fSeaF`zwF`2a$OZ0R#|0009ILKmY**5I_KdJ{Rbp
z^Z^O=qfSCVjwJ=nij3aF^nrKe_y11%K$bi(&+LGGej3v~1Q0*~0R#|0009ILKmdWA
z2^?Jdfb91V!yH2aBQkJL(g$Aakv{PHmh^$w9-lr?NEa6*ePCxNAMZv00R#|0009IL
zKmY**5a=6$Crcj?YgrY4o<f5soqa%(^ildiAzfT>BYA-IfxbCk=?DS{Ab<b at 2q1s}
z0tg_0z at 7-?O&<{NDChDF3gn-CKpxJ)rVqT{BYohTTha$!e at yy7`bo&F2GR%i<dkHm
z2q1s}0tg_000IagfB*sqAh4I|1M+Zwk<KR&*J3MlO;-;S$I%_f_4JzS*Ywy41IxFP
zY8)EojEwG=^nvtqNWcG+K9EXqK>z^+5I_I{1Q0*~0R#{@`~th4J|M~N<rxncm4OGF
zKJd*R=>v!V1Lp`4KmY**5I_I{1Q0*~0R)bwK;8j>-Cw;>dLQfnK(FZskLF3hQ6qo=
z0tg_000IagfB*sr^s~TZA?saHQI(-?M-6#jlK;HPY{P=Q-~9Z?m$p8aQ5UzA%+Ech
zWM14;R5785wT+_r^5<C#+#-Mg0tg_000IagfB*srAn<GoJXy(nWo$@Ei{SH2uJ@!f
z?pH>#edk%F&#Xx8`Lrit<kbDdsp-BQgt}8REWKt2o*vYU#CDv}w(QtEE2C%AXg9I#
zC~yKj^!&Q+Sk+JuUAL~sb;nDr*fA43F{h4;`||t0tq5DZ(=D5S at 7erx=`{ifAb<b@
z2q1s}0tg_000OxL4lZLp&B4zz6d<RzQ{Kum3Ls?QxjX{^<)3}0S&!eo*rR0rwJqrb
z7ax~Cu%;B#^nuG_BKsqKAlJ!85CRAwfB*srAb<b at 2q1t!-wO0k`hX<rml^lv@`J at J
zePCnGebNW|_6(+D2q1s}0tg_000IagfB*u!AaHQ$1G4k|H{ssvTha$!dwlvpL5{1C
zKClZXAiF>S0R#|0009ILKmY**5a<hmCrcj?YgrZlUd%oqIr=DlpdjXz?#u&e3IXW@
zeQ~zZ2?P*8009ILKmY**5I_KdJrc;9J|GkB&*d2u7)vJ=h-<MGx~8j#iR0*w<9d3{
z^=o?Ugn{K-Ni`0QQF%ZIn?CS*kJ$(INSWOtfB*srAb<b at 2q1s}0tg_0z_TkbKBcu+
zZ#A381~qL|i_C0VLxX{ZXZIte=LjHx00IagfB*srAb`N(7nm+&$5>QU<yiM{4S8RZ
z|NQiUyj+*}>2q%~eWoaX`{vf?a#PT)z&Bo=xO=_iZ?C{t2MqJd`t{-R`^tFvjv~Gk
zpNUVzN8$tVuDC01i<{!AsEQf!yim(uls_$hT)tEOX{Rl25I_I{1Q0*~0R#|0009IL
zcuIlOW2#+<TFuq4(TrEx3(G65q_G&c2gk<M8Tr$0YbCr9FRZl|Z!N`b*~SZ6kKmJ|
z>V?faks(Rr#`N%rdS+Ajh)}1r#**A$d!gOBwGzdmtiHOjrP->-?UTcDN8J$FljV42
zwITbWJ*U3djeF^Yda<z4$%T5n)`;ZpvIdLe$5mb1{^Ds(bsrtba;vqt5q_$q+S^{}
zbYkR~?B~Xjcix~4$@a50%7f~eLfWx#wb5#}$4(5WuRLmUTvcb%&ix>cRySHHo+xG!
zosC*S?x`K!jF-Y;sX3Ug;P*h>6Km;2e5~O2lqWH7LjVB;5I_I{1Q0*~0R#}}6M-k2
zg0C_*q+AmPB~P`zdA>55MdvBGH)kp%S#)S5Rnzf2-HlvXpzl``J#cD)o`i-c>-G&d
zOf0if5#8v>sr!jj(|tP#b*E-ndd&_zJ*XLp?Kq)r*|B-HQqH2?#I~cr3G~qO>$+oA
zLp^lex*pdZFR^0BOzgy*Ixat(!E^<`hr$-`i2F>z*C!_{-9P{V1Q0*~0R#|0009IL
z*eij&Q}CrT>*W~~$Ug;Ny2m_20VAEj9HRgt4a_qDP|hmlgC3Rgb6ZyM3%1t}6w~s7
zOT*%UGLWVZbdv{2AK0t2ll>xq00IagfB*srAb<b at 2<(@@(MTVVnzL!zL!QwCnLZ#M
z%J2U&ePAPbfEE1q>vW_Y1Q0*~0R#|0009ILKmdV*5jafg15)r at o&kVv`oLU|^nr_8
z(g)@qmp*V$QPcE+&ddX1g7krdId^Fl0R#|0009ILKmY**5I|t}1fJFO0Vyz_CIo~z
z>JH4aGWv<q2ky)7|K0R~c{%q0=>xlW!m%d=5I_I{1Q0*~0R#|00D*oKcm~o3B->kg
pMgfEje3JBmi#^f@`tc*8PY57 at 00IagfB*srAb<b at 2<)Q3{{lO)@<;#x
literal 0
HcmV?d00001
diff --git a/lxd/cluster/testdata/pre10/snapshots/1-62-1557738365979/meta.json b/lxd/cluster/testdata/pre10/snapshots/1-62-1557738365979/meta.json
new file mode 100644
index 0000000000..96273a6958
--- /dev/null
+++ b/lxd/cluster/testdata/pre10/snapshots/1-62-1557738365979/meta.json
@@ -0,0 +1 @@
+{"Version":1,"ID":"1-62-1557738365979","Index":62,"Term":1,"Peers":"kaEw","Configuration":{"Servers":[{"Suffrage":0,"ID":"1","Address":"0"}]},"ConfigurationIndex":1,"Size":1009439,"CRC":"+xixNbabeXg="}
diff --git a/lxd/cluster/testdata/pre10/snapshots/1-62-1557738365979/state.bin b/lxd/cluster/testdata/pre10/snapshots/1-62-1557738365979/state.bin
new file mode 100644
index 0000000000000000000000000000000000000000..c2a40daf321d2b5349e8ff4a7a65ba661fb9f068
GIT binary patch
literal 1009439
zcmeF)349yHq4;sh at +CX=<{+3P1e8FC9b1l8w?ZKpM+pguLmV4Ofr_=Xas;+zBsnDT
z3e5qf^tGkOYoXAC*R-^y4Ly0ZrMw=WM@#7mw9u16c|adWOADnG`tN^6tCb~NPMq6z
zeoE{}yR)-1-<dr$nVCJ)l6y at z{@u7?Z6c#P;>lFMmT`D2(=BCXma`lVvDTXVF#;ag
zZ>y!%>!_34=Jt1B(XbT)2q1s}0tg_000IagfB*sr9F)KXN_O+WwUs+8&L>YWzo_rI
z#&YiiHEV9T>6z`{{@I+^QV~eD2UyJIfBnuYU;J_9(~s<>{h&+)Un954y;{Y;<>m+R
zZ#KSy&!yg$$@YT^v8G&kUamh|d0qLN@{;m=-MRKxr){lz!v3X7t?HxYH<i6yey(+C
z+3m{TOGPtbmhDK+sA^cg+`6kjqxJOZ(d0lzOAP3#bR?#4NksK at L@bIYddFNbqrEfS
z))jViwVk~->=-kcqp`*zUWu4vbw^itRk+hJm`e0(scnwS^=*!}?ymK#J48(D!W~_X
zj`iZFd+pkm>?Zjrb18}x#8Dhl#?%$QaAX8qw7wxdyTXWY<@(O>>Qx<%bHke)jU{4l
za&(4QhC9O@?ct3jGURAX#F`xII~*&*Yr`Vr?QI*|+g5~)xVt-6Z|Dw}jJ8GQw5g_P
z!_3MCeYw?=7>Max(^vF~)~<*)lu2ekOYPDH5K+fmHFHOoQq`cU)?FLND8$j5D*Gt5
zLs4F17Sw>&ZxqOgqRwm^)bsVjEL%rRk84AH8Ha1H#U~4;WV!8Uc_m^k4w*htQti_#
z8!k}yqohg{(e#}i^;NS2f$Hm*Z%OD^78HbdD~_v3N{^RZm at A+2SBEbstc1f`IT{C3
z$;<R;#$Xvp#`N4fk$U5usBp%c>><$PXxk`{sEwlDws(o+FS~WRRpxd$j>6Bw2`dOR
zyfAlaXwC0t^s4NRvxl)`W#{^JqjO at cb2!$B<G*07JidlwYfZ$?82)7LHorui5sIQQ
zb0#A(-xo*Gnp-?_ONp44n`PgO*hVfB8|+v%t*T+^QtPf$MjmV;LB(fEV?o_mhnlf^
z^pSR6=jwHBotv^JsPUZ{$2`&w?v!{ak16Bi*H%~A(7kjVhm>JcbkZ{x)$i=6$)4=4
zUp3OdM~hxQG?*6Q%u{G?N$prAkXwIPiy(WlJHM^7{hYSWMz_z`G~%QeP3c-DksOG`
zH1Yo)E5x<5YxTO^<wV~)m`H6awI-j*NOGWW+dfa4$-zvbKXH{Fi=;Ejl-4_;PP4V+
zJk91xtEf`smQCArO?6enf(6!hjI0%TsI<MP7AzcIv2b+xkN4&zcTO1PKYS)Gbw{G5
zqMOBKKbjdzjV=gd(~_4$W0gL*S?||VTHmO>id&MEZQX0T%u8kIs+F&QBZH?iq6Wr?
z_8o7Zmd-?m(t4?iDNdl#R6<m@<Uqka*YNpvRHAd48ndRGw^YMX<Sx+0hDCRCx#B3g
z6Ud&*_j(gh6vgmviekwvPLB~$O_Rr7*|2KCIL>SZHwiV{=T%lU%$Z}|F~dBU85a=o
zyQ-+RipnaMjjpHTy;O|zI$&BGC+p^_%7*20_L3!$r0l^c{;l4AZFyD0qD9u7ZAMn~
zR3;HmL`98CM+ at Fp6lbeo&4Fhwo*3xWQ{s>v$do$T_HrnVbt77k;!-i1Oa(g at w~K*_
z%7$|njmcGzRe at IUsq#EqWkYkbbyF5D_l+Q}UooT)i0 at y<7X<TTc at f`;m5tduIT5sJ
zd%M-BSDF!LRGcxiboO^yaSYjI!}RhyntYjG(#BiAwuo<x@`x|k)VzN;4%4AQd3lRP
z#Q*7=m$>|S$OvJg=Asgy&1HM5e<C~LTeSa8;@2sv_;uXL;@7Mh;+OoMpKllV%`ew{
z?}5-$F8P1{R^=6o_{$3c1Q0*~0R#|0009ILKmY**rjmf&T5hqH*I)e0`bv>4o6MTo
zU*N4D`QGw{?|uF|HDZ}ndEFxZ@<IRs1Q0*~0R#|0009ILKmdWMCs1y)+UrHOvSR?u
zc7Y|YwB3K!bN4()e}So2;N%Da1Q0*~0R#|0009ILKmdWPfZb}FV>iZ0h|hAEf4;xK
z$$$LK7uMbP*)7Z+n3due0tg_000IagfB*srAb<b at Q(ZugCCIl6+}^gn>uROvUOA3{
zX9`FJUI-w700IagfB*srAb<b at 2ux{#Dw#WT9KmCOw|w-+zpHqV{sL3F%*h-A2q1s}
z0tg_000IagfB*ufK(%~hwhO%Jn;Uj){OO=he*u$+e-S_c0R#|0009ILKmY**5SY>e
zc9}b~UEq6Xp7;D4ci-|D{RO6UnUgsL5I_I{1Q0*~0R#|0009I{fo#_Rvt8iwmEAvG
zkv{h{`U{vm{EGkr2q1s}0tg_000IagfWVX%sFk at h+XcR!`OMreSMUBU{RO6UnUgsL
z5I_I{1Q0*~0R#|0009I{fobxM*)H&v<9>GElBd=zr at w&7!@me1fB*srAb<b at 2q1s}
z0tifLfo#_Rvt8h!r(gc`ua`eJm;M4%y3ENO0tg_000IagfB*srAb<b at ra--XW3~$n
zI(Gc<{>^9qnf?MM5C0;700IagfB*srAb<b at 2p}+}1r(V(vt8hxHKF4l{nhn9p})YC
zE^{)600IagfB*srAb<b at 2q1ufDKK5WG1~?H-qCT at w@<ox8~p`L9{xoD0R#|0009IL
zKmY**5I|r`3(SzYGus9Bq#u6j(tictM1O%PUFKvC0R#|0009ILKmY**5I_I{Q(&fi
zW3~&_SQef$@0=gbrN4m5!@me1fB*srAb<b at 2q1s}0tifLfmt$lX1l<Roh#lI-ksc7
zr~KVA!@9$wR45PDf4OdJ{Z-RGUw5wk)oEL+p0IzZQmguC`AxP^*~``))}<oovHlVS
z8};Q at OJX3VZ%tp(m&oW5Z77q>evU+w0~swbpr_K2n7$<u)zflOJkc9b$6PgYM{-70
z!}8_UUHuuYr%xX<;Fv3Bw0DNvy26gGwzJoU9b*P_G}bu8D-m<7?&u1y3U at jNQ;B{p
zwasz4zRl6r-L-yohlpuixTDL_v0nUiuU*@c-6S7nE?1F)IEq8cn7YCjj*MW7);FYQ
zR~QklT;Ca9y{f};Zg`WUu|(`mj?VDPaA&xqJ-o3*h8&HFSd(LYhhs%}ZCGTyy=`NA
z+lsIecX!9?4c+0A(YDB(Hq|t>Pp at paK;4f*EKz#XcXTOL4XSG0wQ-DM%jYX!8I*k#
z)S)P$F^gtE>o<yEMB!$(4eI$CVV0&NrpL9RzKp}Q*8-EpP_m@<vxE|{7KcoqD4U%f
z^;NS2f$Hm*Z%OD^78r`R@<2-I at sbO3wCAr5Ur<<MhPQGw4yKZq>Cudlqk&{h&%F~n
zHr|QaVZ6!K*Ct2XMp^wgib~erB at UMC*6CK6(BU`=KMyCYAkgr_+_|7NznjskvOCTm
zbB>jr>(`CWiLuV%SR)SBg0=Es7>=zq5j$h}leydc5^?wzMPuenMq<7%j-oZUc;uE6
zF)cUCz8SHNTqZWyv20pZ!_uYJU8jsZPDFx4iRP1Ko-6CdI!cW2qYskvI#;i2>)ez*
zn~d)?Hs(QcaObZ>c{mtns<yhyhVG@~I8zxmMVA?4QT@)2n(TS%`c)(Sd$j1~Lxbsp
z0>~|?9jgR#>kn%YWY1IQw{^Ck)7IJO_W7DdoUfuOUCShr1Cf}P(H$$qg|BP%y4<;6
z-#VB`Z7a1VpUFsapl{ng&v(heOrk$=l^%<vGs%?JJCV+i?X}~)dgV&1s8ZyXP1|)%
zbydTH1=e?rtQC2vw7sYnEF503aCG^P_f8^rP8j7sd?qe+`=F(wo5hV*G&7VMT at c2m
zC9h(}Dt&OX-mj;$zEOJ>_YW)Ey4QA at SGm$vD_{Rc22W>14U7-%JKjDmorw&k^->j6
zoIs<ggs5)Gfr1;P;q&dNMCUR!W=%Kme}<#TU0aO}i*CJg#Zhz-&z{Qndi^enVt6-2
zvE&w~$B3w=$z!i<ShZjrXSRZSubS=iDytgi%(3p6VV=v33yAn#Ra9F=WfjXt*VFM{
zD#m#oFs+S~b#ql^!}2+M$&yG?_FxqMR&T$yysBZ*BJ0jJBP)6;lZYpxqDG~o1 at 9}0
zvsJL>z%v(54D{+LamWs2N*!%`Ih4k_hb%~OsTfVBf*p%Hy+B1}!?}yb<SNLjKr8rE
zd7iDZp}E<*DGQhT4wlxh7}5vC7cS!~nfbB2h;PKo#_XM(2->v0-D=b;&4 at E9&KO!c
z`@5_-hU~IodijM)zRYh_<E>v?#8*Ul#20L8-g+B{>Cm9Oyu~6~CVz>`pNEVPCTcD!
z0oq))xB6#fC!!kv+Qb+6>fD$4<K(|)yTJB~*IxaPFKh3us|s7{Eo&^(&aV2es&y5s
zD*vCYsq7Z(Q%Z0B%hUd2+HA|3+Q(|UYU^sV0}gUAmHcOC!%0;QO-<J85?NgFm9tlT
zFV^}JT3SyRe>$lMdtvd)5|^Fva*tJSvpX33+w1|Jo6V9LJ~fVg1~yYrbn-OvJ;tfC
z#A$qZM at 7->^Blb0J*TQ6vB0{$H`{zMe1(q;#~9HDVv*c+!gw#f{s^+I!=dNoN^Nz=
zitvR6Uqi+dOYs+%{C70510$cgJ?B2iuZY|0PN-_wAQJ7{SE5HH^n~#yG`HnG6DQNN
zYhFXu?9)%Le*1<IUs#1<Jkckrdhu7*!o|7tjQrAC7$i?J>@kdCygQWNR?S^!zrdFM
zB5DRH`VKi<8;a42zZ1$Dku_<}hZiMX6p<N1Mq|DoE0Wy$5>d$S`Pna~B1raqaTEvh
zebz|r`py-i0qyKfBdWWRIc;#TZ(Agpis`AH-q}?R9*^~U#kkBT^9_V%=JFEhyw34O
zr@;~dhA%+og=f4s;Q7b|TK=k4eCZih9V0+N_C|#m*>Gf at 3k&z#Viv4OLz7X3i*6|k
z!xo*;@+;DNYXB^YYdAASF`4+K<Ed#1A79z9$us7?$>>rn`jTbF-ca*S<08BHxT=Pb
zIAz&Jp0dmnYti|1_E at K>g3!fBnWHhcs_D87v#Mr?Le)36jW`>b?a{?2qk^SloQn!V
zjN?>fgf_}l-bWhm#O?8+KU2wjCr9DomrH5srZ;otaH7oZHF1iL>cg$j`PN#q-7KH_
z#fx*bHJaS#M%i<+k!i7+JZFq-&&#FAIGwBwuk3QH at 9q-klo2g^BmN_yxZFpqEI6wg
zv5Nz&up!VyFi*K7c2yjA at fp+HQ^78ZVi8q-f6uwuy~}#LuNFj$#k%eg|DTJ9^2Yu6
zKbOG5y?s^5-KJ)<5>X33E!=K(R5h$>vTi?F9{I(ew&p_S?t}~Pmc;3xc%`H0lu*2G
z#D=mIiv#43TY0`c{p8Aq^-W`)*@`#JXC{}#lbh%7?HfjJ<D9_!4Kphn4so&Vy7<KG
z_q2D+D)=g4?DoW>Yi91N!$EvyDEO{g>MO0h(2w=4A}tyYM_m|=bHU(HPhTQBYURk5
zrExTPm2Rst_A+)e=+IzaQi~P0Lyh=e8P&vh+|l<2Ba^tdHZA!dq7ma8>^`%aJ8_JC
zJ7V5J3<qp+6ckRW8;IecBQN^KIclt{{uT4Yf5eK*v at -HCJshD(XTdk0c}0ii at a;*7
zLvna!(c!rL__<XL7l?hf?PZ^ZU%`s at KX)AaAF*-qNxm>GM%^)QB*a(w+4AeW*)A~c
z{T;Wy?c5W8%{YRRjT|gQ009ILKmY**5I_I{1Q0-AstFt~Gi0_4v{`T3 at RPN#zJ+-Q
zr&=+S9|RCU009ILKmY**5I_I{1WF1t$fahxz}vp_(uP%UIpsw93zW2CAp!^>fB*sr
zAb<b at 2q1s}0#i-k1eqbTU0~o=>+QY|ed{l at 3rw|QCO-%ufB*srAb<b at 2q1s}0tl28
zm?M{(?E=f*`TyF^{L`an(JoNZiiHRufB*srAb<b at 2q1s}0tiesfw?k6X1l;!$`+`1
z&;H)iv<pnNVkSQbAb<b at 2q1s}0tg_000Ib<6qqNMn(YD~TiJHS<A1tu1?>VQtyqWv
z0tg_000IagfB*srAb`MB6F5<3$ZQw5{N^j~SpLVI&+$BhsaDM72LS{SKmY**5I_I{
z1Q0*~fsz9A<x;a<;AfUQ-~K?=C+5>%prjQG5kLR|1Q0*~0R#|0009ILm}&wi$qbq8
z0<Akw{$_30?>|j{fvHx^<Ocx+5I_I{1Q0*~0R#|00D+PMC(ET~yMXJwTmNZ4t+t;2
z0wt|jhyVfzAb<b at 2q1s}0tg_0z*G}($PAh70<XMp#k$|$`;O1iE-=-KnfxGt00Iag
zfB*srAb<b at 2p~{WV1Zm}whKJD=f;h%zpZ8#;|NMxu at C_S5I_I{1Q0*~0R#|00D-9{
zuux{mY!~?Kw|?;57yjeP>*y~q)ry(?Ab<b at 2q1s}0tg_000IagP*UI&xzubIcy-wq
zqW7);$$hfFz)b5K7V8`JyX!Bnw@>?R?YnF4t$Anl>(%#Gbym%;_+rKN6&KllY1>`)
zHtPb*lNIl;-B2?2qqwkam*xC38myMYKuq77zM?OY(IeVWCYk*lu~@7h-OxPN+QC#Z
zp6Julk!W%tp6HFJBNq5}G<&KVnwzb=>M~kSpFU#sh>za(&Tv~-*wNK?_S&#x#7akF
zjYGT=F~{nTuJEdGr{lcN)$7_iH#yD?Z*sJCcdcLDA+}!^?&xxKtQSAsYuC19H_1g5
zDcH0)z~%ZjM_2g5k=t$2`iAuE3L~EGj at 28w!;Z#WeA}&#s)kif*6k-}r4s#GuYl0{
z5?WgPD*|1nM>Arl;_uetm5y9KS9h!kU+5 at aH)2D_`i>C+vir|P-LzwYTGcRrzIE5^
zB3_fuT?D$&T)gnWEeEuIgJgtpOixEsiNQ=FIZz-rjF2z1V)+8nT$WlKau-d8_{#O2
z;nk};vL!6p&1^e5!z;s`;g0t3Mp;ed;<TeN5o?lME5d8Tq7t>YZESB_5w2<SxGEb~
zEf}i?CkOSECX=0xsKf8o9oH_cYFMzqdfn<Q?C_f5cdlaG;S~qLJ2WIJX^E;hkc{cF
z{+3vk**2(;(iI3R+q&0wIb8WeAZKR=Ew0jXzopD``)w)&m&?Y|K7_^A)LPjv_Gu}b
zh=@D;&(bF6zJsdSpshQ!ma2w^2J3YTvgMjxnEmA}D$VSoy&fe-PRDyV<jZ66Vq-%s
z7E9^rbg2SOM>p&JnpsXGjx%jA5z)71^ntXjvSS3=tZS)EkFI4LE5zxvYxTNte*f*=
zo#MFZipYf<yV};BXC!A(ABZIedPl{V-!4~=%;UA-7&RB=a at 17Qbb51T!}_MN4!8k5
zb7eAhc{-9cj;O_-TXys=scL9yvR-F53aBV_aoFbKA}b2H*RtE|)grf#v9A^~-D9sc
zIfkPVXEbArk_VWPY7t-YF_z2B7$>z7hncyL at gAu)O=mP!HjI5`&r_RM_7Mw~?5u0d
zUfHkPP+U+(_z|I-N*9$8_Q;i$;kKWb`w}JD;t+96u;N_0xU!*r{yvYo{PEVfef??K
zdxsrY6cwOZa*InA*+G=me%(2wGg at Zfm$m%CCT|&Pns%&RRMoI}v31u4*&O9=tkTJ$
zR8%kea9UB`idG$X2D8zOePT%KsV#b{)CphQ!DN!rWZ$T>uTfp2dMcBMC!!Lh#PvUW
z!&z!Y(an>YxT3R+k?}FkF at m${)T)N1ORc-Cg`6U_;+83s(9=1$;RTCIu at 0|3Y_uhB
zf%fZIKWM8<PpNDO%&}fu_*FpUJa^M2YDMlVz^OT+qFevsfWx=^MYsBag_RBGE*k6j
z&Xq;(np*g2;f_rUsu~t8vhErxEXHie!f*>p6<#*fVJo-XHEWD|Zywy)L<+&ZUPsM9
z<FDU`<Qgizwx24$x|{6+7eDx^i>4*hE%X;SiVYg<8UX|lKmY**5I_I{1Q0*~fg>ld
zNbbsP7kJxOU;Sg#Lw}Akzrc|z0QQ6c0tg_000IagfB*srAb`M86gW-p+iVx`I6nN$
zpXy#ZpLT(xSR(8i0R#|0009ILKmY**5I_KdBPXy}?#gTzXuPW2H>>lyuhK4X<O+a2
zA%Fk^2q1s}0tg_000Iaga1;d^<-W~!fdxN(Z{0&5xL`T+3mnA~Vb=&CfB*srAb<b@
z2q1s}0tg&AfhM^tvt8iDGw-<d`@hJXO at DzSR{-n at 0R#|0009ILKmY**5I_KdqbRUM
z?%Qk^xc38_X5D at H-@ZkEfumR=>>2?C5I_I{1Q0*~0R#|00D&VX&@6XlwhMgY7yom6
zGBS4+{RNI(0k9_o5I_I{1Q0*~0R#|0009J!qCkt>x7jYxSN at i5^%s3;4*dm=Vu`S8
z1Q0*~0R#|0009ILKmY**j+}r~?#gTz_~VVMzjyCNZ<Nzt;K&sKdqMyK1Q0*~0R#|0
z009ILK;S3}w90*(?E)WJR{q~>zBvCz`U at Pz5@FW}Ab<b at 2q1s}0tg_000IacIf12e
zS7y7wZ8Z-)^M~Urs%aNEas|Mi5I_I{1Q0*~0R#|0009ILIEn%;xo@*w;O9>~qu;&f
zK80}vN3lfMH3A4AfB*srAb<b at 2q1s}0!L0jmAf+A1ztJ(&HZbZ-TVao1&&++uqOl%
zKmY**5I_I{1Q0*~0R)bsfLrd{Y!~>ybKly0-oMJ%(qG^xmI%8>009ILKmY**5I_I{
z1Q0;r$O(AluFQ4;&;Q){@Y=O+JVbwiBUb?I2>}EUKmY**5I_I{1Q0*~fuktkmHRf^
z1=Jrr@!lDi)@-N0z)>s_c8vf62q1s}0tg_000IagfWVOx at X1}7?E;HtFS+IXJ*Up2
zzrc|z0QQ6c0tg_000IagfB*srAb`M86!6P^o9zO-uU|dqv+mcI%l-nj<-0AFGs^!^
zzPtW|b?fS;)qc8mQ}th}-(GpP?bWhBSUzHTv*pdzCsm)cziF>{cHLsDB{2}wx2CV?
zOJwwjHk3(bKS$D;WJ>GRBZJ9gUpk@|eQdK>Dn2iFwyXW at s@XnY^$pG~3H{2!RPr)C
znn_27(t0e?vn`U+<E56hcZS=#!jAJ-hc9rHTJLBa(E9b38i#mo(fWq;CP&*w at o8gt
zZMeNFze($GsKHcXAQNv~yi{#<EsZ7zGFoClPo<YGN-xjt;EYA-#YT+7tBWEqlAc}E
zl-={n&h_gY!+{-Z*01i!@7u9H|2~q4onhvpH4$@kcdTCDG5*vh`h~JgX)bxj=hk>)
zpjS_2^Hq=>W98vWipWJg(U(okXf|`@IMB?X<4xy|8EsXwJ)Y{F>xycTu#YEtiz?CZ
zqLOuJcy(bFx?JBjx-RXtGUckV5nXG3d$W`yxfR*EnXN01Y-Jg--dHnMwMmP*lCL*r
zjJX(Ol{%}kVMFs+2j_ at AMbr@smhWg;Ue(atY~7_~w4OeF#Oe_r&&*+VwVk~-oUh`!
zG&vfx`4AO*bw^itRk+h}Ugzp{ZJnDO=Y}^q+Pb^MVR43cABoLYt6YTH#1<~_?P&HG
ziLEmeTew;zw(z5OWMT_fjwi8V`*q=tE=R|D at zcF_tx-2}dlB;ad=>`~wZPF8zHsDr
z**cJ2VZ<Yj{tew>N8 at mOEe^41ld*Q?`p)p`RUKKnf*?(f&hW}`XSkz1yixjRv(%V~
zHObS=3USsGXUq1sjqPnK!Zl4x-IWdP^Y>CS3=ml*vhO^*&VG||l5Wl&UPgf8lk})%
zBMz`p>$6#ur+Xu3!{_4hp0SP86`f8F at +2+}%iM`SA~K)76pcKen<&{jy{|(vo9EVC
zRH9}KZ&bXeY`E-7hb*uXD`g}yRPI_~L|i(;Qt`-BMX6Zx%Zd_b=3QJgN2Dc7Qfddo
zUPXH<$a~`Ss)p0&Tenwb?+#2xb33tr at pr4a`1D-mS>3TBd||#qj|iN*gk~-Bl}W_X
zv at 5pEsISSQ`Wn->B%*pccRk2%7AaU(vc49q&+lSneKo=uOEcRBbN3zjlYU`!-up$v
zDAnNuLEgSNhI5}wOHqY4jsY_pt;4%Fwv8q=aXF6a=BXlWhBU7i`BW}moIAtpbCYa-
zjNFQiWMwd~T}7luEm<30+2vT at -6hWY=D|`F%DhGxcSEIC<!io?eNn#hqE{4B)NW&M
zMY|}h9=S_p at m9HLM2y8-XDfMbYvUYHu+M@Ty)IPQpq;+Y^Qc*c#IYvtM)IfC5k;|V
zhZYn^$m!Nyn~E=D#oLd}dk!#^tG)Rva8XuG<#AkzMLc`CxaG^>&|c&6(eCq7mk(L;
zif58K#=7(^4^%d6bdI~oMwL}?$4Y-ygVSl<wY+2 at jVztOUW?^0-fYN|NU6IGd4d>K
zc8-`H*M|Bs4wrFQm%0U!5gx!j$e1Ouqt}<cW#6@}=p3CpNR01&MjakrRPqcxyn4(N
zbp9lF;HTnz!5X_Ox}`5nW@~mA#`&c9v}kNte5T26Ip%pPx2uCaO^I*W0r^$iY!~>Q
z^Z8fr`Q)E_7)Kygcp-oQ0tg_000IagfB*srATZ?xf-;9 at yTI+w+<NT~ulS~gaRgJo
z+{qvU2q1s}0tg_000IagfB*tgAS8d8?E<lHf8 at 6h{p7_v=r15ucp-oQ0tg_000Iag
zfB*srATZ?xmdPBN?E;<;EepTrtv|Vi{sL3J+{qvU2q1s}0tg_000IagfB*tg;B@)R
zY!|3J!E?q*pV{zN+6ANvF9Z-k009ILKmY**5I_I{1g5;en`92nc7aXbxb(TXgYT$h
z9Kn<?cQS|o0tg_000IagfB*srAb@}rI79w2+Xdb_ZOhIN&Z&BY{sK~k7Xk<%fB*sr
zAb<b at 2q1s}0#jb#OqoNoUEn>_uUAjLsq#7c3rzWPCxZwefB*srAb<b at 2q1s}0tiTf
z<?@%=F7TzkXp{SM%Dd<<AXRuFfB*srAb<b at 2q1s}0tg^5<ps`?IW*e^((hdKp;K at B
z-y7&JFy+gg3?hI40tg_000IagfB*srARq<W<S(;b;N+h_f7i4p?tcsY1*8fu1Q0*~
z0R#|0009ILKmY**ro6z}GKXfnz(?CI?cV(Pdw12@=Ub**{?DRxDW}xET=&&!x7S=`
zztp~{YC-iQ73(U0RraBZddvTnt(p8fJ1#l5sv#7z?&{8HJ$-sQlT2y7dSpwoZ>V2S
zN21Arc%rxT($(#q;kK@@qpR)gwP8o;FpkC=hj=Aoj at 2Dq;Z@;I$9bKr*R^$Sa-199
z<Y?>eTEDtOM6)j3(dFn^FMhh$u5HO~QXF5TAco>_m+RXcUEvEyZoWn98`85YjOe;M
zR&VGII~qqt+u{&0G#P<buI~)5Ue%FJ+vt!@j?VDPaA&xqJ-l&bb{&n0Sd(LYhhs%}
zZCIGJw{2{1TM at 2lI(KblLnLIiBnD#o*7Oy9iHsi6hBC?Q=SXpCOzwzUdik1NnblPd
z&CS+#oLs7~Mr^dIsCX8RSiA7ROJ+dpH)O{svB6}rZ|w3KNXGQB*Jid2>iMctT+lH+
z9Ze+$Gl}FtLAj;}w88Y|WM<Sxq9j(fb+7GmxQwzLOeHVVqnWXHJ+g-6veDwmrMSg0
z9EDgY5Lz5ExqGWSxj0JJ9Jww%M!hLf9doHDib^iqk0^%IUldDj at mLYnG@Y at gvSCB>
zc<b=U%6rbvnpIT|9*_09(~D|tHvf at q7R<U?uqa$qdrJg3 at cLUA+1S+~=RK-kl|G&1
z3OZj|CTOLoX$qfH*|5p8ud*yCh~bk_!NQe0UE!(*pU--|twfm?Uo?!(jAiY6DODrH
z5iVAvG!O6^GP2P3c`3<lP;~Sa*2b|8!9#F5sA*cYva;bK-}p-2D2tIrZ!BFQZhETa
zjZbJrWy89~<BKqVQpnbZw(VQns~Q$Bw(eXyy80A>oHcfZIq;e?)&(q<N^H?nrS2xi
zxb|eU%;>At$P15oU6Ffjs%hHQes<MtpRf7`=az(irFrg+45js0q-R?srN>Jx%Q2k4
zI(&h78A=y!<7muYkg|79*;}6`N83j6X`{FuY46H!(mEV!FqIg{#2XhcRa;$4MagEg
z#DJbkFI|*go=d<Pi_(jYnm4?<D1z(-HJx45l-={n&h_gY!+{-Z#NAzf-}3G#7qd9)
zo?+&)RovMLqV*l)Pi>-K+ at i{q=8|W8ZjC1fdi7K`Uj at lARvxaTh+M=I;<mXYo4Ilv
zXlBsyrgO)PwyN14Pxa1qMYYH{LW?TV at S>7+Xn1vD6%r>zqvQ+g(q1c*Q5SOG7h3b%
zo26{7$i|ngD~@bs8L{42Ggh at pi@K7pH)f2v7-W??Yj3wHBlZ+gM=V&rqh)#a=6aWs
zJtvJ=J>uh;MdzgvD-Zn4l#6KWyOMEUsfxbH3`bgY;V~1v*IQ8Y<TV_JagnQOT6$(>
zL;L)(&i-cP!^~%FZ#tu at Vg7vUjzvYjX2>^{6g_atF;DU(zebq(%$=mg)vaA#;>>n|
zU*2xLy7J$(cQTGZs_;Sp0R#|0009ILKmY**5I|tc3&`g0e7itLQ}a8oefyI6j3b!x
z<xU0>KmY**5I_I{1Q0*~0R#|`0%7^f>@N^Jahlq8kF}cq0#bz+0tg_000IagfB*sr
zAb<b at Q(j=D%%RyXuzC921;4uGS3jn|z?3g{GKc^I2q1s}0tg_000IagfPfTOC4ZUi
z0(aanqpIRo&s*s)AXRuFfB*srAb<b at 2q1s}0tg^5<ps`>IW*e^o_M4AL(y-jkJ4XY
z%9lGCL;wK<5I_I{1Q0*~0R#|0Knkpuzsz=l at 9vJg^*=xJnI-fWkSe?oKmY**5I_I{
z1Q0*~0R#}3@&ao_4%e9N0;%#q{}<kV(^mQmO!;yrg9spi00IagfB*srAb<b at 2uOi)
z`O9n<m|k}J&7X~&`B~Xtpj^4%qP(I!t2`{;@In9q1Q0*~0R#|0009ILKmdUWDo|T)
zn{OXXCF6-cJw3~2Yqp9Hm+8?=IucC|#1p+`wPi+#>_WTMHpiZhZr1xXks+tdl-VwD
z;hmO;-)Q>Yd9q!=syt#5e|aH*00IagfB*srAb<b at 2q1vKBobI+wOeM_&9jzyst1w-
zx~nz&r`~d%+ogt_E}v8NI9&c^ZvQgXE3(xpvu3snJn+MH|M|i5x4taL5m=SKTf|>p
z2q1s}0tg_000IagfB*srATY%Qs%*3Eve7_fYpKkd*)DLyYt>g?^6PUSGTH at R5#tD^
zSjNc?0tg_000IagfB*srAb<b@$GSj`ZJs^Xi$GAgC5PEAQ2EbuuK)U_EjD8u!Q&R?
z at nao}{3Cz>0tg_000IagfB*srATVhKPPSH9=6UQfJ+2M)WmXvfIM!ac!jYSeK;+CL
zb7!^-Jo46$U313zr_ML}3%qPmUY at jhC*cSnfB*srAb<b at 2q1s}0tg^bLZG(XKF at 3_
z5UWp at LCki6_r3C`ng_dHs5aUK-moZdl!$;u2q1s}0tg_000IagfB*srATa3#Y&N at H
zWa3RSBWAn63Aewo<sTpTOxzep at Oz8$d%2bu0tg_000IagfB*srAb<b at 2uwzSldR=p
zAVKMI1oD}MB2Q<?e3|V6PXyn2=O;IJ|BHDCCu3=oW&{vG009ILKmY**5I_I{1jZpC
z=N-&76^PJh$}P-xfoZqBu>JAM|J^A23zR88uqaQ9U%U`N009ILKmY**5I_I{1Q0-A
zf(y(otFYLY&W`I^W+<hnTXPc&rZcG_{RA=npuKr`U7}y>)w63vrk0DJHD<d&?`K+{
zT9W>vSN0buQ+{qyo)^D(A%Fk^2q1s}0tg_000IagfWYJtIKJGz)H<qxplnvTz1f~$
zY_tuCOqI)wne75+wl+5X*Q>MMC;JOjDF0<qUQwP^9#-xbZ+Rhr00IagfB*srAb<b@
z2q1vK#1N>fur07B2lbSeNhAl-wdJ<?cC&-PESs&_Dn7{R1JaRbav+}QEvqdvLSz@(
zt+qM#bab=cuZirqWR}c!fv^7VD(@%W|F5qZ^A0{_Q68EY87377Ab<b at 2q1s}0tg_0
z00IagFdl&xYo(=Iv06OQXvpXB_c-0I9-q at 2*WAuvkLq^@wU{fQYCYZ_w at +OxVpPRX
zejLHAKUs2Jdgb?<jd=$jvnY>^XP<;Y009ILKmY**5I_I{1Q0*~fr%ur*w$?yPs at Nv
zbI~}0t>3Hb9Qb`lg)xrc?%{C+cTc3;kro6HKmY**5I_I{1Q0*~0R)a^fqF5Fz&bpP
zpzP3%BiMby&H8iSj$UItk6^bLN1*IJmZix%0tg_000IagfB*srAb<b at M_Zt!%orIE
zbh}+HuP^BAiE6sj>yP at Jp`KvSsrN*^AuSm4dR0w5?cl}{eD1lf^O7AGJB;TM+$F{l
zD0dz0{Ih=q5I_I{1Q0*~0R#|0009IL$O<&tx~=1B9w<A|aRjfgeA_#I<GlTT#t~5C
z5I_I{1Q0*~0R#|0009ILn5Y6H#u3;K-8h1)v>(rS;^L>?EZYTa%9iXn0`U(o1Q0*~
z0R#|0009ILKmY**5IAH4EjD9hfGgzD{cgYC=?!{gPOs|sI(ve;&)K7Tf<2lq77Mzg
zYRi!vM-U&Z)n`B1`;r_-U{kKQDAzNR;E)wN5g~v80tg_000IagfB*srAW%|34i6Yl
z8-eYxk0S`3dfIpVt6O$5j-X^gu at C_S5I_I{1Q0*~0R#|00D%c0Fk&1*`Jo#}@Q;B_
zOTM*Od5>%tC|AzOjUy0mcp-oQ0tg_000IagfB*srAb`M_0xjj{$bfh-<O{`}KDXE7
z^lAZ*Q*-&^PLE&p1zd4eJT)MuE}6LF2>$P at U-(A#wNIQU#}Sk(m&kDhmoSoG%wlH+
z0tg_000IagfB*srAb<b at V+$C=1NPleP(E?S5j1rVG`_lP<~58X7`s?mg8%{uAb<b@
z2q1s}0tg_0z|j>LF^-_((2XOwBXjMGC*RSz#b_6pn;%D@%ssj}X730ffB*srAb<b@
z2q1s}0tg&gftCtmWPsPB`Mqj9=JW;Pn$zp?`J5qDbvZ*`zc=WQ`Ne|+0&4SA8AtHk
zn@{@7!{1!=fE-6qp?J-41YYH?BdgEe5I_I{1Q0*~0R#|0009ILn5Y7>*Z+803o0hz
zID)oipL+el$mL&V9Kl4beWVKk1Q0*~0R#|0009ILKmdUe;|MAb-8cf<uJ^efc(3ae
z*)C97|9rtX0`Y+t0tg_000IagfB*srAb<b at lUSgoGWW~?f6Nz;yFE_7PxU#yQ8nfa
z`GO&*=GEOnJ?QHR_4w4rV{;sV?WNDBKlkiORymHKQmM5lwYet}Fp^*r*L;$W00Iag
zfB*srAb<b at 2p}*Hfo!k;aWolJPO)(WE@`vp8q2*8)U3JTf%=ti`iiostxkE!QX%^b
z>{hlY=O}aQpQpdTIO-5V5I_I{1Q0*~0R#|0009Ihu)wj=UqBQ<Yqw&xc%spe&*SfL
zx?Me at S43QMJA*x{-x<_mu7IlbczfJFb+HJoik~%RyTHbW*4=#bTmJV0HOgaR-obo-
z0nuB4nFl9u)h1~OAb<b at 2q1s}0tg_000IXuFs1qn2#&__v<!%FMg0Yu|N37y9QRMl
zuj-WDmU7u&Am3Yn{sISI#fSj`1Q0*~0R#|0009ILK;UQ!OxFGa!t7$%10d*jyIfvh
z(Ag8!bf?!J6;Dj;2?m{dPt+UIf+4S0)zs5O=!57l at IsrKnf~IBw$_O50p;`;IGWX!
zeItMX0tg_000IagfB*sr99e<M&|g4^Y#2}TK$!@5K>Y<CfA6NqZ~j;RWu3CcVw3#^
za&rqXzrc~LjO+~o1Q0*~0R#|0009ILK;X~|Oq~7#B5ya!9ssV8NB6t^ey2C+jXAxl
z-|OrN>ON<W>IwE}zE~{ij;bvp^bzYX at CEnuir>uK at a-DqdW+5IFK~(C6 at Qpt;Luk{
zVnhG|1Q0*~0R#|0009ILm`DOgvA=*|xN1CY1U3=y at b(wD_!_&Xe8mSJt5eRgl*#@A
z*(VXm-U7 at oFp(-NX+Z!11Q0*~0R#|0009IL*nfe;*<V1U<{qO5Ks*@oh2l=1+v{<9
zwSdQ|xqNY_$FKSVuDB{@0f?zfMCb|HU*P4J|MNGyKAHJejdF>l%;+y5dkfHCVE?Nd
zTOfb{0tg_000IagfB*srOfG>#-(NuF{}Zmzz8eb4MZgKtUtrU_Z#(sG-<k0z*<Zk_
z%%#7;<f_A at 76AkhKmY**5I_I{1Q0-AF9PDLZg!@Amp|r<$K4*M->3SV-l!ULhJ3-0
zQ}gQXpdR$~gnE2xV at pM)MfL#jdNjXRjmMn6KwNWrJwEXi09AE4Ltei(=#TluL;wM`
zS?qR7^cOhm`ENb8=mK|C_7|`!UQ4#G0R08_Qma^n00IagfB*srAb<b at 2p}*;1jIMt
zrtvfxRLYJ8<7q9Z5HORizrfWWdFb}9ocy^<>tvx$x6of;id1g0fdB#sAb<b at 2q1s}
z0tg@=zT at s!e*q!(^s(1pVDoKXyKv5LxBa+ARv6hXV9YMS`~n<62q1s}0tg_000Iag
zfB*thU0~e(1q8$or%-<ZIlF;n&o!2NAE;S#!%tei`?&8`>ou~!K&5i0MR`SeR(V*t
zU%6X+;DrDJ2q1s}0tg_000IagfB*tVOTb=fn`4hA2jYp|`U=|uYjRLeX_-WFAYEBo
zZkul(OeN!qK0Q6lW^1;J50~lDOgdt0P*z)Jgvc(mTSZLi=w`iN6Pc-!88X`iK4g2W
z?xM99-zwV$tjg~!;x8`*5I_I{1Q0*~0R#|0009ILn3MtzYq_OeN%U*Idb(8`$|NI0
zgE1|mM-l at WJ+(#abBSzKi=Q=SyTHyR-+%67AOBXF97kYPp0J3&ybwSD0R#|0009IL
zKmY**5I|tE3Y=tXx9_E4KxC_E9Ki**J3su(cUjMu{ROI(?M34Vlv~6CUI-w700Iag
zfB*srAb<b at 2q19i1dgw^dF<&-G9~^`KQfq1_N5~O$(Wv=US(Tcy!>#DBRETD#_TWf
ziPyh*(Z?U$ez7r*;6;n_;-SkbF(H5e0tg_000IagfB*srAb`Np6qs(Uw9K1TXOHP|
zZKy93OQb|<+GL8%c7fa96KeV0VDBPh9KkE1zrfK<FZ)IS0R#|0009ILKmY**5I_Kd
zLoQHbn`br+h`7TtezRSmd1mI_|Fd<$BBQ^+KP<{W4*A}R5CH at bKmY**5I_I{1Q0*~
z0R)b^K)INc&~6vWSSfz;^A6hH_w%w#w7X9=`U|{nQC>gl2_*ptAb<b at 2q1s}0tg_0
z00IagaOeff%k9~AfueZ_|Mu%=GF8@#BC@}LUD-Tp-a)a57Xk<%fB*srAb<b at 2q1s}
z0toD<K!e at pE1pI$5={=o6TL@&-obX66>}WHV}E?}Z+>#ZIopkK1dmvhNA@$L1V;b?
z1Q0*~0R#|0009ILKmdWGDzHS%E at -#US)EP}rK0-MO#k50zOAvPxrZJ&vkyI3nwxr1
zBy@%N$&Vv={izkVy=9f}<Hk6Gmo3W6M>Q+#903FnKmY**5I_I{1Q0*~0R#@QKy8I+
zHLx441|sUBaRi>T9anwkvCmJF{RL_je~EDf%0{u67Xk<%fB*srAb<b at 2q1s}0>`kx
zyjt7o*1=RVp6Julk(j<E5f%R$0|Uxt)!3Z&kt-+gID&vINOK&)@((Y2{C~e3o^FgI
zpufN|T*=8g0tg_000IagfB*srAb`LW6{xXRSnRW{sbn%!CbAfm*)-b)9<BVt>(@W|
zi=P|g2wt@)uTD_}vV{Nw2q1s}0tg_000IagfWWaUP+e}HZI=xLf*>Rr%yxlq-1)+P
zZhGQ#SIGVXWy+IcegW}|7Xk<%fB*srAb<b at 2q1s}0tie}f$3$kcfdS3+hA_4L8HHb
zGN@%XFBO?uCVuks4!ZvQ?8T3}FI+173zR7 at i2efN7cT@5KmY**5I_I{1Q0*~0R#}3
z`~r5-OTa1{2+Fek1!Tj3$W+n1gYUfe;+~6+`+hI;4o?2!CI<*0fB*srAb<b at 2q1s}
z0*6arg3UWvRa<VGpP%Y*md(~ICq=wWk7n|dA=Z`|A+ihYR-1SpL3Fd;uL&WSR5HgA
z{Av1wgN<jeiy89{K4eiIs(;@6bGS5#7XbtiKmY**5I_I{1Q0*~fg>THoMUP2R;(76
zKjw?a-5#gkr}~`Us2X#Ie8G at Y^Xl%P9`yBudVFf5NYT?WwAZ8gy=pw>^abLY)9dm1
zoFP?pIYVB*H|US~gFRY6Z5E;L$;KUu2SdJ4-05?BJx;F{@HjP>FYfgCRbRjrSA!vM
zOkE;E-)MyP=zh1~@AL+}F{fAcd!0Q&-RJC4JtA6PEEaS})fN%@Vi`K<cDr0&U(neT
z)pV!V9~CL-2?m{dPt+UIM9f}QQ%`HDs<g<^QLif&iiy>sKtQD1-y at Uk)}4MWsK&ki
zxF at Fi)x{!iRs7_~5xmPc_ at m!kp4n~8JNT$Yc~tRc|J<eACBpGS009ILKmY**5I_I{
z1Q0-AstPEVSenMuWKbypemI`if(jAv6RvSI6v&?ZSGhv_ZX>XXfE&isJkV(CwvMM^
zph`qoG>+iTo9Fhvdfz+lVH|;w;e`MK2q1s}0tg_000IagfIvxs2{w+P`p}Id_(HmS
z<~J_8>3Z2NP^Rn_;|P at 9j3X#n30Q~#0tg_000IagfB*srATY at UChItYmTF^UfF?%^
z_&pw{Cl>5+dcCobvqz8moq9Ct_V<Lm at t`k$u;U0m-;#do-(LKA)Oa4j9*eRkH<EyH
z1e2_`lUxK4KmY**5I_I{1Q0*~f&CPi1mg(g at PP5O4pblTID+I`uKd at oO#4p85$tE}
zAvgjEAb<b at 2q1s}0tg_0z%e8+!Nw8T58XI|vp&1v>R+y3xLmdi*pw~VaRka1V<f>b
zRKR2v0R#|0009ILKmY**5I|rG3QU}F1TA)BWPnHX|5tVKtb at 2mjf&?HM7_?C?g}|Q
zZkIn46%QOx0|B+=NRA`;Q`M8N{e8#jSIBV$Hsv~ta$RmD0pkd!;1NNV5I_I{1Q0*~
z0R#|0009IF1diS~0@>?-JWT}l!#|GT%-`Pj>+jrp&pQ}LP*5{?ivR)$Ab<b at 2q1s}
z0tg_0K#{-%8%I!c=*AIz<ECY+zxuUze^j;$l#31)@_7Wx7Udi{lAwry4+tQD00Iag
zfB*srAb<b at 2uw18!#R$irN$T;5OT*uL2pcV26bP+=?%u>;+<POJ-`<Tc>^I|&=b|v
zB@=fX!5srX3ak!3`#*9VLAi3V97k|*b|eAg2qsx=C%Fh9fB*srAb<b at 2q1s}0{ba&
z$j1 at LUjO51C#ab~;|T8g(~rJw`{#SlU>w1I)*gZ*fB*srAb<b at 2q1s}0tg&K0uyW;
zLG7U%M{x7o&i?#$%RcZE*)C9_%*~G at Q06j@;25glWEBAf5I_I{1Q0*~0R#}3k^=iT
zj-aL17#R at I-MY^g@;akp5P*1QfKQAO at VEj_x2F07x;Lr^JzlkWs*EGBU+{n5yz`SU
zu9D*jDin_#N8m9>5=_ZTL8cHu009ILKmY**5I_I{1Q0L;#ygHc_WB=BV?pgC8Asrf
z+23=G<=zKs*4&W#*nO3$`OWq^<smV at faoufopn%|OMd|l69f=I009ILKmY**5I_Kd
z$u4kg^cN7N)Y`3BEm5y47K#Of&QKtrIlcZKmowyb>rTHGRO4QM+!Is%>S7UE6+dgt
zc7gf#{kQFkt!2NeQ63fT0`wP{>~)^>BY*$`2q1s}0tg_000Ib%TVN{n7Z90l8c)MO
zl?YhWUtq=i&sVNK|H^;VDZ4E;*<V0Bqfqu1pufPl>k$DFKmY**5I_I{1Q0*~0R$$f
zz+~+&AUH3UJpeRsNDKNs9;YW36i+_%#zM{>J?eMr(Wu+s6Y|D`zPNgt2z?O!1<qWt
zyJPdc>&t7DJ)&KJ{sI%UDw8Y(5I_I{1Q0*~0R#|00D(g<FbVn#h)j2mr*)uO1U#_*
z0xx~0X4C9z=YP6R*<vY^{RMKp1?Vqu=qnsCB7gt_2q1s}0tg_000IacS%HbuUqIyh
zM%e?vBmSRX)#EW|+ at nUtvkszOXGnL2oF2E!ABy^;J~a?fTSVw1)?eUDJ)06Acye81
zjdGo67ZCjg<kJW)R`w{5Dt8^((qeB2Ab<b at 2q1s}0tg_000IacQGuh^UqEE@>hUxY
z*hRp at ++X0Xmv%I~bo2LG>qH}pRrVLi&Mm<F0!OqavNHq_KmY**5I_I{1Q0*~fx{wj
zIQt8T1m7cj0EFD}P|zFGok86faC(EWxOnFlvjO-5A#Wh$3wolOx<rJYp#23>cU=DB
z;ErGautvF9v<uK*;ILFgB1HfJ1Q0*~0R#|0009ILn1}+0yuX0R<nHmb6V!-+6Q#ev
z&Brxf@%X%OOPwsC=@$A6NC#dBAb<b at 2q1s}0tg_000L7&VE_6Hh%7%XdjJG<x9;<W
zyw0fj&weonfKT)V at VEj_x2F07x;Lr^Jzlk0gq{-p1%7qTTVH<o7dKoc+bXi{0?aQU
zC3qo#00IagfB*srAb<b at 2uyu}@%9%G5<ePGV?nJ5I2rm2l*^*sbB*QR2Wr;bu<DAN
zo_zgn`bTAdfm+3HQC?A=RUTIESMFBsRBlnWE1Q*#VlgiS5I_I{1Q0*~0R#|0009J!
zVS#zIw$rVHsboCSr>7$^eM=%L{*5LF;)&j}Sv5ANedNjpyUk}$XObzcSC0%PlYM6J
z<Ew3+5i16gF+DxK%C at +8xxLah$8PMSzQVS^njF+qS|*VkNKdORx6RM*a+b~3EE9K`
z9?h70DXT3rLSz@(ts<s$bhF;Ci9&SAf;8I&Hac&- at 1h^PW2<ZzuqqE(l!wfj2kW1g
z;dmi{00IagfB*srAb<b at 2p}+71&)nz1TE7lEn at tG-RIMSE^o}|6wf>KJH1ht%Ng>i
zA*Wwe{c7Cp)_PnawOW*=Dt^|O?E>FB|1;ayFFs?X97kYP9<_+)6Y`wG$y(t_Hv$MC
zfB*srAb<b at 2q1vK{s>H^aRiOFZtHj&2Brz1qHzS?|6b94?fXKXk^Keg726o&2$W{!
zII)r!0tg_000IagfB*srAb<b at lV3oow=J?K`b7u-bR^q_e|DX1sWrDi^v4(7^)m at Q
zU16Uldfyw1r_?xt6J-&a{RPUOe(De3tT^LM#yEm=Ey}r*zktaB0tg_000IagfB*sr
zAb<b at M?he^+p1W$*DbJBTbD22SZ%S}eSx^o=W}~?-R;staj#d6>(M|w)Dw^Ogk0`;
zz!i+e-0mK~J1zpmL+)6>=T`%IDC!Mr9?jpkHP)KZQmtFBYX8n7Wp0-k1>o{IRky?C
zT9*CWoc*na0xqXZ6 at N#p$Zz3sxctjJVk5OR6bQMzUbow0Y%5&mOB=IY;GBUS_uTZc
zza)%t1Yfr(Uq1qJ>;eG<5I_I{1Q0*~0R#|0009K{Qy^Sn-(GjnO%GzH3*?T?c7b=s
zFa5#4-Q2#(=r8aOi}H{COcucrKmY**5I_I{1Q0*~0R#|0U=j$FTgye429eN(GO=d6
zK+A(O_N at ESO*a_*1zxu(uTO%kkVFI!KmY**5I_I{1Q0*~0R#}(UxD&+d$wKRRJozq
zE->%h>puJS%(Fd4yTBV3<&FJ~mhcE5fB*srAb<b at 2q1s}0tg^583b&$Y`efBk=Fck
z3h#U7M`xY=^^HH3;|MC0v_*MFc~JSFa=m!V3jqWWKmY**5I_I{1Q0*~0R#?SU`9oy
z#a=ycc1rKlwY09sdiDBpxoF0mWV)?yFfpKeZ1VFd8)AAono10263GE$g+qpRX#Fw2
z*U>1a6?FMjcSQ9p^8}mbmdTCgG^O=aLhCDD*<h9H7By(8=;lO5k7kBadNvl3oucOy
zzW1K)cfIS`&;Cia3n=w3?e#f at Vl^)W5I_I{1Q0*~0R#|0009Ihm%wzzw%B|sVLCaK
zit5Mua|%xr1(@$I at Q>49d*rtj at 0(@JJNSS_d0=uCFR4WU0R#|0009ILKmY**5J2FV
z6!2NAEcOl7x>Gi1GK1-5OP414wO&2l+Lss at +8Rv`WHiwaA(d`Trh3b?Y^Q{!vhRVg
zI#K-O+Xa5scK^Ebzxn7rM!Uc&i?ZsN)FjggAb<b at 2q1s}0tg_000Ic?kHB`nRk3WZ
zTVSiUE?>UUE at t%e1>!!R&+XN9w at VMjy<Ro0M+5OtPdwHWa=GIHS1=ZHyL<fZxCjsr
zxnlvJUk&J?s5hv2G=JaLSZhX0wQjwt{X378$x#7Lm(QuX9WK|h?BC|>Z#5KfIbEvw
zJ7PtC3y;I)U*-`TsjZ<v$mR9A-5z7xi+@@FN0G^*aRk-dpZflXpZm~FvRz<i{g?MK
zj-dX@`UgcIUI-w700IagfB*srAb<b at 2pl<q<7e7D_Tm2lj|`@g at kF1Vo;}01)LO7y
zyp1P%E9<7)78C?$96=^+x7y~|)6va(zb1-JKC3YQJc2iW?T7PU_{h#582ts_ZBgER
z<O-HOA%Fk^2q1s}0tg_000IagaBK?<TI($K#n$TTjn&KR?90=cp at D(KK(Dhu*{2Wp
z87S-l5b~<7pxdA80WfCo0hcS=r(krq0^uzmNtkaJ*!a&|e{*y5(2d4(3SYD+FCN<^
zLhcbj009ILKmY**5I_I{1Q3`y0yE0(i|s|N4I*!{Ye2qTK&kjkGO|YdxLjCS|1XR3
zr1EX$?MjQ%AU^Oy009ILKmY**5I_I{1Q0-AY6%=)St0&|es*?_zSe>V2hOSx%jeHA
zmYYux+$5hS=-FPkCE2HC#K?u#9`TsLNH)yD&L$AFiTy3zVC-+do=$jjnb>ksld)x~
zM--lD6#>)+BS5L=6xPck-E)oQ-Un*d+;CRS<KOt>`&WEb_7|vBuD2+ERKBI$qFk>$
zs at yTPik{pcfB*srAb<b at 2q1s}0tg^5Q3Pg+|Hro)-`sPJ6P0!4_W8L*$#lgD<#vy?
zM0-T}@#XfVxnKp&2{zF&U{P+d(N<6<0;<OI8Cwe*2ZX?EsbIDXeB<)RZe9E{$K}R2
zf;TM68xzHdbRd8L0tg_000IagfB*srAb`N36tLOsa?ZiyWt?Waz{;1tt#10xo>j)Y
zgRfbX*AC?#i3<S)5I_I{1Q0*~0R#|0009Ihgg}KHUXX1UXpreK+XepiySFv%zV^zy
zjs60!Ta?!)L~2O{0tg_000IagfB*srAb<b at 2pn30@^YhH-~<`1*)9-T_Ja#szxM3)
zJdfbeViOwz2q1s}0tg_000IagfB*tVU!dgk2<FIine77i-CxoC{dVVlM!UcZ7UhMb
zpK4No00IagfB*srAb<b at 2q1s}0*63gW`(`Pa|-9mn9X(pZRI<oH{BhwGmhX8coPKz
z2q1s}0tg_000IagfB*s$SHK)cFi#|LjoB{nmm444dE5KG@*bmIfaeiR+~Ol;2q1s}
z0tg_000IagfB*sr9GpPzc?9J$7PDPo-pgNbeP(I9CZ9(ztNvpa<rU>w<zeN1<!<Fp
z<rZbTvRT=v_?2emIK@`~QvH+l57vKK1mlGO0tg_000IagfB*srAb`N(5STg3)@)B_
zk}0iMk8DZy4fX5k<7e7D_Goe-qa_CPR5~)4O2!j?dV2N at +fr-6a`85v=&h`qZd*_g
ze7a&=Y)|xy_|lPdawrwmm3rGEbICw5CL*d65#@r2$OrXQCZVS*?9*&>a^dFH+D^Bc
z`;El(Es3az&e&nutQwosK5}J)-R8?@WH6cRGlL&rEplV#&{&a8#PlkW&Z6b^O0j=q
zAN3Ws1y-3bEt5zNq^H%E+vewYIm>2imWjJek7mrhl+~6QA+ihYRuNM=x>@hnM4_#c
z1!uMk{KvZFzdio%JHy6v3ZJzo&mN9qB~AnoKmY**5I_I{1Q0*~0R#{@k^;wD?Uu!M
zdv#s4Xlxi77)T8CI{TA-x=6!0;wL|j;KO^?1V8fA$DcOF5j<y6o;#AMWM2p%fB*sr
zAb<b at 2q1s}0tg^*m;{ci63-=EJi1+=XdJ;CkDPMKovW at WtE>OA7)NmZc*YT2&p3j^
zRANMn00IagfB*srAb<b at 2pls44Pt0O;Sd0`7k*i(aRfr;go1$q`xr-1$2bBpnqqI`
z2<)<&nd1mv`#{h8KmFWIn~ZS;cUY7=j+shGMiD>&0R#|0009ILKmY**5IFh*Df?n;
zb at j&T<#qPu!#(~lu+~|ISJVx!5JLgV+%7fbbordB$KmoX^SG9|)YeeQtGa at 2zo*$H
zK8pYWm(!(+zmDu at vB+2<GAx?_^6diCt!KRwuHSZ|Tv%1V#-g07IP3pf|GWD8>pv+z
z@<IRs1Q0*~0R#|0009ILKwu1kO?8zPyJvfym>p2ehS#UJ_GoE6lAXTK7l`|OKDSrb
z-7Y;8_j=X19u34pJ at Hsi$mNa)T)|k(?e6ir<03#j<c<Y=el?(nqTZn9(flV*tFYJ?
zHD%*!P3x(I)+fIUyL_rUqI#Bjg2&g20E=fE0aALOE_SWQdi7Z~V)^_z#`0vkt#43F
zRT#93g-bUW3&k{p(NtnkPTyy&b;#`;T7S&%bu<?5rRl_Kv8CEzY^kN9n-du^x8G1o
z&j#P_u95;fmgfZ$v3_l^bwJ;mx#;SJtJ6b?*urI*)R5jHKJ>~Dajh>cKAn?HXXHoK
zC0{KIJJRx#c(YC$OlbqXy17cOerroUuH5Fi$c&ixFRn{OVb;q6HTw&E%(4Ed?`9sq
zT+T00rSw^ppD6bzpH)7jT&MIY&&ZIx5I_I{1Q0*~0R#|0009ILK;S?GPAIo~tocTR
z;bxP%8Rhmx#sd&?tsc{6mW$p2Ma>+wb>;T?xkbrz&GF^-+%pqe3z|0U4dr$<7pSn^
zqPlq3!|fGSA|J=gyqN6*Gar6s%ix=x5n~*|YZm3T1Ia1dAb<b at 2q1s}0tg_000Iag
zfWU+ns3;SCAMA2&fd=uDe;&cVo#9w^$Flcl`wLi=e^|s{UI-w700IagfB*srAb<b@
z2p}-!1S-nyc59i)R?+ha=709nvzx!IT+Q<crd)xOAp{UW009ILKmY**5I|r83yk$V
z0^7kqk6<>>BiP6D2#V$%yuJCPj?b-lx<xLmu3u_Vo>V at qykEImd2{{0>eq+`ybwSD
z0R#|0009ILKmY**j%9&4vqkT7kCGoGZ;VuzL*-rD-N#M7Y4<kGI((BQ&J<L8oh)&N
z2vBOW#OY%BsL2u)vGBkqORN`LjyYMPsLVz44*uf#=l3kW^<0;ncd%O7Xi=V3ex`g^
z`GWFMG4tR?F%cmz1Q0*~0R#|0009ILKmY**5ZIeQsX6{+%gO9gE9Aopjw at a<JR#w%
zQ4<o*96cf7jG_q%r<b0PP#I%FLfIxzH1FUIecjW)czOS0%saTZB4!-|2q1s}0tg_0
z00IagfB*uMT%f`xx)F??cks5euU at fIzx~s)zX0<NPVzD)`3N9 at 00IagfB*srAb<b@
z2#hI^op;bCRu|1Xcw6SKr5E1o{~_}ZjtR#K1Q0*~0R#|0009ILKmdV5E-==-gXIT5
z at 8C(yJGejd4xaYsPyF{&<u7cJ;|T2abr$85%8klq<vith#ajRS`ew0+7Xk<%fB*sr
zAb<b at 2q1vKR24Y!q`gg1x7|H|@=Yvw@`)l#dz<}l);tkl)GU8<#li!c<?r}8V#^Yf
z2|7*?%a_l{PuSR}r8ALqM$71}*|{4px at 1m+2vK@U$4$o{w%HLyRW6!$@cv6*+fni4
z?_ZVU2<(d2qIku8gI6lIDZ28E@?&MKa at SO?{NxJ(1Q0*~0R#|0009ILKmdX935+p^
z-$}*O2O5uRkn;-8A2og8iA8e-&MQ4v;M~!349+>J+`im8X5+#MrBWcLF>EMaF+8)O
zoO7^f-oY!HKlpE7d*6~fnRjq}b%9U_Ab<b at 2q1s}0tg_000Ib1N`Z=U(LG`Gyn{<B
z{!#sjt3DB6-oZ&(sw5f#1Q0*~0R#|0009ILKmdVp3S{RUlyeTAD1O%LxyEwu12t=I
zsJZiFuWefV%%w(ufh87Y$vC4U7y<|&fB*srAb<b at 2q1s}0#jOG`$nr`*<QE6R&8Cr
ze4~B6-R=v-eLkPttLtu;9*TRtYFv*7;-Q{+tS97h#{;flEarCi_}y_4ARcnZ0zSVQ
z&_hvgQ1fX1zOAv=jFxKMdR6;(9w~FX)R5EVbE<BK%e5@~w>kS;4Fz0Imn!~_Sdrht
z<8b+xdBjF)YbX$MdA)A8$Jlms5O=G~?NvQ4zY#zbS<&+d0s}w)^XV5goyhYDrgUW{
za|j at S00IagfB*srAb`LG6Bz4x1QiGWJc34^N3cK7Bd9pD{41|^UU`S?FHlqWM~m_s
z<vwMXl2khCZ>is2e_DO5SjY<j1Q0*~0R#|0009ILKw$C-I2!jkA^zgAC&WMPuuX`6
z{35Z}5)=4ucb_^5pMG$P$jn}!ey~sk81?jn1!Ca=J^kQBhuBhWNc3yHdaIU-Zcb$M
zXl5v-YyC05cj3um?dfy!(-dlhgMHf~8NGk7Pdo=9Dney+QIU(DNAUhDPd?lK#!{D@
zU!X>5uqY2IUr|0Ko>ce&<qyhw#jji~7V<&>0R#|0009ILKmY**5I|sR3+#Q~|Hk6!
z{f#FN$VvVe7q1we_y4rf&lXrz9CrBe1*eXBe8DM2k1tqQ`tbz|#&}GE<HU0N>BG-3
zC~h1$SrlW@^9Y{YV|(P(cYpjIo<}gXD>%7B009ILKmY**5I_I{1Q0mJ1u7~;XM)ks
zBUo at w<JP$|7HsBu1jl&&Ao~a)fB*srAb<b at 2q1s}0#jTd`#gdQkx7@#sQEmCE5GsL
zJ5Jv6%O2TZz%t!pQ65s}8h`4amy38IfB*srAb<b at 2q1s}0tg^5Sp}4HEUn#&)#CEU
zeDS#3<MjJfpVJ#vW6qE-7;<V}-5u0}zMfEzPi+*rd|HMM=x*KT3wfPUU(n<9sy^Kr
z^tb{}x2F07x;Lr^Jzlk0guX|H4!Pr at pf@JM=)Qo{8;r%pJ9m%M>kEXu0kIoTR8!@%
z|8A6_J#K%Gs>fr_xJQkOxT9WYNOy&t9=FRMiu$8IH4so+MCglUXw4haf_{(3>4^nJ
zjNVwt*`r7OPCXhG at r1nbpf9eTCPK?x+I>DP=<>#VP7#k^<kIC5xfG8)@T;m{jl11i
zk1M2Bi_oh0$v>ws at QW4dFXlY3P4*YawhKI}TrB?VQ65$9l8bmDfB*srAb<b at 2q1s}
z0tg^5MFo^gEKTESG7#DS(RdmQYDK`^<7p?5-@&gQPZNP%1neA7>wx^a-!z_vfoURO
z(Q^vV`qvx3*Q){#@SH*+!wUff5I_I{1Q0*~0R#|0;0OuuoWf*EJW?Psm`wJiBLm5p
zo}ON1TWn8f3YOa|ZFB6=<bZhWV10#cfi*d(r?gBWIgqZaEw|0L52ljwM4z6XWwSL~
z#fQtpqY2UxV}r8VG9yHGq1`HCN=G;A{hBD?qHzRI{p6Q5?|NqDN at L!^hZsk2gpL4q
zg8%{uAb<b at 2q1s}0tg^*_yvxQaRe=Ol@>XUz~zYs#4rT6)31fZh=x!w?(FdgqfS>a
z>e1xr1vTbUWmkowaRk%aI(?R{?^z~SSd~XDV*CK(2o8VMWDf`+fB*srAb<b at 2q1s}
z0!Lq9Dvcv(v~^p@(=Z^r1<26}`FRKbFn7_6M_;<=Y@@%xmn_Pcj()+A0t661009IL
zKmY**5I_I{1dbVjlzp+ax_V>v@;dwSbY^H^ATiME>=)VJlv}aZT4%A(FvHYE)iSqB
zOeg5_IaQCt<zMD;Epw@@p^#T~1>Js6vrBvy0Rk?kOBH_|+0SB;u|h<gonp{@9>HzD
z`~02@?l|>9qrbpw7Ui{LMxKl!fB*srAb<b at 2q1s}0tg^5^#x{%Neg6aLuI*rhCM3B
z5S$=0ZMF+M{``IKx#fG?FQr{z>Q at DFhyVfzAb<b at 2q1s}0tg^*3<`{H7nmay^3N%3
ze(zta9ha>6w9zi`cZ>4(W6+2!BY*$`2q1s}0tg_000IagFhvDs$%h<>{sL7McC)|0
zDw$QYzrdG19G&;U)Y%^~<`?*@MfvL#g&<o9Ab<b at 2q1s}0tg_000IacTLLF6w%e=g
zs_nH_yJd#)Ham{s9Kn$9FR<e&Pvy??pZ?D1FYq^u^0#9PiQFQ900IagfB*srAb<b@
z2p}+Z1!mh9SIhna_A0yRE>LaEFW{2-HTw&k^p7)MnDxSvS7m<z%XEuHc}SUS{HcFl
zF5-m%0tg_000IagfB*srAb`MR6;RHxw00|2i_0JL#p7;|)9+J#PH$9=IYYi+$f<dC
zcTf-ddO|%uwNd2qX&E}8yLF#0<aI`UL66g``gCW|;|e(4n(7be-l!h*c-3YR`W_iN
z<c^1e-k1oZ`vOjHFcuf at +&xaOFA(wu#BMxMO<f{F-zY<S-2NU_kH?&Gj~W$mN4?IF
z?g}|QZkIn4^+$bbAfUF0&=<?lnm42c{T`3g6AOwMy|Iw9N00iQdNeBH33=l|UtB#+
zgqFFqkLNLks_YgZ=N`=W7x?V!pQ`xhwQD{uS7h4-9#t+DfA%PkDtF06ybwSD0R#|0
z009ILKmY**5SXF@$|aVj at iZBT?Eh#yjRmzL;O_CX6Ugu2SC6NOz%BxIj;D2?S_Etw
zPs4!h7EmsJ^8E!`uhiDMPrCiz>gvC2nN@$iMR`SeR(V*tU%6YkQ at KUiu54B|Dt at I|
zIZm<Fzf}KZ{e$(_*MCgLzzYEc5I_I{1Q0*~0R#|00D%J)Xqaj9Sfj~-jFuSCQ|ZWH
zDj84o>FJ7 at vuw?x<9jlt_3Du=$-bd}JuP-}g1toG*)wcQ3nLLh;)&kIy6LtB1;M8)
zw#D{Dzlbj#NhgO=QC+FGEi#u3Bx8DdcAbbQ7eu6FP)}tNA}#i5wmG?Q^J;CUTg^m9
zV)~XuR77X&uxwV1&1oOGvcYci<ufvvO!k?<kFU0QMy$vtVtSQGr<wEY2KGv^e`6o@
z6}AOdnJ_JrNDidyYs+o(^Shj7vo*`aU8YAf=3dHb%Zw1&g?6imDIMLc_iF;PXdJ<*
z=lyh`*_S!dn0N3Y#t|H7m1a8x5I_I{1Q0*~0R#|00D%c7aBPetXsNHX$WaHrxW^L>
zdByOEKrrU?y8W6{^Z8;<J?0I>-MUwed*bS1QAkDO2;O$vf873~_OE<L_F=Fpk6Og|
z0mczbxSC9Y5I_I{1Q0*~0R#|0009IJgTPc8N6={NwvMM^pk6o at jU)I(@8>@K`;G7Z
z6ypdELk%NJ1Q0*~0R#|0009ILKmdU;1Q<tPjFd1280_~r0_D(+Blzy$mVe?$H+^oc
z97j;5?3UvQcJn-fF{&+#5kLR|1Q0*~0R#|0009IhmB3^jN6?}eBLlRkHx`V0d&Kh)
zLYmXt<8nKDJfVOy)Z>bKyk5=Y_UP(q2RDvj_ZL5O+p14}_gp!SpiJ2##}Vw|c?6TH
znv+-r5I_I{1Q0*~0R#|00D%J&m;~bp<nVy;v<@f-JdWV}|M>T%SFhOmPR0=&SS4aB
z1Q0*~0R#|0009ILKmdUQ5nvp_L5(Are(1&#Japn$Qvbib`w5Ms3g9 at tX__9Qo5|*o
zL-DB9wEN2ZdGil?k|MPS|J59XJ at _U&4_cw1heZhZuS7IQ4?;!IN`i<qJ%}P0tl)pE
z2ZPXqh3Z8twxwQ*pcLQRO`skM)6H&nHhx+1hwx^1@|kzpx4++;Prf-lYkLHvPOH-+
zXxS#g#N}?TEXN>#00IagfB*srAb<b at 2p~`vf%5bS>XYlu09QwzjAIebg;K~Q(jt{v
zDztLr&<ovAY2T}FWsgAqdE~@r$G%H!k6_eUuswnW(<E3iKhKoqI&vNeAb<b at 2q1s}
z0tg_000Ic~L11t_f_6Qiz!Je^|9b=voSR*fZ!RCCN6?2E##17I00IagfB*srAb<b@
z2;4$|9>J!11hwAv2&TQSuU1=+-fepXJ59i%eLsShbBG?nEv7DiMgRc>5I_I{1Q0*~
z0R#{j8i9WH2<o*?Ga%1$UzrMkP+FR1fDB`ig-IrIsZu}jb(AaD+gIKm!Tj4NUwAhD
z{($Wf>~tC%J%R>3f}xq_oG1baAb<b at 2q1s}0tg_0Kz9N??-AJ4f01>9T8VlDmyW#t
z^MR#Plk^C>n?xLi00IagfB*srAb<b at 2q4gn06l_D^$4bV*CY7x;rUN5zPftB_6WwD
z-Sh~$na&)800IagfB*srAb<b at 2p~{Of!pg5)TcVlfK<z-3?pC2Tm|M^6pJ)cQlx$w
zr)io5sULWIw at Z)UbN|cQUB4_|w>^R}C$K$&pnXSz2KOT<<)r4M5I_I{1Q0*~0R#|0
z009JgB~Y|SU{n7^77M0^q(|Udz?J7JtKW_ver3m#2OoUn{vTeeI%{Ts0h1S4a at xBN
zvhQH8rWwzJ00IagfB*srAb<b at 2p}*hf&WKd!07Iubw(<o4uZK<rb5L at Q%FBjLWN;d
z<V_iCKbM{kw0Ex=?U}E`8|wly=f61hPxH5Y+&OR71x#LGBP~E)U{KSR*F^vU1Q0*~
z0R#|0009IL*cyTDlov2!Ges5#Cd|P9<OQxh^V8#}Z!AAwb(SilHZQQA79cOMHItB6
zK>z^+5I_I{1Q0*~0R#}}pTMx at 1&m0;CIHl&Y$jS}!t80Fgv?xDWI<}SJIGueNT~up
z$h|#g^d|BGS5|L)^?UWgQ{&ExSr=H(3p5=2IMcr?!!sj*00IagfB*srAb<b at 2p}+U
zfg#8X7^BCEtPVJ4V0U?elhf1J&W_#lUe#$;cG$eYA!m2zF>!g|SBMip009ILKmY**
z5I_I{1Q0-A69VPQ3z&amv7G>Lb>zu77U5hdg-jwXQkkVfD>n|k&<&OLy}B8_WqE<+
zzdt;8wep0?3oMv*fp%V?lNKN^u!)(+p$H&=00IagfB*srAb<b at Lm)7?ynw-(R|S>`
zCe6S;=LKGT=aFMCzrXf@%?pf}g%t7vLojnWMFbE)009ILKmY**5I_KdLISs!7cg&(
z+dcgvw<PrT)@!>eHUW at lxv$Je{X%JJ5&$xcMHVKR$fZjC#Me=-TyLK_-IC at 7{y6dO
z^^e}V<k-Bxh|{RFcNSoOfkN|(K?op#00IagfB*srAb<b at LoHA=FJRs=&lFiKn7VIt
zcBIHULCvt0B`>h+=+l2av}Zo8+DmEj3FHNadQNll2q1s}0tg_000IagfB*tF3G_2B
zU|#0q?T{CE>CVMFe$CGY<JQft3vBEz!2SX^%_M$B009ILKmY**5I_I{1Q6JU0zJ<Q
R7^9`K=LMSC{_N=S%D=MJZAbtB
literal 0
HcmV?d00001
diff --git a/lxd/daemon.go b/lxd/daemon.go
index fc7782da36..1d82f168f9 100644
--- a/lxd/daemon.go
+++ b/lxd/daemon.go
@@ -671,6 +671,7 @@ func (d *Daemon) init() error {
for {
logger.Info("Initializing global database")
dir := filepath.Join(d.os.VarDir, "database")
+
store := d.gateway.ServerStore()
contextTimeout := 5 * time.Second
From d55bb37045b6b512459ba92b5ec676cade916a08 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Tue, 14 May 2019 11:30:47 +0200
Subject: [PATCH 32/39] Check dqlite version of connecting nodes
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway.go | 11 ++++++++++-
lxd/cluster/gateway_test.go | 2 ++
lxd/cluster/heartbeat.go | 1 +
lxd/cluster/upgrade.go | 1 +
4 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index 419a8b42c8..440944ec96 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -10,6 +10,7 @@ import (
"net/url"
"os"
"path/filepath"
+ "strconv"
"sync"
"time"
@@ -108,7 +109,12 @@ type Gateway struct {
}
// Current dqlite protocol version.
-const dqliteVersion = 0
+const dqliteVersion = 1
+
+// Set the dqlite version header.
+func setDqliteVersionHeader(request *http.Request) {
+ request.Header.Set("X-Dqlite-Version", fmt.Sprintf("%d", dqliteVersion))
+}
// HandlerFuncs returns the HTTP handlers that should be added to the REST API
// endpoint in order to handle database-related requests.
@@ -478,6 +484,7 @@ func (g *Gateway) LeaderAddress() (string, error) {
if err != nil {
return "", err
}
+ setDqliteVersionHeader(request)
request = request.WithContext(ctx)
client := &http.Client{Transport: &http.Transport{TLSClientConfig: config}}
response, err := client.Do(request)
@@ -686,6 +693,7 @@ func dqliteNetworkDial(ctx context.Context, addr string, g *Gateway, checkLeader
if err != nil {
return nil, err
}
+ setDqliteVersionHeader(request)
request = request.WithContext(ctx)
client := &http.Client{Transport: &http.Transport{TLSClientConfig: config}}
response, err := client.Do(request)
@@ -734,6 +742,7 @@ func dqliteNetworkDial(ctx context.Context, addr string, g *Gateway, checkLeader
}
request.Header.Set("Upgrade", "dqlite")
+ setDqliteVersionHeader(request)
request = request.WithContext(ctx)
deadline, _ := ctx.Deadline()
diff --git a/lxd/cluster/gateway_test.go b/lxd/cluster/gateway_test.go
index d151dd0809..9da768d26f 100644
--- a/lxd/cluster/gateway_test.go
+++ b/lxd/cluster/gateway_test.go
@@ -37,6 +37,8 @@ func TestGateway_Single(t *testing.T) {
require.NoError(t, err)
w := httptest.NewRecorder()
r := &http.Request{}
+ r.Header = http.Header{}
+ r.Header.Set("X-Dqlite-Version", "1")
r.TLS = &tls.ConnectionState{
PeerCertificates: []*x509.Certificate{c},
}
diff --git a/lxd/cluster/heartbeat.go b/lxd/cluster/heartbeat.go
index 709e2fa234..0260ffa364 100644
--- a/lxd/cluster/heartbeat.go
+++ b/lxd/cluster/heartbeat.go
@@ -214,6 +214,7 @@ func heartbeatNode(taskCtx context.Context, address string, cert *shared.CertInf
if err != nil {
return err
}
+ setDqliteVersionHeader(request)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
request = request.WithContext(ctx)
diff --git a/lxd/cluster/upgrade.go b/lxd/cluster/upgrade.go
index df649da997..b489856529 100644
--- a/lxd/cluster/upgrade.go
+++ b/lxd/cluster/upgrade.go
@@ -36,6 +36,7 @@ func NotifyUpgradeCompleted(state *state.State, cert *shared.CertInfo) error {
if err != nil {
return errors.Wrap(err, "failed to create database notify upgrade request")
}
+ setDqliteVersionHeader(request)
httpClient, err := client.GetHTTPClient()
if err != nil {
From d641b130fa35c357c4ac4e4bf258f7c021736133 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Tue, 14 May 2019 11:25:09 +0200
Subject: [PATCH 33/39] Drop the legacy /internal/raft endpoint
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway.go | 44 -------------------------------------
lxd/cluster/gateway_test.go | 2 +-
2 files changed, 1 insertion(+), 45 deletions(-)
diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index 440944ec96..5a6654739e 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -243,53 +243,9 @@ func (g *Gateway) HandlerFuncs() map[string]http.HandlerFunc {
g.acceptCh <- conn
}
- raft := func(w http.ResponseWriter, r *http.Request) {
- g.lock.RLock()
- defer g.lock.RUnlock()
-
- // If we are not part of the raft cluster, reply with a
- // redirect to one of the raft nodes that we know about.
- if g.raft == nil {
- var address string
- err := g.db.Transaction(func(tx *db.NodeTx) error {
- nodes, err := tx.RaftNodes()
- if err != nil {
- return err
- }
- address = nodes[0].Address
- return nil
- })
- if err != nil {
- http.Error(w, "500 failed to fetch raft nodes", http.StatusInternalServerError)
- return
- }
- url := &url.URL{
- Scheme: "http",
- Path: r.URL.Path,
- RawQuery: r.URL.RawQuery,
- Host: address,
- }
- http.Redirect(w, r, url.String(), http.StatusPermanentRedirect)
- return
- }
-
- if !tlsCheckCert(r, g.cert) {
- http.Error(w, "403 invalid client certificate", http.StatusForbidden)
- return
- }
-
- // If this node is not clustered return a 404.
- if g.raft.HandlerFunc() == nil {
- http.NotFound(w, r)
- return
- }
-
- g.raft.HandlerFunc()(w, r)
- }
return map[string]http.HandlerFunc{
databaseEndpoint: database,
- raftEndpoint: raft,
}
}
diff --git a/lxd/cluster/gateway_test.go b/lxd/cluster/gateway_test.go
index 9da768d26f..ebfc706a81 100644
--- a/lxd/cluster/gateway_test.go
+++ b/lxd/cluster/gateway_test.go
@@ -31,7 +31,7 @@ func TestGateway_Single(t *testing.T) {
defer gateway.Shutdown()
handlerFuncs := gateway.HandlerFuncs()
- assert.Len(t, handlerFuncs, 2)
+ assert.Len(t, handlerFuncs, 1)
for endpoint, f := range handlerFuncs {
c, err := x509.ParseCertificate(cert.KeyPair().Certificate[0])
require.NoError(t, err)
From aed0fca8279738746413229109a5d7f2ba371231 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Tue, 14 May 2019 15:41:07 +0200
Subject: [PATCH 34/39] Drop unused hashicorp/raft network transport wrapper
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/raft.go | 33 ---------------------------------
1 file changed, 33 deletions(-)
diff --git a/lxd/cluster/raft.go b/lxd/cluster/raft.go
index a77882b31c..84dbe245dd 100644
--- a/lxd/cluster/raft.go
+++ b/lxd/cluster/raft.go
@@ -3,9 +3,7 @@ package cluster
import (
"bytes"
"fmt"
- "log"
"math"
- "net"
"net/http"
"os"
"path/filepath"
@@ -226,37 +224,6 @@ func raftDial(cert *shared.CertInfo) (rafthttp.Dial, error) {
return dial, nil
}
-// Create a network raft transport that will handle connections using a
-// rafthttp.Handler.
-func raftNetworkTransport(
- db *db.Node,
- address string,
- logger *log.Logger,
- timeout time.Duration,
- dial rafthttp.Dial) (raft.Transport, *rafthttp.Handler, *rafthttp.Layer, error) {
- handler := rafthttp.NewHandlerWithLogger(logger)
- addr, err := net.ResolveTCPAddr("tcp", address)
- if err != nil {
- return nil, nil, nil, errors.Wrap(err, "invalid node address")
- }
-
- layer := rafthttp.NewLayer(raftEndpoint, addr, handler, dial)
- config := &raft.NetworkTransportConfig{
- Logger: logger,
- Stream: layer,
- MaxPool: 2,
- Timeout: timeout,
- ServerAddressProvider: &raftAddressProvider{db: db},
- }
- transport := raft.NewNetworkTransportWithConfig(config)
-
- return transport, handler, layer, nil
-}
-
-// The LXD API endpoint path that gets routed to a rafthttp.Handler for
-// joining/leaving the cluster and exchanging raft commands between nodes.
-const raftEndpoint = "/internal/raft"
-
// An address provider that looks up server addresses in the raft_nodes table.
type raftAddressProvider struct {
db *db.Node
From dc7fe7e39f2f5ea9121042bab2941289101a15eb Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Wed, 15 May 2019 20:01:26 +0200
Subject: [PATCH 35/39] Translate address of first node
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway.go | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index 5a6654739e..4aadbc5900 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -291,7 +291,15 @@ func (g *Gateway) DialFunc() dqlite.DialFunc {
// Dial function for establishing raft connections.
func (g *Gateway) raftDial() dqlite.DialFunc {
return func(ctx context.Context, address string) (net.Conn, error) {
- return dqliteNetworkDial(ctx, address, g.cert, false)
+ if address == "0" {
+ provider := raftAddressProvider{db: g.db}
+ addr, err := provider.ServerAddr(raft.ServerID("1"))
+ if err != nil {
+ return nil, err
+ }
+ address = string(addr)
+ }
+ return dqliteNetworkDial(ctx, address, g, false)
}
}
From 19d06afcf076f65d0f437976f6faf8b58d3ee94e Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Wed, 22 May 2019 13:11:12 +0200
Subject: [PATCH 36/39] Return HTTP code 426 (Upgrade Required) if peer has old
version
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway.go | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index 4aadbc5900..1326e00478 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -148,13 +148,17 @@ func (g *Gateway) HandlerFuncs() map[string]http.HandlerFunc {
return
}
if version != dqliteVersion {
- if !g.upgradeTriggered && version > dqliteVersion {
- err = triggerUpdate()
- if err == nil {
- g.upgradeTriggered = true
+ if version > dqliteVersion {
+ if !g.upgradeTriggered {
+ err = triggerUpdate()
+ if err == nil {
+ g.upgradeTriggered = true
+ }
}
+ http.Error(w, "503 unsupported dqlite version", http.StatusServiceUnavailable)
+ } else {
+ http.Error(w, "426 dqlite version too old ", http.StatusUpgradeRequired)
}
- http.Error(w, "503 dqlite version mismatch", http.StatusServiceUnavailable)
return
}
From 515dd417c5a1ee795031012de87a910aaa14a8ee Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Mon, 17 Jun 2019 14:58:36 +0200
Subject: [PATCH 37/39] Drop dependencies on hashicorp/raft
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/cluster/gateway.go | 15 +-
lxd/cluster/gateway_export_test.go | 6 -
lxd/cluster/heartbeat.go | 3 +-
lxd/cluster/heartbeat_test.go | 3 +-
lxd/cluster/migrate.go | 9 +-
lxd/cluster/raft.go | 210 +----------
lxd/cluster/raft/bolt.go | 301 ++++++++++++++++
lxd/cluster/raft/configuration.go | 23 ++
lxd/cluster/raft/doc.go | 3 +
lxd/cluster/raft/file_snapshot.go | 548 +++++++++++++++++++++++++++++
lxd/cluster/raft/log.go | 72 ++++
lxd/cluster/raft/snapshot.go | 67 ++++
lxd/cluster/raft/transport.go | 8 +
13 files changed, 1041 insertions(+), 227 deletions(-)
create mode 100644 lxd/cluster/raft/bolt.go
create mode 100644 lxd/cluster/raft/configuration.go
create mode 100644 lxd/cluster/raft/doc.go
create mode 100644 lxd/cluster/raft/file_snapshot.go
create mode 100644 lxd/cluster/raft/log.go
create mode 100644 lxd/cluster/raft/snapshot.go
create mode 100644 lxd/cluster/raft/transport.go
diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index 1326e00478..06c5b1dedf 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -15,7 +15,6 @@ import (
"time"
dqlite "github.com/CanonicalLtd/go-dqlite"
- "github.com/hashicorp/raft"
"github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/lxd/util"
"github.com/lxc/lxd/shared"
@@ -258,7 +257,8 @@ func (g *Gateway) Snapshot() error {
g.lock.RLock()
defer g.lock.RUnlock()
- return g.raft.Snapshot()
+ // TODO: implement support for forcing a snapshot in dqlite v1
+ return fmt.Errorf("Not supported")
}
// WaitUpgradeNotification waits for a notification from another node that all
@@ -297,7 +297,7 @@ func (g *Gateway) raftDial() dqlite.DialFunc {
return func(ctx context.Context, address string) (net.Conn, error) {
if address == "0" {
provider := raftAddressProvider{db: g.db}
- addr, err := provider.ServerAddr(raft.ServerID("1"))
+ addr, err := provider.ServerAddr(1)
if err != nil {
return nil, err
}
@@ -597,6 +597,9 @@ func (g *Gateway) isLeader() bool {
return info != nil && info.ID == g.raft.info.ID
}
+// Internal error signalling that a node not the leader.
+var errNotLeader = fmt.Errorf("Not leader")
+
// Return information about the LXD nodes that a currently part of the raft
// cluster, as configured in the raft log. It returns an error if this node is
// not the leader.
@@ -605,7 +608,7 @@ func (g *Gateway) currentRaftNodes() ([]db.RaftNode, error) {
defer g.lock.RUnlock()
if g.raft == nil {
- return nil, raft.ErrNotLeader
+ return nil, errNotLeader
}
servers, err := g.server.Cluster()
if err != nil {
@@ -614,7 +617,7 @@ func (g *Gateway) currentRaftNodes() ([]db.RaftNode, error) {
provider := raftAddressProvider{db: g.db}
nodes := make([]db.RaftNode, len(servers))
for i, server := range servers {
- address, err := provider.ServerAddr(raft.ServerID(fmt.Sprintf("%d", server.ID)))
+ address, err := provider.ServerAddr(int(server.ID))
if err != nil {
if err != db.ErrNoSuchObject {
return nil, errors.Wrap(err, "Failed to fetch raft server address")
@@ -622,7 +625,7 @@ func (g *Gateway) currentRaftNodes() ([]db.RaftNode, error) {
// Use the initial address as fallback. This is an edge
// case that happens when a new leader is elected and
// its raft_nodes table is not fully up-to-date yet.
- address = raft.ServerAddress(server.Address)
+ address = server.Address
}
nodes[i].ID = int64(server.ID)
nodes[i].Address = string(address)
diff --git a/lxd/cluster/gateway_export_test.go b/lxd/cluster/gateway_export_test.go
index a5089643aa..4a78cacd4b 100644
--- a/lxd/cluster/gateway_export_test.go
+++ b/lxd/cluster/gateway_export_test.go
@@ -1,16 +1,10 @@
package cluster
import (
- "github.com/hashicorp/raft"
"github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/shared"
)
-// Raft returns the gateway's internal raft instance.
-func (g *Gateway) Raft() *raft.Raft {
- return g.raft.raft
-}
-
// IsLeader returns true if this node is the leader.
func (g *Gateway) IsLeader() bool {
return g.isLeader()
diff --git a/lxd/cluster/heartbeat.go b/lxd/cluster/heartbeat.go
index 0260ffa364..2833bc92b2 100644
--- a/lxd/cluster/heartbeat.go
+++ b/lxd/cluster/heartbeat.go
@@ -9,7 +9,6 @@ import (
"sync"
"time"
- "github.com/hashicorp/raft"
"github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/lxd/task"
"github.com/lxc/lxd/shared"
@@ -31,7 +30,7 @@ func Heartbeat(gateway *Gateway, cluster *db.Cluster) (task.Func, task.Schedule)
}
raftNodes, err := gateway.currentRaftNodes()
- if err == raft.ErrNotLeader {
+ if err == errNotLeader {
return
}
logger.Debugf("Starting heartbeat round")
diff --git a/lxd/cluster/heartbeat_test.go b/lxd/cluster/heartbeat_test.go
index 7b5f5b793e..579eeefe6b 100644
--- a/lxd/cluster/heartbeat_test.go
+++ b/lxd/cluster/heartbeat_test.go
@@ -7,7 +7,6 @@ import (
"time"
dqlite "github.com/CanonicalLtd/go-dqlite"
- "github.com/hashicorp/raft"
"github.com/lxc/lxd/lxd/cluster"
"github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/lxd/state"
@@ -184,7 +183,7 @@ func (f *heartbeatFixture) Follower() *cluster.Gateway {
for {
for _, gateway := range f.gateways {
- if gateway.Raft().State() == raft.Follower {
+ if !gateway.IsLeader() {
return gateway
}
}
diff --git a/lxd/cluster/migrate.go b/lxd/cluster/migrate.go
index 5b39987798..07caca85c8 100644
--- a/lxd/cluster/migrate.go
+++ b/lxd/cluster/migrate.go
@@ -10,8 +10,7 @@ import (
"github.com/boltdb/bolt"
"github.com/hashicorp/go-msgpack/codec"
- "github.com/hashicorp/raft"
- raftboltdb "github.com/hashicorp/raft-boltdb"
+ "github.com/lxc/lxd/lxd/cluster/raft"
"github.com/lxc/lxd/shared"
"github.com/pkg/errors"
)
@@ -82,12 +81,12 @@ func MigrateToDqlite10(dir string) error {
}
// Open the hashicorp/raft store in the given dir.
-func openLegacyRaftStore(dir string) (*raftboltdb.BoltStore, raft.SnapshotStore, error) {
- options := raftboltdb.Options{
+func openLegacyRaftStore(dir string) (*raft.BoltStore, raft.SnapshotStore, error) {
+ options := raft.Options{
Path: filepath.Join(dir, "logs.db"),
BoltOptions: &bolt.Options{ReadOnly: true},
}
- logs, err := raftboltdb.New(options)
+ logs, err := raft.New(options)
if err != nil {
return nil, nil, errors.Wrap(err, "Open boltdb file")
}
diff --git a/lxd/cluster/raft.go b/lxd/cluster/raft.go
index 84dbe245dd..7ee808d914 100644
--- a/lxd/cluster/raft.go
+++ b/lxd/cluster/raft.go
@@ -1,22 +1,11 @@
package cluster
import (
- "bytes"
"fmt"
- "math"
- "net/http"
"os"
"path/filepath"
- "strconv"
- "strings"
- "time"
dqlite "github.com/CanonicalLtd/go-dqlite"
- rafthttp "github.com/CanonicalLtd/raft-http"
- raftmembership "github.com/CanonicalLtd/raft-membership"
- hclog "github.com/hashicorp/go-hclog"
- "github.com/hashicorp/raft"
- raftboltdb "github.com/hashicorp/raft-boltdb"
"github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/lxd/node"
"github.com/lxc/lxd/shared"
@@ -76,13 +65,7 @@ func newRaft(database *db.Node, cert *shared.CertInfo, latency float64) (*raftIn
// A LXD-specific wrapper around raft.Raft, which also holds a reference to its
// network transport and dqlite FSM.
type raftInstance struct {
- info dqlite.ServerInfo
- layer *rafthttp.Layer // HTTP-based raft transport layer
- handler http.HandlerFunc // Handles join/leave/connect requests
- membershipChanger func(*raft.Raft) // Forwards to raft membership requests from handler
- logs *raftboltdb.BoltStore // Raft logs store, needs to be closed upon shutdown
- fsm raft.FSM // The dqlite FSM linked to the raft instance
- raft *raft.Raft // The actual raft instance
+ info dqlite.ServerInfo
}
// Create a new raftFactory, instantiating all needed raft dependencies.
@@ -124,118 +107,14 @@ func raftInstanceInit(
return instance, nil
}
-var serial = 99
-
-// FSM returns the dqlite FSM associated with the raft instance.
-func (i *raftInstance) FSM() raft.FSM {
- return i.fsm
-}
-
-// Raft returns the actual underlying raft instance.
-func (i *raftInstance) Raft() *raft.Raft {
- return i.raft
-}
-
-// Servers returns the servers that are currently part of the cluster.
-//
-// If this raft instance is not the leader, an error is returned.
-func (i *raftInstance) Servers() ([]raft.Server, error) {
- if i.raft.State() != raft.Leader {
- return nil, raft.ErrNotLeader
- }
- future := i.raft.GetConfiguration()
- err := future.Error()
- if err != nil {
- return nil, err
- }
- configuration := future.Configuration()
- return configuration.Servers, nil
-}
-
-// HandlerFunc can be used to handle HTTP requests performed against the LXD
-// API RaftEndpoint ("/internal/raft"), in order to join/leave/form the raft
-// cluster.
-//
-// If it returns nil, it means that this node is not supposed to expose a raft
-// endpoint over the network, because it's running as a non-clustered single
-// node.
-func (i *raftInstance) HandlerFunc() http.HandlerFunc {
- if i.handler == nil {
- return nil
- }
- return i.handler.ServeHTTP
-}
-
-// MembershipChanger returns the underlying rafthttp.Layer, which can be used
-// to change the membership of this node in the cluster.
-func (i *raftInstance) MembershipChanger() raftmembership.Changer {
- return i.layer
-}
-
-// Shutdown raft and any raft-related resource we have instantiated.
-func (i *raftInstance) Shutdown() error {
- logger.Debug("Stop raft instance")
-
- // Invoke raft APIs asynchronously to allow for a timeout.
- timeout := 10 * time.Second
-
- errCh := make(chan error)
- timer := time.After(timeout)
- go func() {
- errCh <- i.raft.Shutdown().Error()
- }()
- select {
- case err := <-errCh:
- if err != nil {
- return errors.Wrap(err, "failed to shutdown raft")
- }
- case <-timer:
- logger.Debug("Timeout waiting for raft to shutdown")
- return fmt.Errorf("raft did not shutdown within %s", timeout)
-
- }
- err := i.logs.Close()
- if err != nil {
- return errors.Wrap(err, "failed to close boltdb logs store")
- }
- return nil
-}
-
-// Snapshot can be used to manually trigger a RAFT snapshot
-func (i *raftInstance) Snapshot() error {
- return i.raft.Snapshot().Error()
-}
-
-// Create an in-memory raft transport.
-func raftMemoryTransport() raft.Transport {
- _, transport := raft.NewInmemTransport("0")
- return transport
-}
-
-// Create a rafthttp.Dial function that connects over TLS using the given
-// cluster (and optionally CA) certificate both as client and remote
-// certificate.
-func raftDial(cert *shared.CertInfo) (rafthttp.Dial, error) {
- config, err := tlsClientConfig(cert)
- if err != nil {
- return nil, err
- }
- dial := rafthttp.NewDialTLS(config)
- return dial, nil
-}
-
// An address provider that looks up server addresses in the raft_nodes table.
type raftAddressProvider struct {
db *db.Node
}
-func (p *raftAddressProvider) ServerAddr(id raft.ServerID) (raft.ServerAddress, error) {
- databaseID, err := strconv.Atoi(string(id))
- if err != nil {
- return "", errors.Wrap(err, "non-numeric server ID")
- }
+func (p *raftAddressProvider) ServerAddr(databaseID int) (string, error) {
var address string
- err = p.db.Transaction(func(tx *db.NodeTx) error {
+ err := p.db.Transaction(func(tx *db.NodeTx) error {
var err error
address, err = tx.RaftNodeAddress(int64(databaseID))
return err
@@ -243,86 +122,5 @@ func (p *raftAddressProvider) ServerAddr(id raft.ServerID) (raft.ServerAddress,
if err != nil {
return "", err
}
- return raft.ServerAddress(address), nil
-}
-
-// Create a base raft configuration tweaked for a network with the given latency measure.
-func raftConfig(latency float64) *raft.Config {
- config := raft.DefaultConfig()
- scale := func(duration *time.Duration) {
- *duration = time.Duration((math.Ceil(float64(*duration) * latency)))
- }
- durations := []*time.Duration{
- &config.HeartbeatTimeout,
- &config.ElectionTimeout,
- &config.CommitTimeout,
- &config.LeaderLeaseTimeout,
- }
- for _, duration := range durations {
- scale(duration)
- }
-
- config.SnapshotThreshold = 1024
- config.TrailingLogs = 512
-
- return config
-}
-
-func raftHandler(info *shared.CertInfo, handler *rafthttp.Handler) http.HandlerFunc {
- if handler == nil {
- return nil
- }
- return func(w http.ResponseWriter, r *http.Request) {
- if !tlsCheckCert(r, info) {
- http.Error(w, "403 invalid client certificate", http.StatusForbidden)
- return
- }
- handler.ServeHTTP(w, r)
- }
-}
-
-func raftLogger() hclog.Logger {
- return hclog.New(&hclog.LoggerOptions{
- Name: "raft",
- Output: &raftLogWriter{},
- })
-}
-
-// Implement io.Writer on top of LXD's logging system.
-type raftLogWriter struct {
-}
-
-func (o *raftLogWriter) Write(line []byte) (n int, err error) {
- // Parse the log level according to hashicorp's raft pkg convetions.
- level := ""
- msg := ""
- x := bytes.IndexByte(line, '[')
- if x >= 0 {
- y := bytes.IndexByte(line[x:], ']')
- if y >= 0 {
- level = string(line[x+1 : x+y])
-
- // Capitalize the string, to match LXD logging conventions
- first := strings.ToUpper(string(line[x+y+2]))
- rest := string(line[x+y+3 : len(line)-1])
- msg = first + rest
- }
- }
-
- if level == "" {
- // Ignore log entries that don't stick to the convetion.
- return len(line), nil
- }
-
- switch level {
- case "DEBUG":
- logger.Debug(msg)
- case "INFO":
- logger.Debug(msg)
- case "WARN":
- logger.Warn(msg)
- default:
- // Ignore any other log level.
- }
- return len(line), nil
+ return address, nil
}
diff --git a/lxd/cluster/raft/bolt.go b/lxd/cluster/raft/bolt.go
new file mode 100644
index 0000000000..ea7390dd5c
--- /dev/null
+++ b/lxd/cluster/raft/bolt.go
@@ -0,0 +1,301 @@
+package raft
+
+import (
+ "bytes"
+ "encoding/binary"
+ "errors"
+
+ "github.com/boltdb/bolt"
+ "github.com/hashicorp/go-msgpack/codec"
+)
+
+const (
+ // Permissions to use on the db file. This is only used if the
+ // database file does not exist and needs to be created.
+ dbFileMode = 0600
+)
+
+var (
+ // Bucket names we perform transactions in
+ dbLogs = []byte("logs")
+ dbConf = []byte("conf")
+
+ // An error indicating a given key does not exist
+ ErrKeyNotFound = errors.New("not found")
+
+ ErrLogNotFound = errors.New("log not found")
+)
+
+// BoltStore provides access to BoltDB for Raft to store and retrieve
+// log entries. It also provides key/value storage, and can be used as
+// a LogStore and StableStore.
+type BoltStore struct {
+ // conn is the underlying handle to the db.
+ conn *bolt.DB
+
+ // The path to the Bolt database file
+ path string
+}
+
+// Options contains all the configuraiton used to open the BoltDB
+type Options struct {
+ // Path is the file path to the BoltDB to use
+ Path string
+
+ // BoltOptions contains any specific BoltDB options you might
+ // want to specify [e.g. open timeout]
+ BoltOptions *bolt.Options
+
+ // NoSync causes the database to skip fsync calls after each
+ // write to the log. This is unsafe, so it should be used
+ // with caution.
+ NoSync bool
+}
+
+// readOnly returns true if the contained bolt options say to open
+// the DB in readOnly mode [this can be useful to tools that want
+// to examine the log]
+func (o *Options) readOnly() bool {
+ return o != nil && o.BoltOptions != nil && o.BoltOptions.ReadOnly
+}
+
+// NewBoltStore takes a file path and returns a connected Raft backend.
+func NewBoltStore(path string) (*BoltStore, error) {
+ return New(Options{Path: path})
+}
+
+// New uses the supplied options to open the BoltDB and prepare it for use as a raft backend.
+func New(options Options) (*BoltStore, error) {
+ // Try to connect
+ handle, err := bolt.Open(options.Path, dbFileMode, options.BoltOptions)
+ if err != nil {
+ return nil, err
+ }
+ handle.NoSync = options.NoSync
+
+ // Create the new store
+ store := &BoltStore{
+ conn: handle,
+ path: options.Path,
+ }
+
+ // If the store was opened read-only, don't try and create buckets
+ if !options.readOnly() {
+ // Set up our buckets
+ if err := store.initialize(); err != nil {
+ store.Close()
+ return nil, err
+ }
+ }
+ return store, nil
+}
+
+// initialize is used to set up all of the buckets.
+func (b *BoltStore) initialize() error {
+ tx, err := b.conn.Begin(true)
+ if err != nil {
+ return err
+ }
+ defer tx.Rollback()
+
+ // Create all the buckets
+ if _, err := tx.CreateBucketIfNotExists(dbLogs); err != nil {
+ return err
+ }
+ if _, err := tx.CreateBucketIfNotExists(dbConf); err != nil {
+ return err
+ }
+
+ return tx.Commit()
+}
+
+// Close is used to gracefully close the DB connection.
+func (b *BoltStore) Close() error {
+ return b.conn.Close()
+}
+
+// FirstIndex returns the first known index from the Raft log.
+func (b *BoltStore) FirstIndex() (uint64, error) {
+ tx, err := b.conn.Begin(false)
+ if err != nil {
+ return 0, err
+ }
+ defer tx.Rollback()
+
+ curs := tx.Bucket(dbLogs).Cursor()
+ if first, _ := curs.First(); first == nil {
+ return 0, nil
+ } else {
+ return bytesToUint64(first), nil
+ }
+}
+
+// LastIndex returns the last known index from the Raft log.
+func (b *BoltStore) LastIndex() (uint64, error) {
+ tx, err := b.conn.Begin(false)
+ if err != nil {
+ return 0, err
+ }
+ defer tx.Rollback()
+
+ curs := tx.Bucket(dbLogs).Cursor()
+ if last, _ := curs.Last(); last == nil {
+ return 0, nil
+ } else {
+ return bytesToUint64(last), nil
+ }
+}
+
+// GetLog is used to retrieve a log from BoltDB at a given index.
+func (b *BoltStore) GetLog(idx uint64, log *Log) error {
+ tx, err := b.conn.Begin(false)
+ if err != nil {
+ return err
+ }
+ defer tx.Rollback()
+
+ bucket := tx.Bucket(dbLogs)
+ val := bucket.Get(uint64ToBytes(idx))
+
+ if val == nil {
+ return ErrLogNotFound
+ }
+ return decodeMsgPack(val, log)
+}
+
+// StoreLog is used to store a single raft log
+func (b *BoltStore) StoreLog(log *Log) error {
+ return b.StoreLogs([]*Log{log})
+}
+
+// StoreLogs is used to store a set of raft logs
+func (b *BoltStore) StoreLogs(logs []*Log) error {
+ tx, err := b.conn.Begin(true)
+ if err != nil {
+ return err
+ }
+ defer tx.Rollback()
+
+ for _, log := range logs {
+ key := uint64ToBytes(log.Index)
+ val, err := encodeMsgPack(log)
+ if err != nil {
+ return err
+ }
+ bucket := tx.Bucket(dbLogs)
+ if err := bucket.Put(key, val.Bytes()); err != nil {
+ return err
+ }
+ }
+
+ return tx.Commit()
+}
+
+// DeleteRange is used to delete logs within a given range inclusively.
+func (b *BoltStore) DeleteRange(min, max uint64) error {
+ minKey := uint64ToBytes(min)
+
+ tx, err := b.conn.Begin(true)
+ if err != nil {
+ return err
+ }
+ defer tx.Rollback()
+
+ curs := tx.Bucket(dbLogs).Cursor()
+ for k, _ := curs.Seek(minKey); k != nil; k, _ = curs.Next() {
+ // Handle out-of-range log index
+ if bytesToUint64(k) > max {
+ break
+ }
+
+ // Delete in-range log index
+ if err := curs.Delete(); err != nil {
+ return err
+ }
+ }
+
+ return tx.Commit()
+}
+
+// Set is used to set a key/value set outside of the raft log
+func (b *BoltStore) Set(k, v []byte) error {
+ tx, err := b.conn.Begin(true)
+ if err != nil {
+ return err
+ }
+ defer tx.Rollback()
+
+ bucket := tx.Bucket(dbConf)
+ if err := bucket.Put(k, v); err != nil {
+ return err
+ }
+
+ return tx.Commit()
+}
+
+// Get is used to retrieve a value from the k/v store by key
+func (b *BoltStore) Get(k []byte) ([]byte, error) {
+ tx, err := b.conn.Begin(false)
+ if err != nil {
+ return nil, err
+ }
+ defer tx.Rollback()
+
+ bucket := tx.Bucket(dbConf)
+ val := bucket.Get(k)
+
+ if val == nil {
+ return nil, ErrKeyNotFound
+ }
+ return append([]byte(nil), val...), nil
+}
+
+// SetUint64 is like Set, but handles uint64 values
+func (b *BoltStore) SetUint64(key []byte, val uint64) error {
+ return b.Set(key, uint64ToBytes(val))
+}
+
+// GetUint64 is like Get, but handles uint64 values
+func (b *BoltStore) GetUint64(key []byte) (uint64, error) {
+ val, err := b.Get(key)
+ if err != nil {
+ return 0, err
+ }
+ return bytesToUint64(val), nil
+}
+
+// Sync performs an fsync on the database file handle. This is not necessary
+// under normal operation unless NoSync is enabled, in which this forces the
+// database file to sync against the disk.
+func (b *BoltStore) Sync() error {
+ return b.conn.Sync()
+}
+
+// Decode reverses the encode operation on a byte slice input
+func decodeMsgPack(buf []byte, out interface{}) error {
+ r := bytes.NewBuffer(buf)
+ hd := codec.MsgpackHandle{}
+ dec := codec.NewDecoder(r, &hd)
+ return dec.Decode(out)
+}
+
+// Encode writes an encoded object to a new bytes buffer
+func encodeMsgPack(in interface{}) (*bytes.Buffer, error) {
+ buf := bytes.NewBuffer(nil)
+ hd := codec.MsgpackHandle{}
+ enc := codec.NewEncoder(buf, &hd)
+ err := enc.Encode(in)
+ return buf, err
+}
+
+// Converts bytes to an integer
+func bytesToUint64(b []byte) uint64 {
+ return binary.BigEndian.Uint64(b)
+}
+
+// Converts a uint to a byte slice
+func uint64ToBytes(u uint64) []byte {
+ buf := make([]byte, 8)
+ binary.BigEndian.PutUint64(buf, u)
+ return buf
+}
diff --git a/lxd/cluster/raft/configuration.go b/lxd/cluster/raft/configuration.go
new file mode 100644
index 0000000000..f5596f5466
--- /dev/null
+++ b/lxd/cluster/raft/configuration.go
@@ -0,0 +1,23 @@
+package raft
+
+// ServerID is a unique string identifying a server for all time.
+type ServerID string
+
+// ServerAddress is a network address for a server that a transport can contact.
+type ServerAddress string
+
+// Server tracks the information about a single server in a configuration.
+type Server struct {
+ // ID is a unique string identifying this server for all time.
+ ID ServerID
+ // Address is its network address that a transport can contact.
+ Address ServerAddress
+}
+
+// Configuration tracks which servers are in the cluster, and whether they have
+// votes. This should include the local server, if it's a member of the cluster.
+// The servers are listed no particular order, but each should only appear once.
+// These entries are appended to the log during membership changes.
+type Configuration struct {
+ Servers []Server
+}
diff --git a/lxd/cluster/raft/doc.go b/lxd/cluster/raft/doc.go
new file mode 100644
index 0000000000..7ebbed5953
--- /dev/null
+++ b/lxd/cluster/raft/doc.go
@@ -0,0 +1,3 @@
+// Package raft contains the subset of hashicorp/raft needed to perform
+// data migrations to dqlite 1.0.
+package raft
diff --git a/lxd/cluster/raft/file_snapshot.go b/lxd/cluster/raft/file_snapshot.go
new file mode 100644
index 0000000000..43d092e507
--- /dev/null
+++ b/lxd/cluster/raft/file_snapshot.go
@@ -0,0 +1,548 @@
+package raft
+
+import (
+ "bufio"
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "hash"
+ "hash/crc64"
+ "io"
+ "io/ioutil"
+ "log"
+ "os"
+ "path/filepath"
+ "runtime"
+ "sort"
+ "strings"
+ "time"
+)
+
+const (
+ testPath = "permTest"
+ snapPath = "snapshots"
+ metaFilePath = "meta.json"
+ stateFilePath = "state.bin"
+ tmpSuffix = ".tmp"
+)
+
+// FileSnapshotStore implements the SnapshotStore interface and allows
+// snapshots to be made on the local disk.
+type FileSnapshotStore struct {
+ path string
+ retain int
+ logger *log.Logger
+}
+
+type snapMetaSlice []*fileSnapshotMeta
+
+// FileSnapshotSink implements SnapshotSink with a file.
+type FileSnapshotSink struct {
+ store *FileSnapshotStore
+ logger *log.Logger
+ dir string
+ parentDir string
+ meta fileSnapshotMeta
+
+ stateFile *os.File
+ stateHash hash.Hash64
+ buffered *bufio.Writer
+
+ closed bool
+}
+
+// fileSnapshotMeta is stored on disk. We also put a CRC
+// on disk so that we can verify the snapshot.
+type fileSnapshotMeta struct {
+ SnapshotMeta
+ CRC []byte
+}
+
+// bufferedFile is returned when we open a snapshot. This way
+// reads are buffered and the file still gets closed.
+type bufferedFile struct {
+ bh *bufio.Reader
+ fh *os.File
+}
+
+func (b *bufferedFile) Read(p []byte) (n int, err error) {
+ return b.bh.Read(p)
+}
+
+func (b *bufferedFile) Close() error {
+ return b.fh.Close()
+}
+
+// NewFileSnapshotStoreWithLogger creates a new FileSnapshotStore based
+// on a base directory. The `retain` parameter controls how many
+// snapshots are retained. Must be at least 1.
+func NewFileSnapshotStoreWithLogger(base string, retain int, logger *log.Logger) (*FileSnapshotStore, error) {
+ if retain < 1 {
+ return nil, fmt.Errorf("must retain at least one snapshot")
+ }
+ if logger == nil {
+ logger = log.New(os.Stderr, "", log.LstdFlags)
+ }
+
+ // Ensure our path exists
+ path := filepath.Join(base, snapPath)
+ if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) {
+ return nil, fmt.Errorf("snapshot path not accessible: %v", err)
+ }
+
+ // Setup the store
+ store := &FileSnapshotStore{
+ path: path,
+ retain: retain,
+ logger: logger,
+ }
+
+ // Do a permissions test
+ if err := store.testPermissions(); err != nil {
+ return nil, fmt.Errorf("permissions test failed: %v", err)
+ }
+ return store, nil
+}
+
+// NewFileSnapshotStore creates a new FileSnapshotStore based
+// on a base directory. The `retain` parameter controls how many
+// snapshots are retained. Must be at least 1.
+func NewFileSnapshotStore(base string, retain int, logOutput io.Writer) (*FileSnapshotStore, error) {
+ if logOutput == nil {
+ logOutput = os.Stderr
+ }
+ return NewFileSnapshotStoreWithLogger(base, retain, log.New(logOutput, "", log.LstdFlags))
+}
+
+// testPermissions tries to touch a file in our path to see if it works.
+func (f *FileSnapshotStore) testPermissions() error {
+ path := filepath.Join(f.path, testPath)
+ fh, err := os.Create(path)
+ if err != nil {
+ return err
+ }
+
+ if err = fh.Close(); err != nil {
+ return err
+ }
+
+ if err = os.Remove(path); err != nil {
+ return err
+ }
+ return nil
+}
+
+// snapshotName generates a name for the snapshot.
+func snapshotName(term, index uint64) string {
+ now := time.Now()
+ msec := now.UnixNano() / int64(time.Millisecond)
+ return fmt.Sprintf("%d-%d-%d", term, index, msec)
+}
+
+// Create is used to start a new snapshot
+func (f *FileSnapshotStore) Create(version SnapshotVersion, index, term uint64,
+ configuration Configuration, configurationIndex uint64, trans Transport) (SnapshotSink, error) {
+ // We only support version 1 snapshots at this time.
+ if version != 1 {
+ return nil, fmt.Errorf("unsupported snapshot version %d", version)
+ }
+
+ // Create a new path
+ name := snapshotName(term, index)
+ path := filepath.Join(f.path, name+tmpSuffix)
+ f.logger.Printf("[INFO] snapshot: Creating new snapshot at %s", path)
+
+ // Make the directory
+ if err := os.MkdirAll(path, 0755); err != nil {
+ f.logger.Printf("[ERR] snapshot: Failed to make snapshot directory: %v", err)
+ return nil, err
+ }
+
+ // Create the sink
+ sink := &FileSnapshotSink{
+ store: f,
+ logger: f.logger,
+ dir: path,
+ parentDir: f.path,
+ meta: fileSnapshotMeta{
+ SnapshotMeta: SnapshotMeta{
+ Version: version,
+ ID: name,
+ Index: index,
+ Term: term,
+ Peers: encodePeers(configuration, trans),
+ Configuration: configuration,
+ ConfigurationIndex: configurationIndex,
+ },
+ CRC: nil,
+ },
+ }
+
+ // Write out the meta data
+ if err := sink.writeMeta(); err != nil {
+ f.logger.Printf("[ERR] snapshot: Failed to write metadata: %v", err)
+ return nil, err
+ }
+
+ // Open the state file
+ statePath := filepath.Join(path, stateFilePath)
+ fh, err := os.Create(statePath)
+ if err != nil {
+ f.logger.Printf("[ERR] snapshot: Failed to create state file: %v", err)
+ return nil, err
+ }
+ sink.stateFile = fh
+
+ // Create a CRC64 hash
+ sink.stateHash = crc64.New(crc64.MakeTable(crc64.ECMA))
+
+ // Wrap both the hash and file in a MultiWriter with buffering
+ multi := io.MultiWriter(sink.stateFile, sink.stateHash)
+ sink.buffered = bufio.NewWriter(multi)
+
+ // Done
+ return sink, nil
+}
+
+// List returns available snapshots in the store.
+func (f *FileSnapshotStore) List() ([]*SnapshotMeta, error) {
+ // Get the eligible snapshots
+ snapshots, err := f.getSnapshots()
+ if err != nil {
+ f.logger.Printf("[ERR] snapshot: Failed to get snapshots: %v", err)
+ return nil, err
+ }
+
+ var snapMeta []*SnapshotMeta
+ for _, meta := range snapshots {
+ snapMeta = append(snapMeta, &meta.SnapshotMeta)
+ if len(snapMeta) == f.retain {
+ break
+ }
+ }
+ return snapMeta, nil
+}
+
+// getSnapshots returns all the known snapshots.
+func (f *FileSnapshotStore) getSnapshots() ([]*fileSnapshotMeta, error) {
+ // Get the eligible snapshots
+ snapshots, err := ioutil.ReadDir(f.path)
+ if err != nil {
+ f.logger.Printf("[ERR] snapshot: Failed to scan snapshot dir: %v", err)
+ return nil, err
+ }
+
+ // Populate the metadata
+ var snapMeta []*fileSnapshotMeta
+ for _, snap := range snapshots {
+ // Ignore any files
+ if !snap.IsDir() {
+ continue
+ }
+
+ // Ignore any temporary snapshots
+ dirName := snap.Name()
+ if strings.HasSuffix(dirName, tmpSuffix) {
+ f.logger.Printf("[WARN] snapshot: Found temporary snapshot: %v", dirName)
+ continue
+ }
+
+ // Try to read the meta data
+ meta, err := f.readMeta(dirName)
+ if err != nil {
+ f.logger.Printf("[WARN] snapshot: Failed to read metadata for %v: %v", dirName, err)
+ continue
+ }
+
+ // Make sure we can understand this version.
+ if meta.Version < SnapshotVersionMin || meta.Version > SnapshotVersionMax {
+ f.logger.Printf("[WARN] snapshot: Snapshot version for %v not supported: %d", dirName, meta.Version)
+ continue
+ }
+
+ // Append, but only return up to the retain count
+ snapMeta = append(snapMeta, meta)
+ }
+
+ // Sort the snapshot, reverse so we get new -> old
+ sort.Sort(sort.Reverse(snapMetaSlice(snapMeta)))
+
+ return snapMeta, nil
+}
+
+// readMeta is used to read the meta data for a given named backup
+func (f *FileSnapshotStore) readMeta(name string) (*fileSnapshotMeta, error) {
+ // Open the meta file
+ metaPath := filepath.Join(f.path, name, metaFilePath)
+ fh, err := os.Open(metaPath)
+ if err != nil {
+ return nil, err
+ }
+ defer fh.Close()
+
+ // Buffer the file IO
+ buffered := bufio.NewReader(fh)
+
+ // Read in the JSON
+ meta := &fileSnapshotMeta{}
+ dec := json.NewDecoder(buffered)
+ if err := dec.Decode(meta); err != nil {
+ return nil, err
+ }
+ return meta, nil
+}
+
+// Open takes a snapshot ID and returns a ReadCloser for that snapshot.
+func (f *FileSnapshotStore) Open(id string) (*SnapshotMeta, io.ReadCloser, error) {
+ // Get the metadata
+ meta, err := f.readMeta(id)
+ if err != nil {
+ f.logger.Printf("[ERR] snapshot: Failed to get meta data to open snapshot: %v", err)
+ return nil, nil, err
+ }
+
+ // Open the state file
+ statePath := filepath.Join(f.path, id, stateFilePath)
+ fh, err := os.Open(statePath)
+ if err != nil {
+ f.logger.Printf("[ERR] snapshot: Failed to open state file: %v", err)
+ return nil, nil, err
+ }
+
+ // Create a CRC64 hash
+ stateHash := crc64.New(crc64.MakeTable(crc64.ECMA))
+
+ // Compute the hash
+ _, err = io.Copy(stateHash, fh)
+ if err != nil {
+ f.logger.Printf("[ERR] snapshot: Failed to read state file: %v", err)
+ fh.Close()
+ return nil, nil, err
+ }
+
+ // Verify the hash
+ computed := stateHash.Sum(nil)
+ if bytes.Compare(meta.CRC, computed) != 0 {
+ f.logger.Printf("[ERR] snapshot: CRC checksum failed (stored: %v computed: %v)",
+ meta.CRC, computed)
+ fh.Close()
+ return nil, nil, fmt.Errorf("CRC mismatch")
+ }
+
+ // Seek to the start
+ if _, err := fh.Seek(0, 0); err != nil {
+ f.logger.Printf("[ERR] snapshot: State file seek failed: %v", err)
+ fh.Close()
+ return nil, nil, err
+ }
+
+ // Return a buffered file
+ buffered := &bufferedFile{
+ bh: bufio.NewReader(fh),
+ fh: fh,
+ }
+
+ return &meta.SnapshotMeta, buffered, nil
+}
+
+// ReapSnapshots reaps any snapshots beyond the retain count.
+func (f *FileSnapshotStore) ReapSnapshots() error {
+ snapshots, err := f.getSnapshots()
+ if err != nil {
+ f.logger.Printf("[ERR] snapshot: Failed to get snapshots: %v", err)
+ return err
+ }
+
+ for i := f.retain; i < len(snapshots); i++ {
+ path := filepath.Join(f.path, snapshots[i].ID)
+ f.logger.Printf("[INFO] snapshot: reaping snapshot %v", path)
+ if err := os.RemoveAll(path); err != nil {
+ f.logger.Printf("[ERR] snapshot: Failed to reap snapshot %v: %v", path, err)
+ return err
+ }
+ }
+ return nil
+}
+
+// ID returns the ID of the snapshot, can be used with Open()
+// after the snapshot is finalized.
+func (s *FileSnapshotSink) ID() string {
+ return s.meta.ID
+}
+
+// Write is used to append to the state file. We write to the
+// buffered IO object to reduce the amount of context switches.
+func (s *FileSnapshotSink) Write(b []byte) (int, error) {
+ return s.buffered.Write(b)
+}
+
+// Close is used to indicate a successful end.
+func (s *FileSnapshotSink) Close() error {
+ // Make sure close is idempotent
+ if s.closed {
+ return nil
+ }
+ s.closed = true
+
+ // Close the open handles
+ if err := s.finalize(); err != nil {
+ s.logger.Printf("[ERR] snapshot: Failed to finalize snapshot: %v", err)
+ if delErr := os.RemoveAll(s.dir); delErr != nil {
+ s.logger.Printf("[ERR] snapshot: Failed to delete temporary snapshot directory at path %v: %v", s.dir, delErr)
+ return delErr
+ }
+ return err
+ }
+
+ // Write out the meta data
+ if err := s.writeMeta(); err != nil {
+ s.logger.Printf("[ERR] snapshot: Failed to write metadata: %v", err)
+ return err
+ }
+
+ // Move the directory into place
+ newPath := strings.TrimSuffix(s.dir, tmpSuffix)
+ if err := os.Rename(s.dir, newPath); err != nil {
+ s.logger.Printf("[ERR] snapshot: Failed to move snapshot into place: %v", err)
+ return err
+ }
+
+ if runtime.GOOS != "windows" { //skipping fsync for directory entry edits on Windows, only needed for *nix style file systems
+ parentFH, err := os.Open(s.parentDir)
+ defer parentFH.Close()
+ if err != nil {
+ s.logger.Printf("[ERR] snapshot: Failed to open snapshot parent directory %v, error: %v", s.parentDir, err)
+ return err
+ }
+
+ if err = parentFH.Sync(); err != nil {
+ s.logger.Printf("[ERR] snapshot: Failed syncing parent directory %v, error: %v", s.parentDir, err)
+ return err
+ }
+ }
+
+ // Reap any old snapshots
+ if err := s.store.ReapSnapshots(); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// Cancel is used to indicate an unsuccessful end.
+func (s *FileSnapshotSink) Cancel() error {
+ // Make sure close is idempotent
+ if s.closed {
+ return nil
+ }
+ s.closed = true
+
+ // Close the open handles
+ if err := s.finalize(); err != nil {
+ s.logger.Printf("[ERR] snapshot: Failed to finalize snapshot: %v", err)
+ return err
+ }
+
+ // Attempt to remove all artifacts
+ return os.RemoveAll(s.dir)
+}
+
+// finalize is used to close all of our resources.
+func (s *FileSnapshotSink) finalize() error {
+ // Flush any remaining data
+ if err := s.buffered.Flush(); err != nil {
+ return err
+ }
+
+ // Sync to force fsync to disk
+ if err := s.stateFile.Sync(); err != nil {
+ return err
+ }
+
+ // Get the file size
+ stat, statErr := s.stateFile.Stat()
+
+ // Close the file
+ if err := s.stateFile.Close(); err != nil {
+ return err
+ }
+
+ // Set the file size, check after we close
+ if statErr != nil {
+ return statErr
+ }
+ s.meta.Size = stat.Size()
+
+ // Set the CRC
+ s.meta.CRC = s.stateHash.Sum(nil)
+ return nil
+}
+
+// writeMeta is used to write out the metadata we have.
+func (s *FileSnapshotSink) writeMeta() error {
+ // Open the meta file
+ metaPath := filepath.Join(s.dir, metaFilePath)
+ fh, err := os.Create(metaPath)
+ if err != nil {
+ return err
+ }
+ defer fh.Close()
+
+ // Buffer the file IO
+ buffered := bufio.NewWriter(fh)
+
+ // Write out as JSON
+ enc := json.NewEncoder(buffered)
+ if err := enc.Encode(&s.meta); err != nil {
+ return err
+ }
+
+ if err = buffered.Flush(); err != nil {
+ return err
+ }
+
+ if err = fh.Sync(); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// Implement the sort interface for []*fileSnapshotMeta.
+func (s snapMetaSlice) Len() int {
+ return len(s)
+}
+
+func (s snapMetaSlice) Less(i, j int) bool {
+ if s[i].Term != s[j].Term {
+ return s[i].Term < s[j].Term
+ }
+ if s[i].Index != s[j].Index {
+ return s[i].Index < s[j].Index
+ }
+ return s[i].ID < s[j].ID
+}
+
+func (s snapMetaSlice) Swap(i, j int) {
+ s[i], s[j] = s[j], s[i]
+}
+
+// encodePeers is used to serialize a Configuration into the old peers format.
+// This is here for backwards compatibility when operating with a mix of old
+// servers and should be removed once we deprecate support for protocol version 1.
+func encodePeers(configuration Configuration, trans Transport) []byte {
+ // Gather up all the voters, other suffrage types are not supported by
+ // this data format.
+ var encPeers [][]byte
+ for _, server := range configuration.Servers {
+ encPeers = append(encPeers, trans.EncodePeer(server.ID, server.Address))
+ }
+
+ // Encode the entire array.
+ buf, err := encodeMsgPack(encPeers)
+ if err != nil {
+ panic(fmt.Errorf("failed to encode peers: %v", err))
+ }
+
+ return buf.Bytes()
+}
diff --git a/lxd/cluster/raft/log.go b/lxd/cluster/raft/log.go
new file mode 100644
index 0000000000..4ade38ecc1
--- /dev/null
+++ b/lxd/cluster/raft/log.go
@@ -0,0 +1,72 @@
+package raft
+
+// LogType describes various types of log entries.
+type LogType uint8
+
+const (
+ // LogCommand is applied to a user FSM.
+ LogCommand LogType = iota
+
+ // LogNoop is used to assert leadership.
+ LogNoop
+
+ // LogAddPeer is used to add a new peer. This should only be used with
+ // older protocol versions designed to be compatible with unversioned
+ // Raft servers. See comments in config.go for details.
+ LogAddPeerDeprecated
+
+ // LogRemovePeer is used to remove an existing peer. This should only be
+ // used with older protocol versions designed to be compatible with
+ // unversioned Raft servers. See comments in config.go for details.
+ LogRemovePeerDeprecated
+
+ // LogBarrier is used to ensure all preceding operations have been
+ // applied to the FSM. It is similar to LogNoop, but instead of returning
+ // once committed, it only returns once the FSM manager acks it. Otherwise
+ // it is possible there are operations committed but not yet applied to
+ // the FSM.
+ LogBarrier
+
+ // LogConfiguration establishes a membership change configuration. It is
+ // created when a server is added, removed, promoted, etc. Only used
+ // when protocol version 1 or greater is in use.
+ LogConfiguration
+)
+
+// Log entries are replicated to all members of the Raft cluster
+// and form the heart of the replicated state machine.
+type Log struct {
+ // Index holds the index of the log entry.
+ Index uint64
+
+ // Term holds the election term of the log entry.
+ Term uint64
+
+ // Type holds the type of the log entry.
+ Type LogType
+
+ // Data holds the log entry's type-specific data.
+ Data []byte
+}
+
+// LogStore is used to provide an interface for storing
+// and retrieving logs in a durable fashion.
+type LogStore interface {
+ // FirstIndex returns the first index written. 0 for no entries.
+ FirstIndex() (uint64, error)
+
+ // LastIndex returns the last index written. 0 for no entries.
+ LastIndex() (uint64, error)
+
+ // GetLog gets a log entry at a given index.
+ GetLog(index uint64, log *Log) error
+
+ // StoreLog stores a log entry.
+ StoreLog(log *Log) error
+
+ // StoreLogs stores multiple log entries.
+ StoreLogs(logs []*Log) error
+
+ // DeleteRange deletes a range of log entries. The range is inclusive.
+ DeleteRange(min, max uint64) error
+}
diff --git a/lxd/cluster/raft/snapshot.go b/lxd/cluster/raft/snapshot.go
new file mode 100644
index 0000000000..72ba6b8420
--- /dev/null
+++ b/lxd/cluster/raft/snapshot.go
@@ -0,0 +1,67 @@
+package raft
+
+import (
+ "io"
+)
+
+type SnapshotVersion int
+
+const (
+ SnapshotVersionMin SnapshotVersion = 0
+ SnapshotVersionMax = 1
+)
+
+// SnapshotMeta is for metadata of a snapshot.
+type SnapshotMeta struct {
+ // Version is the version number of the snapshot metadata. This does not cover
+ // the application's data in the snapshot, that should be versioned
+ // separately.
+ Version SnapshotVersion
+
+ // ID is opaque to the store, and is used for opening.
+ ID string
+
+ // Index and Term store when the snapshot was taken.
+ Index uint64
+ Term uint64
+
+ // Peers is deprecated and used to support version 0 snapshots, but will
+ // be populated in version 1 snapshots as well to help with upgrades.
+ Peers []byte
+
+ // Configuration and ConfigurationIndex are present in version 1
+ // snapshots and later.
+ Configuration Configuration
+ ConfigurationIndex uint64
+
+ // Size is the size of the snapshot in bytes.
+ Size int64
+}
+
+// SnapshotStore interface is used to allow for flexible implementations
+// of snapshot storage and retrieval. For example, a client could implement
+// a shared state store such as S3, allowing new nodes to restore snapshots
+// without streaming from the leader.
+type SnapshotStore interface {
+ // Create is used to begin a snapshot at a given index and term, and with
+ // the given committed configuration. The version parameter controls
+ // which snapshot version to create.
+ Create(version SnapshotVersion, index, term uint64, configuration Configuration,
+ configurationIndex uint64, trans Transport) (SnapshotSink, error)
+
+ // List is used to list the available snapshots in the store.
+ // It should return then in descending order, with the highest index first.
+ List() ([]*SnapshotMeta, error)
+
+ // Open takes a snapshot ID and provides a ReadCloser. Once close is
+ // called it is assumed the snapshot is no longer needed.
+ Open(id string) (*SnapshotMeta, io.ReadCloser, error)
+}
+
+// SnapshotSink is returned by StartSnapshot. The FSM will Write state
+// to the sink and call Close on completion. On error, Cancel will be invoked.
+type SnapshotSink interface {
+ io.WriteCloser
+ ID() string
+ Cancel() error
+}
diff --git a/lxd/cluster/raft/transport.go b/lxd/cluster/raft/transport.go
new file mode 100644
index 0000000000..ee33a00b2d
--- /dev/null
+++ b/lxd/cluster/raft/transport.go
@@ -0,0 +1,8 @@
+package raft
+
+// Transport provides an interface for network transports
+// to allow Raft to communicate with other nodes.
+type Transport interface {
+ // EncodePeer is used to serialize a peer's address.
+ EncodePeer(id ServerID, addr ServerAddress) []byte
+}
From 3f9769c470d24085eb772e4e4c3f5f1f2e4be62b Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Mon, 17 Jun 2019 15:35:24 +0200
Subject: [PATCH 38/39] Drop vendor directory
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
.../github.com/CanonicalLtd/go-dqlite/AUTHORS | 1 -
.../github.com/CanonicalLtd/go-dqlite/LICENSE | 201 --
.../CanonicalLtd/go-dqlite/Makefile | 4 -
.../CanonicalLtd/go-dqlite/README.md | 99 -
.../CanonicalLtd/go-dqlite/cluster.go | 123 -
.../go-dqlite/cmd/dqlite/bench.go | 130 -
.../go-dqlite/cmd/dqlite/delete.go | 40 -
.../CanonicalLtd/go-dqlite/cmd/dqlite/dump.go | 78 -
.../CanonicalLtd/go-dqlite/cmd/dqlite/main.go | 12 -
.../CanonicalLtd/go-dqlite/cmd/dqlite/root.go | 26 -
.../CanonicalLtd/go-dqlite/config.go | 31 -
.../github.com/CanonicalLtd/go-dqlite/doc.go | 17 -
.../CanonicalLtd/go-dqlite/driver.go | 646 -----
.../github.com/CanonicalLtd/go-dqlite/fsm.go | 17 -
.../go-dqlite/internal/bindings/build.go | 6 -
.../go-dqlite/internal/bindings/cluster.go | 249 --
.../go-dqlite/internal/bindings/config.go | 39 -
.../go-dqlite/internal/bindings/conn.go | 132 -
.../go-dqlite/internal/bindings/datatype.go | 23 -
.../go-dqlite/internal/bindings/errors.go | 58 -
.../go-dqlite/internal/bindings/logger.go | 117 -
.../go-dqlite/internal/bindings/server.go | 223 --
.../go-dqlite/internal/bindings/status.go | 43 -
.../go-dqlite/internal/bindings/testing.go | 24 -
.../go-dqlite/internal/bindings/vfs.go | 106 -
.../go-dqlite/internal/bindings/wal.go | 41 -
.../internal/bindings/wal_replication.go | 409 ---
.../go-dqlite/internal/client/buffer.go | 11 -
.../go-dqlite/internal/client/client.go | 322 ---
.../go-dqlite/internal/client/config.go | 14 -
.../go-dqlite/internal/client/connector.go | 231 --
.../go-dqlite/internal/client/dial.go | 23 -
.../go-dqlite/internal/client/errors.go | 29 -
.../go-dqlite/internal/client/message.go | 585 -----
.../go-dqlite/internal/client/request.go | 98 -
.../go-dqlite/internal/client/response.go | 213 --
.../go-dqlite/internal/client/schema.go | 26 -
.../go-dqlite/internal/client/schema.sh | 150 --
.../go-dqlite/internal/client/store.go | 48 -
.../go-dqlite/internal/connection/open.go | 76 -
.../go-dqlite/internal/connection/snapshot.go | 38 -
.../go-dqlite/internal/connection/uri.go | 61 -
.../go-dqlite/internal/logging/func.go | 26 -
.../go-dqlite/internal/logging/level.go | 27 -
.../go-dqlite/internal/protocol/commands.go | 107 -
.../internal/protocol/commands.pb.go | 2253 -----------------
.../internal/protocol/commands.proto | 87 -
.../go-dqlite/internal/protocol/doc.go | 17 -
.../go-dqlite/internal/registry/conn.go | 154 --
.../go-dqlite/internal/registry/fsm.go | 29 -
.../go-dqlite/internal/registry/hook.go | 191 --
.../go-dqlite/internal/registry/registry.go | 149 --
.../go-dqlite/internal/registry/trace.go | 36 -
.../go-dqlite/internal/registry/txn.go | 218 --
.../go-dqlite/internal/replication/doc.go | 17 -
.../go-dqlite/internal/replication/fsm.go | 809 ------
.../go-dqlite/internal/replication/methods.go | 794 ------
.../go-dqlite/internal/replication/trace.go | 23 -
.../go-dqlite/internal/store/iterate.go | 29 -
.../go-dqlite/internal/store/range.go | 59 -
.../go-dqlite/internal/store/replay.go | 47 -
.../go-dqlite/internal/trace/buffer.go | 88 -
.../go-dqlite/internal/trace/constants.go | 21 -
.../go-dqlite/internal/trace/cursor.go | 40 -
.../go-dqlite/internal/trace/doc.go | 17 -
.../go-dqlite/internal/trace/entry.go | 65 -
.../go-dqlite/internal/trace/field.go | 56 -
.../go-dqlite/internal/trace/set.go | 131 -
.../go-dqlite/internal/trace/time.go | 23 -
.../go-dqlite/internal/trace/tracer.go | 96 -
.../go-dqlite/internal/transaction/state.go | 67 -
.../go-dqlite/internal/transaction/txn.go | 237 --
.../github.com/CanonicalLtd/go-dqlite/log.go | 33 -
.../go-dqlite/recover/delete/delete.go | 36 -
.../go-dqlite/recover/dump/dump.go | 105 -
.../go-dqlite/recover/dump/options.go | 56 -
.../CanonicalLtd/go-dqlite/recover/open.go | 30 -
.../CanonicalLtd/go-dqlite/registry.go | 50 -
.../CanonicalLtd/go-dqlite/run-demo | 62 -
.../CanonicalLtd/go-dqlite/server.go | 233 --
.../CanonicalLtd/go-dqlite/store.go | 141 --
.../github.com/CanonicalLtd/raft-http/AUTHORS | 1 -
.../github.com/CanonicalLtd/raft-http/LICENSE | 201 --
.../CanonicalLtd/raft-http/README.md | 14 -
.../github.com/CanonicalLtd/raft-http/dial.go | 50 -
.../github.com/CanonicalLtd/raft-http/doc.go | 67 -
.../CanonicalLtd/raft-http/handler.go | 212 --
.../CanonicalLtd/raft-http/layer.go | 132 -
.../CanonicalLtd/raft-http/membership.go | 95 -
.../CanonicalLtd/raft-membership/AUTHORS | 1 -
.../CanonicalLtd/raft-membership/LICENSE | 201 --
.../CanonicalLtd/raft-membership/README.md | 11 -
.../CanonicalLtd/raft-membership/changer.go | 36 -
.../CanonicalLtd/raft-membership/errors.go | 49 -
.../CanonicalLtd/raft-membership/handle.go | 65 -
.../CanonicalLtd/raft-membership/request.go | 113 -
.../github.com/CanonicalLtd/raft-test/AUTHORS | 1 -
.../github.com/CanonicalLtd/raft-test/LICENSE | 201 --
.../CanonicalLtd/raft-test/README.md | 11 -
.../CanonicalLtd/raft-test/cluster.go | 303 ---
.../CanonicalLtd/raft-test/control.go | 519 ----
.../CanonicalLtd/raft-test/duration.go | 45 -
.../github.com/CanonicalLtd/raft-test/fsm.go | 60 -
.../raft-test/internal/election/future.go | 61 -
.../raft-test/internal/election/leadership.go | 43 -
.../raft-test/internal/election/notifier.go | 149 --
.../raft-test/internal/election/tracker.go | 112 -
.../raft-test/internal/event/event.go | 54 -
.../raft-test/internal/fsms/watcher.go | 77 -
.../raft-test/internal/fsms/wrapper.go | 188 --
.../raft-test/internal/logging/logger.go | 50 -
.../raft-test/internal/network/logs.go | 76 -
.../raft-test/internal/network/network.go | 147 --
.../raft-test/internal/network/peers.go | 307 ---
.../raft-test/internal/network/pipeline.go | 166 --
.../raft-test/internal/network/schedule.go | 178 --
.../raft-test/internal/network/transport.go | 268 --
.../CanonicalLtd/raft-test/options.go | 107 -
.../CanonicalLtd/raft-test/server.go | 36 -
.../github.com/CanonicalLtd/raft-test/term.go | 219 --
.../github.com/hashicorp/raft-boltdb/LICENSE | 362 ---
.../github.com/hashicorp/raft-boltdb/Makefile | 11 -
.../hashicorp/raft-boltdb/README.md | 11 -
.../hashicorp/raft-boltdb/bolt_store.go | 268 --
.../github.com/hashicorp/raft-boltdb/util.go | 37 -
vendor/github.com/hashicorp/raft/CHANGELOG.md | 16 -
vendor/github.com/hashicorp/raft/LICENSE | 354 ---
vendor/github.com/hashicorp/raft/Makefile | 20 -
vendor/github.com/hashicorp/raft/README.md | 107 -
vendor/github.com/hashicorp/raft/api.go | 1013 --------
vendor/github.com/hashicorp/raft/commands.go | 151 --
.../github.com/hashicorp/raft/commitment.go | 101 -
vendor/github.com/hashicorp/raft/config.go | 265 --
.../hashicorp/raft/configuration.go | 343 ---
.../hashicorp/raft/discard_snapshot.go | 49 -
.../hashicorp/raft/file_snapshot.go | 528 ----
vendor/github.com/hashicorp/raft/fsm.go | 136 -
vendor/github.com/hashicorp/raft/future.go | 289 ---
vendor/github.com/hashicorp/raft/go.mod | 10 -
vendor/github.com/hashicorp/raft/go.sum | 37 -
.../hashicorp/raft/inmem_snapshot.go | 109 -
.../github.com/hashicorp/raft/inmem_store.go | 130 -
.../hashicorp/raft/inmem_transport.go | 335 ---
vendor/github.com/hashicorp/raft/log.go | 72 -
vendor/github.com/hashicorp/raft/log_cache.go | 79 -
.../github.com/hashicorp/raft/membership.md | 83 -
.../hashicorp/raft/net_transport.go | 757 ------
vendor/github.com/hashicorp/raft/observer.go | 131 -
vendor/github.com/hashicorp/raft/peersjson.go | 98 -
vendor/github.com/hashicorp/raft/raft.go | 1486 -----------
.../github.com/hashicorp/raft/replication.go | 572 -----
vendor/github.com/hashicorp/raft/snapshot.go | 239 --
vendor/github.com/hashicorp/raft/stable.go | 15 -
vendor/github.com/hashicorp/raft/state.go | 171 --
vendor/github.com/hashicorp/raft/tag.sh | 16 -
.../hashicorp/raft/tcp_transport.go | 116 -
vendor/github.com/hashicorp/raft/transport.go | 124 -
vendor/github.com/hashicorp/raft/util.go | 133 -
vendor/vendor.json | 45 -
159 files changed, 24924 deletions(-)
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/AUTHORS
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/LICENSE
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/Makefile
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/README.md
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/cluster.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/cmd/dqlite/bench.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/cmd/dqlite/delete.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/cmd/dqlite/dump.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/cmd/dqlite/main.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/cmd/dqlite/root.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/config.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/doc.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/driver.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/fsm.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/build.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/cluster.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/config.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/conn.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/datatype.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/errors.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/logger.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/server.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/status.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/testing.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/vfs.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/wal.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/wal_replication.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/client/buffer.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/client/client.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/client/config.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/client/connector.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/client/dial.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/client/errors.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/client/message.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/client/request.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/client/response.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/client/schema.go
delete mode 100755 vendor/github.com/CanonicalLtd/go-dqlite/internal/client/schema.sh
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/client/store.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/connection/open.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/connection/snapshot.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/connection/uri.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/logging/func.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/logging/level.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/protocol/commands.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/protocol/commands.pb.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/protocol/commands.proto
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/protocol/doc.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/conn.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/fsm.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/hook.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/registry.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/trace.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/txn.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/replication/doc.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/replication/fsm.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/replication/methods.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/replication/trace.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/store/iterate.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/store/range.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/store/replay.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/buffer.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/constants.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/cursor.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/doc.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/entry.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/field.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/set.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/time.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/tracer.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/transaction/state.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/internal/transaction/txn.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/log.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/recover/delete/delete.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/recover/dump/dump.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/recover/dump/options.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/recover/open.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/registry.go
delete mode 100755 vendor/github.com/CanonicalLtd/go-dqlite/run-demo
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/server.go
delete mode 100644 vendor/github.com/CanonicalLtd/go-dqlite/store.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-http/AUTHORS
delete mode 100644 vendor/github.com/CanonicalLtd/raft-http/LICENSE
delete mode 100644 vendor/github.com/CanonicalLtd/raft-http/README.md
delete mode 100644 vendor/github.com/CanonicalLtd/raft-http/dial.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-http/doc.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-http/handler.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-http/layer.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-http/membership.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-membership/AUTHORS
delete mode 100644 vendor/github.com/CanonicalLtd/raft-membership/LICENSE
delete mode 100644 vendor/github.com/CanonicalLtd/raft-membership/README.md
delete mode 100644 vendor/github.com/CanonicalLtd/raft-membership/changer.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-membership/errors.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-membership/handle.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-membership/request.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/AUTHORS
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/LICENSE
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/README.md
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/cluster.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/control.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/duration.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/fsm.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/internal/election/future.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/internal/election/leadership.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/internal/election/notifier.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/internal/election/tracker.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/internal/event/event.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/internal/fsms/watcher.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/internal/fsms/wrapper.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/internal/logging/logger.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/internal/network/logs.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/internal/network/network.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/internal/network/peers.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/internal/network/pipeline.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/internal/network/schedule.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/internal/network/transport.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/options.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/server.go
delete mode 100644 vendor/github.com/CanonicalLtd/raft-test/term.go
delete mode 100644 vendor/github.com/hashicorp/raft-boltdb/LICENSE
delete mode 100644 vendor/github.com/hashicorp/raft-boltdb/Makefile
delete mode 100644 vendor/github.com/hashicorp/raft-boltdb/README.md
delete mode 100644 vendor/github.com/hashicorp/raft-boltdb/bolt_store.go
delete mode 100644 vendor/github.com/hashicorp/raft-boltdb/util.go
delete mode 100644 vendor/github.com/hashicorp/raft/CHANGELOG.md
delete mode 100644 vendor/github.com/hashicorp/raft/LICENSE
delete mode 100644 vendor/github.com/hashicorp/raft/Makefile
delete mode 100644 vendor/github.com/hashicorp/raft/README.md
delete mode 100644 vendor/github.com/hashicorp/raft/api.go
delete mode 100644 vendor/github.com/hashicorp/raft/commands.go
delete mode 100644 vendor/github.com/hashicorp/raft/commitment.go
delete mode 100644 vendor/github.com/hashicorp/raft/config.go
delete mode 100644 vendor/github.com/hashicorp/raft/configuration.go
delete mode 100644 vendor/github.com/hashicorp/raft/discard_snapshot.go
delete mode 100644 vendor/github.com/hashicorp/raft/file_snapshot.go
delete mode 100644 vendor/github.com/hashicorp/raft/fsm.go
delete mode 100644 vendor/github.com/hashicorp/raft/future.go
delete mode 100644 vendor/github.com/hashicorp/raft/go.mod
delete mode 100644 vendor/github.com/hashicorp/raft/go.sum
delete mode 100644 vendor/github.com/hashicorp/raft/inmem_snapshot.go
delete mode 100644 vendor/github.com/hashicorp/raft/inmem_store.go
delete mode 100644 vendor/github.com/hashicorp/raft/inmem_transport.go
delete mode 100644 vendor/github.com/hashicorp/raft/log.go
delete mode 100644 vendor/github.com/hashicorp/raft/log_cache.go
delete mode 100644 vendor/github.com/hashicorp/raft/membership.md
delete mode 100644 vendor/github.com/hashicorp/raft/net_transport.go
delete mode 100644 vendor/github.com/hashicorp/raft/observer.go
delete mode 100644 vendor/github.com/hashicorp/raft/peersjson.go
delete mode 100644 vendor/github.com/hashicorp/raft/raft.go
delete mode 100644 vendor/github.com/hashicorp/raft/replication.go
delete mode 100644 vendor/github.com/hashicorp/raft/snapshot.go
delete mode 100644 vendor/github.com/hashicorp/raft/stable.go
delete mode 100644 vendor/github.com/hashicorp/raft/state.go
delete mode 100755 vendor/github.com/hashicorp/raft/tag.sh
delete mode 100644 vendor/github.com/hashicorp/raft/tcp_transport.go
delete mode 100644 vendor/github.com/hashicorp/raft/transport.go
delete mode 100644 vendor/github.com/hashicorp/raft/util.go
delete mode 100644 vendor/vendor.json
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/AUTHORS b/vendor/github.com/CanonicalLtd/go-dqlite/AUTHORS
deleted file mode 100644
index 6e13f86ebb..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/AUTHORS
+++ /dev/null
@@ -1 +0,0 @@
-Free Ekanayaka <free.ekanayaka at canonical.com>
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/LICENSE b/vendor/github.com/CanonicalLtd/go-dqlite/LICENSE
deleted file mode 100644
index 261eeb9e9f..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/LICENSE
+++ /dev/null
@@ -1,201 +0,0 @@
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/Makefile b/vendor/github.com/CanonicalLtd/go-dqlite/Makefile
deleted file mode 100644
index edf4a49e21..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/Makefile
+++ /dev/null
@@ -1,4 +0,0 @@
-proto:
- protoc --gogofast_out=. internal/protocol/commands.proto
-
-.PHONY: proto
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/README.md b/vendor/github.com/CanonicalLtd/go-dqlite/README.md
deleted file mode 100644
index 4c02176c50..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/README.md
+++ /dev/null
@@ -1,99 +0,0 @@
-go-dqlite [](https://travis-ci.org/CanonicalLtd/go-dqlite) [](https://coveralls.io/github/CanonicalLtd/go-dqlite?branch=master) [](https://goreportcard.com/report/github.com/CanonicalLtd/go-dqlite) [](https://godoc.org/github.com/CanonicalLtd/go-dqlite)
-======
-
-This repository provides the `dqlite` Go package, which can be used to
-replicate a SQLite database across a cluster, using the Raft
-algorithm.
-
-Design higlights
-----------------
-
-* No external processes needed: dqlite is just a Go library, you link it
- it to your application exactly like you would with SQLite.
-* Replication needs a [SQLite patch](https://github.com/CanonicalLtd/sqlite/commit/2a9aa8b056f37ae05f38835182a2856ffc95aee4)
- which is not yet included upstream.
-* The Go [Raft package](https://github.com/hashicorp/raft) from Hashicorp
- is used internally for replicating the write-ahead log frames of SQLite
- across all nodes.
-
-How does it compare to rqlite?
-------------------------------
-
-The main differences from [rqlite](https://github.com/rqlite/rqlite) are:
-
-* Full support for transactions
-* No need for statements to be deterministic (e.g. you can use ```time()```)
-* Frame-based replication instead of statement-based replication, this
- means in dqlite there's more data flowing between nodes, so expect
- lower performance. Should not really matter for most use cases.
-
-Status
-------
-
-This is **beta** software for now, but we'll get to rc/release soon.
-
-Demo
-----
-
-To see dqlite in action, make sure you have the following dependencies
-installed:
-
-* Go (tested on 1.8)
-* gcc
-* any dependency/header that SQLite needs to build from source
-* Python 3
-
-Then run:
-
-```
-go get -d github.com/CanonicalLtd/dqlite
-cd $GOPATH/src/github.com/CanonicalLtd/dqlite
-make dependencies
-./run-demo
-```
-
-This should spawn three dqlite-based nodes, each of one running the
-code in the [demo Go source](testdata/demo.go).
-
-Each node inserts data in a test table and then dies abruptly after a
-random timeout. Leftover transactions and failover to other nodes
-should be handled gracefully.
-
-While the demo is running, to get more details about what's going on
-behind the scenes you can also open another terminal and run a command
-like:
-
-```
-watch ls -l /tmp/dqlite-demo-*/ /tmp/dqlite-demo-*/snapshots/
-```
-
-and see how the data directories of the three nodes evolve in terms
-SQLite databases (```test.db```), write-ahead log files (```test.db-wal```),
-raft logs store (```raft.db```), and raft snapshots.
-
-
-Documentation
--------------
-
-The documentation for this package can be found on [Godoc](http://godoc.org/github.com/CanonicalLtd/dqlite).
-
-FAQ
----
-
-**Q**: How does dqlite behave during conflict situations? Does Raft
-select a winning WAL write and any others in flight are aborted?
-
-**A**: There can't be a conflict situation. Raft's model is that
-only the leader can append new log entries, which translated to dqlite
-means that only the leader can write new WAL frames. So this means
-that any attempt to perform a write transaction on a non-leader node
-will fail with a sqlite3x.ErrNotLeader error (and in this case clients
-are supposed to retry against whoever is the new leader).
-
-**Q**: When not enough nodes are available, are writes hung until
-consensus?
-
-**A**: Yes, however there's a (configurable) timeout. This is a
-consequence of Raft sitting in the CP spectrum of the CAP theorem: in
-case of a network partition it chooses consistency and sacrifices
-availability.
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/cluster.go b/vendor/github.com/CanonicalLtd/go-dqlite/cluster.go
deleted file mode 100644
index 3c8d743980..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/cluster.go
+++ /dev/null
@@ -1,123 +0,0 @@
-package dqlite
-
-import (
- "strconv"
- "time"
-
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
- "github.com/CanonicalLtd/go-dqlite/internal/protocol"
- "github.com/CanonicalLtd/go-dqlite/internal/registry"
- "github.com/hashicorp/raft"
- "github.com/pkg/errors"
-)
-
-type cluster struct {
- raft *raft.Raft // Raft instance
- registry *registry.Registry // Connection registry
- provider raft.ServerAddressProvider // Custom address provider
-}
-
-func (c *cluster) Leader() string {
- return string(c.raft.Leader())
-}
-
-func (c *cluster) Servers() ([]bindings.ServerInfo, error) {
- if c.raft.State() != raft.Leader {
- return nil, raft.ErrNotLeader
- }
-
- future := c.raft.GetConfiguration()
-
- if err := future.Error(); err != nil {
- return nil, errors.Wrap(err, "failed to get raft configuration")
- }
-
- configuration := future.Configuration()
-
- servers := make([]bindings.ServerInfo, len(configuration.Servers))
-
- for i := range servers {
- server := configuration.Servers[i]
-
- id, err := strconv.Atoi(string(server.ID))
- if err != nil {
- return nil, errors.Wrap(err, "server ID is not a number")
- }
- servers[i].ID = uint64(id)
-
- if c.provider != nil {
- address, err := c.provider.ServerAddr(server.ID)
- if err != nil {
- return nil, errors.Wrap(err, "failed to fetch raft server address")
- }
- if address != "" {
- servers[i].Address = string(address)
- continue
- }
- }
- servers[i].Address = string(server.Address)
- }
-
- return servers, nil
-}
-
-func (c *cluster) Register(conn *bindings.Conn) {
- filename := conn.Filename()
- c.registry.Lock()
- defer c.registry.Unlock()
- c.registry.ConnLeaderAdd(filename, conn)
-}
-
-func (c *cluster) Unregister(conn *bindings.Conn) {
- c.registry.Lock()
- defer c.registry.Unlock()
- c.registry.ConnLeaderDel(conn)
-}
-
-func (c *cluster) Barrier() error {
- if c.raft.State() != raft.Leader {
- return bindings.Error{Code: bindings.ErrIoErrNotLeader}
- }
-
- c.registry.Lock()
- index := c.registry.Index()
- c.registry.Unlock()
-
- if index == c.raft.LastIndex() {
- return nil
- }
-
- timeout := time.Minute // TODO: make this configurable
- if err := c.raft.Barrier(timeout).Error(); err != nil {
- if err == raft.ErrLeadershipLost {
- return bindings.Error{Code: bindings.ErrIoErrNotLeader}
- }
-
- // TODO: add an out-of-sync error to SQLite?
- return errors.Wrap(err, "FSM out of sync")
- }
- c.registry.Lock()
- c.registry.IndexUpdate(c.raft.LastIndex())
- c.registry.Unlock()
-
- return nil
-}
-
-func (c *cluster) Recover(token uint64) error {
- return nil
-}
-
-func (c *cluster) Checkpoint(conn *bindings.Conn) error {
- command := protocol.NewCheckpoint(conn.Filename())
-
- data, err := protocol.MarshalCommand(command)
- if err != nil {
- return err
- }
-
- if err := c.raft.Apply(data, time.Second).Error(); err != nil {
- return err
- }
-
- return nil
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/cmd/dqlite/bench.go b/vendor/github.com/CanonicalLtd/go-dqlite/cmd/dqlite/bench.go
deleted file mode 100644
index 425f07ec46..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/cmd/dqlite/bench.go
+++ /dev/null
@@ -1,130 +0,0 @@
-package main
-
-import (
- "context"
- "database/sql"
- "fmt"
- "net"
- "testing"
- "time"
-
- "github.com/CanonicalLtd/go-dqlite"
- "github.com/CanonicalLtd/raft-test"
- "github.com/hashicorp/raft"
- "github.com/pkg/errors"
- "github.com/spf13/cobra"
-)
-
-// Return a new bench command.
-func newBench() *cobra.Command {
- bench := &cobra.Command{
- Use: "bench [address]",
- Short: "Bench all raft logs after the given index (included).",
- Args: cobra.ExactArgs(2),
- RunE: func(cmd *cobra.Command, args []string) error {
- address := args[0]
- role := args[1]
-
- if role == "server" {
- return runServer(address)
- }
-
- return runClient(address)
- },
- }
-
- return bench
-}
-
-func runServer(address string) error {
- registry := dqlite.NewRegistry("0")
- fsm := dqlite.NewFSM(registry)
-
- t := &testing.T{}
- r, cleanup := rafttest.Server(t, fsm, rafttest.Transport(func(i int) raft.Transport {
- _, transport := raft.NewInmemTransport(raft.ServerAddress(address))
- return transport
- }))
- defer cleanup()
-
- listener, err := net.Listen("tcp", address)
- if err != nil {
- return errors.Wrap(err, "failed to listen")
- }
-
- server, err := dqlite.NewServer(r, registry, listener)
- if err != nil {
- return errors.Wrap(err, "failed to create server")
- }
-
- time.Sleep(time.Minute)
-
- return server.Close()
-}
-
-func runClient(address string) error {
- store, err := dqlite.DefaultServerStore(":memory:")
- if err != nil {
- return errors.Wrap(err, "failed to create server store")
- }
-
- if err := store.Set(context.Background(), []dqlite.ServerInfo{{Address: address}}); err != nil {
- return errors.Wrap(err, "failed to set server address")
- }
-
- driver, err := dqlite.NewDriver(store)
- if err != nil {
- return errors.Wrap(err, "failed to create dqlite driver")
- }
-
- sql.Register("dqlite", driver)
-
- ctx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond)
- defer cancel()
-
- db, err := sql.Open("dqlite", "test.db")
- if err != nil {
- return errors.Wrap(err, "failed to open database")
- }
- defer db.Close()
-
- tx, err := db.Begin()
- if err != nil {
- return errors.Wrap(err, "failed to begin transaction")
- }
-
- start := time.Now()
-
- if _, err := tx.ExecContext(ctx, "CREATE TABLE test (n INT, t TEXT)"); err != nil {
- return errors.Wrapf(err, "failed to create test table")
- }
-
- for i := 0; i < 100; i++ {
- if _, err := tx.ExecContext(ctx, "INSERT INTO test(n,t) VALUES(?, ?)", int64(i), "hello"); err != nil {
- return errors.Wrapf(err, "failed to insert test value")
- }
- }
-
- rows, err := tx.QueryContext(ctx, "SELECT n FROM test")
- if err != nil {
- return errors.Wrapf(err, "failed to query test table")
- }
-
- for rows.Next() {
- var n int64
- if err := rows.Scan(&n); err != nil {
- return errors.Wrap(err, "failed to scan row")
- }
- }
- if err := rows.Err(); err != nil {
- return errors.Wrap(err, "result set failure")
- }
-
- if err := tx.Commit(); err != nil {
- return errors.Wrap(err, "failed to commit transaction")
- }
-
- fmt.Printf("time %s\n", time.Since(start))
-
- return nil
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/cmd/dqlite/delete.go b/vendor/github.com/CanonicalLtd/go-dqlite/cmd/dqlite/delete.go
deleted file mode 100644
index b5a1396929..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/cmd/dqlite/delete.go
+++ /dev/null
@@ -1,40 +0,0 @@
-package main
-
-import (
- "strconv"
-
- "github.com/CanonicalLtd/go-dqlite/recover"
- "github.com/CanonicalLtd/go-dqlite/recover/delete"
- "github.com/pkg/errors"
- "github.com/spf13/cobra"
-)
-
-// Return a new delete command.
-func newDelete() *cobra.Command {
- delete := &cobra.Command{
- Use: "delete [path to raft data dir] [index]",
- Short: "Delete all raft logs after the given index (included).",
- Args: cobra.ExactArgs(2),
- RunE: func(cmd *cobra.Command, args []string) error {
- dir := args[0]
-
- index, err := strconv.Atoi(args[1])
- if err != nil {
- return errors.Wrapf(err, "invalid index: %s", args[1])
- }
-
- logs, _, err := recover.Open(dir)
- if err != nil {
- return err
- }
-
- if err := delete.Delete(logs, uint64(index)); err != nil {
- return err
- }
-
- return nil
- },
- }
-
- return delete
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/cmd/dqlite/dump.go b/vendor/github.com/CanonicalLtd/go-dqlite/cmd/dqlite/dump.go
deleted file mode 100644
index 2a688907e8..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/cmd/dqlite/dump.go
+++ /dev/null
@@ -1,78 +0,0 @@
-package main
-
-import (
- "io/ioutil"
- "os"
- "path/filepath"
-
- "github.com/CanonicalLtd/go-dqlite/recover"
- "github.com/CanonicalLtd/go-dqlite/recover/dump"
- "github.com/hashicorp/raft"
- "github.com/hashicorp/raft-boltdb"
- "github.com/pkg/errors"
- "github.com/spf13/cobra"
-)
-
-// Return a new dump command.
-func newDump() *cobra.Command {
- var head int
- var tail int
- var replay string
-
- dump := &cobra.Command{
- Use: "dump [path to raft data dir]",
- Short: "Dump or replay the content of a dqlite raft store.",
- Args: cobra.ExactArgs(1),
- RunE: func(cmd *cobra.Command, args []string) error {
- dir := args[0]
-
- logs, snaps, err := recover.Open(dir)
- if err != nil {
- return err
- }
-
- options := make([]dump.Option, 0)
-
- if head != 0 {
- options = append(options, dump.Head(head))
- }
- if tail != 0 {
- options = append(options, dump.Tail(tail))
- }
- if replay != "" {
- options = append(options, dump.Replay(replay))
- }
-
- if err := dump.Dump(logs, snaps, os.Stdout, options...); err != nil {
- return err
- }
-
- return nil
- },
- }
-
- flags := dump.Flags()
- flags.IntVarP(&head, "head", "H", 0, "limit the dump to the first N log commands")
- flags.IntVarP(&tail, "tail", "t", 0, "limit the dump to the last N log commands")
- flags.StringVarP(&replay, "replay", "r", "", "replay the logs to the given database dir")
-
- return dump
-}
-
-func dumpOpen(dir string) (raft.LogStore, raft.SnapshotStore, error) {
- if _, err := os.Stat(dir); err != nil {
- return nil, nil, errors.Wrap(err, "invalid raft data dir")
- }
-
- logs, err := raftboltdb.NewBoltStore(filepath.Join(dir, "logs.db"))
- if err != nil {
- return nil, nil, errors.Wrap(err, "failed to open boltdb file")
- }
-
- snaps, err := raft.NewFileSnapshotStore(dir, 1, ioutil.Discard)
- if err != nil {
- return nil, nil, errors.Wrap(err, "failed to open snapshot store")
- }
-
- return logs, snaps, nil
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/cmd/dqlite/main.go b/vendor/github.com/CanonicalLtd/go-dqlite/cmd/dqlite/main.go
deleted file mode 100644
index 1f9f41df45..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/cmd/dqlite/main.go
+++ /dev/null
@@ -1,12 +0,0 @@
-package main
-
-import (
- "os"
-)
-
-func main() {
- root := newRoot()
- if err := root.Execute(); err != nil {
- os.Exit(1)
- }
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/cmd/dqlite/root.go b/vendor/github.com/CanonicalLtd/go-dqlite/cmd/dqlite/root.go
deleted file mode 100644
index 87900c761e..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/cmd/dqlite/root.go
+++ /dev/null
@@ -1,26 +0,0 @@
-package main
-
-import (
- "fmt"
-
- "github.com/spf13/cobra"
-)
-
-// Return a new root command.
-func newRoot() *cobra.Command {
- root := &cobra.Command{
- Use: "dqlite",
- Short: "Distributed SQLite for Go applications",
- Long: `Replicate a SQLite database across a cluster, using the Raft algorithm.
-
-Complete documentation is available at https://github.com/CanonicalLtd/go-dqlite`,
- RunE: func(cmd *cobra.Command, args []string) error {
- return fmt.Errorf("not implemented")
- },
- }
- root.AddCommand(newDump())
- root.AddCommand(newDelete())
- root.AddCommand(newBench())
-
- return root
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/config.go b/vendor/github.com/CanonicalLtd/go-dqlite/config.go
deleted file mode 100644
index 37c20fc6dc..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/config.go
+++ /dev/null
@@ -1,31 +0,0 @@
-package dqlite
-
-/*
-import (
- "fmt"
- "os"
-
- "github.com/pkg/errors"
-)
-
-// Ensure that the configured directory exists and is accessible.
-func ensureDir(dir string) error {
- if dir == "" {
- return fmt.Errorf("no data dir provided in config")
- }
- info, err := os.Stat(dir)
- if err != nil {
- if os.IsNotExist(err) {
- if err := os.MkdirAll(dir, 0700); err != nil {
- return errors.Wrap(err, "failed to create data dir")
- }
- return nil
- }
- return errors.Wrap(err, "failed to access data dir")
- }
- if !info.IsDir() {
- return fmt.Errorf("data dir '%s' is not a directory", dir)
- }
- return nil
-}
-*/
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/doc.go b/vendor/github.com/CanonicalLtd/go-dqlite/doc.go
deleted file mode 100644
index 98fc5d42b2..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/doc.go
+++ /dev/null
@@ -1,17 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package dqlite implements a database/sql/driver with raft-based
-// SQLite replication.
-package dqlite
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/driver.go b/vendor/github.com/CanonicalLtd/go-dqlite/driver.go
deleted file mode 100644
index 316c96d4b9..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/driver.go
+++ /dev/null
@@ -1,646 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package dqlite
-
-import (
- "context"
- "database/sql/driver"
- "io"
- "net"
- "reflect"
- "time"
-
- "github.com/Rican7/retry/backoff"
- "github.com/Rican7/retry/strategy"
- "github.com/pkg/errors"
-
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
- "github.com/CanonicalLtd/go-dqlite/internal/client"
- "github.com/CanonicalLtd/go-dqlite/internal/connection"
-)
-
-// Driver perform queries against a dqlite server.
-type Driver struct {
- log LogFunc // Log function to use
- store ServerStore // Holds addresses of dqlite servers
- context context.Context // Global cancellation context
- connectionTimeout time.Duration // Max time to wait for a new connection
- contextTimeout time.Duration // Default client context timeout.
- clientConfig client.Config // Configuration for dqlite client instances
-}
-
-// DriverError is returned in case of database errors.
-type DriverError = bindings.Error
-
-// DriverOption can be used to tweak driver parameters.
-type DriverOption func(*driverOptions)
-
-// WithLogFunc sets a custom logging function.
-func WithLogFunc(log LogFunc) DriverOption {
- return func(options *driverOptions) {
- options.Log = log
- }
-}
-
-// DialFunc is a function that can be used to establish a network connection.
-type DialFunc client.DialFunc
-
-// WithDialFunc sets a custom dial function.
-func WithDialFunc(dial DialFunc) DriverOption {
- return func(options *driverOptions) {
- options.Dial = client.DialFunc(dial)
- }
-}
-
-// WithConnectionTimeout sets the connection timeout.
-//
-// If not used, the default is 5 seconds.
-func WithConnectionTimeout(timeout time.Duration) DriverOption {
- return func(options *driverOptions) {
- options.ConnectionTimeout = timeout
- }
-}
-
-// WithContextTimeout sets the default client context timeout when no context
-// deadline is provided.
-//
-// If not used, the default is 5 seconds.
-func WithContextTimeout(timeout time.Duration) DriverOption {
- return func(options *driverOptions) {
- options.ContextTimeout = timeout
- }
-}
-
-// WithConnectionBackoffFactor sets the exponential backoff factor for retrying
-// failed connection attempts.
-//
-// If not used, the default is 50 milliseconds.
-func WithConnectionBackoffFactor(factor time.Duration) DriverOption {
- return func(options *driverOptions) {
- options.ConnectionBackoffFactor = factor
- }
-}
-
-// WithConnectionBackoffCap sets the maximum connection retry backoff value,
-// (regardless of the backoff factor) for retrying failed connection attempts.
-//
-// If not used, the default is 1 second.
-func WithConnectionBackoffCap(cap time.Duration) DriverOption {
- return func(options *driverOptions) {
- options.ConnectionBackoffCap = cap
- }
-}
-
-// WithContext sets a global cancellation context.
-func WithContext(context context.Context) DriverOption {
- return func(options *driverOptions) {
- options.Context = context
- }
-}
-
-// NewDriver creates a new dqlite driver, which also implements the
-// driver.Driver interface.
-func NewDriver(store ServerStore, options ...DriverOption) (*Driver, error) {
- o := defaultDriverOptions()
-
- for _, option := range options {
- option(o)
- }
-
- driver := &Driver{
- log: o.Log,
- store: store,
- context: o.Context,
- connectionTimeout: o.ConnectionTimeout,
- contextTimeout: o.ContextTimeout,
- }
-
- driver.clientConfig.Dial = o.Dial
- driver.clientConfig.AttemptTimeout = 5 * time.Second
- driver.clientConfig.RetryStrategies = []strategy.Strategy{
- driverConnectionRetryStrategy(
- o.ConnectionBackoffFactor,
- o.ConnectionBackoffCap,
- ),
- }
-
- return driver, nil
-}
-
-// Hold configuration options for a dqlite driver.
-type driverOptions struct {
- Log LogFunc
- Dial client.DialFunc
- ConnectionTimeout time.Duration
- ContextTimeout time.Duration
- ConnectionBackoffFactor time.Duration
- ConnectionBackoffCap time.Duration
- Context context.Context
-}
-
-// Create a driverOptions object with sane defaults.
-func defaultDriverOptions() *driverOptions {
- return &driverOptions{
- Log: defaultLogFunc(),
- Dial: client.TCPDial,
- ConnectionTimeout: 15 * time.Second,
- ContextTimeout: 5 * time.Second,
- ConnectionBackoffFactor: 50 * time.Millisecond,
- ConnectionBackoffCap: time.Second,
- Context: context.Background(),
- }
-}
-
-// Return a retry strategy with jittered exponential backoff, capped at the
-// given amount of time.
-func driverConnectionRetryStrategy(factor, cap time.Duration) strategy.Strategy {
- backoff := backoff.BinaryExponential(factor)
-
- return func(attempt uint) bool {
- if attempt > 0 {
- duration := backoff(attempt)
- if duration > cap {
- duration = cap
- }
- time.Sleep(duration)
- }
-
- return true
- }
-}
-
-// Open establishes a new connection to a SQLite database on the dqlite server.
-//
-// The given name must be a pure file name without any directory segment,
-// dqlite will connect to a database with that name in its data directory.
-//
-// Query parameters are always valid except for "mode=memory".
-//
-// If this node is not the leader, or the leader is unknown an ErrNotLeader
-// error is returned.
-func (d *Driver) Open(uri string) (driver.Conn, error) {
- // Validate the given data source string.
- filename, flags, err := connection.ParseURI(uri)
- if err != nil {
- return nil, errors.Wrapf(err, "invalid URI %s", uri)
- }
-
- ctx, cancel := context.WithTimeout(d.context, d.connectionTimeout)
- defer cancel()
-
- // TODO: generate a client ID.
- connector := client.NewConnector(0, d.store, d.clientConfig, d.log)
-
- conn := &Conn{
- log: d.log,
- contextTimeout: d.contextTimeout,
- }
-
- conn.client, err = connector.Connect(ctx)
- if err != nil {
- return nil, errors.Wrap(err, "failed to create dqlite connection")
- }
- conn.client.SetContextTimeout(d.contextTimeout)
-
- conn.request.Init(4096)
- conn.response.Init(4096)
-
- defer conn.request.Reset()
- defer conn.response.Reset()
-
- client.EncodeOpen(&conn.request, filename, flags, "volatile")
-
- if err := conn.client.Call(ctx, &conn.request, &conn.response); err != nil {
- conn.client.Close()
- return nil, errors.Wrap(err, "failed to open database")
- }
-
- conn.id, err = client.DecodeDb(&conn.response)
- if err != nil {
- conn.client.Close()
- return nil, errors.Wrap(err, "failed to open database")
- }
-
- return conn, nil
-}
-
-// SetContextTimeout sets the default client timeout when no context deadline
-// is provided.
-func (d *Driver) SetContextTimeout(timeout time.Duration) {
- d.contextTimeout = timeout
-}
-
-// ErrNoAvailableLeader is returned as root cause of Open() if there's no
-// leader available in the cluster.
-var ErrNoAvailableLeader = client.ErrNoAvailableLeader
-
-// Conn implements the sql.Conn interface.
-type Conn struct {
- log LogFunc
- client *client.Client
- request client.Message
- response client.Message
- id uint32 // Database ID.
- contextTimeout time.Duration
-}
-
-// PrepareContext returns a prepared statement, bound to this connection.
-// context is for the preparation of the statement, it must not store the
-// context within the statement itself.
-func (c *Conn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) {
- defer c.request.Reset()
- defer c.response.Reset()
-
- stmt := &Stmt{
- client: c.client,
- request: &c.request,
- response: &c.response,
- }
-
- client.EncodePrepare(&c.request, uint64(c.id), query)
-
- if err := c.client.Call(ctx, &c.request, &c.response); err != nil {
- return nil, driverError(err)
- }
-
- var err error
- stmt.db, stmt.id, stmt.params, err = client.DecodeStmt(&c.response)
- if err != nil {
- return nil, driverError(err)
- }
-
- return stmt, nil
-}
-
-// Prepare returns a prepared statement, bound to this connection.
-func (c *Conn) Prepare(query string) (driver.Stmt, error) {
- return c.PrepareContext(context.Background(), query)
-}
-
-// ExecContext is an optional interface that may be implemented by a Conn.
-func (c *Conn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) {
- defer c.request.Reset()
- defer c.response.Reset()
-
- client.EncodeExecSQL(&c.request, uint64(c.id), query, args)
-
- if err := c.client.Call(ctx, &c.request, &c.response); err != nil {
- return nil, driverError(err)
- }
-
- result, err := client.DecodeResult(&c.response)
- if err != nil {
- return nil, driverError(err)
- }
-
- return &Result{result: result}, nil
-}
-
-// Query is an optional interface that may be implemented by a Conn.
-func (c *Conn) Query(query string, args []driver.Value) (driver.Rows, error) {
- return c.QueryContext(context.Background(), query, valuesToNamedValues(args))
-}
-
-// QueryContext is an optional interface that may be implemented by a Conn.
-func (c *Conn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {
- defer c.request.Reset()
-
- client.EncodeQuerySQL(&c.request, uint64(c.id), query, args)
-
- if err := c.client.Call(ctx, &c.request, &c.response); err != nil {
- return nil, driverError(err)
- }
-
- rows, err := client.DecodeRows(&c.response)
- if err != nil {
- return nil, driverError(err)
- }
-
- return &Rows{ctx: ctx, request: &c.request, response: &c.response, client: c.client, rows: rows}, nil
-}
-
-// Exec is an optional interface that may be implemented by a Conn.
-func (c *Conn) Exec(query string, args []driver.Value) (driver.Result, error) {
- return c.ExecContext(context.Background(), query, valuesToNamedValues(args))
-}
-
-// Close invalidates and potentially stops any current prepared statements and
-// transactions, marking this connection as no longer in use.
-//
-// Because the sql package maintains a free pool of connections and only calls
-// Close when there's a surplus of idle connections, it shouldn't be necessary
-// for drivers to do their own connection caching.
-func (c *Conn) Close() error {
- return c.client.Close()
-}
-
-// BeginTx starts and returns a new transaction. If the context is canceled by
-// the user the sql package will call Tx.Rollback before discarding and closing
-// the connection.
-//
-// This must check opts.Isolation to determine if there is a set isolation
-// level. If the driver does not support a non-default level and one is set or
-// if there is a non-default isolation level that is not supported, an error
-// must be returned.
-//
-// This must also check opts.ReadOnly to determine if the read-only value is
-// true to either set the read-only transaction property if supported or return
-// an error if it is not supported.
-func (c *Conn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) {
- if _, err := c.ExecContext(ctx, "BEGIN", nil); err != nil {
- return nil, driverError(err)
- }
-
- tx := &Tx{
- conn: c,
- }
-
- return tx, nil
-}
-
-// Begin starts and returns a new transaction.
-//
-// Deprecated: Drivers should implement ConnBeginTx instead (or additionally).
-func (c *Conn) Begin() (driver.Tx, error) {
- return c.BeginTx(context.Background(), driver.TxOptions{})
-}
-
-// Tx is a transaction.
-type Tx struct {
- conn *Conn
-}
-
-// Commit the transaction.
-func (tx *Tx) Commit() error {
- ctx, cancel := context.WithTimeout(context.Background(), tx.conn.contextTimeout)
- defer cancel()
-
- if _, err := tx.conn.ExecContext(ctx, "COMMIT", nil); err != nil {
- return driverError(err)
- }
-
- return nil
-}
-
-// Rollback the transaction.
-func (tx *Tx) Rollback() error {
- ctx, cancel := context.WithTimeout(context.Background(), tx.conn.contextTimeout)
- defer cancel()
-
- if _, err := tx.conn.ExecContext(ctx, "ROLLBACK", nil); err != nil {
- return driverError(err)
- }
-
- return nil
-}
-
-// Stmt is a prepared statement. It is bound to a Conn and not
-// used by multiple goroutines concurrently.
-type Stmt struct {
- client *client.Client
- request *client.Message
- response *client.Message
- db uint32
- id uint32
- params uint64
-}
-
-// Close closes the statement.
-func (s *Stmt) Close() error {
- defer s.request.Reset()
- defer s.response.Reset()
-
- client.EncodeFinalize(s.request, s.db, s.id)
-
- ctx := context.Background()
-
- if err := s.client.Call(ctx, s.request, s.response); err != nil {
- return driverError(err)
- }
-
- if err := client.DecodeEmpty(s.response); err != nil {
- return driverError(err)
- }
-
- return nil
-}
-
-// NumInput returns the number of placeholder parameters.
-func (s *Stmt) NumInput() int {
- return int(s.params)
-}
-
-// ExecContext executes a query that doesn't return rows, such
-// as an INSERT or UPDATE.
-//
-// ExecContext must honor the context timeout and return when it is canceled.
-func (s *Stmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) {
- defer s.request.Reset()
- defer s.response.Reset()
-
- client.EncodeExec(s.request, s.db, s.id, args)
-
- if err := s.client.Call(ctx, s.request, s.response); err != nil {
- return nil, driverError(err)
- }
-
- result, err := client.DecodeResult(s.response)
- if err != nil {
- return nil, driverError(err)
- }
-
- return &Result{result: result}, nil
-}
-
-// Exec executes a query that doesn't return rows, such
-func (s *Stmt) Exec(args []driver.Value) (driver.Result, error) {
- return s.ExecContext(context.Background(), valuesToNamedValues(args))
-}
-
-// QueryContext executes a query that may return rows, such as a
-// SELECT.
-//
-// QueryContext must honor the context timeout and return when it is canceled.
-func (s *Stmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) {
- defer s.request.Reset()
-
- client.EncodeQuery(s.request, s.db, s.id, args)
-
- if err := s.client.Call(ctx, s.request, s.response); err != nil {
- return nil, driverError(err)
- }
-
- rows, err := client.DecodeRows(s.response)
- if err != nil {
- return nil, driverError(err)
- }
-
- return &Rows{ctx: ctx, request: s.request, response: s.response, client: s.client, rows: rows}, nil
-}
-
-// Query executes a query that may return rows, such as a
-func (s *Stmt) Query(args []driver.Value) (driver.Rows, error) {
- return s.QueryContext(context.Background(), valuesToNamedValues(args))
-}
-
-// Result is the result of a query execution.
-type Result struct {
- result client.Result
-}
-
-// LastInsertId returns the database's auto-generated ID
-// after, for example, an INSERT into a table with primary
-// key.
-func (r *Result) LastInsertId() (int64, error) {
- return int64(r.result.LastInsertID), nil
-}
-
-// RowsAffected returns the number of rows affected by the
-// query.
-func (r *Result) RowsAffected() (int64, error) {
- return int64(r.result.RowsAffected), nil
-}
-
-// Rows is an iterator over an executed query's results.
-type Rows struct {
- ctx context.Context
- client *client.Client
- request *client.Message
- response *client.Message
- rows client.Rows
- consumed bool
-}
-
-// Columns returns the names of the columns. The number of
-// columns of the result is inferred from the length of the
-// slice. If a particular column name isn't known, an empty
-// string should be returned for that entry.
-func (r *Rows) Columns() []string {
- return r.rows.Columns
-}
-
-// Close closes the rows iterator.
-func (r *Rows) Close() error {
- r.rows.Close()
-
- // If we consumed the whole result set, there's nothing to do as
- // there's no pending response from the server.
- if r.consumed {
- return nil
- }
-
- r.rows.Close()
-
- // Let's issue an interrupt request and wait until we get an empty
- // response, signalling that the query was interrupted.
- if err := r.client.Interrupt(r.ctx, r.request, r.response); err != nil {
- return driverError(err)
- }
-
- return nil
-}
-
-// Next is called to populate the next row of data into
-// the provided slice. The provided slice will be the same
-// size as the Columns() are wide.
-//
-// Next should return io.EOF when there are no more rows.
-func (r *Rows) Next(dest []driver.Value) error {
- err := r.rows.Next(dest)
-
- if err == client.ErrRowsPart {
- r.rows.Close()
- if err := r.client.More(r.ctx, r.response); err != nil {
- return driverError(err)
- }
- rows, err := client.DecodeRows(r.response)
- if err != nil {
- return driverError(err)
- }
- r.rows = rows
- return r.rows.Next(dest)
- }
-
- if err == io.EOF {
- r.consumed = true
- }
-
- return err
-}
-
-// ColumnTypeScanType implements RowsColumnTypeScanType.
-func (r *Rows) ColumnTypeScanType(i int) reflect.Type {
- // column := sql.NewColumn(r.rows, i)
-
- // typ, err := r.client.ColumnTypeScanType(context.Background(), column)
- // if err != nil {
- // return nil
- // }
-
- // return typ.DriverType()
- return nil
-}
-
-// ColumnTypeDatabaseTypeName implements RowsColumnTypeDatabaseTypeName.
-func (r *Rows) ColumnTypeDatabaseTypeName(i int) string {
- // column := sql.NewColumn(r.rows, i)
-
- // typeName, err := r.client.ColumnTypeDatabaseTypeName(context.Background(), column)
- // if err != nil {
- // return ""
- // }
-
- // return typeName.Value
- return ""
-}
-
-// Convert a driver.Value slice into a driver.NamedValue slice.
-func valuesToNamedValues(args []driver.Value) []driver.NamedValue {
- namedValues := make([]driver.NamedValue, len(args))
- for i, value := range args {
- namedValues[i] = driver.NamedValue{
- Ordinal: i + 1,
- Value: value,
- }
- }
- return namedValues
-}
-
-func driverError(err error) error {
- switch err := errors.Cause(err).(type) {
- case *net.OpError:
- return driver.ErrBadConn
- case client.ErrRequest:
- switch err.Code {
- case bindings.ErrIoErrNotLeader:
- fallthrough
- case bindings.ErrIoErrLeadershipLost:
- return driver.ErrBadConn
- default:
- return DriverError{
- Code: int(err.Code),
- Message: err.Description,
- }
- }
- }
- return err
-}
-
-func init() {
- err := bindings.Init()
- if err != nil {
- panic(errors.Wrap(err, "failed to initialize dqlite"))
- }
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/fsm.go b/vendor/github.com/CanonicalLtd/go-dqlite/fsm.go
deleted file mode 100644
index c1d15015e9..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/fsm.go
+++ /dev/null
@@ -1,17 +0,0 @@
-package dqlite
-
-import (
- "github.com/CanonicalLtd/go-dqlite/internal/replication"
- "github.com/hashicorp/raft"
-)
-
-// NewFSM creates a new dqlite FSM, suitable to be passed to raft.NewRaft.
-//
-// It will handle replication of the SQLite write-ahead log.
-//
-// This is mostly an internal implementation detail of dqlite, but it needs to
-// be exposed since the raft.Raft parameter that NewDriver accepts doesn't
-// allow access to the FSM that it was passed when created with raft.NewRaft().
-func NewFSM(r *Registry) raft.FSM {
- return replication.NewFSM(r.registry)
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/build.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/build.go
deleted file mode 100644
index e1ac1f9960..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/build.go
+++ /dev/null
@@ -1,6 +0,0 @@
-package bindings
-
-/*
-#cgo linux LDFLAGS: -lsqlite3 -ldqlite
-*/
-import "C"
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/cluster.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/cluster.go
deleted file mode 100644
index 797d3c8a07..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/cluster.go
+++ /dev/null
@@ -1,249 +0,0 @@
-package bindings
-
-/*
-#include <assert.h>
-#include <stdlib.h>
-
-#include <dqlite.h>
-
-// Go land callbacks for dqlite_cluster methods.
-char *clusterLeaderCb(uintptr_t handle);
-int clusterServersCb(uintptr_t handle, dqlite_server_info **servers);
-void clusterRegisterCb(uintptr_t handle, sqlite3 *db);
-void clusterUnregisterCb(uintptr_t handle, sqlite3 *db);
-int clusterBarrierCb(uintptr_t handle);
-int clusterRecoverCb(uintptr_t handle, uint64_t txToken);
-int clusterCheckpointCb(uintptr_t handle, sqlite3 *db);
-
-// Implementation of xLeader.
-static const char* dqlite__cluster_leader(void *ctx) {
- assert(ctx != NULL);
-
- return clusterLeaderCb((uintptr_t)ctx);
-}
-
-// Implementation of xServers.
-static int dqlite__cluster_servers(void *ctx, dqlite_server_info *servers[]) {
- assert(ctx != NULL);
-
- return clusterServersCb((uintptr_t)ctx, servers);
-}
-
-// Implementation of xRegister.
-static void dqlite__cluster_register(void *ctx, sqlite3 *db) {
- assert(ctx != NULL);
-
- clusterRegisterCb((uintptr_t)ctx, db);
-}
-
-// Implementation of xUnregister.
-static void dqlite__cluster_unregister(void *ctx, sqlite3 *db) {
- assert(ctx != NULL);
-
- clusterUnregisterCb((uintptr_t)ctx, db);
-}
-
-// Implementation of xBarrier.
-static int dqlite__cluster_barrier(void *ctx) {
- assert(ctx != NULL);
-
- return clusterBarrierCb((uintptr_t)ctx);
-}
-
-// Implementation of of xRecover.
-static int dqlite__cluster_recover(void *ctx, uint64_t tx_token) {
- assert(ctx != NULL);
-
- return clusterRecoverCb((uintptr_t)ctx, tx_token);
-}
-
-// Implementation of of xCheckpoint.
-static int dqlite__cluster_checkpoint(void *ctx, sqlite3 *db) {
- assert(ctx != NULL);
-
- return clusterCheckpointCb((uintptr_t)ctx, db);
-}
-
-// Constructor.
-static dqlite_cluster *dqlite__cluster_create(uintptr_t handle)
-{
- dqlite_cluster *c = sqlite3_malloc(sizeof *c);
- if (c == NULL) {
- return NULL;
- }
-
- c->ctx = (void*)handle;
- c->xLeader = dqlite__cluster_leader;
- c->xServers = dqlite__cluster_servers;
- c->xRegister = dqlite__cluster_register;
- c->xUnregister = dqlite__cluster_unregister;
- c->xBarrier = dqlite__cluster_barrier;
- c->xRecover = dqlite__cluster_recover;
- c->xCheckpoint = dqlite__cluster_checkpoint;
-
- return c;
-}
-*/
-import "C"
-import (
- "unsafe"
-)
-
-// Cluster is a Go wrapper around the associated dqlite's C type.
-type Cluster C.dqlite_cluster
-
-// NewCluster creates a new Cluster object set with the given method hooks..
-func NewCluster(methods ClusterMethods) (*Cluster, error) {
- handle := clusterMethodsSerial
- clusterMethodsIndex[handle] = methods
- clusterMethodsSerial++
-
- cluster := C.dqlite__cluster_create(handle)
- if cluster == nil {
- return nil, codeToError(C.SQLITE_NOMEM)
- }
-
- return (*Cluster)(unsafe.Pointer(cluster)), nil
-}
-
-// Close releases all memory associated with the cluster object.
-func (c *Cluster) Close() {
- cluster := (*C.dqlite_cluster)(unsafe.Pointer(c))
-
- handle := (C.uintptr_t)(uintptr(cluster.ctx))
- delete(clusterMethodsIndex, handle)
-
- C.sqlite3_free(unsafe.Pointer(cluster))
-}
-
-// ServerInfo is the Go equivalent of dqlite_server_info.
-type ServerInfo struct {
- ID uint64
- Address string
-}
-
-// ClusterMethods implements the interface required by the various hooks
-// dqlite_cluster.
-type ClusterMethods interface {
- // Return the address of the current cluster leader, if any. If not
- // empty, the address string must a be valid network IP or hostname,
- // that clients can use to connect to a dqlite service.
- Leader() string
-
- // If this driver is the current leader of the cluster, return the
- // addresses of all other servers. Each address must be a valid IP or
- // host name name, that clients can use to connect to the relevant
- // dqlite service , in case the current leader is deposed and a new one
- // is elected.
- //
- // If this driver is not the current leader of the cluster, an error
- // implementing the Error interface below and returning true in
- // NotLeader() must be returned.
- Servers() ([]ServerInfo, error)
-
- Register(*Conn)
- Unregister(*Conn)
-
- Barrier() error
-
- Recover(token uint64) error
-
- Checkpoint(*Conn) error
-}
-
-// Map uintptr to Cluster instances to avoid passing Go pointers to C.
-//
-// We do not protect this map with a lock since typically just one long-lived
-// Cluster instance should be registered (except for unit tests).
-var clusterMethodsSerial C.uintptr_t = 100
-var clusterMethodsIndex = map[C.uintptr_t]ClusterMethods{}
-
-//export clusterLeaderCb
-func clusterLeaderCb(handle C.uintptr_t) *C.char {
- cluster := clusterMethodsIndex[handle]
-
- // It's responsibility of calling code to free() this string.
- return C.CString(cluster.Leader())
-}
-
-//export clusterServersCb
-func clusterServersCb(handle C.uintptr_t, out **C.dqlite_server_info) C.int {
- cluster := clusterMethodsIndex[handle]
-
- servers, err := cluster.Servers()
- if err != nil {
- *out = nil
- return C.int(ErrorCode(err))
- }
-
- n := C.size_t(len(servers)) + 1
-
- // It's responsibility of calling code to free() this array of servers.
- size := unsafe.Sizeof(C.dqlite_server_info{})
- *out = (*C.dqlite_server_info)(C.malloc(n * C.size_t(size)))
-
- if *out == nil {
- return C.SQLITE_NOMEM
- }
-
- for i := C.size_t(0); i < n; i++ {
- server := (*C.dqlite_server_info)(unsafe.Pointer(uintptr(unsafe.Pointer(*out)) + size*uintptr(i)))
-
- if i == n-1 {
- server.id = 0
- server.address = nil
- } else {
- server.id = C.uint64_t(servers[i].ID)
- server.address = C.CString(servers[i].Address)
- }
- }
-
- return C.int(0)
-}
-
-//export clusterRegisterCb
-func clusterRegisterCb(handle C.uintptr_t, db *C.sqlite3) {
- cluster := clusterMethodsIndex[handle]
- cluster.Register((*Conn)(unsafe.Pointer(db)))
-}
-
-//export clusterUnregisterCb
-func clusterUnregisterCb(handle C.uintptr_t, db *C.sqlite3) {
- cluster := clusterMethodsIndex[handle]
- cluster.Unregister((*Conn)(unsafe.Pointer(db)))
-}
-
-//export clusterBarrierCb
-func clusterBarrierCb(handle C.uintptr_t) C.int {
- cluster := clusterMethodsIndex[handle]
-
- if err := cluster.Barrier(); err != nil {
- return C.int(ErrorCode(err))
- }
-
- return 0
-}
-
-//export clusterRecoverCb
-func clusterRecoverCb(handle C.uintptr_t, txToken C.uint64_t) C.int {
- cluster := clusterMethodsIndex[handle]
-
- err := cluster.Recover(uint64(txToken))
- if err != nil {
- return C.int(ErrorCode(err))
- }
-
- return C.int(0)
-}
-
-//export clusterCheckpointCb
-func clusterCheckpointCb(handle C.uintptr_t, db *C.sqlite3) C.int {
- cluster := clusterMethodsIndex[handle]
-
- err := cluster.Checkpoint((*Conn)(unsafe.Pointer(db)))
- if err != nil {
- return C.int(ErrorCode(err))
- }
-
- return C.int(0)
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/config.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/config.go
deleted file mode 100644
index 42d0f3b1da..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/config.go
+++ /dev/null
@@ -1,39 +0,0 @@
-package bindings
-
-/*
-#include <sqlite3.h>
-
-// Wrapper around sqlite3_db_config() for invoking the
-// SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE opcode, since there's no way to use C
-// varargs from Go.
-static int sqlite3__db_config_no_ckpt_on_close(sqlite3 *db, int value, int *pValue) {
- return sqlite3_db_config(db, SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, value, pValue);
-}
-*/
-import "C"
-import (
- "unsafe"
-
- "github.com/pkg/errors"
-)
-
-// ConfigNoCkptOnClose switches on or off the automatic WAL checkpoint when a
-// connection is closed.
-func (c *Conn) ConfigNoCkptOnClose(flag bool) (bool, error) {
- db := (*C.sqlite3)(unsafe.Pointer(c))
-
- var in C.int
- var out C.int
-
- if flag {
- in = 1
- }
-
- rc := C.sqlite3__db_config_no_ckpt_on_close(db, in, &out)
- if rc != C.SQLITE_OK {
- err := lastError(db)
- return false, errors.Wrap(err, "failed to config checkpoint on close")
- }
-
- return out == 1, nil
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/conn.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/conn.go
deleted file mode 100644
index 09197a35be..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/conn.go
+++ /dev/null
@@ -1,132 +0,0 @@
-package bindings
-
-/*
-#include <stdlib.h>
-#include <sqlite3.h>
-*/
-import "C"
-import (
- "database/sql/driver"
- "io"
- "unsafe"
-)
-
-// Open modes.
-const (
- OpenReadWrite = C.SQLITE_OPEN_READWRITE
- OpenReadOnly = C.SQLITE_OPEN_READONLY
- OpenCreate = C.SQLITE_OPEN_CREATE
-)
-
-// Conn is a Go wrapper around a SQLite database handle.
-type Conn C.sqlite3
-
-// Open a SQLite connection.
-func Open(name string, vfs string) (*Conn, error) {
- flags := OpenReadWrite | OpenCreate
-
- // Open the database.
- cname := C.CString(name)
- defer C.free(unsafe.Pointer(cname))
-
- cvfs := C.CString(vfs)
- defer C.free(unsafe.Pointer(cvfs))
-
- var db *C.sqlite3
-
- rc := C.sqlite3_open_v2(cname, &db, C.int(flags), cvfs)
- if rc != C.SQLITE_OK {
- err := lastError(db)
- C.sqlite3_close_v2(db)
- return nil, err
- }
-
- return (*Conn)(unsafe.Pointer(db)), nil
-}
-
-// Close the connection.
-func (c *Conn) Close() error {
- db := (*C.sqlite3)(unsafe.Pointer(c))
-
- rc := C.sqlite3_close(db)
- if rc != C.SQLITE_OK {
- return lastError(db)
- }
-
- return nil
-}
-
-// Filename of the underlying database file.
-func (c *Conn) Filename() string {
- db := (*C.sqlite3)(unsafe.Pointer(c))
-
- return C.GoString(C.sqlite3_db_filename(db, walReplicationSchema))
-}
-
-// Exec executes a query.
-func (c *Conn) Exec(query string) error {
- db := (*C.sqlite3)(unsafe.Pointer(c))
-
- sql := C.CString(query)
- defer C.free(unsafe.Pointer(sql))
-
- rc := C.sqlite3_exec(db, sql, nil, nil, nil)
- if rc != C.SQLITE_OK {
- return lastError(db)
- }
-
- return nil
-}
-
-// Query the database.
-func (c *Conn) Query(query string) (*Rows, error) {
- db := (*C.sqlite3)(unsafe.Pointer(c))
-
- var stmt *C.sqlite3_stmt
- var tail *C.char
-
- sql := C.CString(query)
- defer C.free(unsafe.Pointer(sql))
-
- rc := C.sqlite3_prepare(db, sql, C.int(-1), &stmt, &tail)
- if rc != C.SQLITE_OK {
- return nil, lastError(db)
- }
-
- rows := &Rows{db: db, stmt: stmt}
-
- return rows, nil
-}
-
-// Rows represents a result set.
-type Rows struct {
- db *C.sqlite3
- stmt *C.sqlite3_stmt
-}
-
-// Next fetches the next row of a result set.
-func (r *Rows) Next(values []driver.Value) error {
- rc := C.sqlite3_step(r.stmt)
- if rc == C.SQLITE_DONE {
- return io.EOF
- }
- if rc != C.SQLITE_ROW {
- return lastError(r.db)
- }
-
- for i := range values {
- values[i] = int64(C.sqlite3_column_int64(r.stmt, C.int(i)))
- }
-
- return nil
-}
-
-// Close finalizes the underlying statement.
-func (r *Rows) Close() error {
- rc := C.sqlite3_finalize(r.stmt)
- if rc != C.SQLITE_OK {
- return lastError(r.db)
- }
-
- return nil
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/datatype.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/datatype.go
deleted file mode 100644
index 9f4e4dcb77..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/datatype.go
+++ /dev/null
@@ -1,23 +0,0 @@
-package bindings
-
-/*
-#include <sqlite3.h>
-#include <dqlite.h>
-*/
-import "C"
-
-// SQLite datatype codes
-const (
- Integer = C.SQLITE_INTEGER
- Float = C.SQLITE_FLOAT
- Text = C.SQLITE_TEXT
- Blob = C.SQLITE_BLOB
- Null = C.SQLITE_NULL
-)
-
-// Special data types for time values.
-const (
- UnixTime = C.DQLITE_UNIXTIME
- ISO8601 = C.DQLITE_ISO8601
- Boolean = C.DQLITE_BOOLEAN
-)
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/errors.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/errors.go
deleted file mode 100644
index c80a19cfc5..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/errors.go
+++ /dev/null
@@ -1,58 +0,0 @@
-package bindings
-
-/*
-#include <sqlite3.h>
-*/
-import "C"
-import (
- "github.com/pkg/errors"
-)
-
-// Error holds information about a SQLite error.
-type Error struct {
- Code int
- Message string
-}
-
-func (e Error) Error() string {
- if e.Message != "" {
- return e.Message
- }
- return C.GoString(C.sqlite3_errstr(C.int(e.Code)))
-}
-
-// Error codes.
-const (
- ErrError = C.SQLITE_ERROR
- ErrInternal = C.SQLITE_INTERNAL
- ErrNoMem = C.SQLITE_NOMEM
- ErrInterrupt = C.SQLITE_INTERRUPT
- ErrBusy = C.SQLITE_BUSY
- ErrIoErr = C.SQLITE_IOERR
- ErrIoErrNotLeader = C.SQLITE_IOERR_NOT_LEADER
- ErrIoErrLeadershipLost = C.SQLITE_IOERR_LEADERSHIP_LOST
-)
-
-// ErrorCode extracts a SQLite error code from a Go error.
-func ErrorCode(err error) int {
- if err, ok := errors.Cause(err).(Error); ok {
- return err.Code
- }
-
- // Return a generic error.
- return int(C.SQLITE_ERROR)
-}
-
-// Create a Go error with the code and message of the last error happened on
-// the given database.
-func lastError(db *C.sqlite3) Error {
- return Error{
- Code: int(C.sqlite3_extended_errcode(db)),
- Message: C.GoString(C.sqlite3_errmsg(db)),
- }
-}
-
-// codeToError converts a SQLite error code to a Go error.
-func codeToError(rc C.int) error {
- return Error{Code: int(rc)}
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/logger.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/logger.go
deleted file mode 100644
index 5493ff3db0..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/logger.go
+++ /dev/null
@@ -1,117 +0,0 @@
-package bindings
-
-/*
-#include <assert.h>
-#include <stdlib.h>
-
-#include <dqlite.h>
-
-// Silence warnings.
-extern int vasprintf(char **strp, const char *fmt, va_list ap);
-
-// Go land callback for xLogf.
-void dqliteLoggerLogfCb(uintptr_t handle, int level, char *msg);
-
-// Implementation of xLogf.
-static void dqliteLoggerLogf(void *ctx, int level, const char *format, va_list args) {
- uintptr_t handle;
- char *msg;
- int err;
-
- assert(ctx != NULL);
-
- handle = (uintptr_t)ctx;
-
- err = vasprintf(&msg, format, args);
- if (err < 0) {
- // Ignore errors
- return;
- }
-
- dqliteLoggerLogfCb(handle, level, (char*)msg);
-
- free(msg);
-}
-
-// Constructor.
-static dqlite_logger *dqlite__logger_create(uintptr_t handle) {
- dqlite_logger *logger = sqlite3_malloc(sizeof *logger);
-
- if (logger == NULL) {
- return NULL;
- }
-
- logger->data = (void*)handle;
- logger->emit = dqliteLoggerLogf;
-
- return logger;
-}
-*/
-import "C"
-import (
- "unsafe"
-
- "github.com/CanonicalLtd/go-dqlite/internal/logging"
-)
-
-// Logger is a Go wrapper around the associated dqlite's C type.
-type Logger C.dqlite_logger
-
-// Logging levels.
-const (
- LogDebug = C.DQLITE_LOG_DEBUG
- LogInfo = C.DQLITE_LOG_INFO
- LogWarn = C.DQLITE_LOG_WARN
- LogError = C.DQLITE_LOG_ERROR
-)
-
-// NewLogger creates a new Logger object set with the given log function.
-func NewLogger(f logging.Func) *Logger {
- // Register the logger implementation and pass its handle to
- // dqliteLoggerInit.
- handle := loggerFuncsSerial
-
- loggerFuncsIndex[handle] = f
- loggerFuncsSerial++
-
- logger := C.dqlite__logger_create(C.uintptr_t(handle))
- if logger == nil {
- panic("out of memory")
- }
-
- return (*Logger)(unsafe.Pointer(logger))
-}
-
-// Close releases all memory associated with the logger object.
-func (l *Logger) Close() {
- logger := (*C.dqlite_logger)(unsafe.Pointer(l))
- handle := (C.uintptr_t)(uintptr(logger.data))
-
- delete(loggerFuncsIndex, handle)
-
- C.sqlite3_free(unsafe.Pointer(logger))
-}
-
-// Map uintptr to logging.Func instances to avoid passing Go pointers to C.
-//
-// We do not protect this map with a lock since typically just one long-lived
-// Logger instance should be registered (except for unit tests).
-var loggerFuncsSerial C.uintptr_t = 100
-var loggerFuncsIndex = map[C.uintptr_t]logging.Func{}
-
-//export dqliteLoggerLogfCb
-func dqliteLoggerLogfCb(handle C.uintptr_t, level C.int, msg *C.char) {
- f := loggerFuncsIndex[handle]
-
- message := C.GoString(msg)
- switch level {
- case LogDebug:
- f(logging.Debug, message)
- case LogInfo:
- f(logging.Info, message)
- case LogWarn:
- f(logging.Warn, message)
- case LogError:
- f(logging.Error, message)
- }
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/server.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/server.go
deleted file mode 100644
index e173e5a3b5..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/server.go
+++ /dev/null
@@ -1,223 +0,0 @@
-package bindings
-
-/*
-#include <stdlib.h>
-#include <unistd.h>
-#include <string.h>
-#include <errno.h>
-#include <fcntl.h>
-
-#include <dqlite.h>
-#include <sqlite3.h>
-
-int dup_cloexec(int oldfd) {
- int newfd = -1;
-
- newfd = dup(oldfd);
- if (newfd < 0) {
- return -1;
- }
-
- if (fcntl(newfd, F_SETFD, FD_CLOEXEC) < 0) {
- return -1;
- }
-
- return newfd;
-}
-*/
-import "C"
-
-import (
- "fmt"
- "net"
- "os"
- "unsafe"
-
- "github.com/pkg/errors"
-)
-
-// ProtocolVersion is the latest dqlite server protocol version.
-const ProtocolVersion = uint64(C.DQLITE_PROTOCOL_VERSION)
-
-// Request types.
-const (
- RequestLeader = C.DQLITE_REQUEST_LEADER
- RequestClient = C.DQLITE_REQUEST_CLIENT
- RequestHeartbeat = C.DQLITE_REQUEST_HEARTBEAT
- RequestOpen = C.DQLITE_REQUEST_OPEN
- RequestPrepare = C.DQLITE_REQUEST_PREPARE
- RequestExec = C.DQLITE_REQUEST_EXEC
- RequestQuery = C.DQLITE_REQUEST_QUERY
- RequestFinalize = C.DQLITE_REQUEST_FINALIZE
- RequestExecSQL = C.DQLITE_REQUEST_EXEC_SQL
- RequestQuerySQL = C.DQLITE_REQUEST_QUERY_SQL
- RequestInterrupt = C.DQLITE_REQUEST_INTERRUPT
-)
-
-// Response types.
-const (
- ResponseFailure = C.DQLITE_RESPONSE_FAILURE
- ResponseServer = C.DQLITE_RESPONSE_SERVER
- ResponseWelcome = C.DQLITE_RESPONSE_WELCOME
- ResponseServers = C.DQLITE_RESPONSE_SERVERS
- ResponseDb = C.DQLITE_RESPONSE_DB
- ResponseStmt = C.DQLITE_RESPONSE_STMT
- ResponseResult = C.DQLITE_RESPONSE_RESULT
- ResponseRows = C.DQLITE_RESPONSE_ROWS
- ResponseEmpty = C.DQLITE_RESPONSE_EMPTY
-)
-
-// Server is a Go wrapper arround dqlite_server.
-type Server C.dqlite_server
-
-// Init initializes dqlite global state.
-func Init() error {
- var errmsg *C.char
-
- rc := C.dqlite_init(&errmsg)
- if rc != 0 {
- return fmt.Errorf("%s (%d)", C.GoString(errmsg), rc)
- }
- return nil
-}
-
-// NewServer creates a new Server instance.
-func NewServer(cluster *Cluster) (*Server, error) {
- var server *C.dqlite_server
-
- rc := C.dqlite_server_create((*C.dqlite_cluster)(unsafe.Pointer(cluster)), &server)
- if rc != 0 {
- err := codeToError(rc)
- return nil, errors.Wrap(err, "failed to create server object")
- }
-
- return (*Server)(unsafe.Pointer(server)), nil
-}
-
-// Close the server releasing all used resources.
-func (s *Server) Close() {
- server := (*C.dqlite_server)(unsafe.Pointer(s))
-
- C.dqlite_server_destroy(server)
-}
-
-// SetLogger sets the server logger.
-func (s *Server) SetLogger(logger *Logger) {
- server := (*C.dqlite_server)(unsafe.Pointer(s))
-
- rc := C.dqlite_server_config(server, C.DQLITE_CONFIG_LOGGER, unsafe.Pointer(logger))
- if rc != 0 {
- // Setting the logger should never fail.
- panic("failed to set logger")
- }
-}
-
-// SetVfs sets the name of the VFS to use for new connections.
-func (s *Server) SetVfs(name string) {
- server := (*C.dqlite_server)(unsafe.Pointer(s))
-
- cname := C.CString(name)
- defer C.free(unsafe.Pointer(cname))
-
- rc := C.dqlite_server_config(server, C.DQLITE_CONFIG_VFS, unsafe.Pointer(cname))
- if rc != 0 {
- // Setting the logger should never fail.
- panic("failed to set vfs")
- }
-}
-
-// SetWalReplication sets the name of the WAL replication to use for new connections.
-func (s *Server) SetWalReplication(name string) {
- server := (*C.dqlite_server)(unsafe.Pointer(s))
-
- cname := C.CString(name)
- defer C.free(unsafe.Pointer(cname))
-
- rc := C.dqlite_server_config(server, C.DQLITE_CONFIG_WAL_REPLICATION, unsafe.Pointer(cname))
- if rc != 0 {
- // Setting the logger should never fail.
- panic("failed to set WAL replication")
- }
-}
-
-// Run the server.
-//
-// After this method is called it's possible to invoke Handle().
-func (s *Server) Run() error {
- server := (*C.dqlite_server)(unsafe.Pointer(s))
-
- var errmsg *C.char
-
- rc := C.dqlite_server_run(server)
- if rc != 0 {
- return fmt.Errorf(C.GoString(errmsg))
- }
-
- return nil
-}
-
-// Ready waits for the server to be ready to handle connections.
-func (s *Server) Ready() bool {
- server := (*C.dqlite_server)(unsafe.Pointer(s))
-
- return C.dqlite_server_ready(server) == 1
-}
-
-// Handle a new connection.
-func (s *Server) Handle(conn net.Conn) error {
- server := (*C.dqlite_server)(unsafe.Pointer(s))
-
- file, err := conn.(fileConn).File()
- if err != nil {
- return err
- }
- defer file.Close()
-
- fd1 := C.int(file.Fd())
-
- // Duplicate the file descriptor, in order to prevent Go's finalizer to
- // close it.
- fd2 := C.dup_cloexec(fd1)
- if fd2 < 0 {
- return fmt.Errorf("failed to dup socket fd")
- }
-
- conn.Close()
-
- var errmsg *C.char
-
- rc := C.dqlite_server_handle(server, fd2, &errmsg)
- if rc != 0 {
- C.close(fd2)
- defer C.sqlite3_free(unsafe.Pointer(errmsg))
- if rc == C.DQLITE_STOPPED {
- return ErrServerStopped
- }
- return fmt.Errorf(C.GoString(errmsg))
- }
-
- return nil
-}
-
-// Interface that net.Conn must implement in order to extract the underlying
-// file descriptor.
-type fileConn interface {
- File() (*os.File, error)
-}
-
-// Stop the server.
-func (s *Server) Stop() error {
- server := (*C.dqlite_server)(unsafe.Pointer(s))
-
- var errmsg *C.char
-
- rc := C.dqlite_server_stop(server, &errmsg)
- if rc != 0 {
- return fmt.Errorf(C.GoString(errmsg))
- }
-
- return nil
-}
-
-// ErrServerStopped is returned by Server.Handle() is the server was stopped.
-var ErrServerStopped = fmt.Errorf("server was stopped")
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/status.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/status.go
deleted file mode 100644
index 082e68731e..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/status.go
+++ /dev/null
@@ -1,43 +0,0 @@
-package bindings
-
-/*
-#include <sqlite3.h>
-*/
-import "C"
-
-// StatusMallocCount returns the current and highest number of memory
-// allocations performed with sqlite3_malloc.
-func StatusMallocCount(reset bool) (int, int, error) {
- var current C.int
- var highest C.int
- var flag C.int
-
- if reset {
- flag = 1
- }
-
- rc := C.sqlite3_status(C.SQLITE_STATUS_MALLOC_COUNT, ¤t, &highest, flag)
- if rc != C.SQLITE_OK {
- return -1, -1, codeToError(rc)
- }
-
- return int(current), int(highest), nil
-}
-
-// StatusMemoryUsed returns the current and highest allocation size.
-func StatusMemoryUsed(reset bool) (int, int, error) {
- var current C.int
- var highest C.int
- var flag C.int
-
- if reset {
- flag = 1
- }
-
- rc := C.sqlite3_status(C.SQLITE_STATUS_MEMORY_USED, ¤t, &highest, flag)
- if rc != C.SQLITE_OK {
- return -1, -1, codeToError(rc)
- }
-
- return int(current), int(highest), nil
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/testing.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/testing.go
deleted file mode 100644
index 9c8a207e1a..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/testing.go
+++ /dev/null
@@ -1,24 +0,0 @@
-package bindings
-
-import (
- "testing"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-// AssertNoMemoryLeaks is a test helper asserting that current allocation count
-// and used memory are both zero.
-func AssertNoMemoryLeaks(t *testing.T) {
- t.Helper()
-
- current, _, err := StatusMallocCount(true)
- require.NoError(t, err)
-
- assert.Equal(t, 0, current, "malloc count leak")
-
- current, _, err = StatusMemoryUsed(true)
- require.NoError(t, err)
-
- assert.Equal(t, 0, current, "memory leak")
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/vfs.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/vfs.go
deleted file mode 100644
index bcc500ebf6..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/vfs.go
+++ /dev/null
@@ -1,106 +0,0 @@
-package bindings
-
-/*
-#include <stdlib.h>
-
-#include <sqlite3.h>
-#include <dqlite.h>
-*/
-import "C"
-import (
- "unsafe"
-)
-
-// Vfs is a Go wrapper around dqlite's in-memory VFS implementation.
-type Vfs C.sqlite3_vfs
-
-// NewVfs registers an in-memory VFS instance under the given name.
-func NewVfs(name string, logger *Logger) (*Vfs, error) {
- cname := C.CString(name)
- defer C.free(unsafe.Pointer(cname))
-
- if vfs := C.sqlite3_vfs_find(cname); vfs != nil {
- err := Error{Code: C.SQLITE_ERROR, Message: "vfs name already registered"}
- return nil, err
- }
-
- clogger := (*C.dqlite_logger)(unsafe.Pointer(logger))
-
- vfs := C.dqlite_vfs_create(cname, clogger)
- if vfs == nil {
- return nil, codeToError(C.SQLITE_NOMEM)
- }
-
- rc := C.sqlite3_vfs_register(vfs, 0)
- if rc != 0 {
- return nil, codeToError(rc)
- }
-
- return (*Vfs)(unsafe.Pointer(vfs)), nil
-}
-
-// Close unregisters this in-memory VFS instance.
-func (v *Vfs) Close() error {
- vfs := (*C.sqlite3_vfs)(unsafe.Pointer(v))
-
- rc := C.sqlite3_vfs_unregister(vfs)
- if rc != 0 {
- return codeToError(rc)
- }
-
- C.dqlite_vfs_destroy(vfs)
-
- return nil
-}
-
-// Name returns the registration name of the vfs.
-func (v *Vfs) Name() string {
- vfs := (*C.sqlite3_vfs)(unsafe.Pointer(v))
-
- return C.GoString(vfs.zName)
-}
-
-// ReadFile returns the content of the given filename.
-func (v *Vfs) ReadFile(filename string) ([]byte, error) {
- vfs := (*C.sqlite3_vfs)(unsafe.Pointer(v))
-
- cfilename := C.CString(filename)
- defer C.free(unsafe.Pointer(cfilename))
-
- var buf *C.uint8_t
- var n C.size_t
-
- rc := C.dqlite_file_read(vfs.zName, cfilename, &buf, &n)
- if rc != 0 {
- return nil, Error{Code: int(rc)}
- }
-
- content := C.GoBytes(unsafe.Pointer(buf), C.int(n))
-
- C.sqlite3_free(unsafe.Pointer(buf))
-
- return content, nil
-}
-
-// WriteFile write the content of the given filename, overriding it if it
-// exists.
-func (v *Vfs) WriteFile(filename string, bytes []byte) error {
- if len(bytes) == 0 {
- return nil
- }
-
- vfs := (*C.sqlite3_vfs)(unsafe.Pointer(v))
-
- cfilename := C.CString(filename)
- defer C.free(unsafe.Pointer(cfilename))
-
- buf := (*C.uint8_t)(unsafe.Pointer(&bytes[0]))
- n := C.size_t(len(bytes))
-
- rc := C.dqlite_file_write(vfs.zName, cfilename, buf, n)
- if rc != 0 {
- return Error{Code: int(rc & 0xff)}
- }
-
- return nil
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/wal.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/wal.go
deleted file mode 100644
index cba970d7d6..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/wal.go
+++ /dev/null
@@ -1,41 +0,0 @@
-package bindings
-
-/*
-#include <stdlib.h>
-#include <sqlite3.h>
-*/
-import "C"
-import "unsafe"
-
-// WalCheckpointMode defines all valid values for the "checkpoint mode" parameter
-// of the WalCheckpointV2 API. See https://sqlite.org/c3ref/wal_checkpoint_v2.html.
-type WalCheckpointMode int
-
-// WAL checkpoint modes
-const (
- WalCheckpointPassive = WalCheckpointMode(C.SQLITE_CHECKPOINT_PASSIVE)
- WalCheckpointFull = WalCheckpointMode(C.SQLITE_CHECKPOINT_FULL)
- WalCheckpointRestart = WalCheckpointMode(C.SQLITE_CHECKPOINT_RESTART)
- WalCheckpointTruncate = WalCheckpointMode(C.SQLITE_CHECKPOINT_TRUNCATE)
-)
-
-// WalCheckpoint triggers a WAL checkpoint on the given database attached to the
-// connection. See https://sqlite.org/c3ref/wal_checkpoint_v2.html
-func (c *Conn) WalCheckpoint(schema string, mode WalCheckpointMode) (int, int, error) {
- db := (*C.sqlite3)(unsafe.Pointer(c))
-
- var size C.int
- var ckpt C.int
- var err error
-
- // Convert to C types
- zDb := C.CString(schema)
- defer C.free(unsafe.Pointer(zDb))
-
- rc := C.sqlite3_wal_checkpoint_v2(db, zDb, C.int(mode), &size, &ckpt)
- if rc != 0 {
- return -1, -1, lastError(db)
- }
-
- return int(size), int(ckpt), err
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/wal_replication.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/wal_replication.go
deleted file mode 100644
index a3d677bc4f..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/bindings/wal_replication.go
+++ /dev/null
@@ -1,409 +0,0 @@
-package bindings
-
-/*
-#include <stdint.h>
-#include <stdlib.h>
-#include <sqlite3.h>
-#include <string.h>
-
-// WAL replication trampolines.
-int walReplicationBegin(uintptr_t handle, sqlite3 *db);
-int walReplicationAbort(uintptr_t handle, sqlite3 *db);
-int walReplicationFrames(uintptr_t handle, sqlite3 *db,
- int, int, sqlite3_wal_replication_frame*, unsigned, int);
-int walReplicationUndo(uintptr_t handle, sqlite3 *db);
-int walReplicationEnd(uintptr_t handle, sqlite3 *db);
-
-// Wal replication methods.
-static int sqlite3__wal_replication_begin(sqlite3_wal_replication *r, void *arg)
-{
- uintptr_t handle = (uintptr_t)(r->pAppData);
- sqlite3 *db = (sqlite3*)(arg);
- return walReplicationBegin(handle, db);
-}
-
-static int sqlite3__wal_replication_abort(sqlite3_wal_replication *r, void *arg)
-{
- uintptr_t handle = (uintptr_t)(r->pAppData);
- sqlite3 *db = (sqlite3*)(arg);
- return walReplicationAbort(handle, db);
-}
-
-static int sqlite3__wal_replication_frames(sqlite3_wal_replication *r, void *arg,
- int szPage, int nFrame, sqlite3_wal_replication_frame *aFrame,
- unsigned nTruncate, int isCommit)
-{
- uintptr_t handle = (uintptr_t)(r->pAppData);
- sqlite3 *db = (sqlite3*)(arg);
- return walReplicationFrames(handle, db, szPage, nFrame, aFrame, nTruncate, isCommit);
-}
-
-static int sqlite3__wal_replication_undo(sqlite3_wal_replication *r, void *arg)
-{
- uintptr_t handle = (uintptr_t)(r->pAppData);
- sqlite3 *db = (sqlite3*)(arg);
- return walReplicationUndo(handle, db);
-}
-
-static int sqlite3__wal_replication_end(sqlite3_wal_replication *r, void *arg)
-{
- uintptr_t handle = (uintptr_t)(r->pAppData);
- sqlite3 *db = (sqlite3*)(arg);
- return walReplicationEnd(handle, db);
-}
-
-// Constructor.
-static sqlite3_wal_replication *sqlite3__wal_replication_create(char *name, uintptr_t ctx){
- sqlite3_wal_replication *replication;
-
- replication = sqlite3_malloc(sizeof *replication);
- if (replication == NULL) {
- goto oom;
- }
-
- replication->iVersion = 1;
-
- // Copy the name so the Go side can just free it.
- replication->zName = sqlite3_malloc(strlen(name));
- if (replication->zName == NULL) {
- goto oom_after_replication_malloc;
- }
- strcpy((char *)replication->zName, (const char*)name);
-
- replication->pAppData = (void*)ctx;
- replication->xBegin = sqlite3__wal_replication_begin;
- replication->xAbort = sqlite3__wal_replication_abort;
- replication->xFrames = sqlite3__wal_replication_frames;
- replication->xUndo = sqlite3__wal_replication_undo;
- replication->xEnd = sqlite3__wal_replication_end;
-
- return replication;
-
-oom_after_replication_malloc:
- sqlite3_free(replication);
-
-oom:
- return NULL;
-}
-
-// Destructor.
-static void sqlite3__wal_replication_destroy(sqlite3_wal_replication *replication) {
- sqlite3_free((char *)replication->zName);
- sqlite3_free(replication);
-}
-
-*/
-import "C"
-import (
- "unsafe"
-)
-
-// WalReplication is a Go wrapper around the associated SQLite's C type.
-type WalReplication C.sqlite3_wal_replication
-
-// NewWalReplication registers a WAL replication instance under the given
-// name.
-func NewWalReplication(name string, methods WalReplicationMethods) (*WalReplication, error) {
- cname := C.CString(name)
- defer C.free(unsafe.Pointer(cname))
-
- if r := C.sqlite3_wal_replication_find(cname); r != nil {
- err := Error{Code: C.SQLITE_ERROR, Message: "WAL replication name already registered"}
- return nil, err
- }
-
- handle := walReplicationMethodsSerial
- walReplicationMethodsIndex[handle] = methods
- walReplicationMethodsSerial++
-
- replication := C.sqlite3__wal_replication_create(cname, handle)
- if replication == nil {
- return nil, codeToError(C.SQLITE_NOMEM)
- }
-
- rc := C.sqlite3_wal_replication_register(replication, 0)
- if rc != 0 {
- return nil, codeToError(rc)
- }
-
- return (*WalReplication)(unsafe.Pointer(replication)), nil
-}
-
-// Name returns the registration name of the Wal replication.
-func (r *WalReplication) Name() string {
- replication := (*C.sqlite3_wal_replication)(unsafe.Pointer(r))
-
- return C.GoString(replication.zName)
-}
-
-// Close unregisters and destroys this WAL replication instance.
-func (r *WalReplication) Close() error {
- replication := (*C.sqlite3_wal_replication)(unsafe.Pointer(r))
-
- rc := C.sqlite3_wal_replication_unregister(replication)
- if rc != 0 {
- return codeToError(rc)
- }
-
- handle := (C.uintptr_t)(uintptr(replication.pAppData))
- delete(walReplicationMethodsIndex, handle)
-
- C.sqlite3__wal_replication_destroy(replication)
-
- return nil
-}
-
-// WalReplicationLeader switches the SQLite connection to leader WAL
-// replication mode.
-func (c *Conn) WalReplicationLeader(name string) error {
- db := (*C.sqlite3)(unsafe.Pointer(c))
-
- cname := C.CString(name)
- defer C.free(unsafe.Pointer(cname))
-
- rc := C.sqlite3_wal_replication_leader(db, walReplicationSchema, cname, unsafe.Pointer(db))
- if rc != C.SQLITE_OK {
- return lastError(db)
- }
-
- return nil
-}
-
-// WalReplicationFollower switches the given SQLite connection to follower WAL
-// replication mode. In this mode no regular operation is possible, and the
-// connection should be driven with the WalReplicationFrames, and
-// WalReplicationUndo APIs.
-func (c *Conn) WalReplicationFollower() error {
- db := (*C.sqlite3)(unsafe.Pointer(c))
-
- rc := C.sqlite3_wal_replication_follower(db, walReplicationSchema)
- if rc != C.SQLITE_OK {
- return lastError(db)
- }
-
- return nil
-}
-
-// WalReplicationFrames writes the given batch of frames to the write-ahead log
-// linked to the given connection.
-//
-// This method must be called with a "follower" connection, meant to replicate
-// the "leader" one.
-func (c *Conn) WalReplicationFrames(info WalReplicationFrameInfo) error {
- db := (*C.sqlite3)(unsafe.Pointer(c))
-
- rc := C.sqlite3_wal_replication_frames(
- db, walReplicationSchema, info.isBegin, info.szPage, info.nFrame,
- info.aPgno, info.aPage, info.nTruncate, info.isCommit)
- if rc != C.SQLITE_OK {
- return lastError(db)
- }
-
- return nil
-}
-
-// WalReplicationUndo rollbacks a write transaction in the given sqlite
-// connection. This should be called with a "follower" connection, meant to
-// replicate the "leader" one.
-func (c *Conn) WalReplicationUndo() error {
- db := (*C.sqlite3)(unsafe.Pointer(c))
-
- rc := C.sqlite3_wal_replication_undo(db, walReplicationSchema)
- if rc != C.SQLITE_OK {
- return lastError(db)
- }
- return nil
-}
-
-// WalReplicationMethods implements the interface required by the various hooks
-// of sqlite3_wal_replication.
-type WalReplicationMethods interface {
- // Begin a new write transaction. The implementation should check
- // that the database is eligible for starting a replicated write
- // transaction (e.g. this node is the leader), and perform internal
- // state changes as appropriate.
- Begin(*Conn) int
-
- // Abort a write transaction. The implementation should clear any
- // state previously set by the Begin hook.
- Abort(*Conn) int
-
- // Write new frames to the write-ahead log. The implementation should
- // broadcast this write to other nodes and wait for a quorum.
- Frames(*Conn, WalReplicationFrameList) int
-
- // Undo a write transaction. The implementation should broadcast
- // this event to other nodes and wait for a quorum. The return code
- // is currently ignored by SQLite.
- Undo(*Conn) int
-
- // End a write transaction. The implementation should update its
- // internal state and be ready for a new transaction.
- End(*Conn) int
-}
-
-// PageNumber identifies a single database or WAL page.
-type PageNumber C.unsigned
-
-// FrameNumber identifies a single frame in the WAL.
-type FrameNumber C.unsigned
-
-// WalReplicationFrameList holds information about a single batch of WAL frames
-// that are being dispatched for replication by a leader connection.
-//
-// They map to the parameters of the sqlite3_wal_replication.xFrames API
-type WalReplicationFrameList struct {
- szPage C.int
- nFrame C.int
- aFrame *C.sqlite3_wal_replication_frame
- nTruncate C.uint
- isCommit C.int
-}
-
-// PageSize returns the page size of this batch of WAL frames.
-func (l *WalReplicationFrameList) PageSize() int {
- return int(l.szPage)
-}
-
-// Len returns the number of WAL frames in this batch.
-func (l *WalReplicationFrameList) Len() int {
- return int(l.nFrame)
-}
-
-// Truncate returns the size of the database in pages after this batch of WAL
-// frames is applied.
-func (l *WalReplicationFrameList) Truncate() uint {
- return uint(l.nTruncate)
-}
-
-// Frame returns information about the i'th frame in the batch.
-func (l *WalReplicationFrameList) Frame(i int) (unsafe.Pointer, PageNumber, FrameNumber) {
- pFrame := (*C.sqlite3_wal_replication_frame)(unsafe.Pointer(
- uintptr(unsafe.Pointer(l.aFrame)) +
- unsafe.Sizeof(*l.aFrame)*uintptr(i),
- ))
- return pFrame.pBuf, PageNumber(pFrame.pgno), FrameNumber(pFrame.iPrev)
-}
-
-// IsCommit returns whether this batch of WAL frames concludes a transaction.
-func (l *WalReplicationFrameList) IsCommit() bool {
- return l.isCommit > 0
-}
-
-// WalReplicationFrameInfo information about a single batch of WAL frames that
-// are being replicated by a follower connection.
-type WalReplicationFrameInfo struct {
- isBegin C.int
- szPage C.int
- nFrame C.int
- aPgno *C.unsigned
- aPage unsafe.Pointer
- nTruncate C.uint
- isCommit C.int
-}
-
-// IsBegin sets the C isBegin parameter for sqlite3_wal_replication_frames.
-func (i *WalReplicationFrameInfo) IsBegin(flag bool) {
- if flag {
- i.isBegin = C.int(1)
- } else {
- i.isBegin = C.int(0)
- }
-}
-
-// PageSize sets the C szPage parameter for sqlite3_wal_replication_frames.
-func (i *WalReplicationFrameInfo) PageSize(size int) {
- i.szPage = C.int(size)
-}
-
-// Len sets the C nFrame parameter for sqlite3_wal_replication_frames.
-func (i *WalReplicationFrameInfo) Len(n int) {
- i.nFrame = C.int(n)
-}
-
-// Pages sets the C aPgno and aPage parameters for sqlite3_wal_replication_frames.
-func (i *WalReplicationFrameInfo) Pages(numbers []PageNumber, data unsafe.Pointer) {
- i.aPgno = (*C.unsigned)(&numbers[0])
- i.aPage = data
-}
-
-// Truncate sets the nTruncate parameter for sqlite3_wal_replication_frames.
-func (i *WalReplicationFrameInfo) Truncate(truncate uint) {
- i.nTruncate = C.unsigned(truncate)
-}
-
-// IsCommit sets the isCommit parameter for sqlite3_wal_replication_frames.
-func (i *WalReplicationFrameInfo) IsCommit(flag bool) {
- if flag {
- i.isCommit = C.int(1)
- } else {
- i.isCommit = C.int(0)
- }
-}
-
-func (i *WalReplicationFrameInfo) IsCommitGet() bool {
- return i.isCommit > 0
-}
-
-//export walReplicationBegin
-func walReplicationBegin(handle C.uintptr_t, db *C.sqlite3) C.int {
- methods := walReplicationMethodsIndex[handle]
-
- return C.int(methods.Begin((*Conn)(unsafe.Pointer(db))))
-}
-
-//export walReplicationAbort
-func walReplicationAbort(handle C.uintptr_t, db *C.sqlite3) C.int {
- methods := walReplicationMethodsIndex[handle]
- return C.int(methods.Abort((*Conn)(unsafe.Pointer(db))))
-}
-
-//export walReplicationFrames
-func walReplicationFrames(
- handle C.uintptr_t,
- db *C.sqlite3,
- szPage C.int,
- nFrame C.int,
- aFrame *C.sqlite3_wal_replication_frame,
- nTruncate C.uint,
- isCommit C.int,
-) C.int {
- methods := walReplicationMethodsIndex[handle]
-
- list := WalReplicationFrameList{
- szPage: szPage,
- nFrame: nFrame,
- aFrame: aFrame,
- nTruncate: nTruncate,
- isCommit: isCommit,
- }
-
- return C.int(methods.Frames((*Conn)(unsafe.Pointer(db)), list))
-}
-
-//export walReplicationUndo
-func walReplicationUndo(handle C.uintptr_t, db *C.sqlite3) C.int {
- methods := walReplicationMethodsIndex[handle]
-
- return C.int(methods.Undo((*Conn)(unsafe.Pointer(db))))
-}
-
-//export walReplicationEnd
-func walReplicationEnd(handle C.uintptr_t, db *C.sqlite3) C.int {
- methods := walReplicationMethodsIndex[handle]
-
- return C.int(methods.End((*Conn)(unsafe.Pointer(db))))
-}
-
-// Map uintptr to WalReplicationMethods instances to avoid passing Go pointers
-// to C.
-//
-// We do not protect this map with a lock since typically just one long-lived
-// WalReplication instance should be registered (except for unit tests).
-var walReplicationMethodsSerial C.uintptr_t = 100
-var walReplicationMethodsIndex = map[C.uintptr_t]WalReplicationMethods{}
-
-// Hard-coded main schema name.
-//
-// TODO: support replicating also attached databases.
-var walReplicationSchema = C.CString("main")
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/buffer.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/buffer.go
deleted file mode 100644
index 00efe94ffe..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/buffer.go
+++ /dev/null
@@ -1,11 +0,0 @@
-package client
-
-// Buffer for reading responses or writing requests.
-type buffer struct {
- Bytes []byte
- Offset int
-}
-
-func (b *buffer) Advance(amount int) {
- b.Offset += amount
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/client.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/client.go
deleted file mode 100644
index 08c3465caf..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/client.go
+++ /dev/null
@@ -1,322 +0,0 @@
-package client
-
-import (
- "context"
- "encoding/binary"
- "io"
- "net"
- "sync"
- "time"
-
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
- "github.com/CanonicalLtd/go-dqlite/internal/logging"
- "github.com/pkg/errors"
-)
-
-// Client connecting to a dqlite server and speaking the dqlite wire protocol.
-type Client struct {
- log logging.Func // Logging function.
- address string // Address of the connected dqlite server.
- store ServerStore // Update this store upon heartbeats.
- conn net.Conn // Underlying network connection.
- heartbeatTimeout time.Duration // Heartbeat timeout reported at registration.
- contextTimeout time.Duration // Default context timeout.
- closeCh chan struct{} // Stops the heartbeat when the connection gets closed
- mu sync.Mutex // Serialize requests
-}
-
-func newClient(conn net.Conn, address string, store ServerStore, log logging.Func) *Client {
- //logger.With(zap.String("target", address)
- client := &Client{
- conn: conn,
- address: address,
- store: store,
- log: log,
- closeCh: make(chan struct{}),
- contextTimeout: 5 * time.Second,
- }
-
- return client
-}
-
-// SetContextTimeout sets the default context timeout when no deadline is
-// provided.
-func (c *Client) SetContextTimeout(timeout time.Duration) {
- c.contextTimeout = timeout
-}
-
-// Call invokes a dqlite RPC, sending a request message and receiving a
-// response message.
-func (c *Client) Call(ctx context.Context, request, response *Message) error {
- // We need to take a lock since the dqlite server currently does not
- // support concurrent requests.
- c.mu.Lock()
- defer c.mu.Unlock()
-
- // Honor the ctx deadline, if present, or use a default.
- deadline, ok := ctx.Deadline()
- if !ok {
- deadline = time.Now().Add(c.contextTimeout)
- }
-
- c.conn.SetDeadline(deadline)
-
- if err := c.send(request); err != nil {
- return errors.Wrap(err, "failed to send request")
- }
-
- if err := c.recv(response); err != nil {
- return errors.Wrap(err, "failed to receive response")
- }
-
- return nil
-}
-
-// More is used when a request maps to multiple responses.
-func (c *Client) More(ctx context.Context, response *Message) error {
- return c.recv(response)
-}
-
-// Interrupt sends an interrupt request and awaits for the server's empty
-// response.
-func (c *Client) Interrupt(ctx context.Context, request *Message, response *Message) error {
- // We need to take a lock since the dqlite server currently does not
- // support concurrent requests.
- c.mu.Lock()
- defer c.mu.Unlock()
-
- // Honor the ctx deadline, if present, or use a default.
- deadline, ok := ctx.Deadline()
- if !ok {
- deadline = time.Now().Add(2 * time.Second)
- }
- c.conn.SetDeadline(deadline)
-
- defer request.Reset()
-
- EncodeInterrupt(request, 0)
-
- if err := c.send(request); err != nil {
- return errors.Wrap(err, "failed to send interrupt request")
- }
-
- for {
- if err := c.recv(response); err != nil {
- response.Reset()
- return errors.Wrap(err, "failed to receive response")
- }
-
- mtype, _ := response.getHeader()
- response.Reset()
-
- if mtype == bindings.ResponseEmpty {
- break
- }
- }
-
- return nil
-}
-
-// Close the client connection.
-func (c *Client) Close() error {
- c.log(bindings.LogInfo, "closing client")
-
- close(c.closeCh)
-
- return c.conn.Close()
-}
-
-func (c *Client) send(req *Message) error {
- if err := c.sendHeader(req); err != nil {
- return errors.Wrap(err, "failed to send header")
- }
-
- if err := c.sendBody(req); err != nil {
- return errors.Wrap(err, "failed to send body")
- }
-
- return nil
-}
-
-func (c *Client) sendHeader(req *Message) error {
- n, err := c.conn.Write(req.header[:])
- if err != nil {
- return errors.Wrap(err, "failed to send header")
- }
-
- if n != messageHeaderSize {
- return errors.Wrap(io.ErrShortWrite, "failed to send header")
- }
-
- return nil
-}
-
-func (c *Client) sendBody(req *Message) error {
- buf := req.body1.Bytes[:req.body1.Offset]
- n, err := c.conn.Write(buf)
- if err != nil {
- return errors.Wrap(err, "failed to send static body")
- }
-
- if n != len(buf) {
- return errors.Wrap(io.ErrShortWrite, "failed to write body")
- }
-
- if req.body2.Bytes == nil {
- return nil
- }
-
- buf = req.body2.Bytes[:req.body2.Offset]
- n, err = c.conn.Write(buf)
- if err != nil {
- return errors.Wrap(err, "failed to send dynamic body")
- }
-
- if n != len(buf) {
- return errors.Wrap(io.ErrShortWrite, "failed to write body")
- }
-
- return nil
-}
-
-func (c *Client) recv(res *Message) error {
- if err := c.recvHeader(res); err != nil {
- return errors.Wrap(err, "failed to receive header")
- }
-
- if err := c.recvBody(res); err != nil {
- return errors.Wrap(err, "failed to receive body")
- }
-
- return nil
-}
-
-func (c *Client) recvHeader(res *Message) error {
- if err := c.recvPeek(res.header); err != nil {
- return errors.Wrap(err, "failed to receive header")
- }
-
- res.words = binary.LittleEndian.Uint32(res.header[0:])
- res.mtype = res.header[4]
- res.flags = res.header[5]
- res.extra = binary.LittleEndian.Uint16(res.header[6:])
-
- return nil
-}
-
-func (c *Client) recvBody(res *Message) error {
- n := int(res.words) * messageWordSize
- n1 := n
- n2 := 0
-
- if n1 > len(res.body1.Bytes) {
- // We need to allocate the dynamic buffer.
- n1 = len(res.body1.Bytes)
- n2 = n - n1
- }
-
- buf := res.body1.Bytes[:n1]
-
- if err := c.recvPeek(buf); err != nil {
- return errors.Wrap(err, "failed to read body")
- }
-
- if n2 > 0 {
- res.body2.Bytes = make([]byte, n2)
- res.body2.Offset = 0
- buf = res.body2.Bytes
- if err := c.recvPeek(buf); err != nil {
- return errors.Wrap(err, "failed to read body")
- }
- }
-
- return nil
-}
-
-// Read until buf is full.
-func (c *Client) recvPeek(buf []byte) error {
- for offset := 0; offset < len(buf); {
- n, err := c.recvFill(buf[offset:])
- if err != nil {
- return err
- }
- offset += n
- }
-
- return nil
-}
-
-// Try to fill buf, but perform at most one read.
-func (c *Client) recvFill(buf []byte) (int, error) {
- // Read new data: try a limited number of times.
- //
- // This technique is copied from bufio.Reader.
- for i := messageMaxConsecutiveEmptyReads; i > 0; i-- {
- n, err := c.conn.Read(buf)
- if n < 0 {
- panic(errNegativeRead)
- }
- if err != nil {
- return -1, err
- }
- if n > 0 {
- return n, nil
- }
- }
- return -1, io.ErrNoProgress
-}
-
-func (c *Client) heartbeat() {
- request := Message{}
- request.Init(16)
- response := Message{}
- response.Init(512)
-
- for {
- delay := c.heartbeatTimeout / 3
-
- //c.logger.Debug("sending heartbeat", zap.Duration("delay", delay))
- time.Sleep(delay)
-
- // Check if we've been closed.
- select {
- case <-c.closeCh:
- return
- default:
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), time.Second)
-
- EncodeHeartbeat(&request, uint64(time.Now().Unix()))
-
- err := c.Call(ctx, &request, &response)
-
- // We bail out upon failures.
- //
- // TODO: make the client survive temporary disconnections.
- if err != nil {
- cancel()
- //c.logger.Error("heartbeat failed", zap.Error(err))
- return
- }
-
- //addresses, err := DecodeServers(&response)
- _, err = DecodeServers(&response)
- if err != nil {
- cancel()
- //c.logger.Error("invalid heartbeat response", zap.Error(err))
- return
- }
-
- // if err := c.store.Set(ctx, addresses); err != nil {
- // cancel()
- // c.logger.Error("failed to update servers", zap.Error(err))
- // return
- // }
-
- cancel()
-
- request.Reset()
- response.Reset()
- }
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/config.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/config.go
deleted file mode 100644
index 6d349f7bf0..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/config.go
+++ /dev/null
@@ -1,14 +0,0 @@
-package client
-
-import (
- "time"
-
- "github.com/Rican7/retry/strategy"
-)
-
-// Config holds various configuration parameters for a dqlite client.
-type Config struct {
- Dial DialFunc // Network dialer.
- AttemptTimeout time.Duration // Timeout for each individual Dial attempt.
- RetryStrategies []strategy.Strategy // Strategies used for retrying to connect to a leader.
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/connector.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/connector.go
deleted file mode 100644
index ae3df0e772..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/connector.go
+++ /dev/null
@@ -1,231 +0,0 @@
-package client
-
-import (
- "context"
- "encoding/binary"
- "fmt"
- "io"
- "time"
-
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
- "github.com/CanonicalLtd/go-dqlite/internal/logging"
- "github.com/Rican7/retry"
- "github.com/pkg/errors"
-)
-
-// Connector is in charge of creating a dqlite SQL client connected to the
-// current leader of a cluster.
-type Connector struct {
- id uint64 // Client ID to use when registering against the server.
- store ServerStore // Used to get and update current cluster servers.
- config Config // Connection parameters.
- log logging.Func // Logging function.
- protocol []byte // Protocol version
-}
-
-// NewConnector returns a new connector that can be used by a dqlite driver to
-// create new clients connected to a leader dqlite server.
-func NewConnector(id uint64, store ServerStore, config Config, log logging.Func) *Connector {
- connector := &Connector{
- id: id,
- store: store,
- config: config,
- log: log,
- protocol: make([]byte, 8),
- }
-
- // Latest protocol version.
- binary.LittleEndian.PutUint64(
- connector.protocol,
- bindings.ProtocolVersion,
- )
-
- return connector
-}
-
-// Connect finds the leader server and returns a connection to it.
-//
-// If the connector is stopped before a leader is found, nil is returned.
-func (c *Connector) Connect(ctx context.Context) (*Client, error) {
- var client *Client
-
- // The retry strategy should be configured to retry indefinitely, until
- // the given context is done.
- err := retry.Retry(func(attempt uint) error {
- log := func(l logging.Level, format string, a ...interface{}) {
- format += fmt.Sprintf(" attempt=%d", attempt)
- c.log(l, fmt.Sprintf(format, a...))
- }
-
- select {
- case <-ctx.Done():
- // Stop retrying
- return nil
- default:
- }
-
- var err error
- client, err = c.connectAttemptAll(ctx, log)
- if err != nil {
- log(logging.Debug, "connection failed err=%v", err)
- return err
- }
-
- return nil
- }, c.config.RetryStrategies...)
-
- if err != nil {
- // The retry strategy should never give up until success or
- // context expiration.
- panic("connect retry aborted unexpectedly")
- }
-
- if ctx.Err() != nil {
- return nil, ErrNoAvailableLeader
- }
-
- return client, nil
-}
-
-// Make a single attempt to establish a connection to the leader server trying
-// all addresses available in the store.
-func (c *Connector) connectAttemptAll(ctx context.Context, log logging.Func) (*Client, error) {
- servers, err := c.store.Get(ctx)
- if err != nil {
- return nil, errors.Wrap(err, "failed to get cluster servers")
- }
-
- // Make an attempt for each address until we find the leader.
- for _, server := range servers {
- log := func(l logging.Level, format string, a ...interface{}) {
- format += fmt.Sprintf(" address=%s", server.Address)
- log(l, fmt.Sprintf(format, a...))
- }
-
- ctx, cancel := context.WithTimeout(ctx, c.config.AttemptTimeout)
- defer cancel()
-
- conn, leader, err := c.connectAttemptOne(ctx, server.Address)
- if err != nil {
- // This server is unavailable, try with the next target.
- log(logging.Debug, "server connection failed err=%v", err)
- continue
- }
- if conn != nil {
- // We found the leader
- log(logging.Info, "connected")
- return conn, nil
- }
- if leader == "" {
- // This server does not know who the current leader is,
- // try with the next target.
- continue
- }
-
- // If we get here, it means this server reported that another
- // server is the leader, let's close the connection to this
- // server and try with the suggested one.
- //logger = logger.With(zap.String("leader", leader))
- conn, leader, err = c.connectAttemptOne(ctx, leader)
- if err != nil {
- // The leader reported by the previous server is
- // unavailable, try with the next target.
- //logger.Info("leader server connection failed", zap.String("err", err.Error()))
- continue
- }
- if conn == nil {
- // The leader reported by the target server does not consider itself
- // the leader, try with the next target.
- //logger.Info("reported leader server is not the leader")
- continue
- }
- log(logging.Info, "connected")
- return conn, nil
- }
-
- return nil, ErrNoAvailableLeader
-}
-
-// Connect to the given dqlite server and check if it's the leader.
-//
-// Return values:
-//
-// - Any failure is hit: -> nil, "", err
-// - Target not leader and no leader known: -> nil, "", nil
-// - Target not leader and leader known: -> nil, leader, nil
-// - Target is the leader: -> server, "", nil
-//
-func (c *Connector) connectAttemptOne(ctx context.Context, address string) (*Client, string, error) {
- // Establish the connection.
- conn, err := c.config.Dial(ctx, address)
- if err != nil {
- return nil, "", errors.Wrap(err, "failed to establish network connection")
- }
-
- // Perform the protocol handshake.
- n, err := conn.Write(c.protocol)
- if err != nil {
- conn.Close()
- return nil, "", errors.Wrap(err, "failed to send handshake")
- }
- if n != 8 {
- conn.Close()
- return nil, "", errors.Wrap(io.ErrShortWrite, "failed to send handshake")
- }
-
- client := newClient(conn, address, c.store, c.log)
-
- // Send the initial Leader request.
- request := Message{}
- request.Init(16)
- response := Message{}
- response.Init(512)
-
- EncodeLeader(&request)
-
- if err := client.Call(ctx, &request, &response); err != nil {
- client.Close()
- return nil, "", errors.Wrap(err, "failed to send Leader request")
- }
-
- leader, err := DecodeServer(&response)
- if err != nil {
- client.Close()
- return nil, "", errors.Wrap(err, "failed to parse Server response")
- }
-
- switch leader {
- case "":
- // Currently this server does not know about any leader.
- client.Close()
- return nil, "", nil
- case address:
- // This server is the leader, register ourselves and return.
- request.Reset()
- response.Reset()
-
- EncodeClient(&request, c.id)
-
- if err := client.Call(ctx, &request, &response); err != nil {
- client.Close()
- return nil, "", errors.Wrap(err, "failed to send Client request")
- }
-
- heartbeatTimeout, err := DecodeWelcome(&response)
- if err != nil {
- client.Close()
- return nil, "", errors.Wrap(err, "failed to parse Welcome response")
- }
-
- client.heartbeatTimeout = time.Duration(heartbeatTimeout) * time.Millisecond
-
- // TODO: enable heartbeat
- //go client.heartbeat()
-
- return client, "", nil
- default:
- // This server claims to know who the current leader is.
- client.Close()
- return nil, leader, nil
- }
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/dial.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/dial.go
deleted file mode 100644
index 7b58aca0a7..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/dial.go
+++ /dev/null
@@ -1,23 +0,0 @@
-package client
-
-import (
- "context"
- "net"
-)
-
-// DialFunc is a function that can be used to establish a network connection.
-type DialFunc func(context.Context, string) (net.Conn, error)
-
-// TCPDial is a dial function using plain TCP to establish the network
-// connection.
-func TCPDial(ctx context.Context, address string) (net.Conn, error) {
- dialer := net.Dialer{}
- return dialer.DialContext(ctx, "tcp", address)
-}
-
-// UnixDial is a dial function using Unix sockets to establish the network
-// connection.
-func UnixDial(ctx context.Context, address string) (net.Conn, error) {
- dialer := net.Dialer{}
- return dialer.DialContext(ctx, "unix", address)
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/errors.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/errors.go
deleted file mode 100644
index 88665a89c0..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/errors.go
+++ /dev/null
@@ -1,29 +0,0 @@
-package client
-
-import (
- "fmt"
-)
-
-// Client errors.
-var (
- ErrNoAvailableLeader = fmt.Errorf("no available dqlite leader server found")
- errStop = fmt.Errorf("connector was stopped")
- errStaleLeader = fmt.Errorf("server has lost leadership")
- errNotClustered = fmt.Errorf("server is not clustered")
- errNegativeRead = fmt.Errorf("reader returned negative count from Read")
- errMessageEOF = fmt.Errorf("message eof")
-)
-
-// ErrRequest is returned in case of request failure.
-type ErrRequest struct {
- Code uint64
- Description string
-}
-
-func (e ErrRequest) Error() string {
- return fmt.Sprintf("%s (%d)", e.Description, e.Code)
-}
-
-// ErrRowsPart is returned when the first batch of a multi-response result
-// batch is done.
-var ErrRowsPart = fmt.Errorf("not all rows were returned in this response")
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/message.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/message.go
deleted file mode 100644
index c5dbc315aa..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/message.go
+++ /dev/null
@@ -1,585 +0,0 @@
-package client
-
-import (
- "bytes"
- "database/sql/driver"
- "encoding/binary"
- "io"
- "math"
- "strings"
- "time"
-
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
-)
-
-// NamedValues is a type alias of a slice of driver.NamedValue. It's used by
-// schema.sh to generate encoding logic for statement parameters.
-type NamedValues = []driver.NamedValue
-
-// Servers is a type alias of a slice of bindings.ServerInfo. It's used by
-// schema.sh to generate decoding logic for the heartbeat response.
-type Servers []bindings.ServerInfo
-
-// Message holds data about a single request or response.
-type Message struct {
- words uint32
- mtype uint8
- flags uint8
- extra uint16
- header []byte // Statically allocated header buffer
- body1 buffer // Statically allocated body data, using bytes
- body2 buffer // Dynamically allocated body data
-}
-
-// Init initializes the message using the given size of the statically
-// allocated buffer (i.e. a buffer which is re-used across requests or
-// responses encoded or decoded using this message object).
-func (m *Message) Init(staticSize int) {
- if (staticSize % messageWordSize) != 0 {
- panic("static size is not aligned to word boundary")
- }
- m.header = make([]byte, messageHeaderSize)
- m.body1.Bytes = make([]byte, staticSize)
- m.Reset()
-}
-
-// Reset the state of the message so it can be used to encode or decode again.
-func (m *Message) Reset() {
- m.words = 0
- m.mtype = 0
- m.flags = 0
- m.extra = 0
- for i := 0; i < messageHeaderSize; i++ {
- m.header[i] = 0
- }
- m.body1.Offset = 0
- m.body2.Bytes = nil
- m.body2.Offset = 0
-}
-
-// Append a byte slice to the message.
-func (m *Message) putBlob(v []byte) {
- size := len(v)
- pad := 0
- if (size % messageWordSize) != 0 {
- // Account for padding
- pad = messageWordSize - (size % messageWordSize)
- size += pad
- }
-
- b := m.bufferForPut(size)
- defer b.Advance(size)
-
- // Copy the bytes into the buffer.
- offset := b.Offset
- copy(b.Bytes[offset:], v)
- offset += len(v)
-
- // Add padding
- for i := 0; i < pad; i++ {
- b.Bytes[offset] = 0
- offset++
- }
-}
-
-// Append a string to the message.
-func (m *Message) putString(v string) {
- size := len(v) + 1
- pad := 0
- if (size % messageWordSize) != 0 {
- // Account for padding
- pad = messageWordSize - (size % messageWordSize)
- size += pad
- }
-
- b := m.bufferForPut(size)
- defer b.Advance(size)
-
- // Copy the string bytes into the buffer.
- offset := b.Offset
- copy(b.Bytes[offset:], v)
- offset += len(v)
-
- // Add a nul byte
- b.Bytes[offset] = 0
- offset++
-
- // Add padding
- for i := 0; i < pad; i++ {
- b.Bytes[offset] = 0
- offset++
- }
-}
-
-// Append a byte to the message.
-func (m *Message) putUint8(v uint8) {
- b := m.bufferForPut(1)
- defer b.Advance(1)
-
- b.Bytes[b.Offset] = v
-}
-
-// Append a 2-byte word to the message.
-func (m *Message) putUint16(v uint16) {
- b := m.bufferForPut(2)
- defer b.Advance(2)
-
- binary.LittleEndian.PutUint16(b.Bytes[b.Offset:], v)
-}
-
-// Append a 4-byte word to the message.
-func (m *Message) putUint32(v uint32) {
- b := m.bufferForPut(4)
- defer b.Advance(4)
-
- binary.LittleEndian.PutUint32(b.Bytes[b.Offset:], v)
-}
-
-// Append an 8-byte word to the message.
-func (m *Message) putUint64(v uint64) {
- b := m.bufferForPut(8)
- defer b.Advance(8)
-
- binary.LittleEndian.PutUint64(b.Bytes[b.Offset:], v)
-}
-
-// Append a signed 8-byte word to the message.
-func (m *Message) putInt64(v int64) {
- b := m.bufferForPut(8)
- defer b.Advance(8)
-
- binary.LittleEndian.PutUint64(b.Bytes[b.Offset:], uint64(v))
-}
-
-// Append a floating point number to the message.
-func (m *Message) putFloat64(v float64) {
- b := m.bufferForPut(8)
- defer b.Advance(8)
-
- binary.LittleEndian.PutUint64(b.Bytes[b.Offset:], math.Float64bits(v))
-}
-
-// Encode the given driver values as binding parameters.
-func (m *Message) putNamedValues(values NamedValues) {
- n := uint8(len(values)) // N of params
- if n == 0 {
- return
- }
-
- m.putUint8(n)
-
- for i := range values {
- if values[i].Ordinal != i+1 {
- panic("unexpected ordinal")
- }
-
- switch values[i].Value.(type) {
- case int64:
- m.putUint8(bindings.Integer)
- case float64:
- m.putUint8(bindings.Float)
- case bool:
- m.putUint8(bindings.Boolean)
- case []byte:
- m.putUint8(bindings.Blob)
- case string:
- m.putUint8(bindings.Text)
- case nil:
- m.putUint8(bindings.Null)
- case time.Time:
- m.putUint8(bindings.ISO8601)
- default:
- panic("unsupported value type")
- }
- }
-
- b := m.bufferForPut(1)
-
- if trailing := b.Offset % messageWordSize; trailing != 0 {
- // Skip padding bytes
- b.Advance(messageWordSize - trailing)
- }
-
- for i := range values {
- switch v := values[i].Value.(type) {
- case int64:
- m.putInt64(v)
- case float64:
- m.putFloat64(v)
- case bool:
- if v {
- m.putUint64(1)
- } else {
- m.putUint64(0)
- }
- case []byte:
- m.putBlob(v)
- case string:
- m.putString(v)
- case nil:
- m.putInt64(0)
- case time.Time:
- timestamp := v.Format(iso8601Formats[0])
- m.putString(timestamp)
- default:
- panic("unsupported value type")
- }
- }
-
-}
-
-// Finalize the message by setting the message type and the number
-// of words in the body (calculated from the body size).
-func (m *Message) putHeader(mtype uint8) {
- if m.body1.Offset <= 0 {
- panic("static offset is not positive")
- }
-
- if (m.body1.Offset % messageWordSize) != 0 {
- panic("static body is not aligned")
- }
-
- m.mtype = mtype
- m.flags = 0
- m.extra = 0
-
- m.words = uint32(m.body1.Offset) / messageWordSize
-
- if m.body2.Bytes == nil {
- m.finalize()
- return
- }
-
- if m.body2.Offset <= 0 {
- panic("dynamic offset is not positive")
- }
-
- if (m.body2.Offset % messageWordSize) != 0 {
- panic("dynamic body is not aligned")
- }
-
- m.words += uint32(m.body2.Offset) / messageWordSize
-
- m.finalize()
-}
-
-func (m *Message) finalize() {
- if m.words == 0 {
- panic("empty message body")
- }
-
- binary.LittleEndian.PutUint32(m.header[0:], m.words)
- m.header[4] = m.mtype
- m.header[5] = m.flags
- binary.LittleEndian.PutUint16(m.header[6:], m.extra)
-}
-
-func (m *Message) bufferForPut(size int) *buffer {
- if m.body2.Bytes != nil {
- if (m.body2.Offset + size) > len(m.body2.Bytes) {
- // Grow body2.
- //
- // TODO: find a good grow strategy.
- bytes := make([]byte, m.body2.Offset+size)
- copy(bytes, m.body2.Bytes)
- m.body2.Bytes = bytes
- }
-
- return &m.body2
- }
-
- if (m.body1.Offset + size) > len(m.body1.Bytes) {
- m.body2.Bytes = make([]byte, size)
- m.body2.Offset = 0
-
- return &m.body2
- }
-
- return &m.body1
-}
-
-// Return the message type and its flags.
-func (m *Message) getHeader() (uint8, uint8) {
- return m.mtype, m.flags
-}
-
-// Read a string from the message body.
-func (m *Message) getString() string {
- b := m.bufferForGet()
-
- index := bytes.IndexByte(b.Bytes[b.Offset:], 0)
- if index == -1 {
- // Check if the string overflows in the dynamic buffer.
- if b == &m.body1 && m.body2.Bytes != nil {
- // Assert that this is the first read of the dynamic buffer.
- if m.body2.Offset != 0 {
- panic("static buffer read after dynamic buffer one")
- }
- index = bytes.IndexByte(m.body2.Bytes[0:], 0)
- if index != -1 {
- // We found the trailing part of the string.
- data := b.Bytes[b.Offset:]
- data = append(data, m.body2.Bytes[0:index]...)
-
- index++
-
- if trailing := index % messageWordSize; trailing != 0 {
- // Account for padding, moving index to the next word boundary.
- index += messageWordSize - trailing
- }
-
- m.body1.Offset = len(m.body1.Bytes)
- m.body2.Advance(index)
-
- return string(data)
- }
- }
- panic("no string found")
- }
- s := string(b.Bytes[b.Offset : b.Offset+index])
-
- index++
-
- if trailing := index % messageWordSize; trailing != 0 {
- // Account for padding, moving index to the next word boundary.
- index += messageWordSize - trailing
- }
-
- b.Advance(index)
-
- return s
-}
-
-// Read a byte from the message body.
-func (m *Message) getUint8() uint8 {
- b := m.bufferForGet()
- defer b.Advance(1)
-
- return b.Bytes[b.Offset]
-}
-
-// Read a 2-byte word from the message body.
-func (m *Message) getUint16() uint16 {
- b := m.bufferForGet()
- defer b.Advance(2)
-
- return binary.LittleEndian.Uint16(b.Bytes[b.Offset:])
-}
-
-// Read a 4-byte word from the message body.
-func (m *Message) getUint32() uint32 {
- b := m.bufferForGet()
- defer b.Advance(4)
-
- return binary.LittleEndian.Uint32(b.Bytes[b.Offset:])
-}
-
-// Read reads an 8-byte word from the message body.
-func (m *Message) getUint64() uint64 {
- b := m.bufferForGet()
- defer b.Advance(8)
-
- return binary.LittleEndian.Uint64(b.Bytes[b.Offset:])
-}
-
-// Read a signed 8-byte word from the message body.
-func (m *Message) getInt64() int64 {
- b := m.bufferForGet()
- defer b.Advance(8)
-
- return int64(binary.LittleEndian.Uint64(b.Bytes[b.Offset:]))
-}
-
-// Read a floating point number from the message body.
-func (m *Message) getFloat64() float64 {
- b := m.bufferForGet()
- defer b.Advance(8)
-
- return math.Float64frombits(binary.LittleEndian.Uint64(b.Bytes[b.Offset:]))
-}
-
-// Decode a list of server objects from the message body.
-func (m *Message) getServers() (servers Servers) {
- defer func() {
- err := recover()
- if err != errMessageEOF {
- panic(err)
- }
-
- }()
-
- for {
- server := bindings.ServerInfo{
- ID: m.getUint64(),
- Address: m.getString(),
- }
- servers = append(servers, server)
- m.bufferForGet()
- }
-}
-
-// Decode a statement result object from the message body.
-func (m *Message) getResult() Result {
- return Result{
- LastInsertID: m.getUint64(),
- RowsAffected: m.getUint64(),
- }
-}
-
-// Decode a query result set object from the message body.
-func (m *Message) getRows() Rows {
- // Read the column count and column names.
- columns := make([]string, m.getUint64())
-
- for i := range columns {
- columns[i] = m.getString()
- }
-
- rows := Rows{
- Columns: columns,
- message: m,
- }
- return rows
-}
-
-func (m *Message) bufferForGet() *buffer {
- size := int(m.words * messageWordSize)
- if m.body1.Offset == size || m.body1.Offset == len(m.body1.Bytes) {
- // The static body has been exahusted, use the dynamic one.
- if m.body1.Offset+m.body2.Offset == size {
- panic(errMessageEOF)
- }
- return &m.body2
- }
-
- return &m.body1
-}
-
-// Result holds the result of a statement.
-type Result struct {
- LastInsertID uint64
- RowsAffected uint64
-}
-
-// Rows holds a result set encoded in a message body.
-type Rows struct {
- Columns []string
- message *Message
-}
-
-// Next returns the next row in the result set.
-func (r *Rows) Next(dest []driver.Value) error {
- types := make([]uint8, len(r.Columns))
-
- // Each column needs a 4 byte slot to store the column type. The row
- // header must be padded to reach word boundary.
- headerBits := len(types) * 4
- padBits := 0
- if trailingBits := (headerBits % messageWordBits); trailingBits != 0 {
- padBits = (messageWordBits - trailingBits)
- }
-
- headerSize := (headerBits + padBits) / messageWordBits * messageWordSize
-
- for i := 0; i < headerSize; i++ {
- slot := r.message.getUint8()
-
- if slot == 0xee {
- // More rows are available.
- return ErrRowsPart
- }
-
- if slot == 0xff {
- // Rows EOF marker
- return io.EOF
- }
-
- index := i * 2
-
- if index >= len(types) {
- continue // This is padding.
- }
-
- types[index] = slot & 0x0f
-
- index++
-
- if index >= len(types) {
- continue // This is padding byte.
- }
-
- types[index] = slot >> 4
- }
-
- for i := range types {
- switch types[i] {
- case bindings.Integer:
- dest[i] = r.message.getInt64()
- case bindings.Float:
- dest[i] = r.message.getFloat64()
- case bindings.Blob:
- panic("todo")
- case bindings.Text:
- dest[i] = r.message.getString()
- case bindings.Null:
- r.message.getUint64()
- dest[i] = nil
- case bindings.UnixTime:
- timestamp := time.Unix(r.message.getInt64(), 0)
- dest[i] = timestamp
- case bindings.ISO8601:
- value := r.message.getString()
- if value == "" {
- dest[i] = time.Time{}
- break
- }
- var t time.Time
- var timeVal time.Time
- var err error
- value = strings.TrimSuffix(value, "Z")
- for _, format := range iso8601Formats {
- if timeVal, err = time.ParseInLocation(format, value, time.UTC); err == nil {
- t = timeVal
- break
- }
- }
- if err != nil {
- return err
- }
- t = t.In(time.Local)
- dest[i] = t
- case bindings.Boolean:
- dest[i] = r.message.getInt64() != 0
- default:
- panic("unknown data type")
- }
- }
-
- return nil
-}
-
-// Close the result set and reset the underlying message.
-func (r *Rows) Close() {
- r.message.Reset()
-}
-
-const (
- messageWordSize = 8
- messageWordBits = messageWordSize * 8
- messageHeaderSize = messageWordSize
- messageMaxConsecutiveEmptyReads = 100
-)
-
-var iso8601Formats = []string{
- // By default, store timestamps with whatever timezone they come with.
- // When parsed, they will be returned with the same timezone.
- "2006-01-02 15:04:05.999999999-07:00",
- "2006-01-02T15:04:05.999999999-07:00",
- "2006-01-02 15:04:05.999999999",
- "2006-01-02T15:04:05.999999999",
- "2006-01-02 15:04:05",
- "2006-01-02T15:04:05",
- "2006-01-02 15:04",
- "2006-01-02T15:04",
- "2006-01-02",
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/request.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/request.go
deleted file mode 100644
index 1cfe9a55c9..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/request.go
+++ /dev/null
@@ -1,98 +0,0 @@
-package client
-
-// DO NOT EDIT
-//
-// This file was generated by ./schema.sh
-
-import (
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
-)
-
-// EncodeLeader encodes a Leader request.
-func EncodeLeader(request *Message) {
- request.putUint64(0)
-
- request.putHeader(bindings.RequestLeader)
-}
-
-// EncodeClient encodes a Client request.
-func EncodeClient(request *Message, id uint64) {
- request.putUint64(id)
-
- request.putHeader(bindings.RequestClient)
-}
-
-// EncodeHeartbeat encodes a Heartbeat request.
-func EncodeHeartbeat(request *Message, timestamp uint64) {
- request.putUint64(timestamp)
-
- request.putHeader(bindings.RequestHeartbeat)
-}
-
-// EncodeOpen encodes a Open request.
-func EncodeOpen(request *Message, name string, flags uint64, vfs string) {
- request.putString(name)
- request.putUint64(flags)
- request.putString(vfs)
-
- request.putHeader(bindings.RequestOpen)
-}
-
-// EncodePrepare encodes a Prepare request.
-func EncodePrepare(request *Message, db uint64, sql string) {
- request.putUint64(db)
- request.putString(sql)
-
- request.putHeader(bindings.RequestPrepare)
-}
-
-// EncodeExec encodes a Exec request.
-func EncodeExec(request *Message, db uint32, stmt uint32, values NamedValues) {
- request.putUint32(db)
- request.putUint32(stmt)
- request.putNamedValues(values)
-
- request.putHeader(bindings.RequestExec)
-}
-
-// EncodeQuery encodes a Query request.
-func EncodeQuery(request *Message, db uint32, stmt uint32, values NamedValues) {
- request.putUint32(db)
- request.putUint32(stmt)
- request.putNamedValues(values)
-
- request.putHeader(bindings.RequestQuery)
-}
-
-// EncodeFinalize encodes a Finalize request.
-func EncodeFinalize(request *Message, db uint32, stmt uint32) {
- request.putUint32(db)
- request.putUint32(stmt)
-
- request.putHeader(bindings.RequestFinalize)
-}
-
-// EncodeExecSQL encodes a ExecSQL request.
-func EncodeExecSQL(request *Message, db uint64, sql string, values NamedValues) {
- request.putUint64(db)
- request.putString(sql)
- request.putNamedValues(values)
-
- request.putHeader(bindings.RequestExecSQL)
-}
-
-// EncodeQuerySQL encodes a QuerySQL request.
-func EncodeQuerySQL(request *Message, db uint64, sql string, values NamedValues) {
- request.putUint64(db)
- request.putString(sql)
- request.putNamedValues(values)
-
- request.putHeader(bindings.RequestQuerySQL)
-}
-
-// EncodeInterrupt encodes a Interrupt request.
-func EncodeInterrupt(request *Message, db uint64) {
- request.putUint64(db)
-
- request.putHeader(bindings.RequestInterrupt)
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/response.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/response.go
deleted file mode 100644
index 6d706d98b2..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/response.go
+++ /dev/null
@@ -1,213 +0,0 @@
-package client
-
-// DO NOT EDIT
-//
-// This file was generated by ./schema.sh
-
-import (
- "fmt"
-
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
-)
-
-// DecodeFailure decodes a Failure response.
-func DecodeFailure(response *Message) (code uint64, message string, err error) {
- mtype, _ := response.getHeader()
-
- if mtype == bindings.ResponseFailure {
- e := ErrRequest{}
- e.Code = response.getUint64()
- e.Description = response.getString()
- err = e
- return
- }
-
- if mtype != bindings.ResponseFailure {
- err = fmt.Errorf("unexpected response type %d", mtype)
- return
- }
-
- code = response.getUint64()
- message = response.getString()
-
- return
-}
-
-// DecodeWelcome decodes a Welcome response.
-func DecodeWelcome(response *Message) (heartbeatTimeout uint64, err error) {
- mtype, _ := response.getHeader()
-
- if mtype == bindings.ResponseFailure {
- e := ErrRequest{}
- e.Code = response.getUint64()
- e.Description = response.getString()
- err = e
- return
- }
-
- if mtype != bindings.ResponseWelcome {
- err = fmt.Errorf("unexpected response type %d", mtype)
- return
- }
-
- heartbeatTimeout = response.getUint64()
-
- return
-}
-
-// DecodeServer decodes a Server response.
-func DecodeServer(response *Message) (address string, err error) {
- mtype, _ := response.getHeader()
-
- if mtype == bindings.ResponseFailure {
- e := ErrRequest{}
- e.Code = response.getUint64()
- e.Description = response.getString()
- err = e
- return
- }
-
- if mtype != bindings.ResponseServer {
- err = fmt.Errorf("unexpected response type %d", mtype)
- return
- }
-
- address = response.getString()
-
- return
-}
-
-// DecodeServers decodes a Servers response.
-func DecodeServers(response *Message) (servers Servers, err error) {
- mtype, _ := response.getHeader()
-
- if mtype == bindings.ResponseFailure {
- e := ErrRequest{}
- e.Code = response.getUint64()
- e.Description = response.getString()
- err = e
- return
- }
-
- if mtype != bindings.ResponseServers {
- err = fmt.Errorf("unexpected response type %d", mtype)
- return
- }
-
- servers = response.getServers()
-
- return
-}
-
-// DecodeDb decodes a Db response.
-func DecodeDb(response *Message) (id uint32, err error) {
- mtype, _ := response.getHeader()
-
- if mtype == bindings.ResponseFailure {
- e := ErrRequest{}
- e.Code = response.getUint64()
- e.Description = response.getString()
- err = e
- return
- }
-
- if mtype != bindings.ResponseDb {
- err = fmt.Errorf("unexpected response type %d", mtype)
- return
- }
-
- id = response.getUint32()
- response.getUint32()
-
- return
-}
-
-// DecodeStmt decodes a Stmt response.
-func DecodeStmt(response *Message) (db uint32, id uint32, params uint64, err error) {
- mtype, _ := response.getHeader()
-
- if mtype == bindings.ResponseFailure {
- e := ErrRequest{}
- e.Code = response.getUint64()
- e.Description = response.getString()
- err = e
- return
- }
-
- if mtype != bindings.ResponseStmt {
- err = fmt.Errorf("unexpected response type %d", mtype)
- return
- }
-
- db = response.getUint32()
- id = response.getUint32()
- params = response.getUint64()
-
- return
-}
-
-// DecodeEmpty decodes a Empty response.
-func DecodeEmpty(response *Message) (err error) {
- mtype, _ := response.getHeader()
-
- if mtype == bindings.ResponseFailure {
- e := ErrRequest{}
- e.Code = response.getUint64()
- e.Description = response.getString()
- err = e
- return
- }
-
- if mtype != bindings.ResponseEmpty {
- err = fmt.Errorf("unexpected response type %d", mtype)
- return
- }
-
- response.getUint64()
-
- return
-}
-
-// DecodeResult decodes a Result response.
-func DecodeResult(response *Message) (result Result, err error) {
- mtype, _ := response.getHeader()
-
- if mtype == bindings.ResponseFailure {
- e := ErrRequest{}
- e.Code = response.getUint64()
- e.Description = response.getString()
- err = e
- return
- }
-
- if mtype != bindings.ResponseResult {
- err = fmt.Errorf("unexpected response type %d", mtype)
- return
- }
-
- result = response.getResult()
-
- return
-}
-
-// DecodeRows decodes a Rows response.
-func DecodeRows(response *Message) (rows Rows, err error) {
- mtype, _ := response.getHeader()
-
- if mtype == bindings.ResponseFailure {
- e := ErrRequest{}
- e.Code = response.getUint64()
- e.Description = response.getString()
- err = e
- return
- }
-
- if mtype != bindings.ResponseRows {
- err = fmt.Errorf("unexpected response type %d", mtype)
- return
- }
-
- rows = response.getRows()
-
- return
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/schema.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/schema.go
deleted file mode 100644
index bc7b6d4f82..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/schema.go
+++ /dev/null
@@ -1,26 +0,0 @@
-package client
-
-//go:generate ./schema.sh --request init
-
-//go:generate ./schema.sh --request Leader unused:uint64
-//go:generate ./schema.sh --request Client id:uint64
-//go:generate ./schema.sh --request Heartbeat timestamp:uint64
-//go:generate ./schema.sh --request Open name:string flags:uint64 vfs:string
-//go:generate ./schema.sh --request Prepare db:uint64 sql:string
-//go:generate ./schema.sh --request Exec db:uint32 stmt:uint32 values:NamedValues
-//go:generate ./schema.sh --request Query db:uint32 stmt:uint32 values:NamedValues
-//go:generate ./schema.sh --request Finalize db:uint32 stmt:uint32
-//go:generate ./schema.sh --request ExecSQL db:uint64 sql:string values:NamedValues
-//go:generate ./schema.sh --request QuerySQL db:uint64 sql:string values:NamedValues
-//go:generate ./schema.sh --request Interrupt db:uint64
-
-//go:generate ./schema.sh --response init
-//go:generate ./schema.sh --response Failure code:uint64 message:string
-//go:generate ./schema.sh --response Welcome heartbeatTimeout:uint64
-//go:generate ./schema.sh --response Server address:string
-//go:generate ./schema.sh --response Servers servers:Servers
-//go:generate ./schema.sh --response Db id:uint32 unused:uint32
-//go:generate ./schema.sh --response Stmt db:uint32 id:uint32 params:uint64
-//go:generate ./schema.sh --response Empty unused:uint64
-//go:generate ./schema.sh --response Result result:Result
-//go:generate ./schema.sh --response Rows rows:Rows
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/schema.sh b/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/schema.sh
deleted file mode 100755
index ca012cea46..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/schema.sh
+++ /dev/null
@@ -1,150 +0,0 @@
-#!/bin/bash
-
-request_init() {
- cat > request.go <<EOF
-package client
-
-// DO NOT EDIT
-//
-// This file was generated by ./schema.sh
-
-import (
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
-)
-EOF
-}
-
-response_init() {
- cat > response.go <<EOF
-package client
-
-// DO NOT EDIT
-//
-// This file was generated by ./schema.sh
-
-import (
- "fmt"
-
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
-)
-EOF
-}
-
-entity=$1
-shift
-
-cmd=$1
-shift
-
-if [ "$entity" = "--request" ]; then
- if [ "$cmd" = "init" ]; then
- request_init
- exit
- fi
-
- args=""
-
- for i in "${@}"
- do
- name=$(echo "$i" | cut -f 1 -d :)
- type=$(echo "$i" | cut -f 2 -d :)
-
- if [ "$name" = "unused" ]; then
- continue
- fi
-
- args=$(echo "${args}, ${name} ${type}")
- done
-
- cat >> request.go <<EOF
-
-// Encode${cmd} encodes a $cmd request.
-func Encode${cmd}(request *Message${args}) {
-EOF
-
- for i in "${@}"
- do
- name=$(echo "$i" | cut -f 1 -d :)
- type=$(echo "$i" | cut -f 2 -d :)
-
- if [ "$name" = "unused" ]; then
- name=$(echo "0")
- fi
-
- cat >> request.go <<EOF
- request.put${type^}(${name})
-EOF
- done
-
- cat >> request.go <<EOF
-
- request.putHeader(bindings.Request${cmd})
-}
-EOF
-
-fi
-
-if [ "$entity" = "--response" ]; then
- if [ "$cmd" = "init" ]; then
- response_init
- exit
- fi
-
- returns=""
-
- for i in "${@}"
- do
- name=$(echo "$i" | cut -f 1 -d :)
- type=$(echo "$i" | cut -f 2 -d :)
-
- if [ "$name" = "unused" ]; then
- continue
- fi
-
- returns=$(echo "${returns}${name} ${type}, ")
- done
-
- cat >> response.go <<EOF
-
-// Decode${cmd} decodes a $cmd response.
-func Decode${cmd}(response *Message) (${returns}err error) {
- mtype, _ := response.getHeader()
-
- if mtype == bindings.ResponseFailure {
- e := ErrRequest{}
- e.Code = response.getUint64()
- e.Description = response.getString()
- err = e
- return
- }
-
- if mtype != bindings.Response${cmd} {
- err = fmt.Errorf("unexpected response type %d", mtype)
- return
- }
-
-EOF
-
- for i in "${@}"
- do
- name=$(echo "$i" | cut -f 1 -d :)
- type=$(echo "$i" | cut -f 2 -d :)
-
- assign=$(echo "${name} = ")
-
- if [ "$name" = "unused" ]; then
- assign=$(echo "")
- fi
-
- cat >> response.go <<EOF
- ${assign}response.get${type^}()
-EOF
- done
-
- cat >> response.go <<EOF
-
- return
-}
-EOF
-
-fi
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/store.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/store.go
deleted file mode 100644
index 1fb71fffdc..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/client/store.go
+++ /dev/null
@@ -1,48 +0,0 @@
-package client
-
-import (
- "context"
-
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
-)
-
-// ServerInfo holds information about a single server.
-type ServerInfo = bindings.ServerInfo
-
-// ServerStore is used by a dqlite client to get an initial list of candidate
-// dqlite servers that it can dial in order to find a leader server to connect
-// to.
-//
-// Once connected, the client periodically updates the server addresses in the
-// store by querying the leader about changes in the cluster (such as servers
-// being added or removed).
-type ServerStore interface {
- // Get return the list of known servers.
- Get(context.Context) ([]ServerInfo, error)
-
- // Set updates the list of known cluster servers.
- Set(context.Context, []ServerInfo) error
-}
-
-// InmemServerStore keeps the list of servers in memory.
-type InmemServerStore struct {
- servers []ServerInfo
-}
-
-// NewInmemServerStore creates ServerStore which stores its data in-memory.
-func NewInmemServerStore() *InmemServerStore {
- return &InmemServerStore{
- servers: make([]ServerInfo, 0),
- }
-}
-
-// Get the current servers.
-func (i *InmemServerStore) Get(ctx context.Context) ([]ServerInfo, error) {
- return i.servers, nil
-}
-
-// Set the servers.
-func (i *InmemServerStore) Set(ctx context.Context, servers []ServerInfo) error {
- i.servers = servers
- return nil
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/connection/open.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/connection/open.go
deleted file mode 100644
index a031d5476c..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/connection/open.go
+++ /dev/null
@@ -1,76 +0,0 @@
-package connection
-
-/*
-import (
- "github.com/CanonicalLtd/go-sqlite3"
- "github.com/pkg/errors"
-)
-
-// OpenLeader is a wrapper around SQLiteDriver.Open that opens connection in
-// leader replication mode, and sets any additional dqlite-related options.
-//
-// The 'methods' argument is used to set the replication methods.
-func OpenLeader(dsn string, methods sqlite3.ReplicationMethods) (*sqlite3.SQLiteConn, error) {
- conn, err := open(dsn)
- if err != nil {
- return nil, err
- }
-
- // Swith to leader replication mode for this connection.
- if err := conn.ReplicationLeader(methods); err != nil {
- return nil, err
- }
-
- return conn, nil
-
-}
-
-// OpenFollower is a wrapper around SQLiteDriver.Open that opens connection in
-// follower replication mode, and sets any additional dqlite-related options.
-func OpenFollower(dsn string) (*sqlite3.SQLiteConn, error) {
- conn, err := open(dsn)
- if err != nil {
- return nil, err
- }
-
- // Switch to leader replication mode for this connection.
- if err := conn.ReplicationFollower(); err != nil {
- return nil, err
- }
-
- return conn, nil
-}
-
-// Open a SQLite connection, setting anything that is common between leader and
-// follower connections.
-func open(dsn string) (*sqlite3.SQLiteConn, error) {
- // Open a plain connection.
- driver := &sqlite3.SQLiteDriver{}
- conn, err := driver.Open(dsn)
- if err != nil {
- return nil, errors.Wrapf(err, "open error for %s", dsn)
- }
-
- // Convert driver.Conn interface to concrete sqlite3.SQLiteConn.
- sqliteConn := conn.(*sqlite3.SQLiteConn)
-
- // Ensure journal mode is set to WAL, as this is a requirement for
- // replication.
- if _, err := sqliteConn.Exec("PRAGMA journal_mode=wal", nil); err != nil {
- return nil, err
- }
-
- // Ensure WAL autocheckpoint disabled, since checkpoints are triggered
- // by explicitly by dqlite.
- if _, err := sqliteConn.Exec("PRAGMA wal_autocheckpoint=0", nil); err != nil {
- return nil, err
- }
-
- // Ensure we don't truncate the WAL.
- if _, err := sqliteConn.Exec("PRAGMA journal_size_limit=-1", nil); err != nil {
- return nil, err
- }
-
- return sqliteConn, nil
-}
-*/
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/connection/snapshot.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/connection/snapshot.go
deleted file mode 100644
index 414a1748bd..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/connection/snapshot.go
+++ /dev/null
@@ -1,38 +0,0 @@
-package connection
-
-import (
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
- "github.com/pkg/errors"
-)
-
-// Snapshot returns a snapshot of the SQLite database with the given path.
-//
-// The snapshot is comprised of two byte slices, one with the content of the
-// database and one is the content of the WAL file.
-func Snapshot(vfs *bindings.Vfs, path string) ([]byte, []byte, error) {
- database, err := vfs.ReadFile(path)
- if err != nil {
- return nil, nil, errors.Wrap(err, "failed to get database file content")
- }
-
- wal, err := vfs.ReadFile(path + "-wal")
- if err != nil {
- return nil, nil, errors.Wrap(err, "failed to get WAL file content")
- }
-
- return database, wal, nil
-}
-
-// Restore the given database and WAL backups, writing them at the given
-// database path.
-func Restore(vfs *bindings.Vfs, path string, database, wal []byte) error {
- if err := vfs.WriteFile(path, database); err != nil {
- return errors.Wrap(err, "failed to restore database file")
- }
-
- if err := vfs.WriteFile(path+"-wal", wal); err != nil {
- return errors.Wrap(err, "failed to restore WAL file")
- }
-
- return nil
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/connection/uri.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/connection/uri.go
deleted file mode 100644
index ee7ca302c3..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/connection/uri.go
+++ /dev/null
@@ -1,61 +0,0 @@
-package connection
-
-import (
- "fmt"
- "net/url"
- "strings"
-
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
-)
-
-// ParseURI parses the given sqlite3 URI checking if it's compatible with
-// dqlite.
-//
-// Only pure file names without any directory segment are accepted
-// (e.g. "test.db"). Query parameters are always valid except for
-// "mode=memory".
-//
-// It returns the filename and query parameters.
-func ParseURI(uri string) (string, uint64, error) {
- filename := uri
- flags := uint64(bindings.OpenReadWrite | bindings.OpenCreate)
-
- pos := strings.IndexRune(uri, '?')
- if pos >= 1 {
- params, err := url.ParseQuery(uri[pos+1:])
- if err != nil {
- return "", 0, err
- }
-
- mode := params.Get("mode")
- switch mode {
- case "":
- case "memory":
- return "", 0, fmt.Errorf("memory database not supported")
- case "ro":
- flags = bindings.OpenReadOnly
- case "rw":
- flags = bindings.OpenReadWrite
- case "rwc":
- flags = bindings.OpenReadWrite | bindings.OpenCreate
- default:
- return "", 0, fmt.Errorf("unknown mode %s", mode)
- }
-
- filename = filename[:pos]
- }
-
- if strings.HasPrefix(filename, "file:") {
- filename = filename[len("file:"):]
- }
-
- if filename == ":memory:" {
- return "", 0, fmt.Errorf("memory database not supported")
- }
-
- if strings.IndexRune(filename, '/') >= 0 {
- return "", 0, fmt.Errorf("directory segments are invalid")
- }
-
- return filename, flags, nil
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/logging/func.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/logging/func.go
deleted file mode 100644
index 57e0525fa8..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/logging/func.go
+++ /dev/null
@@ -1,26 +0,0 @@
-package logging
-
-import (
- "fmt"
- "testing"
-)
-
-// Func is a function that can be used for logging.
-type Func func(Level, string, ...interface{})
-
-// Test returns a logging function that forwards messages to the test logger.
-func Test(t *testing.T) Func {
- return func(l Level, format string, a ...interface{}) {
- format = fmt.Sprintf("%s: %s", l.String(), format)
- t.Logf(format, a...)
- }
-}
-
-// Stdout returns a logging function that prints log messages on standard
-// output.
-func Stdout() Func {
- return func(l Level, format string, a ...interface{}) {
- format = fmt.Sprintf("%s: %s\n", l.String(), format)
- fmt.Printf(format, a...)
- }
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/logging/level.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/logging/level.go
deleted file mode 100644
index 0f412c2970..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/logging/level.go
+++ /dev/null
@@ -1,27 +0,0 @@
-package logging
-
-// Level defines the logging level.
-type Level int
-
-// Available logging levels.
-const (
- Debug Level = iota
- Info
- Warn
- Error
-)
-
-func (l Level) String() string {
- switch l {
- case Debug:
- return "DEBUG"
- case Info:
- return "INFO"
- case Warn:
- return "WARN"
- case Error:
- return "ERROR"
- default:
- return "UNKNOWN"
- }
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/protocol/commands.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/protocol/commands.go
deleted file mode 100644
index 96de2178c6..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/protocol/commands.go
+++ /dev/null
@@ -1,107 +0,0 @@
-package protocol
-
-import (
- "reflect"
- "strings"
- "unsafe"
-
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
- "github.com/golang/protobuf/proto"
- "github.com/pkg/errors"
-)
-
-// MarshalCommand marshals a dqlite FSM command.
-func MarshalCommand(command *Command) ([]byte, error) {
- return proto.Marshal(command)
-}
-
-// UnmarshalCommand unmarshals a dqlite FSM command.
-func UnmarshalCommand(data []byte) (*Command, error) {
- command := &Command{}
- if err := proto.Unmarshal(data, command); err != nil {
- return nil, errors.Wrap(err, "protobuf failure")
- }
- return command, nil
-}
-
-// NewOpen returns a new Command with Open parameters.
-func NewOpen(name string) *Command {
- params := &Command_Open{Open: &Open{Name: name}}
- return newCommand(params)
-}
-
-// NewBegin returns a new Command with Begin parameters.
-func NewBegin(txid uint64, name string) *Command {
- params := &Command_Begin{Begin: &Begin{Txid: txid, Name: name}}
- return newCommand(params)
-}
-
-// NewFrames returns a new WalFrames protobuf message.
-func NewFrames(txid uint64, filename string, list bindings.WalReplicationFrameList) *Command {
- length := list.Len()
- pageSize := list.PageSize()
-
- numbers := make([]uint32, length)
- pages := make([]byte, length*pageSize)
-
- for i := range numbers {
- data, pgno, _ := list.Frame(i)
- numbers[i] = uint32(pgno)
- header := reflect.SliceHeader{Data: uintptr(data), Len: pageSize, Cap: pageSize}
- var slice []byte
- slice = reflect.NewAt(reflect.TypeOf(slice), unsafe.Pointer(&header)).Elem().Interface().([]byte)
- copy(pages[i*pageSize:(i+1)*pageSize], slice)
- }
-
- isCommit := int32(0)
- if list.IsCommit() {
- isCommit = int32(1)
- }
-
- params := &Command_Frames{Frames: &Frames{
- Txid: txid,
- PageSize: int32(pageSize),
- PageNumbers: numbers,
- PageData: pages,
- Truncate: uint32(list.Truncate()),
- IsCommit: isCommit,
- Filename: filename,
- }}
-
- return newCommand(params)
-}
-
-// NewUndo returns a new Undo protobuf message.
-func NewUndo(txid uint64) *Command {
- params := &Command_Undo{Undo: &Undo{
- Txid: txid,
- }}
- return newCommand(params)
-}
-
-// NewEnd returns a new End protobuf message.
-func NewEnd(txid uint64) *Command {
- params := &Command_End{End: &End{
- Txid: txid,
- }}
- return newCommand(params)
-}
-
-// NewCheckpoint returns a new Checkpoint protobuf message.
-func NewCheckpoint(name string) *Command {
- params := &Command_Checkpoint{Checkpoint: &Checkpoint{
- Name: name,
- }}
- return newCommand(params)
-}
-
-func newCommand(payload isCommand_Payload) *Command {
- return &Command{Payload: payload}
-}
-
-// Name returns a human readable name for the command, based on its Params
-// type.
-func (c *Command) Name() string {
- typeName := reflect.TypeOf(c.Payload).Elem().String()
- return strings.ToLower(strings.Replace(typeName, "protocol.Command_", "", 1))
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/protocol/commands.pb.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/protocol/commands.pb.go
deleted file mode 100644
index 479ce51dec..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/protocol/commands.pb.go
+++ /dev/null
@@ -1,2253 +0,0 @@
-// Code generated by protoc-gen-gogo. DO NOT EDIT.
-// source: internal/protocol/commands.proto
-
-/*
- Package protocol is a generated protocol buffer package.
-
- It is generated from these files:
- internal/protocol/commands.proto
-
- It has these top-level messages:
- Command
- Open
- Begin
- Frames
- FramesPage
- Undo
- End
- Checkpoint
-*/
-package protocol
-
-import proto "github.com/gogo/protobuf/proto"
-import fmt "fmt"
-import math "math"
-
-import io "io"
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
-
-// Command encapsulates the payload fo a dqlite Raft FSM command.
-//
-// On the wire this will be a varint indentifying the command type,
-// followed by the command payload.
-type Command struct {
- // Types that are valid to be assigned to Payload:
- // *Command_Open
- // *Command_Begin
- // *Command_Frames
- // *Command_Undo
- // *Command_End
- // *Command_Checkpoint
- Payload isCommand_Payload `protobuf_oneof:"Payload"`
-}
-
-func (m *Command) Reset() { *m = Command{} }
-func (m *Command) String() string { return proto.CompactTextString(m) }
-func (*Command) ProtoMessage() {}
-func (*Command) Descriptor() ([]byte, []int) { return fileDescriptorCommands, []int{0} }
-
-type isCommand_Payload interface {
- isCommand_Payload()
- MarshalTo([]byte) (int, error)
- Size() int
-}
-
-type Command_Open struct {
- Open *Open `protobuf:"bytes,1,opt,name=open,oneof"`
-}
-type Command_Begin struct {
- Begin *Begin `protobuf:"bytes,2,opt,name=begin,oneof"`
-}
-type Command_Frames struct {
- Frames *Frames `protobuf:"bytes,3,opt,name=frames,oneof"`
-}
-type Command_Undo struct {
- Undo *Undo `protobuf:"bytes,4,opt,name=undo,oneof"`
-}
-type Command_End struct {
- End *End `protobuf:"bytes,5,opt,name=end,oneof"`
-}
-type Command_Checkpoint struct {
- Checkpoint *Checkpoint `protobuf:"bytes,6,opt,name=checkpoint,oneof"`
-}
-
-func (*Command_Open) isCommand_Payload() {}
-func (*Command_Begin) isCommand_Payload() {}
-func (*Command_Frames) isCommand_Payload() {}
-func (*Command_Undo) isCommand_Payload() {}
-func (*Command_End) isCommand_Payload() {}
-func (*Command_Checkpoint) isCommand_Payload() {}
-
-func (m *Command) GetPayload() isCommand_Payload {
- if m != nil {
- return m.Payload
- }
- return nil
-}
-
-func (m *Command) GetOpen() *Open {
- if x, ok := m.GetPayload().(*Command_Open); ok {
- return x.Open
- }
- return nil
-}
-
-func (m *Command) GetBegin() *Begin {
- if x, ok := m.GetPayload().(*Command_Begin); ok {
- return x.Begin
- }
- return nil
-}
-
-func (m *Command) GetFrames() *Frames {
- if x, ok := m.GetPayload().(*Command_Frames); ok {
- return x.Frames
- }
- return nil
-}
-
-func (m *Command) GetUndo() *Undo {
- if x, ok := m.GetPayload().(*Command_Undo); ok {
- return x.Undo
- }
- return nil
-}
-
-func (m *Command) GetEnd() *End {
- if x, ok := m.GetPayload().(*Command_End); ok {
- return x.End
- }
- return nil
-}
-
-func (m *Command) GetCheckpoint() *Checkpoint {
- if x, ok := m.GetPayload().(*Command_Checkpoint); ok {
- return x.Checkpoint
- }
- return nil
-}
-
-// XXX_OneofFuncs is for the internal use of the proto package.
-func (*Command) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) {
- return _Command_OneofMarshaler, _Command_OneofUnmarshaler, _Command_OneofSizer, []interface{}{
- (*Command_Open)(nil),
- (*Command_Begin)(nil),
- (*Command_Frames)(nil),
- (*Command_Undo)(nil),
- (*Command_End)(nil),
- (*Command_Checkpoint)(nil),
- }
-}
-
-func _Command_OneofMarshaler(msg proto.Message, b *proto.Buffer) error {
- m := msg.(*Command)
- // Payload
- switch x := m.Payload.(type) {
- case *Command_Open:
- _ = b.EncodeVarint(1<<3 | proto.WireBytes)
- if err := b.EncodeMessage(x.Open); err != nil {
- return err
- }
- case *Command_Begin:
- _ = b.EncodeVarint(2<<3 | proto.WireBytes)
- if err := b.EncodeMessage(x.Begin); err != nil {
- return err
- }
- case *Command_Frames:
- _ = b.EncodeVarint(3<<3 | proto.WireBytes)
- if err := b.EncodeMessage(x.Frames); err != nil {
- return err
- }
- case *Command_Undo:
- _ = b.EncodeVarint(4<<3 | proto.WireBytes)
- if err := b.EncodeMessage(x.Undo); err != nil {
- return err
- }
- case *Command_End:
- _ = b.EncodeVarint(5<<3 | proto.WireBytes)
- if err := b.EncodeMessage(x.End); err != nil {
- return err
- }
- case *Command_Checkpoint:
- _ = b.EncodeVarint(6<<3 | proto.WireBytes)
- if err := b.EncodeMessage(x.Checkpoint); err != nil {
- return err
- }
- case nil:
- default:
- return fmt.Errorf("Command.Payload has unexpected type %T", x)
- }
- return nil
-}
-
-func _Command_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) {
- m := msg.(*Command)
- switch tag {
- case 1: // Payload.open
- if wire != proto.WireBytes {
- return true, proto.ErrInternalBadWireType
- }
- msg := new(Open)
- err := b.DecodeMessage(msg)
- m.Payload = &Command_Open{msg}
- return true, err
- case 2: // Payload.begin
- if wire != proto.WireBytes {
- return true, proto.ErrInternalBadWireType
- }
- msg := new(Begin)
- err := b.DecodeMessage(msg)
- m.Payload = &Command_Begin{msg}
- return true, err
- case 3: // Payload.frames
- if wire != proto.WireBytes {
- return true, proto.ErrInternalBadWireType
- }
- msg := new(Frames)
- err := b.DecodeMessage(msg)
- m.Payload = &Command_Frames{msg}
- return true, err
- case 4: // Payload.undo
- if wire != proto.WireBytes {
- return true, proto.ErrInternalBadWireType
- }
- msg := new(Undo)
- err := b.DecodeMessage(msg)
- m.Payload = &Command_Undo{msg}
- return true, err
- case 5: // Payload.end
- if wire != proto.WireBytes {
- return true, proto.ErrInternalBadWireType
- }
- msg := new(End)
- err := b.DecodeMessage(msg)
- m.Payload = &Command_End{msg}
- return true, err
- case 6: // Payload.checkpoint
- if wire != proto.WireBytes {
- return true, proto.ErrInternalBadWireType
- }
- msg := new(Checkpoint)
- err := b.DecodeMessage(msg)
- m.Payload = &Command_Checkpoint{msg}
- return true, err
- default:
- return false, nil
- }
-}
-
-func _Command_OneofSizer(msg proto.Message) (n int) {
- m := msg.(*Command)
- // Payload
- switch x := m.Payload.(type) {
- case *Command_Open:
- s := proto.Size(x.Open)
- n += proto.SizeVarint(1<<3 | proto.WireBytes)
- n += proto.SizeVarint(uint64(s))
- n += s
- case *Command_Begin:
- s := proto.Size(x.Begin)
- n += proto.SizeVarint(2<<3 | proto.WireBytes)
- n += proto.SizeVarint(uint64(s))
- n += s
- case *Command_Frames:
- s := proto.Size(x.Frames)
- n += proto.SizeVarint(3<<3 | proto.WireBytes)
- n += proto.SizeVarint(uint64(s))
- n += s
- case *Command_Undo:
- s := proto.Size(x.Undo)
- n += proto.SizeVarint(4<<3 | proto.WireBytes)
- n += proto.SizeVarint(uint64(s))
- n += s
- case *Command_End:
- s := proto.Size(x.End)
- n += proto.SizeVarint(5<<3 | proto.WireBytes)
- n += proto.SizeVarint(uint64(s))
- n += s
- case *Command_Checkpoint:
- s := proto.Size(x.Checkpoint)
- n += proto.SizeVarint(6<<3 | proto.WireBytes)
- n += proto.SizeVarint(uint64(s))
- n += s
- case nil:
- default:
- panic(fmt.Sprintf("proto: unexpected type %T in oneof", x))
- }
- return n
-}
-
-// Parameters to open a new database and setup the needed data
-// structures for replication.
-type Open struct {
- Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-}
-
-func (m *Open) Reset() { *m = Open{} }
-func (m *Open) String() string { return proto.CompactTextString(m) }
-func (*Open) ProtoMessage() {}
-func (*Open) Descriptor() ([]byte, []int) { return fileDescriptorCommands, []int{1} }
-
-func (m *Open) GetName() string {
- if m != nil {
- return m.Name
- }
- return ""
-}
-
-// Parameters to begin a new write transaction.
-//
-// This command is not used anymore, but it's
-// kept for backward-compatibility.
-type Begin struct {
- Txid uint64 `protobuf:"varint,1,opt,name=txid,proto3" json:"txid,omitempty"`
- Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
-}
-
-func (m *Begin) Reset() { *m = Begin{} }
-func (m *Begin) String() string { return proto.CompactTextString(m) }
-func (*Begin) ProtoMessage() {}
-func (*Begin) Descriptor() ([]byte, []int) { return fileDescriptorCommands, []int{2} }
-
-func (m *Begin) GetTxid() uint64 {
- if m != nil {
- return m.Txid
- }
- return 0
-}
-
-func (m *Begin) GetName() string {
- if m != nil {
- return m.Name
- }
- return ""
-}
-
-// Parameters to append new frames to the WAL within a write transaction.
-type Frames struct {
- Txid uint64 `protobuf:"varint,1,opt,name=txid,proto3" json:"txid,omitempty"`
- PageSize int32 `protobuf:"varint,2,opt,name=page_size,json=pageSize,proto3" json:"page_size,omitempty"`
- Pages []*FramesPage `protobuf:"bytes,3,rep,name=pages" json:"pages,omitempty"`
- Truncate uint32 `protobuf:"varint,4,opt,name=truncate,proto3" json:"truncate,omitempty"`
- IsCommit int32 `protobuf:"varint,5,opt,name=is_commit,json=isCommit,proto3" json:"is_commit,omitempty"`
- SyncFlags uint32 `protobuf:"varint,6,opt,name=sync_flags,json=syncFlags,proto3" json:"sync_flags,omitempty"`
- Filename string `protobuf:"bytes,7,opt,name=filename,proto3" json:"filename,omitempty"`
- PageNumbers []uint32 `protobuf:"varint,8,rep,packed,name=page_numbers,json=pageNumbers" json:"page_numbers,omitempty"`
- PageData []byte `protobuf:"bytes,9,opt,name=page_data,json=pageData,proto3" json:"page_data,omitempty"`
-}
-
-func (m *Frames) Reset() { *m = Frames{} }
-func (m *Frames) String() string { return proto.CompactTextString(m) }
-func (*Frames) ProtoMessage() {}
-func (*Frames) Descriptor() ([]byte, []int) { return fileDescriptorCommands, []int{3} }
-
-func (m *Frames) GetTxid() uint64 {
- if m != nil {
- return m.Txid
- }
- return 0
-}
-
-func (m *Frames) GetPageSize() int32 {
- if m != nil {
- return m.PageSize
- }
- return 0
-}
-
-func (m *Frames) GetPages() []*FramesPage {
- if m != nil {
- return m.Pages
- }
- return nil
-}
-
-func (m *Frames) GetTruncate() uint32 {
- if m != nil {
- return m.Truncate
- }
- return 0
-}
-
-func (m *Frames) GetIsCommit() int32 {
- if m != nil {
- return m.IsCommit
- }
- return 0
-}
-
-func (m *Frames) GetSyncFlags() uint32 {
- if m != nil {
- return m.SyncFlags
- }
- return 0
-}
-
-func (m *Frames) GetFilename() string {
- if m != nil {
- return m.Filename
- }
- return ""
-}
-
-func (m *Frames) GetPageNumbers() []uint32 {
- if m != nil {
- return m.PageNumbers
- }
- return nil
-}
-
-func (m *Frames) GetPageData() []byte {
- if m != nil {
- return m.PageData
- }
- return nil
-}
-
-// A single frame of data in a Frames command.
-//
-// FIXME: this is a legacy message only used in v1 of the protocol.
-type FramesPage struct {
- Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"`
- Flags uint32 `protobuf:"varint,2,opt,name=flags,proto3" json:"flags,omitempty"`
- Number uint32 `protobuf:"varint,3,opt,name=number,proto3" json:"number,omitempty"`
-}
-
-func (m *FramesPage) Reset() { *m = FramesPage{} }
-func (m *FramesPage) String() string { return proto.CompactTextString(m) }
-func (*FramesPage) ProtoMessage() {}
-func (*FramesPage) Descriptor() ([]byte, []int) { return fileDescriptorCommands, []int{4} }
-
-func (m *FramesPage) GetData() []byte {
- if m != nil {
- return m.Data
- }
- return nil
-}
-
-func (m *FramesPage) GetFlags() uint32 {
- if m != nil {
- return m.Flags
- }
- return 0
-}
-
-func (m *FramesPage) GetNumber() uint32 {
- if m != nil {
- return m.Number
- }
- return 0
-}
-
-// Parameters to undo any previous WAL change in a write transaction.
-type Undo struct {
- Txid uint64 `protobuf:"varint,1,opt,name=txid,proto3" json:"txid,omitempty"`
-}
-
-func (m *Undo) Reset() { *m = Undo{} }
-func (m *Undo) String() string { return proto.CompactTextString(m) }
-func (*Undo) ProtoMessage() {}
-func (*Undo) Descriptor() ([]byte, []int) { return fileDescriptorCommands, []int{5} }
-
-func (m *Undo) GetTxid() uint64 {
- if m != nil {
- return m.Txid
- }
- return 0
-}
-
-// Parameters to end a write transaction, and update the WAL commit
-// pointer.
-//
-// This command is not used anymore, but it's
-// kept for backward-compatibility.
-type End struct {
- Txid uint64 `protobuf:"varint,1,opt,name=txid,proto3" json:"txid,omitempty"`
-}
-
-func (m *End) Reset() { *m = End{} }
-func (m *End) String() string { return proto.CompactTextString(m) }
-func (*End) ProtoMessage() {}
-func (*End) Descriptor() ([]byte, []int) { return fileDescriptorCommands, []int{6} }
-
-func (m *End) GetTxid() uint64 {
- if m != nil {
- return m.Txid
- }
- return 0
-}
-
-// Parameters to perform a WAL checkpoint.
-type Checkpoint struct {
- Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-}
-
-func (m *Checkpoint) Reset() { *m = Checkpoint{} }
-func (m *Checkpoint) String() string { return proto.CompactTextString(m) }
-func (*Checkpoint) ProtoMessage() {}
-func (*Checkpoint) Descriptor() ([]byte, []int) { return fileDescriptorCommands, []int{7} }
-
-func (m *Checkpoint) GetName() string {
- if m != nil {
- return m.Name
- }
- return ""
-}
-
-func init() {
- proto.RegisterType((*Command)(nil), "protocol.Command")
- proto.RegisterType((*Open)(nil), "protocol.Open")
- proto.RegisterType((*Begin)(nil), "protocol.Begin")
- proto.RegisterType((*Frames)(nil), "protocol.Frames")
- proto.RegisterType((*FramesPage)(nil), "protocol.FramesPage")
- proto.RegisterType((*Undo)(nil), "protocol.Undo")
- proto.RegisterType((*End)(nil), "protocol.End")
- proto.RegisterType((*Checkpoint)(nil), "protocol.Checkpoint")
-}
-func (m *Command) Marshal() (dAtA []byte, err error) {
- size := m.Size()
- dAtA = make([]byte, size)
- n, err := m.MarshalTo(dAtA)
- if err != nil {
- return nil, err
- }
- return dAtA[:n], nil
-}
-
-func (m *Command) MarshalTo(dAtA []byte) (int, error) {
- var i int
- _ = i
- var l int
- _ = l
- if m.Payload != nil {
- nn1, err := m.Payload.MarshalTo(dAtA[i:])
- if err != nil {
- return 0, err
- }
- i += nn1
- }
- return i, nil
-}
-
-func (m *Command_Open) MarshalTo(dAtA []byte) (int, error) {
- i := 0
- if m.Open != nil {
- dAtA[i] = 0xa
- i++
- i = encodeVarintCommands(dAtA, i, uint64(m.Open.Size()))
- n2, err := m.Open.MarshalTo(dAtA[i:])
- if err != nil {
- return 0, err
- }
- i += n2
- }
- return i, nil
-}
-func (m *Command_Begin) MarshalTo(dAtA []byte) (int, error) {
- i := 0
- if m.Begin != nil {
- dAtA[i] = 0x12
- i++
- i = encodeVarintCommands(dAtA, i, uint64(m.Begin.Size()))
- n3, err := m.Begin.MarshalTo(dAtA[i:])
- if err != nil {
- return 0, err
- }
- i += n3
- }
- return i, nil
-}
-func (m *Command_Frames) MarshalTo(dAtA []byte) (int, error) {
- i := 0
- if m.Frames != nil {
- dAtA[i] = 0x1a
- i++
- i = encodeVarintCommands(dAtA, i, uint64(m.Frames.Size()))
- n4, err := m.Frames.MarshalTo(dAtA[i:])
- if err != nil {
- return 0, err
- }
- i += n4
- }
- return i, nil
-}
-func (m *Command_Undo) MarshalTo(dAtA []byte) (int, error) {
- i := 0
- if m.Undo != nil {
- dAtA[i] = 0x22
- i++
- i = encodeVarintCommands(dAtA, i, uint64(m.Undo.Size()))
- n5, err := m.Undo.MarshalTo(dAtA[i:])
- if err != nil {
- return 0, err
- }
- i += n5
- }
- return i, nil
-}
-func (m *Command_End) MarshalTo(dAtA []byte) (int, error) {
- i := 0
- if m.End != nil {
- dAtA[i] = 0x2a
- i++
- i = encodeVarintCommands(dAtA, i, uint64(m.End.Size()))
- n6, err := m.End.MarshalTo(dAtA[i:])
- if err != nil {
- return 0, err
- }
- i += n6
- }
- return i, nil
-}
-func (m *Command_Checkpoint) MarshalTo(dAtA []byte) (int, error) {
- i := 0
- if m.Checkpoint != nil {
- dAtA[i] = 0x32
- i++
- i = encodeVarintCommands(dAtA, i, uint64(m.Checkpoint.Size()))
- n7, err := m.Checkpoint.MarshalTo(dAtA[i:])
- if err != nil {
- return 0, err
- }
- i += n7
- }
- return i, nil
-}
-func (m *Open) Marshal() (dAtA []byte, err error) {
- size := m.Size()
- dAtA = make([]byte, size)
- n, err := m.MarshalTo(dAtA)
- if err != nil {
- return nil, err
- }
- return dAtA[:n], nil
-}
-
-func (m *Open) MarshalTo(dAtA []byte) (int, error) {
- var i int
- _ = i
- var l int
- _ = l
- if len(m.Name) > 0 {
- dAtA[i] = 0xa
- i++
- i = encodeVarintCommands(dAtA, i, uint64(len(m.Name)))
- i += copy(dAtA[i:], m.Name)
- }
- return i, nil
-}
-
-func (m *Begin) Marshal() (dAtA []byte, err error) {
- size := m.Size()
- dAtA = make([]byte, size)
- n, err := m.MarshalTo(dAtA)
- if err != nil {
- return nil, err
- }
- return dAtA[:n], nil
-}
-
-func (m *Begin) MarshalTo(dAtA []byte) (int, error) {
- var i int
- _ = i
- var l int
- _ = l
- if m.Txid != 0 {
- dAtA[i] = 0x8
- i++
- i = encodeVarintCommands(dAtA, i, uint64(m.Txid))
- }
- if len(m.Name) > 0 {
- dAtA[i] = 0x12
- i++
- i = encodeVarintCommands(dAtA, i, uint64(len(m.Name)))
- i += copy(dAtA[i:], m.Name)
- }
- return i, nil
-}
-
-func (m *Frames) Marshal() (dAtA []byte, err error) {
- size := m.Size()
- dAtA = make([]byte, size)
- n, err := m.MarshalTo(dAtA)
- if err != nil {
- return nil, err
- }
- return dAtA[:n], nil
-}
-
-func (m *Frames) MarshalTo(dAtA []byte) (int, error) {
- var i int
- _ = i
- var l int
- _ = l
- if m.Txid != 0 {
- dAtA[i] = 0x8
- i++
- i = encodeVarintCommands(dAtA, i, uint64(m.Txid))
- }
- if m.PageSize != 0 {
- dAtA[i] = 0x10
- i++
- i = encodeVarintCommands(dAtA, i, uint64(m.PageSize))
- }
- if len(m.Pages) > 0 {
- for _, msg := range m.Pages {
- dAtA[i] = 0x1a
- i++
- i = encodeVarintCommands(dAtA, i, uint64(msg.Size()))
- n, err := msg.MarshalTo(dAtA[i:])
- if err != nil {
- return 0, err
- }
- i += n
- }
- }
- if m.Truncate != 0 {
- dAtA[i] = 0x20
- i++
- i = encodeVarintCommands(dAtA, i, uint64(m.Truncate))
- }
- if m.IsCommit != 0 {
- dAtA[i] = 0x28
- i++
- i = encodeVarintCommands(dAtA, i, uint64(m.IsCommit))
- }
- if m.SyncFlags != 0 {
- dAtA[i] = 0x30
- i++
- i = encodeVarintCommands(dAtA, i, uint64(m.SyncFlags))
- }
- if len(m.Filename) > 0 {
- dAtA[i] = 0x3a
- i++
- i = encodeVarintCommands(dAtA, i, uint64(len(m.Filename)))
- i += copy(dAtA[i:], m.Filename)
- }
- if len(m.PageNumbers) > 0 {
- dAtA9 := make([]byte, len(m.PageNumbers)*10)
- var j8 int
- for _, num := range m.PageNumbers {
- for num >= 1<<7 {
- dAtA9[j8] = uint8(uint64(num)&0x7f | 0x80)
- num >>= 7
- j8++
- }
- dAtA9[j8] = uint8(num)
- j8++
- }
- dAtA[i] = 0x42
- i++
- i = encodeVarintCommands(dAtA, i, uint64(j8))
- i += copy(dAtA[i:], dAtA9[:j8])
- }
- if len(m.PageData) > 0 {
- dAtA[i] = 0x4a
- i++
- i = encodeVarintCommands(dAtA, i, uint64(len(m.PageData)))
- i += copy(dAtA[i:], m.PageData)
- }
- return i, nil
-}
-
-func (m *FramesPage) Marshal() (dAtA []byte, err error) {
- size := m.Size()
- dAtA = make([]byte, size)
- n, err := m.MarshalTo(dAtA)
- if err != nil {
- return nil, err
- }
- return dAtA[:n], nil
-}
-
-func (m *FramesPage) MarshalTo(dAtA []byte) (int, error) {
- var i int
- _ = i
- var l int
- _ = l
- if len(m.Data) > 0 {
- dAtA[i] = 0xa
- i++
- i = encodeVarintCommands(dAtA, i, uint64(len(m.Data)))
- i += copy(dAtA[i:], m.Data)
- }
- if m.Flags != 0 {
- dAtA[i] = 0x10
- i++
- i = encodeVarintCommands(dAtA, i, uint64(m.Flags))
- }
- if m.Number != 0 {
- dAtA[i] = 0x18
- i++
- i = encodeVarintCommands(dAtA, i, uint64(m.Number))
- }
- return i, nil
-}
-
-func (m *Undo) Marshal() (dAtA []byte, err error) {
- size := m.Size()
- dAtA = make([]byte, size)
- n, err := m.MarshalTo(dAtA)
- if err != nil {
- return nil, err
- }
- return dAtA[:n], nil
-}
-
-func (m *Undo) MarshalTo(dAtA []byte) (int, error) {
- var i int
- _ = i
- var l int
- _ = l
- if m.Txid != 0 {
- dAtA[i] = 0x8
- i++
- i = encodeVarintCommands(dAtA, i, uint64(m.Txid))
- }
- return i, nil
-}
-
-func (m *End) Marshal() (dAtA []byte, err error) {
- size := m.Size()
- dAtA = make([]byte, size)
- n, err := m.MarshalTo(dAtA)
- if err != nil {
- return nil, err
- }
- return dAtA[:n], nil
-}
-
-func (m *End) MarshalTo(dAtA []byte) (int, error) {
- var i int
- _ = i
- var l int
- _ = l
- if m.Txid != 0 {
- dAtA[i] = 0x8
- i++
- i = encodeVarintCommands(dAtA, i, uint64(m.Txid))
- }
- return i, nil
-}
-
-func (m *Checkpoint) Marshal() (dAtA []byte, err error) {
- size := m.Size()
- dAtA = make([]byte, size)
- n, err := m.MarshalTo(dAtA)
- if err != nil {
- return nil, err
- }
- return dAtA[:n], nil
-}
-
-func (m *Checkpoint) MarshalTo(dAtA []byte) (int, error) {
- var i int
- _ = i
- var l int
- _ = l
- if len(m.Name) > 0 {
- dAtA[i] = 0xa
- i++
- i = encodeVarintCommands(dAtA, i, uint64(len(m.Name)))
- i += copy(dAtA[i:], m.Name)
- }
- return i, nil
-}
-
-func encodeVarintCommands(dAtA []byte, offset int, v uint64) int {
- for v >= 1<<7 {
- dAtA[offset] = uint8(v&0x7f | 0x80)
- v >>= 7
- offset++
- }
- dAtA[offset] = uint8(v)
- return offset + 1
-}
-func (m *Command) Size() (n int) {
- var l int
- _ = l
- if m.Payload != nil {
- n += m.Payload.Size()
- }
- return n
-}
-
-func (m *Command_Open) Size() (n int) {
- var l int
- _ = l
- if m.Open != nil {
- l = m.Open.Size()
- n += 1 + l + sovCommands(uint64(l))
- }
- return n
-}
-func (m *Command_Begin) Size() (n int) {
- var l int
- _ = l
- if m.Begin != nil {
- l = m.Begin.Size()
- n += 1 + l + sovCommands(uint64(l))
- }
- return n
-}
-func (m *Command_Frames) Size() (n int) {
- var l int
- _ = l
- if m.Frames != nil {
- l = m.Frames.Size()
- n += 1 + l + sovCommands(uint64(l))
- }
- return n
-}
-func (m *Command_Undo) Size() (n int) {
- var l int
- _ = l
- if m.Undo != nil {
- l = m.Undo.Size()
- n += 1 + l + sovCommands(uint64(l))
- }
- return n
-}
-func (m *Command_End) Size() (n int) {
- var l int
- _ = l
- if m.End != nil {
- l = m.End.Size()
- n += 1 + l + sovCommands(uint64(l))
- }
- return n
-}
-func (m *Command_Checkpoint) Size() (n int) {
- var l int
- _ = l
- if m.Checkpoint != nil {
- l = m.Checkpoint.Size()
- n += 1 + l + sovCommands(uint64(l))
- }
- return n
-}
-func (m *Open) Size() (n int) {
- var l int
- _ = l
- l = len(m.Name)
- if l > 0 {
- n += 1 + l + sovCommands(uint64(l))
- }
- return n
-}
-
-func (m *Begin) Size() (n int) {
- var l int
- _ = l
- if m.Txid != 0 {
- n += 1 + sovCommands(uint64(m.Txid))
- }
- l = len(m.Name)
- if l > 0 {
- n += 1 + l + sovCommands(uint64(l))
- }
- return n
-}
-
-func (m *Frames) Size() (n int) {
- var l int
- _ = l
- if m.Txid != 0 {
- n += 1 + sovCommands(uint64(m.Txid))
- }
- if m.PageSize != 0 {
- n += 1 + sovCommands(uint64(m.PageSize))
- }
- if len(m.Pages) > 0 {
- for _, e := range m.Pages {
- l = e.Size()
- n += 1 + l + sovCommands(uint64(l))
- }
- }
- if m.Truncate != 0 {
- n += 1 + sovCommands(uint64(m.Truncate))
- }
- if m.IsCommit != 0 {
- n += 1 + sovCommands(uint64(m.IsCommit))
- }
- if m.SyncFlags != 0 {
- n += 1 + sovCommands(uint64(m.SyncFlags))
- }
- l = len(m.Filename)
- if l > 0 {
- n += 1 + l + sovCommands(uint64(l))
- }
- if len(m.PageNumbers) > 0 {
- l = 0
- for _, e := range m.PageNumbers {
- l += sovCommands(uint64(e))
- }
- n += 1 + sovCommands(uint64(l)) + l
- }
- l = len(m.PageData)
- if l > 0 {
- n += 1 + l + sovCommands(uint64(l))
- }
- return n
-}
-
-func (m *FramesPage) Size() (n int) {
- var l int
- _ = l
- l = len(m.Data)
- if l > 0 {
- n += 1 + l + sovCommands(uint64(l))
- }
- if m.Flags != 0 {
- n += 1 + sovCommands(uint64(m.Flags))
- }
- if m.Number != 0 {
- n += 1 + sovCommands(uint64(m.Number))
- }
- return n
-}
-
-func (m *Undo) Size() (n int) {
- var l int
- _ = l
- if m.Txid != 0 {
- n += 1 + sovCommands(uint64(m.Txid))
- }
- return n
-}
-
-func (m *End) Size() (n int) {
- var l int
- _ = l
- if m.Txid != 0 {
- n += 1 + sovCommands(uint64(m.Txid))
- }
- return n
-}
-
-func (m *Checkpoint) Size() (n int) {
- var l int
- _ = l
- l = len(m.Name)
- if l > 0 {
- n += 1 + l + sovCommands(uint64(l))
- }
- return n
-}
-
-func sovCommands(x uint64) (n int) {
- for {
- n++
- x >>= 7
- if x == 0 {
- break
- }
- }
- return n
-}
-func sozCommands(x uint64) (n int) {
- return sovCommands(uint64((x << 1) ^ uint64((int64(x) >> 63))))
-}
-func (m *Command) Unmarshal(dAtA []byte) error {
- l := len(dAtA)
- iNdEx := 0
- for iNdEx < l {
- preIndex := iNdEx
- var wire uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- wire |= (uint64(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- fieldNum := int32(wire >> 3)
- wireType := int(wire & 0x7)
- if wireType == 4 {
- return fmt.Errorf("proto: Command: wiretype end group for non-group")
- }
- if fieldNum <= 0 {
- return fmt.Errorf("proto: Command: illegal tag %d (wire type %d)", fieldNum, wire)
- }
- switch fieldNum {
- case 1:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field Open", wireType)
- }
- var msglen int
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- msglen |= (int(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- if msglen < 0 {
- return ErrInvalidLengthCommands
- }
- postIndex := iNdEx + msglen
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- v := &Open{}
- if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
- return err
- }
- m.Payload = &Command_Open{v}
- iNdEx = postIndex
- case 2:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field Begin", wireType)
- }
- var msglen int
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- msglen |= (int(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- if msglen < 0 {
- return ErrInvalidLengthCommands
- }
- postIndex := iNdEx + msglen
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- v := &Begin{}
- if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
- return err
- }
- m.Payload = &Command_Begin{v}
- iNdEx = postIndex
- case 3:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field Frames", wireType)
- }
- var msglen int
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- msglen |= (int(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- if msglen < 0 {
- return ErrInvalidLengthCommands
- }
- postIndex := iNdEx + msglen
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- v := &Frames{}
- if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
- return err
- }
- m.Payload = &Command_Frames{v}
- iNdEx = postIndex
- case 4:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field Undo", wireType)
- }
- var msglen int
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- msglen |= (int(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- if msglen < 0 {
- return ErrInvalidLengthCommands
- }
- postIndex := iNdEx + msglen
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- v := &Undo{}
- if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
- return err
- }
- m.Payload = &Command_Undo{v}
- iNdEx = postIndex
- case 5:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field End", wireType)
- }
- var msglen int
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- msglen |= (int(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- if msglen < 0 {
- return ErrInvalidLengthCommands
- }
- postIndex := iNdEx + msglen
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- v := &End{}
- if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
- return err
- }
- m.Payload = &Command_End{v}
- iNdEx = postIndex
- case 6:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field Checkpoint", wireType)
- }
- var msglen int
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- msglen |= (int(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- if msglen < 0 {
- return ErrInvalidLengthCommands
- }
- postIndex := iNdEx + msglen
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- v := &Checkpoint{}
- if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
- return err
- }
- m.Payload = &Command_Checkpoint{v}
- iNdEx = postIndex
- default:
- iNdEx = preIndex
- skippy, err := skipCommands(dAtA[iNdEx:])
- if err != nil {
- return err
- }
- if skippy < 0 {
- return ErrInvalidLengthCommands
- }
- if (iNdEx + skippy) > l {
- return io.ErrUnexpectedEOF
- }
- iNdEx += skippy
- }
- }
-
- if iNdEx > l {
- return io.ErrUnexpectedEOF
- }
- return nil
-}
-func (m *Open) Unmarshal(dAtA []byte) error {
- l := len(dAtA)
- iNdEx := 0
- for iNdEx < l {
- preIndex := iNdEx
- var wire uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- wire |= (uint64(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- fieldNum := int32(wire >> 3)
- wireType := int(wire & 0x7)
- if wireType == 4 {
- return fmt.Errorf("proto: Open: wiretype end group for non-group")
- }
- if fieldNum <= 0 {
- return fmt.Errorf("proto: Open: illegal tag %d (wire type %d)", fieldNum, wire)
- }
- switch fieldNum {
- case 1:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType)
- }
- var stringLen uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- stringLen |= (uint64(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- intStringLen := int(stringLen)
- if intStringLen < 0 {
- return ErrInvalidLengthCommands
- }
- postIndex := iNdEx + intStringLen
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- m.Name = string(dAtA[iNdEx:postIndex])
- iNdEx = postIndex
- default:
- iNdEx = preIndex
- skippy, err := skipCommands(dAtA[iNdEx:])
- if err != nil {
- return err
- }
- if skippy < 0 {
- return ErrInvalidLengthCommands
- }
- if (iNdEx + skippy) > l {
- return io.ErrUnexpectedEOF
- }
- iNdEx += skippy
- }
- }
-
- if iNdEx > l {
- return io.ErrUnexpectedEOF
- }
- return nil
-}
-func (m *Begin) Unmarshal(dAtA []byte) error {
- l := len(dAtA)
- iNdEx := 0
- for iNdEx < l {
- preIndex := iNdEx
- var wire uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- wire |= (uint64(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- fieldNum := int32(wire >> 3)
- wireType := int(wire & 0x7)
- if wireType == 4 {
- return fmt.Errorf("proto: Begin: wiretype end group for non-group")
- }
- if fieldNum <= 0 {
- return fmt.Errorf("proto: Begin: illegal tag %d (wire type %d)", fieldNum, wire)
- }
- switch fieldNum {
- case 1:
- if wireType != 0 {
- return fmt.Errorf("proto: wrong wireType = %d for field Txid", wireType)
- }
- m.Txid = 0
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- m.Txid |= (uint64(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- case 2:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType)
- }
- var stringLen uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- stringLen |= (uint64(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- intStringLen := int(stringLen)
- if intStringLen < 0 {
- return ErrInvalidLengthCommands
- }
- postIndex := iNdEx + intStringLen
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- m.Name = string(dAtA[iNdEx:postIndex])
- iNdEx = postIndex
- default:
- iNdEx = preIndex
- skippy, err := skipCommands(dAtA[iNdEx:])
- if err != nil {
- return err
- }
- if skippy < 0 {
- return ErrInvalidLengthCommands
- }
- if (iNdEx + skippy) > l {
- return io.ErrUnexpectedEOF
- }
- iNdEx += skippy
- }
- }
-
- if iNdEx > l {
- return io.ErrUnexpectedEOF
- }
- return nil
-}
-func (m *Frames) Unmarshal(dAtA []byte) error {
- l := len(dAtA)
- iNdEx := 0
- for iNdEx < l {
- preIndex := iNdEx
- var wire uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- wire |= (uint64(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- fieldNum := int32(wire >> 3)
- wireType := int(wire & 0x7)
- if wireType == 4 {
- return fmt.Errorf("proto: Frames: wiretype end group for non-group")
- }
- if fieldNum <= 0 {
- return fmt.Errorf("proto: Frames: illegal tag %d (wire type %d)", fieldNum, wire)
- }
- switch fieldNum {
- case 1:
- if wireType != 0 {
- return fmt.Errorf("proto: wrong wireType = %d for field Txid", wireType)
- }
- m.Txid = 0
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- m.Txid |= (uint64(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- case 2:
- if wireType != 0 {
- return fmt.Errorf("proto: wrong wireType = %d for field PageSize", wireType)
- }
- m.PageSize = 0
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- m.PageSize |= (int32(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- case 3:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field Pages", wireType)
- }
- var msglen int
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- msglen |= (int(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- if msglen < 0 {
- return ErrInvalidLengthCommands
- }
- postIndex := iNdEx + msglen
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- m.Pages = append(m.Pages, &FramesPage{})
- if err := m.Pages[len(m.Pages)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
- return err
- }
- iNdEx = postIndex
- case 4:
- if wireType != 0 {
- return fmt.Errorf("proto: wrong wireType = %d for field Truncate", wireType)
- }
- m.Truncate = 0
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- m.Truncate |= (uint32(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- case 5:
- if wireType != 0 {
- return fmt.Errorf("proto: wrong wireType = %d for field IsCommit", wireType)
- }
- m.IsCommit = 0
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- m.IsCommit |= (int32(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- case 6:
- if wireType != 0 {
- return fmt.Errorf("proto: wrong wireType = %d for field SyncFlags", wireType)
- }
- m.SyncFlags = 0
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- m.SyncFlags |= (uint32(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- case 7:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field Filename", wireType)
- }
- var stringLen uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- stringLen |= (uint64(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- intStringLen := int(stringLen)
- if intStringLen < 0 {
- return ErrInvalidLengthCommands
- }
- postIndex := iNdEx + intStringLen
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- m.Filename = string(dAtA[iNdEx:postIndex])
- iNdEx = postIndex
- case 8:
- if wireType == 0 {
- var v uint32
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- v |= (uint32(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- m.PageNumbers = append(m.PageNumbers, v)
- } else if wireType == 2 {
- var packedLen int
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- packedLen |= (int(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- if packedLen < 0 {
- return ErrInvalidLengthCommands
- }
- postIndex := iNdEx + packedLen
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- for iNdEx < postIndex {
- var v uint32
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- v |= (uint32(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- m.PageNumbers = append(m.PageNumbers, v)
- }
- } else {
- return fmt.Errorf("proto: wrong wireType = %d for field PageNumbers", wireType)
- }
- case 9:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field PageData", wireType)
- }
- var byteLen int
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- byteLen |= (int(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- if byteLen < 0 {
- return ErrInvalidLengthCommands
- }
- postIndex := iNdEx + byteLen
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- m.PageData = append(m.PageData[:0], dAtA[iNdEx:postIndex]...)
- if m.PageData == nil {
- m.PageData = []byte{}
- }
- iNdEx = postIndex
- default:
- iNdEx = preIndex
- skippy, err := skipCommands(dAtA[iNdEx:])
- if err != nil {
- return err
- }
- if skippy < 0 {
- return ErrInvalidLengthCommands
- }
- if (iNdEx + skippy) > l {
- return io.ErrUnexpectedEOF
- }
- iNdEx += skippy
- }
- }
-
- if iNdEx > l {
- return io.ErrUnexpectedEOF
- }
- return nil
-}
-func (m *FramesPage) Unmarshal(dAtA []byte) error {
- l := len(dAtA)
- iNdEx := 0
- for iNdEx < l {
- preIndex := iNdEx
- var wire uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- wire |= (uint64(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- fieldNum := int32(wire >> 3)
- wireType := int(wire & 0x7)
- if wireType == 4 {
- return fmt.Errorf("proto: FramesPage: wiretype end group for non-group")
- }
- if fieldNum <= 0 {
- return fmt.Errorf("proto: FramesPage: illegal tag %d (wire type %d)", fieldNum, wire)
- }
- switch fieldNum {
- case 1:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType)
- }
- var byteLen int
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- byteLen |= (int(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- if byteLen < 0 {
- return ErrInvalidLengthCommands
- }
- postIndex := iNdEx + byteLen
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...)
- if m.Data == nil {
- m.Data = []byte{}
- }
- iNdEx = postIndex
- case 2:
- if wireType != 0 {
- return fmt.Errorf("proto: wrong wireType = %d for field Flags", wireType)
- }
- m.Flags = 0
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- m.Flags |= (uint32(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- case 3:
- if wireType != 0 {
- return fmt.Errorf("proto: wrong wireType = %d for field Number", wireType)
- }
- m.Number = 0
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- m.Number |= (uint32(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- default:
- iNdEx = preIndex
- skippy, err := skipCommands(dAtA[iNdEx:])
- if err != nil {
- return err
- }
- if skippy < 0 {
- return ErrInvalidLengthCommands
- }
- if (iNdEx + skippy) > l {
- return io.ErrUnexpectedEOF
- }
- iNdEx += skippy
- }
- }
-
- if iNdEx > l {
- return io.ErrUnexpectedEOF
- }
- return nil
-}
-func (m *Undo) Unmarshal(dAtA []byte) error {
- l := len(dAtA)
- iNdEx := 0
- for iNdEx < l {
- preIndex := iNdEx
- var wire uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- wire |= (uint64(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- fieldNum := int32(wire >> 3)
- wireType := int(wire & 0x7)
- if wireType == 4 {
- return fmt.Errorf("proto: Undo: wiretype end group for non-group")
- }
- if fieldNum <= 0 {
- return fmt.Errorf("proto: Undo: illegal tag %d (wire type %d)", fieldNum, wire)
- }
- switch fieldNum {
- case 1:
- if wireType != 0 {
- return fmt.Errorf("proto: wrong wireType = %d for field Txid", wireType)
- }
- m.Txid = 0
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- m.Txid |= (uint64(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- default:
- iNdEx = preIndex
- skippy, err := skipCommands(dAtA[iNdEx:])
- if err != nil {
- return err
- }
- if skippy < 0 {
- return ErrInvalidLengthCommands
- }
- if (iNdEx + skippy) > l {
- return io.ErrUnexpectedEOF
- }
- iNdEx += skippy
- }
- }
-
- if iNdEx > l {
- return io.ErrUnexpectedEOF
- }
- return nil
-}
-func (m *End) Unmarshal(dAtA []byte) error {
- l := len(dAtA)
- iNdEx := 0
- for iNdEx < l {
- preIndex := iNdEx
- var wire uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- wire |= (uint64(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- fieldNum := int32(wire >> 3)
- wireType := int(wire & 0x7)
- if wireType == 4 {
- return fmt.Errorf("proto: End: wiretype end group for non-group")
- }
- if fieldNum <= 0 {
- return fmt.Errorf("proto: End: illegal tag %d (wire type %d)", fieldNum, wire)
- }
- switch fieldNum {
- case 1:
- if wireType != 0 {
- return fmt.Errorf("proto: wrong wireType = %d for field Txid", wireType)
- }
- m.Txid = 0
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- m.Txid |= (uint64(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- default:
- iNdEx = preIndex
- skippy, err := skipCommands(dAtA[iNdEx:])
- if err != nil {
- return err
- }
- if skippy < 0 {
- return ErrInvalidLengthCommands
- }
- if (iNdEx + skippy) > l {
- return io.ErrUnexpectedEOF
- }
- iNdEx += skippy
- }
- }
-
- if iNdEx > l {
- return io.ErrUnexpectedEOF
- }
- return nil
-}
-func (m *Checkpoint) Unmarshal(dAtA []byte) error {
- l := len(dAtA)
- iNdEx := 0
- for iNdEx < l {
- preIndex := iNdEx
- var wire uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- wire |= (uint64(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- fieldNum := int32(wire >> 3)
- wireType := int(wire & 0x7)
- if wireType == 4 {
- return fmt.Errorf("proto: Checkpoint: wiretype end group for non-group")
- }
- if fieldNum <= 0 {
- return fmt.Errorf("proto: Checkpoint: illegal tag %d (wire type %d)", fieldNum, wire)
- }
- switch fieldNum {
- case 1:
- if wireType != 2 {
- return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType)
- }
- var stringLen uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- stringLen |= (uint64(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- intStringLen := int(stringLen)
- if intStringLen < 0 {
- return ErrInvalidLengthCommands
- }
- postIndex := iNdEx + intStringLen
- if postIndex > l {
- return io.ErrUnexpectedEOF
- }
- m.Name = string(dAtA[iNdEx:postIndex])
- iNdEx = postIndex
- default:
- iNdEx = preIndex
- skippy, err := skipCommands(dAtA[iNdEx:])
- if err != nil {
- return err
- }
- if skippy < 0 {
- return ErrInvalidLengthCommands
- }
- if (iNdEx + skippy) > l {
- return io.ErrUnexpectedEOF
- }
- iNdEx += skippy
- }
- }
-
- if iNdEx > l {
- return io.ErrUnexpectedEOF
- }
- return nil
-}
-func skipCommands(dAtA []byte) (n int, err error) {
- l := len(dAtA)
- iNdEx := 0
- for iNdEx < l {
- var wire uint64
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return 0, ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return 0, io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- wire |= (uint64(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- wireType := int(wire & 0x7)
- switch wireType {
- case 0:
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return 0, ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return 0, io.ErrUnexpectedEOF
- }
- iNdEx++
- if dAtA[iNdEx-1] < 0x80 {
- break
- }
- }
- return iNdEx, nil
- case 1:
- iNdEx += 8
- return iNdEx, nil
- case 2:
- var length int
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return 0, ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return 0, io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- length |= (int(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- iNdEx += length
- if length < 0 {
- return 0, ErrInvalidLengthCommands
- }
- return iNdEx, nil
- case 3:
- for {
- var innerWire uint64
- var start int = iNdEx
- for shift := uint(0); ; shift += 7 {
- if shift >= 64 {
- return 0, ErrIntOverflowCommands
- }
- if iNdEx >= l {
- return 0, io.ErrUnexpectedEOF
- }
- b := dAtA[iNdEx]
- iNdEx++
- innerWire |= (uint64(b) & 0x7F) << shift
- if b < 0x80 {
- break
- }
- }
- innerWireType := int(innerWire & 0x7)
- if innerWireType == 4 {
- break
- }
- next, err := skipCommands(dAtA[start:])
- if err != nil {
- return 0, err
- }
- iNdEx = start + next
- }
- return iNdEx, nil
- case 4:
- return iNdEx, nil
- case 5:
- iNdEx += 4
- return iNdEx, nil
- default:
- return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
- }
- }
- panic("unreachable")
-}
-
-var (
- ErrInvalidLengthCommands = fmt.Errorf("proto: negative length found during unmarshaling")
- ErrIntOverflowCommands = fmt.Errorf("proto: integer overflow")
-)
-
-func init() { proto.RegisterFile("internal/protocol/commands.proto", fileDescriptorCommands) }
-
-var fileDescriptorCommands = []byte{
- // 478 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x52, 0xc1, 0x6e, 0xd3, 0x40,
- 0x10, 0x8d, 0x13, 0xdb, 0xb1, 0xa7, 0x31, 0xa0, 0x55, 0x85, 0x4c, 0x11, 0x91, 0x6b, 0x21, 0x11,
- 0xf5, 0x90, 0x48, 0x20, 0xf1, 0x01, 0x09, 0xad, 0x7c, 0x2a, 0xd5, 0x22, 0xce, 0xd1, 0xc6, 0xde,
- 0x84, 0x15, 0xf6, 0xda, 0xb2, 0x1d, 0x89, 0xf6, 0x2b, 0xb8, 0xf2, 0x47, 0x1c, 0xf9, 0x04, 0x94,
- 0x2f, 0x41, 0x33, 0xdb, 0xda, 0xa5, 0xca, 0x6d, 0xe7, 0xbd, 0x37, 0x3b, 0xfb, 0xde, 0x0e, 0x44,
- 0x4a, 0xb7, 0xb2, 0xd6, 0x22, 0x5f, 0x54, 0x75, 0xd9, 0x96, 0x69, 0x99, 0x2f, 0xd2, 0xb2, 0x28,
- 0x84, 0xce, 0x9a, 0x39, 0x21, 0xcc, 0x7b, 0x20, 0xe2, 0x9f, 0x43, 0x18, 0xaf, 0x0c, 0xc9, 0xde,
- 0x82, 0x5d, 0x56, 0x52, 0x87, 0x56, 0x64, 0xcd, 0x4e, 0xde, 0x3f, 0x9b, 0x3f, 0x88, 0xe6, 0x9f,
- 0x2b, 0xa9, 0x93, 0x01, 0x27, 0x96, 0xbd, 0x03, 0x67, 0x23, 0x77, 0x4a, 0x87, 0x43, 0x92, 0x3d,
- 0xef, 0x65, 0x4b, 0x84, 0x93, 0x01, 0x37, 0x3c, 0xbb, 0x00, 0x77, 0x5b, 0x8b, 0x42, 0x36, 0xe1,
- 0x88, 0x94, 0x2f, 0x7a, 0xe5, 0x15, 0xe1, 0xc9, 0x80, 0xdf, 0x2b, 0x70, 0xf4, 0x5e, 0x67, 0x65,
- 0x68, 0x3f, 0x1d, 0xfd, 0x55, 0x67, 0x25, 0x8e, 0x46, 0x96, 0x9d, 0xc3, 0x48, 0xea, 0x2c, 0x74,
- 0x48, 0x14, 0xf4, 0xa2, 0x4b, 0x9d, 0x25, 0x03, 0x8e, 0x1c, 0xfb, 0x08, 0x90, 0x7e, 0x93, 0xe9,
- 0xf7, 0xaa, 0x54, 0xba, 0x0d, 0x5d, 0x52, 0x9e, 0xf6, 0xca, 0x55, 0xc7, 0x25, 0x03, 0xfe, 0x48,
- 0xb9, 0xf4, 0x61, 0x7c, 0x23, 0x6e, 0xf3, 0x52, 0x64, 0xf1, 0x19, 0xd8, 0x68, 0x98, 0x31, 0xb0,
- 0xb5, 0x28, 0x24, 0xc5, 0xe1, 0x73, 0x3a, 0xc7, 0x0b, 0x70, 0xc8, 0x25, 0x92, 0xed, 0x0f, 0x95,
- 0x11, 0x69, 0x73, 0x3a, 0x77, 0x0d, 0xc3, 0x47, 0x0d, 0xbf, 0x86, 0xe0, 0x1a, 0xb7, 0x47, 0x5b,
- 0x5e, 0x83, 0x5f, 0x89, 0x9d, 0x5c, 0x37, 0xea, 0xce, 0xf4, 0x39, 0xdc, 0x43, 0xe0, 0x8b, 0xba,
- 0x93, 0xec, 0x02, 0x1c, 0x3c, 0x63, 0x7e, 0xa3, 0xff, 0x6d, 0x98, 0x1b, 0x6f, 0xc4, 0x4e, 0x72,
- 0x23, 0x61, 0x67, 0xe0, 0xb5, 0xf5, 0x5e, 0xa7, 0xa2, 0x95, 0x14, 0x62, 0xc0, 0xbb, 0x1a, 0x87,
- 0xa8, 0x66, 0x8d, 0x2b, 0xa0, 0x5a, 0x0a, 0xcf, 0xe1, 0x9e, 0x6a, 0x56, 0x54, 0xb3, 0x37, 0x00,
- 0xcd, 0xad, 0x4e, 0xd7, 0xdb, 0x5c, 0xec, 0x1a, 0x0a, 0x2c, 0xe0, 0x3e, 0x22, 0x57, 0x08, 0xe0,
- 0xbd, 0x5b, 0x95, 0x4b, 0xf2, 0x35, 0x26, 0x5f, 0x5d, 0xcd, 0xce, 0x61, 0x42, 0x8f, 0xd7, 0xfb,
- 0x62, 0x23, 0xeb, 0x26, 0xf4, 0xa2, 0xd1, 0x2c, 0xe0, 0x27, 0x88, 0x5d, 0x1b, 0xa8, 0xf3, 0x97,
- 0x89, 0x56, 0x84, 0x7e, 0x64, 0xcd, 0x26, 0xc6, 0xdf, 0x27, 0xd1, 0x8a, 0xf8, 0x1a, 0xa0, 0x37,
- 0x82, 0xf1, 0x90, 0xca, 0x22, 0x15, 0x9d, 0xd9, 0x29, 0x38, 0xe6, 0x5d, 0x43, 0x7a, 0x97, 0x29,
- 0xd8, 0x4b, 0x70, 0xcd, 0x48, 0x5a, 0xac, 0x80, 0xdf, 0x57, 0xf8, 0x71, 0xb8, 0x2e, 0xc7, 0x82,
- 0x8e, 0x5f, 0xc1, 0xe8, 0x52, 0x67, 0x47, 0xa9, 0x08, 0xa0, 0x5f, 0x8b, 0x63, 0xbf, 0xbe, 0x9c,
- 0xfc, 0x3e, 0x4c, 0xad, 0x3f, 0x87, 0xa9, 0xf5, 0xf7, 0x30, 0xb5, 0x36, 0x2e, 0x7d, 0xc3, 0x87,
- 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x7a, 0x3c, 0x28, 0xf7, 0x67, 0x03, 0x00, 0x00,
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/protocol/commands.proto b/vendor/github.com/CanonicalLtd/go-dqlite/internal/protocol/commands.proto
deleted file mode 100644
index d3cde8c564..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/protocol/commands.proto
+++ /dev/null
@@ -1,87 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-syntax = "proto3";
-
-package protocol;
-
-// Command encapsulates the payload fo a dqlite Raft FSM command.
-//
-// On the wire this will be a varint indentifying the command type,
-// followed by the command payload.
-message Command {
- oneof Payload {
- Open open = 1;
- Begin begin = 2;
- Frames frames = 3;
- Undo undo = 4;
- End end = 5;
- Checkpoint checkpoint = 6;
- }
-}
-
-// Parameters to open a new database and setup the needed data
-// structures for replication.
-message Open {
- string name = 1; // Name of the database file.
-}
-
-// Parameters to begin a new write transaction.
-//
-// This command is not used anymore, but it's
-// kept for backward-compatibility.
-message Begin {
- uint64 txid = 1; // Transaction identifier.
- string name = 2; // Name of the database file.
-}
-
-// Parameters to append new frames to the WAL within a write transaction.
-message Frames {
- uint64 txid = 1; // Transaction identifier.
- int32 page_size = 2; // Size of the data field of each frame.
- repeated FramesPage pages = 3; // List of the frames to write (legacy from v1).
- uint32 truncate = 4; // Flag telling if the WAL should be truncated.
- int32 is_commit = 5; // Flag telling if this is a final frames write.
- uint32 sync_flags = 6; // Flags for disk syncing (legacy from v1).
- string filename = 7; // Filename of the database file.
- repeated uint32 page_numbers = 8; // Page numbers of the frames to write (v2).
- bytes page_data = 9; // Contiguous page data (v2).
-}
-
-// A single frame of data in a Frames command.
-//
-// FIXME: this is a legacy message only used in v1 of the protocol.
-message FramesPage {
- bytes data = 1; // Frame data.
- uint32 flags = 2; // WAL write flags.
- uint32 number = 3; // Page number (i.e. position in the database file).
-}
-
-// Parameters to undo any previous WAL change in a write transaction.
-message Undo {
- uint64 txid = 1; // Transaction identifier.
-}
-
-// Parameters to end a write transaction, and update the WAL commit
-// pointer.
-//
-// This command is not used anymore, but it's
-// kept for backward-compatibility.
-message End {
- uint64 txid = 1; // Transaction identifier.
-}
-
-// Parameters to perform a WAL checkpoint.
-message Checkpoint {
- string name = 1; // Name of the database file.
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/protocol/doc.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/protocol/doc.go
deleted file mode 100644
index 3d4fc5404b..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/protocol/doc.go
+++ /dev/null
@@ -1,17 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// package protocol implements serialization and deserialization logic for
-// dqlite-specific Raft commands, that will be exeuted by the dqlite's FSM.
-package protocol
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/conn.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/conn.go
deleted file mode 100644
index acc3fd1cbe..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/conn.go
+++ /dev/null
@@ -1,154 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package registry
-
-import (
- "fmt"
- "sync/atomic"
-
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
-)
-
-// ConnLeaderAdd adds a new leader connection to the registry.
-func (r *Registry) ConnLeaderAdd(filename string, conn *bindings.Conn) {
- r.connAdd(conn)
- r.leaders[conn] = filename
-
- // Add a tracer specific to this connection. It will be used by
- // replication.Methods when firing replication hooks triggered by WAL
- // events on this connection.
- r.tracers.Add(fmt.Sprintf("methods %d", r.ConnSerial(conn)))
-}
-
-// ConnLeaderDel removes the given leader connection from the registry.
-func (r *Registry) ConnLeaderDel(conn *bindings.Conn) {
- // Dell the connection-specific tracer.
- r.tracers.Del(fmt.Sprintf("methods %d", r.ConnSerial(conn)))
-
- r.connDel(conn)
- delete(r.leaders, conn)
-
-}
-
-// ConnLeaderFilename returns the filename of the database associated with the
-// given leader connection.
-//
-// If conn is not a registered leader connection, this method will panic.
-func (r *Registry) ConnLeaderFilename(conn *bindings.Conn) string {
- name, ok := r.leaders[conn]
- if !ok {
- panic("no database for the given connection")
- }
- return name
-}
-
-// ConnLeaders returns all open leader connections for the database with
-// the given filename.
-func (r *Registry) ConnLeaders(filename string) []*bindings.Conn {
- conns := []*bindings.Conn{}
- for conn := range r.leaders {
- if r.leaders[conn] == filename {
- conns = append(conns, conn)
- }
- }
- return conns
-}
-
-// ConnFollowerAdd adds a new follower connection to the registry.
-//
-// If a follower connection for the database with the given filename is already
-// registered, this method panics.
-func (r *Registry) ConnFollowerAdd(filename string, conn *bindings.Conn) {
- r.connAdd(conn)
- r.followers[filename] = conn
-}
-
-// ConnFollowerDel removes the follower registered against the database with the
-// given filename.
-func (r *Registry) ConnFollowerDel(filename string) {
- conn, ok := r.followers[filename]
- if !ok {
- panic(fmt.Sprintf("follower connection for '%s' is not registered", filename))
- }
-
- delete(r.followers, filename)
- r.connDel(conn)
-}
-
-// ConnFollowerFilenames returns the filenames for all databases which currently
-// have registered follower connections.
-func (r *Registry) ConnFollowerFilenames() []string {
- names := []string{}
- for name := range r.followers {
- names = append(names, name)
- }
- return names
-}
-
-// ConnFollower returns the follower connection used to replicate the
-// database identified by the given filename.
-//
-// If there's no follower connection registered for the database with the given
-// filename, this method panics.
-func (r *Registry) ConnFollower(filename string) *bindings.Conn {
- conn, ok := r.followers[filename]
- if !ok {
- panic(fmt.Sprintf("no follower connection for '%s'", filename))
- }
- return conn
-}
-
-// ConnFollowerExists checks whether the registry has a follower connection registered
-// against the database with the given filename.
-func (r *Registry) ConnFollowerExists(filename string) bool {
- _, ok := r.followers[filename]
- return ok
-}
-
-// ConnSerial returns a serial number uniquely identifying the given registered
-// connection.
-func (r *Registry) ConnSerial(conn *bindings.Conn) uint64 {
- serial, ok := r.serial[conn]
-
- if !ok {
- panic("connection is not registered")
- }
-
- return serial
-}
-
-// Add a new connection (either leader or follower) to the registry and assign
-// it a serial number.
-func (r *Registry) connAdd(conn *bindings.Conn) {
- if serial, ok := r.serial[conn]; ok {
- panic(fmt.Sprintf("connection is already registered with serial %d", serial))
- }
-
- atomic.AddUint64(&serial, 1)
- r.serial[conn] = serial
-}
-
-// Delete a connection (either leader or follower) from the registry
-func (r *Registry) connDel(conn *bindings.Conn) {
- if _, ok := r.serial[conn]; !ok {
- panic("connection is not registered")
- }
-
- delete(r.serial, conn)
-}
-
-// Monotonic counter for identifying connections for tracing and debugging
-// purposes.
-var serial uint64
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/fsm.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/fsm.go
deleted file mode 100644
index df9a559d66..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/fsm.go
+++ /dev/null
@@ -1,29 +0,0 @@
-package registry
-
-// Index returns the last Raft log index that was successfully applied the FSM.
-func (r *Registry) Index() uint64 {
- return r.index
-}
-
-// IndexUpdate updates the index of the last log applied by the FSM we're
-// associated with.
-func (r *Registry) IndexUpdate(index uint64) {
- r.index = index
-}
-
-// Frames returns the number of frames that have been written to the WAL so
-// far.
-func (r *Registry) Frames() uint64 {
- return r.frames
-}
-
-// FramesIncrease increases by the given amount the number of frames written to
-// the WAL so far.
-func (r *Registry) FramesIncrease(n uint64) {
- r.frames += n
-}
-
-// FramesReset resets the WAL frames counter to zero.
-func (r *Registry) FramesReset() {
- r.frames = 0
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/hook.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/hook.go
deleted file mode 100644
index 4c464cd437..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/hook.go
+++ /dev/null
@@ -1,191 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package registry
-
-import (
- "reflect"
- "sync"
-)
-
-// HookSyncSet creates a new hookSync instance associated with this Registry.
-func (r *Registry) HookSyncSet() {
- r.hookSyncEnsureUnset()
- r.hookSync = newHookSync()
-}
-
-// HookSyncAdd adds a new log command data to the underlying hookSync, which is
-// expected to match the Log.Data bytes received next FSM.Apply() call.
-func (r *Registry) HookSyncAdd(data []byte) {
- r.hookSyncEnsureSet()
- r.hookSync.Add(data)
-}
-
-// HookSyncPresent checks whether a hook sync was set by methods hook.
-func (r *Registry) HookSyncPresent() bool {
- return r.hookSync != nil
-}
-
-// HookSyncMatches checks whether the Log.Data bytes that an FSM.Apply() call
-// is about to process match the ones that were last added to the hookSync via
-// HookSyncAdd().
-func (r *Registry) HookSyncMatches(data []byte) bool {
- r.hookSyncEnsureSet()
- return r.hookSync.Matches(data)
-}
-
-// HookSyncWait blocks until the underlying hookSync is done.
-//
-// It assumes that the lock is held, releasing it before blocking and requiring
-// it thereafter.
-func (r *Registry) HookSyncWait() {
- r.hookSyncEnsureSet()
- hookSync := r.hookSync
- r.Unlock()
- hookSync.Wait()
- r.Lock()
-}
-
-// HookSyncReset clears the hookSync instance created with HookSyncSet.
-func (r *Registry) HookSyncReset() {
- r.hookSyncEnsureSet()
- r.hookSync.Done() // Unblock any FSM.Apply() call waiting on this hookSync.
- r.hookSync = nil
-}
-
-// Ensure that a hookSync instance is set.
-func (r *Registry) hookSyncEnsureSet() {
- if r.hookSync == nil {
- panic("no hookSync instance set on this registry")
- }
-}
-
-// Ensure that a hookSync instance is not set.
-func (r *Registry) hookSyncEnsureUnset() {
- if r.hookSync != nil {
- panic("a hookSync instance is set on this registry")
- }
-}
-
-// HookSync is used to synchronize a Methods instance and an FSM instance
-// between each other.
-//
-// The goal is that if a replication hook of Methods instance is in progress,
-// the associated FSM instance should only execute log commands applied by that
-// hook, and block the execution of any log command not applied by the hook
-// until the hook returns.
-//
-// The semantics of HookSync is somewhat similar to sync.WaitGroup, and indeed
-// it uses WaitGroup internally. The only additional behavior is really the
-// additional API that checks if a certain command log that the FSM is about to
-// apply was originated by an Apply() call on the same server during a
-// concurrent Methods hook call, or if's a replicated command log that was sent
-// by another server over the network, perhaps right while a Methods hook is
-// finishing up after leadership was lost.
-//
-// The synchronization protocol goes through the following steps:
-//
-// - The Methods instance starts executing a replication hook.
-//
-// - The Methods instance acquires the the Registry lock and creates a new
-// HookSync instance.
-//
-// - Whenever the Methods instance is about to apply a log command, it calls
-// HookSync.Add(), which saves the reference to the data bytes slice to be
-// applied in HookSync.data, and increases by one the the WaitGroup count on
-// HookSync.wg. The Methods instance then releases the Registry lock.
-//
-// - The FSM starts executing a log command.
-//
-// - The FSM acquires the Registry lock and check if a HookSync instance
-// is set.
-//
-// - If no HookSync instance is set, the FSM continues normally. This is the
-// typical case when the FSM is applying logs as follower.
-//
-// - If the HookSync instance is set and HookSync.Matches() returns true, then
-// the HookSync.data field matches the Log.Data field of the log command
-// being applied, and the FSM continues normally.
-//
-// - If the HookSync instance is set and HookSync.Matches() returns false,
-// then the HookSync.data field does not match the Log.Data field of the log
-// command being applied. This means that the FSM is about to apply a log
-// command that did not originate on this node during the hook execution
-// (e.g. the FSM is about to apply a log sent from a new leader after this
-// leader was deposed). The FSM releases the lock on the Registry and calls
-// HookSync.Wait() which tries to acquire the HookSync.mu lock (which is
-// being held by the Methods instance running the replication hook).
-//
-// - When control eventually returns to the Methods instance after the
-// Raft.Apply() call returns, the Methods instance re-acquires the Registry
-// lock and resumes the execution of the replication hook. When the hook is
-// about to finish, the Methods instance calls HookSync.Done(), which
-// releases all the hookSync.mu reader locks previously acquired. Finally,
-// the Methods instance releases the Registry lock.
-//
-// - If the FSM was blocked on HookSync.Wait(), it's now free to proceed.
-//
-// See also Methods.Begin, Methods.Frames, Methods.Undo and FSM.Apply for
-// details.
-type hookSync struct {
- // A Methods instance hook must call Add(1) aginst this wait group each
- // time it applies a log command.
- wg sync.WaitGroup
-
- // Track the number of Add(1) calls agaist the WaitGroup.
- n int
-
- // Reference to the Log.Data payload of the last log command applied by
- // a Methods hook running on this server.
- data []byte
-}
-
-func newHookSync() *hookSync {
- return &hookSync{}
-}
-
-// Add is invoked by a Methods instance before calling Raft.Apply(). It
-// sets the data beying applied and increases the number of lock readers by
-// one.
-//
-// This can be called multiple times by a Methods instance during the execution
-// of a replication hook.
-func (s *hookSync) Add(data []byte) {
- s.wg.Add(1)
- s.n++
- s.data = data
-}
-
-// Matches returns true if the data referenced by this HookSync matches the
-// one of the given raft.Log.Data.
-//
-// This assumes that the hashicorp/raft package does not make a copy of the
-// data slice when invoking FSM.Apply() to apply a log command that originated
-// from a call to Raft.Apply() on this server.
-func (s *hookSync) Matches(data []byte) bool {
- return reflect.ValueOf(s.data).Pointer() == reflect.ValueOf(data).Pointer()
-}
-
-// Wait blocks until our mutex has no more readers, i.e. the replication hook
-// that created us has completed.
-func (s *hookSync) Wait() {
- s.wg.Wait()
-}
-
-// Done releases all reader locks acquired during our life cycle.
-func (s *hookSync) Done() {
- for i := 0; i < s.n; i++ {
- s.wg.Done()
- }
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/registry.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/registry.go
deleted file mode 100644
index a88b75d728..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/registry.go
+++ /dev/null
@@ -1,149 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package registry
-
-import (
- "bytes"
- "fmt"
- "sync"
- "testing"
-
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
- "github.com/CanonicalLtd/go-dqlite/internal/trace"
- "github.com/CanonicalLtd/go-dqlite/internal/transaction"
-)
-
-// Registry is a dqlite node-level data structure that tracks:
-//
-// - The directory where dqlite data for this node lives.
-//
-// - All SQLite connections opened on the node, either in leader replication
-// mode or follower replication mode.
-//
-// - All inflight WAL write transactions, either for leader or follower
-// connections.
-//
-// - All tracers used to emit trace messages.
-//
-// - Last log index applied by the FSM.
-//
-// A single Registry instance is shared by a single replication.FSM instance, a
-// single replication.Methods instance and a single dqlite.Driver instance.
-//
-// Methods that access or mutate the registry are not thread-safe and must be
-// performed after acquiring the lock. See Lock() and Unlock().
-type Registry struct {
- mu sync.Mutex // Serialize access to internal state.
- vfs *bindings.Vfs // In-memory file-system
- leaders map[*bindings.Conn]string // Map leader connections to database filenames.
- followers map[string]*bindings.Conn // Map database filenames to follower connections.
- txns map[uint64]*transaction.Txn // Transactions by ID
- tracers *trace.Set // Tracers used by this dqlite instance.
- index uint64 // Last log index applied by the dqlite FSM.
- frames uint64 // Number of frames written to the WAL so far.
- hookSync *hookSync // Used for synchronizing Methods and FSM.
-
- // Map a connection to its serial number. Serial numbers are guaranteed
- // to be unique inside the same process.
- serial map[*bindings.Conn]uint64
-
- // Circular buffer holding the IDs of the last N transactions that
- // where successfully committed. It is used to recover a transaction
- // that errored because of lost leadership but that might actually get
- // completed because a quorum was reached for the lost commit frames
- // command log.
- committed []uint64
- committedCursor int
-
- // Map a leader connection to the ID of the last transaction executed
- // on it. Used by the driver's Tx implementation to know its ID in case
- // a client asks for it for recovering a lost commit.
- lastTxnIDs map[*bindings.Conn]uint64
-
- // Flag indicating whether transactions state transitions
- // should actually callback the relevant SQLite APIs. Some
- // tests need set this flag to true because there's no public
- // API to acquire the WAL read lock in leader connections.
- txnDryRun bool
-}
-
-// New creates a new registry.
-//
-// The 'dir' parameter sets the directory where the node associated with this
-// registry will save the SQLite database files.
-func New(vfs *bindings.Vfs) *Registry {
- tracers := trace.NewSet(250)
-
- // Register the is the tracer that will be used by the FSM associated
- // with this registry.
- tracers.Add("fsm")
-
- return &Registry{
- vfs: vfs,
- leaders: map[*bindings.Conn]string{},
- followers: map[string]*bindings.Conn{},
- txns: map[uint64]*transaction.Txn{},
- tracers: tracers,
- serial: map[*bindings.Conn]uint64{},
- committed: make([]uint64, committedBufferSize),
- lastTxnIDs: make(map[*bindings.Conn]uint64),
- }
-}
-
-// Lock the registry.
-func (r *Registry) Lock() {
- r.mu.Lock()
-}
-
-// Unlock the registry.
-func (r *Registry) Unlock() {
- r.mu.Unlock()
-}
-
-// Vfs is in-memory VFS used for SQLite databases.
-func (r *Registry) Vfs() *bindings.Vfs {
- return r.vfs
-}
-
-// Testing sets up this registry for unit-testing.
-//
-// The tracers will forward all entries to the testing logger, using the given
-// node prefix.
-func (r *Registry) Testing(t *testing.T, node int) {
- r.tracers.Testing(t, node)
-}
-
-// Dump the content of the registry, useful for debugging.
-func (r *Registry) Dump() string {
- buffer := bytes.NewBuffer(nil)
- fmt.Fprintf(buffer, "leaders:\n")
- for conn, name := range r.leaders {
- fmt.Fprintf(buffer, "-> %d: %s\n", r.ConnSerial(conn), name)
- }
- fmt.Fprintf(buffer, "followers:\n")
- for name, conn := range r.followers {
- fmt.Fprintf(buffer, "-> %d: %s\n", r.ConnSerial(conn), name)
- }
- fmt.Fprintf(buffer, "transactions:\n")
- for _, txn := range r.txns {
- fmt.Fprintf(buffer, "-> %s\n", txn)
- }
- return buffer.String()
-}
-
-// Keep track of at most this much comitted transactions. This number should be
-// large enough for any real-world situation, where it's unlikely that a client
-// tries to recover a transaction that is so old.
-const committedBufferSize = 10000
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/trace.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/trace.go
deleted file mode 100644
index db7527ffe1..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/trace.go
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package registry
-
-import (
- "fmt"
-
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
- "github.com/CanonicalLtd/go-dqlite/internal/trace"
-)
-
-// TracerFSM returns the tracer that should be used by the replication.FSM
-// instance associated with this registry.
-func (r *Registry) TracerFSM() *trace.Tracer {
- return r.tracers.Get("fsm")
-}
-
-// TracerConn returns the tracer that should be used by the replication.Methods
-// instance associated with this registry when running the given hook for the
-// given connection, which is assumed to be a registered leader connection.
-func (r *Registry) TracerConn(conn *bindings.Conn, hook string) *trace.Tracer {
- tracer := r.tracers.Get(fmt.Sprintf("methods %d", r.ConnSerial(conn)))
- return tracer.With(trace.String("hook", hook))
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/txn.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/txn.go
deleted file mode 100644
index 68929193b9..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/registry/txn.go
+++ /dev/null
@@ -1,218 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package registry
-
-import (
- "fmt"
-
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
- "github.com/CanonicalLtd/go-dqlite/internal/transaction"
-)
-
-// TxnLeaderAdd adds a new transaction to the registry.
-//
-// The given connection is assumed to be in leader replication mode.
-func (r *Registry) TxnLeaderAdd(conn *bindings.Conn, id uint64) *transaction.Txn {
- // Check that no other leader connection is registered for the same
- // filename.
- filename := r.ConnLeaderFilename(conn)
- for other := range r.leaders {
- if other != conn && r.leaders[other] == filename {
- if txn := r.TxnByConn(other); txn != nil {
- serial := r.ConnSerial(other)
- panic(fmt.Sprintf("transaction %s registered on connection %d", txn, serial))
- }
- }
- }
-
- // Keep track of the ID of the last transaction executed on this
- // connection.
- r.lastTxnIDs[conn] = id
-
- return r.txnAdd(conn, id, true)
-}
-
-// TxnLeaderByFilename returns the leader transaction associated with the given
-// database filename, if any.
-//
-// If there is more than one leader transaction for the same filename, this
-// method panics.
-func (r *Registry) TxnLeaderByFilename(filename string) *transaction.Txn {
- var found *transaction.Txn
- for _, txn := range r.txns {
- if r.leaders[txn.Conn()] == filename {
- if found != nil {
- panic("found more than one leader transaction for this database")
- }
- found = txn
- }
- }
- return found
-}
-
-// TxnFollowerAdd adds a new follower transaction to the registry.
-//
-// The given connection is assumed to be in follower replication mode. The new
-// transaction will be associated with the given transaction ID, which should
-// match the one of the leader transaction that initiated the write.
-func (r *Registry) TxnFollowerAdd(conn *bindings.Conn, id uint64) *transaction.Txn {
- return r.txnAdd(conn, id, false)
-}
-
-// TxnFollowerSurrogate creates a surrogate follower transaction.
-//
-// Surrogate follower transactions are used to replace leader transactions when
-// a node loses leadership and are supposed to be undone by the next leader.
-func (r *Registry) TxnFollowerSurrogate(txn *transaction.Txn) *transaction.Txn {
- if !txn.IsLeader() {
- panic("expected leader transaction")
- }
- r.TxnDel(txn.ID())
- filename := r.ConnLeaderFilename(txn.Conn())
- conn := r.ConnFollower(filename)
- txn = r.TxnFollowerAdd(conn, txn.ID())
- txn.DryRun()
-
- return txn
-}
-
-// TxnFollowerResurrected registers a follower transaction created by
-// resurrecting a zombie leader transaction.
-func (r *Registry) TxnFollowerResurrected(txn *transaction.Txn) {
- if txn.IsLeader() {
- panic("expected follower transaction")
- }
-
- // Delete the zombie leader transaction, which has the same ID.
- r.TxnDel(txn.ID())
-
- // Register the new follower transaction.
- r.txnAdd(txn.Conn(), txn.ID(), false)
-}
-
-// TxnDel deletes the transaction with the given ID.
-func (r *Registry) TxnDel(id uint64) {
- if _, ok := r.txns[id]; !ok {
- panic(fmt.Sprintf("attempt to remove unregistered transaction %d", id))
- }
-
- delete(r.txns, id)
-}
-
-// TxnByID returns the transaction with the given ID, if it exists.
-func (r *Registry) TxnByID(id uint64) *transaction.Txn {
- txn, _ := r.txns[id]
- return txn
-}
-
-// TxnByConn returns the transaction associated with the given connection, if
-// any.
-func (r *Registry) TxnByConn(conn *bindings.Conn) *transaction.Txn {
- for _, txn := range r.txns {
- if txn.Conn() == conn {
- return txn
- }
- }
- return nil
-}
-
-// TxnByFilename returns the transaction associated with the given database
-// filename, if any.
-//
-// If there is more than one transaction for the same filename, this method
-// panics.
-func (r *Registry) TxnByFilename(filename string) *transaction.Txn {
- conns := make([]*bindings.Conn, 0)
-
- if conn, ok := r.followers[filename]; ok {
- conns = append(conns, conn)
- }
-
- for conn := range r.leaders {
- if r.leaders[conn] == filename {
- conns = append(conns, conn)
- }
- }
-
- var found *transaction.Txn
- for _, conn := range conns {
- if txn := r.TxnByConn(conn); txn != nil {
- if found == nil {
- found = txn
- } else {
- panic("found more than one transaction for this database")
- }
- }
- }
-
- return found
-}
-
-// TxnDryRun makes transactions only transition between states, without
-// actually invoking the relevant SQLite APIs. This is used by tests and by
-// surrogate followers.
-func (r *Registry) TxnDryRun() {
- r.txnDryRun = true
-}
-
-// TxnLastID returns the ID of the last transaction executed on the given
-// leader connection.
-func (r *Registry) TxnLastID(conn *bindings.Conn) uint64 {
- return r.lastTxnIDs[conn]
-}
-
-// TxnCommittedAdd saves the ID of the given transaction in the committed buffer,
-// in case a client needs to check if it can be recovered.
-func (r *Registry) TxnCommittedAdd(txn *transaction.Txn) {
- r.committed[r.committedCursor] = txn.ID()
- r.committedCursor++
- if r.committedCursor == len(r.committed) {
- // Rollover
- r.committedCursor = 0
- }
-}
-
-// TxnCommittedFind scans the comitted buffer and returns true if the given ID
-// is present.
-func (r *Registry) TxnCommittedFind(id uint64) bool {
- for i := range r.committed {
- if r.committed[i] == id {
- return true
- }
- }
- return false
-}
-
-func (r *Registry) txnAdd(conn *bindings.Conn, id uint64, isLeader bool) *transaction.Txn {
- // Sanity check that a transaction for the same connection hasn't been
- // registered already. Iterating is fast since there will always be few
- // write transactions active at given time.
- if txn := r.TxnByConn(conn); txn != nil {
- panic(fmt.Sprintf(
- "a transaction for this connection is already registered with ID %d", txn.ID()))
- }
-
- txn := transaction.New(conn, id)
-
- if isLeader {
- txn.Leader()
- } else if r.txnDryRun {
- txn.DryRun()
- }
-
- r.txns[id] = txn
-
- return txn
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/replication/doc.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/replication/doc.go
deleted file mode 100644
index 29838ef0a8..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/replication/doc.go
+++ /dev/null
@@ -1,17 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package replication implements the core part of dqlite, setting up
-// raft-based replication of the SQLite WAL.
-package replication
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/replication/fsm.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/replication/fsm.go
deleted file mode 100644
index 09a49752b6..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/replication/fsm.go
+++ /dev/null
@@ -1,809 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package replication
-
-import (
- "bufio"
- "bytes"
- "encoding/binary"
- "fmt"
- "io"
- "unsafe"
-
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
- "github.com/CanonicalLtd/go-dqlite/internal/connection"
- "github.com/CanonicalLtd/go-dqlite/internal/protocol"
- "github.com/CanonicalLtd/go-dqlite/internal/registry"
- "github.com/CanonicalLtd/go-dqlite/internal/trace"
- "github.com/CanonicalLtd/go-dqlite/internal/transaction"
- "github.com/hashicorp/raft"
- "github.com/pkg/errors"
-)
-
-// FSM implements the raft finite-state machine used to replicate
-// SQLite data.
-type FSM struct {
- registry *registry.Registry
-
- // Whether to make Apply panic when an error occurs, or to simply
- // return an error. This should always be is true except for unit
- // tests.
- panicOnFailure bool
-
- noopBeginTxn uint64 // For upgrades
-}
-
-// NewFSM creates a new Raft state machine for executing dqlite-specific
-// command.
-func NewFSM(registry *registry.Registry) *FSM {
- return &FSM{
- registry: registry,
- panicOnFailure: true,
- }
-}
-
-// Apply log is invoked once a log entry is committed. It returns a value
-// which will be made available in the ApplyFuture returned by Raft.Apply
-// method if that method was called on the same Raft node as the FSM.
-func (f *FSM) Apply(log *raft.Log) interface{} {
- // Lock the registry for the entire duration of the command
- // handlers. This is fine since no other write change can happen
- // anyways while we're running. This might slowdown a bit opening new
- // leader connections, but since application should be designed to open
- // their leaders once for all, it shouldn't be a problem in
- // practice. Read transactions are not be affected by the locking.
- f.registry.Lock()
- defer f.registry.Unlock()
-
- tracer := f.registry.TracerFSM()
-
- // If we're being invoked in the context of a Methods replication hook
- // applying a log command, block execution of any log commands coming
- // on the wire from other leaders until the hook as completed.
- if f.registry.HookSyncPresent() && !f.registry.HookSyncMatches(log.Data) {
- tracer.Message("wait for methods hook to complete")
- // This will temporarily release and re-acquire the registry lock.
- f.registry.HookSyncWait()
- }
-
- err := f.apply(tracer, log)
- if err != nil {
- if f.panicOnFailure {
- tracer.Panic("%v", err)
- }
- tracer.Error("apply failed", err)
- return err
- }
-
- return nil
-}
-
-func (f *FSM) apply(tracer *trace.Tracer, log *raft.Log) error {
- tracer = tracer.With(
- trace.Integer("term", int64(log.Term)),
- trace.Integer("index", int64(log.Index)),
- )
-
- cmd, err := protocol.UnmarshalCommand(log.Data)
- if err != nil {
- return errors.Wrap(err, "corrupted command data")
- }
- tracer = tracer.With(trace.String("cmd", cmd.Name()))
-
- switch payload := cmd.Payload.(type) {
- case *protocol.Command_Open:
- err = f.applyOpen(tracer, payload.Open)
- err = errors.Wrapf(err, "open %s", payload.Open.Name)
- case *protocol.Command_Begin:
- err = f.applyBegin(tracer, payload.Begin)
- err = errors.Wrapf(err, "begin txn %d on %s", payload.Begin.Txid, payload.Begin.Name)
- case *protocol.Command_Frames:
- err = f.applyFrames(tracer, payload.Frames)
- err = errors.Wrapf(err, "wal frames txn %d (%v)", payload.Frames.Txid, payload.Frames.IsCommit)
- case *protocol.Command_Undo:
- err = f.applyUndo(tracer, payload.Undo)
- err = errors.Wrapf(err, "undo txn %d", payload.Undo.Txid)
- case *protocol.Command_End:
- err = f.applyEnd(tracer, payload.End)
- err = errors.Wrapf(err, "end txn %d", payload.End.Txid)
- case *protocol.Command_Checkpoint:
- err = f.applyCheckpoint(tracer, payload.Checkpoint)
- err = errors.Wrapf(err, "checkpoint")
- default:
- err = fmt.Errorf("unknown command")
- }
-
- if err != nil {
- tracer.Error("failed", err)
- return err
- }
-
- f.registry.IndexUpdate(log.Index)
-
- return nil
-}
-
-func (f *FSM) applyOpen(tracer *trace.Tracer, params *protocol.Open) error {
- tracer = tracer.With(
- trace.String("name", params.Name),
- )
- tracer.Message("start")
-
- if err := f.openFollower(params.Name); err != nil {
- return err
- }
-
- tracer.Message("done")
-
- return nil
-}
-
-func (f *FSM) applyBegin(tracer *trace.Tracer, params *protocol.Begin) error {
- tracer = tracer.With(
- trace.Integer("txn", int64(params.Txid)),
- )
-
- // This FSM command is not needed anymore. We make it a no-op, for
- // backward compatibility with deployments that do have it stored in
- // their raft logs.
- tracer.Message("no-op")
- f.noopBeginTxn = params.Txid
-
- return nil
-}
-
-func (f *FSM) applyFrames(tracer *trace.Tracer, params *protocol.Frames) error {
- tracer = tracer.With(
- trace.Integer("txn", int64(params.Txid)),
- trace.Integer("pages", int64(len(params.PageNumbers))),
- trace.Integer("commit", int64(params.IsCommit)))
- tracer.Message("start")
-
- if params.Filename == "" {
- // Backward compatibility with existing LXD deployments.
- params.Filename = "db.bin"
- }
-
- txn := f.registry.TxnByID(params.Txid)
- begin := true
-
- if txn != nil {
- // We know about this transaction.
- tracer.Message("txn found %s", txn)
-
- if txn.IsLeader() {
- // We're executing a Frames command triggered by the
- // Methods.Frames hook on this servers.
- if txn.IsZombie() {
- // The only way that this can be a zombie is if
- // this Frames command is being executed by
- // this FSM after this leader failed with
- // ErrLeadershipLost, and 1) this server was
- // re-elected right away and has successfully
- // retried to apply this command or 2) another
- // server was elected and a quorum was still
- // reached for this command log despite the
- // previous leader not getting notified about
- // it.
- if params.IsCommit == 0 {
- // This is not a commit frames
- // command. Regardless of whether 1) or
- // 2) happened, it's safe to create a
- // surrogate follower transaction and
- // transition it to Writing.
- //
- // If 1) happens, then the next
- // Methods.Begin hook on this server
- // will find a leftover Writing
- // follower and will roll it back with
- // an Undo command. If 2) happens, same.
- tracer.Message("create surrogate follower", txn)
- txn = f.registry.TxnFollowerSurrogate(txn)
- } else {
- // This is a commit frames
- // command. Regardless of whether 1) or
- // 2) happened, we need to resurrect
- // the zombie into a follower and
- // possibly re-apply any non-commit
- // frames that were applied so far in
- // the transaction.
- tracer.Message("recover commit")
- conn := f.registry.ConnFollower(params.Filename)
- var err error
- txn, err = txn.Resurrect(conn)
- if err != nil {
- return err
- }
- f.registry.TxnFollowerResurrected(txn)
- begin = txn.State() == transaction.Pending
- }
- } else {
- // We're executing this FSM command in during
- // the execution of the Methods.Frames hook.
- }
-
- } else {
- // We're executing the Frames command as followers. The
- // transaction must be in the Writing state.
- if txn.State() != transaction.Writing {
- tracer.Panic("unexpected transaction %s", txn)
- }
- begin = false
- }
- } else {
- // We don't know about this transaction.
- //
- // This is must be a new follower transaction. Let's make sure
- // that no other transaction against this database is happening
- // on this server.
- if txn := f.registry.TxnByFilename(params.Filename); txn != nil {
- if txn.IsZombie() {
- // This transactions was left around by a
- // leader that lost leadership during a Frames
- // hook that was the first to be sent and did
- // not reach a quorum, so no other server knows
- // about it, and now we're starting a new
- // trasaction initiated by a new leader. We can
- // just purge it from the registry, since its
- // state was already rolled back by SQLite
- // after the xFrames hook failure.
- tracer.Message("found zombie transaction %s", txn)
-
- // Perform some sanity checks.
- if txn.ID() > params.Txid {
- tracer.Panic("zombie transaction too recent %s", txn)
- }
- if txn.State() != transaction.Pending {
- tracer.Panic("unexpected transaction state %s", txn)
- }
-
- tracer.Message("removing stale zombie transaction %s", txn)
- f.registry.TxnDel(txn.ID())
- } else {
- tracer.Panic("unexpected transaction %s", txn)
- }
- }
-
- conn := f.registry.ConnFollower(params.Filename)
- txn = f.registry.TxnFollowerAdd(conn, params.Txid)
- }
-
- if len(params.Pages) != 0 {
- // This should be a v1 log entry.
- if len(params.PageNumbers) != 0 || len(params.PageData) != 0 {
- tracer.Panic("unexpected data mix between v1 and v2")
- }
-
- // Convert to v2.
- params.PageNumbers = make([]uint32, 0)
- params.PageData = make([]byte, int(params.PageSize)*len(params.Pages))
-
- for i := range params.Pages {
- params.PageNumbers = append(params.PageNumbers, params.Pages[i].Number)
- copy(
- params.PageData[(i*int(params.PageSize)):((i+1)*int(params.PageSize))],
- params.Pages[i].Data,
- )
- }
- }
-
- info := bindings.WalReplicationFrameInfo{}
- info.IsBegin(begin)
- info.PageSize(int(params.PageSize))
- info.Len(len(params.PageNumbers))
- info.Truncate(uint(params.Truncate))
-
- isCommit := false
- if params.IsCommit > 0 {
- isCommit = true
- }
- info.IsCommit(isCommit)
-
- numbers := make([]bindings.PageNumber, len(params.PageNumbers))
- for i, pgno := range params.PageNumbers {
- numbers[i] = bindings.PageNumber(pgno)
- }
-
- info.Pages(numbers, unsafe.Pointer(¶ms.PageData[0]))
-
- if err := txn.Frames(begin, info); err != nil {
- return err
- }
-
- // If the commit flag is on, this is the final write of a transaction,
- if isCommit {
- // Save the ID of this transaction in the buffer of recently committed
- // transactions.
- f.registry.TxnCommittedAdd(txn)
-
- // If it's a follower, we also unregister it.
- if !txn.IsLeader() {
- tracer.Message("unregister txn")
- f.registry.TxnDel(params.Txid)
- }
- }
-
- tracer.Message("done")
-
- f.noopBeginTxn = 0 // Backward compat.
-
- return nil
-}
-
-func (f *FSM) applyUndo(tracer *trace.Tracer, params *protocol.Undo) error {
- tracer = tracer.With(
- trace.Integer("txn", int64(params.Txid)),
- )
- tracer.Message("start")
-
- txn := f.registry.TxnByID(params.Txid)
-
- if txn != nil {
- // We know about this transaction.
- tracer.Message("txn found: %s", txn)
- } else {
- if f.noopBeginTxn != params.Txid {
- tracer.Panic("txn not found")
- }
- f.noopBeginTxn = 0
- return nil
- }
-
- if err := txn.Undo(); err != nil {
- return err
- }
-
- // Let's decide whether to remove the transaction from the registry or
- // not. The following scenarios are possible:
- //
- // 1. This is a non-zombie leader transaction. We can assume that this
- // command is being applied in the context of a Methods.Undo() hook
- // execution, which will wait for the command to succeed and then
- // remove the transaction by itself in the End hook, so no need to
- // remove it here.
- //
- // 2. This is a follower transaction. We're done here, since undone is
- // a final state, so let's remove the transaction.
- //
- // 3. This is a zombie leader transaction. This can happen if the
- // leader lost leadership when applying the a non-commit frames, but
- // the command was still committed (either by us is we were
- // re-elected, or by another server if the command still reached a
- // quorum). In that case the we're handling an Undo command to
- // rollback a dangling transaction, and we have to remove the zombie
- // ourselves, because nobody else would do it otherwise.
- if !txn.IsLeader() || txn.IsZombie() {
- tracer.Message("unregister txn")
- f.registry.TxnDel(params.Txid)
- }
-
- tracer.Message("done")
-
- return nil
-}
-
-func (f *FSM) applyEnd(tracer *trace.Tracer, params *protocol.End) error {
- tracer = tracer.With(
- trace.Integer("txn", int64(params.Txid)),
- )
-
- // This FSM command is not needed anymore. We make it a no-op, for
- // backward compatibility with deployments that do have it stored in
- // their raft logs.
- tracer.Message("no-op")
-
- return nil
-}
-
-func (f *FSM) applyCheckpoint(tracer *trace.Tracer, params *protocol.Checkpoint) error {
- tracer = tracer.With(
- trace.String("file", params.Name),
- )
- tracer.Message("start")
-
- conn := f.registry.ConnFollower(params.Name)
-
- if txn := f.registry.TxnByConn(conn); txn != nil {
- // Something went really wrong, a checkpoint should never be issued
- // while a follower transaction is in flight.
- tracer.Panic("can't run checkpoint concurrently with transaction %s", txn)
- }
-
- // Run the checkpoint.
- logFrames, checkpointedFrames, err := conn.WalCheckpoint("main", bindings.WalCheckpointTruncate)
- if err != nil {
- return err
- }
- if logFrames != 0 {
- tracer.Panic("%d frames are still in the WAL", logFrames)
- }
- if checkpointedFrames != 0 {
- tracer.Panic("only %d frames were checkpointed", checkpointedFrames)
- }
-
- tracer.Message("done")
-
- return nil
-}
-
-// Snapshot is used to support log compaction.
-//
-// From the raft's package documentation:
-//
-// "This call should return an FSMSnapshot which can be used to save a
-// point-in-time snapshot of the FSM. Apply and Snapshot are not called in
-// multiple threads, but Apply will be called concurrently with Persist. This
-// means the FSM should be implemented in a fashion that allows for
-// concurrent updates while a snapshot is happening."
-//
-// In dqlite's case we do the following:
-//
-// - For each database that we track (i.e. that we have a follower connection
-// for), create a backup using sqlite3_backup() and then read the content of
-// the backup file and the current WAL file. Since nothing else is writing to
-// the database (FSM.Apply won't be called until FSM.Snapshot completes), we
-// could probably read the database bytes directly to increase efficiency,
-// but for now we do concurrent-write-safe backup as good measure.
-//
-// - For each database we track, look for ongoing transactions and include
-// their ID in the FSM snapshot, so their state can be re-created upon
-// snapshot Restore.
-//
-// This is a bit heavy-weight but should be safe. Optimizations can be added as
-// needed.
-func (f *FSM) Snapshot() (raft.FSMSnapshot, error) {
- f.registry.Lock()
- defer f.registry.Unlock()
-
- tracer := f.registry.TracerFSM()
-
- databases := []*fsmDatabaseSnapshot{}
-
- // Loop through all known databases and create a backup for each of
- // them. The filenames associated with follower connections uniquely
- // identify all known databases, since there will be one and only
- // follower connection for each known database (we never close follower
- // connections since database deletion is not supported).
- for _, filename := range f.registry.ConnFollowerFilenames() {
- database, err := f.snapshotDatabase(tracer, filename)
- if err != nil {
- err = errors.Wrapf(err, "%s", filename)
- tracer.Error("database snapshot failed", err)
- return nil, err
- }
- databases = append(databases, database)
- }
-
- return &FSMSnapshot{
- index: f.registry.Index(),
- databases: databases,
- }, nil
-}
-
-// Backup a single database.
-func (f *FSM) snapshotDatabase(tracer *trace.Tracer, filename string) (*fsmDatabaseSnapshot, error) {
- tracer = tracer.With(trace.String("snapshot", filename))
- tracer.Message("start")
-
- // Figure out if there is an ongoing transaction associated with any of
- // the database connections, if so we'll return an error.
- conns := f.registry.ConnLeaders(filename)
- conns = append(conns, f.registry.ConnFollower(filename))
- txid := ""
- for _, conn := range conns {
- if txn := f.registry.TxnByConn(conn); txn != nil {
- // XXX TODO: If we let started transaction in the
- // snapshot, the TestIntegration_Snapshot crashes with:
- //
- // panic: unexpected follower transaction 7 started as follower
- //
- // figure out why.
- //if txn.State() == transaction.Writing {
- tracer.Message("transaction %s is in progress", txn)
- return nil, fmt.Errorf("transaction %s is in progress", txn)
- //}
- // We'll save the transaction ID in the snapshot.
- //tracer.Message("idle transaction %s", txn)
- //txid = strconv.FormatUint(txn.ID(), 10)
- }
- }
-
- database, wal, err := connection.Snapshot(f.registry.Vfs(), filename)
- if err != nil {
- return nil, err
- }
-
- tracer.Message("done")
-
- return &fsmDatabaseSnapshot{
- filename: filename,
- database: database,
- wal: wal,
- txid: txid,
- }, nil
-}
-
-// Restore is used to restore an FSM from a snapshot. It is not called
-// concurrently with any other command. The FSM must discard all previous
-// state.
-func (f *FSM) Restore(reader io.ReadCloser) error {
- f.registry.Lock()
- defer f.registry.Unlock()
-
- tracer := f.registry.TracerFSM()
-
- // The first 8 bytes contain the FSM Raft log index.
- var index uint64
- if err := binary.Read(reader, binary.LittleEndian, &index); err != nil {
- return errors.Wrap(err, "failed to read FSM index")
- }
-
- tracer = tracer.With(trace.Integer("restore", int64(index)))
- tracer.Message("start")
-
- f.registry.IndexUpdate(index)
-
- for {
- done, err := f.restoreDatabase(tracer, reader)
- if err != nil {
- return err
- }
- if done {
- break
- }
- }
-
- tracer.Message("done")
-
- return nil
-}
-
-// Restore a single database. Returns true if there are more databases
-// to restore, false otherwise.
-func (f *FSM) restoreDatabase(tracer *trace.Tracer, reader io.ReadCloser) (bool, error) {
- done := false
-
- // The first 8 bytes contain the size of database.
- var dataSize uint64
- if err := binary.Read(reader, binary.LittleEndian, &dataSize); err != nil {
- return false, errors.Wrap(err, "failed to read database size")
- }
- tracer.Message("database size: %d", dataSize)
-
- // Then there's the database data.
- data := make([]byte, dataSize)
- if _, err := io.ReadFull(reader, data); err != nil {
- return false, errors.Wrap(err, "failed to read database data")
- }
-
- // Next, the size of the WAL.
- var walSize uint64
- if err := binary.Read(reader, binary.LittleEndian, &walSize); err != nil {
- return false, errors.Wrap(err, "failed to read wal size")
- }
- tracer.Message("wal size: %d", walSize)
-
- // Read the WAL data.
- wal := make([]byte, walSize)
- if _, err := io.ReadFull(reader, wal); err != nil {
- return false, errors.Wrap(err, "failed to read wal data")
- }
-
- // Read the database path.
- bufReader := bufio.NewReader(reader)
- filename, err := bufReader.ReadString(0)
- if err != nil {
- return false, errors.Wrap(err, "failed to read database name")
- }
- filename = filename[:len(filename)-1] // Strip the trailing 0
- tracer.Message("filename: %s", filename)
-
- // XXX TODO: reason about this situation, is it harmful?
- // Check that there are no leader connections for this database.
- //
- // FIXME: we should relax this, as it prevents restoring snapshots "on
- // the fly".
- // conns := f.registry.ConnLeaders(filename)
- // if len(conns) > 0 {
- // tracer.Panic("found %d leader connections", len(conns))
- // }
-
- // XXX TODO: reason about this situation, is it possible?
- //txn := f.transactions.GetByConn(f.connections.Follower(name))
- //if txn != nil {
- // f.logger.Printf("[WARN] dqlite: fsm: closing follower in-flight transaction %s", txn)
- // f.transactions.Remove(txn.ID())
- //}
-
- // Close any follower connection, since we're going to overwrite the
- // database file.
- if f.registry.ConnFollowerExists(filename) {
- tracer.Message("close follower: %s", filename)
- follower := f.registry.ConnFollower(filename)
- f.registry.ConnFollowerDel(filename)
- if err := follower.Close(); err != nil {
- return false, err
- }
- }
-
- // At this point there should be not connection open against this
- // database, so it's safe to overwrite it.
- txid, err := bufReader.ReadString(0)
- if err != nil {
- if err != io.EOF {
- return false, errors.Wrap(err, "failed to read txid")
- }
- done = true // This is the last database.
- }
- tracer.Message("transaction ID: %s", txid)
-
- vfs := f.registry.Vfs()
-
- if err := connection.Restore(vfs, filename, data, wal); err != nil {
- return false, err
- }
-
- tracer.Message("open follower: %s", filename)
- if err := f.openFollower(filename); err != nil {
- return false, err
- }
-
- if txid != "" {
- // txid, err := strconv.ParseUint(txid, 10, 64)
- // if err != nil {
- // return false, err
- // }
- // tracer.Message("add transaction: %d", txid)
- // conn := f.registry.ConnFollower(filename)
- // txn := f.registry.TxnFollowerAdd(conn, txid)
- // if err := txn.Begin(); err != nil {
- // return false, err
- // }
- }
-
- return done, nil
-}
-
-func (f *FSM) openFollower(filename string) error {
- vfs := f.registry.Vfs().Name()
- conn, err := bindings.Open(filename, vfs)
- if err != nil {
- return errors.Wrap(err, "failed to open connection")
- }
-
- err = conn.Exec("PRAGMA synchronous=OFF")
- if err != nil {
- return errors.Wrap(err, "failed to disable syncs")
- }
-
- err = conn.Exec("PRAGMA journal_mode=wal")
- if err != nil {
- return errors.Wrap(err, "failed to set WAL mode")
- }
-
- _, err = conn.ConfigNoCkptOnClose(true)
- if err != nil {
- return errors.Wrap(err, "failed to disable checkpoints on close")
- }
-
- err = conn.WalReplicationFollower()
- if err != nil {
- return errors.Wrap(err, "failed to set follower replication mode")
- }
-
- f.registry.ConnFollowerAdd(filename, conn)
-
- return nil
-}
-
-// FSMSnapshot is returned by an FSM in response to a Snapshot
-// It must be safe to invoke FSMSnapshot methods with concurrent
-// calls to Apply.
-type FSMSnapshot struct {
- index uint64
- databases []*fsmDatabaseSnapshot
-}
-
-// Persist should dump all necessary state to the WriteCloser 'sink',
-// and call sink.Close() when finished or call sink.Cancel() on error.
-func (s *FSMSnapshot) Persist(sink raft.SnapshotSink) error {
- // First, write the FSM index.
- buffer := new(bytes.Buffer)
- if err := binary.Write(buffer, binary.LittleEndian, s.index); err != nil {
- return errors.Wrap(err, "failed to FSM index")
- }
- if _, err := sink.Write(buffer.Bytes()); err != nil {
- return errors.Wrap(err, "failed to write FSM index to sink")
- }
-
- // Then write the individual databases.
- for _, database := range s.databases {
- if err := s.persistDatabase(sink, database); err != nil {
- sink.Cancel()
- return err
- }
-
- }
-
- if err := sink.Close(); err != nil {
- sink.Cancel()
- return err
- }
-
- return nil
-}
-
-// Persist a single daabase snapshot.
-func (s *FSMSnapshot) persistDatabase(sink raft.SnapshotSink, database *fsmDatabaseSnapshot) error {
- // Start by writing the size of the backup
- buffer := new(bytes.Buffer)
- dataSize := uint64(len(database.database))
- if err := binary.Write(buffer, binary.LittleEndian, dataSize); err != nil {
- return errors.Wrap(err, "failed to encode data size")
- }
- if _, err := sink.Write(buffer.Bytes()); err != nil {
- return errors.Wrap(err, "failed to write data size to sink")
- }
-
- // Next write the data to the sink.
- if _, err := sink.Write(database.database); err != nil {
- return errors.Wrap(err, "failed to write backup data to sink")
-
- }
-
- buffer.Reset()
- walSize := uint64(len(database.wal))
- if err := binary.Write(buffer, binary.LittleEndian, walSize); err != nil {
- return errors.Wrap(err, "failed to encode wal size")
- }
- if _, err := sink.Write(buffer.Bytes()); err != nil {
- return errors.Wrap(err, "failed to write wal size to sink")
- }
- if _, err := sink.Write(database.wal); err != nil {
- return errors.Wrap(err, "failed to write backup data to sink")
-
- }
-
- // Next write the database name.
- buffer.Reset()
- buffer.WriteString(database.filename)
- if _, err := sink.Write(buffer.Bytes()); err != nil {
- return errors.Wrap(err, "failed to write database name to sink")
- }
- if _, err := sink.Write([]byte{0}); err != nil {
- return errors.Wrap(err, "failed to write database name delimiter to sink")
- }
-
- // FInally write the current transaction ID, if any.
- buffer.Reset()
- buffer.WriteString(database.txid)
- if _, err := sink.Write(buffer.Bytes()); err != nil {
- return errors.Wrap(err, "failed to write txid to sink")
- }
-
- return nil
-}
-
-// Release is invoked when we are finished with the snapshot.
-func (s *FSMSnapshot) Release() {
-}
-
-// fsmDatabaseSnapshot holds backup information for a single database.
-type fsmDatabaseSnapshot struct {
- filename string
- database []byte
- wal []byte
- txid string
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/replication/methods.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/replication/methods.go
deleted file mode 100644
index 7278b48136..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/replication/methods.go
+++ /dev/null
@@ -1,794 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package replication
-
-import (
- "sync"
- "time"
-
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
- "github.com/CanonicalLtd/go-dqlite/internal/protocol"
- "github.com/CanonicalLtd/go-dqlite/internal/registry"
- "github.com/CanonicalLtd/go-dqlite/internal/trace"
- "github.com/CanonicalLtd/go-dqlite/internal/transaction"
- "github.com/hashicorp/raft"
-)
-
-// Methods implements the SQLite replication C API using the sqlite3 bindings.
-type Methods struct {
- registry *registry.Registry
- raft *raft.Raft // Raft engine to use
- mu sync.RWMutex // TODO: make this lock per-database.
- applyTimeout time.Duration
-
- // If greater than zero, skip the initial not-leader checks, this
- // amount of times. Only used for testing.
- noLeaderCheck int
-}
-
-// NewMethods returns a new Methods instance that can be used as callbacks API
-// for raft-based SQLite replication of a single connection.
-func NewMethods(reg *registry.Registry, raft *raft.Raft) *Methods {
- return &Methods{
- registry: reg,
- raft: raft,
- applyTimeout: 10 * time.Second,
- }
-}
-
-// ApplyTimeout sets the maximum amount of time to wait before giving
-// up applying a raft command. The default is 10 seconds.
-func (m *Methods) ApplyTimeout(timeout time.Duration) {
- m.applyTimeout = timeout
-}
-
-// Begin is the hook invoked by SQLite when a new write transaction is
-// being started within a connection in leader replication mode on
-// this server.
-func (m *Methods) Begin(conn *bindings.Conn) int {
- // We take a the lock for the entire duration of the hook to avoid
- // races between to concurrent hooks.
- //
- // The main tasks of Begin are to check that no other write transaction
- // is in progress and to cleanup any dangling follower transactions
- // that might have been left open after a leadership change.
- //
- // Concurrent calls can happen because the xBegin hook is fired by
- // SQLite before acquiring a write lock on the WAL (i.e. before calling
- // WalBeginWriteTransaction), so different connections can enter the
- // Begin hook at any time .
- //
- // Some races that is worth mentioning are:
- //
- // - Two concurrent calls of Begin: without the lock, they would race
- // to open a follower connection if it does not exists yet.
- //
- // - Two concurrent calls of Begin and another hook: without the lock,
- // they would race to mutate the registry (e.g. add/delete
- // transactions).
- //
- // The most common errors that Begin returns are:
- //
- // - SQLITE_BUSY: If we detect that a write transaction is in progress
- // on another connection. The SQLite's call to sqlite3PagerBegin
- // that triggered the xBegin hook will propagate the error
- // to sqlite3BtreeBeginTrans, which will invoke the busy
- // handler (if set), calling xBegin/Begin again, which will
- // keep returning SQLITE_BUSY as long as the other transaction
- // is in progress. If the busy handler gives up, the SQLITE_BUSY
- // error will bubble up, and the statement that triggered the
- // write attempt will fail. The client should then execute a
- // ROLLBACK and then decide what to do.
- //
- // - SQLITE_IOERR: This is returned if we are not the leader when the
- // hook fires or if we fail to apply the Open follower
- // command log (in case no follower for this database
- // is open yet). We include the relevant extended
- // code, either SQLITE_IOERR_NOT_LEADER if this server
- // is not the leader anymore or it is being shutdown,
- // or SQLITE_IOERR_LEADERSHIP_LOST if leadership was
- // lost while applying the Open command. The SQLite's
- // call to sqlite3PagerBegin that triggered the xBegin
- // hook will propagate the error to sqlite3BtreeBeginTrans,
- // which will in turn propagate it to the OP_Transaction case
- // of vdbe.c, which will goto abort_due_to_error and finally
- // call sqlite3VdbeHalt, automatically rolling back the
- // transaction. Since no write transaction was actually started the
- // xEnd hook is not fired.
- //
- // We might return SQLITE_INTERRUPT or SQLITE_INTERNAL in case of more exotic
- // failures. See the apply() method for details.
- m.mu.Lock()
- defer m.mu.Unlock()
-
- // Lock the registry, to avoid races with the FSM when checking for the
- // presence of a hook sync and when modifying transactions.
- m.registry.Lock()
- defer m.registry.Unlock()
-
- // Enable synchronization with the FSM: it will only execute commands
- // applied during this hook, and block applying any other command until
- // this hook is done.
- m.registry.HookSyncSet()
- defer m.registry.HookSyncReset()
-
- tracer := m.registry.TracerConn(conn, "begin")
- tracer.Message("start")
-
- // Check if we're the leader.
- if m.noLeaderCheck == 0 && m.raft.State() != raft.Leader {
- // No dqlite state has been modified, and the WAL write lock
- // has not been acquired. Just return ErrIoErrNotLeader.
- tracer.Message("not leader")
- return bindings.ErrIoErrNotLeader
- }
-
- // Update the noLeaderCheck counter (used for tests).
- if m.noLeaderCheck > 0 {
- m.noLeaderCheck--
- }
-
- // Possibly open a follower for this database if it doesn't exist yet.
- if err := m.beginMaybeAddFollowerConn(tracer, conn); err != nil {
- // Since we haven't yet registered a transaction, there's no
- // cleanup to do here. The worst that can happen is that the
- // Raft.Apply() call failed with ErrLeadershipLost and a quorum
- // for the log will actually be reached. In that case all FSMs
- // (including our own) will apply the open command.
- return errno(err)
- }
-
- // Check whether there is already an an ongoing transaction.
- if err := m.beginMaybeHandleInProgressTxn(tracer, conn); err != nil {
- return errno(err)
- }
-
- // Use the last applied index as transaction ID.
- //
- // If this server is still the leader, this number is guaranteed to be
- // strictly higher than any previous transaction ID, since after a
- // leadership change we always call raft.Barrier() to advance the FSM
- // up to the latest committed log, raft.Barrier() itself will increment
- // the applied index by one.
- //
- // If it's not the leader anymore, it does not matter which ID we pick
- // because any coming Frames or Undo hook will fail with ErrNotLeader.
- txid := m.raft.AppliedIndex()
-
- tracer = tracer.With(trace.Integer("txn", int64(txid)))
- tracer.Message("register transaction")
-
- // Create a new transaction.
- m.registry.TxnLeaderAdd(conn, txid)
-
- tracer.Message("done")
-
- return 0
-}
-
-// Check if a follower connection is already open for this database, if not
-// open one with the Open raft command.
-func (m *Methods) beginMaybeAddFollowerConn(tracer *trace.Tracer, conn *bindings.Conn) error {
- filename := m.registry.ConnLeaderFilename(conn)
-
- if m.registry.ConnFollowerExists(filename) {
- return nil
- }
-
- tracer.Message("open follower for %s", filename)
- return m.apply(tracer, conn, protocol.NewOpen(filename))
-}
-
-// This method ensures that there is no other write transactions happening
-// on this node against database associated to the given connection.
-//
-// If one is found, this method will try take appropriate measures.
-//
-// If an error is returned, Begin should stop and return it.
-func (m *Methods) beginMaybeHandleInProgressTxn(tracer *trace.Tracer, conn *bindings.Conn) error {
- filename := m.registry.ConnLeaderFilename(conn)
- txn := m.registry.TxnByFilename(filename)
- if txn == nil {
- return nil
- }
-
- tracer.Message("found in-progress transaction %s", txn)
-
- // Check if the in-progress transaction is a concurrent leader.
- if txn.IsLeader() {
- if txn.Conn() != conn {
- // This means that there is a transaction in progress
- // originated on this Methods instance for another
- // connection.
- //
- // We can't proceed as the Begin method would then try
- // to add a new transaction to the registry and crash.
- //
- // No dqlite state has been modified, and the WAL write
- // lock has not been acquired.
- //
- // We just return ErrBusy, which has the same effect as
- // the call to sqlite3WalBeginWriteTransaction (invoked
- // in pager.c after a successful xBegin) would have, i.e.
- // the busy handler will end up polling us again until
- // the concurrent write transaction ends and we're free
- // to go.
- tracer.Message("busy")
- return bindings.Error{Code: bindings.ErrBusy}
- }
-
- // There a's transaction originated on this Methods instance for
- // the same connection.
- if !txn.IsZombie() {
- // This should be an impossible situation since it
- // would mean that the same connection managed to begin
- // a new transaction on the same connection, something
- // that SQLite prevents.
- tracer.Panic("unexpected transaction on same connection %s", txn)
- }
-
- // If we have a zombie for this connection it, must mean that a
- // Frames command failed because we were not leaders anymore at
- // that time and this was a commit frames command following one
- // or more non-commit frames commands that were successfully
- // applied.
- if txn.State() != transaction.Writing {
- tracer.Panic("unexpected transaction %s", txn)
- }
-
- // Create a surrogate follower and revert the transaction just
- // below.
- m.surrogateWriting(tracer, txn)
- }
-
- tracer.Message("undo stale transaction %s", txn)
- if err := m.apply(tracer, conn, protocol.NewUndo(txn.ID())); err != nil {
- // Whatever the reason of the failure is (not leader or
- // leadeship lost), we can leave things as they are,
- // since the next leader should try to run again the
- // undo command.
- return err
- }
-
- return nil
-}
-
-// Abort is the hook invoked by SQLite when a write transaction fails
-// to begin.
-func (m *Methods) Abort(conn *bindings.Conn) int {
- // We take a the lock for the entire duration of the hook to avoid
- // races between to cocurrent hooks.
- m.mu.Lock()
- defer m.mu.Unlock()
-
- // Lock the registry.
- m.registry.Lock()
- defer m.registry.Unlock()
-
- tracer := m.registry.TracerConn(conn, "abort")
- tracer.Message("start")
-
- // This is only called if SQLite fails to start a WAL write transaction.
- txn := m.registry.TxnByConn(conn)
- if txn == nil {
- tracer.Panic("no in-progress transaction")
- }
- tracer.Message("found txn %s", txn)
-
- // Sanity checks.
- if !txn.IsLeader() || txn.Conn() != conn {
- tracer.Panic("unexpected transaction %s", txn)
- }
- if txn.State() != transaction.Pending {
- tracer.Panic("unexpected transaction state %s", txn)
- }
-
- tracer.Message("discard aborted transaction")
- m.registry.TxnDel(txn.ID())
-
- return 0
-}
-
-// Frames is the hook invoked by sqlite when new frames need to be
-// flushed to the write-ahead log.
-func (m *Methods) Frames(conn *bindings.Conn, frames bindings.WalReplicationFrameList) int {
- // We take a the lock for the entire duration of the hook to avoid
- // races between to cocurrent hooks. See the comments in Begin for more
- // details.
- //
- // The xFrames hook is invoked by the SQLite pager in two cases, either
- // in sqlite3PagerCommitPhaseOne (for committing) or in pagerStress (for
- // flushing dirty pages to disk, invoked by the pcache implementation when
- // no more space is available in the built-in page cache).
- //
- // In the first case, any error returned here will be propagated up to
- // sqlite3VdbeHalt (the final step of SQLite's VDBE), which will rollback
- // the transaction and indirectly invoke sqlite3PagerRollback which in turn
- // will indirectly fire xUndo and xEnd.
- //
- // In the second case, any error returned here will transition the
- // pager to the ERROR state (see the final pager_error call in
- // pagerStress) and will be propagated first to sqlite3PcacheFetchStress
- // and the indirectly to the btree layer which will automatically rollback
- // the transaction. The xUndo and xEnd hooks won't be fired, since the
- // pager is in an error state.
- //
- // The most common errors that Frames returns are:
- //
- m.mu.Lock()
- defer m.mu.Unlock()
-
- // Lock the registry.
- m.registry.Lock()
- defer m.registry.Unlock()
-
- // Enable synchronization with the FSM: it will only execute commands
- // applied during this hook, and block applying any other command until
- // this hook is done.
- m.registry.HookSyncSet()
- defer m.registry.HookSyncReset()
-
- tracer := m.registry.TracerConn(conn, "frames")
- tracer.Message("start (commit=%v)", frames.IsCommit())
-
- txn := m.registry.TxnByConn(conn)
- if txn == nil {
- tracer.Panic("no in-progress transaction")
- }
- tracer.Message("found txn %s", txn)
-
- // Sanity checks.
- if !txn.IsLeader() {
- tracer.Panic("unexpected transaction %s", txn)
- }
- if txn.State() != transaction.Pending && txn.State() != transaction.Writing {
- tracer.Panic("unexpected transaction state %s", txn)
- }
-
- // Check if we're the leader.
- if m.noLeaderCheck == 0 && m.raft.State() != raft.Leader {
- return errno(m.framesNotLeader(tracer, txn))
- }
-
- // Update the noLeaderCheck counter.
- if m.noLeaderCheck > 0 {
- m.noLeaderCheck--
- }
-
- filename := m.registry.ConnLeaderFilename(conn)
- command := protocol.NewFrames(txn.ID(), filename, frames)
- if err := m.apply(tracer, conn, command); err != nil {
- // Check that transaction is still Pending or Writing. The hook-sync
- // mechanism prevents our FSM to apply anything else, but let's
- // assert it for sanity.
- if txn.State() != transaction.Pending && txn.State() != transaction.Writing {
- tracer.Panic("unexpected transaction state: %s", txn)
- }
-
- if isErrNotLeader(err) {
- // This is relatively unlikely since we already checked
- // for leadership at the beginning of the hook, but
- // still possible in principle with a particular bad
- // timing.
- //
- // The same logic applies.
- //
- // We can be sure that the Frames command didn't get
- // committed, so we can just mark the transaction as
- // stale, create a surrogate follower and return. The
- // Undo hook that will be fired right after and will
- // no-op.
- return errno(m.framesNotLeader(tracer, txn))
- } else if isErrLeadershipLost(err) {
- if frames.IsCommit() {
- // Mark the transaction as zombie. Possible scenarios:
- //
- // 1. This server gets re-elected right away as leader.
- //
- // In this case we'll try to apply this lost
- // command log again. If we succeed, our FSM
- // will transition this zombie transaction
- // into to a surrogate follower and our next
- // Begin hook invokation will issue an Undo
- // command, which (if successfull) will be a
- // no-op on our FSM and an actual rollback
- // on the followers (regardless of whether
- // this was the first non-commit frames
- // command or a further one). If we fail to
- // re-apply the command there will be a new
- // election, and we'll end up again in
- // either this case (1) or the next one
- // (2). Same if the Undo command fails.
- //
- // 2. Another server gets elected as leader.
- //
- // In this case there are two possible
- // scenarios.
- //
- // 2.1. No quorum was reached for the lost
- // commit command. This means that no
- // FSM (including ours) will ever try
- // to apply it. If this lost non-commit
- // frames command was the first one of
- // a transaction, the new leader will
- // see no dangling follower and will
- // just start a new transaction with a
- // new ID, sending a Frames command to
- // our FSM. Our FSM will detect the
- // zombie transaction and simply purge
- // it from the registry.
- //
- // 2.2 A quorum was reached for the lost
- // commit command. This means that the
- // new leader will replicate it to every
- // server that didn't apply it yet,
- // which includes us, and then issue an
- // Undo command to abort the
- // transaction. In this case our FSM
- // will behave like in case 1.
- tracer.Message("marking as zombie")
- txn.Zombie()
- } else {
- // Mark the transaction as zombie. Possible scenarios:
- //
- // 1. This server gets re-elected right away as leader.
- //
- // In this case we'll try to apply this lost
- // command log again. If we succeed, two
- // scenarios are possible:
- //
- // 1.1 This was the only Frames command in
- // the transaction, our FSM will convert
- // the zombie transaction into a
- // follower transaction, and apply it
- // normally, effectively recovering the
- // commit failure.
- //
- // 2.1 This was the last Frames command in a
- // series of one or more non-commit
- // Frames commands, which were all
- // applied.
- //
- // 2. Another server gets elected as leader.
- //
- // In this case there are two possible
- // scenarios.
- //
- // 2.1. No quorum was reached for the lost
- // commit command. This means that no
- // FSM (including ours) will ever try
- // to apply it. If this lost commit
- // frames command was the first one of
- // a transaction, the new leader will
- // see no dangling follower and will
- // just start a new transaction with a
- // new ID, sending a Frames command to
- // our FSM. Our FSM will detect the
- // zombie transaction and simply purge
- // it from the registry.
- //
- // 2.2 A quorum was reached for the lost
- // commit command. This means that the
- // new leader will replicate it to every
- // server that didn't apply it yet,
- // which includes us. In this case our
- // FSM will detect the zombie and
- // resurrect it using the follower
- // connection for this database, and
- // possibly writing all preceeding
- // non-commit frames to fully recover
- // the transaction (which was originally
- // rolled back on this server).
- tracer.Message("marking as zombie")
- txn.Zombie()
- }
- }
-
- // TODO: under which circumstances can we get errors other than
- // NotLeader/RaftShutdown and LeadershipLost? How to handle
- // them? See also the comments in the apply() method.
-
- return errno(err)
- }
-
- tracer.Message("done")
-
- return 0
-}
-
-// Handle Frames failures due to this server not not being the leader.
-func (m *Methods) framesNotLeader(tracer *trace.Tracer, txn *transaction.Txn) error {
- if txn.State() == transaction.Pending {
- // No Frames command was applied, so followers don't
- // know about this transaction. We don't need to do
- // anything special, the xUndo hook will just remove
- // it.
- tracer.Message("no frames command applied")
- } else {
- // At least one Frames command was applied, so the transaction
- // exists on the followers. We mark the transaction as zombie,
- // the Begin() hook of next leader (either us or somebody else)
- // will detect a dangling transaction and issue an Undo command
- // to roll it back. In its applyUndo method our FSM will detect
- // that the rollback is for a zombie and just no-op it.
- tracer.Message("marking as zombie")
- txn.Zombie()
- }
-
- // When we return an error, SQLite will fire the End hook.
- tracer.Message("not leader")
-
- return bindings.Error{Code: bindings.ErrIoErrNotLeader}
-}
-
-// Undo is the hook invoked by sqlite when a write transaction needs
-// to be rolled back.
-func (m *Methods) Undo(conn *bindings.Conn) int {
- // We take a the lock for the entire duration of the hook to avoid
- // races between to cocurrent hooks.
- m.mu.Lock()
- defer m.mu.Unlock()
-
- // Lock the registry.
- m.registry.Lock()
- defer m.registry.Unlock()
-
- // Enable synchronization with the FSM: it will only execute commands
- // applied during this hook, and block applying any other command until
- // this hook is done.
- m.registry.HookSyncSet()
- defer m.registry.HookSyncReset()
-
- tracer := m.registry.TracerConn(conn, "undo")
- tracer.Message("start")
-
- txn := m.registry.TxnByConn(conn)
- if txn == nil {
- tracer.Panic("no in-progress transaction")
- }
- tracer.Message("found txn %s", txn)
-
- // Sanity check.
- if !txn.IsLeader() {
- tracer.Panic("unexpected transaction %s", txn)
- }
-
- if txn.IsZombie() {
- // This zombie originated from the Frames hook. There are two scenarios:
- //
- // 1. Leadership was lost while applying the Frames command.
- //
- // We can't simply remove the transaction since the Frames
- // command might eventually get committed. We just ignore
- // it, and let it handle by our FSM in that case (i.e. if we
- // are re-elected or a quorum was reached and another leader
- // tries to apply it).
- //
- // 2. This server was not the leader anymore when the Frames
- // hook fired for a commit frames batch which was the last
- // of a sequence of non-commit ones.
- //
- // In this case we're being invoked by SQLite which is
- // trying to rollback the transaction. We can't simply
- // remove the transaction since the next leader will detect
- // a dangling transaction and try to issue an Undo
- // command. We just ignore the zombie and let our FSM handle
- // it when the Undo command will be applied.
- tracer.Message("done: ignore zombie")
- return 0
- }
-
- if txn.State() == transaction.Pending {
- // This means that the Undo hook fired because this node was
- // not the leader when trying to apply the first Frames
- // command, so no follower knows about it. We can just return,
- // the transaction will be removed by the End hook.
- tracer.Message("done: no frames command was sent")
- return 0
- }
-
- // Check if we're the leader.
- if m.noLeaderCheck == 0 && m.raft.State() != raft.Leader {
- // If we have lost leadership we're in a state where the
- // transaction began on this node and a quorum of followers. We
- // return an error, and SQLite will ignore it, however we
- // need to create a surrogate follower, so the next leader will try to
- // undo it across all nodes.
- tracer.Message("not leader")
- m.surrogateWriting(tracer, txn)
- return bindings.ErrIoErrNotLeader
- }
-
- // Update the noLeaderCheck counter.
- if m.noLeaderCheck > 0 {
- m.noLeaderCheck--
- }
-
- // We don't really care whether the Undo command applied just below here
- // will be committed or not.If the command fails, we'll create a
- // surrogate follower: if the command still gets committed, then the
- // rollback succeeds and the next leader will start fresh, if if the
- // command does not get committed, the next leader will find a stale
- // follower and re-try to roll it back.
- if txn.State() != transaction.Pending {
- command := protocol.NewUndo(txn.ID())
- if err := m.apply(tracer, conn, command); err != nil {
- m.surrogateWriting(tracer, txn)
- return errno(err)
- }
- }
-
- tracer.Message("done")
-
- return 0
-}
-
-// End is the hook invoked by sqlite when ending a write transaction.
-func (m *Methods) End(conn *bindings.Conn) int {
- // We take a the lock for the entire duration of the hook to avoid
- // races between to cocurrent hooks.
- m.mu.Lock()
- defer m.mu.Unlock()
-
- // Lock the registry.
- m.registry.Lock()
- defer m.registry.Unlock()
-
- tracer := m.registry.TracerConn(conn, "end")
- tracer.Message("start")
-
- txn := m.registry.TxnByConn(conn)
- if txn == nil {
- // Check if we have a surrogate transaction instead.
- filename := m.registry.ConnLeaderFilename(conn)
- conn := m.registry.ConnFollower(filename)
- txn = m.registry.TxnByConn(conn)
- if txn == nil {
- // Ignore missing transactions that might have been removed by a
- // particularly bad timing where a new leader has already sent
- // some Undo command following a leadership change and our FSM
- // applied it against a surrogate, removing it from the
- // registry.
- tracer.Message("done: ignore missing transaction")
- return 0
- }
- } else {
- // Sanity check
- if txn.Conn() != conn {
- tracer.Panic("unexpected transaction", conn)
- }
- }
- tracer.Message("found txn %s", txn)
-
- if !txn.IsLeader() {
- // This must be a surrogate follower created by the Frames or
- // Undo hooks. Let's ignore it, has it will be handled by the
- // next leader of FSM.
- tracer.Message("done: ignore surrogate follower")
- return 0
- }
-
- if txn.IsZombie() {
- // Ignore zombie transactions as we don't know what will happen
- // to them (either committed or not).
- tracer.Message("done: ignore zombie")
- return 0
- }
-
- tracer.Message("unregister transaction")
- m.registry.TxnDel(txn.ID())
-
- tracer.Message("done")
-
- return 0
-}
-
-// Create a surrogate follower transaction, transiting it to the Writing state.
-func (m *Methods) surrogateWriting(tracer *trace.Tracer, txn *transaction.Txn) {
- tracer.Message("surrogate to Writing")
- // TODO: fix
- //txn = m.registry.TxnFollowerSurrogate(txn)
- //txn.Frames(true, &sqlite3.ReplicationFramesParams{IsCommit: 0})
-}
-
-// Apply the given command through raft.
-func (m *Methods) apply(tracer *trace.Tracer, conn *bindings.Conn, cmd *protocol.Command) error {
- tracer = tracer.With(trace.String("cmd", cmd.Name()))
- tracer.Message("apply start")
-
- data, err := protocol.MarshalCommand(cmd)
- if err != nil {
- return err
- }
-
- // We need to release the lock while the command is being applied,
- // since the FSM of this raft instance needs to be able to acquire
- // it. However, since we don't want the FSM to execute more than one
- // log we also configure the registry's HookSync so the FSM will block
- // on executing any log command otherwise than the one we are sending
- // now. See also internal/registry/hook.go.
- m.registry.HookSyncAdd(data)
- m.registry.Unlock()
- err = m.raft.Apply(data, m.applyTimeout).Error()
- m.registry.Lock()
-
- if err != nil {
- tracer.Error("apply error", err)
-
- // If the server has lost leadership or is shutting down, we
- // return a dedicated error, so clients will typically retry
- // against the new leader.
- switch err {
- case raft.ErrRaftShutdown:
- // For our purposes, this is semantically equivalent
- // to not being the leader anymore.
- fallthrough
- case raft.ErrNotLeader:
- return bindings.Error{Code: bindings.ErrIoErrNotLeader}
- case raft.ErrLeadershipLost:
- return bindings.Error{Code: bindings.ErrIoErrLeadershipLost}
- case raft.ErrEnqueueTimeout:
- // This should be pretty much impossible, since Methods
- // hooks are the only way to apply command logs, and
- // hooks always wait for a command log to finish before
- // applying a new one (see the Apply().Error() call
- // above). We return SQLITE_INTERRUPT, which for our
- // purposes has the same semantics as SQLITE_IOERR,
- // i.e. it will automatically rollback the transaction.
- return bindings.Error{Code: bindings.ErrInterrupt}
- default:
- // This is an unexpected raft error of some kind.
- //
- // TODO: We should investigate what this could be,
- // for example how to properly handle ErrAbortedByRestore
- // or log-store related errors. We should also
- // examine what SQLite exactly does if we return
- // SQLITE_INTERNAL.
- return bindings.Error{Code: bindings.ErrInternal}
- }
-
- }
-
- tracer.Message("apply done")
- return nil
-}
-
-// Convert a Go error into a SQLite error number.
-func errno(err error) int {
- switch e := err.(type) {
- case bindings.Error:
- return e.Code
- default:
- return bindings.ErrInternal
- }
-}
-
-func isErrNotLeader(err error) bool {
- if err, ok := err.(bindings.Error); ok {
- if err.Code == bindings.ErrIoErrNotLeader {
- return true
- }
- }
- return false
-}
-
-func isErrLeadershipLost(err error) bool {
- if err, ok := err.(bindings.Error); ok {
- if err.Code == bindings.ErrIoErrLeadershipLost {
- return true
- }
- }
- return false
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/replication/trace.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/replication/trace.go
deleted file mode 100644
index b085496efc..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/replication/trace.go
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package replication
-
-/*
-// TracerName returns the name of the methods tracer to be used for the given
-// connection.
-func TracerName(connections *connection.Registry, conn *sqlite3.SQLiteConn) string {
- return fmt.Sprintf("methods %d", connections.Serial(conn))
-}
-*/
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/store/iterate.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/store/iterate.go
deleted file mode 100644
index efeeff85b7..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/store/iterate.go
+++ /dev/null
@@ -1,29 +0,0 @@
-package store
-
-import (
- "github.com/hashicorp/raft"
- "github.com/pkg/errors"
-)
-
-// Handler handles a single command log yielded by Iterate.
-type Handler func(uint64, *raft.Log) error
-
-// Iterate through all command logs in the given store within the given range.
-func Iterate(logs raft.LogStore, r *Range, handler Handler) error {
- for index := r.First; index <= r.Last; index++ {
- log := &raft.Log{}
- if err := logs.GetLog(index, log); err != nil {
- return errors.Wrapf(err, "failed to get log %d", index)
- }
-
- if log.Type != raft.LogCommand {
- continue
- }
-
- if err := handler(index, log); err != nil {
- return err
- }
- }
-
- return nil
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/store/range.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/store/range.go
deleted file mode 100644
index 7b3380145e..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/store/range.go
+++ /dev/null
@@ -1,59 +0,0 @@
-package store
-
-import (
- "github.com/hashicorp/raft"
- "github.com/pkg/errors"
-)
-
-// DefaultRange returns a Range spanning all available indexes.
-func DefaultRange(logs raft.LogStore) (*Range, error) {
- first, err := logs.FirstIndex()
- if err != nil {
- return nil, errors.Wrap(err, "failed to get first index")
- }
- last, err := logs.LastIndex()
- if err != nil {
- return nil, errors.Wrap(err, "failed to get last index")
- }
-
- return &Range{First: first, Last: last}, nil
-}
-
-// HeadRange returns a range that includes only the first n entries.
-func HeadRange(logs raft.LogStore, n int) (*Range, error) {
- first, err := logs.FirstIndex()
- if err != nil {
- return nil, errors.Wrap(err, "failed to get first index")
- }
- last, err := logs.LastIndex()
- if err != nil {
- return nil, errors.Wrap(err, "failed to get last index")
- }
-
- if first+uint64(n) < last {
- last = first + uint64(n)
- }
-
- return &Range{First: first, Last: last}, nil
-}
-
-// TailRange returns a range that includes only the last n entries.
-func TailRange(logs raft.LogStore, n int) (*Range, error) {
- last, err := logs.LastIndex()
- if err != nil {
- return nil, errors.Wrap(err, "failed to get last index")
- }
-
- first := uint64(0)
- if last > uint64(n) {
- first = last - uint64(n)
- }
-
- return &Range{First: first, Last: last}, nil
-}
-
-// Range contains the first and last index of a dump.
-type Range struct {
- First uint64
- Last uint64
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/store/replay.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/store/replay.go
deleted file mode 100644
index a5ea4d7f8b..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/store/replay.go
+++ /dev/null
@@ -1,47 +0,0 @@
-package store
-
-import (
- "github.com/hashicorp/raft"
-)
-
-// Replay the commands in the given logs and snapshot stores using the given
-// dir as database directory.
-func Replay(logs raft.LogStore, snaps raft.SnapshotStore, r *Range, dir string) error {
- return nil
-
- /*
- // Create a registry and a FSM.
- registry := registry.New(dir)
- fsm := replication.NewFSM(registry)
-
- // Figure out if we have a snapshot to restore.
- metas, err := snaps.List()
- if err != nil {
- return errors.Wrap(err, "failed to get snapshots list")
- }
-
- if len(metas) > 0 {
- meta := metas[0] // The most recent.
- _, reader, err := snaps.Open(meta.ID)
- if err != nil {
- return errors.Wrapf(err, "failed to open snapshot %s", meta.ID)
- }
- if err := fsm.Restore(reader); err != nil {
- return errors.Wrapf(err, "failed to restore snapshot %s", meta.ID)
- }
-
- // Update the range
- r.First = meta.Index + 1
- }
-
- // Replay the logs.
- err = Iterate(logs, r, func(index uint64, log *raft.Log) error {
- fsm.Apply(log)
- return nil
- })
- if err != nil {
- errors.Wrap(err, "failed to iterate through the logs")
- }
- return nil
- */
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/buffer.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/buffer.go
deleted file mode 100644
index 6feec95d04..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/buffer.go
+++ /dev/null
@@ -1,88 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package trace
-
-import (
- "time"
-)
-
-// A circular buffer of trace entries.
-type buffer struct {
- // Fixed-size slice of entries in the buffer. When the slice fills, new
- // entries will replace old ones at the beginning on the slice.
- entries []entry
-
- // Track the position of the last entry in the buffer.
- cursor *cursor
-}
-
-// Create a new circular buffer of trace entries, retaining at most the given
-// number of entries.
-func newBuffer(n int) *buffer {
- return &buffer{
- entries: make([]entry, n),
- cursor: newCursor(0, n),
- }
-}
-
-// Append a new entry to the buffer, possibly replacing an older one.
-func (b *buffer) Append(timestamp time.Time, message string, args []interface{}, err error, fields *fields) {
- i := b.cursor.Position()
-
- b.entries[i].timestamp = timestamp
- b.entries[i].message = message
- for j := range b.entries[i].args {
- // Set arg j to either the provided arg or nil
- if j < len(args) {
- b.entries[i].args[j] = args[j]
- } else {
- b.entries[i].args[j] = nil
- }
- }
- b.entries[i].error = err
- b.entries[i].fields = fields
-
- b.cursor.Advance()
-}
-
-// Return the last inserted entry
-func (b *buffer) Last() entry {
- cursor := newCursor(b.cursor.Position(), len(b.entries))
- cursor.Retract()
- return b.entries[cursor.Position()]
-}
-
-// Return the list of current entries in the buffer.
-func (b *buffer) Entries() []entry {
- entries := make([]entry, 0)
-
- // We don't keep track of the actual number of entries in the buffer,
- // instead we iterate them backwards until we find a "null" entry.
- //
- // A "null" entry is detected by looking at its timestamp and seeting
- // that it's set to the Unix epoch.
- n := len(b.entries)
- cursor := newCursor(b.cursor.Position(), n)
- for i := 0; i < n; i++ {
- cursor.Retract()
- previous := b.entries[cursor.Position()]
- if previous.timestamp.Unix() == epoch {
- break
- }
- entries = append([]entry{previous}, entries...)
- }
-
- return entries
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/constants.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/constants.go
deleted file mode 100644
index f4e8c83846..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/constants.go
+++ /dev/null
@@ -1,21 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package trace
-
-// Maximum number of key/value fields each trace entry might have.
-const maxFields = 6
-
-// Maximum number of format arguments each trace entry might have.
-const maxArgs = 4
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/cursor.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/cursor.go
deleted file mode 100644
index f2810af43c..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/cursor.go
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package trace
-
-// A cursor holds the index of an entry of a circular buffer.
-type cursor struct {
- position int // Current position of the cursor
- length int // Lenght of the circular buffer.
-}
-
-func newCursor(position, length int) *cursor {
- return &cursor{
- position: position,
- length: length,
- }
-}
-
-func (c *cursor) Position() int {
- return c.position
-}
-
-func (c *cursor) Advance() {
- c.position = (c.position + c.length + 1) % c.length
-}
-
-func (c *cursor) Retract() {
- c.position = (c.position + c.length - 1) % c.length
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/doc.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/doc.go
deleted file mode 100644
index 875ab66ab2..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/doc.go
+++ /dev/null
@@ -1,17 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package trace implements a tracing system that can handle emitting large
-// amounts of entries with minimal performance overhead.
-package trace
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/entry.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/entry.go
deleted file mode 100644
index 3d6ea9ad60..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/entry.go
+++ /dev/null
@@ -1,65 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package trace
-
-import (
- "fmt"
- "time"
-)
-
-// A single trace entry.
-type entry struct {
- timestamp time.Time // Time at which the entry was created.
- message string // Message of the entry.
- args args // Additional format arguments for the message.
- error error // Error associated with the entry.
-
- // Key/value fields associated with the entry. This is a pointer
- // because all entries of a specific tracer share the same fields.
- fields *fields
-}
-
-// Timestamp returns a string representation of the entry's timestamp.
-func (e entry) Timestamp() string {
- return e.timestamp.Format("2006-01-02 15:04:05.00000")
-}
-
-// Message returns a string with the entry's message along with its fields,
-// arguments and error.
-func (e entry) Message() string {
- message := e.message
-
- if e.args[0] != nil {
- args := make([]interface{}, 0)
- for i := 0; e.args[i] != nil; i++ {
- args = append(args, e.args[i])
- }
- message = fmt.Sprintf(message, args...)
- }
-
- fields := ""
- for i := 0; i < len(e.fields) && e.fields[i].key != ""; i++ {
- fields += fmt.Sprintf("%s ", e.fields[i])
- }
-
- if e.error != nil {
- message += fmt.Sprintf(": %v", e.error)
- }
-
- return fmt.Sprintf("%s%s", fields, message)
-}
-
-type args [maxArgs]interface{}
-type fields [maxFields]Field
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/field.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/field.go
deleted file mode 100644
index f7f48cbc73..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/field.go
+++ /dev/null
@@ -1,56 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package trace
-
-import "fmt"
-
-// String returns a Field with a string value.
-func String(key string, value string) Field {
- return Field{
- key: key,
- isString: true,
- string: value,
- }
-}
-
-// Integer returns a Field with an integer value.
-func Integer(key string, value int64) Field {
- return Field{
- key: key,
- integer: value,
- }
-}
-
-// Field holds a single key/value pair in a trace Entry.
-type Field struct {
- key string // Name of the key
- isString bool // Whether the value is a string or an integer
- string string // String value
- integer int64 // Integer value
-}
-
-func (f Field) String() string {
- format := "%s="
- args := []interface{}{f.key}
- if f.isString {
- format += "%s"
- args = append(args, f.string)
- } else {
- format += "%d"
- args = append(args, f.integer)
- }
-
- return fmt.Sprintf(format, args...)
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/set.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/set.go
deleted file mode 100644
index 6fb40142c3..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/set.go
+++ /dev/null
@@ -1,131 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package trace
-
-import (
- "fmt"
- "sort"
- "sync"
- "testing"
- "time"
-)
-
-// Set manages the lifecycle of a set of Tracers.
-//
-// When Tracer.Panic() is invoked on any of the tracers of this, the entries of
-// all tracers in the set will be dumped as part of the panic message, ordered
-// by timestamp.
-type Set struct {
- tracers map[string]*Tracer // Index of available tracers by name.
- retain int // Number of entries each tracer will retain.
-
- mu sync.RWMutex
-
- // For testing only.
- now now // Function returning the current time.
- testing testing.TB // Emitted entries will also be sent to the test logger.
- node int // Index of the node emitting the entries.
-}
-
-// NewSet creates a new tracer Set.
-//
-// Each Set has a number of 'tracers', each holding a different buffer
-// of trace entries, and each retaining at most 'retain' entrier.
-func NewSet(retain int) *Set {
- return &Set{
- tracers: make(map[string]*Tracer),
- retain: retain,
- now: time.Now,
- }
-}
-
-// Add a new tracer to the registry.
-func (s *Set) Add(name string) *Tracer {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- _, ok := s.tracers[name]
- if ok {
- panic(fmt.Sprintf("a tracer named %s is already registered", name))
- }
- buffer := newBuffer(s.retain)
- tracer := newTracer(s, name, buffer)
- s.tracers[name] = tracer
- return tracer
-}
-
-// Get the tracer with the given name, add one if does not exists.
-func (s *Set) Get(name string) *Tracer {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- tracer, ok := s.tracers[name]
- if !ok {
- panic(fmt.Sprintf("no tracer named %s is registered", name))
- }
- return tracer
-}
-
-// Del removes the tracer with the given name.
-func (s *Set) Del(name string) {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- _, ok := s.tracers[name]
- if !ok {
- panic(fmt.Sprintf("no tracer named %s is registered", name))
- }
- delete(s.tracers, name)
-}
-
-// String returns a string representing all current entries, in all current
-// tracers, ordered by timestamp.
-func (s *Set) String() string {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- entries := make([]struct {
- e entry // Actual entry object
- t *Tracer // Tracer that emitted the entry
- }, 0)
-
- for _, tracer := range s.tracers {
- for _, e := range tracer.buffer.Entries() {
- entries = append(entries, struct {
- e entry
- t *Tracer
- }{e, tracer})
- }
- }
- sort.Slice(entries, func(i, j int) bool {
- return entries[i].e.timestamp.Before(entries[j].e.timestamp)
- })
-
- result := ""
-
- for _, entry := range entries {
- result += fmt.Sprintf(
- "%s: %s: %s\n", entry.e.Timestamp(), entry.t.name, entry.e.Message())
- }
-
- return result
-}
-
-// Testing sets the tracers to log emitted entries through the given testing
-// instance.
-func (s *Set) Testing(t testing.TB, node int) {
- s.testing = t
- s.node = node
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/time.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/time.go
deleted file mode 100644
index 74a687d60b..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/time.go
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package trace
-
-import "time"
-
-// A function returning the current time.
-type now func() time.Time
-
-// Nil value of a time.Time instance
-var epoch = (time.Time{}).Unix()
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/tracer.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/tracer.go
deleted file mode 100644
index 09167db5cf..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/trace/tracer.go
+++ /dev/null
@@ -1,96 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package trace
-
-import (
- "fmt"
-)
-
-// Tracer holds a buffer of recent trace entries in a trace Registry.
-type Tracer struct {
- set *Set // Set this tracer is part of.
- name string // Name of the tracer.
- buffer *buffer // Ring buffer for trace entries.
- fields fields // Tracer-specific key/value pairs.
-}
-
-// Creates a new tracer.
-func newTracer(set *Set, name string, buffer *buffer) *Tracer {
- return &Tracer{
- set: set,
- name: name,
- buffer: buffer,
- fields: fields{},
- }
-}
-
-// Message emits a new trace message.
-func (t *Tracer) Message(message string, args ...interface{}) {
- if n := len(args); n > maxArgs {
- panic(fmt.Sprintf("a trace entry can have at most %d args, but %d were given", maxArgs, n))
- }
- t.emit(message, args, nil)
-}
-
-// Emit a new trace entry with an error attached.
-func (t *Tracer) Error(message string, err error) {
- t.emit(message, nil, err)
-}
-
-// Panic causes a Go panic which will print all trace entries across all
-// tracers.
-func (t *Tracer) Panic(message string, v ...interface{}) {
- message = fmt.Sprintf(message, v...)
- if t.set.testing == nil {
- message += "\n\ntrace:\n" + t.set.String()
- }
- panic(message)
-}
-
-// With returns a new Tracer instance emitting entries in the same buffer of this
-// tracer, but with additional predefined fields.
-func (t *Tracer) With(fields ...Field) *Tracer {
- if n := len(fields); n > maxFields {
- panic(fmt.Sprintf("a trace entry can have at most %d fields, but %d were given", maxFields, n))
- }
-
- // Create the child tracer, cloning the parent and using its entries
- // buffer.
- tracer := newTracer(t.set, t.name, t.buffer)
-
- // Copy the fields of the parent into the child.
- i := 0
- for ; t.fields[i].key != ""; i++ {
- tracer.fields[i] = t.fields[i]
- }
-
- // Add the child fields.
- for j := range fields {
- tracer.fields[i+j] = fields[j]
- }
-
- return tracer
-}
-
-// Emit a new trace entry.
-func (t *Tracer) emit(message string, args []interface{}, err error) {
- t.buffer.Append(t.set.now(), message, args, err, &t.fields)
-
- if t.set.testing != nil {
- entry := t.buffer.Last()
- format := "%d: %s: %s: %s\n"
- t.set.testing.Logf(format, t.set.node, entry.Timestamp(), t.name, entry.Message())
- }
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/transaction/state.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/transaction/state.go
deleted file mode 100644
index 3ec38c4408..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/transaction/state.go
+++ /dev/null
@@ -1,67 +0,0 @@
-package transaction
-
-import (
- "github.com/ryanfaerman/fsm"
-)
-
-// Possible transaction states. Most states are associated with SQLite
-// replication hooks that are invoked upon transitioning from one lifecycle
-// state to the next.
-const (
- Pending = fsm.State("pending") // Initial state right after creation.
- Writing = fsm.State("writing") // After a non-commit frames command has been executed.
- Written = fsm.State("written") // After a final commit frames command has been executed.
- Undone = fsm.State("undone") // After an undo command has been executed.
- Doomed = fsm.State("doomed") // The transaction has errored.
-)
-
-// Create a new FSM initialized with a fresh state object set to Pending.
-func newMachine() fsm.Machine {
- return fsm.New(
- fsm.WithRules(newRules()),
- fsm.WithSubject(newState()),
- )
-}
-
-// Capture valid state transitions within a transaction.
-func newRules() fsm.Ruleset {
- rules := fsm.Ruleset{}
-
- for o, states := range transitions {
- for _, e := range states {
- rules.AddTransition(fsm.T{O: o, E: e})
- }
- }
-
- return rules
-}
-
-// Map of all valid state transitions.
-var transitions = map[fsm.State][]fsm.State{
- Pending: {Writing, Written, Undone},
- Writing: {Writing, Written, Undone, Doomed},
- Written: {Doomed},
- Undone: {Doomed},
-}
-
-// Track the state of transaction. Implements the fsm.Stater interface.
-type state struct {
- state fsm.State
-}
-
-// Return a new transaction state object, set to Pending.
-func newState() *state {
- return &state{
- state: Pending,
- }
-}
-
-// CurrentState returns the current state, implementing fsm.Stater.
-func (s *state) CurrentState() fsm.State {
- return s.state
-}
-
-// SetState switches the current state, implementing fsm.Stater.
-func (s *state) SetState(state fsm.State) {
- s.state = state
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/internal/transaction/txn.go b/vendor/github.com/CanonicalLtd/go-dqlite/internal/transaction/txn.go
deleted file mode 100644
index 76f64e1502..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/internal/transaction/txn.go
+++ /dev/null
@@ -1,237 +0,0 @@
-package transaction
-
-import (
- "fmt"
-
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
- "github.com/ryanfaerman/fsm"
-)
-
-// Txn captures information about an active WAL write transaction that has been
-// started on a SQLite connection configured to be in either leader or follower
-// replication mode.
-type Txn struct {
- conn *bindings.Conn // Underlying SQLite db.
- id uint64 // Transaction ID.
- machine fsm.Machine // Internal fsm for validating state changes.
- isLeader bool // Whether our connection is in leader mode.
- isZombie bool // Whether this is a zombie transaction, see Zombie().
- dryRun bool // Dry run mode, don't invoke actual SQLite hooks.
-
- // For leader transactions, these are the parameters of all non-commit
- // frames commands that were executed so far during this
- // transaction.
- //
- // They are used in case the last commit frames command failed with
- // ErrLeadershipLost, and either the same server gets re-elected or a
- // quorum was reached despite the glitch and another server was
- // elected. In that situation the server that lost leadership in the
- // first place will need to replay the whole transaction using a
- // follower connection, since its transaction (associated with a leader
- // connection) was rolled back by SQLite.
- frames []bindings.WalReplicationFrameInfo
-}
-
-// New creates a new Txn instance.
-func New(conn *bindings.Conn, id uint64) *Txn {
- return &Txn{
- conn: conn,
- id: id,
- machine: newMachine(),
- }
-}
-
-func (t *Txn) String() string {
- s := fmt.Sprintf("%d %s as ", t.id, t.State())
- if t.IsLeader() {
- s += "leader"
- if t.IsZombie() {
- s += " (zombie)"
- }
- } else {
- s += "follower"
- }
- return s
-}
-
-// Leader marks this transaction as a leader transaction.
-//
-// A leader transaction is automatically set to dry-run, since the SQLite will
-// trigger itself the relevant WAL APIs when transitioning between states.
-//
-// Depending on the particular replication hook being executed SQLite might do
-// that before or after the hook. See src/pager.c in SQLite source code for
-// details about when WAL APis are invoked exactly with respect to the various
-// sqlite3_replication_methods hooks.
-func (t *Txn) Leader() {
- if t.isLeader {
- panic("transaction is already marked as leader")
- }
- t.isLeader = true
- t.DryRun()
-}
-
-// IsLeader returns true if the underlying connection is in leader
-// replication mode.
-func (t *Txn) IsLeader() bool {
- return t.isLeader
-}
-
-// DryRun makes this transaction only transition between states, without
-// actually invoking the relevant SQLite APIs.
-//
-// This is used to create a surrogate follower, and for tests.
-func (t *Txn) DryRun() {
- if t.dryRun {
- panic("transaction is already in dry-run mode")
- }
- t.dryRun = true
-}
-
-// Conn returns the sqlite connection that started this write
-// transaction.
-func (t *Txn) Conn() *bindings.Conn {
- return t.conn
-}
-
-// ID returns the ID associated with this transaction.
-func (t *Txn) ID() uint64 {
- return t.id
-}
-
-// State returns the current state of the transaction.
-func (t *Txn) State() fsm.State {
- return t.machine.Subject.CurrentState()
-}
-
-// Frames writes frames to the WAL.
-func (t *Txn) Frames(begin bool, info bindings.WalReplicationFrameInfo) error {
- state := Writing
- if info.IsCommitGet() {
- state = Written
- }
- return t.transition(state, begin, info)
-}
-
-// Undo reverts all changes to the WAL since the start of the
-// transaction.
-func (t *Txn) Undo() error {
- return t.transition(Undone)
-}
-
-// Zombie marks this transaction as zombie. It must be called only for leader
-// transactions.
-//
-// A zombie transaction is one whose leader has lost leadership while applying
-// the associated FSM command. The transaction is left in state passed as
-// argument.
-func (t *Txn) Zombie() {
- if !t.isLeader {
- panic("follower transactions can't be marked as zombie")
- }
- if t.isZombie {
- panic("transaction is already marked as zombie")
- }
- t.isZombie = true
-}
-
-// IsZombie returns true if this is a zombie transaction.
-func (t *Txn) IsZombie() bool {
- if !t.isLeader {
- panic("follower transactions can't be zombie")
- }
- return t.isZombie
-}
-
-// Resurrect a zombie transaction.
-//
-// This should be called only on zombie transactions in Pending or Writing
-// state, in case a leader that lost leadership was re-elected right away or a
-// quorum for a lost commit frames command was reached and the new leader is
-// replicating it on the former leader.
-//
-// A new follower transaction will be created with the given connection (which
-// is assumed to be in follower replication mode), and set to the same ID as
-// this zombie.
-//
-// All preceeding non-commit frames commands (if any) will be re-applied on the
-// follower transaction.
-//
-// If no error occurrs, the newly created follower transaction is returned.
-func (t *Txn) Resurrect(conn *bindings.Conn) (*Txn, error) {
- if !t.isLeader {
- panic("attempt to resurrect follower transaction")
- }
- if !t.isZombie {
- panic("attempt to resurrect non-zombie transaction")
- }
- if t.State() != Pending && t.State() != Writing {
- panic("attempt to resurrect a transaction not in pending or writing state")
- }
- txn := New(conn, t.ID())
-
- for i, frames := range t.frames {
- begin := i == 0
- if err := txn.transition(Writing, begin, frames); err != nil {
- return nil, err
- }
- }
-
- return txn, nil
-}
-
-// Try to transition to the given state. If the transition is invalid,
-// panic out.
-func (t *Txn) transition(state fsm.State, args ...interface{}) error {
- if err := t.machine.Transition(state); err != nil {
- panic(fmt.Sprintf("invalid %s -> %s transition", t.State(), state))
- }
-
- if t.isLeader {
- // In leader mode, don't actually invoke SQLite replication
- // API, since that will be done by SQLite internally.
- switch state {
- case Writing:
- // Save non-commit frames in case the last commit fails
- // and gets recovered by the same leader.
- begin := args[0].(bool)
- frames := args[1].(bindings.WalReplicationFrameInfo)
- if begin {
- t.frames = append(t.frames, frames)
- }
- case Written:
- fallthrough
- case Undone:
- // Reset saved frames. They are not going to be used
- // anymore and they help garbage-collecting them, since
- // the tracer holds references to a number of
- // transaction objects.
- t.frames = nil
- }
- }
-
- if t.dryRun {
- // In dry run mode, don't actually invoke any SQLite API.
- return nil
- }
-
- var err error
- switch state {
- case Writing:
- fallthrough
- case Written:
- //begin := args[0].(bool)
- info := args[1].(bindings.WalReplicationFrameInfo)
- err = t.conn.WalReplicationFrames(info)
- case Undone:
- err = t.conn.WalReplicationUndo()
- }
-
- if err != nil {
- if err := t.machine.Transition(Doomed); err != nil {
- panic(fmt.Sprintf("cannot doom from %s", t.State()))
- }
- }
-
- return err
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/log.go b/vendor/github.com/CanonicalLtd/go-dqlite/log.go
deleted file mode 100644
index 6f2106a124..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/log.go
+++ /dev/null
@@ -1,33 +0,0 @@
-package dqlite
-
-import (
- "fmt"
- "log"
- "os"
-
- "github.com/CanonicalLtd/go-dqlite/internal/logging"
-)
-
-// LogFunc is a function that can be used for logging.
-type LogFunc = logging.Func
-
-// LogLevel defines the logging level.
-type LogLevel = logging.Level
-
-// Available logging levels.
-const (
- LogDebug = logging.Debug
- LogInfo = logging.Info
- LogWarn = logging.Warn
- LogError = logging.Error
-)
-
-// Create a LogFunc with reasonable defaults.
-func defaultLogFunc() LogFunc {
- logger := log.New(os.Stdout, "", log.LstdFlags|log.Lmicroseconds)
-
- return func(l LogLevel, format string, a ...interface{}) {
- format = fmt.Sprintf("[%s]: %s", l.String(), format)
- logger.Printf(format, a...)
- }
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/recover/delete/delete.go b/vendor/github.com/CanonicalLtd/go-dqlite/recover/delete/delete.go
deleted file mode 100644
index d6355b81cd..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/recover/delete/delete.go
+++ /dev/null
@@ -1,36 +0,0 @@
-package delete
-
-import (
- "fmt"
-
- "github.com/CanonicalLtd/go-dqlite/internal/store"
- "github.com/hashicorp/raft"
- "github.com/pkg/errors"
-)
-
-// Delete the given log entries from the given store.
-func Delete(logs raft.LogStore, index uint64) error {
- r, err := store.DefaultRange(logs)
- if err != nil {
- return errors.Wrap(err, "failed to get current log store range")
- }
-
- found := false
-
- store.Iterate(logs, r, func(i uint64, log *raft.Log) error {
- if i == index {
- found = true
- }
- return nil
- })
-
- if !found {
- return fmt.Errorf("log %d not found", index)
- }
-
- if err := logs.DeleteRange(index, r.Last); err != nil {
- return errors.Wrapf(err, "failed to delete range %d -> %d", index, r.Last)
- }
-
- return nil
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/recover/dump/dump.go b/vendor/github.com/CanonicalLtd/go-dqlite/recover/dump/dump.go
deleted file mode 100644
index 9b1f00a1e3..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/recover/dump/dump.go
+++ /dev/null
@@ -1,105 +0,0 @@
-package dump
-
-import (
- "fmt"
- "io"
- "log"
-
- "github.com/CanonicalLtd/go-dqlite/internal/protocol"
- "github.com/CanonicalLtd/go-dqlite/internal/store"
- "github.com/hashicorp/raft"
- "github.com/pkg/errors"
-)
-
-// Dump the content of a dqlite store.
-func Dump(logs raft.LogStore, snaps raft.SnapshotStore, out io.Writer, options ...Option) error {
- o := defaultOptions()
- for _, option := range options {
- err := option(logs, o)
- if err != nil {
- return err
- }
- }
-
- if o.r == nil {
- r, err := store.DefaultRange(logs)
- if err != nil {
- return err
- }
- o.r = r
- }
-
- if o.dir != "" {
- // Replay the logs.
- if err := store.Replay(logs, snaps, o.r, o.dir); err != nil {
- return errors.Wrap(err, "failed to replay logs")
- }
- return nil
- }
-
- logger := log.New(out, "", 0)
-
- h := func(index uint64, log *raft.Log) error {
- cmd, err := protocol.UnmarshalCommand(log.Data)
- if err != nil {
- return errors.Wrapf(err, "index %d: failed to unmarshal command", index)
- }
-
- logger.Printf(dumpCommand(index, cmd))
- return nil
- }
-
- return store.Iterate(logs, o.r, h)
-}
-
-func dumpCommand(index uint64, cmd *protocol.Command) string {
- var name string
- var dump string
- switch payload := cmd.Payload.(type) {
- case *protocol.Command_Open:
- name = "open"
- dump = dumpOpen(payload.Open)
- case *protocol.Command_Begin:
- name = "begin"
- dump = dumpBegin(payload.Begin)
- case *protocol.Command_Frames:
- name = "frames"
- dump = dumpFrames(payload.Frames)
- case *protocol.Command_Undo:
- name = "undo"
- dump = dumpUndo(payload.Undo)
- case *protocol.Command_End:
- name = "end"
- dump = dumpEnd(payload.End)
- case *protocol.Command_Checkpoint:
- name = "checkpoint"
- dump = dumpCheckpoint(payload.Checkpoint)
- }
-
- return fmt.Sprintf("index %6d: %-8s: %s", index, name, dump)
-}
-
-func dumpOpen(params *protocol.Open) string {
- return fmt.Sprintf("name: %8s", params.Name)
-}
-
-func dumpBegin(params *protocol.Begin) string {
- return fmt.Sprintf("name: %8s txn: %6d", params.Name, params.Txid)
-}
-
-func dumpFrames(params *protocol.Frames) string {
- return fmt.Sprintf("name: %8s txn: %6d commit: %d pages: %2d",
- params.Filename, params.Txid, params.IsCommit, len(params.PageNumbers))
-}
-
-func dumpUndo(params *protocol.Undo) string {
- return fmt.Sprintf("txn: %6d", params.Txid)
-}
-
-func dumpEnd(params *protocol.End) string {
- return fmt.Sprintf("txn: %6d", params.Txid)
-}
-
-func dumpCheckpoint(params *protocol.Checkpoint) string {
- return fmt.Sprintf("file: %-8s", params.Name)
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/recover/dump/options.go b/vendor/github.com/CanonicalLtd/go-dqlite/recover/dump/options.go
deleted file mode 100644
index a443cdb8b6..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/recover/dump/options.go
+++ /dev/null
@@ -1,56 +0,0 @@
-package dump
-
-import (
- "github.com/CanonicalLtd/go-dqlite/internal/store"
- "github.com/hashicorp/raft"
-)
-
-// Option to tweak the output of dump
-type Option func(logs raft.LogStore, o *options) error
-
-// Tail limits the output to the last N entries.
-func Tail(n int) Option {
- return func(logs raft.LogStore, o *options) error {
- r, err := store.TailRange(logs, n)
- if err != nil {
- return err
- }
-
- o.r = r
-
- return nil
- }
-}
-
-// Head limits the output to the first N entries.
-func Head(n int) Option {
- return func(logs raft.LogStore, o *options) error {
- r, err := store.HeadRange(logs, n)
- if err != nil {
- return err
- }
-
- o.r = r
-
- return nil
- }
-}
-
-// Replay the commands generating SQLite databases in the given dir.
-func Replay(dir string) Option {
- return func(logs raft.LogStore, o *options) error {
- o.dir = dir
- return nil
- }
-}
-
-// Hold options for the Dump function.
-type options struct {
- r *store.Range
- dir string
-}
-
-// Return the default Dump options.
-func defaultOptions() *options {
- return &options{}
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/recover/open.go b/vendor/github.com/CanonicalLtd/go-dqlite/recover/open.go
deleted file mode 100644
index bdc57a4bce..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/recover/open.go
+++ /dev/null
@@ -1,30 +0,0 @@
-package recover
-
-import (
- "io/ioutil"
- "os"
- "path/filepath"
-
- "github.com/hashicorp/raft"
- "github.com/hashicorp/raft-boltdb"
- "github.com/pkg/errors"
-)
-
-// Open a raft store in the given dir.
-func Open(dir string) (raft.LogStore, raft.SnapshotStore, error) {
- if _, err := os.Stat(dir); err != nil {
- return nil, nil, errors.Wrap(err, "invalid raft data dir")
- }
-
- logs, err := raftboltdb.NewBoltStore(filepath.Join(dir, "logs.db"))
- if err != nil {
- return nil, nil, errors.Wrap(err, "failed to open boltdb file")
- }
-
- snaps, err := raft.NewFileSnapshotStore(dir, 1, ioutil.Discard)
- if err != nil {
- return nil, nil, errors.Wrap(err, "failed to open snapshot store")
- }
-
- return logs, snaps, nil
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/registry.go b/vendor/github.com/CanonicalLtd/go-dqlite/registry.go
deleted file mode 100644
index 33a180e773..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/registry.go
+++ /dev/null
@@ -1,50 +0,0 @@
-package dqlite
-
-import (
- "fmt"
-
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
- "github.com/CanonicalLtd/go-dqlite/internal/logging"
- "github.com/CanonicalLtd/go-dqlite/internal/registry"
-)
-
-// Registry tracks internal data shared by the dqlite Driver and FSM.
-type Registry struct {
- name string
- vfs *bindings.Vfs
- logger *bindings.Logger
- registry *registry.Registry
-}
-
-// NewRegistry creates a new Registry, which is expected to be passed to both
-// NewFSM and NewDriver.
-//
-// The ID parameter is a string identifying the local node.
-func NewRegistry(id string) *Registry {
- return NewRegistryWithLogger(id, logging.Stdout())
-}
-
-// NewRegistryWithLogger returns a registry configured with the given logger.
-func NewRegistryWithLogger(id string, log LogFunc) *Registry {
- name := fmt.Sprintf("dqlite-%s", id)
-
- logger := bindings.NewLogger(log)
-
- vfs, err := bindings.NewVfs(name, logger)
- if err != nil {
- panic("failed to register VFS")
- }
-
- return &Registry{
- name: name,
- vfs: vfs,
- registry: registry.New(vfs),
- logger: logger,
- }
-}
-
-// Close the registry.
-func (r *Registry) Close() {
- r.vfs.Close()
- r.logger.Close()
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/run-demo b/vendor/github.com/CanonicalLtd/go-dqlite/run-demo
deleted file mode 100755
index dfdb280059..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/run-demo
+++ /dev/null
@@ -1,62 +0,0 @@
-#!/usr/bin/python3
-
-import os
-import sys
-import shutil
-import subprocess
-import time
-
-BUILD_DQLITE = "go build -tags libsqlite3 .".split(" ")
-BUILD_DEMO = "go build -tags libsqlite3 testdata/demo.go".split(" ")
-BASE_PORT = 9980
-
-
-def spawnNode(n, purge=False):
- data = "/tmp/dqlite-demo-%d" % n
- port = BASE_PORT + n
-
- if purge and os.path.exists(data):
- shutil.rmtree(data)
- if not os.path.exists(data):
- os.mkdir(data)
-
- args = ["./demo", "-data", data, "-addr", "127.0.0.1:%d" % port]
- if n > 0:
- args.extend(["-join", "127.0.0.1:%d" % BASE_PORT])
-
- env = os.environ.copy()
- env.update({
- "LD_LIBRARY_PATH": os.path.join(os.getcwd(), ".sqlite"),
- })
-
- fd = open(os.path.join(data, "log"), "a+")
- return subprocess.Popen(args, env=env, stdout=fd, stderr=fd)
-
-def spawnTail():
- args = ["tail", "-f"]
- for i in range(3):
- args.append("/tmp/dqlite-demo-%d/log" % i)
- return subprocess.Popen(args, stderr=subprocess.STDOUT)
-
-if __name__ == "__main__":
- subprocess.check_call(BUILD_DQLITE)
- subprocess.check_call(BUILD_DEMO)
-
- processes = []
- for i in range(3):
- processes.append(spawnNode(i, purge=True))
- spawnTail()
-
- while True:
- for i, process in enumerate(processes):
- rc = process.poll()
- if rc is not None:
- if rc != 0:
- print("ERROR node %d exited with return code %d" % (i, rc))
- for j, process in enumerate(processes):
- if j != i:
- process.kill()
- sys.exit(1)
- # respawn
- processes[i] = spawnNode(i)
- time.sleep(0.5)
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/server.go b/vendor/github.com/CanonicalLtd/go-dqlite/server.go
deleted file mode 100644
index ce06510454..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/server.go
+++ /dev/null
@@ -1,233 +0,0 @@
-package dqlite
-
-import (
- "fmt"
- "io/ioutil"
- "net"
- "path/filepath"
- "runtime"
- "time"
-
- "github.com/CanonicalLtd/go-dqlite/internal/bindings"
- "github.com/CanonicalLtd/go-dqlite/internal/replication"
- "github.com/hashicorp/raft"
- "github.com/pkg/errors"
-)
-
-// Server implements the dqlite network protocol.
-type Server struct {
- log LogFunc // Logger
- registry *Registry // Registry wrapper
- server *bindings.Server // Low-level C implementation
- listener net.Listener // Queue of new connections
- runCh chan error // Receives the low-level C server return code
- acceptCh chan error // Receives connection handling errors
- replication *bindings.WalReplication
- logger *bindings.Logger
- cluster *bindings.Cluster
-}
-
-// ServerOption can be used to tweak server parameters.
-type ServerOption func(*serverOptions)
-
-// WithServerLogFunc sets a custom log function for the server.
-func WithServerLogFunc(log LogFunc) ServerOption {
- return func(options *serverOptions) {
- options.Log = log
- }
-}
-
-// WithServerAddressProvider sets a custom resolver for server addresses.
-func WithServerAddressProvider(provider raft.ServerAddressProvider) ServerOption {
- return func(options *serverOptions) {
- options.AddressProvider = provider
- }
-}
-
-// NewServer creates a new Server instance.
-func NewServer(raft *raft.Raft, registry *Registry, listener net.Listener, options ...ServerOption) (*Server, error) {
- o := defaultServerOptions()
-
- for _, option := range options {
- option(o)
- }
-
- replication, err := newWalReplication(registry, raft)
- if err != nil {
- return nil, err
- }
-
- cluster, err := newCluster(registry, raft, o.AddressProvider)
- if err != nil {
- return nil, err
- }
-
- server, err := bindings.NewServer(cluster)
- if err != nil {
- return nil, err
- }
-
- logger := bindings.NewLogger(o.Log)
-
- server.SetLogger(logger)
- server.SetVfs(registry.name)
- server.SetWalReplication(registry.name)
-
- s := &Server{
- log: o.Log,
- registry: registry,
- server: server,
- listener: listener,
- runCh: make(chan error),
- acceptCh: make(chan error, 1),
- logger: logger,
- cluster: cluster,
- replication: replication,
- }
-
- go s.run()
-
- if !s.server.Ready() {
- return nil, fmt.Errorf("server failed to start")
- }
-
- go s.acceptLoop()
-
- return s, nil
-}
-
-func newWalReplication(registry *Registry, raft *raft.Raft) (*bindings.WalReplication, error) {
- methods := replication.NewMethods(registry.registry, raft)
-
- replication, err := bindings.NewWalReplication(registry.name, methods)
- if err != nil {
- return nil, errors.Wrap(err, "failed to register WAL replication")
- }
-
- return replication, nil
-}
-
-func newCluster(registry *Registry, raft *raft.Raft, provider raft.ServerAddressProvider) (*bindings.Cluster, error) {
- methods := &cluster{
- raft: raft,
- registry: registry.registry,
- provider: provider,
- }
-
- return bindings.NewCluster(methods)
-}
-
-// Hold configuration options for a dqlite server.
-type serverOptions struct {
- Log LogFunc
- AddressProvider raft.ServerAddressProvider
-}
-
-// Run the server.
-func (s *Server) run() {
- runtime.LockOSThread()
- defer runtime.UnlockOSThread()
-
- s.runCh <- s.server.Run()
-}
-
-func (s *Server) acceptLoop() {
- s.log(LogDebug, "accepting connections")
-
- for {
- conn, err := s.listener.Accept()
- if err != nil {
- s.acceptCh <- nil
- return
- }
-
- err = s.server.Handle(conn)
- if err != nil {
- if err == bindings.ErrServerStopped {
- // Ignore failures due to the server being
- // stopped.
- err = nil
- }
- s.acceptCh <- err
- return
- }
- }
-}
-
-// Dump the files of a database to disk.
-func (s *Server) Dump(name string, dir string) error {
- // Dump the database file.
- bytes, err := s.registry.vfs.ReadFile(name)
- if err != nil {
- return errors.Wrap(err, "failed to get database file content")
- }
-
- path := filepath.Join(dir, name)
- if err := ioutil.WriteFile(path, bytes, 0600); err != nil {
- return errors.Wrap(err, "failed to write database file")
- }
-
- // Dump the WAL file.
- bytes, err = s.registry.vfs.ReadFile(name + "-wal")
- if err != nil {
- return errors.Wrap(err, "failed to get WAL file content")
- }
-
- path = filepath.Join(dir, name+"-wal")
- if err := ioutil.WriteFile(path, bytes, 0600); err != nil {
- return errors.Wrap(err, "failed to write WAL file")
- }
-
- return nil
-}
-
-// Close the server, releasing all resources it created.
-func (s *Server) Close() error {
- // Close the listener, which will make the listener.Accept() call in
- // acceptLoop() return an error.
- if err := s.listener.Close(); err != nil {
- return err
- }
-
- // Wait for the acceptLoop goroutine to exit.
- select {
- case err := <-s.acceptCh:
- if err != nil {
- return errors.Wrap(err, "accept goroutine failed")
- }
- case <-time.After(time.Second):
- return fmt.Errorf("accept goroutine did not stop within a second")
- }
-
- // Send a stop signal to the dqlite event loop.
- if err := s.server.Stop(); err != nil {
- return errors.Wrap(err, "server failed to stop")
- }
-
- // Wait for the run goroutine to exit.
- select {
- case err := <-s.runCh:
- if err != nil {
- return errors.Wrap(err, "accept goroutine failed")
- }
- case <-time.After(time.Second):
- return fmt.Errorf("server did not stop within a second")
- }
-
- s.server.Close()
-
- s.logger.Close()
- s.cluster.Close()
- s.replication.Close()
- s.registry.Close()
-
- return nil
-}
-
-// Create a serverOptions object with sane defaults.
-func defaultServerOptions() *serverOptions {
- return &serverOptions{
- Log: defaultLogFunc(),
- AddressProvider: nil,
- }
-}
diff --git a/vendor/github.com/CanonicalLtd/go-dqlite/store.go b/vendor/github.com/CanonicalLtd/go-dqlite/store.go
deleted file mode 100644
index bb1255cda2..0000000000
--- a/vendor/github.com/CanonicalLtd/go-dqlite/store.go
+++ /dev/null
@@ -1,141 +0,0 @@
-package dqlite
-
-import (
- "context"
- "database/sql"
- "fmt"
-
- "github.com/pkg/errors"
-
- "github.com/CanonicalLtd/go-dqlite/internal/client"
- _ "github.com/mattn/go-sqlite3" // Go SQLite bindings
-)
-
-// ServerInfo holds information about a single server.
-type ServerInfo = client.ServerInfo
-
-// ServerStore is used by a dqlite client to get an initial list of candidate
-// dqlite server addresses that it can dial in order to find a leader dqlite
-// server to use.
-//
-// Once connected, the client periodically updates the addresses in the store
-// by querying the leader server about changes in the cluster (such as servers
-// being added or removed).
-type ServerStore = client.ServerStore
-
-// InmemServerStore keeps the list of target gRPC SQL servers in memory.
-type InmemServerStore = client.InmemServerStore
-
-// NewInmemServerStore creates ServerStore which stores its data in-memory.
-var NewInmemServerStore = client.NewInmemServerStore
-
-// DatabaseServerStore persists a list addresses of dqlite servers in a SQL table.
-type DatabaseServerStore struct {
- db *sql.DB // Database handle to use.
- schema string // Name of the schema holding the servers table.
- table string // Name of the servers table.
- column string // Column name in the servers table holding the server address.
-}
-
-// DefaultServerStore creates a new ServerStore using the given filename to
-// open a SQLite database, with default names for the schema, table and column
-// parameters.
-//
-// It also creates the table if it doesn't exist yet.
-func DefaultServerStore(filename string) (*DatabaseServerStore, error) {
- // Open the database.
- db, err := sql.Open("sqlite3", filename)
- if err != nil {
- return nil, errors.Wrap(err, "failed to open database")
- }
-
- // Since we're setting SQLite single-thread mode, we need to have one
- // connection at most.
- db.SetMaxOpenConns(1)
-
- // Create the servers table if it does not exist yet.
- _, err = db.Exec("CREATE TABLE IF NOT EXISTS servers (address TEXT, UNIQUE(address))")
- if err != nil {
- return nil, errors.Wrap(err, "failed to create servers table")
- }
-
- store := NewServerStore(db, "main", "servers", "address")
-
- return store, nil
-}
-
-// NewServerStore creates a new ServerStore.
-func NewServerStore(db *sql.DB, schema, table, column string) *DatabaseServerStore {
- return &DatabaseServerStore{
- db: db,
- schema: schema,
- table: table,
- column: column,
- }
-}
-
-// Get the current servers.
-func (d *DatabaseServerStore) Get(ctx context.Context) ([]ServerInfo, error) {
- tx, err := d.db.Begin()
- if err != nil {
- return nil, errors.Wrap(err, "failed to begin transaction")
- }
- defer tx.Rollback()
-
- query := fmt.Sprintf("SELECT %s FROM %s.%s", d.column, d.schema, d.table)
- rows, err := tx.QueryContext(ctx, query)
- if err != nil {
- return nil, errors.Wrap(err, "failed to query servers table")
- }
- defer rows.Close()
-
- servers := make([]ServerInfo, 0)
- for rows.Next() {
- var address string
- err := rows.Scan(&address)
- if err != nil {
- return nil, errors.Wrap(err, "failed to fetch server address")
- }
- servers = append(servers, ServerInfo{ID: 1, Address: address})
- }
- if err := rows.Err(); err != nil {
- return nil, errors.Wrap(err, "result set failure")
- }
-
- return servers, nil
-}
-
-// Set the servers addresses.
-func (d *DatabaseServerStore) Set(ctx context.Context, servers []ServerInfo) error {
- tx, err := d.db.Begin()
- if err != nil {
- return errors.Wrap(err, "failed to begin transaction")
- }
-
- query := fmt.Sprintf("DELETE FROM %s.%s", d.schema, d.table)
- if _, err := tx.ExecContext(ctx, query); err != nil {
- tx.Rollback()
- return errors.Wrap(err, "failed to delete existing servers rows")
- }
-
- query = fmt.Sprintf("INSERT INTO %s.%s(%s) VALUES (?)", d.schema, d.table, d.column)
- stmt, err := tx.PrepareContext(ctx, query)
- if err != nil {
- tx.Rollback()
- return errors.Wrap(err, "failed to prepare insert statement")
- }
- defer stmt.Close()
-
- for _, server := range servers {
- if _, err := stmt.ExecContext(ctx, server.Address); err != nil {
- tx.Rollback()
- return errors.Wrapf(err, "failed to insert server %s", server.Address)
- }
- }
-
- if err := tx.Commit(); err != nil {
- return errors.Wrap(err, "failed to commit transaction")
- }
-
- return nil
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-http/AUTHORS b/vendor/github.com/CanonicalLtd/raft-http/AUTHORS
deleted file mode 100644
index 6e13f86ebb..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-http/AUTHORS
+++ /dev/null
@@ -1 +0,0 @@
-Free Ekanayaka <free.ekanayaka at canonical.com>
diff --git a/vendor/github.com/CanonicalLtd/raft-http/LICENSE b/vendor/github.com/CanonicalLtd/raft-http/LICENSE
deleted file mode 100644
index 261eeb9e9f..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-http/LICENSE
+++ /dev/null
@@ -1,201 +0,0 @@
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/vendor/github.com/CanonicalLtd/raft-http/README.md b/vendor/github.com/CanonicalLtd/raft-http/README.md
deleted file mode 100644
index 5a7bd20085..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-http/README.md
+++ /dev/null
@@ -1,14 +0,0 @@
-raft-http [](https://travis-ci.org/CanonicalLtd/raft-http) [](https://coveralls.io/github/CanonicalLtd/raft-http?branch=master) [](https://goreportcard.com/report/github.com/CanonicalLtd/raft-http) [](https://godoc.org/github.com/CanonicalLtd/raft-http)
-=========
-
-This repository provides the `rafthttp` package, which can be used to
-establish a network connection between to raft nodes using HTTP. Once
-the HTTP connection is established, the Upgrade header will be used to
-switch it to raw TCP mode, and the regular TCP-based network transport
-of the `raft` [package](https://github.com/hashicorp/raft) can take it
-from there.
-
-Documentation
-==============
-
-The documentation for this package can be found on [Godoc](http://godoc.org/github.com/CanonicalLtd/raft-http).
diff --git a/vendor/github.com/CanonicalLtd/raft-http/dial.go b/vendor/github.com/CanonicalLtd/raft-http/dial.go
deleted file mode 100644
index c11c029431..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-http/dial.go
+++ /dev/null
@@ -1,50 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package rafthttp
-
-import (
- "crypto/tls"
- "net"
- "time"
-)
-
-// Dial is a function that given an address and a timeout returns a
-// new network connection (typically TCP or TLS over TCP).
-type Dial func(addr string, timeout time.Duration) (net.Conn, error)
-
-// NewDialTCP returns a Dial function that establishes a network
-// connection using raw TCP.
-func NewDialTCP() Dial {
- dial := func(addr string, timeout time.Duration) (net.Conn, error) {
- dialer := newDialerWithTimeout(timeout)
- return dialer.Dial("tcp", addr)
- }
- return dial
-}
-
-// NewDialTLS returns a Dial function that enstablishes a network
-// connection using TLS over TCP.
-func NewDialTLS(config *tls.Config) Dial {
- dial := func(addr string, timeout time.Duration) (net.Conn, error) {
- dialer := newDialerWithTimeout(timeout)
- return tls.DialWithDialer(dialer, "tcp", addr, config)
- }
- return dial
-}
-
-// Convenience to create a Dialer configured with the give timeout.
-func newDialerWithTimeout(timeout time.Duration) *net.Dialer {
- return &net.Dialer{Timeout: timeout}
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-http/doc.go b/vendor/github.com/CanonicalLtd/raft-http/doc.go
deleted file mode 100644
index 39e4ea1d91..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-http/doc.go
+++ /dev/null
@@ -1,67 +0,0 @@
-// Package rafthttp provides an extension for the github.com/hashicorp/raft
-// package.
-//
-// It implements a raft.StreamLayer that a raft.NetworkTransport can use to
-// connect to and accept connections from other raft.Transport's using
-// HTTP/WebSocket rather than straight TCP.
-//
-// This is handy for applications that expose an HTTP endpoint and don't want
-// to open an extra TCP port for handling raft-level traffic.
-//
-// In addition to the regular raft.StreamLayer interface, rafthttp.Layer
-// implements extra methods to join and leave a cluster.
-//
-// Typical usage of this package is as follows:
-//
-// - Create a rafthttp.Handler object which implements the standard
-// http.Handler interface.
-//
-// - Create a standard http.Server and configure it to route an endpoint path
-// of your choice to the rafthttp.Handler above. All your raft servers must
-// use the same endpoint path. You'll probably want to gate the
-// rafthttp.Handler behind some authorization mechanism of your choice.
-//
-// - Create a net.Listener and use it to start a the http.Server create
-// above. From this point the rafthttp.Handler will start accepting
-// raft-related requests.
-//
-// - Create a rafthttp.Layer object passing it:
-//
-// 1) The endpoint path you chose above, which will be used to establish
-// outbound raft.Transport connections to other raft servers over
-// HTTP/WebSocket.
-//
-// 2) The network address of the net.Listener you used to start the
-// http.Server, which will be used by the local raft server to know its
-// own network address.
-//
-// 3) The rafthttp.Handler object you created above, which will be used to
-// accept inbound raft.NetworkTransport connections from other raft
-// servers over HTTP/WebSocket.
-//
-// 4) A rafthttp.Dial function, which will be used to establish outbound
-// raft.NetworkTransport connections to other raft servers over
-// HTTP/WebSocket (the rafthttp.Layer will use it to perform HTTP requests
-// to other servers using your chosen endpoint path).
-//
-// - Create a raft.NetworkTransport passing it the rafthttp.Layer you created
-// above.
-//
-// - Create a raft.Raft server using the raft.NetworkTransport created above.
-//
-// - Spawn a goroutine running the raftmembership.HandleChangeRequests function
-// from the github.com/Canonical/raft-membership package, passing it the
-// raft.Raft server you created above and the channel returned by Request()
-// method of the rafthttp.Handler created above. This will process join and
-// leave requests, that you can perform using the Join() and Leave() methods
-// of the rafthttp.Layer object you created above. This goroutine will
-// terminate automatically when you shutdown your raft.Raft server, since
-// that will close your raft.NetworkTransport, which in turn closes the your
-// rafttest.Layer, which closes your rafttest.Handler, which will ultimately
-// close the channel returned by its Requests() method and signal the
-// raftmembership.HandleChangeRequests function to return.
-//
-// To cleanly shutdown the service, first shutdown your raft.Raft instance,
-// then call the CloseStreams() method of your raft.NetworkTransport instance
-// (to close all connections) and then stop your http.Server.
-package rafthttp
diff --git a/vendor/github.com/CanonicalLtd/raft-http/handler.go b/vendor/github.com/CanonicalLtd/raft-http/handler.go
deleted file mode 100644
index 2a0d9d7355..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-http/handler.go
+++ /dev/null
@@ -1,212 +0,0 @@
-package rafthttp
-
-import (
- "fmt"
- "log"
- "net"
- "net/http"
- "net/url"
- "os"
- "sync"
- "time"
-
- "github.com/CanonicalLtd/raft-membership"
- "github.com/hashicorp/raft"
- "github.com/pkg/errors"
-)
-
-// Handler implements an HTTP handler that will look for an Upgrade
-// header in the request to switch the HTTP connection to raw TCP
-// mode, so it can be used as raft.NetworkTransport stream.
-type Handler struct {
- requests chan *raftmembership.ChangeRequest // Membership requests are pushed to this channel
- connections chan net.Conn // New Raft connections are pushed to this channel.
- shutdown chan struct{} // Used to stop processing membership requests.
- timeout time.Duration // Maximum time to wait for requests to be processed.
- logger *log.Logger // Logger to use.
- mu sync.RWMutex // Blocks closing until all membership requests are handled
-}
-
-// NewHandler returns a new Handler.
-//
-// Incoming raft membership requests (received via POST and DELETE) are
-// forwarded to the given channel, which is supposed to be processed using
-// raftmembership.HandleChangeRequests().
-func NewHandler() *Handler {
- logger := log.New(os.Stderr, "", log.LstdFlags)
- return NewHandlerWithLogger(logger)
-}
-
-// NewHandlerWithLogger returns a new Handler configured with the given logger.
-func NewHandlerWithLogger(logger *log.Logger) *Handler {
- return &Handler{
- requests: make(chan *raftmembership.ChangeRequest),
- connections: make(chan net.Conn),
- shutdown: make(chan struct{}),
- timeout: 10 * time.Second,
- logger: logger,
- }
-}
-
-// Requests returns a channel of inbound Raft membership change requests
-// received over HTTP. Consumer code is supposed to process this channel by
-// invoking raftmembership.HandleChangeRequests.
-func (h *Handler) Requests() <-chan *raftmembership.ChangeRequest {
- return h.requests
-}
-
-// Timeout sets the maximum amount of time for a request to be processed. It
-// defaults to 10 seconds if not set.
-func (h *Handler) Timeout(timeout time.Duration) {
- h.timeout = timeout
-}
-
-// Close stops handling incoming requests.
-func (h *Handler) Close() {
- close(h.shutdown)
-
- // Block until all pending requests are done. After that no new
- // requests will be sent to the requests channel since the shutdown
- // channel is closed.
- h.mu.Lock()
- defer h.mu.Unlock()
-
- close(h.requests)
-}
-
-// ServerHTTP upgrades the given HTTP connection to a raw TCP one for
-// use by raft.
-func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
- switch r.Method {
- case "GET":
- h.handleGet(w, r)
- case "POST":
- h.handlePost(w, r)
- case "DELETE":
- h.handleDelete(w, r)
- default:
- http.Error(w, "unknown action", http.StatusMethodNotAllowed)
- }
-}
-
-func (h *Handler) handleGet(w http.ResponseWriter, r *http.Request) {
- if r.Header.Get("Upgrade") != "raft" {
- http.Error(w, "missing or invalid upgrade header", http.StatusBadRequest)
- return
- }
-
- hijacker, ok := w.(http.Hijacker)
- if !ok {
- http.Error(w, "webserver doesn't support hijacking", http.StatusInternalServerError)
- return
- }
-
- conn, _, err := hijacker.Hijack()
- if err != nil {
- message := errors.Wrap(err, "failed to hijack connection").Error()
- http.Error(w, message, http.StatusInternalServerError)
- return
- }
-
- // Write the status line and upgrade header by hand since w.WriteHeader()
- // would fail after Hijack()
- data := []byte("HTTP/1.1 101 Switching Protocols\r\nUpgrade: raft\r\n\r\n")
- if n, err := conn.Write(data); err != nil || n != len(data) {
- conn.Close()
- return
- }
-
- // We don't need to watch for the shutdown channel here, because if the
- // shutdown channel gets closed, Layer.Accept() will eventually return
- // error causing the raft.NetworkTransport.listen() method to return
- // (the assumption is that the raft instance is being shutdown). At
- // that point, nobody will be calling Layer.Accept() anymore and we'll
- // block sending to the h.connections channel until h.timeout expires.
- h.logger.Printf("[INFO] raft-http: Establishing new connection with %s", r.Host)
- select {
- case <-h.shutdown:
- h.logger.Printf("[ERR] raft-http: Connection from %s dropped since we have shutdowns", r.Host)
- conn.Close()
- case h.connections <- conn:
- case <-time.After(h.timeout):
- h.logger.Printf("[ERR] raft-http: Connection from %s not processed within %s", r.Host, h.timeout)
- conn.Close()
- }
-}
-
-func (h *Handler) handlePost(w http.ResponseWriter, r *http.Request) {
- query := r.URL.Query()
- id := raft.ServerID(query.Get("id"))
- address := raft.ServerAddress(query.Get("address"))
-
- h.logger.Printf("[INFO] raft-http: Handling join request for node %s (%s)", id, address)
-
- request := raftmembership.NewJoinRequest(id, address)
- h.changeMembership(w, r, request)
-}
-
-func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request) {
- query := r.URL.Query()
- id := raft.ServerID(query.Get("id"))
-
- h.logger.Printf("[INFO] raft-http: Handling leave request for node %s", id)
-
- request := raftmembership.NewLeaveRequest(id)
- h.changeMembership(w, r, request)
-}
-
-func (h *Handler) changeMembership(w http.ResponseWriter, r *http.Request, request *raftmembership.ChangeRequest) {
- // Acquire a read lock, so Close() will block until all change
- // membership requests are done.
- h.mu.RLock()
- defer h.mu.RUnlock()
-
- // Fail immediately if we've been closed.
- select {
- case <-h.shutdown:
- http.Error(w, "raft transport closed", http.StatusForbidden)
- return
- default:
- }
-
- // Sanity check before actually trying to process the request.
- if request.ID() == "" {
- http.Error(w, "no server ID provided", http.StatusBadRequest)
- return
- }
-
- // It's safe to block here, since HandleChangeRequests has an internal
- // timeout, which will abort a request if takes too long.
- h.requests <- request
-
- err := request.Error(h.timeout)
- if err == nil {
- return
- }
-
- var code int
-
- switch err := err.(type) {
- case *raftmembership.ErrDifferentLeader:
- // If we fail because the current node is not the leader, send
- // a redirect.
- url := &url.URL{
- Scheme: "http", // XXX TODO: handle HTTPS
- Path: r.URL.Path,
- RawQuery: r.URL.RawQuery,
- Host: err.Leader(),
- }
- http.Redirect(w, r, url.String(), http.StatusPermanentRedirect)
- return
- case *raftmembership.ErrUnknownLeader:
- // If we fail because we currently don't know the leader, hint
- // the client to retry.
- code = http.StatusServiceUnavailable
- default:
- code = http.StatusForbidden
- }
-
- message := errors.Wrap(err, fmt.Sprintf(
- "failed to %s server %s", request.Kind(), request.ID())).Error()
- http.Error(w, message, code)
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-http/layer.go b/vendor/github.com/CanonicalLtd/raft-http/layer.go
deleted file mode 100644
index cdba5dbb42..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-http/layer.go
+++ /dev/null
@@ -1,132 +0,0 @@
-package rafthttp
-
-import (
- "bufio"
- "fmt"
- "io"
- "io/ioutil"
- "log"
- "net"
- "net/http"
- "time"
-
- "github.com/CanonicalLtd/raft-membership"
- "github.com/hashicorp/raft"
- "github.com/pkg/errors"
-)
-
-// NewLayer returns a new raft stream layer that initiates connections
-// with HTTP and then uses Upgrade to switch them into raw TCP.
-func NewLayer(path string, localAddr net.Addr, handler *Handler, dial Dial) *Layer {
- //logger := log.New(os.Stderr, "", log.LstdFlags)
- logger := log.New(ioutil.Discard, "", 0)
- return NewLayerWithLogger(path, localAddr, handler, dial, logger)
-}
-
-// NewLayerWithLogger returns a Layer using the specified logger.
-func NewLayerWithLogger(path string, localAddr net.Addr, handler *Handler, dial Dial, logger *log.Logger) *Layer {
- return &Layer{
- path: path,
- localAddr: localAddr,
- handler: handler,
- dial: dial,
- logger: logger,
- }
-}
-
-// Layer represents the connection between raft nodes.
-type Layer struct {
- path string
- localAddr net.Addr
- handler *Handler
- dial Dial
- logger *log.Logger
-}
-
-// Accept waits for the next connection.
-func (l *Layer) Accept() (net.Conn, error) {
- select {
- case conn := <-l.handler.connections:
- return conn, nil
- case <-l.handler.shutdown:
- return nil, io.EOF
- }
-}
-
-// Close closes the layer.
-func (l *Layer) Close() error {
- l.handler.Close()
- return nil
-}
-
-// Addr returns the local address for the layer.
-func (l *Layer) Addr() net.Addr {
- return l.localAddr
-}
-
-// Dial creates a new network connection.
-func (l *Layer) Dial(addr raft.ServerAddress, timeout time.Duration) (net.Conn, error) {
- l.logger.Printf("[INFO] raft-http: Connecting to %s", addr)
-
- url := makeURL(l.path)
- request := &http.Request{
- Method: "GET",
- URL: url,
- Proto: "HTTP/1.1",
- ProtoMajor: 1,
- ProtoMinor: 1,
- Header: make(http.Header),
- Host: l.Addr().String(),
- }
- request.Header.Set("Upgrade", "raft")
-
- conn, err := l.dial(string(addr), timeout)
- if err != nil {
- return nil, errors.Wrap(err, "dialing failed")
- }
-
- if err := request.Write(conn); err != nil {
- return nil, errors.Wrap(err, "sending HTTP request failed")
- }
-
- response, err := http.ReadResponse(bufio.NewReader(conn), request)
- if err != nil {
- return nil, errors.Wrap(err, "failed to read response")
- }
- if response.StatusCode != http.StatusSwitchingProtocols {
- return nil, fmt.Errorf("dialing fail: expected status code 101 got %d", response.StatusCode)
- }
- if response.Header.Get("Upgrade") != "raft" {
- return nil, fmt.Errorf("missing or unexpected Upgrade header in response")
- }
- return conn, err
-}
-
-// Join tries to join the cluster by contacting the leader at the given
-// address. The raft node associated with this layer must have the given server
-// identity.
-func (l *Layer) Join(id raft.ServerID, addr raft.ServerAddress, timeout time.Duration) error {
- l.logger.Printf("[INFO] raft-http: Joining cluster at %s as node %s", addr, id)
-
- return l.changeMemberhip(raftmembership.JoinRequest, id, addr, timeout)
-}
-
-// Leave tries to leave the cluster by contacting the leader at the given
-// address. The raft node associated with this layer must have the given
-// server identity.
-func (l *Layer) Leave(id raft.ServerID, addr raft.ServerAddress, timeout time.Duration) error {
- l.logger.Printf("[INFO] raft-http: Leaving cluster at %s as node %s", addr, id)
-
- return l.changeMemberhip(raftmembership.LeaveRequest, id, addr, timeout)
-}
-
-// Change the membership of the server associated with this layer.
-func (l *Layer) changeMemberhip(kind raftmembership.ChangeRequestKind, id raft.ServerID, addr raft.ServerAddress, timeout time.Duration) error {
- return ChangeMembership(kind, l.path, l.dial, id, l.Addr().String(), string(addr), timeout)
-}
-
-// Map a membership ChangeRequest kind code to an HTTP method name.
-var membershipChangeRequestKindToMethod = map[raftmembership.ChangeRequestKind]string{
- raftmembership.JoinRequest: "POST",
- raftmembership.LeaveRequest: "DELETE",
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-http/membership.go b/vendor/github.com/CanonicalLtd/raft-http/membership.go
deleted file mode 100644
index a83100b3a7..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-http/membership.go
+++ /dev/null
@@ -1,95 +0,0 @@
-package rafthttp
-
-import (
- "fmt"
- "io/ioutil"
- "net"
- "net/http"
- "net/url"
- "strings"
- "time"
-
- "github.com/CanonicalLtd/raft-membership"
- "github.com/hashicorp/raft"
- "github.com/pkg/errors"
-)
-
-// ChangeMembership can be used to join or leave a cluster over HTTP.
-func ChangeMembership(
- kind raftmembership.ChangeRequestKind,
- path string,
- dial Dial,
- id raft.ServerID,
- address, target string,
- timeout time.Duration) error {
- url := makeURL(path)
- url.RawQuery = fmt.Sprintf("id=%s", id)
- if kind == raftmembership.JoinRequest {
- url.RawQuery += fmt.Sprintf("&address=%s", address)
- }
- url.Host = target
- url.Scheme = "http"
- method := membershipChangeRequestKindToMethod[kind]
- request := &http.Request{
- Method: method,
- URL: url,
- Proto: "HTTP/1.1",
- ProtoMajor: 1,
- ProtoMinor: 1,
- Header: make(http.Header),
- }
-
- remaining := timeout
- var response *http.Response
- var err error
- for remaining > 0 {
- start := time.Now()
- netDial := func(network, addr string) (net.Conn, error) {
- return dial(addr, remaining)
- }
- client := &http.Client{
- Timeout: remaining,
- Transport: &http.Transport{Dial: netDial},
- }
- response, err = client.Do(request)
-
- // If we got a system or network error, just return it.
- if err != nil {
- break
- }
-
- // If we got an HTTP error, let's capture its details,
- // and possibly return it if it's not retriable or we
- // have hit our timeout.
- if response.StatusCode != http.StatusOK {
- body, _ := ioutil.ReadAll(response.Body)
- err = fmt.Errorf(
- "http code %d '%s'", response.StatusCode,
- strings.TrimSpace(string(body)))
- }
- // If there's a temporary failure, let's retry.
- if response.StatusCode == http.StatusServiceUnavailable {
- // XXX TODO: use an exponential backoff
- // relative to the timeout?
- time.Sleep(100 * time.Millisecond)
-
- remaining -= time.Since(start)
- continue
- }
-
- break
- }
- if err != nil {
- return errors.Wrap(err, fmt.Sprintf("server %s failed", kind))
- }
- return nil
-}
-
-// Build a full url.URL object out of our path.
-func makeURL(path string) *url.URL {
- url, err := url.Parse(path)
- if err != nil {
- panic(fmt.Sprintf("invalid URL path %s", path))
- }
- return url
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-membership/AUTHORS b/vendor/github.com/CanonicalLtd/raft-membership/AUTHORS
deleted file mode 100644
index 6e13f86ebb..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-membership/AUTHORS
+++ /dev/null
@@ -1 +0,0 @@
-Free Ekanayaka <free.ekanayaka at canonical.com>
diff --git a/vendor/github.com/CanonicalLtd/raft-membership/LICENSE b/vendor/github.com/CanonicalLtd/raft-membership/LICENSE
deleted file mode 100644
index 261eeb9e9f..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-membership/LICENSE
+++ /dev/null
@@ -1,201 +0,0 @@
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/vendor/github.com/CanonicalLtd/raft-membership/README.md b/vendor/github.com/CanonicalLtd/raft-membership/README.md
deleted file mode 100644
index f3a36a9601..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-membership/README.md
+++ /dev/null
@@ -1,11 +0,0 @@
-raft-membership [](https://travis-ci.org/CanonicalLtd/raft-membership) [](https://coveralls.io/github/CanonicalLtd/raft-membership?branch=master) [](https://goreportcard.com/report/github.com/CanonicalLtd/raft-membership) [](https://godoc.org/github.com/CanonicalLtd/raft-membership)
-=========
-
-This repository provides the `raftmembership` package, which contains
-an extensions of the `raft` Go [package](https://github.com/hashicorp/raft)
-from Hashicorp to easily make a node join or leave a cluster.
-
-Documentation
-==============
-
-The documentation for this package can be found on [Godoc](http://godoc.org/github.com/CanonicalLtd/raft-membership).
diff --git a/vendor/github.com/CanonicalLtd/raft-membership/changer.go b/vendor/github.com/CanonicalLtd/raft-membership/changer.go
deleted file mode 100644
index 06d3d01f77..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-membership/changer.go
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package raftmembership
-
-import (
- "time"
-
- "github.com/hashicorp/raft"
-)
-
-// Changer is an API that can be used by a raft server to change its
-// membership in a cluster (i.e. either join it or leave it).
-//
-// It works by using some transport layer (e.g. HTTP, TCP, etc) to
-// send a membership change request to a target server that is part of
-// the cluster and that can handle such requests, possibly redirecting
-// the requesting server to another server (e.g. the cluster leader).
-//
-// It is effectively an extensions of the raft.Transport interface,
-// with additional semantics for joining/leaving a raft cluster.
-type Changer interface {
- Join(raft.ServerID, raft.ServerAddress, time.Duration) error
- Leave(raft.ServerID, raft.ServerAddress, time.Duration) error
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-membership/errors.go b/vendor/github.com/CanonicalLtd/raft-membership/errors.go
deleted file mode 100644
index 15391126a1..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-membership/errors.go
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package raftmembership
-
-import (
- "fmt"
-)
-
-// ErrDifferentLeader is returned by ChangeRequest.Error() when the
-// request to join or leave a cluster failed because the target peer
-// is not the leader. The network address of the leader as currently
-// known by the target peer is attached to the error, so clients can
-// perform again the request, this time using the given leader address
-// as target peer.
-type ErrDifferentLeader struct {
- leader string
-}
-
-// Leader is the address of the leader as currently known.
-func (e *ErrDifferentLeader) Leader() string {
- return e.leader
-}
-
-func (e *ErrDifferentLeader) Error() string {
- return fmt.Sprintf("node is not leader, current leader at: %s", e.leader)
-}
-
-// ErrUnknownLeader is returned by ChangeRequest.Error() when the
-// request to join or leave a cluster failed because the target peer
-// is not the leader, and at the moment it also does not know the
-// address of the leader (this can happen for example during leader
-// elections). Clients typically want to retry after a short time.
-type ErrUnknownLeader struct{}
-
-func (e *ErrUnknownLeader) Error() string {
- return "node is not leader, current leader unknown"
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-membership/handle.go b/vendor/github.com/CanonicalLtd/raft-membership/handle.go
deleted file mode 100644
index fb467459ae..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-membership/handle.go
+++ /dev/null
@@ -1,65 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package raftmembership
-
-import (
- "fmt"
- "time"
-
- "github.com/hashicorp/raft"
-)
-
-// HandleChangeRequests processes ChangeRequest's received through the
-// given channel, using the given raft.Raft instance to add or remove
-// peers to the cluster according to the received requests.
-func HandleChangeRequests(r *raft.Raft, requests <-chan *ChangeRequest) {
- for request := range requests {
-
- // If we currently think we're the leader, let's try
- // to handle the request, otherwise let's bail out
- // directly.
- var err error
- if r.State() == raft.Leader {
- err = changeMembership(r, request).Error()
- } else {
- err = raft.ErrNotLeader
- }
-
- // Wrap not-leader errors.
- if err == raft.ErrNotLeader {
- if r.Leader() != "" {
- err = &ErrDifferentLeader{leader: string(r.Leader())}
- } else {
- err = &ErrUnknownLeader{}
- }
- }
-
- request.Done(err)
- }
-}
-
-// Execute the appropriate Raft to handle the given request.
-func changeMembership(raft *raft.Raft, request *ChangeRequest) raft.Future {
- kind := request.Kind()
- timeout := 10 * time.Second // FIXME: should be configurable
- switch kind {
- case JoinRequest:
- return raft.AddVoter(request.ID(), request.Address(), 0, timeout)
- case LeaveRequest:
- return raft.RemoveServer(request.ID(), 0, timeout)
- default:
- panic(fmt.Sprintf("invalid change request kind: %d", int(kind)))
- }
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-membership/request.go b/vendor/github.com/CanonicalLtd/raft-membership/request.go
deleted file mode 100644
index 59efaa44f7..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-membership/request.go
+++ /dev/null
@@ -1,113 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package raftmembership
-
-import (
- "fmt"
- "time"
-
- "github.com/hashicorp/raft"
-)
-
-// ChangeRequest represents a request to change a server's membership in
-// a raft cluster (either join or leave).
-//
-// A requesting server uses an implementation of the membership Changer
-// interface to connect to a target server through some network transport layer
-// and to ask to join or leave the target server's cluster. The target server
-// internally uses ChangeRequest and HandleChangeRequests as helpers to
-// implement handlers to process such requests coming from the network
-// transport layer.
-type ChangeRequest struct {
- id raft.ServerID // ID of the server requesting the membership change
- address raft.ServerAddress // Address of the server requesting the membership change
- kind ChangeRequestKind // Kind of membership change being requested
- done chan error // Notify client code of request success or failure
-}
-
-// NewJoinRequest creates a new membership ChangeRequest to join a
-// cluster.
-func NewJoinRequest(id raft.ServerID, address raft.ServerAddress) *ChangeRequest {
- return &ChangeRequest{
- id: id,
- address: address,
- kind: JoinRequest,
- done: make(chan error, 1),
- }
-}
-
-// NewLeaveRequest creates a new membership ChangeRequest to leave a
-// cluster.
-func NewLeaveRequest(id raft.ServerID) *ChangeRequest {
- return &ChangeRequest{
- id: id,
- kind: LeaveRequest,
- done: make(chan error, 1),
- }
-}
-
-// ID of the server requesting to change its membership.
-func (r *ChangeRequest) ID() raft.ServerID {
- return r.id
-}
-
-// Address of the server requesting to change its membership.
-func (r *ChangeRequest) Address() raft.ServerAddress {
- return r.address
-}
-
-// Kind is the type of membership change requested, either join leave.
-func (r *ChangeRequest) Kind() ChangeRequestKind {
- return r.kind
-}
-
-// Error blocks until this ChangeRequest is fully processed or the given
-// timeout is reached and returns any error hit while handling the request, or
-// nil if none was met.
-func (r *ChangeRequest) Error(timeout time.Duration) error {
- var err error
- select {
- case err = <-r.done:
- case <-time.After(timeout):
- err = fmt.Errorf("timeout waiting for membership change")
- }
- return err
-}
-
-// Done should be invoked by the code handling this request (such as
-// HandleChangeRequests) to notify callers that the it has been
-// processed, either successfully or not.
-func (r *ChangeRequest) Done(err error) {
- r.done <- err
- close(r.done)
-}
-
-// ChangeRequestKind is kind of membership change being requested.
-type ChangeRequestKind int
-
-func (k ChangeRequestKind) String() string {
- return changeRequestKindToString[k]
-}
-
-// Possible values for ChangeRequestKind
-const (
- JoinRequest ChangeRequestKind = iota
- LeaveRequest
-)
-
-var changeRequestKindToString = []string{
- JoinRequest: "join",
- LeaveRequest: "leave",
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-test/AUTHORS b/vendor/github.com/CanonicalLtd/raft-test/AUTHORS
deleted file mode 100644
index 6e13f86ebb..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/AUTHORS
+++ /dev/null
@@ -1 +0,0 @@
-Free Ekanayaka <free.ekanayaka at canonical.com>
diff --git a/vendor/github.com/CanonicalLtd/raft-test/LICENSE b/vendor/github.com/CanonicalLtd/raft-test/LICENSE
deleted file mode 100644
index 261eeb9e9f..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/LICENSE
+++ /dev/null
@@ -1,201 +0,0 @@
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/vendor/github.com/CanonicalLtd/raft-test/README.md b/vendor/github.com/CanonicalLtd/raft-test/README.md
deleted file mode 100644
index 7a6b645b52..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/README.md
+++ /dev/null
@@ -1,11 +0,0 @@
-raft-test [](https://travis-ci.org/CanonicalLtd/raft-test) [](https://coveralls.io/github/CanonicalLtd/raft-test?branch=master) [](https://goreportcard.com/report/github.com/CanonicalLtd/raft-test) [](https://godoc.org/github.com/CanonicalLtd/raft-test)
-=========
-
-This repository provides the `rafttest` package, which contains
-helpers to test code based on the `raft` Go [package](https://github.com/hashicorp/raft)
-from Hashicorp.
-
-Documentation
-==============
-
-The documentation for this package can be found on [Godoc](http://godoc.org/github.com/CanonicalLtd/raft-test).
diff --git a/vendor/github.com/CanonicalLtd/raft-test/cluster.go b/vendor/github.com/CanonicalLtd/raft-test/cluster.go
deleted file mode 100644
index c5db316c8e..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/cluster.go
+++ /dev/null
@@ -1,303 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package rafttest
-
-import (
- "fmt"
- "strconv"
- "testing"
- "time"
-
- "github.com/CanonicalLtd/raft-test/internal/election"
- "github.com/CanonicalLtd/raft-test/internal/fsms"
- "github.com/CanonicalLtd/raft-test/internal/logging"
- "github.com/CanonicalLtd/raft-test/internal/network"
- "github.com/hashicorp/go-hclog"
- "github.com/hashicorp/raft"
-)
-
-// Cluster creates n raft servers, one for each of the given FSMs, and returns
-// a Control object that can be used to create deterministic test scenarios,
-// deciding which server is elected as leader and if and when a failure should
-// happen during its term.
-//
-// Each raft.Raft instance is created with sane test-oriented default
-// dependencies, which include:
-//
-// - very low configuration timeouts
-// - in-memory transports
-// - in-memory log and stable stores
-// - in-memory snapshot stores
-//
-// You can tweak the default dependencies using the Config, Transport and
-// LogStore options.
-//
-// All created raft servers will be part of the cluster and act as voting
-// servers, unless the Servers option is used.
-//
-// If a GO_RAFT_TEST_LATENCY environment is found, the default configuration
-// timeouts will be scaled up accordingly (useful when running tests on slow
-// hardware). A latency of 1.0 is a no-op, since it just keeps the default
-// values unchanged. A value greater than 1.0 increases the default timeouts by
-// that factor. See also the Duration helper.
-func Cluster(t testing.TB, fsms []raft.FSM, options ...Option) (map[raft.ServerID]*raft.Raft, *Control) {
- logger := logging.New(t, "DEBUG")
- logger.Debug(fmt.Sprintf("[DEBUG] raft-test: setup: start (%d servers)", len(fsms)))
-
- // Create a set of default dependencies for each server.
- dependencies := make([]*dependencies, len(fsms))
- for i, fsm := range fsms {
- dependencies[i] = newDefaultDependencies(t, logger, i, fsm)
- }
-
- // Customize the default dependencies by applying the given options.
- for _, option := range options {
- option(dependencies)
- }
-
- // Honor the GO_RAFT_TEST_LATENCY env var, if set.
- setTimeouts(dependencies)
-
- // Instrument the Config of each server with a NotifyCh and return a
- // leadership object for watching them.
- leadership := instrumentConfigs(t, logger, dependencies)
-
- // Instrument all servers by replacing their transports with transport
- // wrappers, creating a network object to control them.
- network := instrumentTransports(logger, dependencies)
-
- // Instrument all servers by replacing their fsms with wrapper fsms,
- // creating a watcher to observe them.
- watcher := instrumentFSMs(logger, dependencies)
-
- // Bootstrap the initial cluster configuration.
- bootstrapCluster(t, logger, dependencies)
-
- // Start the individual servers.
- servers := make(map[raft.ServerID]*raft.Raft)
- confs := make(map[raft.ServerID]*raft.Config)
- for _, d := range dependencies {
- id := d.Conf.LocalID
- logger.Debug(fmt.Sprintf("[DEBUG] raft-test: setup: server %s: start", id))
- raft, err := newRaft(d)
- if err != nil {
- logger.Debug(fmt.Sprintf("[DEBUG] raft-test: setup: error: server %s failed to start: %v", id, err))
- }
- confs[id] = d.Conf
- servers[id] = raft
- }
-
- // Create the Control instance for this cluster
- control := &Control{
- t: t,
- logger: logger,
- election: leadership,
- network: network,
- watcher: watcher,
- confs: confs,
- servers: servers,
- }
-
- logger.Debug("[DEBUG] raft-test: setup: done")
-
- return servers, control
-}
-
-// Option can be used to tweak the dependencies of test Raft servers created with
-// Cluster() or Server().
-type Option func([]*dependencies)
-
-// Hold dependencies for a single dependencies.
-type dependencies struct {
- Conf *raft.Config
- FSM raft.FSM
- Logs raft.LogStore
- Stable raft.StableStore
- Snaps raft.SnapshotStore
- Configuration *raft.Configuration
- Trans raft.Transport
- Voter bool // Whether this is voter server in the initial configuration
-}
-
-// Create default dependencies for a single raft server.
-func newDefaultDependencies(t testing.TB, logger hclog.Logger, i int, fsm raft.FSM) *dependencies {
- // Use the server's index as its server ID and address.
- addr := strconv.Itoa(i)
- _, transport := raft.NewInmemTransport(raft.ServerAddress(addr))
-
- conf := raft.DefaultConfig()
- conf.LocalID = raft.ServerID(addr)
- conf.Logger = logger
-
- // Set low timeouts.
- conf.HeartbeatTimeout = 15 * time.Millisecond
- conf.ElectionTimeout = 15 * time.Millisecond
- conf.CommitTimeout = 1 * time.Millisecond
- conf.LeaderLeaseTimeout = 10 * time.Millisecond
-
- // Set very high values to prevent snapshots to happen randomly.
- conf.SnapshotInterval = 24 * time.Hour
- conf.SnapshotThreshold = 4096
-
- // Set the snapshot to retain only one log, since the most common use
- // case is to test an FSM restore from a snapshot.
- conf.TrailingLogs = 1
-
- store := raft.NewInmemStore()
- return &dependencies{
- Conf: conf,
- FSM: fsm,
- Logs: store,
- Stable: store,
- Snaps: raft.NewInmemSnapshotStore(),
- Trans: transport,
- Voter: true,
- }
-}
-
-// Set scaled timeouts on all servers, to match GO_RAFT_TEST_LATENCY (if set).
-func setTimeouts(dependencies []*dependencies) {
- for _, d := range dependencies {
- d.Conf.HeartbeatTimeout = Duration(d.Conf.HeartbeatTimeout)
- d.Conf.ElectionTimeout = Duration(d.Conf.ElectionTimeout)
- d.Conf.CommitTimeout = Duration(d.Conf.CommitTimeout)
- d.Conf.LeaderLeaseTimeout = Duration(d.Conf.LeaderLeaseTimeout)
- }
-}
-
-// Set leader notification channels on all servers.
-func instrumentConfigs(t testing.TB, logger hclog.Logger, dependencies []*dependencies) *election.Tracker {
- t.Helper()
-
- tracker := election.NewTracker(logger)
-
- for _, d := range dependencies {
- id := d.Conf.LocalID
- if d.Conf.NotifyCh != nil {
- t.Fatalf("raft-test: setup: error: found NotifyCh on server %s set via Config option", id)
- }
- // Use an unbuffered channel, so raft will block on us.
- notifyCh := make(chan bool)
- d.Conf.NotifyCh = notifyCh
- tracker.Track(id, notifyCh)
- }
-
- return tracker
-}
-
-// Replace the dependencies.Trans object on each server with a faulty transport
-// that wraps the real transport. Return a network object that knows about the
-// these wrappers and that inject various kind of failures.
-func instrumentTransports(logger hclog.Logger, dependencies []*dependencies) *network.Network {
- // Connect to each others all the servers that use a LoopbackTransport
- // (the default). However, actual connectivity control will be
- // performed by the network object
- connectLoobackTransports(dependencies)
-
- network := network.New(logger)
-
- for _, d := range dependencies {
- d.Trans = network.Add(d.Conf.LocalID, d.Trans)
- }
-
- return network
-}
-
-// Replace the dependencies.FSM object on each server with a wrapper FSM that
-// wraps the real FSM. Return a watcher object that can be used to get notified
-// of various events.
-func instrumentFSMs(logger hclog.Logger, dependencies []*dependencies) *fsms.Watcher {
- watcher := fsms.New(logger)
-
- for _, d := range dependencies {
- d.FSM = watcher.Add(d.Conf.LocalID, d.FSM)
- }
-
- return watcher
-}
-
-// Connect loopback transports from servers that have them.
-func connectLoobackTransports(dependencies []*dependencies) {
- loopbacks := make([]raft.LoopbackTransport, 0)
- for _, d := range dependencies {
- loopback, ok := d.Trans.(raft.LoopbackTransport)
- if ok {
- loopbacks = append(loopbacks, loopback)
- }
- }
-
- for i, t1 := range loopbacks {
- for j, t2 := range loopbacks {
- if i == j {
- continue
- }
- t1.Connect(t2.LocalAddr(), t2)
- t2.Connect(t1.LocalAddr(), t1)
- }
- }
-}
-
-// Bootstrap the cluster, including in the initial configuration of each voting
-// server.
-func bootstrapCluster(t testing.TB, logger hclog.Logger, dependencies []*dependencies) {
- t.Helper()
-
- // Figure out which servers should be part of the initial
- // configuration.
- servers := make([]raft.Server, 0)
- for _, d := range dependencies {
- id := d.Conf.LocalID
- if !d.Voter {
- // If the server is not initially part of the cluster,
- // there's nothing to do.
- logger.Debug(fmt.Sprintf("[DEBUG] raft-test: setup: server %s: skip bootstrap (not part of initial configuration)", id))
- continue
- }
- server := raft.Server{
- ID: id,
- Address: d.Trans.LocalAddr(),
- }
- servers = append(servers, server)
- }
-
- // Create the initial cluster configuration.
- configuration := raft.Configuration{Servers: servers}
- for i := 0; i < len(dependencies); i++ {
- d := dependencies[i]
- id := d.Conf.LocalID
- if !d.Voter {
- continue
- }
- logger.Debug(fmt.Sprintf("[DEBUG] raft-test: setup: server %s: bootstrap", id))
- err := raft.BootstrapCluster(
- d.Conf,
- d.Logs,
- d.Stable,
- d.Snaps,
- d.Trans,
- configuration,
- )
- if err != nil {
- t.Fatalf("raft-test: setup: error: server %s failed to bootstrap: %v", id, err)
- }
- }
-
-}
-
-// Convenience around raft.NewRaft for creating a new Raft instance using the
-// given dependencies.
-func newRaft(d *dependencies) (*raft.Raft, error) {
- return raft.NewRaft(d.Conf, d.FSM, d.Logs, d.Stable, d.Snaps, d.Trans)
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-test/control.go b/vendor/github.com/CanonicalLtd/raft-test/control.go
deleted file mode 100644
index 2299072b04..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/control.go
+++ /dev/null
@@ -1,519 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package rafttest
-
-import (
- "context"
- "fmt"
- "runtime"
- "testing"
- "time"
-
- "github.com/CanonicalLtd/raft-test/internal/election"
- "github.com/CanonicalLtd/raft-test/internal/event"
- "github.com/CanonicalLtd/raft-test/internal/fsms"
- "github.com/CanonicalLtd/raft-test/internal/network"
- "github.com/hashicorp/raft"
- "github.com/hashicorp/go-hclog"
-)
-
-// Control the events happening in a cluster of raft servers, such has leadership
-// changes, failures and shutdowns.
-type Control struct {
- t testing.TB
- logger hclog.Logger
- election *election.Tracker
- network *network.Network
- watcher *fsms.Watcher
- confs map[raft.ServerID]*raft.Config
- servers map[raft.ServerID]*raft.Raft
- errored bool
- deposing chan struct{}
-
- // Current Term after Elect() was called, if any.
- term *Term
-
- // Future of any pending snapshot that has been scheduled with an
- // event.
- snapshotFuture raft.SnapshotFuture
-}
-
-// Close the control for this raft cluster, shutting down all servers and
-// stopping all monitoring goroutines.
-//
-// It must be called by every test creating a test cluster with Cluster().
-func (c *Control) Close() {
- c.logger.Debug("[DEBUG] raft-test: close: start")
-
- // First tell the election tracker that we don't care anymore about
- // notifications. Any value received from the NotifyCh's will be dropped
- // on the floor.
- c.election.Ignore()
-
- // Now shutdown the servers.
- c.shutdownServers()
-
- // Finally shutdown the election tracker since nothing will be
- // sending to NotifyCh's.
- c.election.Close()
-
- c.logger.Debug("[DEBUG] raft-test: close: done")
-}
-
-// Elect a server as leader.
-//
-// When calling this method there must be no leader in the cluster and server
-// transports must all be disconnected from eacher.
-func (c *Control) Elect(id raft.ServerID) *Term {
- c.t.Helper()
-
- c.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: elect: start (server %s)", id))
-
- // Wait for the current leader (if any) to be fully deposed.
- if c.deposing != nil {
- <-c.deposing
- }
-
- // Sanity check that no server is the leader.
- for id, r := range c.servers {
- if r.State() == raft.Leader {
- c.t.Fatalf("raft-test: error: cluster has already a leader (server %s)", id)
- }
- }
-
- // We might need to repeat the logic below a few times in case a
- // follower hits its heartbeat timeout before the leader has chance to
- // append entries to it and refresh the last contact timestamp (hence
- // transitioning to candidate and starting a new election).
- for n := 0; n < maxElectionRounds; n++ {
- leadership := c.waitLeadershipAcquired(id)
-
- // We did not acquire leadership, let's retry.
- if leadership == nil {
- if n < maxElectionRounds {
- c.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: elect: server %s: retry %d ", id, n+1))
- continue
- }
- }
-
- // The given node became the leader, let's make sure
- // that leadership is stable and that other nodes
- // become followers.
- if !c.waitLeadershipPropagated(id, leadership) {
- if n < maxElectionRounds {
- c.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: elect: server %s: retry %d ", id, n+1))
- continue
- }
- }
- // Now establish all remaining connections. E.g. for three nodes:
- //
- // L <--- F1
- // L <--- F2
- //
- // and:
- //
- // F1 <--- F2
- // F1 ---> F2
- //
- // This way the cluster is fully connected. foo
- c.logger.Debug("[DEBUG] raft-test: elect: done")
- term := &Term{
- control: c,
- id: id,
- leadership: leadership,
- }
- c.term = term
-
- return term
- }
- c.t.Fatalf("raft-test: server %s: did not acquire stable leadership", id)
-
- return nil
-}
-
-// Barrier is used to wait for the cluster to settle to a stable state, where
-// all in progress Apply() commands are committed across all FSM associated
-// with servers that are not disconnected and all in progress snapshots and
-// restores have been performed.
-//
-// Usually you don't wan't to concurrently keep invoking Apply() on the cluster
-// raft instances while Barrier() is running.
-func (c *Control) Barrier() {
- // Wait for snapshots to complete.
- if c.snapshotFuture != nil {
- if err := c.snapshotFuture.Error(); err != nil {
- c.t.Fatalf("raft-test: snapshot failed: %v", err)
- }
- }
-
- // Wait for inflight commands to be applied to the leader's FSM.
- if c.term.id != "" {
- // Set a relatively high timeout.
- //
- // TODO: let users specify the maximum amount of time a single
- // Apply() to their FSM should take, and calculate this value
- // accordingly.
- timeout := Duration(time.Second)
-
- if err := c.servers[c.term.id].Barrier(timeout).Error(); err != nil {
- c.t.Fatalf("raft-test: leader barrier: %v", err)
- }
-
- // Wait for follower FSMs to catch up.
- n := c.Commands(c.term.id)
- events := make([]*event.Event, 0)
- for id := range c.servers {
- if id == c.term.id {
- continue
- }
- // Skip disconnected followers.
- if !c.network.PeerConnected(c.term.id, id) {
- continue
- }
- event := c.watcher.WhenApplied(id, n)
- events = append(events, event)
- }
- for _, event := range events {
- <-event.Watch()
- event.Ack()
- }
- }
-}
-
-// Depose the current leader.
-//
-// When calling this method a leader must have been previously elected with
-// Elect().
-//
-// It must not be called if the current term has scheduled a depose action with
-// Action.Depose().
-func (c *Control) Depose() {
- event := event.New()
- go c.deposeUponEvent(event, c.term.id, c.term.leadership)
- event.Fire()
- event.Block()
-}
-
-// Commands returns the total number of command logs applied by the FSM of the
-// server with the given ID.
-func (c *Control) Commands(id raft.ServerID) uint64 {
- return c.watcher.Commands(id)
-}
-
-// Snapshots returns the total number of snapshots performed by the FSM of the
-// server with the given ID.
-func (c *Control) Snapshots(id raft.ServerID) uint64 {
- return c.watcher.Snapshots(id)
-}
-
-// Restores returns the total number of restores performed by the FSM of the
-// server with the given ID.
-func (c *Control) Restores(id raft.ServerID) uint64 {
- return c.watcher.Restores(id)
-}
-
-// Shutdown all raft nodes and fail the test if any of them errors out while
-// doing so.
-func (c *Control) shutdownServers() {
- // Find the leader if there is one, and shut it down first. This should
- // prevent it from getting stuck on shutdown while trying to send RPCs
- // to the followers.
- //
- // TODO: this is arguably a workaround for a bug in the transport
- // wrapper.
- ids := make([]raft.ServerID, 0)
- for id, r := range c.servers {
- if r.State() == raft.Leader {
- c.shutdownServer(id)
- ids = append(ids, id)
- }
- }
-
- // Shutdown the rest.
- for id := range c.servers {
- hasShutdown := false
- for i := range ids {
- if ids[i] == id {
- hasShutdown = true
- break
- }
- }
- if !hasShutdown {
- c.shutdownServer(id)
- ids = append(ids, id)
- }
- }
-}
-
-// Shutdown a single server.
-func (c *Control) shutdownServer(id raft.ServerID) {
- r := c.servers[id]
- future := r.Shutdown()
-
- // Expect the shutdown to happen within two seconds by default.
- timeout := Duration(2 * time.Second)
-
- // Watch for errors.
- ch := make(chan error, 1)
- go func(future raft.Future) {
- ch <- future.Error()
- }(future)
-
- var err error
- select {
- case err = <-ch:
- c.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: close: server %s: shutdown done", id))
- case <-time.After(timeout):
- err = fmt.Errorf("timeout (%s)", timeout)
- }
- if err == nil {
- return
- }
-
- c.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: close: server %s: shutdown failed: %s", id, err))
-
- buf := make([]byte, 1<<16)
- n := runtime.Stack(buf, true)
-
- c.t.Errorf("\n\t%s", buf[:n])
- c.t.Fatalf("raft-test: close: error: server %s: shutdown error: %v", id, err)
-}
-
-// Wait for the given server to acquire leadership. Returns true on success,
-// false otherwise (i.e. if the timeout expires).
-func (c *Control) waitLeadershipAcquired(id raft.ServerID) *election.Leadership {
- timeout := maximumElectionTimeout(c.confs) * maxElectionRounds
- future := c.election.Expect(id, timeout)
-
- c.watcher.Electing(id)
-
- // Reset any leader-related state on the transport of the given server
- // and connect it to all other servers, letting it send them RPCs
- // messages but not viceversa. E.g. for three nodes:
- //
- // L ---> F1
- // L ---> F2
- //
- // This way we are sure we are the only server that can possibly acquire
- // leadership.
- c.network.Electing(id)
-
- // First wait for the given node to become leader.
- c.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: elect: server %s: wait to become leader within %s", id, timeout))
-
- leadership, err := future.Done()
- if err != nil {
- c.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: elect: server %s: did not become leader", id))
- }
- return leadership
-
-}
-
-// Wait that the leadership just acquired by server with the given id is
-// acknowledged by all other servers and they all permanently transition to the
-// follower state.
-func (c *Control) waitLeadershipPropagated(id raft.ServerID, leadership *election.Leadership) bool {
- // The leadership propagation needs to happen within the leader lease
- // timeout, otherwise the newly elected leader will step down.
- timeout := maximumLeaderLeaseTimeout(c.confs)
- c.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: elect: server %s: wait for other servers to become followers within %s", id, timeout))
-
- // Get the current configuration, so we wait only for servers that are
- // actually currently part of the cluster (some of them might have been
- // excluded with the Servers option).
- r := c.servers[id]
- future := r.GetConfiguration()
- if err := future.Error(); err != nil {
- c.t.Fatalf("raft-test: control: server %s: failed to get configuration: %v", id, err)
- }
- servers := future.Configuration().Servers
-
- timer := time.After(timeout)
- address := c.network.Address(id)
- for _, server := range servers {
- other := server.ID
- if other == id {
- continue
- }
- r := c.servers[server.ID]
- for {
- // Check that we didn't lose leadership in the meantime.
- select {
- case <-leadership.Lost():
- c.network.Deposing(id)
- c.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: elect: server %s: lost leadership", id))
- return false
- case <-timer:
- c.t.Fatalf("raft-test: elect: server %s: followers did not settle", id)
- default:
- }
-
- // Check that this server is in follower mode, that it
- // has set the elected sever as leader and that we were
- // able to append at least one log entry to it (when a
- // server becomes leader, it always sends a LogNoop).
- if r.State() == raft.Follower && r.Leader() == address && c.network.HasAppendedLogsFromTo(id, other) {
- c.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: elect: server %s: became follower", other))
- break
- }
- time.Sleep(time.Millisecond)
- }
- }
-
- return true
-}
-
-// Return an event that gets fired when the n'th log command gets enqueued by
-// the given leader server.
-func (c *Control) whenCommandEnqueued(id raft.ServerID, n uint64) *event.Event {
- return c.network.ScheduleEnqueueFailure(id, n)
-}
-
-// Return an event that gets fired when the n'th log command gets appended by
-// server with the given ID (which is supposed to be the leader) to all other
-// servers.
-func (c *Control) whenCommandAppended(id raft.ServerID, n uint64) *event.Event {
- return c.network.ScheduleAppendFailure(id, n)
-}
-
-// Return an event that gets fired when the n'th log command gets committed on
-// server with the given ID (which is supposed to be the leader).
-func (c *Control) whenCommandCommitted(id raft.ServerID, n uint64) *event.Event {
- return c.watcher.WhenApplied(id, n)
-}
-
-// Depose the server with the given ID when the given event fires.
-func (c *Control) deposeUponEvent(event *event.Event, id raft.ServerID, leadership *election.Leadership) {
- // Sanity checks.
- r := c.servers[id]
- if r.State() != raft.Leader {
- panic(fmt.Errorf("raft-test: server %s: is not leader", id))
- }
-
- <-event.Watch()
-
- c.network.Deposing(id)
-
- timeout := maximumLeaderLeaseTimeout(c.confs)
-
- c.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: node %s: state: wait leadership lost (timeout=%s)", id, timeout))
-
- select {
- case <-leadership.Lost():
- case <-time.After(timeout):
- c.t.Errorf("raft-test: server %s: error: timeout: leadership not lost", id)
- c.errored = true
- }
- event.Ack()
-
- if !c.errored {
- c.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: server %s: leadership lost", id))
- }
-
- c.deposing <- struct{}{}
- c.deposing = nil
- c.term = nil
-}
-
-// Take a snapshot on the server with the given ID when the given event fires.
-func (c *Control) snapshotUponEvent(event *event.Event, id raft.ServerID) {
- <-event.Watch()
-
- c.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: server %s: control: take snapshot", id))
-
- r := c.servers[id]
- c.snapshotFuture = r.Snapshot()
-
- event.Ack()
-}
-
-// Compute the maximum time a leader election should take, according to the
-// given nodes configs.
-func maximumElectionTimeout(confs map[raft.ServerID]*raft.Config) time.Duration {
- timeout := time.Duration(0)
-
- for _, conf := range confs {
- if conf.ElectionTimeout > timeout {
- timeout = conf.ElectionTimeout
- }
- }
-
- return timeout * timeoutRandomizationFactor
-}
-
-// Return the maximum leader lease timeout among the given nodes configs.
-func maximumLeaderLeaseTimeout(confs map[raft.ServerID]*raft.Config) time.Duration {
- timeout := time.Duration(0)
-
- for _, conf := range confs {
- if conf.LeaderLeaseTimeout > timeout {
- timeout = conf.LeaderLeaseTimeout
- }
- }
-
- // Multiply the timeout by three to account for randomization.
- return timeout * timeoutRandomizationFactor
-}
-
-const (
- // Assume that a leader is elected within 25 rounds. Should be safe enough.
- maxElectionRounds = 25
-
- // Hashicorp's raft implementation randomizes timeouts between 1x and
- // 2x. Multiplying by 4x makes it sure to expire the timeout.
- timeoutRandomizationFactor = 4
-)
-
-// WaitLeader blocks until the given raft instance sets a leader (which
-// could possibly be the instance itself).
-//
-// It fails the test if this doesn't happen within the specified timeout.
-func WaitLeader(t testing.TB, raft *raft.Raft, timeout time.Duration) {
- ctx, cancel := context.WithTimeout(context.Background(), timeout)
- defer cancel()
-
- waitLeader(ctx, t, raft)
-}
-
-func waitLeader(ctx context.Context, t testing.TB, raft *raft.Raft) {
- t.Helper()
-
- check := func() bool {
- return raft.Leader() != ""
- }
- wait(ctx, t, check, 25*time.Millisecond, "no leader was set")
-}
-
-// Poll the given function at the given internval, until it returns true, or
-// the given context expires.
-func wait(ctx context.Context, t testing.TB, f func() bool, interval time.Duration, message string) {
- t.Helper()
-
- start := time.Now()
- for {
- select {
- case <-ctx.Done():
- if err := ctx.Err(); err == context.Canceled {
- return
- }
- t.Fatalf("%s within %s", message, time.Since(start))
- default:
- }
- if f() {
- return
- }
- time.Sleep(interval)
- }
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-test/duration.go b/vendor/github.com/CanonicalLtd/raft-test/duration.go
deleted file mode 100644
index a6142aa44f..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/duration.go
+++ /dev/null
@@ -1,45 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package rafttest
-
-import (
- "fmt"
- "math"
- "os"
- "strconv"
- "time"
-)
-
-// Duration is a convenience to scale the given duration according to the
-// GO_RAFT_TEST_LATENCY environment variable.
-func Duration(duration time.Duration) time.Duration {
- factor := 1.0
- if env := os.Getenv("GO_RAFT_TEST_LATENCY"); env != "" {
- var err error
- factor, err = strconv.ParseFloat(env, 64)
- if err != nil {
- panic(fmt.Sprintf("invalid value '%s' for GO_RAFT_TEST_LATENCY", env))
- }
- }
- return scaleDuration(duration, factor)
-}
-
-func scaleDuration(duration time.Duration, factor float64) time.Duration {
- if factor == 1.0 {
- return duration
- }
-
- return time.Duration((math.Ceil(float64(duration) * factor)))
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-test/fsm.go b/vendor/github.com/CanonicalLtd/raft-test/fsm.go
deleted file mode 100644
index cd3bf5df33..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/fsm.go
+++ /dev/null
@@ -1,60 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package rafttest
-
-import (
- "io"
-
- "github.com/hashicorp/raft"
-)
-
-// FSM create a dummy FSMs.
-func FSM() raft.FSM {
- return &fsm{}
-}
-
-// FSMs creates the given number of dummy FSMs.
-func FSMs(n int) []raft.FSM {
- fsms := make([]raft.FSM, n)
- for i := range fsms {
- fsms[i] = FSM()
- }
- return fsms
-}
-
-// fsm is a dummy raft finite state machine that does nothing and
-// always no-ops.
-type fsm struct{}
-
-// Apply always return a nil error without doing anything.
-func (f *fsm) Apply(*raft.Log) interface{} { return nil }
-
-// Snapshot always return a dummy snapshot and no error without doing
-// anything.
-func (f *fsm) Snapshot() (raft.FSMSnapshot, error) { return &fsmSnapshot{}, nil }
-
-// Restore always return a nil error without reading anything from
-// the reader.
-func (f *fsm) Restore(io.ReadCloser) error { return nil }
-
-// fsmSnapshot a dummy implementation of an fsm snapshot.
-type fsmSnapshot struct{}
-
-// Persist always return a nil error without writing anything
-// to the sink.
-func (s *fsmSnapshot) Persist(sink raft.SnapshotSink) error { return nil }
-
-// Release is a no-op.
-func (s *fsmSnapshot) Release() {}
diff --git a/vendor/github.com/CanonicalLtd/raft-test/internal/election/future.go b/vendor/github.com/CanonicalLtd/raft-test/internal/election/future.go
deleted file mode 100644
index 5adcf16f8f..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/internal/election/future.go
+++ /dev/null
@@ -1,61 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package election
-
-import (
- "fmt"
- "time"
-
- "github.com/hashicorp/raft"
-)
-
-// Future represents a request to acquire leadership that will eventually
-// succeed or fail.
-type Future struct {
- // ID of the raft server that should acquire leadership.
- id raft.ServerID
-
- // If leadership is not acquire within this timeout, the future fails.
- timeout time.Duration
-
- // Notification about leadership being acquired.
- acquiredCh chan struct{}
-
- // Notification about leadership being lost.
- lostCh chan struct{}
-}
-
-// Creates a new leadership future of the given server.
-func newFuture(id raft.ServerID, timeout time.Duration) *Future {
- future := &Future{
- id: id,
- timeout: timeout,
- acquiredCh: make(chan struct{}),
- lostCh: make(chan struct{}),
- }
- return future
-}
-
-// Done returns a Leadership object if leadership was acquired withing the
-// timeout, or an error otherwise.
-func (f *Future) Done() (*Leadership, error) {
- select {
- case <-f.acquiredCh:
- leadership := newLeadership(f.id, f.lostCh)
- return leadership, nil
- case <-time.After(f.timeout):
- return nil, fmt.Errorf("server %s: leadership not acquired within %s", f.id, f.timeout)
- }
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-test/internal/election/leadership.go b/vendor/github.com/CanonicalLtd/raft-test/internal/election/leadership.go
deleted file mode 100644
index b54885de92..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/internal/election/leadership.go
+++ /dev/null
@@ -1,43 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package election
-
-import (
- "github.com/hashicorp/raft"
-)
-
-// Leadership represents the leadership acquired by a server that was elected
-// as leader. It exposes methods to be notified about its loss, with the server
-// stepping down as leader.
-type Leadership struct {
- // ID of the raft server that acquired the leadership.
- id raft.ServerID
-
- // Notification about leadership being lost.
- lostCh chan struct{}
-}
-
-// Create new leadership object.
-func newLeadership(id raft.ServerID, lostCh chan struct{}) *Leadership {
- return &Leadership{
- id: id,
- lostCh: lostCh,
- }
-}
-
-// Lost returns a channel that gets closed when leadership is lost.
-func (l *Leadership) Lost() chan struct{} {
- return l.lostCh
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-test/internal/election/notifier.go b/vendor/github.com/CanonicalLtd/raft-test/internal/election/notifier.go
deleted file mode 100644
index 7ec4213f30..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/internal/election/notifier.go
+++ /dev/null
@@ -1,149 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package election
-
-import (
- "fmt"
- "time"
-
- "github.com/hashicorp/go-hclog"
- "github.com/hashicorp/raft"
-)
-
-// Notifiy about leadership changes in a single raft server.
-type notifier struct {
- // For debugging raft-test itself or its consumers.
- logger hclog.Logger
-
- // ID of the raft server we're observing.
- id raft.ServerID
-
- // Reference to the Config.NotifyCh object set in this server's Config.
- notifyCh chan bool
-
- // Channel used to tell the notification loop to expect the server to
- // acquire leadership. The leadership future sent to this channel will
- // be used both for notifying that leadership was acquired.
- futureCh chan *Future
-
- // Channel used to tell the notification loop to ignore any
- // notification received from the notifyCh.
- ignoreCh chan struct{}
-
- // Stop observing leadership changes when this channel gets closed.
- shutdownCh chan struct{}
-}
-
-// Create a new notifier.
-func newNotifier(logger hclog.Logger, id raft.ServerID, notifyCh chan bool) *notifier {
- observer := ¬ifier{
- logger: logger,
- id: id,
- notifyCh: notifyCh,
- futureCh: make(chan *Future),
- ignoreCh: make(chan struct{}),
- shutdownCh: make(chan struct{}),
- }
- go observer.start()
- return observer
-}
-
-// Ignore any notifications received on the notifyCh.
-func (n *notifier) Ignore() {
- close(n.ignoreCh)
-}
-
-// Close stops observing leadership changes.
-func (n *notifier) Close() {
- n.shutdownCh <- struct{}{}
- <-n.shutdownCh
-}
-
-// Acquired returns a Leadership object when the server acquires leadership, or
-// an error if the timeout expires.
-//
-// It must be called before this server has any chance to become leader
-// (e.g. it's disconnected from the other servers).
-//
-// Once called, it must not be called again until leadership is lost.
-func (n *notifier) Acquired(timeout time.Duration) *Future {
- future := newFuture(n.id, timeout)
- n.futureCh <- future
- return future
-}
-
-// Start observing leadership changes using the notify channel of our server
-// and eed notification to our consumers.
-//
-// The loop will be terminated once the stopCh is closed.
-func (n *notifier) start() {
- // Record the last leadership change observation. For asserting that a
- // leadership lost notification always follows a leadership acquired
- // one.
- var last bool
-
- // Record the last request for leadership change for this server, if
- // any.
- var future *Future
- for {
- select {
- case f := <-n.futureCh:
- if future != nil {
- panic(fmt.Sprintf("server %s: duplicate leadership request", n.id))
- }
- future = f
- case acquired := <-n.notifyCh:
- ignore := false
- select {
- case <-n.ignoreCh:
- // Just drop the notification on the floor.
- ignore = true
- default:
- }
- if ignore {
- break
- }
- if future == nil {
- panic(fmt.Sprintf("server %s: unexpected leadership change", n.id))
- }
- verb := ""
- var ch chan struct{}
- if acquired {
- verb = "acquired"
- ch = future.acquiredCh
- } else {
- verb = "lost"
- ch = future.lostCh
- future = nil
-
- }
- if acquired == last {
- panic(fmt.Sprintf("server %s %s leadership twice in a row", n.id, verb))
- }
- last = acquired
- n.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: server %s: leadership: %s", n.id, verb))
- select {
- case <-ch:
- panic(fmt.Sprintf("server %s: duplicate leadership %s notification", n.id, verb))
- default:
- close(ch)
- }
- case <-n.shutdownCh:
- n.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: server %s: leadership: stop watching", n.id))
- close(n.shutdownCh)
- return
- }
- }
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-test/internal/election/tracker.go b/vendor/github.com/CanonicalLtd/raft-test/internal/election/tracker.go
deleted file mode 100644
index e8090380f1..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/internal/election/tracker.go
+++ /dev/null
@@ -1,112 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package election
-
-import (
- "fmt"
- "sync"
- "time"
-
- "github.com/hashicorp/go-hclog"
- "github.com/hashicorp/raft"
-)
-
-// Tracker consumes the raft.Config.NotifyCh set on each server of a cluster,
-// tracking when elections occur.
-type Tracker struct {
- // For debugging raft-test itself or its consumers.
- logger hclog.Logger
-
- // Watchers for individual servers.
- //
- // Note that this map is not protected by a mutex, since it should be
- // written once when the cluster is created, and never written again.
- observers map[raft.ServerID]*notifier
-
- // Flag indicating if Acquired() has been called on this Observer. It's
- // used to as sanity check that Add() is not called after the first
- // call to Acquired().
- observing bool
-
- // Current leadership future, if any. It's used as sanity check to
- // prevent further leadership requests.
- future *Future
-
- // Serialize access to internal state.
- mu sync.Mutex
-}
-
-// NewTracker creates a new Tracker for watching leadership
-// changes in a raft cluster.
-func NewTracker(logger hclog.Logger) *Tracker {
- return &Tracker{
- logger: logger,
- observers: make(map[raft.ServerID]*notifier),
- }
-}
-
-// Ignore stops propagating leadership change notifications, which will be
-// simply dropped on the floor. Should be called before the final Close().
-func (t *Tracker) Ignore() {
- for _, observer := range t.observers {
- observer.Ignore()
- }
-}
-
-// Close stops watching for leadership changes in the cluster.
-func (t *Tracker) Close() {
- for _, observer := range t.observers {
- observer.Close()
- }
-}
-
-// Track leadership changes on the server with the given ID using the given
-// Config.NotifyCh.
-func (t *Tracker) Track(id raft.ServerID, notifyCh chan bool) {
- if t.observing {
- panic("can't track new server while observing")
- }
- if _, ok := t.observers[id]; ok {
- panic(fmt.Sprintf("an observer for server %s is already registered", id))
- }
- t.observers[id] = newNotifier(t.logger, id, notifyCh)
-}
-
-// Expect returns an election Future object whose Done() method will return
-// a Leadership object when the server with the given ID acquires leadership,
-// or an error if the given timeout expires.
-//
-// It must be called before this server has any chance to become leader
-// (e.g. it's disconnected from the other servers).
-//
-// Once called, it must not be called again until leadership is lost.
-func (t *Tracker) Expect(id raft.ServerID, timeout time.Duration) *Future {
- t.mu.Lock()
- defer t.mu.Unlock()
- t.observing = true
-
- if t.future != nil {
- select {
- case <-t.future.lostCh:
- // Leadership was acquired, but has been lost, so let's proceed.
- t.future = nil
- default:
- panic(fmt.Sprintf("server %s has already requested leadership", t.future.id))
- }
- }
-
- t.future = t.observers[id].Acquired(timeout)
- return t.future
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-test/internal/event/event.go b/vendor/github.com/CanonicalLtd/raft-test/internal/event/event.go
deleted file mode 100644
index 9f73987ab4..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/internal/event/event.go
+++ /dev/null
@@ -1,54 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package event
-
-// An Event that occurrs when certain log command is either enqueued, appended
-// or committed. Events may be fired in the transport layer (i.e. in the
-// eventTransport wrappers) or in the state machine layer (i.e. in the eventFSM
-// wrapper).
-type Event struct {
- fireCh chan struct{}
- ackCh chan struct{}
-}
-
-// New creates a new event.
-func New() *Event {
- return &Event{
- fireCh: make(chan struct{}),
- ackCh: make(chan struct{}),
- }
-}
-
-// Watch the event. Return a channel that gets closed when the event gets
-// fired.
-func (e *Event) Watch() <-chan struct{} {
- return e.fireCh
-}
-
-// Fire the event. A watcher on the event will be awaken.
-func (e *Event) Fire() {
- close(e.fireCh)
-}
-
-// Block until the watcher of the event has acknowledged that the event has
-// been handled.
-func (e *Event) Block() {
- <-e.ackCh
-}
-
-// Ack acknowledges that the event has been handled.
-func (e *Event) Ack() {
- close(e.ackCh)
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-test/internal/fsms/watcher.go b/vendor/github.com/CanonicalLtd/raft-test/internal/fsms/watcher.go
deleted file mode 100644
index f10aa1bffd..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/internal/fsms/watcher.go
+++ /dev/null
@@ -1,77 +0,0 @@
-// Copyright 2017 Canonical Ld.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package fsms
-
-import (
- "github.com/CanonicalLtd/raft-test/internal/event"
- "github.com/hashicorp/go-hclog"
- "github.com/hashicorp/raft"
-)
-
-// Watcher watches all FSMs of a cluster, firing events at certain moments.
-type Watcher struct {
- logger hclog.Logger
-
- // FSM wrappers.
- fsms map[raft.ServerID]*fsmWrapper
-}
-
-// New create a new FSMs watcher for watching the underlying FSMs.
-func New(logger hclog.Logger) *Watcher {
- return &Watcher{
- logger: logger,
- fsms: make(map[raft.ServerID]*fsmWrapper),
- }
-}
-
-// Add an FSM to the watcher. Returns an FSM that wraps the given FSM with
-// instrumentation for firing events.
-func (w *Watcher) Add(id raft.ServerID, fsm raft.FSM) raft.FSM {
- w.fsms[id] = newFSMWrapper(w.logger, id, fsm)
- return w.fsms[id]
-}
-
-// WhenApplied returns an event that will fire when the n'th command log for
-// the term is applied on the FSM associated with the server with the given
-// ID. It's that such server is currently the leader.
-func (w *Watcher) WhenApplied(id raft.ServerID, n uint64) *event.Event {
- return w.fsms[id].whenApplied(n)
-}
-
-// Commands returns the total number of command logs applied by the FSM of
-// the server with the given ID.
-func (w *Watcher) Commands(id raft.ServerID) uint64 {
- return w.fsms[id].Commands()
-}
-
-// Snapshots returns the total number of snapshots performed by the FSM of the
-// server with the given ID.
-func (w *Watcher) Snapshots(id raft.ServerID) uint64 {
- return w.fsms[id].Snapshots()
-}
-
-// Restores returns the total number of restores performed by the FSM of the
-// server with the given ID.
-func (w *Watcher) Restores(id raft.ServerID) uint64 {
- return w.fsms[id].Restores()
-}
-
-// Electing must be called whenever the given server is about to transition to
-// the leader state, and before any new command log is applied.
-//
-// It resets the internal state of the FSN, such the the commands counter.
-func (w *Watcher) Electing(id raft.ServerID) {
- w.fsms[id].electing()
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-test/internal/fsms/wrapper.go b/vendor/github.com/CanonicalLtd/raft-test/internal/fsms/wrapper.go
deleted file mode 100644
index ea86a49550..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/internal/fsms/wrapper.go
+++ /dev/null
@@ -1,188 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package fsms
-
-import (
- "encoding/binary"
- "fmt"
- "io"
- "sync"
-
- "github.com/CanonicalLtd/raft-test/internal/event"
- "github.com/hashicorp/go-hclog"
- "github.com/hashicorp/raft"
- "github.com/pkg/errors"
-)
-
-// Wraps a raft.FSM, adding control on logs, snapshots and restores.
-type fsmWrapper struct {
- logger hclog.Logger
-
- // ID of of the raft server associated with this FSM.
- id raft.ServerID
-
- // Wrapped FSM
- fsm raft.FSM
-
- // Total number of commands applied by this FSM.
- commands uint64
-
- // Total number of snapshots performed on this FSM.
- snapshots uint64
-
- // Total number of restores performed on this FSM.
- restores uint64
-
- // Events that should be fired when a certain command log is events.
- events map[uint64][]*event.Event
-
- mu sync.RWMutex
-}
-
-func newFSMWrapper(logger hclog.Logger, id raft.ServerID, fsm raft.FSM) *fsmWrapper {
- return &fsmWrapper{
- logger: logger,
- id: id,
- fsm: fsm,
- events: make(map[uint64][]*event.Event),
- }
-}
-
-func (f *fsmWrapper) Apply(log *raft.Log) interface{} {
- result := f.fsm.Apply(log)
-
- f.mu.Lock()
- f.commands++
- f.mu.Unlock()
-
- f.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: fsm %s: applied %d", f.id, f.commands))
- if events, ok := f.events[f.commands]; ok {
- for _, event := range events {
- event.Fire()
- event.Block()
- }
- }
-
- return result
-}
-
-// Snapshot always return a dummy snapshot and no error without doing
-// anything.
-func (f *fsmWrapper) Snapshot() (raft.FSMSnapshot, error) {
- snapshot, err := f.fsm.Snapshot()
-
- if snapshot != nil {
- f.mu.Lock()
- f.snapshots++
- snapshot = &fsmSnapshotWrapper{
- commands: f.commands,
- snapshot: snapshot,
- }
- f.mu.Unlock()
- }
-
- return snapshot, err
-}
-
-// Restore always return a nil error without reading anything from
-// the reader.
-func (f *fsmWrapper) Restore(reader io.ReadCloser) error {
- if err := binary.Read(reader, binary.LittleEndian, &f.commands); err != nil {
- return errors.Wrap(err, "failed to restore commands count")
- }
- if err := f.fsm.Restore(reader); err != nil {
- return errors.Wrap(err, "failed to perform restore on user's FSM")
- }
-
- if events, ok := f.events[f.commands]; ok {
- for _, event := range events {
- event.Fire()
- event.Block()
- }
- }
-
- f.mu.Lock()
- f.restores++
- f.mu.Unlock()
-
- return nil
-}
-
-// This method must be called whenever the server associated with this FSM is
-// about to transition to the leader state, and before any new command log is
-// applied.
-//
-// It resets the internal state of the fsm, such as the list of applied command
-// logs and the scheduled events.
-func (f *fsmWrapper) electing() {
- f.mu.Lock()
- defer f.mu.Unlock()
- for n := range f.events {
- delete(f.events, n)
- }
-}
-
-// Return an event that will fire when the n'th command log for the term is
-// applied on this FSM. It's assumed that this FSM is associated with the
-// current leader.
-func (f *fsmWrapper) whenApplied(n uint64) *event.Event {
- e := event.New()
- f.mu.RLock()
- defer f.mu.RUnlock()
- if f.commands >= n {
- // Fire immediately.
- go e.Fire()
- } else {
- _, ok := f.events[n]
- if !ok {
- f.events[n] = make([]*event.Event, 0)
- }
- f.events[n] = append(f.events[n], e)
- }
- return e
-}
-
-// Return the total number of command logs applied by this FSM.
-func (f *fsmWrapper) Commands() uint64 {
- return f.commands
-}
-
-// Return the total number of snapshots performed by this FSM.
-func (f *fsmWrapper) Snapshots() uint64 {
- return f.snapshots
-}
-
-// Return the total number of restores performed by this FSM.
-func (f *fsmWrapper) Restores() uint64 {
- return f.restores
-}
-
-type fsmSnapshotWrapper struct {
- commands uint64
- snapshot raft.FSMSnapshot
-}
-
-func (s *fsmSnapshotWrapper) Persist(sink raft.SnapshotSink) error {
- // Augment the snapshot with the current command count.
- if err := binary.Write(sink, binary.LittleEndian, s.commands); err != nil {
- return errors.Wrap(err, "failed to augment snapshot with commands count")
- }
- if err := s.snapshot.Persist(sink); err != nil {
- return errors.Wrap(err, "failed to perform snapshot on user's FSM")
- }
- return nil
-}
-
-func (s *fsmSnapshotWrapper) Release() {}
diff --git a/vendor/github.com/CanonicalLtd/raft-test/internal/logging/logger.go b/vendor/github.com/CanonicalLtd/raft-test/internal/logging/logger.go
deleted file mode 100644
index 07ec09f63c..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/internal/logging/logger.go
+++ /dev/null
@@ -1,50 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package logging
-
-import (
- "testing"
-
- "github.com/hashicorp/logutils"
- "github.com/hashicorp/go-hclog"
-)
-
-// New returns a standard hclog.Logger that will write entries at or above the
-// specified level to the testing log.
-func New(t testing.TB, level logutils.LogLevel) hclog.Logger {
- filter := &logutils.LevelFilter{
- Levels: []logutils.LogLevel{"DEBUG", "WARN", "ERROR", "INFO"},
- MinLevel: level,
- Writer: &testingWriter{t},
- }
-
- return hclog.New(&hclog.LoggerOptions{
- Name: "raft-test",
- Output: filter,
- })
-}
-
-// Implement io.Writer and forward what it receives to a
-// testing logger.
-type testingWriter struct {
- t testing.TB
-}
-
-// Write a single log entry. It's assumed that p is always a \n-terminated UTF
-// string.
-func (w *testingWriter) Write(p []byte) (n int, err error) {
- w.t.Logf(string(p))
- return len(p), nil
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-test/internal/network/logs.go b/vendor/github.com/CanonicalLtd/raft-test/internal/network/logs.go
deleted file mode 100644
index df88af9e31..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/internal/network/logs.go
+++ /dev/null
@@ -1,76 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package network
-
-import (
- "fmt"
- "strings"
-
- "github.com/hashicorp/raft"
-)
-
-// Return a string representation of the given log entries.
-func stringifyLogs(logs []*raft.Log) string {
- n := len(logs)
- description := fmt.Sprintf("%d ", n)
- if n == 1 {
- description += "entry"
- } else {
- description += "entries"
- }
-
- if n > 0 {
- entries := make([]string, n)
- for i, log := range logs {
- name := "Other"
- switch log.Type {
- case raft.LogCommand:
- name = "Command"
- case raft.LogNoop:
- name = "Noop"
- }
- entries[i] = fmt.Sprintf("%s:term=%d,index=%d", name, log.Term, log.Index)
- }
- description += fmt.Sprintf(" [%s]", strings.Join(entries, " "))
- }
-
- return description
-}
-
-// This function takes a set of log entries that have been successfully
-// appended to a peer and filters out any log entry with an older term relative
-// to the others.
-//
-// The returned entries are guaranted to have the same term and that term is
-// the highest among the ones in this batch.
-func filterLogsWithOlderTerms(logs []*raft.Log) []*raft.Log {
- // Find the highest term.
- var highestTerm uint64
- for _, log := range logs {
- if log.Term > highestTerm {
- highestTerm = log.Term
- }
- }
-
- // Discard any log with an older term than the highest one.
- filteredLogs := make([]*raft.Log, 0)
- for _, log := range logs {
- if log.Term == highestTerm {
- filteredLogs = append(filteredLogs, log)
- }
- }
-
- return filteredLogs
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-test/internal/network/network.go b/vendor/github.com/CanonicalLtd/raft-test/internal/network/network.go
deleted file mode 100644
index b41cc6e0a2..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/internal/network/network.go
+++ /dev/null
@@ -1,147 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package network
-
-import (
- "fmt"
-
- "github.com/CanonicalLtd/raft-test/internal/event"
- "github.com/hashicorp/go-hclog"
- "github.com/hashicorp/raft"
-)
-
-// Network provides control over all transports of a cluster, injecting
-// disconnections and failures.
-type Network struct {
- logger hclog.Logger
-
- // Transport wrappers.
- transports map[raft.ServerID]*eventTransport
-}
-
-// New create a new network for controlling the underlying transports.
-func New(logger hclog.Logger) *Network {
- return &Network{
- logger: logger,
- transports: make(map[raft.ServerID]*eventTransport),
- }
-}
-
-// Add a new transport to the network. Returns a transport that wraps the given
-// transport with instrumentation to inject disconnections and failures.
-func (n *Network) Add(id raft.ServerID, trans raft.Transport) raft.Transport {
- transport := newEventTransport(n.logger, id, trans)
-
- for _, other := range n.transports {
- transport.AddPeer(other)
- other.AddPeer(transport)
- }
-
- n.transports[id] = transport
- return transport
-}
-
-// Electing resets any leader-related state in the transport associated with
-// given server ID (such as the track of logs appended by the peers), and it
-// connects the transport to all its peers, enabling it to send them RPCs. It
-// must be called whenever the server associated with this transport is about
-// to transition to the leader state, and before any append entries RPC is
-// made.
-func (n *Network) Electing(id raft.ServerID) {
- n.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: server %s: establish outbound connection to all other nodes", id))
-
- // Sanity check that the network is fully disconnected at this time.
- for id, transport := range n.transports {
- if transport.Connected() {
- panic(fmt.Sprintf("expected a fully disconected network, but server %s is connected", id))
- }
- }
-
- transport := n.transports[id]
- transport.Electing()
-}
-
-// Deposing disables connectivity from the transport of the server with the
-// given ID to all its peers, allowing only append entries RPCs for peers that
-// are lagging behind in terms of applied logs to be performed.
-func (n *Network) Deposing(id raft.ServerID) {
- n.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: server %s: dropping outbound connection to all other nodes", id))
- n.transports[id].Deposing()
-}
-
-// ConnectAllServers establishes full cluster connectivity after an
-// election. The given ID is the one of the leader, which is already connected.
-func (n *Network) ConnectAllServers(id raft.ServerID) {
- // Sanity check that the network is fully disconnected at this time.
- for other, transport := range n.transports {
- if other == id {
- continue
- }
- transport.peers.Connect()
- }
-}
-
-// Disconnect disables connectivity from the transport of the leader
-// server with the given ID to the peer with the given ID.
-func (n *Network) Disconnect(id, follower raft.ServerID) {
- n.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: server %s: disconnecting follower %s", id, follower))
- n.transports[id].Disconnect(follower)
-}
-
-// Reconnect re-enables connectivity from the transport of the leader
-// server with the given ID to the peer with the given ID.
-func (n *Network) Reconnect(id, follower raft.ServerID) {
- n.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: server %s: reconnecting follower %s", id, follower))
- n.transports[id].Reconnect(follower)
-}
-
-// PeerConnected returns whether the peer with the given server ID is connected
-// with the transport of the server with the given ID.
-func (n *Network) PeerConnected(id, peer raft.ServerID) bool {
- return n.transports[id].PeerConnected(peer)
-}
-
-// Address returns the address of the server with the given id.
-func (n *Network) Address(id raft.ServerID) raft.ServerAddress {
- return n.transports[id].LocalAddr()
-}
-
-// HasAppendedLogsFromTo returns true if at least one log entry has been appended
-// by server with id1 to server with id2.
-//
-// It is assumed that id1 is a leader that has just been elected and has been
-// trying to append a noop log to all its followers.
-func (n *Network) HasAppendedLogsFromTo(id1, id2 raft.ServerID) bool {
- transport := n.transports[id1]
- return transport.HasAppendedLogsTo(id2)
-}
-
-// ScheduleEnqueueFailure will make all followers of the given server fail when
-// the leader tries to append the n'th log command. Return an event that will
-// fire when all of them have failed and will block them all until
-// acknowledged.
-func (n *Network) ScheduleEnqueueFailure(id raft.ServerID, command uint64) *event.Event {
- transport := n.transports[id]
- return transport.ScheduleEnqueueFailure(command)
-}
-
-// ScheduleAppendFailure will make all followers of the given leader server
-// append the n'th log command sent by the leader, but they will fail to
-// acknowledge the leader about it. Return an event that will fire when all of
-// them have failed and will block them all until acknowledged.
-func (n *Network) ScheduleAppendFailure(id raft.ServerID, command uint64) *event.Event {
- transport := n.transports[id]
- return transport.ScheduleAppendFailure(command)
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-test/internal/network/peers.go b/vendor/github.com/CanonicalLtd/raft-test/internal/network/peers.go
deleted file mode 100644
index a386c2982a..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/internal/network/peers.go
+++ /dev/null
@@ -1,307 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package network
-
-import (
- "fmt"
- "sync"
-
- "github.com/hashicorp/raft"
-)
-
-// Small wrapper around a map of raft.ServerID->peer, offering concurrency
-// safety. This bit of information is not on faultyTransport directly, since it
-// needs to be shared between faultyTransport and faultyPipeline.
-type peers struct {
- peers map[raft.ServerID]*peer
- mu sync.RWMutex
-}
-
-// Create a new empty peers map.
-func newPeers() *peers {
- return &peers{
- peers: make(map[raft.ServerID]*peer),
- }
-}
-
-// Add a new peer for the given source and target server IDs.
-func (p *peers) Add(source, target raft.ServerID) {
- p.peers[target] = newPeer(source, target)
-}
-
-// Get the peer with the given ID.
-func (p *peers) Get(id raft.ServerID) *peer {
- // Sinces peers entries are inserted at initialization time by the
- // Cluster() function, and currently they never change afterwise,
- // there's no need to protect this method with the mutex.
- return p.peers[id]
-}
-
-// Return all the peers
-func (p *peers) All() map[raft.ServerID]*peer {
- // Sinces peers entries are inserted at initialization time by the
- // Cluster() function, and currently they never change afterwise,
- // there's no need to protect this method with the mutex.
- return p.peers
-}
-
-// Enable connectivity to all the peers in this map.
-func (p *peers) Connect() {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- for _, peer := range p.peers {
- peer.Connect()
- }
-}
-
-// Returns true if all peers are connected, false otherwise.
-//
-// It panics if some nodes are connected and others are not.
-func (p *peers) Connected() bool {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- connected := false
- for id, peer := range p.peers {
- if !connected {
- connected = peer.Connected()
- } else if !peer.Connected() {
- panic(fmt.Sprintf("server %s is not not connected while some others are", id))
- }
- }
- return connected
-}
-
-// Disable connectivity to all the peers in this map. However allow for peers
-// that are lagging behind in terms of received entries to still receive
-// AppendEntries RPCs.
-func (p *peers) SoftDisconnect() {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- for _, peer := range p.peers {
- peer.SoftDisconnect()
- }
-}
-
-// Whether the given target peer is both disconnected from its source
-// transport, and it's not syncing logs with other peers (i.e. either they are
-// at the same index of the peer with the highest index of appended logs, or
-// the peer has been hard-disconnected)
-func (p *peers) DisconnectedAndNotSyncing(id raft.ServerID) bool {
- p.mu.RLock()
- defer p.mu.RUnlock()
-
- for _, peer := range p.peers {
- peer.mu.RLock()
- defer peer.mu.RUnlock()
- }
-
- this := p.peers[id]
- if this.connected {
- return false
- }
-
- if !this.allowSyncing {
- return true
- }
-
- count := this.LogsCount()
-
- for _, other := range p.peers {
- if other.target == this.target {
- continue
- }
- if count < other.LogsCount() {
- return false
- }
- }
-
- return true
-}
-
-// Hold information about a single peer server that a faultyTransport is
-// sending RPCs to.
-type peer struct {
- // Server ID of the server sending RPCs to the peer.
- source raft.ServerID
-
- // Server ID of the peer server.
- target raft.ServerID
-
- // Whether connectivity is up. The transport can send RPCs to the peer
- // server only if this value is true.
- connected bool
-
- // Whether to allow appending entries to this peer even if the
- // connected field is false. Used for bringing the logs appended by a
- // peer in sync with the others.
- allowSyncing bool
-
- // Logs successfully appended to this peer since the server of the
- // transport we're associated with has acquired leadership. This keeps
- // only logs tagged with the same term the leader was elected at.
- logs []*raft.Log
-
- // Serialize access to internal state.
- mu sync.RWMutex
-}
-
-// Create a new peer for the given server.
-func newPeer(source, target raft.ServerID) *peer {
- return &peer{
- target: target,
- logs: make([]*raft.Log, 0),
- }
-}
-
-// Enable connectivity between the source transport and the target peer.
-func (p *peer) Connect() {
- p.mu.Lock()
- defer p.mu.Unlock()
- if p.connected {
- panic(fmt.Sprintf("server %s is already connected with server %s", p.source, p.target))
- }
- p.connected = true
- p.allowSyncing = false
-}
-
-// Disable connectivity between the source transport and the target
-// peer.
-func (p *peer) Disconnect() {
- p.mu.Lock()
- defer p.mu.Unlock()
- if !p.connected {
- panic(fmt.Sprintf("server %s is already disconnected from server %s", p.source, p.target))
- }
- p.connected = false
- p.allowSyncing = false
-}
-
-// Re-enables connectivity between the source transport and the target
-// peer.
-func (p *peer) Reconnect() {
- p.mu.Lock()
- defer p.mu.Unlock()
- if p.connected {
- panic(fmt.Sprintf("server %s is already connected with server %s", p.source, p.target))
- }
- p.connected = true
- p.allowSyncing = false
-}
-
-// Disable connectivity between the source transport and the target
-// peer. However allow for peers that are lagging behind in terms of received
-// entries to still receive AppendEntries RPCs.
-func (p *peer) SoftDisconnect() {
- p.mu.Lock()
- defer p.mu.Unlock()
- if !p.connected {
- panic(fmt.Sprintf("server %s is already disconnected from server %s", p.source, p.target))
- }
- p.connected = false
- p.allowSyncing = true
-}
-
-// Return whether this source transport is connected to the target peer.
-func (p *peer) Connected() bool {
- p.mu.RLock()
- defer p.mu.RUnlock()
- return p.connected
-}
-
-// Reset all recorded logs. Should be called when a new server is elected.
-func (p *peer) ResetLogs() {
- p.logs = p.logs[:0]
-}
-
-// This method updates the logs that the peer successfully appended. It must be
-// called whenever the transport is confident that logs have been
-// appended. There are two cases:
-//
-// - Transport.AppendEntries(): this is synchronous so UpdateLogs() can be invoked
-// as soon as the AppendEntries() call returns.
-//
-// - AppendPipeline.AppendEntries(): this is asynchronous, so UpdateLogs() should
-// be invoked only when the AppendFuture returned by AppendEntries() completes.
-//
-// In practice, the current implementation of faultyTransport and
-// faultyPipeline is a bit sloppy about the above rules, since we can make some
-// assumptions about the flow of entries. See comments in faultyTransport and
-// faultyPipeline for more details.
-func (p *peer) UpdateLogs(logs []*raft.Log) {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- if len(logs) == 0 {
- return // Nothing to do.
- }
-
- // Discard any log with an older term (relative to the others).
- newLogs := filterLogsWithOlderTerms(logs)
-
- // If no logs have been received yet, just append everything.
- if len(p.logs) == 0 {
- p.logs = newLogs
- return
- }
-
- // Check if we have stored entries for older terms, and if so, discard
- // them.
- //
- // We only need to check the first entry, because we always store
- // entries that all have the same term.
- if p.logs[0].Term < newLogs[0].Term {
- p.logs = p.logs[:0]
- }
-
- // Append new logs that aren't duplicates.
- for _, newLog := range newLogs {
- duplicate := false
- for _, log := range p.logs {
- if newLog.Index == log.Index {
- duplicate = true
- break
- }
- }
- if duplicate {
- continue
- }
- p.logs = append(p.logs, newLog)
- }
-}
-
-// Return then number of all logs appended so far to this peer.
-func (p *peer) LogsCount() int {
- p.mu.RLock()
- defer p.mu.RUnlock()
-
- return len(p.logs)
-}
-
-// Return then number of command logs appended so far to this peer.
-func (p *peer) CommandLogsCount() uint64 {
- p.mu.RLock()
- defer p.mu.RUnlock()
-
- n := uint64(0)
- for _, log := range p.logs {
- if log.Type == raft.LogCommand {
- n++
- }
- }
- return n
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-test/internal/network/pipeline.go b/vendor/github.com/CanonicalLtd/raft-test/internal/network/pipeline.go
deleted file mode 100644
index 29d77e56f8..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/internal/network/pipeline.go
+++ /dev/null
@@ -1,166 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package network
-
-import (
- "fmt"
- "time"
-
- "github.com/hashicorp/go-hclog"
- "github.com/hashicorp/raft"
-)
-
-// Wrap a regular raft.AppendPipeline, adding support for triggering events at
-// specific times.
-type eventPipeline struct {
- logger hclog.Logger
-
- // Server ID sending RPCs.
- source raft.ServerID
-
- // Server ID this pipeline is sending RPCs to.
- target raft.ServerID
-
- // Regular pipeline that we are wrapping.
- pipeline raft.AppendPipeline
-
- // All other peers connected to our transport. Syncing logs after a
- // disconnection.
- peers *peers
-
- // Fault that should happen in this transport during a term.
- schedule *schedule
-
- // If non-zero, the pipeline will artificially return an error to its
- // consumer when firing the response of a request whose entries contain
- // a log with this index. This happens after the peer as actually
- // appended the request's entries and it effectively simulates a
- // follower disconnecting before it can acknowledge the leader of a
- // successful request.
- failure uint64
-
- // To stop the pipeline.
- shutdownCh chan struct{}
-}
-
-// AppendEntries is used to add another request to the pipeline.
-// The send may block which is an effective form of back-pressure.
-func (p *eventPipeline) AppendEntries(
- args *raft.AppendEntriesRequest, resp *raft.AppendEntriesResponse) (raft.AppendFuture, error) {
-
- p.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: server %s: pipeline: append to %s: %s", p.source, p.target, stringifyLogs(args.Entries)))
-
- peer := p.peers.Get(p.target)
- faulty := false
- if p.schedule != nil {
- n := peer.CommandLogsCount()
- args, faulty = p.schedule.FilterRequest(n, args)
- if faulty && p.schedule.IsEnqueueFault() {
- p.logger.Debug(fmt.Sprintf(
- "[DEBUG] raft-test: server %s: pipeline: append to: %s: enqueue fault: command %d", p.source, p.target, p.schedule.Command()))
- }
- }
-
- if p.peers.DisconnectedAndNotSyncing(p.target) {
- p.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: server %s: pipeline: append to %s: not connected", p.source, p.target))
- return nil, fmt.Errorf("cannot reach server %s", p.target)
- }
-
- if faulty && p.schedule.IsAppendFault() {
- p.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: server %s: pipeline: append to %s: append fault: command %d", p.source, p.target, p.schedule.n))
- p.failure = args.Entries[0].Index
- }
-
- future, err := p.pipeline.AppendEntries(args, resp)
- if err != nil {
- return nil, err
- }
- peer.UpdateLogs(args.Entries)
-
- if faulty && p.schedule.IsEnqueueFault() {
- p.schedule.OccurredOn(p.target)
- p.schedule.event.Block()
- return nil, fmt.Errorf("cannot reach server %s", p.target)
- }
-
- return &appendFutureWrapper{
- id: p.target,
- future: future,
- }, nil
-}
-
-// Consumer returns a channel that can be used to consume
-// response futures when they are ready.
-func (p *eventPipeline) Consumer() <-chan raft.AppendFuture {
- ch := make(chan raft.AppendFuture)
-
- go func() {
- for {
- select {
- case future := <-p.pipeline.Consumer():
- entries := future.Request().Entries
- fail := false
- if len(entries) > 0 && entries[0].Index == p.failure {
- fail = true
- }
- if fail {
- p.schedule.OccurredOn(p.target)
- p.schedule.event.Block()
- future = &appendFutureWrapper{id: p.target, future: future, failing: true}
- }
- ch <- future
- case <-p.shutdownCh:
- return
- }
- }
- }()
- return ch
-
-}
-
-// Close closes the pipeline and cancels all inflight RPCs
-func (p *eventPipeline) Close() error {
- err := p.pipeline.Close()
- close(p.shutdownCh)
- return err
-}
-
-type appendFutureWrapper struct {
- id raft.ServerID
- future raft.AppendFuture
- failing bool
-}
-
-func (f *appendFutureWrapper) Error() error {
- if f.failing {
- return fmt.Errorf("cannot reach server %s", f.id)
- }
- return f.future.Error()
-}
-
-func (f *appendFutureWrapper) Start() time.Time {
- return f.future.Start()
-}
-
-func (f *appendFutureWrapper) Request() *raft.AppendEntriesRequest {
- return f.future.Request()
-}
-func (f *appendFutureWrapper) Response() *raft.AppendEntriesResponse {
- response := f.future.Response()
- if f.failing {
- response.Success = false
- }
- return response
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-test/internal/network/schedule.go b/vendor/github.com/CanonicalLtd/raft-test/internal/network/schedule.go
deleted file mode 100644
index 3b5c0293c5..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/internal/network/schedule.go
+++ /dev/null
@@ -1,178 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package network
-
-import (
- "sync"
-
- "github.com/CanonicalLtd/raft-test/internal/event"
- "github.com/hashicorp/raft"
-)
-
-// Schedule contains details about under when a certain event should occur.
-type schedule struct {
- // List of peers that the event should occurr on.
- peers []raft.ServerID
-
- // The event should fire when the transport tries to append n'th
- // command log command in this term.
- n uint64
-
- // Event object that should be fired when all peers have been trying to
- // append the given command.
- event *event.Event
-
- // Track peers where the event already occurred.
- occurred []bool
-
- // If true, the event should occur after the command log has been
- // appended to all followers.
- append bool
-
- // Serialize access to internal state.
- mu sync.RWMutex
-}
-
-// Return a zero value fault that will never occurr.
-func newSchedule() *schedule {
- return &schedule{}
-}
-
-// Add a server to the list of peers where the event should occurr.
-func (s *schedule) AddPeer(id raft.ServerID) {
- s.peers = append(s.peers, id)
- s.occurred = append(s.occurred, false)
-}
-
-// Resets this fault to not occur.
-func (s *schedule) NoEvent() {
- s.n = 0
- s.event = nil
- for i := range s.occurred {
- s.occurred[i] = false
- }
- s.append = false
-}
-
-// Configure this scheduler to fire the given event when the append entries RPC to
-// apply the n'th command log has failed on all given peers.
-func (s *schedule) EnqueueFailure(n uint64, event *event.Event) {
- s.n = n
- s.event = event
- for i := range s.occurred {
- s.occurred[i] = false
- }
-}
-
-// Configure this scheduler to fire the given event after the n'th command log has
-// been appended by all peers but has a failed to be notified to all consumers.
-func (s *schedule) AppendFailure(n uint64, event *event.Event) {
- s.n = n
- s.event = event
- for i := range s.occurred {
- s.occurred[i] = false
- }
- s.append = true
-}
-
-// FilterRequest scans the entries in the given append request, to see whether they
-// contain the command log that this fault is supposed to trigger upon.
-//
-// The n parameter is the number of command logs successfully appended so far
-// in the current term.
-//
-// It returns a request object and a boolean value.
-//
-// If the fault should not be triggered by this request, the returned request
-// object is the same as the given one and the boolean value is false.
-//
-// If the fault should be be triggered by this request, the bolean value will
-// be true and for the returned request object the are two cases:
-//
-// 1) If this is an enqueue fault, the returned request object will have its
-// Entries truncated to exclude the failing command log entry and every
-// entry beyond that. This way all logs preceeding the failing command log
-// will still be appended to the peer and the associated apply futures will
-// succeed, although the failing command log won't be applied and its apply
-// future will fail with ErrLeadershipLost.
-//
-// 1) If this is an append fault, the returned request object will be the same
-// as the given one. This way all logs willl be appended to the peer,
-// although the transport pretend that the append entries RPC has failed,
-// simulating a disconnection when delivering the RPC reply.
-//
-func (s *schedule) FilterRequest(n uint64, args *raft.AppendEntriesRequest) (*raft.AppendEntriesRequest, bool) {
- if s.n == 0 {
- return args, false
- }
-
- for i, log := range args.Entries {
- // Only consider command log entries.
- if log.Type != raft.LogCommand {
- continue
- }
- n++
- if n != s.n {
- continue
- }
-
- // We found a match.
- if !s.append {
- truncatedArgs := *args
- truncatedArgs.Entries = args.Entries[:i]
- args = &truncatedArgs
- }
- return args, true
-
- }
- return args, false
-}
-
-// Return the command log sequence number that should trigger this fault.
-//
-// For example if the fault was set to fail at the n'th command log appended
-// during the term, the n is returned.
-func (s *schedule) Command() uint64 {
- return s.n
-}
-
-// Return true if this is an enqueue fault.
-func (s *schedule) IsEnqueueFault() bool {
- return !s.append
-}
-
-// Return true if this is an append fault.
-func (s *schedule) IsAppendFault() bool {
- return s.append
-}
-
-// Mark the fault as occurred on the given server, and fire the event if it has
-// occurred on all servers.
-func (s *schedule) OccurredOn(id raft.ServerID) {
- s.mu.Lock()
- defer s.mu.Unlock()
- for i, other := range s.peers {
- if other == id {
- s.occurred[i] = true
- }
- }
-
- for _, flag := range s.occurred {
- if !flag {
- return
- }
- }
- s.event.Fire()
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-test/internal/network/transport.go b/vendor/github.com/CanonicalLtd/raft-test/internal/network/transport.go
deleted file mode 100644
index 599b83d06e..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/internal/network/transport.go
+++ /dev/null
@@ -1,268 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package network
-
-import (
- "fmt"
- "io"
-
- "github.com/CanonicalLtd/raft-test/internal/event"
- "github.com/hashicorp/go-hclog"
- "github.com/hashicorp/raft"
-)
-
-// Wrap a regular raft.Transport, adding support for trigger events at
-// specific times.
-type eventTransport struct {
- logger hclog.Logger
-
- // ID of of the raft server associated with this transport.
- id raft.ServerID
-
- // The regular raft.Transport beging wrapped.
- trans raft.Transport
-
- // Track the peers we are sending RPCs to.
- peers *peers
-
- // Schedule and event that should happen in this transport during a
- // term.
- schedule *schedule
-}
-
-// Create a new transport wrapper..
-func newEventTransport(logger hclog.Logger, id raft.ServerID, trans raft.Transport) *eventTransport {
- return &eventTransport{
- logger: logger,
- id: id,
- trans: trans,
- peers: newPeers(),
- schedule: newSchedule(),
- }
-}
-
-// Consumer returns a channel that can be used to
-// consume and respond to RPC requests.
-func (t *eventTransport) Consumer() <-chan raft.RPC {
- return t.trans.Consumer()
-}
-
-// LocalAddr is used to return our local address to distinguish from our peers.
-func (t *eventTransport) LocalAddr() raft.ServerAddress {
- return t.trans.LocalAddr()
-}
-
-// AppendEntriesPipeline returns an interface that can be used to pipeline
-// AppendEntries requests.
-func (t *eventTransport) AppendEntriesPipeline(
- id raft.ServerID, target raft.ServerAddress) (raft.AppendPipeline, error) {
-
- if t.peers.DisconnectedAndNotSyncing(id) {
- t.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: server %s: transport: append to %s: not connected", t.id, id))
- return nil, fmt.Errorf("cannot reach server %s", id)
- }
- if !t.peers.Get(id).Connected() {
- t.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: server %s: transport: append to %s: syncing logs", t.id, id))
- }
-
- pipeline, err := t.trans.AppendEntriesPipeline(id, target)
- if err != nil {
- return nil, err
- }
-
- pipeline = &eventPipeline{
- logger: t.logger,
- source: t.id,
- target: id,
- pipeline: pipeline,
- peers: t.peers,
- schedule: t.schedule,
- shutdownCh: make(chan struct{}),
- }
-
- return pipeline, nil
-}
-
-// AppendEntries sends the appropriate RPC to the target node.
-func (t *eventTransport) AppendEntries(
- id raft.ServerID, target raft.ServerAddress, args *raft.AppendEntriesRequest,
- resp *raft.AppendEntriesResponse) error {
-
- peer := t.peers.Get(id)
- t.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: server %s: transport: append to %s: %s", t.id, id, stringifyLogs(args.Entries)))
-
- // If a fault is set, check if this batch of entries contains a command
- // log matching the one configured in the fault.
- faulty := false
- if t.schedule != nil {
- n := peer.CommandLogsCount()
- args, faulty = t.schedule.FilterRequest(n, args)
- if faulty && t.schedule.IsEnqueueFault() {
- t.logger.Debug(fmt.Sprintf(
- "[DEBUG] raft-test: server %s: transport: append to %s: enqueue fault: command %d", t.id, id, t.schedule.Command()))
- }
- }
-
- if t.peers.DisconnectedAndNotSyncing(id) {
- t.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: server %s: transport: append to %s: not connected", t.id, id))
- return fmt.Errorf("cannot reach server %s", id)
- }
- if !t.peers.Get(id).Connected() {
- t.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: server %s: transport: append to %s: syncing logs", t.id, id))
- }
-
- if err := t.trans.AppendEntries(id, target, args, resp); err != nil {
- return err
- }
-
- // Check for a newer term, stop running
- if resp.Term > args.Term {
- t.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: server %s: transport: append to %s: newer term", t.id, id))
- }
-
- peer.UpdateLogs(args.Entries)
-
- if faulty && t.schedule.IsEnqueueFault() {
- t.schedule.OccurredOn(id)
- t.schedule.event.Block()
- return fmt.Errorf("cannot reach server %s", id)
- }
-
- return nil
-}
-
-// RequestVote sends the appropriate RPC to the target node.
-func (t *eventTransport) RequestVote(
- id raft.ServerID, target raft.ServerAddress, args *raft.RequestVoteRequest,
- resp *raft.RequestVoteResponse) error {
-
- if !t.peers.Get(id).Connected() {
- return fmt.Errorf("connectivity to server %s is down", id)
- }
-
- return t.trans.RequestVote(id, target, args, resp)
-}
-
-// InstallSnapshot is used to push a snapshot down to a follower. The data is read from
-// the ReadCloser and streamed to the client.
-func (t *eventTransport) InstallSnapshot(
- id raft.ServerID, target raft.ServerAddress, args *raft.InstallSnapshotRequest,
- resp *raft.InstallSnapshotResponse, data io.Reader) error {
-
- if !t.peers.Get(id).Connected() {
- return fmt.Errorf("connectivity to server %s is down", id)
- }
- return t.trans.InstallSnapshot(id, target, args, resp, data)
-}
-
-// EncodePeer is used to serialize a peer's address.
-func (t *eventTransport) EncodePeer(id raft.ServerID, addr raft.ServerAddress) []byte {
- return t.trans.EncodePeer(id, addr)
-}
-
-// DecodePeer is used to deserialize a peer's address.
-func (t *eventTransport) DecodePeer(data []byte) raft.ServerAddress {
- return t.trans.DecodePeer(data)
-}
-
-// SetHeartbeatHandler is used to setup a heartbeat handler
-// as a fast-pass. This is to avoid head-of-line blocking from
-// disk IO. If a Transport does not support this, it can simply
-// ignore the call, and push the heartbeat onto the Consumer channel.
-func (t *eventTransport) SetHeartbeatHandler(cb func(rpc raft.RPC)) {
- t.trans.SetHeartbeatHandler(cb)
-}
-
-func (t *eventTransport) Close() error {
- if closer, ok := t.trans.(raft.WithClose); ok {
- return closer.Close()
- }
- return nil
-}
-
-// AddPeer adds a new transport as peer of this transport. Once the other
-// transport has become a peer, this transport will be able to send RPCs to it,
-// if the peer object 'connected' flag is on.
-func (t *eventTransport) AddPeer(transport *eventTransport) {
- t.peers.Add(t.id, transport.id)
- t.schedule.AddPeer(transport.id)
-}
-
-// Electing resets any leader-related state in this transport (such as the
-// track of logs appended by the peers), and it connects the transport to all
-// its peers, enabling it to send them RPCs. It must be called whenever the
-// server associated with this transport is about to transition to the leader
-// state, and before any append entries RPC is made.
-func (t *eventTransport) Electing() {
- t.schedule.NoEvent()
- for _, peer := range t.peers.All() {
- peer.ResetLogs()
- }
- t.peers.Connect()
-}
-
-// Deposing disables connectivity from this transport to all its peers,
-// allowing only append entries RPCs for peers that are lagging behind in terms
-// of applied logs to be performed.
-func (t *eventTransport) Deposing() {
- t.peers.SoftDisconnect()
-}
-
-// Disable connectivity from this transport to the given peer.
-func (t *eventTransport) Disconnect(id raft.ServerID) {
- t.peers.Get(id).Disconnect()
-}
-
-// Re-nable connectivity from this transport to the given peer.
-func (t *eventTransport) Reconnect(id raft.ServerID) {
- t.peers.Get(id).Reconnect()
-}
-
-// Returns true if all peers are connected, false otherwise.
-//
-// It panics if some nodes are connected and others are not.
-func (t *eventTransport) Connected() bool {
- return t.peers.Connected()
-}
-
-// Returns true if the given peer is connected.
-func (t *eventTransport) PeerConnected(id raft.ServerID) bool {
- return t.peers.Get(id).Connected()
-}
-
-// Returns true if this transport has appended logs to the given peer during
-// the term.
-func (t *eventTransport) HasAppendedLogsTo(id raft.ServerID) bool {
- peer := t.peers.Get(id)
- return peer.LogsCount() > 0
-}
-
-// Schedule the n'th command log to fail to be appended to the
-// followers. Return an event that will fire when all followers have reached
-// this failure.
-func (t *eventTransport) ScheduleEnqueueFailure(n uint64) *event.Event {
- event := event.New()
- t.schedule.EnqueueFailure(n, event)
- return event
-}
-
-// Schedule the n'th command log to fail to acknowledge that it has been
-// appended to the followers. Return an event that will fire when all followers
-// have reached this failure.
-func (t *eventTransport) ScheduleAppendFailure(n uint64) *event.Event {
- event := event.New()
- t.schedule.AppendFailure(n, event)
- return event
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-test/options.go b/vendor/github.com/CanonicalLtd/raft-test/options.go
deleted file mode 100644
index 77e7feef98..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/options.go
+++ /dev/null
@@ -1,107 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package rafttest
-
-import (
- "io/ioutil"
- "time"
-
- "github.com/hashicorp/go-hclog"
- "github.com/hashicorp/raft"
-)
-
-// Config sets a hook for tweaking the raft configuration of individual nodes.
-func Config(f func(int, *raft.Config)) Option {
- return func(nodes []*dependencies) {
- for i, node := range nodes {
- f(i, node.Conf)
- }
- }
-}
-
-// LogStore can be used to create custom log stores.
-//
-// The given function takes a node index as argument and returns the LogStore
-// that the node should use.
-func LogStore(factory func(int) raft.LogStore) Option {
- return func(nodes []*dependencies) {
- for i, node := range nodes {
- node.Logs = factory(i)
- }
- }
-}
-
-// Transport can be used to create custom transports.
-//
-// The given function takes a node index as argument and returns the Transport
-// that the node should use.
-//
-// If the transports returned by the factory do not implement
-// LoopbackTransport, the Disconnect API won't work.
-func Transport(factory func(int) raft.Transport) Option {
- return func(nodes []*dependencies) {
- for i, node := range nodes {
- node.Trans = factory(i)
- }
- }
-}
-
-// Latency is a convenience around Config that scales the values of the various
-// raft timeouts that would be set by default by Cluster.
-//
-// This option is orthogonal to the GO_RAFT_TEST_LATENCY environment
-// variable. If this option is used and GO_RAFT_TEST_LATENCY is set, they will
-// compound. E.g. passing a factor of 2.0 to this option and setting
-// GO_RAFT_TEST_LATENCY to 3.0 will have the net effect that default timeouts
-// are scaled by a factor of 6.0.
-func Latency(factor float64) Option {
- return Config(func(i int, config *raft.Config) {
- timeouts := []*time.Duration{
- &config.HeartbeatTimeout,
- &config.ElectionTimeout,
- &config.LeaderLeaseTimeout,
- &config.CommitTimeout,
- }
- for _, timeout := range timeouts {
- *timeout = scaleDuration(*timeout, factor)
- }
- })
-}
-
-// DiscardLogger is a convenience around Config that sets the output stream of
-// raft's logger to ioutil.Discard.
-func DiscardLogger() Option {
- return Config(func(i int, config *raft.Config) {
- config.Logger = hclog.New(&hclog.LoggerOptions{
- Name: "raft-test",
- Output: ioutil.Discard})
- })
-}
-
-// Servers can be used to indicate which nodes should be initially part of the
-// created cluster.
-//
-// If this option is not used, the default is to have all nodes be part of the
-// cluster.
-func Servers(indexes ...int) Option {
- return func(nodes []*dependencies) {
- for _, node := range nodes {
- node.Voter = false
- }
- for _, index := range indexes {
- nodes[index].Voter = true
- }
- }
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-test/server.go b/vendor/github.com/CanonicalLtd/raft-test/server.go
deleted file mode 100644
index 035774cdb0..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/server.go
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package rafttest
-
-import (
- "testing"
-
- "github.com/hashicorp/raft"
-)
-
-// Server is a convenience for creating a cluster with a single raft.Raft server
-// that immediately be elected as leader.
-//
-// The default network address of a test node is "0".
-//
-// Dependencies can be replaced or mutated using the various options.
-func Server(t *testing.T, fsm raft.FSM, options ...Option) (*raft.Raft, func()) {
- fsms := []raft.FSM{fsm}
-
- rafts, control := Cluster(t, fsms, options...)
- control.Elect("0")
-
- return rafts["0"], control.Close
-}
diff --git a/vendor/github.com/CanonicalLtd/raft-test/term.go b/vendor/github.com/CanonicalLtd/raft-test/term.go
deleted file mode 100644
index c60957e87f..0000000000
--- a/vendor/github.com/CanonicalLtd/raft-test/term.go
+++ /dev/null
@@ -1,219 +0,0 @@
-// Copyright 2017 Canonical Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package rafttest
-
-import (
- "fmt"
-
- "github.com/CanonicalLtd/raft-test/internal/election"
- "github.com/CanonicalLtd/raft-test/internal/event"
- "github.com/hashicorp/raft"
-)
-
-// A Term holds information about an event that should happen while a certain
-// node is the leader.
-type Term struct {
- control *Control
- id raft.ServerID
- leadership *election.Leadership
- events []*Event
-
- // Server ID of a follower that has been disconnect.
- disconnected raft.ServerID
-}
-
-// When can be used to schedule a certain action when a certain expected
-// event occurs in the cluster during this Term.
-func (t *Term) When() *Event {
- // TODO: check that we're not using Connect()
- t.control.t.Helper()
-
- event := &Event{
- term: t,
- }
-
- t.events = append(t.events, event)
- return event
-}
-
-// Disconnect a follower, which will stop receiving RPCs.
-func (t *Term) Disconnect(id raft.ServerID) {
- t.control.t.Helper()
-
- if t.disconnected != "" {
- t.control.t.Fatalf("raft-test: term: disconnecting more than one server is not supported")
- }
-
- if id == t.id {
- t.control.t.Fatalf("raft-test: term: disconnect error: server %s is the leader", t.id)
- }
-
- t.control.logger.Debug(fmt.Sprintf("[DEBUG] raft-test: term: disconnect %s", id))
-
- t.disconnected = id
- t.control.network.Disconnect(t.id, id)
-}
-
-// Reconnect a previously disconnected follower.
-func (t *Term) Reconnect(id raft.ServerID) {
- t.control.t.Helper()
-
- if id != t.disconnected {
- t.control.t.Fatalf("raft-test: term: reconnect error: server %s was not disconnected", id)
- }
-
- // Reconnecting a server might end up in a new election round, so we
- // have to be prepared for that.
- t.control.network.Reconnect(t.id, id)
- if t.control.waitLeadershipPropagated(t.id, t.leadership) {
- // Leadership was not lost and all followers are back
- // on track.
- return
- }
-
- // Leadership was lost, we must undergo a new election.
- //
- // FIXME: this prevents When() hooks to function properly. It's not a
- // big deal at the moment, since Disconnect() is mainly used for
- // snapshots, but it should be sorted.
- term := t.control.Elect(t.id)
- t.leadership = term.leadership
-}
-
-// Snapshot performs a snapshot on the given server.
-func (t *Term) Snapshot(id raft.ServerID) {
- t.control.t.Helper()
-
- r := t.control.servers[id]
- if err := r.Snapshot().Error(); err != nil {
- t.control.t.Fatalf("raft-test: term: snapshot error on server %s: %v", id, err)
- }
-}
-
-// Event that is expected to happen during a Term.
-type Event struct {
- term *Term
- isScheduled bool
-}
-
-// Command schedules the event to occur when the Raft.Apply() method is called
-// on the leader raft instance in order to apply the n'th command log during
-// the current term.
-func (e *Event) Command(n uint64) *Dispatch {
- e.term.control.t.Helper()
-
- if e.isScheduled {
- e.term.control.t.Fatal("raft-test: error: term event already scheduled")
- }
- e.isScheduled = true
-
- return &Dispatch{
- term: e.term,
- n: n,
- }
-}
-
-// Dispatch defines at which phase of the dispatch process a command log event
-// should fire.
-type Dispatch struct {
- term *Term
- n uint64
- event *event.Event
-}
-
-// Enqueued configures the command log event to occurr when the command log is
-// enqueued, but not yet appended by the followers.
-func (d *Dispatch) Enqueued() *Action {
- d.term.control.t.Helper()
-
- if d.event != nil {
- d.term.control.t.Fatal("raft-test: error: dispatch event already defined")
- }
- d.event = d.term.control.whenCommandEnqueued(d.term.id, d.n)
-
- return &Action{
- term: d.term,
- event: d.event,
- }
-}
-
-// Appended configures the command log event to occurr when the command log is
-// appended by all followers, but not yet committed by the leader.
-func (d *Dispatch) Appended() *Action {
- d.term.control.t.Helper()
-
- if d.event != nil {
- d.term.control.t.Fatal("raft-test: error: dispatch event already defined")
- }
-
- d.event = d.term.control.whenCommandAppended(d.term.id, d.n)
-
- return &Action{
- term: d.term,
- event: d.event,
- }
-}
-
-// Committed configures the command log event to occurr when the command log is
-// committed.
-func (d *Dispatch) Committed() *Action {
- d.term.control.t.Helper()
-
- if d.event != nil {
- d.term.control.t.Fatal("raft-test: error: dispatch event already defined")
- }
-
- d.event = d.term.control.whenCommandCommitted(d.term.id, d.n)
-
- return &Action{
- term: d.term,
- event: d.event,
- }
-}
-
-// Action defines what should happen when the event defined in the term occurs.
-type Action struct {
- term *Term
- event *event.Event
-}
-
-// Depose makes the action depose the current leader.
-func (a *Action) Depose() {
- a.term.control.t.Helper()
- //a.control.t.Logf(
- //"raft-test: event: schedule depose server %s when command %d gets %s", a.id, a.n, a.phase)
-
- a.term.control.deposing = make(chan struct{})
-
- go func() {
- //c.t.Logf("raft-test: node %d: fsm: wait log command %d", i, n)
- a.term.control.deposeUponEvent(a.event, a.term.id, a.term.leadership)
- }()
-}
-
-// Snapshot makes the action trigger a snapshot on the leader.
-//
-// The typical use is to take the snapshot after a certain command log gets
-// committed (see Dispatch.Committed()).
-func (a *Action) Snapshot() {
- a.term.control.t.Helper()
- // a.control.t.Logf(
- // "raft-test: event: schedule snapshot server %s when command %d gets %s", a.id, a.n, a.phase)
-
- go func() {
- //c.t.Logf("raft-test: node %d: fsm: wait log command %d", i, n)
- a.term.control.snapshotUponEvent(a.event, a.term.id)
- }()
-}
diff --git a/vendor/github.com/hashicorp/raft-boltdb/LICENSE b/vendor/github.com/hashicorp/raft-boltdb/LICENSE
deleted file mode 100644
index f0e5c79e18..0000000000
--- a/vendor/github.com/hashicorp/raft-boltdb/LICENSE
+++ /dev/null
@@ -1,362 +0,0 @@
-Mozilla Public License, version 2.0
-
-1. Definitions
-
-1.1. "Contributor"
-
- means each individual or legal entity that creates, contributes to the
- creation of, or owns Covered Software.
-
-1.2. "Contributor Version"
-
- means the combination of the Contributions of others (if any) used by a
- Contributor and that particular Contributor's Contribution.
-
-1.3. "Contribution"
-
- means Covered Software of a particular Contributor.
-
-1.4. "Covered Software"
-
- means Source Code Form to which the initial Contributor has attached the
- notice in Exhibit A, the Executable Form of such Source Code Form, and
- Modifications of such Source Code Form, in each case including portions
- thereof.
-
-1.5. "Incompatible With Secondary Licenses"
- means
-
- a. that the initial Contributor has attached the notice described in
- Exhibit B to the Covered Software; or
-
- b. that the Covered Software was made available under the terms of
- version 1.1 or earlier of the License, but not also under the terms of
- a Secondary License.
-
-1.6. "Executable Form"
-
- means any form of the work other than Source Code Form.
-
-1.7. "Larger Work"
-
- means a work that combines Covered Software with other material, in a
- separate file or files, that is not Covered Software.
-
-1.8. "License"
-
- means this document.
-
-1.9. "Licensable"
-
- means having the right to grant, to the maximum extent possible, whether
- at the time of the initial grant or subsequently, any and all of the
- rights conveyed by this License.
-
-1.10. "Modifications"
-
- means any of the following:
-
- a. any file in Source Code Form that results from an addition to,
- deletion from, or modification of the contents of Covered Software; or
-
- b. any new file in Source Code Form that contains any Covered Software.
-
-1.11. "Patent Claims" of a Contributor
-
- means any patent claim(s), including without limitation, method,
- process, and apparatus claims, in any patent Licensable by such
- Contributor that would be infringed, but for the grant of the License,
- by the making, using, selling, offering for sale, having made, import,
- or transfer of either its Contributions or its Contributor Version.
-
-1.12. "Secondary License"
-
- means either the GNU General Public License, Version 2.0, the GNU Lesser
- General Public License, Version 2.1, the GNU Affero General Public
- License, Version 3.0, or any later versions of those licenses.
-
-1.13. "Source Code Form"
-
- means the form of the work preferred for making modifications.
-
-1.14. "You" (or "Your")
-
- means an individual or a legal entity exercising rights under this
- License. For legal entities, "You" includes any entity that controls, is
- controlled by, or is under common control with You. For purposes of this
- definition, "control" means (a) the power, direct or indirect, to cause
- the direction or management of such entity, whether by contract or
- otherwise, or (b) ownership of more than fifty percent (50%) of the
- outstanding shares or beneficial ownership of such entity.
-
-
-2. License Grants and Conditions
-
-2.1. Grants
-
- Each Contributor hereby grants You a world-wide, royalty-free,
- non-exclusive license:
-
- a. under intellectual property rights (other than patent or trademark)
- Licensable by such Contributor to use, reproduce, make available,
- modify, display, perform, distribute, and otherwise exploit its
- Contributions, either on an unmodified basis, with Modifications, or
- as part of a Larger Work; and
-
- b. under Patent Claims of such Contributor to make, use, sell, offer for
- sale, have made, import, and otherwise transfer either its
- Contributions or its Contributor Version.
-
-2.2. Effective Date
-
- The licenses granted in Section 2.1 with respect to any Contribution
- become effective for each Contribution on the date the Contributor first
- distributes such Contribution.
-
-2.3. Limitations on Grant Scope
-
- The licenses granted in this Section 2 are the only rights granted under
- this License. No additional rights or licenses will be implied from the
- distribution or licensing of Covered Software under this License.
- Notwithstanding Section 2.1(b) above, no patent license is granted by a
- Contributor:
-
- a. for any code that a Contributor has removed from Covered Software; or
-
- b. for infringements caused by: (i) Your and any other third party's
- modifications of Covered Software, or (ii) the combination of its
- Contributions with other software (except as part of its Contributor
- Version); or
-
- c. under Patent Claims infringed by Covered Software in the absence of
- its Contributions.
-
- This License does not grant any rights in the trademarks, service marks,
- or logos of any Contributor (except as may be necessary to comply with
- the notice requirements in Section 3.4).
-
-2.4. Subsequent Licenses
-
- No Contributor makes additional grants as a result of Your choice to
- distribute the Covered Software under a subsequent version of this
- License (see Section 10.2) or under the terms of a Secondary License (if
- permitted under the terms of Section 3.3).
-
-2.5. Representation
-
- Each Contributor represents that the Contributor believes its
- Contributions are its original creation(s) or it has sufficient rights to
- grant the rights to its Contributions conveyed by this License.
-
-2.6. Fair Use
-
- This License is not intended to limit any rights You have under
- applicable copyright doctrines of fair use, fair dealing, or other
- equivalents.
-
-2.7. Conditions
-
- Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in
- Section 2.1.
-
-
-3. Responsibilities
-
-3.1. Distribution of Source Form
-
- All distribution of Covered Software in Source Code Form, including any
- Modifications that You create or to which You contribute, must be under
- the terms of this License. You must inform recipients that the Source
- Code Form of the Covered Software is governed by the terms of this
- License, and how they can obtain a copy of this License. You may not
- attempt to alter or restrict the recipients' rights in the Source Code
- Form.
-
-3.2. Distribution of Executable Form
-
- If You distribute Covered Software in Executable Form then:
-
- a. such Covered Software must also be made available in Source Code Form,
- as described in Section 3.1, and You must inform recipients of the
- Executable Form how they can obtain a copy of such Source Code Form by
- reasonable means in a timely manner, at a charge no more than the cost
- of distribution to the recipient; and
-
- b. You may distribute such Executable Form under the terms of this
- License, or sublicense it under different terms, provided that the
- license for the Executable Form does not attempt to limit or alter the
- recipients' rights in the Source Code Form under this License.
-
-3.3. Distribution of a Larger Work
-
- You may create and distribute a Larger Work under terms of Your choice,
- provided that You also comply with the requirements of this License for
- the Covered Software. If the Larger Work is a combination of Covered
- Software with a work governed by one or more Secondary Licenses, and the
- Covered Software is not Incompatible With Secondary Licenses, this
- License permits You to additionally distribute such Covered Software
- under the terms of such Secondary License(s), so that the recipient of
- the Larger Work may, at their option, further distribute the Covered
- Software under the terms of either this License or such Secondary
- License(s).
-
-3.4. Notices
-
- You may not remove or alter the substance of any license notices
- (including copyright notices, patent notices, disclaimers of warranty, or
- limitations of liability) contained within the Source Code Form of the
- Covered Software, except that You may alter any license notices to the
- extent required to remedy known factual inaccuracies.
-
-3.5. Application of Additional Terms
-
- You may choose to offer, and to charge a fee for, warranty, support,
- indemnity or liability obligations to one or more recipients of Covered
- Software. However, You may do so only on Your own behalf, and not on
- behalf of any Contributor. You must make it absolutely clear that any
- such warranty, support, indemnity, or liability obligation is offered by
- You alone, and You hereby agree to indemnify every Contributor for any
- liability incurred by such Contributor as a result of warranty, support,
- indemnity or liability terms You offer. You may include additional
- disclaimers of warranty and limitations of liability specific to any
- jurisdiction.
-
-4. Inability to Comply Due to Statute or Regulation
-
- If it is impossible for You to comply with any of the terms of this License
- with respect to some or all of the Covered Software due to statute,
- judicial order, or regulation then You must: (a) comply with the terms of
- this License to the maximum extent possible; and (b) describe the
- limitations and the code they affect. Such description must be placed in a
- text file included with all distributions of the Covered Software under
- this License. Except to the extent prohibited by statute or regulation,
- such description must be sufficiently detailed for a recipient of ordinary
- skill to be able to understand it.
-
-5. Termination
-
-5.1. The rights granted under this License will terminate automatically if You
- fail to comply with any of its terms. However, if You become compliant,
- then the rights granted under this License from a particular Contributor
- are reinstated (a) provisionally, unless and until such Contributor
- explicitly and finally terminates Your grants, and (b) on an ongoing
- basis, if such Contributor fails to notify You of the non-compliance by
- some reasonable means prior to 60 days after You have come back into
- compliance. Moreover, Your grants from a particular Contributor are
- reinstated on an ongoing basis if such Contributor notifies You of the
- non-compliance by some reasonable means, this is the first time You have
- received notice of non-compliance with this License from such
- Contributor, and You become compliant prior to 30 days after Your receipt
- of the notice.
-
-5.2. If You initiate litigation against any entity by asserting a patent
- infringement claim (excluding declaratory judgment actions,
- counter-claims, and cross-claims) alleging that a Contributor Version
- directly or indirectly infringes any patent, then the rights granted to
- You by any and all Contributors for the Covered Software under Section
- 2.1 of this License shall terminate.
-
-5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user
- license agreements (excluding distributors and resellers) which have been
- validly granted by You or Your distributors under this License prior to
- termination shall survive termination.
-
-6. Disclaimer of Warranty
-
- Covered Software is provided under this License on an "as is" basis,
- without warranty of any kind, either expressed, implied, or statutory,
- including, without limitation, warranties that the Covered Software is free
- of defects, merchantable, fit for a particular purpose or non-infringing.
- The entire risk as to the quality and performance of the Covered Software
- is with You. Should any Covered Software prove defective in any respect,
- You (not any Contributor) assume the cost of any necessary servicing,
- repair, or correction. This disclaimer of warranty constitutes an essential
- part of this License. No use of any Covered Software is authorized under
- this License except under this disclaimer.
-
-7. Limitation of Liability
-
- Under no circumstances and under no legal theory, whether tort (including
- negligence), contract, or otherwise, shall any Contributor, or anyone who
- distributes Covered Software as permitted above, be liable to You for any
- direct, indirect, special, incidental, or consequential damages of any
- character including, without limitation, damages for lost profits, loss of
- goodwill, work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses, even if such party shall have been
- informed of the possibility of such damages. This limitation of liability
- shall not apply to liability for death or personal injury resulting from
- such party's negligence to the extent applicable law prohibits such
- limitation. Some jurisdictions do not allow the exclusion or limitation of
- incidental or consequential damages, so this exclusion and limitation may
- not apply to You.
-
-8. Litigation
-
- Any litigation relating to this License may be brought only in the courts
- of a jurisdiction where the defendant maintains its principal place of
- business and such litigation shall be governed by laws of that
- jurisdiction, without reference to its conflict-of-law provisions. Nothing
- in this Section shall prevent a party's ability to bring cross-claims or
- counter-claims.
-
-9. Miscellaneous
-
- This License represents the complete agreement concerning the subject
- matter hereof. If any provision of this License is held to be
- unenforceable, such provision shall be reformed only to the extent
- necessary to make it enforceable. Any law or regulation which provides that
- the language of a contract shall be construed against the drafter shall not
- be used to construe this License against a Contributor.
-
-
-10. Versions of the License
-
-10.1. New Versions
-
- Mozilla Foundation is the license steward. Except as provided in Section
- 10.3, no one other than the license steward has the right to modify or
- publish new versions of this License. Each version will be given a
- distinguishing version number.
-
-10.2. Effect of New Versions
-
- You may distribute the Covered Software under the terms of the version
- of the License under which You originally received the Covered Software,
- or under the terms of any subsequent version published by the license
- steward.
-
-10.3. Modified Versions
-
- If you create software not governed by this License, and you want to
- create a new license for such software, you may create and use a
- modified version of this License if you rename the license and remove
- any references to the name of the license steward (except to note that
- such modified license differs from this License).
-
-10.4. Distributing Source Code Form that is Incompatible With Secondary
- Licenses If You choose to distribute Source Code Form that is
- Incompatible With Secondary Licenses under the terms of this version of
- the License, the notice described in Exhibit B of this License must be
- attached.
-
-Exhibit A - Source Code Form License Notice
-
- This Source Code Form is subject to the
- terms of the Mozilla Public License, v.
- 2.0. If a copy of the MPL was not
- distributed with this file, You can
- obtain one at
- http://mozilla.org/MPL/2.0/.
-
-If it is not possible or desirable to put the notice in a particular file,
-then You may include the notice in a location (such as a LICENSE file in a
-relevant directory) where a recipient would be likely to look for such a
-notice.
-
-You may add additional accurate notices of copyright ownership.
-
-Exhibit B - "Incompatible With Secondary Licenses" Notice
-
- This Source Code Form is "Incompatible
- With Secondary Licenses", as defined by
- the Mozilla Public License, v. 2.0.
\ No newline at end of file
diff --git a/vendor/github.com/hashicorp/raft-boltdb/Makefile b/vendor/github.com/hashicorp/raft-boltdb/Makefile
deleted file mode 100644
index bc5c6cc011..0000000000
--- a/vendor/github.com/hashicorp/raft-boltdb/Makefile
+++ /dev/null
@@ -1,11 +0,0 @@
-DEPS = $(go list -f '{{range .TestImports}}{{.}} {{end}}' ./...)
-
-.PHONY: test deps
-
-test:
- go test -timeout=30s ./...
-
-deps:
- go get -d -v ./...
- echo $(DEPS) | xargs -n1 go get -d
-
diff --git a/vendor/github.com/hashicorp/raft-boltdb/README.md b/vendor/github.com/hashicorp/raft-boltdb/README.md
deleted file mode 100644
index 5d7180ab9e..0000000000
--- a/vendor/github.com/hashicorp/raft-boltdb/README.md
+++ /dev/null
@@ -1,11 +0,0 @@
-raft-boltdb
-===========
-
-This repository provides the `raftboltdb` package. The package exports the
-`BoltStore` which is an implementation of both a `LogStore` and `StableStore`.
-
-It is meant to be used as a backend for the `raft` [package
-here](https://github.com/hashicorp/raft).
-
-This implementation uses [BoltDB](https://github.com/boltdb/bolt). BoltDB is
-a simple key/value store implemented in pure Go, and inspired by LMDB.
diff --git a/vendor/github.com/hashicorp/raft-boltdb/bolt_store.go b/vendor/github.com/hashicorp/raft-boltdb/bolt_store.go
deleted file mode 100644
index a1f9f0ba61..0000000000
--- a/vendor/github.com/hashicorp/raft-boltdb/bolt_store.go
+++ /dev/null
@@ -1,268 +0,0 @@
-package raftboltdb
-
-import (
- "errors"
-
- "github.com/boltdb/bolt"
- "github.com/hashicorp/raft"
-)
-
-const (
- // Permissions to use on the db file. This is only used if the
- // database file does not exist and needs to be created.
- dbFileMode = 0600
-)
-
-var (
- // Bucket names we perform transactions in
- dbLogs = []byte("logs")
- dbConf = []byte("conf")
-
- // An error indicating a given key does not exist
- ErrKeyNotFound = errors.New("not found")
-)
-
-// BoltStore provides access to BoltDB for Raft to store and retrieve
-// log entries. It also provides key/value storage, and can be used as
-// a LogStore and StableStore.
-type BoltStore struct {
- // conn is the underlying handle to the db.
- conn *bolt.DB
-
- // The path to the Bolt database file
- path string
-}
-
-// Options contains all the configuraiton used to open the BoltDB
-type Options struct {
- // Path is the file path to the BoltDB to use
- Path string
-
- // BoltOptions contains any specific BoltDB options you might
- // want to specify [e.g. open timeout]
- BoltOptions *bolt.Options
-
- // NoSync causes the database to skip fsync calls after each
- // write to the log. This is unsafe, so it should be used
- // with caution.
- NoSync bool
-}
-
-// readOnly returns true if the contained bolt options say to open
-// the DB in readOnly mode [this can be useful to tools that want
-// to examine the log]
-func (o *Options) readOnly() bool {
- return o != nil && o.BoltOptions != nil && o.BoltOptions.ReadOnly
-}
-
-// NewBoltStore takes a file path and returns a connected Raft backend.
-func NewBoltStore(path string) (*BoltStore, error) {
- return New(Options{Path: path})
-}
-
-// New uses the supplied options to open the BoltDB and prepare it for use as a raft backend.
-func New(options Options) (*BoltStore, error) {
- // Try to connect
- handle, err := bolt.Open(options.Path, dbFileMode, options.BoltOptions)
- if err != nil {
- return nil, err
- }
- handle.NoSync = options.NoSync
-
- // Create the new store
- store := &BoltStore{
- conn: handle,
- path: options.Path,
- }
-
- // If the store was opened read-only, don't try and create buckets
- if !options.readOnly() {
- // Set up our buckets
- if err := store.initialize(); err != nil {
- store.Close()
- return nil, err
- }
- }
- return store, nil
-}
-
-// initialize is used to set up all of the buckets.
-func (b *BoltStore) initialize() error {
- tx, err := b.conn.Begin(true)
- if err != nil {
- return err
- }
- defer tx.Rollback()
-
- // Create all the buckets
- if _, err := tx.CreateBucketIfNotExists(dbLogs); err != nil {
- return err
- }
- if _, err := tx.CreateBucketIfNotExists(dbConf); err != nil {
- return err
- }
-
- return tx.Commit()
-}
-
-// Close is used to gracefully close the DB connection.
-func (b *BoltStore) Close() error {
- return b.conn.Close()
-}
-
-// FirstIndex returns the first known index from the Raft log.
-func (b *BoltStore) FirstIndex() (uint64, error) {
- tx, err := b.conn.Begin(false)
- if err != nil {
- return 0, err
- }
- defer tx.Rollback()
-
- curs := tx.Bucket(dbLogs).Cursor()
- if first, _ := curs.First(); first == nil {
- return 0, nil
- } else {
- return bytesToUint64(first), nil
- }
-}
-
-// LastIndex returns the last known index from the Raft log.
-func (b *BoltStore) LastIndex() (uint64, error) {
- tx, err := b.conn.Begin(false)
- if err != nil {
- return 0, err
- }
- defer tx.Rollback()
-
- curs := tx.Bucket(dbLogs).Cursor()
- if last, _ := curs.Last(); last == nil {
- return 0, nil
- } else {
- return bytesToUint64(last), nil
- }
-}
-
-// GetLog is used to retrieve a log from BoltDB at a given index.
-func (b *BoltStore) GetLog(idx uint64, log *raft.Log) error {
- tx, err := b.conn.Begin(false)
- if err != nil {
- return err
- }
- defer tx.Rollback()
-
- bucket := tx.Bucket(dbLogs)
- val := bucket.Get(uint64ToBytes(idx))
-
- if val == nil {
- return raft.ErrLogNotFound
- }
- return decodeMsgPack(val, log)
-}
-
-// StoreLog is used to store a single raft log
-func (b *BoltStore) StoreLog(log *raft.Log) error {
- return b.StoreLogs([]*raft.Log{log})
-}
-
-// StoreLogs is used to store a set of raft logs
-func (b *BoltStore) StoreLogs(logs []*raft.Log) error {
- tx, err := b.conn.Begin(true)
- if err != nil {
- return err
- }
- defer tx.Rollback()
-
- for _, log := range logs {
- key := uint64ToBytes(log.Index)
- val, err := encodeMsgPack(log)
- if err != nil {
- return err
- }
- bucket := tx.Bucket(dbLogs)
- if err := bucket.Put(key, val.Bytes()); err != nil {
- return err
- }
- }
-
- return tx.Commit()
-}
-
-// DeleteRange is used to delete logs within a given range inclusively.
-func (b *BoltStore) DeleteRange(min, max uint64) error {
- minKey := uint64ToBytes(min)
-
- tx, err := b.conn.Begin(true)
- if err != nil {
- return err
- }
- defer tx.Rollback()
-
- curs := tx.Bucket(dbLogs).Cursor()
- for k, _ := curs.Seek(minKey); k != nil; k, _ = curs.Next() {
- // Handle out-of-range log index
- if bytesToUint64(k) > max {
- break
- }
-
- // Delete in-range log index
- if err := curs.Delete(); err != nil {
- return err
- }
- }
-
- return tx.Commit()
-}
-
-// Set is used to set a key/value set outside of the raft log
-func (b *BoltStore) Set(k, v []byte) error {
- tx, err := b.conn.Begin(true)
- if err != nil {
- return err
- }
- defer tx.Rollback()
-
- bucket := tx.Bucket(dbConf)
- if err := bucket.Put(k, v); err != nil {
- return err
- }
-
- return tx.Commit()
-}
-
-// Get is used to retrieve a value from the k/v store by key
-func (b *BoltStore) Get(k []byte) ([]byte, error) {
- tx, err := b.conn.Begin(false)
- if err != nil {
- return nil, err
- }
- defer tx.Rollback()
-
- bucket := tx.Bucket(dbConf)
- val := bucket.Get(k)
-
- if val == nil {
- return nil, ErrKeyNotFound
- }
- return append([]byte(nil), val...), nil
-}
-
-// SetUint64 is like Set, but handles uint64 values
-func (b *BoltStore) SetUint64(key []byte, val uint64) error {
- return b.Set(key, uint64ToBytes(val))
-}
-
-// GetUint64 is like Get, but handles uint64 values
-func (b *BoltStore) GetUint64(key []byte) (uint64, error) {
- val, err := b.Get(key)
- if err != nil {
- return 0, err
- }
- return bytesToUint64(val), nil
-}
-
-// Sync performs an fsync on the database file handle. This is not necessary
-// under normal operation unless NoSync is enabled, in which this forces the
-// database file to sync against the disk.
-func (b *BoltStore) Sync() error {
- return b.conn.Sync()
-}
diff --git a/vendor/github.com/hashicorp/raft-boltdb/util.go b/vendor/github.com/hashicorp/raft-boltdb/util.go
deleted file mode 100644
index 68dd786b7a..0000000000
--- a/vendor/github.com/hashicorp/raft-boltdb/util.go
+++ /dev/null
@@ -1,37 +0,0 @@
-package raftboltdb
-
-import (
- "bytes"
- "encoding/binary"
-
- "github.com/hashicorp/go-msgpack/codec"
-)
-
-// Decode reverses the encode operation on a byte slice input
-func decodeMsgPack(buf []byte, out interface{}) error {
- r := bytes.NewBuffer(buf)
- hd := codec.MsgpackHandle{}
- dec := codec.NewDecoder(r, &hd)
- return dec.Decode(out)
-}
-
-// Encode writes an encoded object to a new bytes buffer
-func encodeMsgPack(in interface{}) (*bytes.Buffer, error) {
- buf := bytes.NewBuffer(nil)
- hd := codec.MsgpackHandle{}
- enc := codec.NewEncoder(buf, &hd)
- err := enc.Encode(in)
- return buf, err
-}
-
-// Converts bytes to an integer
-func bytesToUint64(b []byte) uint64 {
- return binary.BigEndian.Uint64(b)
-}
-
-// Converts a uint to a byte slice
-func uint64ToBytes(u uint64) []byte {
- buf := make([]byte, 8)
- binary.BigEndian.PutUint64(buf, u)
- return buf
-}
diff --git a/vendor/github.com/hashicorp/raft/CHANGELOG.md b/vendor/github.com/hashicorp/raft/CHANGELOG.md
deleted file mode 100644
index 06f1d8a6c5..0000000000
--- a/vendor/github.com/hashicorp/raft/CHANGELOG.md
+++ /dev/null
@@ -1,16 +0,0 @@
-# 1.0.1 (April 12th, 2019)
-
-IMPROVEMENTS
-
-* InMemTransport: Add timeout for sending a message [[GH-313](https://github.com/hashicorp/raft/pull/313)]
-* ensure 'make deps' downloads test dependencies like testify [[GH-310](https://github.com/hashicorp/raft/pull/310)]
-* Clarifies function of CommitTimeout [[GH-309](https://github.com/hashicorp/raft/pull/309)]
-* Add additional metrics regarding log dispatching and committal [[GH-316](https://github.com/hashicorp/raft/pull/316)]
-
-# 1.0.0 (October 3rd, 2017)
-
-v1.0.0 takes the changes that were staged in the library-v2-stage-one branch. This version manages server identities using a UUID, so introduces some breaking API changes. It also versions the Raft protocol, and requires some special steps when interoperating with Raft servers running older versions of the library (see the detailed comment in config.go about version compatibility). You can reference https://github.com/hashicorp/consul/pull/2222 for an idea of what was required to port Consul to these new interfaces.
-
-# 0.1.0 (September 29th, 2017)
-
-v0.1.0 is the original stable version of the library that was in master and has been maintained with no breaking API changes. This was in use by Consul prior to version 0.7.0.
diff --git a/vendor/github.com/hashicorp/raft/LICENSE b/vendor/github.com/hashicorp/raft/LICENSE
deleted file mode 100644
index c33dcc7c92..0000000000
--- a/vendor/github.com/hashicorp/raft/LICENSE
+++ /dev/null
@@ -1,354 +0,0 @@
-Mozilla Public License, version 2.0
-
-1. Definitions
-
-1.1. “Contributor”
-
- means each individual or legal entity that creates, contributes to the
- creation of, or owns Covered Software.
-
-1.2. “Contributor Version”
-
- means the combination of the Contributions of others (if any) used by a
- Contributor and that particular Contributor’s Contribution.
-
-1.3. “Contribution”
-
- means Covered Software of a particular Contributor.
-
-1.4. “Covered Software”
-
- means Source Code Form to which the initial Contributor has attached the
- notice in Exhibit A, the Executable Form of such Source Code Form, and
- Modifications of such Source Code Form, in each case including portions
- thereof.
-
-1.5. “Incompatible With Secondary Licenses”
- means
-
- a. that the initial Contributor has attached the notice described in
- Exhibit B to the Covered Software; or
-
- b. that the Covered Software was made available under the terms of version
- 1.1 or earlier of the License, but not also under the terms of a
- Secondary License.
-
-1.6. “Executable Form”
-
- means any form of the work other than Source Code Form.
-
-1.7. “Larger Work”
-
- means a work that combines Covered Software with other material, in a separate
- file or files, that is not Covered Software.
-
-1.8. “License”
-
- means this document.
-
-1.9. “Licensable”
-
- means having the right to grant, to the maximum extent possible, whether at the
- time of the initial grant or subsequently, any and all of the rights conveyed by
- this License.
-
-1.10. “Modifications”
-
- means any of the following:
-
- a. any file in Source Code Form that results from an addition to, deletion
- from, or modification of the contents of Covered Software; or
-
- b. any new file in Source Code Form that contains any Covered Software.
-
-1.11. “Patent Claims” of a Contributor
-
- means any patent claim(s), including without limitation, method, process,
- and apparatus claims, in any patent Licensable by such Contributor that
- would be infringed, but for the grant of the License, by the making,
- using, selling, offering for sale, having made, import, or transfer of
- either its Contributions or its Contributor Version.
-
-1.12. “Secondary License”
-
- means either the GNU General Public License, Version 2.0, the GNU Lesser
- General Public License, Version 2.1, the GNU Affero General Public
- License, Version 3.0, or any later versions of those licenses.
-
-1.13. “Source Code Form”
-
- means the form of the work preferred for making modifications.
-
-1.14. “You” (or “Your”)
-
- means an individual or a legal entity exercising rights under this
- License. For legal entities, “You” includes any entity that controls, is
- controlled by, or is under common control with You. For purposes of this
- definition, “control” means (a) the power, direct or indirect, to cause
- the direction or management of such entity, whether by contract or
- otherwise, or (b) ownership of more than fifty percent (50%) of the
- outstanding shares or beneficial ownership of such entity.
-
-
-2. License Grants and Conditions
-
-2.1. Grants
-
- Each Contributor hereby grants You a world-wide, royalty-free,
- non-exclusive license:
-
- a. under intellectual property rights (other than patent or trademark)
- Licensable by such Contributor to use, reproduce, make available,
- modify, display, perform, distribute, and otherwise exploit its
- Contributions, either on an unmodified basis, with Modifications, or as
- part of a Larger Work; and
-
- b. under Patent Claims of such Contributor to make, use, sell, offer for
- sale, have made, import, and otherwise transfer either its Contributions
- or its Contributor Version.
-
-2.2. Effective Date
-
- The licenses granted in Section 2.1 with respect to any Contribution become
- effective for each Contribution on the date the Contributor first distributes
- such Contribution.
-
-2.3. Limitations on Grant Scope
-
- The licenses granted in this Section 2 are the only rights granted under this
- License. No additional rights or licenses will be implied from the distribution
- or licensing of Covered Software under this License. Notwithstanding Section
- 2.1(b) above, no patent license is granted by a Contributor:
-
- a. for any code that a Contributor has removed from Covered Software; or
-
- b. for infringements caused by: (i) Your and any other third party’s
- modifications of Covered Software, or (ii) the combination of its
- Contributions with other software (except as part of its Contributor
- Version); or
-
- c. under Patent Claims infringed by Covered Software in the absence of its
- Contributions.
-
- This License does not grant any rights in the trademarks, service marks, or
- logos of any Contributor (except as may be necessary to comply with the
- notice requirements in Section 3.4).
-
-2.4. Subsequent Licenses
-
- No Contributor makes additional grants as a result of Your choice to
- distribute the Covered Software under a subsequent version of this License
- (see Section 10.2) or under the terms of a Secondary License (if permitted
- under the terms of Section 3.3).
-
-2.5. Representation
-
- Each Contributor represents that the Contributor believes its Contributions
- are its original creation(s) or it has sufficient rights to grant the
- rights to its Contributions conveyed by this License.
-
-2.6. Fair Use
-
- This License is not intended to limit any rights You have under applicable
- copyright doctrines of fair use, fair dealing, or other equivalents.
-
-2.7. Conditions
-
- Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in
- Section 2.1.
-
-
-3. Responsibilities
-
-3.1. Distribution of Source Form
-
- All distribution of Covered Software in Source Code Form, including any
- Modifications that You create or to which You contribute, must be under the
- terms of this License. You must inform recipients that the Source Code Form
- of the Covered Software is governed by the terms of this License, and how
- they can obtain a copy of this License. You may not attempt to alter or
- restrict the recipients’ rights in the Source Code Form.
-
-3.2. Distribution of Executable Form
-
- If You distribute Covered Software in Executable Form then:
-
- a. such Covered Software must also be made available in Source Code Form,
- as described in Section 3.1, and You must inform recipients of the
- Executable Form how they can obtain a copy of such Source Code Form by
- reasonable means in a timely manner, at a charge no more than the cost
- of distribution to the recipient; and
-
- b. You may distribute such Executable Form under the terms of this License,
- or sublicense it under different terms, provided that the license for
- the Executable Form does not attempt to limit or alter the recipients’
- rights in the Source Code Form under this License.
-
-3.3. Distribution of a Larger Work
-
- You may create and distribute a Larger Work under terms of Your choice,
- provided that You also comply with the requirements of this License for the
- Covered Software. If the Larger Work is a combination of Covered Software
- with a work governed by one or more Secondary Licenses, and the Covered
- Software is not Incompatible With Secondary Licenses, this License permits
- You to additionally distribute such Covered Software under the terms of
- such Secondary License(s), so that the recipient of the Larger Work may, at
- their option, further distribute the Covered Software under the terms of
- either this License or such Secondary License(s).
-
-3.4. Notices
-
- You may not remove or alter the substance of any license notices (including
- copyright notices, patent notices, disclaimers of warranty, or limitations
- of liability) contained within the Source Code Form of the Covered
- Software, except that You may alter any license notices to the extent
- required to remedy known factual inaccuracies.
-
-3.5. Application of Additional Terms
-
- You may choose to offer, and to charge a fee for, warranty, support,
- indemnity or liability obligations to one or more recipients of Covered
- Software. However, You may do so only on Your own behalf, and not on behalf
- of any Contributor. You must make it absolutely clear that any such
- warranty, support, indemnity, or liability obligation is offered by You
- alone, and You hereby agree to indemnify every Contributor for any
- liability incurred by such Contributor as a result of warranty, support,
- indemnity or liability terms You offer. You may include additional
- disclaimers of warranty and limitations of liability specific to any
- jurisdiction.
-
-4. Inability to Comply Due to Statute or Regulation
-
- If it is impossible for You to comply with any of the terms of this License
- with respect to some or all of the Covered Software due to statute, judicial
- order, or regulation then You must: (a) comply with the terms of this License
- to the maximum extent possible; and (b) describe the limitations and the code
- they affect. Such description must be placed in a text file included with all
- distributions of the Covered Software under this License. Except to the
- extent prohibited by statute or regulation, such description must be
- sufficiently detailed for a recipient of ordinary skill to be able to
- understand it.
-
-5. Termination
-
-5.1. The rights granted under this License will terminate automatically if You
- fail to comply with any of its terms. However, if You become compliant,
- then the rights granted under this License from a particular Contributor
- are reinstated (a) provisionally, unless and until such Contributor
- explicitly and finally terminates Your grants, and (b) on an ongoing basis,
- if such Contributor fails to notify You of the non-compliance by some
- reasonable means prior to 60 days after You have come back into compliance.
- Moreover, Your grants from a particular Contributor are reinstated on an
- ongoing basis if such Contributor notifies You of the non-compliance by
- some reasonable means, this is the first time You have received notice of
- non-compliance with this License from such Contributor, and You become
- compliant prior to 30 days after Your receipt of the notice.
-
-5.2. If You initiate litigation against any entity by asserting a patent
- infringement claim (excluding declaratory judgment actions, counter-claims,
- and cross-claims) alleging that a Contributor Version directly or
- indirectly infringes any patent, then the rights granted to You by any and
- all Contributors for the Covered Software under Section 2.1 of this License
- shall terminate.
-
-5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user
- license agreements (excluding distributors and resellers) which have been
- validly granted by You or Your distributors under this License prior to
- termination shall survive termination.
-
-6. Disclaimer of Warranty
-
- Covered Software is provided under this License on an “as is” basis, without
- warranty of any kind, either expressed, implied, or statutory, including,
- without limitation, warranties that the Covered Software is free of defects,
- merchantable, fit for a particular purpose or non-infringing. The entire
- risk as to the quality and performance of the Covered Software is with You.
- Should any Covered Software prove defective in any respect, You (not any
- Contributor) assume the cost of any necessary servicing, repair, or
- correction. This disclaimer of warranty constitutes an essential part of this
- License. No use of any Covered Software is authorized under this License
- except under this disclaimer.
-
-7. Limitation of Liability
-
- Under no circumstances and under no legal theory, whether tort (including
- negligence), contract, or otherwise, shall any Contributor, or anyone who
- distributes Covered Software as permitted above, be liable to You for any
- direct, indirect, special, incidental, or consequential damages of any
- character including, without limitation, damages for lost profits, loss of
- goodwill, work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses, even if such party shall have been
- informed of the possibility of such damages. This limitation of liability
- shall not apply to liability for death or personal injury resulting from such
- party’s negligence to the extent applicable law prohibits such limitation.
- Some jurisdictions do not allow the exclusion or limitation of incidental or
- consequential damages, so this exclusion and limitation may not apply to You.
-
-8. Litigation
-
- Any litigation relating to this License may be brought only in the courts of
- a jurisdiction where the defendant maintains its principal place of business
- and such litigation shall be governed by laws of that jurisdiction, without
- reference to its conflict-of-law provisions. Nothing in this Section shall
- prevent a party’s ability to bring cross-claims or counter-claims.
-
-9. Miscellaneous
-
- This License represents the complete agreement concerning the subject matter
- hereof. If any provision of this License is held to be unenforceable, such
- provision shall be reformed only to the extent necessary to make it
- enforceable. Any law or regulation which provides that the language of a
- contract shall be construed against the drafter shall not be used to construe
- this License against a Contributor.
-
-
-10. Versions of the License
-
-10.1. New Versions
-
- Mozilla Foundation is the license steward. Except as provided in Section
- 10.3, no one other than the license steward has the right to modify or
- publish new versions of this License. Each version will be given a
- distinguishing version number.
-
-10.2. Effect of New Versions
-
- You may distribute the Covered Software under the terms of the version of
- the License under which You originally received the Covered Software, or
- under the terms of any subsequent version published by the license
- steward.
-
-10.3. Modified Versions
-
- If you create software not governed by this License, and you want to
- create a new license for such software, you may create and use a modified
- version of this License if you rename the license and remove any
- references to the name of the license steward (except to note that such
- modified license differs from this License).
-
-10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses
- If You choose to distribute Source Code Form that is Incompatible With
- Secondary Licenses under the terms of this version of the License, the
- notice described in Exhibit B of this License must be attached.
-
-Exhibit A - Source Code Form License Notice
-
- This Source Code Form is subject to the
- terms of the Mozilla Public License, v.
- 2.0. If a copy of the MPL was not
- distributed with this file, You can
- obtain one at
- http://mozilla.org/MPL/2.0/.
-
-If it is not possible or desirable to put the notice in a particular file, then
-You may include the notice in a location (such as a LICENSE file in a relevant
-directory) where a recipient would be likely to look for such a notice.
-
-You may add additional accurate notices of copyright ownership.
-
-Exhibit B - “Incompatible With Secondary Licenses” Notice
-
- This Source Code Form is “Incompatible
- With Secondary Licenses”, as defined by
- the Mozilla Public License, v. 2.0.
-
diff --git a/vendor/github.com/hashicorp/raft/Makefile b/vendor/github.com/hashicorp/raft/Makefile
deleted file mode 100644
index 46849d88c0..0000000000
--- a/vendor/github.com/hashicorp/raft/Makefile
+++ /dev/null
@@ -1,20 +0,0 @@
-DEPS = $(go list -f '{{range .TestImports}}{{.}} {{end}}' ./...)
-
-test:
- go test -timeout=60s .
-
-integ: test
- INTEG_TESTS=yes go test -timeout=25s -run=Integ .
-
-fuzz:
- go test -timeout=300s ./fuzzy
-
-deps:
- go get -t -d -v ./...
- echo $(DEPS) | xargs -n1 go get -d
-
-cov:
- INTEG_TESTS=yes gocov test github.com/hashicorp/raft | gocov-html > /tmp/coverage.html
- open /tmp/coverage.html
-
-.PHONY: test cov integ deps
diff --git a/vendor/github.com/hashicorp/raft/README.md b/vendor/github.com/hashicorp/raft/README.md
deleted file mode 100644
index 43208ebba8..0000000000
--- a/vendor/github.com/hashicorp/raft/README.md
+++ /dev/null
@@ -1,107 +0,0 @@
-raft [](https://travis-ci.org/hashicorp/raft)
-====
-
-raft is a [Go](http://www.golang.org) library that manages a replicated
-log and can be used with an FSM to manage replicated state machines. It
-is a library for providing [consensus](http://en.wikipedia.org/wiki/Consensus_(computer_science)).
-
-The use cases for such a library are far-reaching as replicated state
-machines are a key component of many distributed systems. They enable
-building Consistent, Partition Tolerant (CP) systems, with limited
-fault tolerance as well.
-
-## Building
-
-If you wish to build raft you'll need Go version 1.2+ installed.
-
-Please check your installation with:
-
-```
-go version
-```
-
-## Documentation
-
-For complete documentation, see the associated [Godoc](http://godoc.org/github.com/hashicorp/raft).
-
-To prevent complications with cgo, the primary backend `MDBStore` is in a separate repository,
-called [raft-mdb](http://github.com/hashicorp/raft-mdb). That is the recommended implementation
-for the `LogStore` and `StableStore`.
-
-A pure Go backend using [BoltDB](https://github.com/boltdb/bolt) is also available called
-[raft-boltdb](https://github.com/hashicorp/raft-boltdb). It can also be used as a `LogStore`
-and `StableStore`.
-
-## Tagged Releases
-
-As of September 2017, HashiCorp will start using tags for this library to clearly indicate
-major version updates. We recommend you vendor your application's dependency on this library.
-
-* v0.1.0 is the original stable version of the library that was in master and has been maintained
-with no breaking API changes. This was in use by Consul prior to version 0.7.0.
-
-* v1.0.0 takes the changes that were staged in the library-v2-stage-one branch. This version
-manages server identities using a UUID, so introduces some breaking API changes. It also versions
-the Raft protocol, and requires some special steps when interoperating with Raft servers running
-older versions of the library (see the detailed comment in config.go about version compatibility).
-You can reference https://github.com/hashicorp/consul/pull/2222 for an idea of what was required
-to port Consul to these new interfaces.
-
- This version includes some new features as well, including non voting servers, a new address
- provider abstraction in the transport layer, and more resilient snapshots.
-
-## Protocol
-
-raft is based on ["Raft: In Search of an Understandable Consensus Algorithm"](https://ramcloud.stanford.edu/wiki/download/attachments/11370504/raft.pdf)
-
-A high level overview of the Raft protocol is described below, but for details please read the full
-[Raft paper](https://ramcloud.stanford.edu/wiki/download/attachments/11370504/raft.pdf)
-followed by the raft source. Any questions about the raft protocol should be sent to the
-[raft-dev mailing list](https://groups.google.com/forum/#!forum/raft-dev).
-
-### Protocol Description
-
-Raft nodes are always in one of three states: follower, candidate or leader. All
-nodes initially start out as a follower. In this state, nodes can accept log entries
-from a leader and cast votes. If no entries are received for some time, nodes
-self-promote to the candidate state. In the candidate state nodes request votes from
-their peers. If a candidate receives a quorum of votes, then it is promoted to a leader.
-The leader must accept new log entries and replicate to all the other followers.
-In addition, if stale reads are not acceptable, all queries must also be performed on
-the leader.
-
-Once a cluster has a leader, it is able to accept new log entries. A client can
-request that a leader append a new log entry, which is an opaque binary blob to
-Raft. The leader then writes the entry to durable storage and attempts to replicate
-to a quorum of followers. Once the log entry is considered *committed*, it can be
-*applied* to a finite state machine. The finite state machine is application specific,
-and is implemented using an interface.
-
-An obvious question relates to the unbounded nature of a replicated log. Raft provides
-a mechanism by which the current state is snapshotted, and the log is compacted. Because
-of the FSM abstraction, restoring the state of the FSM must result in the same state
-as a replay of old logs. This allows Raft to capture the FSM state at a point in time,
-and then remove all the logs that were used to reach that state. This is performed automatically
-without user intervention, and prevents unbounded disk usage as well as minimizing
-time spent replaying logs.
-
-Lastly, there is the issue of updating the peer set when new servers are joining
-or existing servers are leaving. As long as a quorum of nodes is available, this
-is not an issue as Raft provides mechanisms to dynamically update the peer set.
-If a quorum of nodes is unavailable, then this becomes a very challenging issue.
-For example, suppose there are only 2 peers, A and B. The quorum size is also
-2, meaning both nodes must agree to commit a log entry. If either A or B fails,
-it is now impossible to reach quorum. This means the cluster is unable to add,
-or remove a node, or commit any additional log entries. This results in *unavailability*.
-At this point, manual intervention would be required to remove either A or B,
-and to restart the remaining node in bootstrap mode.
-
-A Raft cluster of 3 nodes can tolerate a single node failure, while a cluster
-of 5 can tolerate 2 node failures. The recommended configuration is to either
-run 3 or 5 raft servers. This maximizes availability without
-greatly sacrificing performance.
-
-In terms of performance, Raft is comparable to Paxos. Assuming stable leadership,
-committing a log entry requires a single round trip to half of the cluster.
-Thus performance is bound by disk I/O and network latency.
-
diff --git a/vendor/github.com/hashicorp/raft/api.go b/vendor/github.com/hashicorp/raft/api.go
deleted file mode 100644
index c6f947f241..0000000000
--- a/vendor/github.com/hashicorp/raft/api.go
+++ /dev/null
@@ -1,1013 +0,0 @@
-package raft
-
-import (
- "errors"
- "fmt"
- "io"
- "os"
- "strconv"
- "sync"
- "time"
-
- "github.com/hashicorp/go-hclog"
-
- "github.com/armon/go-metrics"
-)
-
-var (
- // ErrLeader is returned when an operation can't be completed on a
- // leader node.
- ErrLeader = errors.New("node is the leader")
-
- // ErrNotLeader is returned when an operation can't be completed on a
- // follower or candidate node.
- ErrNotLeader = errors.New("node is not the leader")
-
- // ErrLeadershipLost is returned when a leader fails to commit a log entry
- // because it's been deposed in the process.
- ErrLeadershipLost = errors.New("leadership lost while committing log")
-
- // ErrAbortedByRestore is returned when a leader fails to commit a log
- // entry because it's been superseded by a user snapshot restore.
- ErrAbortedByRestore = errors.New("snapshot restored while committing log")
-
- // ErrRaftShutdown is returned when operations are requested against an
- // inactive Raft.
- ErrRaftShutdown = errors.New("raft is already shutdown")
-
- // ErrEnqueueTimeout is returned when a command fails due to a timeout.
- ErrEnqueueTimeout = errors.New("timed out enqueuing operation")
-
- // ErrNothingNewToSnapshot is returned when trying to create a snapshot
- // but there's nothing new commited to the FSM since we started.
- ErrNothingNewToSnapshot = errors.New("nothing new to snapshot")
-
- // ErrUnsupportedProtocol is returned when an operation is attempted
- // that's not supported by the current protocol version.
- ErrUnsupportedProtocol = errors.New("operation not supported with current protocol version")
-
- // ErrCantBootstrap is returned when attempt is made to bootstrap a
- // cluster that already has state present.
- ErrCantBootstrap = errors.New("bootstrap only works on new clusters")
-)
-
-// Raft implements a Raft node.
-type Raft struct {
- raftState
-
- // protocolVersion is used to inter-operate with Raft servers running
- // different versions of the library. See comments in config.go for more
- // details.
- protocolVersion ProtocolVersion
-
- // applyCh is used to async send logs to the main thread to
- // be committed and applied to the FSM.
- applyCh chan *logFuture
-
- // Configuration provided at Raft initialization
- conf Config
-
- // FSM is the client state machine to apply commands to
- fsm FSM
-
- // fsmMutateCh is used to send state-changing updates to the FSM. This
- // receives pointers to commitTuple structures when applying logs or
- // pointers to restoreFuture structures when restoring a snapshot. We
- // need control over the order of these operations when doing user
- // restores so that we finish applying any old log applies before we
- // take a user snapshot on the leader, otherwise we might restore the
- // snapshot and apply old logs to it that were in the pipe.
- fsmMutateCh chan interface{}
-
- // fsmSnapshotCh is used to trigger a new snapshot being taken
- fsmSnapshotCh chan *reqSnapshotFuture
-
- // lastContact is the last time we had contact from the
- // leader node. This can be used to gauge staleness.
- lastContact time.Time
- lastContactLock sync.RWMutex
-
- // Leader is the current cluster leader
- leader ServerAddress
- leaderLock sync.RWMutex
-
- // leaderCh is used to notify of leadership changes
- leaderCh chan bool
-
- // leaderState used only while state is leader
- leaderState leaderState
-
- // Stores our local server ID, used to avoid sending RPCs to ourself
- localID ServerID
-
- // Stores our local addr
- localAddr ServerAddress
-
- // Used for our logging
- logger hclog.Logger
-
- // LogStore provides durable storage for logs
- logs LogStore
-
- // Used to request the leader to make configuration changes.
- configurationChangeCh chan *configurationChangeFuture
-
- // Tracks the latest configuration and latest committed configuration from
- // the log/snapshot.
- configurations configurations
-
- // RPC chan comes from the transport layer
- rpcCh <-chan RPC
-
- // Shutdown channel to exit, protected to prevent concurrent exits
- shutdown bool
- shutdownCh chan struct{}
- shutdownLock sync.Mutex
-
- // snapshots is used to store and retrieve snapshots
- snapshots SnapshotStore
-
- // userSnapshotCh is used for user-triggered snapshots
- userSnapshotCh chan *userSnapshotFuture
-
- // userRestoreCh is used for user-triggered restores of external
- // snapshots
- userRestoreCh chan *userRestoreFuture
-
- // stable is a StableStore implementation for durable state
- // It provides stable storage for many fields in raftState
- stable StableStore
-
- // The transport layer we use
- trans Transport
-
- // verifyCh is used to async send verify futures to the main thread
- // to verify we are still the leader
- verifyCh chan *verifyFuture
-
- // configurationsCh is used to get the configuration data safely from
- // outside of the main thread.
- configurationsCh chan *configurationsFuture
-
- // bootstrapCh is used to attempt an initial bootstrap from outside of
- // the main thread.
- bootstrapCh chan *bootstrapFuture
-
- // List of observers and the mutex that protects them. The observers list
- // is indexed by an artificial ID which is used for deregistration.
- observersLock sync.RWMutex
- observers map[uint64]*Observer
-}
-
-// BootstrapCluster initializes a server's storage with the given cluster
-// configuration. This should only be called at the beginning of time for the
-// cluster, and you absolutely must make sure that you call it with the same
-// configuration on all the Voter servers. There is no need to bootstrap
-// Nonvoter and Staging servers.
-//
-// One sane approach is to bootstrap a single server with a configuration
-// listing just itself as a Voter, then invoke AddVoter() on it to add other
-// servers to the cluster.
-func BootstrapCluster(conf *Config, logs LogStore, stable StableStore,
- snaps SnapshotStore, trans Transport, configuration Configuration) error {
- // Validate the Raft server config.
- if err := ValidateConfig(conf); err != nil {
- return err
- }
-
- // Sanity check the Raft peer configuration.
- if err := checkConfiguration(configuration); err != nil {
- return err
- }
-
- // Make sure the cluster is in a clean state.
- hasState, err := HasExistingState(logs, stable, snaps)
- if err != nil {
- return fmt.Errorf("failed to check for existing state: %v", err)
- }
- if hasState {
- return ErrCantBootstrap
- }
-
- // Set current term to 1.
- if err := stable.SetUint64(keyCurrentTerm, 1); err != nil {
- return fmt.Errorf("failed to save current term: %v", err)
- }
-
- // Append configuration entry to log.
- entry := &Log{
- Index: 1,
- Term: 1,
- }
- if conf.ProtocolVersion < 3 {
- entry.Type = LogRemovePeerDeprecated
- entry.Data = encodePeers(configuration, trans)
- } else {
- entry.Type = LogConfiguration
- entry.Data = encodeConfiguration(configuration)
- }
- if err := logs.StoreLog(entry); err != nil {
- return fmt.Errorf("failed to append configuration entry to log: %v", err)
- }
-
- return nil
-}
-
-// RecoverCluster is used to manually force a new configuration in order to
-// recover from a loss of quorum where the current configuration cannot be
-// restored, such as when several servers die at the same time. This works by
-// reading all the current state for this server, creating a snapshot with the
-// supplied configuration, and then truncating the Raft log. This is the only
-// safe way to force a given configuration without actually altering the log to
-// insert any new entries, which could cause conflicts with other servers with
-// different state.
-//
-// WARNING! This operation implicitly commits all entries in the Raft log, so
-// in general this is an extremely unsafe operation. If you've lost your other
-// servers and are performing a manual recovery, then you've also lost the
-// commit information, so this is likely the best you can do, but you should be
-// aware that calling this can cause Raft log entries that were in the process
-// of being replicated but not yet be committed to be committed.
-//
-// Note the FSM passed here is used for the snapshot operations and will be
-// left in a state that should not be used by the application. Be sure to
-// discard this FSM and any associated state and provide a fresh one when
-// calling NewRaft later.
-//
-// A typical way to recover the cluster is to shut down all servers and then
-// run RecoverCluster on every server using an identical configuration. When
-// the cluster is then restarted, and election should occur and then Raft will
-// resume normal operation. If it's desired to make a particular server the
-// leader, this can be used to inject a new configuration with that server as
-// the sole voter, and then join up other new clean-state peer servers using
-// the usual APIs in order to bring the cluster back into a known state.
-func RecoverCluster(conf *Config, fsm FSM, logs LogStore, stable StableStore,
- snaps SnapshotStore, trans Transport, configuration Configuration) error {
- // Validate the Raft server config.
- if err := ValidateConfig(conf); err != nil {
- return err
- }
-
- // Sanity check the Raft peer configuration.
- if err := checkConfiguration(configuration); err != nil {
- return err
- }
-
- // Refuse to recover if there's no existing state. This would be safe to
- // do, but it is likely an indication of an operator error where they
- // expect data to be there and it's not. By refusing, we force them
- // to show intent to start a cluster fresh by explicitly doing a
- // bootstrap, rather than quietly fire up a fresh cluster here.
- hasState, err := HasExistingState(logs, stable, snaps)
- if err != nil {
- return fmt.Errorf("failed to check for existing state: %v", err)
- }
- if !hasState {
- return fmt.Errorf("refused to recover cluster with no initial state, this is probably an operator error")
- }
-
- // Attempt to restore any snapshots we find, newest to oldest.
- var snapshotIndex uint64
- var snapshotTerm uint64
- snapshots, err := snaps.List()
- if err != nil {
- return fmt.Errorf("failed to list snapshots: %v", err)
- }
- for _, snapshot := range snapshots {
- _, source, err := snaps.Open(snapshot.ID)
- if err != nil {
- // Skip this one and try the next. We will detect if we
- // couldn't open any snapshots.
- continue
- }
- defer source.Close()
-
- if err := fsm.Restore(source); err != nil {
- // Same here, skip and try the next one.
- continue
- }
-
- snapshotIndex = snapshot.Index
- snapshotTerm = snapshot.Term
- break
- }
- if len(snapshots) > 0 && (snapshotIndex == 0 || snapshotTerm == 0) {
- return fmt.Errorf("failed to restore any of the available snapshots")
- }
-
- // The snapshot information is the best known end point for the data
- // until we play back the Raft log entries.
- lastIndex := snapshotIndex
- lastTerm := snapshotTerm
-
- // Apply any Raft log entries past the snapshot.
- lastLogIndex, err := logs.LastIndex()
- if err != nil {
- return fmt.Errorf("failed to find last log: %v", err)
- }
- for index := snapshotIndex + 1; index <= lastLogIndex; index++ {
- var entry Log
- if err := logs.GetLog(index, &entry); err != nil {
- return fmt.Errorf("failed to get log at index %d: %v", index, err)
- }
- if entry.Type == LogCommand {
- _ = fsm.Apply(&entry)
- }
- lastIndex = entry.Index
- lastTerm = entry.Term
- }
-
- // Create a new snapshot, placing the configuration in as if it was
- // committed at index 1.
- snapshot, err := fsm.Snapshot()
- if err != nil {
- return fmt.Errorf("failed to snapshot FSM: %v", err)
- }
- version := getSnapshotVersion(conf.ProtocolVersion)
- sink, err := snaps.Create(version, lastIndex, lastTerm, configuration, 1, trans)
- if err != nil {
- return fmt.Errorf("failed to create snapshot: %v", err)
- }
- if err := snapshot.Persist(sink); err != nil {
- return fmt.Errorf("failed to persist snapshot: %v", err)
- }
- if err := sink.Close(); err != nil {
- return fmt.Errorf("failed to finalize snapshot: %v", err)
- }
-
- // Compact the log so that we don't get bad interference from any
- // configuration change log entries that might be there.
- firstLogIndex, err := logs.FirstIndex()
- if err != nil {
- return fmt.Errorf("failed to get first log index: %v", err)
- }
- if err := logs.DeleteRange(firstLogIndex, lastLogIndex); err != nil {
- return fmt.Errorf("log compaction failed: %v", err)
- }
-
- return nil
-}
-
-// HasExistingState returns true if the server has any existing state (logs,
-// knowledge of a current term, or any snapshots).
-func HasExistingState(logs LogStore, stable StableStore, snaps SnapshotStore) (bool, error) {
- // Make sure we don't have a current term.
- currentTerm, err := stable.GetUint64(keyCurrentTerm)
- if err == nil {
- if currentTerm > 0 {
- return true, nil
- }
- } else {
- if err.Error() != "not found" {
- return false, fmt.Errorf("failed to read current term: %v", err)
- }
- }
-
- // Make sure we have an empty log.
- lastIndex, err := logs.LastIndex()
- if err != nil {
- return false, fmt.Errorf("failed to get last log index: %v", err)
- }
- if lastIndex > 0 {
- return true, nil
- }
-
- // Make sure we have no snapshots
- snapshots, err := snaps.List()
- if err != nil {
- return false, fmt.Errorf("failed to list snapshots: %v", err)
- }
- if len(snapshots) > 0 {
- return true, nil
- }
-
- return false, nil
-}
-
-// NewRaft is used to construct a new Raft node. It takes a configuration, as well
-// as implementations of various interfaces that are required. If we have any
-// old state, such as snapshots, logs, peers, etc, all those will be restored
-// when creating the Raft node.
-func NewRaft(conf *Config, fsm FSM, logs LogStore, stable StableStore, snaps SnapshotStore, trans Transport) (*Raft, error) {
- // Validate the configuration.
- if err := ValidateConfig(conf); err != nil {
- return nil, err
- }
-
- // Ensure we have a LogOutput.
- var logger hclog.Logger
- if conf.Logger != nil {
- logger = conf.Logger
- } else {
- if conf.LogOutput == nil {
- conf.LogOutput = os.Stderr
- }
-
- logger = hclog.New(&hclog.LoggerOptions{
- Name: "raft",
- Level: hclog.LevelFromString(conf.LogLevel),
- Output: conf.LogOutput,
- })
- }
-
- // Try to restore the current term.
- currentTerm, err := stable.GetUint64(keyCurrentTerm)
- if err != nil && err.Error() != "not found" {
- return nil, fmt.Errorf("failed to load current term: %v", err)
- }
-
- // Read the index of the last log entry.
- lastIndex, err := logs.LastIndex()
- if err != nil {
- return nil, fmt.Errorf("failed to find last log: %v", err)
- }
-
- // Get the last log entry.
- var lastLog Log
- if lastIndex > 0 {
- if err = logs.GetLog(lastIndex, &lastLog); err != nil {
- return nil, fmt.Errorf("failed to get last log at index %d: %v", lastIndex, err)
- }
- }
-
- // Make sure we have a valid server address and ID.
- protocolVersion := conf.ProtocolVersion
- localAddr := ServerAddress(trans.LocalAddr())
- localID := conf.LocalID
-
- // TODO (slackpad) - When we deprecate protocol version 2, remove this
- // along with the AddPeer() and RemovePeer() APIs.
- if protocolVersion < 3 && string(localID) != string(localAddr) {
- return nil, fmt.Errorf("when running with ProtocolVersion < 3, LocalID must be set to the network address")
- }
-
- // Create Raft struct.
- r := &Raft{
- protocolVersion: protocolVersion,
- applyCh: make(chan *logFuture),
- conf: *conf,
- fsm: fsm,
- fsmMutateCh: make(chan interface{}, 128),
- fsmSnapshotCh: make(chan *reqSnapshotFuture),
- leaderCh: make(chan bool),
- localID: localID,
- localAddr: localAddr,
- logger: logger,
- logs: logs,
- configurationChangeCh: make(chan *configurationChangeFuture),
- configurations: configurations{},
- rpcCh: trans.Consumer(),
- snapshots: snaps,
- userSnapshotCh: make(chan *userSnapshotFuture),
- userRestoreCh: make(chan *userRestoreFuture),
- shutdownCh: make(chan struct{}),
- stable: stable,
- trans: trans,
- verifyCh: make(chan *verifyFuture, 64),
- configurationsCh: make(chan *configurationsFuture, 8),
- bootstrapCh: make(chan *bootstrapFuture),
- observers: make(map[uint64]*Observer),
- }
-
- // Initialize as a follower.
- r.setState(Follower)
-
- // Start as leader if specified. This should only be used
- // for testing purposes.
- if conf.StartAsLeader {
- r.setState(Leader)
- r.setLeader(r.localAddr)
- }
-
- // Restore the current term and the last log.
- r.setCurrentTerm(currentTerm)
- r.setLastLog(lastLog.Index, lastLog.Term)
-
- // Attempt to restore a snapshot if there are any.
- if err := r.restoreSnapshot(); err != nil {
- return nil, err
- }
-
- // Scan through the log for any configuration change entries.
- snapshotIndex, _ := r.getLastSnapshot()
- for index := snapshotIndex + 1; index <= lastLog.Index; index++ {
- var entry Log
- if err := r.logs.GetLog(index, &entry); err != nil {
- r.logger.Error(fmt.Sprintf("Failed to get log at %d: %v", index, err))
- panic(err)
- }
- r.processConfigurationLogEntry(&entry)
- }
- r.logger.Info(fmt.Sprintf("Initial configuration (index=%d): %+v",
- r.configurations.latestIndex, r.configurations.latest.Servers))
-
- // Setup a heartbeat fast-path to avoid head-of-line
- // blocking where possible. It MUST be safe for this
- // to be called concurrently with a blocking RPC.
- trans.SetHeartbeatHandler(r.processHeartbeat)
-
- // Start the background work.
- r.goFunc(r.run)
- r.goFunc(r.runFSM)
- r.goFunc(r.runSnapshots)
- return r, nil
-}
-
-// restoreSnapshot attempts to restore the latest snapshots, and fails if none
-// of them can be restored. This is called at initialization time, and is
-// completely unsafe to call at any other time.
-func (r *Raft) restoreSnapshot() error {
- snapshots, err := r.snapshots.List()
- if err != nil {
- r.logger.Error(fmt.Sprintf("Failed to list snapshots: %v", err))
- return err
- }
-
- // Try to load in order of newest to oldest
- for _, snapshot := range snapshots {
- _, source, err := r.snapshots.Open(snapshot.ID)
- if err != nil {
- r.logger.Error(fmt.Sprintf("Failed to open snapshot %v: %v", snapshot.ID, err))
- continue
- }
- defer source.Close()
-
- if err := r.fsm.Restore(source); err != nil {
- r.logger.Error(fmt.Sprintf("Failed to restore snapshot %v: %v", snapshot.ID, err))
- continue
- }
-
- // Log success
- r.logger.Info(fmt.Sprintf("Restored from snapshot %v", snapshot.ID))
-
- // Update the lastApplied so we don't replay old logs
- r.setLastApplied(snapshot.Index)
-
- // Update the last stable snapshot info
- r.setLastSnapshot(snapshot.Index, snapshot.Term)
-
- // Update the configuration
- if snapshot.Version > 0 {
- r.configurations.committed = snapshot.Configuration
- r.configurations.committedIndex = snapshot.ConfigurationIndex
- r.configurations.latest = snapshot.Configuration
- r.configurations.latestIndex = snapshot.ConfigurationIndex
- } else {
- configuration := decodePeers(snapshot.Peers, r.trans)
- r.configurations.committed = configuration
- r.configurations.committedIndex = snapshot.Index
- r.configurations.latest = configuration
- r.configurations.latestIndex = snapshot.Index
- }
-
- // Success!
- return nil
- }
-
- // If we had snapshots and failed to load them, its an error
- if len(snapshots) > 0 {
- return fmt.Errorf("failed to load any existing snapshots")
- }
- return nil
-}
-
-// BootstrapCluster is equivalent to non-member BootstrapCluster but can be
-// called on an un-bootstrapped Raft instance after it has been created. This
-// should only be called at the beginning of time for the cluster, and you
-// absolutely must make sure that you call it with the same configuration on all
-// the Voter servers. There is no need to bootstrap Nonvoter and Staging
-// servers.
-func (r *Raft) BootstrapCluster(configuration Configuration) Future {
- bootstrapReq := &bootstrapFuture{}
- bootstrapReq.init()
- bootstrapReq.configuration = configuration
- select {
- case <-r.shutdownCh:
- return errorFuture{ErrRaftShutdown}
- case r.bootstrapCh <- bootstrapReq:
- return bootstrapReq
- }
-}
-
-// Leader is used to return the current leader of the cluster.
-// It may return empty string if there is no current leader
-// or the leader is unknown.
-func (r *Raft) Leader() ServerAddress {
- r.leaderLock.RLock()
- leader := r.leader
- r.leaderLock.RUnlock()
- return leader
-}
-
-// Apply is used to apply a command to the FSM in a highly consistent
-// manner. This returns a future that can be used to wait on the application.
-// An optional timeout can be provided to limit the amount of time we wait
-// for the command to be started. This must be run on the leader or it
-// will fail.
-func (r *Raft) Apply(cmd []byte, timeout time.Duration) ApplyFuture {
- metrics.IncrCounter([]string{"raft", "apply"}, 1)
- var timer <-chan time.Time
- if timeout > 0 {
- timer = time.After(timeout)
- }
-
- // Create a log future, no index or term yet
- logFuture := &logFuture{
- log: Log{
- Type: LogCommand,
- Data: cmd,
- },
- }
- logFuture.init()
-
- select {
- case <-timer:
- return errorFuture{ErrEnqueueTimeout}
- case <-r.shutdownCh:
- return errorFuture{ErrRaftShutdown}
- case r.applyCh <- logFuture:
- return logFuture
- }
-}
-
-// Barrier is used to issue a command that blocks until all preceeding
-// operations have been applied to the FSM. It can be used to ensure the
-// FSM reflects all queued writes. An optional timeout can be provided to
-// limit the amount of time we wait for the command to be started. This
-// must be run on the leader or it will fail.
-func (r *Raft) Barrier(timeout time.Duration) Future {
- metrics.IncrCounter([]string{"raft", "barrier"}, 1)
- var timer <-chan time.Time
- if timeout > 0 {
- timer = time.After(timeout)
- }
-
- // Create a log future, no index or term yet
- logFuture := &logFuture{
- log: Log{
- Type: LogBarrier,
- },
- }
- logFuture.init()
-
- select {
- case <-timer:
- return errorFuture{ErrEnqueueTimeout}
- case <-r.shutdownCh:
- return errorFuture{ErrRaftShutdown}
- case r.applyCh <- logFuture:
- return logFuture
- }
-}
-
-// VerifyLeader is used to ensure the current node is still
-// the leader. This can be done to prevent stale reads when a
-// new leader has potentially been elected.
-func (r *Raft) VerifyLeader() Future {
- metrics.IncrCounter([]string{"raft", "verify_leader"}, 1)
- verifyFuture := &verifyFuture{}
- verifyFuture.init()
- select {
- case <-r.shutdownCh:
- return errorFuture{ErrRaftShutdown}
- case r.verifyCh <- verifyFuture:
- return verifyFuture
- }
-}
-
-// GetConfiguration returns the latest configuration and its associated index
-// currently in use. This may not yet be committed. This must not be called on
-// the main thread (which can access the information directly).
-func (r *Raft) GetConfiguration() ConfigurationFuture {
- configReq := &configurationsFuture{}
- configReq.init()
- select {
- case <-r.shutdownCh:
- configReq.respond(ErrRaftShutdown)
- return configReq
- case r.configurationsCh <- configReq:
- return configReq
- }
-}
-
-// AddPeer (deprecated) is used to add a new peer into the cluster. This must be
-// run on the leader or it will fail. Use AddVoter/AddNonvoter instead.
-func (r *Raft) AddPeer(peer ServerAddress) Future {
- if r.protocolVersion > 2 {
- return errorFuture{ErrUnsupportedProtocol}
- }
-
- return r.requestConfigChange(configurationChangeRequest{
- command: AddStaging,
- serverID: ServerID(peer),
- serverAddress: peer,
- prevIndex: 0,
- }, 0)
-}
-
-// RemovePeer (deprecated) is used to remove a peer from the cluster. If the
-// current leader is being removed, it will cause a new election
-// to occur. This must be run on the leader or it will fail.
-// Use RemoveServer instead.
-func (r *Raft) RemovePeer(peer ServerAddress) Future {
- if r.protocolVersion > 2 {
- return errorFuture{ErrUnsupportedProtocol}
- }
-
- return r.requestConfigChange(configurationChangeRequest{
- command: RemoveServer,
- serverID: ServerID(peer),
- prevIndex: 0,
- }, 0)
-}
-
-// AddVoter will add the given server to the cluster as a staging server. If the
-// server is already in the cluster as a voter, this updates the server's address.
-// This must be run on the leader or it will fail. The leader will promote the
-// staging server to a voter once that server is ready. If nonzero, prevIndex is
-// the index of the only configuration upon which this change may be applied; if
-// another configuration entry has been added in the meantime, this request will
-// fail. If nonzero, timeout is how long this server should wait before the
-// configuration change log entry is appended.
-func (r *Raft) AddVoter(id ServerID, address ServerAddress, prevIndex uint64, timeout time.Duration) IndexFuture {
- if r.protocolVersion < 2 {
- return errorFuture{ErrUnsupportedProtocol}
- }
-
- return r.requestConfigChange(configurationChangeRequest{
- command: AddStaging,
- serverID: id,
- serverAddress: address,
- prevIndex: prevIndex,
- }, timeout)
-}
-
-// AddNonvoter will add the given server to the cluster but won't assign it a
-// vote. The server will receive log entries, but it won't participate in
-// elections or log entry commitment. If the server is already in the cluster,
-// this updates the server's address. This must be run on the leader or it will
-// fail. For prevIndex and timeout, see AddVoter.
-func (r *Raft) AddNonvoter(id ServerID, address ServerAddress, prevIndex uint64, timeout time.Duration) IndexFuture {
- if r.protocolVersion < 3 {
- return errorFuture{ErrUnsupportedProtocol}
- }
-
- return r.requestConfigChange(configurationChangeRequest{
- command: AddNonvoter,
- serverID: id,
- serverAddress: address,
- prevIndex: prevIndex,
- }, timeout)
-}
-
-// RemoveServer will remove the given server from the cluster. If the current
-// leader is being removed, it will cause a new election to occur. This must be
-// run on the leader or it will fail. For prevIndex and timeout, see AddVoter.
-func (r *Raft) RemoveServer(id ServerID, prevIndex uint64, timeout time.Duration) IndexFuture {
- if r.protocolVersion < 2 {
- return errorFuture{ErrUnsupportedProtocol}
- }
-
- return r.requestConfigChange(configurationChangeRequest{
- command: RemoveServer,
- serverID: id,
- prevIndex: prevIndex,
- }, timeout)
-}
-
-// DemoteVoter will take away a server's vote, if it has one. If present, the
-// server will continue to receive log entries, but it won't participate in
-// elections or log entry commitment. If the server is not in the cluster, this
-// does nothing. This must be run on the leader or it will fail. For prevIndex
-// and timeout, see AddVoter.
-func (r *Raft) DemoteVoter(id ServerID, prevIndex uint64, timeout time.Duration) IndexFuture {
- if r.protocolVersion < 3 {
- return errorFuture{ErrUnsupportedProtocol}
- }
-
- return r.requestConfigChange(configurationChangeRequest{
- command: DemoteVoter,
- serverID: id,
- prevIndex: prevIndex,
- }, timeout)
-}
-
-// Shutdown is used to stop the Raft background routines.
-// This is not a graceful operation. Provides a future that
-// can be used to block until all background routines have exited.
-func (r *Raft) Shutdown() Future {
- r.shutdownLock.Lock()
- defer r.shutdownLock.Unlock()
-
- if !r.shutdown {
- close(r.shutdownCh)
- r.shutdown = true
- r.setState(Shutdown)
- return &shutdownFuture{r}
- }
-
- // avoid closing transport twice
- return &shutdownFuture{nil}
-}
-
-// Snapshot is used to manually force Raft to take a snapshot. Returns a future
-// that can be used to block until complete, and that contains a function that
-// can be used to open the snapshot.
-func (r *Raft) Snapshot() SnapshotFuture {
- future := &userSnapshotFuture{}
- future.init()
- select {
- case r.userSnapshotCh <- future:
- return future
- case <-r.shutdownCh:
- future.respond(ErrRaftShutdown)
- return future
- }
-}
-
-// Restore is used to manually force Raft to consume an external snapshot, such
-// as if restoring from a backup. We will use the current Raft configuration,
-// not the one from the snapshot, so that we can restore into a new cluster. We
-// will also use the higher of the index of the snapshot, or the current index,
-// and then add 1 to that, so we force a new state with a hole in the Raft log,
-// so that the snapshot will be sent to followers and used for any new joiners.
-// This can only be run on the leader, and blocks until the restore is complete
-// or an error occurs.
-//
-// WARNING! This operation has the leader take on the state of the snapshot and
-// then sets itself up so that it replicates that to its followers though the
-// install snapshot process. This involves a potentially dangerous period where
-// the leader commits ahead of its followers, so should only be used for disaster
-// recovery into a fresh cluster, and should not be used in normal operations.
-func (r *Raft) Restore(meta *SnapshotMeta, reader io.Reader, timeout time.Duration) error {
- metrics.IncrCounter([]string{"raft", "restore"}, 1)
- var timer <-chan time.Time
- if timeout > 0 {
- timer = time.After(timeout)
- }
-
- // Perform the restore.
- restore := &userRestoreFuture{
- meta: meta,
- reader: reader,
- }
- restore.init()
- select {
- case <-timer:
- return ErrEnqueueTimeout
- case <-r.shutdownCh:
- return ErrRaftShutdown
- case r.userRestoreCh <- restore:
- // If the restore is ingested then wait for it to complete.
- if err := restore.Error(); err != nil {
- return err
- }
- }
-
- // Apply a no-op log entry. Waiting for this allows us to wait until the
- // followers have gotten the restore and replicated at least this new
- // entry, which shows that we've also faulted and installed the
- // snapshot with the contents of the restore.
- noop := &logFuture{
- log: Log{
- Type: LogNoop,
- },
- }
- noop.init()
- select {
- case <-timer:
- return ErrEnqueueTimeout
- case <-r.shutdownCh:
- return ErrRaftShutdown
- case r.applyCh <- noop:
- return noop.Error()
- }
-}
-
-// State is used to return the current raft state.
-func (r *Raft) State() RaftState {
- return r.getState()
-}
-
-// LeaderCh is used to get a channel which delivers signals on
-// acquiring or losing leadership. It sends true if we become
-// the leader, and false if we lose it. The channel is not buffered,
-// and does not block on writes.
-func (r *Raft) LeaderCh() <-chan bool {
- return r.leaderCh
-}
-
-// String returns a string representation of this Raft node.
-func (r *Raft) String() string {
- return fmt.Sprintf("Node at %s [%v]", r.localAddr, r.getState())
-}
-
-// LastContact returns the time of last contact by a leader.
-// This only makes sense if we are currently a follower.
-func (r *Raft) LastContact() time.Time {
- r.lastContactLock.RLock()
- last := r.lastContact
- r.lastContactLock.RUnlock()
- return last
-}
-
-// Stats is used to return a map of various internal stats. This
-// should only be used for informative purposes or debugging.
-//
-// Keys are: "state", "term", "last_log_index", "last_log_term",
-// "commit_index", "applied_index", "fsm_pending",
-// "last_snapshot_index", "last_snapshot_term",
-// "latest_configuration", "last_contact", and "num_peers".
-//
-// The value of "state" is a numerical value representing a
-// RaftState const.
-//
-// The value of "latest_configuration" is a string which contains
-// the id of each server, its suffrage status, and its address.
-//
-// The value of "last_contact" is either "never" if there
-// has been no contact with a leader, "0" if the node is in the
-// leader state, or the time since last contact with a leader
-// formatted as a string.
-//
-// The value of "num_peers" is the number of other voting servers in the
-// cluster, not including this node. If this node isn't part of the
-// configuration then this will be "0".
-//
-// All other values are uint64s, formatted as strings.
-func (r *Raft) Stats() map[string]string {
- toString := func(v uint64) string {
- return strconv.FormatUint(v, 10)
- }
- lastLogIndex, lastLogTerm := r.getLastLog()
- lastSnapIndex, lastSnapTerm := r.getLastSnapshot()
- s := map[string]string{
- "state": r.getState().String(),
- "term": toString(r.getCurrentTerm()),
- "last_log_index": toString(lastLogIndex),
- "last_log_term": toString(lastLogTerm),
- "commit_index": toString(r.getCommitIndex()),
- "applied_index": toString(r.getLastApplied()),
- "fsm_pending": toString(uint64(len(r.fsmMutateCh))),
- "last_snapshot_index": toString(lastSnapIndex),
- "last_snapshot_term": toString(lastSnapTerm),
- "protocol_version": toString(uint64(r.protocolVersion)),
- "protocol_version_min": toString(uint64(ProtocolVersionMin)),
- "protocol_version_max": toString(uint64(ProtocolVersionMax)),
- "snapshot_version_min": toString(uint64(SnapshotVersionMin)),
- "snapshot_version_max": toString(uint64(SnapshotVersionMax)),
- }
-
- future := r.GetConfiguration()
- if err := future.Error(); err != nil {
- r.logger.Warn(fmt.Sprintf("could not get configuration for Stats: %v", err))
- } else {
- configuration := future.Configuration()
- s["latest_configuration_index"] = toString(future.Index())
- s["latest_configuration"] = fmt.Sprintf("%+v", configuration.Servers)
-
- // This is a legacy metric that we've seen people use in the wild.
- hasUs := false
- numPeers := 0
- for _, server := range configuration.Servers {
- if server.Suffrage == Voter {
- if server.ID == r.localID {
- hasUs = true
- } else {
- numPeers++
- }
- }
- }
- if !hasUs {
- numPeers = 0
- }
- s["num_peers"] = toString(uint64(numPeers))
- }
-
- last := r.LastContact()
- if r.getState() == Leader {
- s["last_contact"] = "0"
- } else if last.IsZero() {
- s["last_contact"] = "never"
- } else {
- s["last_contact"] = fmt.Sprintf("%v", time.Now().Sub(last))
- }
- return s
-}
-
-// LastIndex returns the last index in stable storage,
-// either from the last log or from the last snapshot.
-func (r *Raft) LastIndex() uint64 {
- return r.getLastIndex()
-}
-
-// AppliedIndex returns the last index applied to the FSM. This is generally
-// lagging behind the last index, especially for indexes that are persisted but
-// have not yet been considered committed by the leader. NOTE - this reflects
-// the last index that was sent to the application's FSM over the apply channel
-// but DOES NOT mean that the application's FSM has yet consumed it and applied
-// it to its internal state. Thus, the application's state may lag behind this
-// index.
-func (r *Raft) AppliedIndex() uint64 {
- return r.getLastApplied()
-}
diff --git a/vendor/github.com/hashicorp/raft/commands.go b/vendor/github.com/hashicorp/raft/commands.go
deleted file mode 100644
index 5d89e7bcdb..0000000000
--- a/vendor/github.com/hashicorp/raft/commands.go
+++ /dev/null
@@ -1,151 +0,0 @@
-package raft
-
-// RPCHeader is a common sub-structure used to pass along protocol version and
-// other information about the cluster. For older Raft implementations before
-// versioning was added this will default to a zero-valued structure when read
-// by newer Raft versions.
-type RPCHeader struct {
- // ProtocolVersion is the version of the protocol the sender is
- // speaking.
- ProtocolVersion ProtocolVersion
-}
-
-// WithRPCHeader is an interface that exposes the RPC header.
-type WithRPCHeader interface {
- GetRPCHeader() RPCHeader
-}
-
-// AppendEntriesRequest is the command used to append entries to the
-// replicated log.
-type AppendEntriesRequest struct {
- RPCHeader
-
- // Provide the current term and leader
- Term uint64
- Leader []byte
-
- // Provide the previous entries for integrity checking
- PrevLogEntry uint64
- PrevLogTerm uint64
-
- // New entries to commit
- Entries []*Log
-
- // Commit index on the leader
- LeaderCommitIndex uint64
-}
-
-// See WithRPCHeader.
-func (r *AppendEntriesRequest) GetRPCHeader() RPCHeader {
- return r.RPCHeader
-}
-
-// AppendEntriesResponse is the response returned from an
-// AppendEntriesRequest.
-type AppendEntriesResponse struct {
- RPCHeader
-
- // Newer term if leader is out of date
- Term uint64
-
- // Last Log is a hint to help accelerate rebuilding slow nodes
- LastLog uint64
-
- // We may not succeed if we have a conflicting entry
- Success bool
-
- // There are scenarios where this request didn't succeed
- // but there's no need to wait/back-off the next attempt.
- NoRetryBackoff bool
-}
-
-// See WithRPCHeader.
-func (r *AppendEntriesResponse) GetRPCHeader() RPCHeader {
- return r.RPCHeader
-}
-
-// RequestVoteRequest is the command used by a candidate to ask a Raft peer
-// for a vote in an election.
-type RequestVoteRequest struct {
- RPCHeader
-
- // Provide the term and our id
- Term uint64
- Candidate []byte
-
- // Used to ensure safety
- LastLogIndex uint64
- LastLogTerm uint64
-}
-
-// See WithRPCHeader.
-func (r *RequestVoteRequest) GetRPCHeader() RPCHeader {
- return r.RPCHeader
-}
-
-// RequestVoteResponse is the response returned from a RequestVoteRequest.
-type RequestVoteResponse struct {
- RPCHeader
-
- // Newer term if leader is out of date.
- Term uint64
-
- // Peers is deprecated, but required by servers that only understand
- // protocol version 0. This is not populated in protocol version 2
- // and later.
- Peers []byte
-
- // Is the vote granted.
- Granted bool
-}
-
-// See WithRPCHeader.
-func (r *RequestVoteResponse) GetRPCHeader() RPCHeader {
- return r.RPCHeader
-}
-
-// InstallSnapshotRequest is the command sent to a Raft peer to bootstrap its
-// log (and state machine) from a snapshot on another peer.
-type InstallSnapshotRequest struct {
- RPCHeader
- SnapshotVersion SnapshotVersion
-
- Term uint64
- Leader []byte
-
- // These are the last index/term included in the snapshot
- LastLogIndex uint64
- LastLogTerm uint64
-
- // Peer Set in the snapshot. This is deprecated in favor of Configuration
- // but remains here in case we receive an InstallSnapshot from a leader
- // that's running old code.
- Peers []byte
-
- // Cluster membership.
- Configuration []byte
- // Log index where 'Configuration' entry was originally written.
- ConfigurationIndex uint64
-
- // Size of the snapshot
- Size int64
-}
-
-// See WithRPCHeader.
-func (r *InstallSnapshotRequest) GetRPCHeader() RPCHeader {
- return r.RPCHeader
-}
-
-// InstallSnapshotResponse is the response returned from an
-// InstallSnapshotRequest.
-type InstallSnapshotResponse struct {
- RPCHeader
-
- Term uint64
- Success bool
-}
-
-// See WithRPCHeader.
-func (r *InstallSnapshotResponse) GetRPCHeader() RPCHeader {
- return r.RPCHeader
-}
diff --git a/vendor/github.com/hashicorp/raft/commitment.go b/vendor/github.com/hashicorp/raft/commitment.go
deleted file mode 100644
index 7aa36464ae..0000000000
--- a/vendor/github.com/hashicorp/raft/commitment.go
+++ /dev/null
@@ -1,101 +0,0 @@
-package raft
-
-import (
- "sort"
- "sync"
-)
-
-// Commitment is used to advance the leader's commit index. The leader and
-// replication goroutines report in newly written entries with Match(), and
-// this notifies on commitCh when the commit index has advanced.
-type commitment struct {
- // protects matchIndexes and commitIndex
- sync.Mutex
- // notified when commitIndex increases
- commitCh chan struct{}
- // voter ID to log index: the server stores up through this log entry
- matchIndexes map[ServerID]uint64
- // a quorum stores up through this log entry. monotonically increases.
- commitIndex uint64
- // the first index of this leader's term: this needs to be replicated to a
- // majority of the cluster before this leader may mark anything committed
- // (per Raft's commitment rule)
- startIndex uint64
-}
-
-// newCommitment returns an commitment struct that notifies the provided
-// channel when log entries have been committed. A new commitment struct is
-// created each time this server becomes leader for a particular term.
-// 'configuration' is the servers in the cluster.
-// 'startIndex' is the first index created in this term (see
-// its description above).
-func newCommitment(commitCh chan struct{}, configuration Configuration, startIndex uint64) *commitment {
- matchIndexes := make(map[ServerID]uint64)
- for _, server := range configuration.Servers {
- if server.Suffrage == Voter {
- matchIndexes[server.ID] = 0
- }
- }
- return &commitment{
- commitCh: commitCh,
- matchIndexes: matchIndexes,
- commitIndex: 0,
- startIndex: startIndex,
- }
-}
-
-// Called when a new cluster membership configuration is created: it will be
-// used to determine commitment from now on. 'configuration' is the servers in
-// the cluster.
-func (c *commitment) setConfiguration(configuration Configuration) {
- c.Lock()
- defer c.Unlock()
- oldMatchIndexes := c.matchIndexes
- c.matchIndexes = make(map[ServerID]uint64)
- for _, server := range configuration.Servers {
- if server.Suffrage == Voter {
- c.matchIndexes[server.ID] = oldMatchIndexes[server.ID] // defaults to 0
- }
- }
- c.recalculate()
-}
-
-// Called by leader after commitCh is notified
-func (c *commitment) getCommitIndex() uint64 {
- c.Lock()
- defer c.Unlock()
- return c.commitIndex
-}
-
-// Match is called once a server completes writing entries to disk: either the
-// leader has written the new entry or a follower has replied to an
-// AppendEntries RPC. The given server's disk agrees with this server's log up
-// through the given index.
-func (c *commitment) match(server ServerID, matchIndex uint64) {
- c.Lock()
- defer c.Unlock()
- if prev, hasVote := c.matchIndexes[server]; hasVote && matchIndex > prev {
- c.matchIndexes[server] = matchIndex
- c.recalculate()
- }
-}
-
-// Internal helper to calculate new commitIndex from matchIndexes.
-// Must be called with lock held.
-func (c *commitment) recalculate() {
- if len(c.matchIndexes) == 0 {
- return
- }
-
- matched := make([]uint64, 0, len(c.matchIndexes))
- for _, idx := range c.matchIndexes {
- matched = append(matched, idx)
- }
- sort.Sort(uint64Slice(matched))
- quorumMatchIndex := matched[(len(matched)-1)/2]
-
- if quorumMatchIndex > c.commitIndex && quorumMatchIndex >= c.startIndex {
- c.commitIndex = quorumMatchIndex
- asyncNotifyCh(c.commitCh)
- }
-}
diff --git a/vendor/github.com/hashicorp/raft/config.go b/vendor/github.com/hashicorp/raft/config.go
deleted file mode 100644
index 66d4d0fa08..0000000000
--- a/vendor/github.com/hashicorp/raft/config.go
+++ /dev/null
@@ -1,265 +0,0 @@
-package raft
-
-import (
- "fmt"
- "io"
- "time"
-
- "github.com/hashicorp/go-hclog"
-)
-
-// These are the versions of the protocol (which includes RPC messages as
-// well as Raft-specific log entries) that this server can _understand_. Use
-// the ProtocolVersion member of the Config object to control the version of
-// the protocol to use when _speaking_ to other servers. Note that depending on
-// the protocol version being spoken, some otherwise understood RPC messages
-// may be refused. See dispositionRPC for details of this logic.
-//
-// There are notes about the upgrade path in the description of the versions
-// below. If you are starting a fresh cluster then there's no reason not to
-// jump right to the latest protocol version. If you need to interoperate with
-// older, version 0 Raft servers you'll need to drive the cluster through the
-// different versions in order.
-//
-// The version details are complicated, but here's a summary of what's required
-// to get from a version 0 cluster to version 3:
-//
-// 1. In version N of your app that starts using the new Raft library with
-// versioning, set ProtocolVersion to 1.
-// 2. Make version N+1 of your app require version N as a prerequisite (all
-// servers must be upgraded). For version N+1 of your app set ProtocolVersion
-// to 2.
-// 3. Similarly, make version N+2 of your app require version N+1 as a
-// prerequisite. For version N+2 of your app, set ProtocolVersion to 3.
-//
-// During this upgrade, older cluster members will still have Server IDs equal
-// to their network addresses. To upgrade an older member and give it an ID, it
-// needs to leave the cluster and re-enter:
-//
-// 1. Remove the server from the cluster with RemoveServer, using its network
-// address as its ServerID.
-// 2. Update the server's config to use a UUID or something else that is
-// not tied to the machine as the ServerID (restarting the server).
-// 3. Add the server back to the cluster with AddVoter, using its new ID.
-//
-// You can do this during the rolling upgrade from N+1 to N+2 of your app, or
-// as a rolling change at any time after the upgrade.
-//
-// Version History
-//
-// 0: Original Raft library before versioning was added. Servers running this
-// version of the Raft library use AddPeerDeprecated/RemovePeerDeprecated
-// for all configuration changes, and have no support for LogConfiguration.
-// 1: First versioned protocol, used to interoperate with old servers, and begin
-// the migration path to newer versions of the protocol. Under this version
-// all configuration changes are propagated using the now-deprecated
-// RemovePeerDeprecated Raft log entry. This means that server IDs are always
-// set to be the same as the server addresses (since the old log entry type
-// cannot transmit an ID), and only AddPeer/RemovePeer APIs are supported.
-// Servers running this version of the protocol can understand the new
-// LogConfiguration Raft log entry but will never generate one so they can
-// remain compatible with version 0 Raft servers in the cluster.
-// 2: Transitional protocol used when migrating an existing cluster to the new
-// server ID system. Server IDs are still set to be the same as server
-// addresses, but all configuration changes are propagated using the new
-// LogConfiguration Raft log entry type, which can carry full ID information.
-// This version supports the old AddPeer/RemovePeer APIs as well as the new
-// ID-based AddVoter/RemoveServer APIs which should be used when adding
-// version 3 servers to the cluster later. This version sheds all
-// interoperability with version 0 servers, but can interoperate with newer
-// Raft servers running with protocol version 1 since they can understand the
-// new LogConfiguration Raft log entry, and this version can still understand
-// their RemovePeerDeprecated Raft log entries. We need this protocol version
-// as an intermediate step between 1 and 3 so that servers will propagate the
-// ID information that will come from newly-added (or -rolled) servers using
-// protocol version 3, but since they are still using their address-based IDs
-// from the previous step they will still be able to track commitments and
-// their own voting status properly. If we skipped this step, servers would
-// be started with their new IDs, but they wouldn't see themselves in the old
-// address-based configuration, so none of the servers would think they had a
-// vote.
-// 3: Protocol adding full support for server IDs and new ID-based server APIs
-// (AddVoter, AddNonvoter, etc.), old AddPeer/RemovePeer APIs are no longer
-// supported. Version 2 servers should be swapped out by removing them from
-// the cluster one-by-one and re-adding them with updated configuration for
-// this protocol version, along with their server ID. The remove/add cycle
-// is required to populate their server ID. Note that removing must be done
-// by ID, which will be the old server's address.
-type ProtocolVersion int
-
-const (
- ProtocolVersionMin ProtocolVersion = 0
- ProtocolVersionMax = 3
-)
-
-// These are versions of snapshots that this server can _understand_. Currently,
-// it is always assumed that this server generates the latest version, though
-// this may be changed in the future to include a configurable version.
-//
-// Version History
-//
-// 0: Original Raft library before versioning was added. The peers portion of
-// these snapshots is encoded in the legacy format which requires decodePeers
-// to parse. This version of snapshots should only be produced by the
-// unversioned Raft library.
-// 1: New format which adds support for a full configuration structure and its
-// associated log index, with support for server IDs and non-voting server
-// modes. To ease upgrades, this also includes the legacy peers structure but
-// that will never be used by servers that understand version 1 snapshots.
-// Since the original Raft library didn't enforce any versioning, we must
-// include the legacy peers structure for this version, but we can deprecate
-// it in the next snapshot version.
-type SnapshotVersion int
-
-const (
- SnapshotVersionMin SnapshotVersion = 0
- SnapshotVersionMax = 1
-)
-
-// Config provides any necessary configuration for the Raft server.
-type Config struct {
- // ProtocolVersion allows a Raft server to inter-operate with older
- // Raft servers running an older version of the code. This is used to
- // version the wire protocol as well as Raft-specific log entries that
- // the server uses when _speaking_ to other servers. There is currently
- // no auto-negotiation of versions so all servers must be manually
- // configured with compatible versions. See ProtocolVersionMin and
- // ProtocolVersionMax for the versions of the protocol that this server
- // can _understand_.
- ProtocolVersion ProtocolVersion
-
- // HeartbeatTimeout specifies the time in follower state without
- // a leader before we attempt an election.
- HeartbeatTimeout time.Duration
-
- // ElectionTimeout specifies the time in candidate state without
- // a leader before we attempt an election.
- ElectionTimeout time.Duration
-
- // CommitTimeout controls the time without an Apply() operation
- // before we heartbeat to ensure a timely commit. Due to random
- // staggering, may be delayed as much as 2x this value.
- CommitTimeout time.Duration
-
- // MaxAppendEntries controls the maximum number of append entries
- // to send at once. We want to strike a balance between efficiency
- // and avoiding waste if the follower is going to reject because of
- // an inconsistent log.
- MaxAppendEntries int
-
- // If we are a member of a cluster, and RemovePeer is invoked for the
- // local node, then we forget all peers and transition into the follower state.
- // If ShutdownOnRemove is is set, we additional shutdown Raft. Otherwise,
- // we can become a leader of a cluster containing only this node.
- ShutdownOnRemove bool
-
- // TrailingLogs controls how many logs we leave after a snapshot. This is
- // used so that we can quickly replay logs on a follower instead of being
- // forced to send an entire snapshot.
- TrailingLogs uint64
-
- // SnapshotInterval controls how often we check if we should perform a snapshot.
- // We randomly stagger between this value and 2x this value to avoid the entire
- // cluster from performing a snapshot at once.
- SnapshotInterval time.Duration
-
- // SnapshotThreshold controls how many outstanding logs there must be before
- // we perform a snapshot. This is to prevent excessive snapshots when we can
- // just replay a small set of logs.
- SnapshotThreshold uint64
-
- // LeaderLeaseTimeout is used to control how long the "lease" lasts
- // for being the leader without being able to contact a quorum
- // of nodes. If we reach this interval without contact, we will
- // step down as leader.
- LeaderLeaseTimeout time.Duration
-
- // StartAsLeader forces Raft to start in the leader state. This should
- // never be used except for testing purposes, as it can cause a split-brain.
- StartAsLeader bool
-
- // The unique ID for this server across all time. When running with
- // ProtocolVersion < 3, you must set this to be the same as the network
- // address of your transport.
- LocalID ServerID
-
- // NotifyCh is used to provide a channel that will be notified of leadership
- // changes. Raft will block writing to this channel, so it should either be
- // buffered or aggressively consumed.
- NotifyCh chan<- bool
-
- // LogOutput is used as a sink for logs, unless Logger is specified.
- // Defaults to os.Stderr.
- LogOutput io.Writer
-
- // LogLevel represents a log level. If a no matching string is specified,
- // hclog.NoLevel is assumed.
- LogLevel string
-
- // Logger is a user-provided hc-log logger. If nil, a logger writing to
- // LogOutput with LogLevel is used.
- Logger hclog.Logger
-}
-
-// DefaultConfig returns a Config with usable defaults.
-func DefaultConfig() *Config {
- return &Config{
- ProtocolVersion: ProtocolVersionMax,
- HeartbeatTimeout: 1000 * time.Millisecond,
- ElectionTimeout: 1000 * time.Millisecond,
- CommitTimeout: 50 * time.Millisecond,
- MaxAppendEntries: 64,
- ShutdownOnRemove: true,
- TrailingLogs: 10240,
- SnapshotInterval: 120 * time.Second,
- SnapshotThreshold: 8192,
- LeaderLeaseTimeout: 500 * time.Millisecond,
- LogLevel: "DEBUG",
- }
-}
-
-// ValidateConfig is used to validate a sane configuration
-func ValidateConfig(config *Config) error {
- // We don't actually support running as 0 in the library any more, but
- // we do understand it.
- protocolMin := ProtocolVersionMin
- if protocolMin == 0 {
- protocolMin = 1
- }
- if config.ProtocolVersion < protocolMin ||
- config.ProtocolVersion > ProtocolVersionMax {
- return fmt.Errorf("Protocol version %d must be >= %d and <= %d",
- config.ProtocolVersion, protocolMin, ProtocolVersionMax)
- }
- if len(config.LocalID) == 0 {
- return fmt.Errorf("LocalID cannot be empty")
- }
- if config.HeartbeatTimeout < 5*time.Millisecond {
- return fmt.Errorf("Heartbeat timeout is too low")
- }
- if config.ElectionTimeout < 5*time.Millisecond {
- return fmt.Errorf("Election timeout is too low")
- }
- if config.CommitTimeout < time.Millisecond {
- return fmt.Errorf("Commit timeout is too low")
- }
- if config.MaxAppendEntries <= 0 {
- return fmt.Errorf("MaxAppendEntries must be positive")
- }
- if config.MaxAppendEntries > 1024 {
- return fmt.Errorf("MaxAppendEntries is too large")
- }
- if config.SnapshotInterval < 5*time.Millisecond {
- return fmt.Errorf("Snapshot interval is too low")
- }
- if config.LeaderLeaseTimeout < 5*time.Millisecond {
- return fmt.Errorf("Leader lease timeout is too low")
- }
- if config.LeaderLeaseTimeout > config.HeartbeatTimeout {
- return fmt.Errorf("Leader lease timeout cannot be larger than heartbeat timeout")
- }
- if config.ElectionTimeout < config.HeartbeatTimeout {
- return fmt.Errorf("Election timeout must be equal or greater than Heartbeat Timeout")
- }
- return nil
-}
diff --git a/vendor/github.com/hashicorp/raft/configuration.go b/vendor/github.com/hashicorp/raft/configuration.go
deleted file mode 100644
index 4bb784d0bf..0000000000
--- a/vendor/github.com/hashicorp/raft/configuration.go
+++ /dev/null
@@ -1,343 +0,0 @@
-package raft
-
-import "fmt"
-
-// ServerSuffrage determines whether a Server in a Configuration gets a vote.
-type ServerSuffrage int
-
-// Note: Don't renumber these, since the numbers are written into the log.
-const (
- // Voter is a server whose vote is counted in elections and whose match index
- // is used in advancing the leader's commit index.
- Voter ServerSuffrage = iota
- // Nonvoter is a server that receives log entries but is not considered for
- // elections or commitment purposes.
- Nonvoter
- // Staging is a server that acts like a nonvoter with one exception: once a
- // staging server receives enough log entries to be sufficiently caught up to
- // the leader's log, the leader will invoke a membership change to change
- // the Staging server to a Voter.
- Staging
-)
-
-func (s ServerSuffrage) String() string {
- switch s {
- case Voter:
- return "Voter"
- case Nonvoter:
- return "Nonvoter"
- case Staging:
- return "Staging"
- }
- return "ServerSuffrage"
-}
-
-// ServerID is a unique string identifying a server for all time.
-type ServerID string
-
-// ServerAddress is a network address for a server that a transport can contact.
-type ServerAddress string
-
-// Server tracks the information about a single server in a configuration.
-type Server struct {
- // Suffrage determines whether the server gets a vote.
- Suffrage ServerSuffrage
- // ID is a unique string identifying this server for all time.
- ID ServerID
- // Address is its network address that a transport can contact.
- Address ServerAddress
-}
-
-// Configuration tracks which servers are in the cluster, and whether they have
-// votes. This should include the local server, if it's a member of the cluster.
-// The servers are listed no particular order, but each should only appear once.
-// These entries are appended to the log during membership changes.
-type Configuration struct {
- Servers []Server
-}
-
-// Clone makes a deep copy of a Configuration.
-func (c *Configuration) Clone() (copy Configuration) {
- copy.Servers = append(copy.Servers, c.Servers...)
- return
-}
-
-// ConfigurationChangeCommand is the different ways to change the cluster
-// configuration.
-type ConfigurationChangeCommand uint8
-
-const (
- // AddStaging makes a server Staging unless its Voter.
- AddStaging ConfigurationChangeCommand = iota
- // AddNonvoter makes a server Nonvoter unless its Staging or Voter.
- AddNonvoter
- // DemoteVoter makes a server Nonvoter unless its absent.
- DemoteVoter
- // RemoveServer removes a server entirely from the cluster membership.
- RemoveServer
- // Promote is created automatically by a leader; it turns a Staging server
- // into a Voter.
- Promote
-)
-
-func (c ConfigurationChangeCommand) String() string {
- switch c {
- case AddStaging:
- return "AddStaging"
- case AddNonvoter:
- return "AddNonvoter"
- case DemoteVoter:
- return "DemoteVoter"
- case RemoveServer:
- return "RemoveServer"
- case Promote:
- return "Promote"
- }
- return "ConfigurationChangeCommand"
-}
-
-// configurationChangeRequest describes a change that a leader would like to
-// make to its current configuration. It's used only within a single server
-// (never serialized into the log), as part of `configurationChangeFuture`.
-type configurationChangeRequest struct {
- command ConfigurationChangeCommand
- serverID ServerID
- serverAddress ServerAddress // only present for AddStaging, AddNonvoter
- // prevIndex, if nonzero, is the index of the only configuration upon which
- // this change may be applied; if another configuration entry has been
- // added in the meantime, this request will fail.
- prevIndex uint64
-}
-
-// configurations is state tracked on every server about its Configurations.
-// Note that, per Diego's dissertation, there can be at most one uncommitted
-// configuration at a time (the next configuration may not be created until the
-// prior one has been committed).
-//
-// One downside to storing just two configurations is that if you try to take a
-// snapshot when your state machine hasn't yet applied the committedIndex, we
-// have no record of the configuration that would logically fit into that
-// snapshot. We disallow snapshots in that case now. An alternative approach,
-// which LogCabin uses, is to track every configuration change in the
-// log.
-type configurations struct {
- // committed is the latest configuration in the log/snapshot that has been
- // committed (the one with the largest index).
- committed Configuration
- // committedIndex is the log index where 'committed' was written.
- committedIndex uint64
- // latest is the latest configuration in the log/snapshot (may be committed
- // or uncommitted)
- latest Configuration
- // latestIndex is the log index where 'latest' was written.
- latestIndex uint64
-}
-
-// Clone makes a deep copy of a configurations object.
-func (c *configurations) Clone() (copy configurations) {
- copy.committed = c.committed.Clone()
- copy.committedIndex = c.committedIndex
- copy.latest = c.latest.Clone()
- copy.latestIndex = c.latestIndex
- return
-}
-
-// hasVote returns true if the server identified by 'id' is a Voter in the
-// provided Configuration.
-func hasVote(configuration Configuration, id ServerID) bool {
- for _, server := range configuration.Servers {
- if server.ID == id {
- return server.Suffrage == Voter
- }
- }
- return false
-}
-
-// checkConfiguration tests a cluster membership configuration for common
-// errors.
-func checkConfiguration(configuration Configuration) error {
- idSet := make(map[ServerID]bool)
- addressSet := make(map[ServerAddress]bool)
- var voters int
- for _, server := range configuration.Servers {
- if server.ID == "" {
- return fmt.Errorf("Empty ID in configuration: %v", configuration)
- }
- if server.Address == "" {
- return fmt.Errorf("Empty address in configuration: %v", server)
- }
- if idSet[server.ID] {
- return fmt.Errorf("Found duplicate ID in configuration: %v", server.ID)
- }
- idSet[server.ID] = true
- if addressSet[server.Address] {
- return fmt.Errorf("Found duplicate address in configuration: %v", server.Address)
- }
- addressSet[server.Address] = true
- if server.Suffrage == Voter {
- voters++
- }
- }
- if voters == 0 {
- return fmt.Errorf("Need at least one voter in configuration: %v", configuration)
- }
- return nil
-}
-
-// nextConfiguration generates a new Configuration from the current one and a
-// configuration change request. It's split from appendConfigurationEntry so
-// that it can be unit tested easily.
-func nextConfiguration(current Configuration, currentIndex uint64, change configurationChangeRequest) (Configuration, error) {
- if change.prevIndex > 0 && change.prevIndex != currentIndex {
- return Configuration{}, fmt.Errorf("Configuration changed since %v (latest is %v)", change.prevIndex, currentIndex)
- }
-
- configuration := current.Clone()
- switch change.command {
- case AddStaging:
- // TODO: barf on new address?
- newServer := Server{
- // TODO: This should add the server as Staging, to be automatically
- // promoted to Voter later. However, the promotion to Voter is not yet
- // implemented, and doing so is not trivial with the way the leader loop
- // coordinates with the replication goroutines today. So, for now, the
- // server will have a vote right away, and the Promote case below is
- // unused.
- Suffrage: Voter,
- ID: change.serverID,
- Address: change.serverAddress,
- }
- found := false
- for i, server := range configuration.Servers {
- if server.ID == change.serverID {
- if server.Suffrage == Voter {
- configuration.Servers[i].Address = change.serverAddress
- } else {
- configuration.Servers[i] = newServer
- }
- found = true
- break
- }
- }
- if !found {
- configuration.Servers = append(configuration.Servers, newServer)
- }
- case AddNonvoter:
- newServer := Server{
- Suffrage: Nonvoter,
- ID: change.serverID,
- Address: change.serverAddress,
- }
- found := false
- for i, server := range configuration.Servers {
- if server.ID == change.serverID {
- if server.Suffrage != Nonvoter {
- configuration.Servers[i].Address = change.serverAddress
- } else {
- configuration.Servers[i] = newServer
- }
- found = true
- break
- }
- }
- if !found {
- configuration.Servers = append(configuration.Servers, newServer)
- }
- case DemoteVoter:
- for i, server := range configuration.Servers {
- if server.ID == change.serverID {
- configuration.Servers[i].Suffrage = Nonvoter
- break
- }
- }
- case RemoveServer:
- for i, server := range configuration.Servers {
- if server.ID == change.serverID {
- configuration.Servers = append(configuration.Servers[:i], configuration.Servers[i+1:]...)
- break
- }
- }
- case Promote:
- for i, server := range configuration.Servers {
- if server.ID == change.serverID && server.Suffrage == Staging {
- configuration.Servers[i].Suffrage = Voter
- break
- }
- }
- }
-
- // Make sure we didn't do something bad like remove the last voter
- if err := checkConfiguration(configuration); err != nil {
- return Configuration{}, err
- }
-
- return configuration, nil
-}
-
-// encodePeers is used to serialize a Configuration into the old peers format.
-// This is here for backwards compatibility when operating with a mix of old
-// servers and should be removed once we deprecate support for protocol version 1.
-func encodePeers(configuration Configuration, trans Transport) []byte {
- // Gather up all the voters, other suffrage types are not supported by
- // this data format.
- var encPeers [][]byte
- for _, server := range configuration.Servers {
- if server.Suffrage == Voter {
- encPeers = append(encPeers, trans.EncodePeer(server.ID, server.Address))
- }
- }
-
- // Encode the entire array.
- buf, err := encodeMsgPack(encPeers)
- if err != nil {
- panic(fmt.Errorf("failed to encode peers: %v", err))
- }
-
- return buf.Bytes()
-}
-
-// decodePeers is used to deserialize an old list of peers into a Configuration.
-// This is here for backwards compatibility with old log entries and snapshots;
-// it should be removed eventually.
-func decodePeers(buf []byte, trans Transport) Configuration {
- // Decode the buffer first.
- var encPeers [][]byte
- if err := decodeMsgPack(buf, &encPeers); err != nil {
- panic(fmt.Errorf("failed to decode peers: %v", err))
- }
-
- // Deserialize each peer.
- var servers []Server
- for _, enc := range encPeers {
- p := trans.DecodePeer(enc)
- servers = append(servers, Server{
- Suffrage: Voter,
- ID: ServerID(p),
- Address: ServerAddress(p),
- })
- }
-
- return Configuration{
- Servers: servers,
- }
-}
-
-// encodeConfiguration serializes a Configuration using MsgPack, or panics on
-// errors.
-func encodeConfiguration(configuration Configuration) []byte {
- buf, err := encodeMsgPack(configuration)
- if err != nil {
- panic(fmt.Errorf("failed to encode configuration: %v", err))
- }
- return buf.Bytes()
-}
-
-// decodeConfiguration deserializes a Configuration using MsgPack, or panics on
-// errors.
-func decodeConfiguration(buf []byte) Configuration {
- var configuration Configuration
- if err := decodeMsgPack(buf, &configuration); err != nil {
- panic(fmt.Errorf("failed to decode configuration: %v", err))
- }
- return configuration
-}
diff --git a/vendor/github.com/hashicorp/raft/discard_snapshot.go b/vendor/github.com/hashicorp/raft/discard_snapshot.go
deleted file mode 100644
index 5e93a9fe01..0000000000
--- a/vendor/github.com/hashicorp/raft/discard_snapshot.go
+++ /dev/null
@@ -1,49 +0,0 @@
-package raft
-
-import (
- "fmt"
- "io"
-)
-
-// DiscardSnapshotStore is used to successfully snapshot while
-// always discarding the snapshot. This is useful for when the
-// log should be truncated but no snapshot should be retained.
-// This should never be used for production use, and is only
-// suitable for testing.
-type DiscardSnapshotStore struct{}
-
-type DiscardSnapshotSink struct{}
-
-// NewDiscardSnapshotStore is used to create a new DiscardSnapshotStore.
-func NewDiscardSnapshotStore() *DiscardSnapshotStore {
- return &DiscardSnapshotStore{}
-}
-
-func (d *DiscardSnapshotStore) Create(version SnapshotVersion, index, term uint64,
- configuration Configuration, configurationIndex uint64, trans Transport) (SnapshotSink, error) {
- return &DiscardSnapshotSink{}, nil
-}
-
-func (d *DiscardSnapshotStore) List() ([]*SnapshotMeta, error) {
- return nil, nil
-}
-
-func (d *DiscardSnapshotStore) Open(id string) (*SnapshotMeta, io.ReadCloser, error) {
- return nil, nil, fmt.Errorf("open is not supported")
-}
-
-func (d *DiscardSnapshotSink) Write(b []byte) (int, error) {
- return len(b), nil
-}
-
-func (d *DiscardSnapshotSink) Close() error {
- return nil
-}
-
-func (d *DiscardSnapshotSink) ID() string {
- return "discard"
-}
-
-func (d *DiscardSnapshotSink) Cancel() error {
- return nil
-}
diff --git a/vendor/github.com/hashicorp/raft/file_snapshot.go b/vendor/github.com/hashicorp/raft/file_snapshot.go
deleted file mode 100644
index ffc9414542..0000000000
--- a/vendor/github.com/hashicorp/raft/file_snapshot.go
+++ /dev/null
@@ -1,528 +0,0 @@
-package raft
-
-import (
- "bufio"
- "bytes"
- "encoding/json"
- "fmt"
- "hash"
- "hash/crc64"
- "io"
- "io/ioutil"
- "log"
- "os"
- "path/filepath"
- "runtime"
- "sort"
- "strings"
- "time"
-)
-
-const (
- testPath = "permTest"
- snapPath = "snapshots"
- metaFilePath = "meta.json"
- stateFilePath = "state.bin"
- tmpSuffix = ".tmp"
-)
-
-// FileSnapshotStore implements the SnapshotStore interface and allows
-// snapshots to be made on the local disk.
-type FileSnapshotStore struct {
- path string
- retain int
- logger *log.Logger
-}
-
-type snapMetaSlice []*fileSnapshotMeta
-
-// FileSnapshotSink implements SnapshotSink with a file.
-type FileSnapshotSink struct {
- store *FileSnapshotStore
- logger *log.Logger
- dir string
- parentDir string
- meta fileSnapshotMeta
-
- stateFile *os.File
- stateHash hash.Hash64
- buffered *bufio.Writer
-
- closed bool
-}
-
-// fileSnapshotMeta is stored on disk. We also put a CRC
-// on disk so that we can verify the snapshot.
-type fileSnapshotMeta struct {
- SnapshotMeta
- CRC []byte
-}
-
-// bufferedFile is returned when we open a snapshot. This way
-// reads are buffered and the file still gets closed.
-type bufferedFile struct {
- bh *bufio.Reader
- fh *os.File
-}
-
-func (b *bufferedFile) Read(p []byte) (n int, err error) {
- return b.bh.Read(p)
-}
-
-func (b *bufferedFile) Close() error {
- return b.fh.Close()
-}
-
-// NewFileSnapshotStoreWithLogger creates a new FileSnapshotStore based
-// on a base directory. The `retain` parameter controls how many
-// snapshots are retained. Must be at least 1.
-func NewFileSnapshotStoreWithLogger(base string, retain int, logger *log.Logger) (*FileSnapshotStore, error) {
- if retain < 1 {
- return nil, fmt.Errorf("must retain at least one snapshot")
- }
- if logger == nil {
- logger = log.New(os.Stderr, "", log.LstdFlags)
- }
-
- // Ensure our path exists
- path := filepath.Join(base, snapPath)
- if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) {
- return nil, fmt.Errorf("snapshot path not accessible: %v", err)
- }
-
- // Setup the store
- store := &FileSnapshotStore{
- path: path,
- retain: retain,
- logger: logger,
- }
-
- // Do a permissions test
- if err := store.testPermissions(); err != nil {
- return nil, fmt.Errorf("permissions test failed: %v", err)
- }
- return store, nil
-}
-
-// NewFileSnapshotStore creates a new FileSnapshotStore based
-// on a base directory. The `retain` parameter controls how many
-// snapshots are retained. Must be at least 1.
-func NewFileSnapshotStore(base string, retain int, logOutput io.Writer) (*FileSnapshotStore, error) {
- if logOutput == nil {
- logOutput = os.Stderr
- }
- return NewFileSnapshotStoreWithLogger(base, retain, log.New(logOutput, "", log.LstdFlags))
-}
-
-// testPermissions tries to touch a file in our path to see if it works.
-func (f *FileSnapshotStore) testPermissions() error {
- path := filepath.Join(f.path, testPath)
- fh, err := os.Create(path)
- if err != nil {
- return err
- }
-
- if err = fh.Close(); err != nil {
- return err
- }
-
- if err = os.Remove(path); err != nil {
- return err
- }
- return nil
-}
-
-// snapshotName generates a name for the snapshot.
-func snapshotName(term, index uint64) string {
- now := time.Now()
- msec := now.UnixNano() / int64(time.Millisecond)
- return fmt.Sprintf("%d-%d-%d", term, index, msec)
-}
-
-// Create is used to start a new snapshot
-func (f *FileSnapshotStore) Create(version SnapshotVersion, index, term uint64,
- configuration Configuration, configurationIndex uint64, trans Transport) (SnapshotSink, error) {
- // We only support version 1 snapshots at this time.
- if version != 1 {
- return nil, fmt.Errorf("unsupported snapshot version %d", version)
- }
-
- // Create a new path
- name := snapshotName(term, index)
- path := filepath.Join(f.path, name+tmpSuffix)
- f.logger.Printf("[INFO] snapshot: Creating new snapshot at %s", path)
-
- // Make the directory
- if err := os.MkdirAll(path, 0755); err != nil {
- f.logger.Printf("[ERR] snapshot: Failed to make snapshot directory: %v", err)
- return nil, err
- }
-
- // Create the sink
- sink := &FileSnapshotSink{
- store: f,
- logger: f.logger,
- dir: path,
- parentDir: f.path,
- meta: fileSnapshotMeta{
- SnapshotMeta: SnapshotMeta{
- Version: version,
- ID: name,
- Index: index,
- Term: term,
- Peers: encodePeers(configuration, trans),
- Configuration: configuration,
- ConfigurationIndex: configurationIndex,
- },
- CRC: nil,
- },
- }
-
- // Write out the meta data
- if err := sink.writeMeta(); err != nil {
- f.logger.Printf("[ERR] snapshot: Failed to write metadata: %v", err)
- return nil, err
- }
-
- // Open the state file
- statePath := filepath.Join(path, stateFilePath)
- fh, err := os.Create(statePath)
- if err != nil {
- f.logger.Printf("[ERR] snapshot: Failed to create state file: %v", err)
- return nil, err
- }
- sink.stateFile = fh
-
- // Create a CRC64 hash
- sink.stateHash = crc64.New(crc64.MakeTable(crc64.ECMA))
-
- // Wrap both the hash and file in a MultiWriter with buffering
- multi := io.MultiWriter(sink.stateFile, sink.stateHash)
- sink.buffered = bufio.NewWriter(multi)
-
- // Done
- return sink, nil
-}
-
-// List returns available snapshots in the store.
-func (f *FileSnapshotStore) List() ([]*SnapshotMeta, error) {
- // Get the eligible snapshots
- snapshots, err := f.getSnapshots()
- if err != nil {
- f.logger.Printf("[ERR] snapshot: Failed to get snapshots: %v", err)
- return nil, err
- }
-
- var snapMeta []*SnapshotMeta
- for _, meta := range snapshots {
- snapMeta = append(snapMeta, &meta.SnapshotMeta)
- if len(snapMeta) == f.retain {
- break
- }
- }
- return snapMeta, nil
-}
-
-// getSnapshots returns all the known snapshots.
-func (f *FileSnapshotStore) getSnapshots() ([]*fileSnapshotMeta, error) {
- // Get the eligible snapshots
- snapshots, err := ioutil.ReadDir(f.path)
- if err != nil {
- f.logger.Printf("[ERR] snapshot: Failed to scan snapshot dir: %v", err)
- return nil, err
- }
-
- // Populate the metadata
- var snapMeta []*fileSnapshotMeta
- for _, snap := range snapshots {
- // Ignore any files
- if !snap.IsDir() {
- continue
- }
-
- // Ignore any temporary snapshots
- dirName := snap.Name()
- if strings.HasSuffix(dirName, tmpSuffix) {
- f.logger.Printf("[WARN] snapshot: Found temporary snapshot: %v", dirName)
- continue
- }
-
- // Try to read the meta data
- meta, err := f.readMeta(dirName)
- if err != nil {
- f.logger.Printf("[WARN] snapshot: Failed to read metadata for %v: %v", dirName, err)
- continue
- }
-
- // Make sure we can understand this version.
- if meta.Version < SnapshotVersionMin || meta.Version > SnapshotVersionMax {
- f.logger.Printf("[WARN] snapshot: Snapshot version for %v not supported: %d", dirName, meta.Version)
- continue
- }
-
- // Append, but only return up to the retain count
- snapMeta = append(snapMeta, meta)
- }
-
- // Sort the snapshot, reverse so we get new -> old
- sort.Sort(sort.Reverse(snapMetaSlice(snapMeta)))
-
- return snapMeta, nil
-}
-
-// readMeta is used to read the meta data for a given named backup
-func (f *FileSnapshotStore) readMeta(name string) (*fileSnapshotMeta, error) {
- // Open the meta file
- metaPath := filepath.Join(f.path, name, metaFilePath)
- fh, err := os.Open(metaPath)
- if err != nil {
- return nil, err
- }
- defer fh.Close()
-
- // Buffer the file IO
- buffered := bufio.NewReader(fh)
-
- // Read in the JSON
- meta := &fileSnapshotMeta{}
- dec := json.NewDecoder(buffered)
- if err := dec.Decode(meta); err != nil {
- return nil, err
- }
- return meta, nil
-}
-
-// Open takes a snapshot ID and returns a ReadCloser for that snapshot.
-func (f *FileSnapshotStore) Open(id string) (*SnapshotMeta, io.ReadCloser, error) {
- // Get the metadata
- meta, err := f.readMeta(id)
- if err != nil {
- f.logger.Printf("[ERR] snapshot: Failed to get meta data to open snapshot: %v", err)
- return nil, nil, err
- }
-
- // Open the state file
- statePath := filepath.Join(f.path, id, stateFilePath)
- fh, err := os.Open(statePath)
- if err != nil {
- f.logger.Printf("[ERR] snapshot: Failed to open state file: %v", err)
- return nil, nil, err
- }
-
- // Create a CRC64 hash
- stateHash := crc64.New(crc64.MakeTable(crc64.ECMA))
-
- // Compute the hash
- _, err = io.Copy(stateHash, fh)
- if err != nil {
- f.logger.Printf("[ERR] snapshot: Failed to read state file: %v", err)
- fh.Close()
- return nil, nil, err
- }
-
- // Verify the hash
- computed := stateHash.Sum(nil)
- if bytes.Compare(meta.CRC, computed) != 0 {
- f.logger.Printf("[ERR] snapshot: CRC checksum failed (stored: %v computed: %v)",
- meta.CRC, computed)
- fh.Close()
- return nil, nil, fmt.Errorf("CRC mismatch")
- }
-
- // Seek to the start
- if _, err := fh.Seek(0, 0); err != nil {
- f.logger.Printf("[ERR] snapshot: State file seek failed: %v", err)
- fh.Close()
- return nil, nil, err
- }
-
- // Return a buffered file
- buffered := &bufferedFile{
- bh: bufio.NewReader(fh),
- fh: fh,
- }
-
- return &meta.SnapshotMeta, buffered, nil
-}
-
-// ReapSnapshots reaps any snapshots beyond the retain count.
-func (f *FileSnapshotStore) ReapSnapshots() error {
- snapshots, err := f.getSnapshots()
- if err != nil {
- f.logger.Printf("[ERR] snapshot: Failed to get snapshots: %v", err)
- return err
- }
-
- for i := f.retain; i < len(snapshots); i++ {
- path := filepath.Join(f.path, snapshots[i].ID)
- f.logger.Printf("[INFO] snapshot: reaping snapshot %v", path)
- if err := os.RemoveAll(path); err != nil {
- f.logger.Printf("[ERR] snapshot: Failed to reap snapshot %v: %v", path, err)
- return err
- }
- }
- return nil
-}
-
-// ID returns the ID of the snapshot, can be used with Open()
-// after the snapshot is finalized.
-func (s *FileSnapshotSink) ID() string {
- return s.meta.ID
-}
-
-// Write is used to append to the state file. We write to the
-// buffered IO object to reduce the amount of context switches.
-func (s *FileSnapshotSink) Write(b []byte) (int, error) {
- return s.buffered.Write(b)
-}
-
-// Close is used to indicate a successful end.
-func (s *FileSnapshotSink) Close() error {
- // Make sure close is idempotent
- if s.closed {
- return nil
- }
- s.closed = true
-
- // Close the open handles
- if err := s.finalize(); err != nil {
- s.logger.Printf("[ERR] snapshot: Failed to finalize snapshot: %v", err)
- if delErr := os.RemoveAll(s.dir); delErr != nil {
- s.logger.Printf("[ERR] snapshot: Failed to delete temporary snapshot directory at path %v: %v", s.dir, delErr)
- return delErr
- }
- return err
- }
-
- // Write out the meta data
- if err := s.writeMeta(); err != nil {
- s.logger.Printf("[ERR] snapshot: Failed to write metadata: %v", err)
- return err
- }
-
- // Move the directory into place
- newPath := strings.TrimSuffix(s.dir, tmpSuffix)
- if err := os.Rename(s.dir, newPath); err != nil {
- s.logger.Printf("[ERR] snapshot: Failed to move snapshot into place: %v", err)
- return err
- }
-
- if runtime.GOOS != "windows" { //skipping fsync for directory entry edits on Windows, only needed for *nix style file systems
- parentFH, err := os.Open(s.parentDir)
- defer parentFH.Close()
- if err != nil {
- s.logger.Printf("[ERR] snapshot: Failed to open snapshot parent directory %v, error: %v", s.parentDir, err)
- return err
- }
-
- if err = parentFH.Sync(); err != nil {
- s.logger.Printf("[ERR] snapshot: Failed syncing parent directory %v, error: %v", s.parentDir, err)
- return err
- }
- }
-
- // Reap any old snapshots
- if err := s.store.ReapSnapshots(); err != nil {
- return err
- }
-
- return nil
-}
-
-// Cancel is used to indicate an unsuccessful end.
-func (s *FileSnapshotSink) Cancel() error {
- // Make sure close is idempotent
- if s.closed {
- return nil
- }
- s.closed = true
-
- // Close the open handles
- if err := s.finalize(); err != nil {
- s.logger.Printf("[ERR] snapshot: Failed to finalize snapshot: %v", err)
- return err
- }
-
- // Attempt to remove all artifacts
- return os.RemoveAll(s.dir)
-}
-
-// finalize is used to close all of our resources.
-func (s *FileSnapshotSink) finalize() error {
- // Flush any remaining data
- if err := s.buffered.Flush(); err != nil {
- return err
- }
-
- // Sync to force fsync to disk
- if err := s.stateFile.Sync(); err != nil {
- return err
- }
-
- // Get the file size
- stat, statErr := s.stateFile.Stat()
-
- // Close the file
- if err := s.stateFile.Close(); err != nil {
- return err
- }
-
- // Set the file size, check after we close
- if statErr != nil {
- return statErr
- }
- s.meta.Size = stat.Size()
-
- // Set the CRC
- s.meta.CRC = s.stateHash.Sum(nil)
- return nil
-}
-
-// writeMeta is used to write out the metadata we have.
-func (s *FileSnapshotSink) writeMeta() error {
- // Open the meta file
- metaPath := filepath.Join(s.dir, metaFilePath)
- fh, err := os.Create(metaPath)
- if err != nil {
- return err
- }
- defer fh.Close()
-
- // Buffer the file IO
- buffered := bufio.NewWriter(fh)
-
- // Write out as JSON
- enc := json.NewEncoder(buffered)
- if err := enc.Encode(&s.meta); err != nil {
- return err
- }
-
- if err = buffered.Flush(); err != nil {
- return err
- }
-
- if err = fh.Sync(); err != nil {
- return err
- }
-
- return nil
-}
-
-// Implement the sort interface for []*fileSnapshotMeta.
-func (s snapMetaSlice) Len() int {
- return len(s)
-}
-
-func (s snapMetaSlice) Less(i, j int) bool {
- if s[i].Term != s[j].Term {
- return s[i].Term < s[j].Term
- }
- if s[i].Index != s[j].Index {
- return s[i].Index < s[j].Index
- }
- return s[i].ID < s[j].ID
-}
-
-func (s snapMetaSlice) Swap(i, j int) {
- s[i], s[j] = s[j], s[i]
-}
diff --git a/vendor/github.com/hashicorp/raft/fsm.go b/vendor/github.com/hashicorp/raft/fsm.go
deleted file mode 100644
index c89986c0fa..0000000000
--- a/vendor/github.com/hashicorp/raft/fsm.go
+++ /dev/null
@@ -1,136 +0,0 @@
-package raft
-
-import (
- "fmt"
- "io"
- "time"
-
- "github.com/armon/go-metrics"
-)
-
-// FSM provides an interface that can be implemented by
-// clients to make use of the replicated log.
-type FSM interface {
- // Apply log is invoked once a log entry is committed.
- // It returns a value which will be made available in the
- // ApplyFuture returned by Raft.Apply method if that
- // method was called on the same Raft node as the FSM.
- Apply(*Log) interface{}
-
- // Snapshot is used to support log compaction. This call should
- // return an FSMSnapshot which can be used to save a point-in-time
- // snapshot of the FSM. Apply and Snapshot are not called in multiple
- // threads, but Apply will be called concurrently with Persist. This means
- // the FSM should be implemented in a fashion that allows for concurrent
- // updates while a snapshot is happening.
- Snapshot() (FSMSnapshot, error)
-
- // Restore is used to restore an FSM from a snapshot. It is not called
- // concurrently with any other command. The FSM must discard all previous
- // state.
- Restore(io.ReadCloser) error
-}
-
-// FSMSnapshot is returned by an FSM in response to a Snapshot
-// It must be safe to invoke FSMSnapshot methods with concurrent
-// calls to Apply.
-type FSMSnapshot interface {
- // Persist should dump all necessary state to the WriteCloser 'sink',
- // and call sink.Close() when finished or call sink.Cancel() on error.
- Persist(sink SnapshotSink) error
-
- // Release is invoked when we are finished with the snapshot.
- Release()
-}
-
-// runFSM is a long running goroutine responsible for applying logs
-// to the FSM. This is done async of other logs since we don't want
-// the FSM to block our internal operations.
-func (r *Raft) runFSM() {
- var lastIndex, lastTerm uint64
-
- commit := func(req *commitTuple) {
- // Apply the log if a command
- var resp interface{}
- if req.log.Type == LogCommand {
- start := time.Now()
- resp = r.fsm.Apply(req.log)
- metrics.MeasureSince([]string{"raft", "fsm", "apply"}, start)
- }
-
- // Update the indexes
- lastIndex = req.log.Index
- lastTerm = req.log.Term
-
- // Invoke the future if given
- if req.future != nil {
- req.future.response = resp
- req.future.respond(nil)
- }
- }
-
- restore := func(req *restoreFuture) {
- // Open the snapshot
- meta, source, err := r.snapshots.Open(req.ID)
- if err != nil {
- req.respond(fmt.Errorf("failed to open snapshot %v: %v", req.ID, err))
- return
- }
-
- // Attempt to restore
- start := time.Now()
- if err := r.fsm.Restore(source); err != nil {
- req.respond(fmt.Errorf("failed to restore snapshot %v: %v", req.ID, err))
- source.Close()
- return
- }
- source.Close()
- metrics.MeasureSince([]string{"raft", "fsm", "restore"}, start)
-
- // Update the last index and term
- lastIndex = meta.Index
- lastTerm = meta.Term
- req.respond(nil)
- }
-
- snapshot := func(req *reqSnapshotFuture) {
- // Is there something to snapshot?
- if lastIndex == 0 {
- req.respond(ErrNothingNewToSnapshot)
- return
- }
-
- // Start a snapshot
- start := time.Now()
- snap, err := r.fsm.Snapshot()
- metrics.MeasureSince([]string{"raft", "fsm", "snapshot"}, start)
-
- // Respond to the request
- req.index = lastIndex
- req.term = lastTerm
- req.snapshot = snap
- req.respond(err)
- }
-
- for {
- select {
- case ptr := <-r.fsmMutateCh:
- switch req := ptr.(type) {
- case *commitTuple:
- commit(req)
-
- case *restoreFuture:
- restore(req)
-
- default:
- panic(fmt.Errorf("bad type passed to fsmMutateCh: %#v", ptr))
- }
-
- case req := <-r.fsmSnapshotCh:
- snapshot(req)
-
- case <-r.shutdownCh:
- return
- }
- }
-}
diff --git a/vendor/github.com/hashicorp/raft/future.go b/vendor/github.com/hashicorp/raft/future.go
deleted file mode 100644
index fac59a5cc4..0000000000
--- a/vendor/github.com/hashicorp/raft/future.go
+++ /dev/null
@@ -1,289 +0,0 @@
-package raft
-
-import (
- "fmt"
- "io"
- "sync"
- "time"
-)
-
-// Future is used to represent an action that may occur in the future.
-type Future interface {
- // Error blocks until the future arrives and then
- // returns the error status of the future.
- // This may be called any number of times - all
- // calls will return the same value.
- // Note that it is not OK to call this method
- // twice concurrently on the same Future instance.
- Error() error
-}
-
-// IndexFuture is used for future actions that can result in a raft log entry
-// being created.
-type IndexFuture interface {
- Future
-
- // Index holds the index of the newly applied log entry.
- // This must not be called until after the Error method has returned.
- Index() uint64
-}
-
-// ApplyFuture is used for Apply and can return the FSM response.
-type ApplyFuture interface {
- IndexFuture
-
- // Response returns the FSM response as returned
- // by the FSM.Apply method. This must not be called
- // until after the Error method has returned.
- Response() interface{}
-}
-
-// ConfigurationFuture is used for GetConfiguration and can return the
-// latest configuration in use by Raft.
-type ConfigurationFuture interface {
- IndexFuture
-
- // Configuration contains the latest configuration. This must
- // not be called until after the Error method has returned.
- Configuration() Configuration
-}
-
-// SnapshotFuture is used for waiting on a user-triggered snapshot to complete.
-type SnapshotFuture interface {
- Future
-
- // Open is a function you can call to access the underlying snapshot and
- // its metadata. This must not be called until after the Error method
- // has returned.
- Open() (*SnapshotMeta, io.ReadCloser, error)
-}
-
-// errorFuture is used to return a static error.
-type errorFuture struct {
- err error
-}
-
-func (e errorFuture) Error() error {
- return e.err
-}
-
-func (e errorFuture) Response() interface{} {
- return nil
-}
-
-func (e errorFuture) Index() uint64 {
- return 0
-}
-
-// deferError can be embedded to allow a future
-// to provide an error in the future.
-type deferError struct {
- err error
- errCh chan error
- responded bool
-}
-
-func (d *deferError) init() {
- d.errCh = make(chan error, 1)
-}
-
-func (d *deferError) Error() error {
- if d.err != nil {
- // Note that when we've received a nil error, this
- // won't trigger, but the channel is closed after
- // send so we'll still return nil below.
- return d.err
- }
- if d.errCh == nil {
- panic("waiting for response on nil channel")
- }
- d.err = <-d.errCh
- return d.err
-}
-
-func (d *deferError) respond(err error) {
- if d.errCh == nil {
- return
- }
- if d.responded {
- return
- }
- d.errCh <- err
- close(d.errCh)
- d.responded = true
-}
-
-// There are several types of requests that cause a configuration entry to
-// be appended to the log. These are encoded here for leaderLoop() to process.
-// This is internal to a single server.
-type configurationChangeFuture struct {
- logFuture
- req configurationChangeRequest
-}
-
-// bootstrapFuture is used to attempt a live bootstrap of the cluster. See the
-// Raft object's BootstrapCluster member function for more details.
-type bootstrapFuture struct {
- deferError
-
- // configuration is the proposed bootstrap configuration to apply.
- configuration Configuration
-}
-
-// logFuture is used to apply a log entry and waits until
-// the log is considered committed.
-type logFuture struct {
- deferError
- log Log
- response interface{}
- dispatch time.Time
-}
-
-func (l *logFuture) Response() interface{} {
- return l.response
-}
-
-func (l *logFuture) Index() uint64 {
- return l.log.Index
-}
-
-type shutdownFuture struct {
- raft *Raft
-}
-
-func (s *shutdownFuture) Error() error {
- if s.raft == nil {
- return nil
- }
- s.raft.waitShutdown()
- if closeable, ok := s.raft.trans.(WithClose); ok {
- closeable.Close()
- }
- return nil
-}
-
-// userSnapshotFuture is used for waiting on a user-triggered snapshot to
-// complete.
-type userSnapshotFuture struct {
- deferError
-
- // opener is a function used to open the snapshot. This is filled in
- // once the future returns with no error.
- opener func() (*SnapshotMeta, io.ReadCloser, error)
-}
-
-// Open is a function you can call to access the underlying snapshot and its
-// metadata.
-func (u *userSnapshotFuture) Open() (*SnapshotMeta, io.ReadCloser, error) {
- if u.opener == nil {
- return nil, nil, fmt.Errorf("no snapshot available")
- } else {
- // Invalidate the opener so it can't get called multiple times,
- // which isn't generally safe.
- defer func() {
- u.opener = nil
- }()
- return u.opener()
- }
-}
-
-// userRestoreFuture is used for waiting on a user-triggered restore of an
-// external snapshot to complete.
-type userRestoreFuture struct {
- deferError
-
- // meta is the metadata that belongs with the snapshot.
- meta *SnapshotMeta
-
- // reader is the interface to read the snapshot contents from.
- reader io.Reader
-}
-
-// reqSnapshotFuture is used for requesting a snapshot start.
-// It is only used internally.
-type reqSnapshotFuture struct {
- deferError
-
- // snapshot details provided by the FSM runner before responding
- index uint64
- term uint64
- snapshot FSMSnapshot
-}
-
-// restoreFuture is used for requesting an FSM to perform a
-// snapshot restore. Used internally only.
-type restoreFuture struct {
- deferError
- ID string
-}
-
-// verifyFuture is used to verify the current node is still
-// the leader. This is to prevent a stale read.
-type verifyFuture struct {
- deferError
- notifyCh chan *verifyFuture
- quorumSize int
- votes int
- voteLock sync.Mutex
-}
-
-// configurationsFuture is used to retrieve the current configurations. This is
-// used to allow safe access to this information outside of the main thread.
-type configurationsFuture struct {
- deferError
- configurations configurations
-}
-
-// Configuration returns the latest configuration in use by Raft.
-func (c *configurationsFuture) Configuration() Configuration {
- return c.configurations.latest
-}
-
-// Index returns the index of the latest configuration in use by Raft.
-func (c *configurationsFuture) Index() uint64 {
- return c.configurations.latestIndex
-}
-
-// vote is used to respond to a verifyFuture.
-// This may block when responding on the notifyCh.
-func (v *verifyFuture) vote(leader bool) {
- v.voteLock.Lock()
- defer v.voteLock.Unlock()
-
- // Guard against having notified already
- if v.notifyCh == nil {
- return
- }
-
- if leader {
- v.votes++
- if v.votes >= v.quorumSize {
- v.notifyCh <- v
- v.notifyCh = nil
- }
- } else {
- v.notifyCh <- v
- v.notifyCh = nil
- }
-}
-
-// appendFuture is used for waiting on a pipelined append
-// entries RPC.
-type appendFuture struct {
- deferError
- start time.Time
- args *AppendEntriesRequest
- resp *AppendEntriesResponse
-}
-
-func (a *appendFuture) Start() time.Time {
- return a.start
-}
-
-func (a *appendFuture) Request() *AppendEntriesRequest {
- return a.args
-}
-
-func (a *appendFuture) Response() *AppendEntriesResponse {
- return a.resp
-}
diff --git a/vendor/github.com/hashicorp/raft/go.mod b/vendor/github.com/hashicorp/raft/go.mod
deleted file mode 100644
index 09803b688f..0000000000
--- a/vendor/github.com/hashicorp/raft/go.mod
+++ /dev/null
@@ -1,10 +0,0 @@
-module github.com/hashicorp/raft
-
-go 1.12
-
-require (
- github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878
- github.com/hashicorp/go-hclog v0.9.1
- github.com/hashicorp/go-msgpack v0.5.5
- github.com/stretchr/testify v1.3.0
-)
diff --git a/vendor/github.com/hashicorp/raft/go.sum b/vendor/github.com/hashicorp/raft/go.sum
deleted file mode 100644
index b06b6a7a4f..0000000000
--- a/vendor/github.com/hashicorp/raft/go.sum
+++ /dev/null
@@ -1,37 +0,0 @@
-github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
-github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
-github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
-github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-hclog v0.9.1 h1:9PZfAcVEvez4yhLH2TBU64/h/z4xlFI80cWXRrxuKuM=
-github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
-github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
-github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
-github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
-github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
-github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
-github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
-golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
diff --git a/vendor/github.com/hashicorp/raft/inmem_snapshot.go b/vendor/github.com/hashicorp/raft/inmem_snapshot.go
deleted file mode 100644
index ad52f93aef..0000000000
--- a/vendor/github.com/hashicorp/raft/inmem_snapshot.go
+++ /dev/null
@@ -1,109 +0,0 @@
-package raft
-
-import (
- "bytes"
- "fmt"
- "io"
- "io/ioutil"
- "sync"
-)
-
-// InmemSnapshotStore implements the SnapshotStore interface and
-// retains only the most recent snapshot
-type InmemSnapshotStore struct {
- latest *InmemSnapshotSink
- hasSnapshot bool
- sync.RWMutex
-}
-
-// InmemSnapshotSink implements SnapshotSink in memory
-type InmemSnapshotSink struct {
- meta SnapshotMeta
- contents *bytes.Buffer
-}
-
-// NewInmemSnapshotStore creates a blank new InmemSnapshotStore
-func NewInmemSnapshotStore() *InmemSnapshotStore {
- return &InmemSnapshotStore{
- latest: &InmemSnapshotSink{
- contents: &bytes.Buffer{},
- },
- }
-}
-
-// Create replaces the stored snapshot with a new one using the given args
-func (m *InmemSnapshotStore) Create(version SnapshotVersion, index, term uint64,
- configuration Configuration, configurationIndex uint64, trans Transport) (SnapshotSink, error) {
- // We only support version 1 snapshots at this time.
- if version != 1 {
- return nil, fmt.Errorf("unsupported snapshot version %d", version)
- }
-
- name := snapshotName(term, index)
-
- m.Lock()
- defer m.Unlock()
-
- sink := &InmemSnapshotSink{
- meta: SnapshotMeta{
- Version: version,
- ID: name,
- Index: index,
- Term: term,
- Peers: encodePeers(configuration, trans),
- Configuration: configuration,
- ConfigurationIndex: configurationIndex,
- },
- contents: &bytes.Buffer{},
- }
- m.hasSnapshot = true
- m.latest = sink
-
- return sink, nil
-}
-
-// List returns the latest snapshot taken
-func (m *InmemSnapshotStore) List() ([]*SnapshotMeta, error) {
- m.RLock()
- defer m.RUnlock()
-
- if !m.hasSnapshot {
- return []*SnapshotMeta{}, nil
- }
- return []*SnapshotMeta{&m.latest.meta}, nil
-}
-
-// Open wraps an io.ReadCloser around the snapshot contents
-func (m *InmemSnapshotStore) Open(id string) (*SnapshotMeta, io.ReadCloser, error) {
- m.RLock()
- defer m.RUnlock()
-
- if m.latest.meta.ID != id {
- return nil, nil, fmt.Errorf("[ERR] snapshot: failed to open snapshot id: %s", id)
- }
-
- // Make a copy of the contents, since a bytes.Buffer can only be read
- // once.
- contents := bytes.NewBuffer(m.latest.contents.Bytes())
- return &m.latest.meta, ioutil.NopCloser(contents), nil
-}
-
-// Write appends the given bytes to the snapshot contents
-func (s *InmemSnapshotSink) Write(p []byte) (n int, err error) {
- written, err := io.Copy(s.contents, bytes.NewReader(p))
- s.meta.Size += written
- return int(written), err
-}
-
-// Close updates the Size and is otherwise a no-op
-func (s *InmemSnapshotSink) Close() error {
- return nil
-}
-
-func (s *InmemSnapshotSink) ID() string {
- return s.meta.ID
-}
-
-func (s *InmemSnapshotSink) Cancel() error {
- return nil
-}
diff --git a/vendor/github.com/hashicorp/raft/inmem_store.go b/vendor/github.com/hashicorp/raft/inmem_store.go
deleted file mode 100644
index 6285610f9a..0000000000
--- a/vendor/github.com/hashicorp/raft/inmem_store.go
+++ /dev/null
@@ -1,130 +0,0 @@
-package raft
-
-import (
- "errors"
- "sync"
-)
-
-// InmemStore implements the LogStore and StableStore interface.
-// It should NOT EVER be used for production. It is used only for
-// unit tests. Use the MDBStore implementation instead.
-type InmemStore struct {
- l sync.RWMutex
- lowIndex uint64
- highIndex uint64
- logs map[uint64]*Log
- kv map[string][]byte
- kvInt map[string]uint64
-}
-
-// NewInmemStore returns a new in-memory backend. Do not ever
-// use for production. Only for testing.
-func NewInmemStore() *InmemStore {
- i := &InmemStore{
- logs: make(map[uint64]*Log),
- kv: make(map[string][]byte),
- kvInt: make(map[string]uint64),
- }
- return i
-}
-
-// FirstIndex implements the LogStore interface.
-func (i *InmemStore) FirstIndex() (uint64, error) {
- i.l.RLock()
- defer i.l.RUnlock()
- return i.lowIndex, nil
-}
-
-// LastIndex implements the LogStore interface.
-func (i *InmemStore) LastIndex() (uint64, error) {
- i.l.RLock()
- defer i.l.RUnlock()
- return i.highIndex, nil
-}
-
-// GetLog implements the LogStore interface.
-func (i *InmemStore) GetLog(index uint64, log *Log) error {
- i.l.RLock()
- defer i.l.RUnlock()
- l, ok := i.logs[index]
- if !ok {
- return ErrLogNotFound
- }
- *log = *l
- return nil
-}
-
-// StoreLog implements the LogStore interface.
-func (i *InmemStore) StoreLog(log *Log) error {
- return i.StoreLogs([]*Log{log})
-}
-
-// StoreLogs implements the LogStore interface.
-func (i *InmemStore) StoreLogs(logs []*Log) error {
- i.l.Lock()
- defer i.l.Unlock()
- for _, l := range logs {
- i.logs[l.Index] = l
- if i.lowIndex == 0 {
- i.lowIndex = l.Index
- }
- if l.Index > i.highIndex {
- i.highIndex = l.Index
- }
- }
- return nil
-}
-
-// DeleteRange implements the LogStore interface.
-func (i *InmemStore) DeleteRange(min, max uint64) error {
- i.l.Lock()
- defer i.l.Unlock()
- for j := min; j <= max; j++ {
- delete(i.logs, j)
- }
- if min <= i.lowIndex {
- i.lowIndex = max + 1
- }
- if max >= i.highIndex {
- i.highIndex = min - 1
- }
- if i.lowIndex > i.highIndex {
- i.lowIndex = 0
- i.highIndex = 0
- }
- return nil
-}
-
-// Set implements the StableStore interface.
-func (i *InmemStore) Set(key []byte, val []byte) error {
- i.l.Lock()
- defer i.l.Unlock()
- i.kv[string(key)] = val
- return nil
-}
-
-// Get implements the StableStore interface.
-func (i *InmemStore) Get(key []byte) ([]byte, error) {
- i.l.RLock()
- defer i.l.RUnlock()
- val := i.kv[string(key)]
- if val == nil {
- return nil, errors.New("not found")
- }
- return val, nil
-}
-
-// SetUint64 implements the StableStore interface.
-func (i *InmemStore) SetUint64(key []byte, val uint64) error {
- i.l.Lock()
- defer i.l.Unlock()
- i.kvInt[string(key)] = val
- return nil
-}
-
-// GetUint64 implements the StableStore interface.
-func (i *InmemStore) GetUint64(key []byte) (uint64, error) {
- i.l.RLock()
- defer i.l.RUnlock()
- return i.kvInt[string(key)], nil
-}
diff --git a/vendor/github.com/hashicorp/raft/inmem_transport.go b/vendor/github.com/hashicorp/raft/inmem_transport.go
deleted file mode 100644
index bb42eeb68b..0000000000
--- a/vendor/github.com/hashicorp/raft/inmem_transport.go
+++ /dev/null
@@ -1,335 +0,0 @@
-package raft
-
-import (
- "fmt"
- "io"
- "sync"
- "time"
-)
-
-// NewInmemAddr returns a new in-memory addr with
-// a randomly generate UUID as the ID.
-func NewInmemAddr() ServerAddress {
- return ServerAddress(generateUUID())
-}
-
-// inmemPipeline is used to pipeline requests for the in-mem transport.
-type inmemPipeline struct {
- trans *InmemTransport
- peer *InmemTransport
- peerAddr ServerAddress
-
- doneCh chan AppendFuture
- inprogressCh chan *inmemPipelineInflight
-
- shutdown bool
- shutdownCh chan struct{}
- shutdownLock sync.Mutex
-}
-
-type inmemPipelineInflight struct {
- future *appendFuture
- respCh <-chan RPCResponse
-}
-
-// InmemTransport Implements the Transport interface, to allow Raft to be
-// tested in-memory without going over a network.
-type InmemTransport struct {
- sync.RWMutex
- consumerCh chan RPC
- localAddr ServerAddress
- peers map[ServerAddress]*InmemTransport
- pipelines []*inmemPipeline
- timeout time.Duration
-}
-
-// NewInmemTransportWithTimeout is used to initialize a new transport and
-// generates a random local address if none is specified. The given timeout
-// will be used to decide how long to wait for a connected peer to process the
-// RPCs that we're sending it. See also Connect() and Consumer().
-func NewInmemTransportWithTimeout(addr ServerAddress, timeout time.Duration) (ServerAddress, *InmemTransport) {
- if string(addr) == "" {
- addr = NewInmemAddr()
- }
- trans := &InmemTransport{
- consumerCh: make(chan RPC, 16),
- localAddr: addr,
- peers: make(map[ServerAddress]*InmemTransport),
- timeout: timeout,
- }
- return addr, trans
-}
-
-// NewInmemTransport is used to initialize a new transport
-// and generates a random local address if none is specified
-func NewInmemTransport(addr ServerAddress) (ServerAddress, *InmemTransport) {
- return NewInmemTransportWithTimeout(addr, 50*time.Millisecond)
-}
-
-// SetHeartbeatHandler is used to set optional fast-path for
-// heartbeats, not supported for this transport.
-func (i *InmemTransport) SetHeartbeatHandler(cb func(RPC)) {
-}
-
-// Consumer implements the Transport interface.
-func (i *InmemTransport) Consumer() <-chan RPC {
- return i.consumerCh
-}
-
-// LocalAddr implements the Transport interface.
-func (i *InmemTransport) LocalAddr() ServerAddress {
- return i.localAddr
-}
-
-// AppendEntriesPipeline returns an interface that can be used to pipeline
-// AppendEntries requests.
-func (i *InmemTransport) AppendEntriesPipeline(id ServerID, target ServerAddress) (AppendPipeline, error) {
- i.Lock()
- defer i.Unlock()
-
- peer, ok := i.peers[target]
- if !ok {
- return nil, fmt.Errorf("failed to connect to peer: %v", target)
- }
- pipeline := newInmemPipeline(i, peer, target)
- i.pipelines = append(i.pipelines, pipeline)
- return pipeline, nil
-}
-
-// AppendEntries implements the Transport interface.
-func (i *InmemTransport) AppendEntries(id ServerID, target ServerAddress, args *AppendEntriesRequest, resp *AppendEntriesResponse) error {
- rpcResp, err := i.makeRPC(target, args, nil, i.timeout)
- if err != nil {
- return err
- }
-
- // Copy the result back
- out := rpcResp.Response.(*AppendEntriesResponse)
- *resp = *out
- return nil
-}
-
-// RequestVote implements the Transport interface.
-func (i *InmemTransport) RequestVote(id ServerID, target ServerAddress, args *RequestVoteRequest, resp *RequestVoteResponse) error {
- rpcResp, err := i.makeRPC(target, args, nil, i.timeout)
- if err != nil {
- return err
- }
-
- // Copy the result back
- out := rpcResp.Response.(*RequestVoteResponse)
- *resp = *out
- return nil
-}
-
-// InstallSnapshot implements the Transport interface.
-func (i *InmemTransport) InstallSnapshot(id ServerID, target ServerAddress, args *InstallSnapshotRequest, resp *InstallSnapshotResponse, data io.Reader) error {
- rpcResp, err := i.makeRPC(target, args, data, 10*i.timeout)
- if err != nil {
- return err
- }
-
- // Copy the result back
- out := rpcResp.Response.(*InstallSnapshotResponse)
- *resp = *out
- return nil
-}
-
-func (i *InmemTransport) makeRPC(target ServerAddress, args interface{}, r io.Reader, timeout time.Duration) (rpcResp RPCResponse, err error) {
- i.RLock()
- peer, ok := i.peers[target]
- i.RUnlock()
-
- if !ok {
- err = fmt.Errorf("failed to connect to peer: %v", target)
- return
- }
-
- // Send the RPC over
- respCh := make(chan RPCResponse)
- req := RPC{
- Command: args,
- Reader: r,
- RespChan: respCh,
- }
- select {
- case peer.consumerCh <- req:
- case <-time.After(timeout):
- err = fmt.Errorf("send timed out")
- return
- }
-
- // Wait for a response
- select {
- case rpcResp = <-respCh:
- if rpcResp.Error != nil {
- err = rpcResp.Error
- }
- case <-time.After(timeout):
- err = fmt.Errorf("command timed out")
- }
- return
-}
-
-// EncodePeer implements the Transport interface.
-func (i *InmemTransport) EncodePeer(id ServerID, p ServerAddress) []byte {
- return []byte(p)
-}
-
-// DecodePeer implements the Transport interface.
-func (i *InmemTransport) DecodePeer(buf []byte) ServerAddress {
- return ServerAddress(buf)
-}
-
-// Connect is used to connect this transport to another transport for
-// a given peer name. This allows for local routing.
-func (i *InmemTransport) Connect(peer ServerAddress, t Transport) {
- trans := t.(*InmemTransport)
- i.Lock()
- defer i.Unlock()
- i.peers[peer] = trans
-}
-
-// Disconnect is used to remove the ability to route to a given peer.
-func (i *InmemTransport) Disconnect(peer ServerAddress) {
- i.Lock()
- defer i.Unlock()
- delete(i.peers, peer)
-
- // Disconnect any pipelines
- n := len(i.pipelines)
- for idx := 0; idx < n; idx++ {
- if i.pipelines[idx].peerAddr == peer {
- i.pipelines[idx].Close()
- i.pipelines[idx], i.pipelines[n-1] = i.pipelines[n-1], nil
- idx--
- n--
- }
- }
- i.pipelines = i.pipelines[:n]
-}
-
-// DisconnectAll is used to remove all routes to peers.
-func (i *InmemTransport) DisconnectAll() {
- i.Lock()
- defer i.Unlock()
- i.peers = make(map[ServerAddress]*InmemTransport)
-
- // Handle pipelines
- for _, pipeline := range i.pipelines {
- pipeline.Close()
- }
- i.pipelines = nil
-}
-
-// Close is used to permanently disable the transport
-func (i *InmemTransport) Close() error {
- i.DisconnectAll()
- return nil
-}
-
-func newInmemPipeline(trans *InmemTransport, peer *InmemTransport, addr ServerAddress) *inmemPipeline {
- i := &inmemPipeline{
- trans: trans,
- peer: peer,
- peerAddr: addr,
- doneCh: make(chan AppendFuture, 16),
- inprogressCh: make(chan *inmemPipelineInflight, 16),
- shutdownCh: make(chan struct{}),
- }
- go i.decodeResponses()
- return i
-}
-
-func (i *inmemPipeline) decodeResponses() {
- timeout := i.trans.timeout
- for {
- select {
- case inp := <-i.inprogressCh:
- var timeoutCh <-chan time.Time
- if timeout > 0 {
- timeoutCh = time.After(timeout)
- }
-
- select {
- case rpcResp := <-inp.respCh:
- // Copy the result back
- *inp.future.resp = *rpcResp.Response.(*AppendEntriesResponse)
- inp.future.respond(rpcResp.Error)
-
- select {
- case i.doneCh <- inp.future:
- case <-i.shutdownCh:
- return
- }
-
- case <-timeoutCh:
- inp.future.respond(fmt.Errorf("command timed out"))
- select {
- case i.doneCh <- inp.future:
- case <-i.shutdownCh:
- return
- }
-
- case <-i.shutdownCh:
- return
- }
- case <-i.shutdownCh:
- return
- }
- }
-}
-
-func (i *inmemPipeline) AppendEntries(args *AppendEntriesRequest, resp *AppendEntriesResponse) (AppendFuture, error) {
- // Create a new future
- future := &appendFuture{
- start: time.Now(),
- args: args,
- resp: resp,
- }
- future.init()
-
- // Handle a timeout
- var timeout <-chan time.Time
- if i.trans.timeout > 0 {
- timeout = time.After(i.trans.timeout)
- }
-
- // Send the RPC over
- respCh := make(chan RPCResponse, 1)
- rpc := RPC{
- Command: args,
- RespChan: respCh,
- }
- select {
- case i.peer.consumerCh <- rpc:
- case <-timeout:
- return nil, fmt.Errorf("command enqueue timeout")
- case <-i.shutdownCh:
- return nil, ErrPipelineShutdown
- }
-
- // Send to be decoded
- select {
- case i.inprogressCh <- &inmemPipelineInflight{future, respCh}:
- return future, nil
- case <-i.shutdownCh:
- return nil, ErrPipelineShutdown
- }
-}
-
-func (i *inmemPipeline) Consumer() <-chan AppendFuture {
- return i.doneCh
-}
-
-func (i *inmemPipeline) Close() error {
- i.shutdownLock.Lock()
- defer i.shutdownLock.Unlock()
- if i.shutdown {
- return nil
- }
-
- i.shutdown = true
- close(i.shutdownCh)
- return nil
-}
diff --git a/vendor/github.com/hashicorp/raft/log.go b/vendor/github.com/hashicorp/raft/log.go
deleted file mode 100644
index 4ade38ecc1..0000000000
--- a/vendor/github.com/hashicorp/raft/log.go
+++ /dev/null
@@ -1,72 +0,0 @@
-package raft
-
-// LogType describes various types of log entries.
-type LogType uint8
-
-const (
- // LogCommand is applied to a user FSM.
- LogCommand LogType = iota
-
- // LogNoop is used to assert leadership.
- LogNoop
-
- // LogAddPeer is used to add a new peer. This should only be used with
- // older protocol versions designed to be compatible with unversioned
- // Raft servers. See comments in config.go for details.
- LogAddPeerDeprecated
-
- // LogRemovePeer is used to remove an existing peer. This should only be
- // used with older protocol versions designed to be compatible with
- // unversioned Raft servers. See comments in config.go for details.
- LogRemovePeerDeprecated
-
- // LogBarrier is used to ensure all preceding operations have been
- // applied to the FSM. It is similar to LogNoop, but instead of returning
- // once committed, it only returns once the FSM manager acks it. Otherwise
- // it is possible there are operations committed but not yet applied to
- // the FSM.
- LogBarrier
-
- // LogConfiguration establishes a membership change configuration. It is
- // created when a server is added, removed, promoted, etc. Only used
- // when protocol version 1 or greater is in use.
- LogConfiguration
-)
-
-// Log entries are replicated to all members of the Raft cluster
-// and form the heart of the replicated state machine.
-type Log struct {
- // Index holds the index of the log entry.
- Index uint64
-
- // Term holds the election term of the log entry.
- Term uint64
-
- // Type holds the type of the log entry.
- Type LogType
-
- // Data holds the log entry's type-specific data.
- Data []byte
-}
-
-// LogStore is used to provide an interface for storing
-// and retrieving logs in a durable fashion.
-type LogStore interface {
- // FirstIndex returns the first index written. 0 for no entries.
- FirstIndex() (uint64, error)
-
- // LastIndex returns the last index written. 0 for no entries.
- LastIndex() (uint64, error)
-
- // GetLog gets a log entry at a given index.
- GetLog(index uint64, log *Log) error
-
- // StoreLog stores a log entry.
- StoreLog(log *Log) error
-
- // StoreLogs stores multiple log entries.
- StoreLogs(logs []*Log) error
-
- // DeleteRange deletes a range of log entries. The range is inclusive.
- DeleteRange(min, max uint64) error
-}
diff --git a/vendor/github.com/hashicorp/raft/log_cache.go b/vendor/github.com/hashicorp/raft/log_cache.go
deleted file mode 100644
index 952e98c228..0000000000
--- a/vendor/github.com/hashicorp/raft/log_cache.go
+++ /dev/null
@@ -1,79 +0,0 @@
-package raft
-
-import (
- "fmt"
- "sync"
-)
-
-// LogCache wraps any LogStore implementation to provide an
-// in-memory ring buffer. This is used to cache access to
-// the recently written entries. For implementations that do not
-// cache themselves, this can provide a substantial boost by
-// avoiding disk I/O on recent entries.
-type LogCache struct {
- store LogStore
-
- cache []*Log
- l sync.RWMutex
-}
-
-// NewLogCache is used to create a new LogCache with the
-// given capacity and backend store.
-func NewLogCache(capacity int, store LogStore) (*LogCache, error) {
- if capacity <= 0 {
- return nil, fmt.Errorf("capacity must be positive")
- }
- c := &LogCache{
- store: store,
- cache: make([]*Log, capacity),
- }
- return c, nil
-}
-
-func (c *LogCache) GetLog(idx uint64, log *Log) error {
- // Check the buffer for an entry
- c.l.RLock()
- cached := c.cache[idx%uint64(len(c.cache))]
- c.l.RUnlock()
-
- // Check if entry is valid
- if cached != nil && cached.Index == idx {
- *log = *cached
- return nil
- }
-
- // Forward request on cache miss
- return c.store.GetLog(idx, log)
-}
-
-func (c *LogCache) StoreLog(log *Log) error {
- return c.StoreLogs([]*Log{log})
-}
-
-func (c *LogCache) StoreLogs(logs []*Log) error {
- // Insert the logs into the ring buffer
- c.l.Lock()
- for _, l := range logs {
- c.cache[l.Index%uint64(len(c.cache))] = l
- }
- c.l.Unlock()
-
- return c.store.StoreLogs(logs)
-}
-
-func (c *LogCache) FirstIndex() (uint64, error) {
- return c.store.FirstIndex()
-}
-
-func (c *LogCache) LastIndex() (uint64, error) {
- return c.store.LastIndex()
-}
-
-func (c *LogCache) DeleteRange(min, max uint64) error {
- // Invalidate the cache on deletes
- c.l.Lock()
- c.cache = make([]*Log, len(c.cache))
- c.l.Unlock()
-
- return c.store.DeleteRange(min, max)
-}
diff --git a/vendor/github.com/hashicorp/raft/membership.md b/vendor/github.com/hashicorp/raft/membership.md
deleted file mode 100644
index df1f83e27f..0000000000
--- a/vendor/github.com/hashicorp/raft/membership.md
+++ /dev/null
@@ -1,83 +0,0 @@
-Simon (@superfell) and I (@ongardie) talked through reworking this library's cluster membership changes last Friday. We don't see a way to split this into independent patches, so we're taking the next best approach: submitting the plan here for review, then working on an enormous PR. Your feedback would be appreciated. (@superfell is out this week, however, so don't expect him to respond quickly.)
-
-These are the main goals:
- - Bringing things in line with the description in my PhD dissertation;
- - Catching up new servers prior to granting them a vote, as well as allowing permanent non-voting members; and
- - Eliminating the `peers.json` file, to avoid issues of consistency between that and the log/snapshot.
-
-## Data-centric view
-
-We propose to re-define a *configuration* as a set of servers, where each server includes an address (as it does today) and a mode that is either:
- - *Voter*: a server whose vote is counted in elections and whose match index is used in advancing the leader's commit index.
- - *Nonvoter*: a server that receives log entries but is not considered for elections or commitment purposes.
- - *Staging*: a server that acts like a nonvoter with one exception: once a staging server receives enough log entries to catch up sufficiently to the leader's log, the leader will invoke a membership change to change the staging server to a voter.
-
-All changes to the configuration will be done by writing a new configuration to the log. The new configuration will be in affect as soon as it is appended to the log (not when it is committed like a normal state machine command). Note that, per my dissertation, there can be at most one uncommitted configuration at a time (the next configuration may not be created until the prior one has been committed). It's not strictly necessary to follow these same rules for the nonvoter/staging servers, but we think its best to treat all changes uniformly.
-
-Each server will track two configurations:
- 1. its *committed configuration*: the latest configuration in the log/snapshot that has been committed, along with its index.
- 2. its *latest configuration*: the latest configuration in the log/snapshot (may be committed or uncommitted), along with its index.
-
-When there's no membership change happening, these two will be the same. The latest configuration is almost always the one used, except:
- - When followers truncate the suffix of their logs, they may need to fall back to the committed configuration.
- - When snapshotting, the committed configuration is written, to correspond with the committed log prefix that is being snapshotted.
-
-
-## Application API
-
-We propose the following operations for clients to manipulate the cluster configuration:
- - AddVoter: server becomes staging unless voter,
- - AddNonvoter: server becomes nonvoter unless staging or voter,
- - DemoteVoter: server becomes nonvoter unless absent,
- - RemovePeer: server removed from configuration,
- - GetConfiguration: waits for latest config to commit, returns committed config.
-
-This diagram, of which I'm quite proud, shows the possible transitions:
-```
-+-----------------------------------------------------------------------------+
-| |
-| Start -> +--------+ |
-| ,------<------------| | |
-| / | absent | |
-| / RemovePeer--> | | <---RemovePeer |
-| / | +--------+ \ |
-| / | | \ |
-| AddNonvoter | AddVoter \ |
-| | ,->---' `--<-. | \ |
-| v / \ v \ |
-| +----------+ +----------+ +----------+ |
-| | | ---AddVoter--> | | -log caught up --> | | |
-| | nonvoter | | staging | | voter | |
-| | | <-DemoteVoter- | | ,- | | |
-| +----------+ \ +----------+ / +----------+ |
-| \ / |
-| `--------------<---------------' |
-| |
-+-----------------------------------------------------------------------------+
-```
-
-While these operations aren't quite symmetric, we think they're a good set to capture
-the possible intent of the user. For example, if I want to make sure a server doesn't have a vote, but the server isn't part of the configuration at all, it probably shouldn't be added as a nonvoting server.
-
-Each of these application-level operations will be interpreted by the leader and, if it has an effect, will cause the leader to write a new configuration entry to its log. Which particular application-level operation caused the log entry to be written need not be part of the log entry.
-
-## Code implications
-
-This is a non-exhaustive list, but we came up with a few things:
-- Remove the PeerStore: the `peers.json` file introduces the possibility of getting out of sync with the log and snapshot, and it's hard to maintain this atomically as the log changes. It's not clear whether it's meant to track the committed or latest configuration, either.
-- Servers will have to search their snapshot and log to find the committed configuration and the latest configuration on startup.
-- Bootstrap will no longer use `peers.json` but should initialize the log or snapshot with an application-provided configuration entry.
-- Snapshots should store the index of their configuration along with the configuration itself. In my experience with LogCabin, the original log index of the configuration is very useful to include in debug log messages.
-- As noted in hashicorp/raft#84, configuration change requests should come in via a separate channel, and one may not proceed until the last has been committed.
-- As to deciding when a log is sufficiently caught up, implementing a sophisticated algorithm *is* something that can be done in a separate PR. An easy and decent placeholder is: once the staging server has reached 95% of the leader's commit index, promote it.
-
-## Feedback
-
-Again, we're looking for feedback here before we start working on this. Here are some questions to think about:
- - Does this seem like where we want things to go?
- - Is there anything here that should be left out?
- - Is there anything else we're forgetting about?
- - Is there a good way to break this up?
- - What do we need to worry about in terms of backwards compatibility?
- - What implication will this have on current tests?
- - What's the best way to test this code, in particular the small changes that will be sprinkled all over the library?
diff --git a/vendor/github.com/hashicorp/raft/net_transport.go b/vendor/github.com/hashicorp/raft/net_transport.go
deleted file mode 100644
index 4f1f101e00..0000000000
--- a/vendor/github.com/hashicorp/raft/net_transport.go
+++ /dev/null
@@ -1,757 +0,0 @@
-package raft
-
-import (
- "bufio"
- "context"
- "errors"
- "fmt"
- "io"
- "log"
- "net"
- "os"
- "sync"
- "time"
-
- "github.com/hashicorp/go-msgpack/codec"
-)
-
-const (
- rpcAppendEntries uint8 = iota
- rpcRequestVote
- rpcInstallSnapshot
-
- // DefaultTimeoutScale is the default TimeoutScale in a NetworkTransport.
- DefaultTimeoutScale = 256 * 1024 // 256KB
-
- // rpcMaxPipeline controls the maximum number of outstanding
- // AppendEntries RPC calls.
- rpcMaxPipeline = 128
-)
-
-var (
- // ErrTransportShutdown is returned when operations on a transport are
- // invoked after it's been terminated.
- ErrTransportShutdown = errors.New("transport shutdown")
-
- // ErrPipelineShutdown is returned when the pipeline is closed.
- ErrPipelineShutdown = errors.New("append pipeline closed")
-)
-
-/*
-
-NetworkTransport provides a network based transport that can be
-used to communicate with Raft on remote machines. It requires
-an underlying stream layer to provide a stream abstraction, which can
-be simple TCP, TLS, etc.
-
-This transport is very simple and lightweight. Each RPC request is
-framed by sending a byte that indicates the message type, followed
-by the MsgPack encoded request.
-
-The response is an error string followed by the response object,
-both are encoded using MsgPack.
-
-InstallSnapshot is special, in that after the RPC request we stream
-the entire state. That socket is not re-used as the connection state
-is not known if there is an error.
-
-*/
-type NetworkTransport struct {
- connPool map[ServerAddress][]*netConn
- connPoolLock sync.Mutex
-
- consumeCh chan RPC
-
- heartbeatFn func(RPC)
- heartbeatFnLock sync.Mutex
-
- logger *log.Logger
-
- maxPool int
-
- serverAddressProvider ServerAddressProvider
-
- shutdown bool
- shutdownCh chan struct{}
- shutdownLock sync.Mutex
-
- stream StreamLayer
-
- // streamCtx is used to cancel existing connection handlers.
- streamCtx context.Context
- streamCancel context.CancelFunc
- streamCtxLock sync.RWMutex
-
- timeout time.Duration
- TimeoutScale int
-}
-
-// NetworkTransportConfig encapsulates configuration for the network transport layer.
-type NetworkTransportConfig struct {
- // ServerAddressProvider is used to override the target address when establishing a connection to invoke an RPC
- ServerAddressProvider ServerAddressProvider
-
- Logger *log.Logger
-
- // Dialer
- Stream StreamLayer
-
- // MaxPool controls how many connections we will pool
- MaxPool int
-
- // Timeout is used to apply I/O deadlines. For InstallSnapshot, we multiply
- // the timeout by (SnapshotSize / TimeoutScale).
- Timeout time.Duration
-}
-
-type ServerAddressProvider interface {
- ServerAddr(id ServerID) (ServerAddress, error)
-}
-
-// StreamLayer is used with the NetworkTransport to provide
-// the low level stream abstraction.
-type StreamLayer interface {
- net.Listener
-
- // Dial is used to create a new outgoing connection
- Dial(address ServerAddress, timeout time.Duration) (net.Conn, error)
-}
-
-type netConn struct {
- target ServerAddress
- conn net.Conn
- r *bufio.Reader
- w *bufio.Writer
- dec *codec.Decoder
- enc *codec.Encoder
-}
-
-func (n *netConn) Release() error {
- return n.conn.Close()
-}
-
-type netPipeline struct {
- conn *netConn
- trans *NetworkTransport
-
- doneCh chan AppendFuture
- inprogressCh chan *appendFuture
-
- shutdown bool
- shutdownCh chan struct{}
- shutdownLock sync.Mutex
-}
-
-// NewNetworkTransportWithConfig creates a new network transport with the given config struct
-func NewNetworkTransportWithConfig(
- config *NetworkTransportConfig,
-) *NetworkTransport {
- if config.Logger == nil {
- config.Logger = log.New(os.Stderr, "", log.LstdFlags)
- }
- trans := &NetworkTransport{
- connPool: make(map[ServerAddress][]*netConn),
- consumeCh: make(chan RPC),
- logger: config.Logger,
- maxPool: config.MaxPool,
- shutdownCh: make(chan struct{}),
- stream: config.Stream,
- timeout: config.Timeout,
- TimeoutScale: DefaultTimeoutScale,
- serverAddressProvider: config.ServerAddressProvider,
- }
-
- // Create the connection context and then start our listener.
- trans.setupStreamContext()
- go trans.listen()
-
- return trans
-}
-
-// NewNetworkTransport creates a new network transport with the given dialer
-// and listener. The maxPool controls how many connections we will pool. The
-// timeout is used to apply I/O deadlines. For InstallSnapshot, we multiply
-// the timeout by (SnapshotSize / TimeoutScale).
-func NewNetworkTransport(
- stream StreamLayer,
- maxPool int,
- timeout time.Duration,
- logOutput io.Writer,
-) *NetworkTransport {
- if logOutput == nil {
- logOutput = os.Stderr
- }
- logger := log.New(logOutput, "", log.LstdFlags)
- config := &NetworkTransportConfig{Stream: stream, MaxPool: maxPool, Timeout: timeout, Logger: logger}
- return NewNetworkTransportWithConfig(config)
-}
-
-// NewNetworkTransportWithLogger creates a new network transport with the given logger, dialer
-// and listener. The maxPool controls how many connections we will pool. The
-// timeout is used to apply I/O deadlines. For InstallSnapshot, we multiply
-// the timeout by (SnapshotSize / TimeoutScale).
-func NewNetworkTransportWithLogger(
- stream StreamLayer,
- maxPool int,
- timeout time.Duration,
- logger *log.Logger,
-) *NetworkTransport {
- config := &NetworkTransportConfig{Stream: stream, MaxPool: maxPool, Timeout: timeout, Logger: logger}
- return NewNetworkTransportWithConfig(config)
-}
-
-// setupStreamContext is used to create a new stream context. This should be
-// called with the stream lock held.
-func (n *NetworkTransport) setupStreamContext() {
- ctx, cancel := context.WithCancel(context.Background())
- n.streamCtx = ctx
- n.streamCancel = cancel
-}
-
-// getStreamContext is used retrieve the current stream context.
-func (n *NetworkTransport) getStreamContext() context.Context {
- n.streamCtxLock.RLock()
- defer n.streamCtxLock.RUnlock()
- return n.streamCtx
-}
-
-// SetHeartbeatHandler is used to setup a heartbeat handler
-// as a fast-pass. This is to avoid head-of-line blocking from
-// disk IO.
-func (n *NetworkTransport) SetHeartbeatHandler(cb func(rpc RPC)) {
- n.heartbeatFnLock.Lock()
- defer n.heartbeatFnLock.Unlock()
- n.heartbeatFn = cb
-}
-
-// CloseStreams closes the current streams.
-func (n *NetworkTransport) CloseStreams() {
- n.connPoolLock.Lock()
- defer n.connPoolLock.Unlock()
-
- // Close all the connections in the connection pool and then remove their
- // entry.
- for k, e := range n.connPool {
- for _, conn := range e {
- conn.Release()
- }
-
- delete(n.connPool, k)
- }
-
- // Cancel the existing connections and create a new context. Both these
- // operations must always be done with the lock held otherwise we can create
- // connection handlers that are holding a context that will never be
- // cancelable.
- n.streamCtxLock.Lock()
- n.streamCancel()
- n.setupStreamContext()
- n.streamCtxLock.Unlock()
-}
-
-// Close is used to stop the network transport.
-func (n *NetworkTransport) Close() error {
- n.shutdownLock.Lock()
- defer n.shutdownLock.Unlock()
-
- if !n.shutdown {
- close(n.shutdownCh)
- n.stream.Close()
- n.shutdown = true
- }
- return nil
-}
-
-// Consumer implements the Transport interface.
-func (n *NetworkTransport) Consumer() <-chan RPC {
- return n.consumeCh
-}
-
-// LocalAddr implements the Transport interface.
-func (n *NetworkTransport) LocalAddr() ServerAddress {
- return ServerAddress(n.stream.Addr().String())
-}
-
-// IsShutdown is used to check if the transport is shutdown.
-func (n *NetworkTransport) IsShutdown() bool {
- select {
- case <-n.shutdownCh:
- return true
- default:
- return false
- }
-}
-
-// getExistingConn is used to grab a pooled connection.
-func (n *NetworkTransport) getPooledConn(target ServerAddress) *netConn {
- n.connPoolLock.Lock()
- defer n.connPoolLock.Unlock()
-
- conns, ok := n.connPool[target]
- if !ok || len(conns) == 0 {
- return nil
- }
-
- var conn *netConn
- num := len(conns)
- conn, conns[num-1] = conns[num-1], nil
- n.connPool[target] = conns[:num-1]
- return conn
-}
-
-// getConnFromAddressProvider returns a connection from the server address provider if available, or defaults to a connection using the target server address
-func (n *NetworkTransport) getConnFromAddressProvider(id ServerID, target ServerAddress) (*netConn, error) {
- address := n.getProviderAddressOrFallback(id, target)
- return n.getConn(address)
-}
-
-func (n *NetworkTransport) getProviderAddressOrFallback(id ServerID, target ServerAddress) ServerAddress {
- if n.serverAddressProvider != nil {
- serverAddressOverride, err := n.serverAddressProvider.ServerAddr(id)
- if err != nil {
- n.logger.Printf("[WARN] raft: Unable to get address for server id %v, using fallback address %v: %v", id, target, err)
- } else {
- return serverAddressOverride
- }
- }
- return target
-}
-
-// getConn is used to get a connection from the pool.
-func (n *NetworkTransport) getConn(target ServerAddress) (*netConn, error) {
- // Check for a pooled conn
- if conn := n.getPooledConn(target); conn != nil {
- return conn, nil
- }
-
- // Dial a new connection
- conn, err := n.stream.Dial(target, n.timeout)
- if err != nil {
- return nil, err
- }
-
- // Wrap the conn
- netConn := &netConn{
- target: target,
- conn: conn,
- r: bufio.NewReader(conn),
- w: bufio.NewWriter(conn),
- }
-
- // Setup encoder/decoders
- netConn.dec = codec.NewDecoder(netConn.r, &codec.MsgpackHandle{})
- netConn.enc = codec.NewEncoder(netConn.w, &codec.MsgpackHandle{})
-
- // Done
- return netConn, nil
-}
-
-// returnConn returns a connection back to the pool.
-func (n *NetworkTransport) returnConn(conn *netConn) {
- n.connPoolLock.Lock()
- defer n.connPoolLock.Unlock()
-
- key := conn.target
- conns, _ := n.connPool[key]
-
- if !n.IsShutdown() && len(conns) < n.maxPool {
- n.connPool[key] = append(conns, conn)
- } else {
- conn.Release()
- }
-}
-
-// AppendEntriesPipeline returns an interface that can be used to pipeline
-// AppendEntries requests.
-func (n *NetworkTransport) AppendEntriesPipeline(id ServerID, target ServerAddress) (AppendPipeline, error) {
- // Get a connection
- conn, err := n.getConnFromAddressProvider(id, target)
- if err != nil {
- return nil, err
- }
-
- // Create the pipeline
- return newNetPipeline(n, conn), nil
-}
-
-// AppendEntries implements the Transport interface.
-func (n *NetworkTransport) AppendEntries(id ServerID, target ServerAddress, args *AppendEntriesRequest, resp *AppendEntriesResponse) error {
- return n.genericRPC(id, target, rpcAppendEntries, args, resp)
-}
-
-// RequestVote implements the Transport interface.
-func (n *NetworkTransport) RequestVote(id ServerID, target ServerAddress, args *RequestVoteRequest, resp *RequestVoteResponse) error {
- return n.genericRPC(id, target, rpcRequestVote, args, resp)
-}
-
-// genericRPC handles a simple request/response RPC.
-func (n *NetworkTransport) genericRPC(id ServerID, target ServerAddress, rpcType uint8, args interface{}, resp interface{}) error {
- // Get a conn
- conn, err := n.getConnFromAddressProvider(id, target)
- if err != nil {
- return err
- }
-
- // Set a deadline
- if n.timeout > 0 {
- conn.conn.SetDeadline(time.Now().Add(n.timeout))
- }
-
- // Send the RPC
- if err = sendRPC(conn, rpcType, args); err != nil {
- return err
- }
-
- // Decode the response
- canReturn, err := decodeResponse(conn, resp)
- if canReturn {
- n.returnConn(conn)
- }
- return err
-}
-
-// InstallSnapshot implements the Transport interface.
-func (n *NetworkTransport) InstallSnapshot(id ServerID, target ServerAddress, args *InstallSnapshotRequest, resp *InstallSnapshotResponse, data io.Reader) error {
- // Get a conn, always close for InstallSnapshot
- conn, err := n.getConnFromAddressProvider(id, target)
- if err != nil {
- return err
- }
- defer conn.Release()
-
- // Set a deadline, scaled by request size
- if n.timeout > 0 {
- timeout := n.timeout * time.Duration(args.Size/int64(n.TimeoutScale))
- if timeout < n.timeout {
- timeout = n.timeout
- }
- conn.conn.SetDeadline(time.Now().Add(timeout))
- }
-
- // Send the RPC
- if err = sendRPC(conn, rpcInstallSnapshot, args); err != nil {
- return err
- }
-
- // Stream the state
- if _, err = io.Copy(conn.w, data); err != nil {
- return err
- }
-
- // Flush
- if err = conn.w.Flush(); err != nil {
- return err
- }
-
- // Decode the response, do not return conn
- _, err = decodeResponse(conn, resp)
- return err
-}
-
-// EncodePeer implements the Transport interface.
-func (n *NetworkTransport) EncodePeer(id ServerID, p ServerAddress) []byte {
- address := n.getProviderAddressOrFallback(id, p)
- return []byte(address)
-}
-
-// DecodePeer implements the Transport interface.
-func (n *NetworkTransport) DecodePeer(buf []byte) ServerAddress {
- return ServerAddress(buf)
-}
-
-// listen is used to handling incoming connections.
-func (n *NetworkTransport) listen() {
- const baseDelay = 5 * time.Millisecond
- const maxDelay = 1 * time.Second
-
- var loopDelay time.Duration
- for {
- // Accept incoming connections
- conn, err := n.stream.Accept()
- if err != nil {
- if loopDelay == 0 {
- loopDelay = baseDelay
- } else {
- loopDelay *= 2
- }
-
- if loopDelay > maxDelay {
- loopDelay = maxDelay
- }
-
- if !n.IsShutdown() {
- n.logger.Printf("[ERR] raft-net: Failed to accept connection: %v", err)
- }
-
- select {
- case <-n.shutdownCh:
- return
- case <-time.After(loopDelay):
- continue
- }
- }
- // No error, reset loop delay
- loopDelay = 0
-
- n.logger.Printf("[DEBUG] raft-net: %v accepted connection from: %v", n.LocalAddr(), conn.RemoteAddr())
-
- // Handle the connection in dedicated routine
- go n.handleConn(n.getStreamContext(), conn)
- }
-}
-
-// handleConn is used to handle an inbound connection for its lifespan. The
-// handler will exit when the passed context is cancelled or the connection is
-// closed.
-func (n *NetworkTransport) handleConn(connCtx context.Context, conn net.Conn) {
- defer conn.Close()
- r := bufio.NewReader(conn)
- w := bufio.NewWriter(conn)
- dec := codec.NewDecoder(r, &codec.MsgpackHandle{})
- enc := codec.NewEncoder(w, &codec.MsgpackHandle{})
-
- for {
- select {
- case <-connCtx.Done():
- n.logger.Println("[DEBUG] raft-net: stream layer is closed")
- return
- default:
- }
-
- if err := n.handleCommand(r, dec, enc); err != nil {
- if err != io.EOF {
- n.logger.Printf("[ERR] raft-net: Failed to decode incoming command: %v", err)
- }
- return
- }
- if err := w.Flush(); err != nil {
- n.logger.Printf("[ERR] raft-net: Failed to flush response: %v", err)
- return
- }
- }
-}
-
-// handleCommand is used to decode and dispatch a single command.
-func (n *NetworkTransport) handleCommand(r *bufio.Reader, dec *codec.Decoder, enc *codec.Encoder) error {
- // Get the rpc type
- rpcType, err := r.ReadByte()
- if err != nil {
- return err
- }
-
- // Create the RPC object
- respCh := make(chan RPCResponse, 1)
- rpc := RPC{
- RespChan: respCh,
- }
-
- // Decode the command
- isHeartbeat := false
- switch rpcType {
- case rpcAppendEntries:
- var req AppendEntriesRequest
- if err := dec.Decode(&req); err != nil {
- return err
- }
- rpc.Command = &req
-
- // Check if this is a heartbeat
- if req.Term != 0 && req.Leader != nil &&
- req.PrevLogEntry == 0 && req.PrevLogTerm == 0 &&
- len(req.Entries) == 0 && req.LeaderCommitIndex == 0 {
- isHeartbeat = true
- }
-
- case rpcRequestVote:
- var req RequestVoteRequest
- if err := dec.Decode(&req); err != nil {
- return err
- }
- rpc.Command = &req
-
- case rpcInstallSnapshot:
- var req InstallSnapshotRequest
- if err := dec.Decode(&req); err != nil {
- return err
- }
- rpc.Command = &req
- rpc.Reader = io.LimitReader(r, req.Size)
-
- default:
- return fmt.Errorf("unknown rpc type %d", rpcType)
- }
-
- // Check for heartbeat fast-path
- if isHeartbeat {
- n.heartbeatFnLock.Lock()
- fn := n.heartbeatFn
- n.heartbeatFnLock.Unlock()
- if fn != nil {
- fn(rpc)
- goto RESP
- }
- }
-
- // Dispatch the RPC
- select {
- case n.consumeCh <- rpc:
- case <-n.shutdownCh:
- return ErrTransportShutdown
- }
-
- // Wait for response
-RESP:
- select {
- case resp := <-respCh:
- // Send the error first
- respErr := ""
- if resp.Error != nil {
- respErr = resp.Error.Error()
- }
- if err := enc.Encode(respErr); err != nil {
- return err
- }
-
- // Send the response
- if err := enc.Encode(resp.Response); err != nil {
- return err
- }
- case <-n.shutdownCh:
- return ErrTransportShutdown
- }
- return nil
-}
-
-// decodeResponse is used to decode an RPC response and reports whether
-// the connection can be reused.
-func decodeResponse(conn *netConn, resp interface{}) (bool, error) {
- // Decode the error if any
- var rpcError string
- if err := conn.dec.Decode(&rpcError); err != nil {
- conn.Release()
- return false, err
- }
-
- // Decode the response
- if err := conn.dec.Decode(resp); err != nil {
- conn.Release()
- return false, err
- }
-
- // Format an error if any
- if rpcError != "" {
- return true, fmt.Errorf(rpcError)
- }
- return true, nil
-}
-
-// sendRPC is used to encode and send the RPC.
-func sendRPC(conn *netConn, rpcType uint8, args interface{}) error {
- // Write the request type
- if err := conn.w.WriteByte(rpcType); err != nil {
- conn.Release()
- return err
- }
-
- // Send the request
- if err := conn.enc.Encode(args); err != nil {
- conn.Release()
- return err
- }
-
- // Flush
- if err := conn.w.Flush(); err != nil {
- conn.Release()
- return err
- }
- return nil
-}
-
-// newNetPipeline is used to construct a netPipeline from a given
-// transport and connection.
-func newNetPipeline(trans *NetworkTransport, conn *netConn) *netPipeline {
- n := &netPipeline{
- conn: conn,
- trans: trans,
- doneCh: make(chan AppendFuture, rpcMaxPipeline),
- inprogressCh: make(chan *appendFuture, rpcMaxPipeline),
- shutdownCh: make(chan struct{}),
- }
- go n.decodeResponses()
- return n
-}
-
-// decodeResponses is a long running routine that decodes the responses
-// sent on the connection.
-func (n *netPipeline) decodeResponses() {
- timeout := n.trans.timeout
- for {
- select {
- case future := <-n.inprogressCh:
- if timeout > 0 {
- n.conn.conn.SetReadDeadline(time.Now().Add(timeout))
- }
-
- _, err := decodeResponse(n.conn, future.resp)
- future.respond(err)
- select {
- case n.doneCh <- future:
- case <-n.shutdownCh:
- return
- }
- case <-n.shutdownCh:
- return
- }
- }
-}
-
-// AppendEntries is used to pipeline a new append entries request.
-func (n *netPipeline) AppendEntries(args *AppendEntriesRequest, resp *AppendEntriesResponse) (AppendFuture, error) {
- // Create a new future
- future := &appendFuture{
- start: time.Now(),
- args: args,
- resp: resp,
- }
- future.init()
-
- // Add a send timeout
- if timeout := n.trans.timeout; timeout > 0 {
- n.conn.conn.SetWriteDeadline(time.Now().Add(timeout))
- }
-
- // Send the RPC
- if err := sendRPC(n.conn, rpcAppendEntries, future.args); err != nil {
- return nil, err
- }
-
- // Hand-off for decoding, this can also cause back-pressure
- // to prevent too many inflight requests
- select {
- case n.inprogressCh <- future:
- return future, nil
- case <-n.shutdownCh:
- return nil, ErrPipelineShutdown
- }
-}
-
-// Consumer returns a channel that can be used to consume complete futures.
-func (n *netPipeline) Consumer() <-chan AppendFuture {
- return n.doneCh
-}
-
-// Closed is used to shutdown the pipeline connection.
-func (n *netPipeline) Close() error {
- n.shutdownLock.Lock()
- defer n.shutdownLock.Unlock()
- if n.shutdown {
- return nil
- }
-
- // Release the connection
- n.conn.Release()
-
- n.shutdown = true
- close(n.shutdownCh)
- return nil
-}
diff --git a/vendor/github.com/hashicorp/raft/observer.go b/vendor/github.com/hashicorp/raft/observer.go
deleted file mode 100644
index 2d4f37db12..0000000000
--- a/vendor/github.com/hashicorp/raft/observer.go
+++ /dev/null
@@ -1,131 +0,0 @@
-package raft
-
-import (
- "sync/atomic"
-)
-
-// Observation is sent along the given channel to observers when an event occurs.
-type Observation struct {
- // Raft holds the Raft instance generating the observation.
- Raft *Raft
- // Data holds observation-specific data. Possible types are
- // *RequestVoteRequest
- // RaftState
- // PeerObservation
- // LeaderObservation
- Data interface{}
-}
-
-// LeaderObservation is used for the data when leadership changes.
-type LeaderObservation struct {
- leader ServerAddress
-}
-
-// PeerObservation is sent to observers when peers change.
-type PeerObservation struct {
- Removed bool
- Peer Server
-}
-
-// nextObserverId is used to provide a unique ID for each observer to aid in
-// deregistration.
-var nextObserverID uint64
-
-// FilterFn is a function that can be registered in order to filter observations.
-// The function reports whether the observation should be included - if
-// it returns false, the observation will be filtered out.
-type FilterFn func(o *Observation) bool
-
-// Observer describes what to do with a given observation.
-type Observer struct {
- // numObserved and numDropped are performance counters for this observer.
- // 64 bit types must be 64 bit aligned to use with atomic operations on
- // 32 bit platforms, so keep them at the top of the struct.
- numObserved uint64
- numDropped uint64
-
- // channel receives observations.
- channel chan Observation
-
- // blocking, if true, will cause Raft to block when sending an observation
- // to this observer. This should generally be set to false.
- blocking bool
-
- // filter will be called to determine if an observation should be sent to
- // the channel.
- filter FilterFn
-
- // id is the ID of this observer in the Raft map.
- id uint64
-}
-
-// NewObserver creates a new observer that can be registered
-// to make observations on a Raft instance. Observations
-// will be sent on the given channel if they satisfy the
-// given filter.
-//
-// If blocking is true, the observer will block when it can't
-// send on the channel, otherwise it may discard events.
-func NewObserver(channel chan Observation, blocking bool, filter FilterFn) *Observer {
- return &Observer{
- channel: channel,
- blocking: blocking,
- filter: filter,
- id: atomic.AddUint64(&nextObserverID, 1),
- }
-}
-
-// GetNumObserved returns the number of observations.
-func (or *Observer) GetNumObserved() uint64 {
- return atomic.LoadUint64(&or.numObserved)
-}
-
-// GetNumDropped returns the number of dropped observations due to blocking.
-func (or *Observer) GetNumDropped() uint64 {
- return atomic.LoadUint64(&or.numDropped)
-}
-
-// RegisterObserver registers a new observer.
-func (r *Raft) RegisterObserver(or *Observer) {
- r.observersLock.Lock()
- defer r.observersLock.Unlock()
- r.observers[or.id] = or
-}
-
-// DeregisterObserver deregisters an observer.
-func (r *Raft) DeregisterObserver(or *Observer) {
- r.observersLock.Lock()
- defer r.observersLock.Unlock()
- delete(r.observers, or.id)
-}
-
-// observe sends an observation to every observer.
-func (r *Raft) observe(o interface{}) {
- // In general observers should not block. But in any case this isn't
- // disastrous as we only hold a read lock, which merely prevents
- // registration / deregistration of observers.
- r.observersLock.RLock()
- defer r.observersLock.RUnlock()
- for _, or := range r.observers {
- // It's wasteful to do this in the loop, but for the common case
- // where there are no observers we won't create any objects.
- ob := Observation{Raft: r, Data: o}
- if or.filter != nil && !or.filter(&ob) {
- continue
- }
- if or.channel == nil {
- continue
- }
- if or.blocking {
- or.channel <- ob
- atomic.AddUint64(&or.numObserved, 1)
- } else {
- select {
- case or.channel <- ob:
- atomic.AddUint64(&or.numObserved, 1)
- default:
- atomic.AddUint64(&or.numDropped, 1)
- }
- }
- }
-}
diff --git a/vendor/github.com/hashicorp/raft/peersjson.go b/vendor/github.com/hashicorp/raft/peersjson.go
deleted file mode 100644
index 38ca2a8b84..0000000000
--- a/vendor/github.com/hashicorp/raft/peersjson.go
+++ /dev/null
@@ -1,98 +0,0 @@
-package raft
-
-import (
- "bytes"
- "encoding/json"
- "io/ioutil"
-)
-
-// ReadPeersJSON consumes a legacy peers.json file in the format of the old JSON
-// peer store and creates a new-style configuration structure. This can be used
-// to migrate this data or perform manual recovery when running protocol versions
-// that can interoperate with older, unversioned Raft servers. This should not be
-// used once server IDs are in use, because the old peers.json file didn't have
-// support for these, nor non-voter suffrage types.
-func ReadPeersJSON(path string) (Configuration, error) {
- // Read in the file.
- buf, err := ioutil.ReadFile(path)
- if err != nil {
- return Configuration{}, err
- }
-
- // Parse it as JSON.
- var peers []string
- dec := json.NewDecoder(bytes.NewReader(buf))
- if err := dec.Decode(&peers); err != nil {
- return Configuration{}, err
- }
-
- // Map it into the new-style configuration structure. We can only specify
- // voter roles here, and the ID has to be the same as the address.
- var configuration Configuration
- for _, peer := range peers {
- server := Server{
- Suffrage: Voter,
- ID: ServerID(peer),
- Address: ServerAddress(peer),
- }
- configuration.Servers = append(configuration.Servers, server)
- }
-
- // We should only ingest valid configurations.
- if err := checkConfiguration(configuration); err != nil {
- return Configuration{}, err
- }
- return configuration, nil
-}
-
-// configEntry is used when decoding a new-style peers.json.
-type configEntry struct {
- // ID is the ID of the server (a UUID, usually).
- ID ServerID `json:"id"`
-
- // Address is the host:port of the server.
- Address ServerAddress `json:"address"`
-
- // NonVoter controls the suffrage. We choose this sense so people
- // can leave this out and get a Voter by default.
- NonVoter bool `json:"non_voter"`
-}
-
-// ReadConfigJSON reads a new-style peers.json and returns a configuration
-// structure. This can be used to perform manual recovery when running protocol
-// versions that use server IDs.
-func ReadConfigJSON(path string) (Configuration, error) {
- // Read in the file.
- buf, err := ioutil.ReadFile(path)
- if err != nil {
- return Configuration{}, err
- }
-
- // Parse it as JSON.
- var peers []configEntry
- dec := json.NewDecoder(bytes.NewReader(buf))
- if err := dec.Decode(&peers); err != nil {
- return Configuration{}, err
- }
-
- // Map it into the new-style configuration structure.
- var configuration Configuration
- for _, peer := range peers {
- suffrage := Voter
- if peer.NonVoter {
- suffrage = Nonvoter
- }
- server := Server{
- Suffrage: suffrage,
- ID: peer.ID,
- Address: peer.Address,
- }
- configuration.Servers = append(configuration.Servers, server)
- }
-
- // We should only ingest valid configurations.
- if err := checkConfiguration(configuration); err != nil {
- return Configuration{}, err
- }
- return configuration, nil
-}
diff --git a/vendor/github.com/hashicorp/raft/raft.go b/vendor/github.com/hashicorp/raft/raft.go
deleted file mode 100644
index a759230bc9..0000000000
--- a/vendor/github.com/hashicorp/raft/raft.go
+++ /dev/null
@@ -1,1486 +0,0 @@
-package raft
-
-import (
- "bytes"
- "container/list"
- "fmt"
- "io"
- "io/ioutil"
- "time"
-
- "github.com/armon/go-metrics"
-)
-
-const (
- minCheckInterval = 10 * time.Millisecond
-)
-
-var (
- keyCurrentTerm = []byte("CurrentTerm")
- keyLastVoteTerm = []byte("LastVoteTerm")
- keyLastVoteCand = []byte("LastVoteCand")
-)
-
-// getRPCHeader returns an initialized RPCHeader struct for the given
-// Raft instance. This structure is sent along with RPC requests and
-// responses.
-func (r *Raft) getRPCHeader() RPCHeader {
- return RPCHeader{
- ProtocolVersion: r.conf.ProtocolVersion,
- }
-}
-
-// checkRPCHeader houses logic about whether this instance of Raft can process
-// the given RPC message.
-func (r *Raft) checkRPCHeader(rpc RPC) error {
- // Get the header off the RPC message.
- wh, ok := rpc.Command.(WithRPCHeader)
- if !ok {
- return fmt.Errorf("RPC does not have a header")
- }
- header := wh.GetRPCHeader()
-
- // First check is to just make sure the code can understand the
- // protocol at all.
- if header.ProtocolVersion < ProtocolVersionMin ||
- header.ProtocolVersion > ProtocolVersionMax {
- return ErrUnsupportedProtocol
- }
-
- // Second check is whether we should support this message, given the
- // current protocol we are configured to run. This will drop support
- // for protocol version 0 starting at protocol version 2, which is
- // currently what we want, and in general support one version back. We
- // may need to revisit this policy depending on how future protocol
- // changes evolve.
- if header.ProtocolVersion < r.conf.ProtocolVersion-1 {
- return ErrUnsupportedProtocol
- }
-
- return nil
-}
-
-// getSnapshotVersion returns the snapshot version that should be used when
-// creating snapshots, given the protocol version in use.
-func getSnapshotVersion(protocolVersion ProtocolVersion) SnapshotVersion {
- // Right now we only have two versions and they are backwards compatible
- // so we don't need to look at the protocol version.
- return 1
-}
-
-// commitTuple is used to send an index that was committed,
-// with an optional associated future that should be invoked.
-type commitTuple struct {
- log *Log
- future *logFuture
-}
-
-// leaderState is state that is used while we are a leader.
-type leaderState struct {
- commitCh chan struct{}
- commitment *commitment
- inflight *list.List // list of logFuture in log index order
- replState map[ServerID]*followerReplication
- notify map[*verifyFuture]struct{}
- stepDown chan struct{}
-}
-
-// setLeader is used to modify the current leader of the cluster
-func (r *Raft) setLeader(leader ServerAddress) {
- r.leaderLock.Lock()
- oldLeader := r.leader
- r.leader = leader
- r.leaderLock.Unlock()
- if oldLeader != leader {
- r.observe(LeaderObservation{leader: leader})
- }
-}
-
-// requestConfigChange is a helper for the above functions that make
-// configuration change requests. 'req' describes the change. For timeout,
-// see AddVoter.
-func (r *Raft) requestConfigChange(req configurationChangeRequest, timeout time.Duration) IndexFuture {
- var timer <-chan time.Time
- if timeout > 0 {
- timer = time.After(timeout)
- }
- future := &configurationChangeFuture{
- req: req,
- }
- future.init()
- select {
- case <-timer:
- return errorFuture{ErrEnqueueTimeout}
- case r.configurationChangeCh <- future:
- return future
- case <-r.shutdownCh:
- return errorFuture{ErrRaftShutdown}
- }
-}
-
-// run is a long running goroutine that runs the Raft FSM.
-func (r *Raft) run() {
- for {
- // Check if we are doing a shutdown
- select {
- case <-r.shutdownCh:
- // Clear the leader to prevent forwarding
- r.setLeader("")
- return
- default:
- }
-
- // Enter into a sub-FSM
- switch r.getState() {
- case Follower:
- r.runFollower()
- case Candidate:
- r.runCandidate()
- case Leader:
- r.runLeader()
- }
- }
-}
-
-// runFollower runs the FSM for a follower.
-func (r *Raft) runFollower() {
- didWarn := false
- r.logger.Info(fmt.Sprintf("%v entering Follower state (Leader: %q)", r, r.Leader()))
- metrics.IncrCounter([]string{"raft", "state", "follower"}, 1)
- heartbeatTimer := randomTimeout(r.conf.HeartbeatTimeout)
- for {
- select {
- case rpc := <-r.rpcCh:
- r.processRPC(rpc)
-
- case c := <-r.configurationChangeCh:
- // Reject any operations since we are not the leader
- c.respond(ErrNotLeader)
-
- case a := <-r.applyCh:
- // Reject any operations since we are not the leader
- a.respond(ErrNotLeader)
-
- case v := <-r.verifyCh:
- // Reject any operations since we are not the leader
- v.respond(ErrNotLeader)
-
- case r := <-r.userRestoreCh:
- // Reject any restores since we are not the leader
- r.respond(ErrNotLeader)
-
- case c := <-r.configurationsCh:
- c.configurations = r.configurations.Clone()
- c.respond(nil)
-
- case b := <-r.bootstrapCh:
- b.respond(r.liveBootstrap(b.configuration))
-
- case <-heartbeatTimer:
- // Restart the heartbeat timer
- heartbeatTimer = randomTimeout(r.conf.HeartbeatTimeout)
-
- // Check if we have had a successful contact
- lastContact := r.LastContact()
- if time.Now().Sub(lastContact) < r.conf.HeartbeatTimeout {
- continue
- }
-
- // Heartbeat failed! Transition to the candidate state
- lastLeader := r.Leader()
- r.setLeader("")
-
- if r.configurations.latestIndex == 0 {
- if !didWarn {
- r.logger.Warn("no known peers, aborting election")
- didWarn = true
- }
- } else if r.configurations.latestIndex == r.configurations.committedIndex &&
- !hasVote(r.configurations.latest, r.localID) {
- if !didWarn {
- r.logger.Warn("not part of stable configuration, aborting election")
- didWarn = true
- }
- } else {
- r.logger.Warn(fmt.Sprintf("Heartbeat timeout from %q reached, starting election", lastLeader))
- metrics.IncrCounter([]string{"raft", "transition", "heartbeat_timeout"}, 1)
- r.setState(Candidate)
- return
- }
-
- case <-r.shutdownCh:
- return
- }
- }
-}
-
-// liveBootstrap attempts to seed an initial configuration for the cluster. See
-// the Raft object's member BootstrapCluster for more details. This must only be
-// called on the main thread, and only makes sense in the follower state.
-func (r *Raft) liveBootstrap(configuration Configuration) error {
- // Use the pre-init API to make the static updates.
- err := BootstrapCluster(&r.conf, r.logs, r.stable, r.snapshots,
- r.trans, configuration)
- if err != nil {
- return err
- }
-
- // Make the configuration live.
- var entry Log
- if err := r.logs.GetLog(1, &entry); err != nil {
- panic(err)
- }
- r.setCurrentTerm(1)
- r.setLastLog(entry.Index, entry.Term)
- r.processConfigurationLogEntry(&entry)
- return nil
-}
-
-// runCandidate runs the FSM for a candidate.
-func (r *Raft) runCandidate() {
- r.logger.Info(fmt.Sprintf("%v entering Candidate state in term %v", r, r.getCurrentTerm()+1))
- metrics.IncrCounter([]string{"raft", "state", "candidate"}, 1)
-
- // Start vote for us, and set a timeout
- voteCh := r.electSelf()
- electionTimer := randomTimeout(r.conf.ElectionTimeout)
-
- // Tally the votes, need a simple majority
- grantedVotes := 0
- votesNeeded := r.quorumSize()
- r.logger.Debug(fmt.Sprintf("Votes needed: %d", votesNeeded))
-
- for r.getState() == Candidate {
- select {
- case rpc := <-r.rpcCh:
- r.processRPC(rpc)
-
- case vote := <-voteCh:
- // Check if the term is greater than ours, bail
- if vote.Term > r.getCurrentTerm() {
- r.logger.Debug("Newer term discovered, fallback to follower")
- r.setState(Follower)
- r.setCurrentTerm(vote.Term)
- return
- }
-
- // Check if the vote is granted
- if vote.Granted {
- grantedVotes++
- r.logger.Debug(fmt.Sprintf("Vote granted from %s in term %v. Tally: %d",
- vote.voterID, vote.Term, grantedVotes))
- }
-
- // Check if we've become the leader
- if grantedVotes >= votesNeeded {
- r.logger.Info(fmt.Sprintf("Election won. Tally: %d", grantedVotes))
- r.setState(Leader)
- r.setLeader(r.localAddr)
- return
- }
-
- case c := <-r.configurationChangeCh:
- // Reject any operations since we are not the leader
- c.respond(ErrNotLeader)
-
- case a := <-r.applyCh:
- // Reject any operations since we are not the leader
- a.respond(ErrNotLeader)
-
- case v := <-r.verifyCh:
- // Reject any operations since we are not the leader
- v.respond(ErrNotLeader)
-
- case r := <-r.userRestoreCh:
- // Reject any restores since we are not the leader
- r.respond(ErrNotLeader)
-
- case c := <-r.configurationsCh:
- c.configurations = r.configurations.Clone()
- c.respond(nil)
-
- case b := <-r.bootstrapCh:
- b.respond(ErrCantBootstrap)
-
- case <-electionTimer:
- // Election failed! Restart the election. We simply return,
- // which will kick us back into runCandidate
- r.logger.Warn("Election timeout reached, restarting election")
- return
-
- case <-r.shutdownCh:
- return
- }
- }
-}
-
-// runLeader runs the FSM for a leader. Do the setup here and drop into
-// the leaderLoop for the hot loop.
-func (r *Raft) runLeader() {
- r.logger.Info(fmt.Sprintf("%v entering Leader state", r))
- metrics.IncrCounter([]string{"raft", "state", "leader"}, 1)
-
- // Notify that we are the leader
- asyncNotifyBool(r.leaderCh, true)
-
- // Push to the notify channel if given
- if notify := r.conf.NotifyCh; notify != nil {
- select {
- case notify <- true:
- case <-r.shutdownCh:
- }
- }
-
- // Setup leader state
- r.leaderState.commitCh = make(chan struct{}, 1)
- r.leaderState.commitment = newCommitment(r.leaderState.commitCh,
- r.configurations.latest,
- r.getLastIndex()+1 /* first index that may be committed in this term */)
- r.leaderState.inflight = list.New()
- r.leaderState.replState = make(map[ServerID]*followerReplication)
- r.leaderState.notify = make(map[*verifyFuture]struct{})
- r.leaderState.stepDown = make(chan struct{}, 1)
-
- // Cleanup state on step down
- defer func() {
- // Since we were the leader previously, we update our
- // last contact time when we step down, so that we are not
- // reporting a last contact time from before we were the
- // leader. Otherwise, to a client it would seem our data
- // is extremely stale.
- r.setLastContact()
-
- // Stop replication
- for _, p := range r.leaderState.replState {
- close(p.stopCh)
- }
-
- // Respond to all inflight operations
- for e := r.leaderState.inflight.Front(); e != nil; e = e.Next() {
- e.Value.(*logFuture).respond(ErrLeadershipLost)
- }
-
- // Respond to any pending verify requests
- for future := range r.leaderState.notify {
- future.respond(ErrLeadershipLost)
- }
-
- // Clear all the state
- r.leaderState.commitCh = nil
- r.leaderState.commitment = nil
- r.leaderState.inflight = nil
- r.leaderState.replState = nil
- r.leaderState.notify = nil
- r.leaderState.stepDown = nil
-
- // If we are stepping down for some reason, no known leader.
- // We may have stepped down due to an RPC call, which would
- // provide the leader, so we cannot always blank this out.
- r.leaderLock.Lock()
- if r.leader == r.localAddr {
- r.leader = ""
- }
- r.leaderLock.Unlock()
-
- // Notify that we are not the leader
- asyncNotifyBool(r.leaderCh, false)
-
- // Push to the notify channel if given
- if notify := r.conf.NotifyCh; notify != nil {
- select {
- case notify <- false:
- case <-r.shutdownCh:
- // On shutdown, make a best effort but do not block
- select {
- case notify <- false:
- default:
- }
- }
- }
- }()
-
- // Start a replication routine for each peer
- r.startStopReplication()
-
- // Dispatch a no-op log entry first. This gets this leader up to the latest
- // possible commit index, even in the absence of client commands. This used
- // to append a configuration entry instead of a noop. However, that permits
- // an unbounded number of uncommitted configurations in the log. We now
- // maintain that there exists at most one uncommitted configuration entry in
- // any log, so we have to do proper no-ops here.
- noop := &logFuture{
- log: Log{
- Type: LogNoop,
- },
- }
- r.dispatchLogs([]*logFuture{noop})
-
- // Sit in the leader loop until we step down
- r.leaderLoop()
-}
-
-// startStopReplication will set up state and start asynchronous replication to
-// new peers, and stop replication to removed peers. Before removing a peer,
-// it'll instruct the replication routines to try to replicate to the current
-// index. This must only be called from the main thread.
-func (r *Raft) startStopReplication() {
- inConfig := make(map[ServerID]bool, len(r.configurations.latest.Servers))
- lastIdx := r.getLastIndex()
-
- // Start replication goroutines that need starting
- for _, server := range r.configurations.latest.Servers {
- if server.ID == r.localID {
- continue
- }
- inConfig[server.ID] = true
- if _, ok := r.leaderState.replState[server.ID]; !ok {
- r.logger.Info(fmt.Sprintf("Added peer %v, starting replication", server.ID))
- s := &followerReplication{
- peer: server,
- commitment: r.leaderState.commitment,
- stopCh: make(chan uint64, 1),
- triggerCh: make(chan struct{}, 1),
- currentTerm: r.getCurrentTerm(),
- nextIndex: lastIdx + 1,
- lastContact: time.Now(),
- notify: make(map[*verifyFuture]struct{}),
- notifyCh: make(chan struct{}, 1),
- stepDown: r.leaderState.stepDown,
- }
- r.leaderState.replState[server.ID] = s
- r.goFunc(func() { r.replicate(s) })
- asyncNotifyCh(s.triggerCh)
- r.observe(PeerObservation{Peer: server, Removed: false})
- }
- }
-
- // Stop replication goroutines that need stopping
- for serverID, repl := range r.leaderState.replState {
- if inConfig[serverID] {
- continue
- }
- // Replicate up to lastIdx and stop
- r.logger.Info(fmt.Sprintf("Removed peer %v, stopping replication after %v", serverID, lastIdx))
- repl.stopCh <- lastIdx
- close(repl.stopCh)
- delete(r.leaderState.replState, serverID)
- r.observe(PeerObservation{Peer: repl.peer, Removed: true})
- }
-}
-
-// configurationChangeChIfStable returns r.configurationChangeCh if it's safe
-// to process requests from it, or nil otherwise. This must only be called
-// from the main thread.
-//
-// Note that if the conditions here were to change outside of leaderLoop to take
-// this from nil to non-nil, we would need leaderLoop to be kicked.
-func (r *Raft) configurationChangeChIfStable() chan *configurationChangeFuture {
- // Have to wait until:
- // 1. The latest configuration is committed, and
- // 2. This leader has committed some entry (the noop) in this term
- // https://groups.google.com/forum/#!msg/raft-dev/t4xj6dJTP6E/d2D9LrWRza8J
- if r.configurations.latestIndex == r.configurations.committedIndex &&
- r.getCommitIndex() >= r.leaderState.commitment.startIndex {
- return r.configurationChangeCh
- }
- return nil
-}
-
-// leaderLoop is the hot loop for a leader. It is invoked
-// after all the various leader setup is done.
-func (r *Raft) leaderLoop() {
- // stepDown is used to track if there is an inflight log that
- // would cause us to lose leadership (specifically a RemovePeer of
- // ourselves). If this is the case, we must not allow any logs to
- // be processed in parallel, otherwise we are basing commit on
- // only a single peer (ourself) and replicating to an undefined set
- // of peers.
- stepDown := false
-
- lease := time.After(r.conf.LeaderLeaseTimeout)
- for r.getState() == Leader {
- select {
- case rpc := <-r.rpcCh:
- r.processRPC(rpc)
-
- case <-r.leaderState.stepDown:
- r.setState(Follower)
-
- case <-r.leaderState.commitCh:
- // Process the newly committed entries
- oldCommitIndex := r.getCommitIndex()
- commitIndex := r.leaderState.commitment.getCommitIndex()
- r.setCommitIndex(commitIndex)
-
- if r.configurations.latestIndex > oldCommitIndex &&
- r.configurations.latestIndex <= commitIndex {
- r.configurations.committed = r.configurations.latest
- r.configurations.committedIndex = r.configurations.latestIndex
- if !hasVote(r.configurations.committed, r.localID) {
- stepDown = true
- }
- }
-
- var numProcessed int
- start := time.Now()
-
- for {
- e := r.leaderState.inflight.Front()
- if e == nil {
- break
- }
- commitLog := e.Value.(*logFuture)
- idx := commitLog.log.Index
- if idx > commitIndex {
- break
- }
- // Measure the commit time
- metrics.MeasureSince([]string{"raft", "commitTime"}, commitLog.dispatch)
-
- r.processLogs(idx, commitLog)
-
- r.leaderState.inflight.Remove(e)
- numProcessed++
- }
-
- // Measure the time to enqueue batch of logs for FSM to apply
- metrics.MeasureSince([]string{"raft", "fsm", "enqueue"}, start)
-
- // Count the number of logs enqueued
- metrics.SetGauge([]string{"raft", "commitNumLogs"}, float32(numProcessed))
-
- if stepDown {
- if r.conf.ShutdownOnRemove {
- r.logger.Info("Removed ourself, shutting down")
- r.Shutdown()
- } else {
- r.logger.Info("Removed ourself, transitioning to follower")
- r.setState(Follower)
- }
- }
-
- case v := <-r.verifyCh:
- if v.quorumSize == 0 {
- // Just dispatched, start the verification
- r.verifyLeader(v)
-
- } else if v.votes < v.quorumSize {
- // Early return, means there must be a new leader
- r.logger.Warn("New leader elected, stepping down")
- r.setState(Follower)
- delete(r.leaderState.notify, v)
- for _, repl := range r.leaderState.replState {
- repl.cleanNotify(v)
- }
- v.respond(ErrNotLeader)
-
- } else {
- // Quorum of members agree, we are still leader
- delete(r.leaderState.notify, v)
- for _, repl := range r.leaderState.replState {
- repl.cleanNotify(v)
- }
- v.respond(nil)
- }
-
- case future := <-r.userRestoreCh:
- err := r.restoreUserSnapshot(future.meta, future.reader)
- future.respond(err)
-
- case c := <-r.configurationsCh:
- c.configurations = r.configurations.Clone()
- c.respond(nil)
-
- case future := <-r.configurationChangeChIfStable():
- r.appendConfigurationEntry(future)
-
- case b := <-r.bootstrapCh:
- b.respond(ErrCantBootstrap)
-
- case newLog := <-r.applyCh:
- // Group commit, gather all the ready commits
- ready := []*logFuture{newLog}
- for i := 0; i < r.conf.MaxAppendEntries; i++ {
- select {
- case newLog := <-r.applyCh:
- ready = append(ready, newLog)
- default:
- break
- }
- }
-
- // Dispatch the logs
- if stepDown {
- // we're in the process of stepping down as leader, don't process anything new
- for i := range ready {
- ready[i].respond(ErrNotLeader)
- }
- } else {
- r.dispatchLogs(ready)
- }
-
- case <-lease:
- // Check if we've exceeded the lease, potentially stepping down
- maxDiff := r.checkLeaderLease()
-
- // Next check interval should adjust for the last node we've
- // contacted, without going negative
- checkInterval := r.conf.LeaderLeaseTimeout - maxDiff
- if checkInterval < minCheckInterval {
- checkInterval = minCheckInterval
- }
-
- // Renew the lease timer
- lease = time.After(checkInterval)
-
- case <-r.shutdownCh:
- return
- }
- }
-}
-
-// verifyLeader must be called from the main thread for safety.
-// Causes the followers to attempt an immediate heartbeat.
-func (r *Raft) verifyLeader(v *verifyFuture) {
- // Current leader always votes for self
- v.votes = 1
-
- // Set the quorum size, hot-path for single node
- v.quorumSize = r.quorumSize()
- if v.quorumSize == 1 {
- v.respond(nil)
- return
- }
-
- // Track this request
- v.notifyCh = r.verifyCh
- r.leaderState.notify[v] = struct{}{}
-
- // Trigger immediate heartbeats
- for _, repl := range r.leaderState.replState {
- repl.notifyLock.Lock()
- repl.notify[v] = struct{}{}
- repl.notifyLock.Unlock()
- asyncNotifyCh(repl.notifyCh)
- }
-}
-
-// checkLeaderLease is used to check if we can contact a quorum of nodes
-// within the last leader lease interval. If not, we need to step down,
-// as we may have lost connectivity. Returns the maximum duration without
-// contact. This must only be called from the main thread.
-func (r *Raft) checkLeaderLease() time.Duration {
- // Track contacted nodes, we can always contact ourself
- contacted := 1
-
- // Check each follower
- var maxDiff time.Duration
- now := time.Now()
- for peer, f := range r.leaderState.replState {
- diff := now.Sub(f.LastContact())
- if diff <= r.conf.LeaderLeaseTimeout {
- contacted++
- if diff > maxDiff {
- maxDiff = diff
- }
- } else {
- // Log at least once at high value, then debug. Otherwise it gets very verbose.
- if diff <= 3*r.conf.LeaderLeaseTimeout {
- r.logger.Warn(fmt.Sprintf("Failed to contact %v in %v", peer, diff))
- } else {
- r.logger.Debug(fmt.Sprintf("Failed to contact %v in %v", peer, diff))
- }
- }
- metrics.AddSample([]string{"raft", "leader", "lastContact"}, float32(diff/time.Millisecond))
- }
-
- // Verify we can contact a quorum
- quorum := r.quorumSize()
- if contacted < quorum {
- r.logger.Warn("Failed to contact quorum of nodes, stepping down")
- r.setState(Follower)
- metrics.IncrCounter([]string{"raft", "transition", "leader_lease_timeout"}, 1)
- }
- return maxDiff
-}
-
-// quorumSize is used to return the quorum size. This must only be called on
-// the main thread.
-// TODO: revisit usage
-func (r *Raft) quorumSize() int {
- voters := 0
- for _, server := range r.configurations.latest.Servers {
- if server.Suffrage == Voter {
- voters++
- }
- }
- return voters/2 + 1
-}
-
-// restoreUserSnapshot is used to manually consume an external snapshot, such
-// as if restoring from a backup. We will use the current Raft configuration,
-// not the one from the snapshot, so that we can restore into a new cluster. We
-// will also use the higher of the index of the snapshot, or the current index,
-// and then add 1 to that, so we force a new state with a hole in the Raft log,
-// so that the snapshot will be sent to followers and used for any new joiners.
-// This can only be run on the leader, and returns a future that can be used to
-// block until complete.
-func (r *Raft) restoreUserSnapshot(meta *SnapshotMeta, reader io.Reader) error {
- defer metrics.MeasureSince([]string{"raft", "restoreUserSnapshot"}, time.Now())
-
- // Sanity check the version.
- version := meta.Version
- if version < SnapshotVersionMin || version > SnapshotVersionMax {
- return fmt.Errorf("unsupported snapshot version %d", version)
- }
-
- // We don't support snapshots while there's a config change
- // outstanding since the snapshot doesn't have a means to
- // represent this state.
- committedIndex := r.configurations.committedIndex
- latestIndex := r.configurations.latestIndex
- if committedIndex != latestIndex {
- return fmt.Errorf("cannot restore snapshot now, wait until the configuration entry at %v has been applied (have applied %v)",
- latestIndex, committedIndex)
- }
-
- // Cancel any inflight requests.
- for {
- e := r.leaderState.inflight.Front()
- if e == nil {
- break
- }
- e.Value.(*logFuture).respond(ErrAbortedByRestore)
- r.leaderState.inflight.Remove(e)
- }
-
- // We will overwrite the snapshot metadata with the current term,
- // an index that's greater than the current index, or the last
- // index in the snapshot. It's important that we leave a hole in
- // the index so we know there's nothing in the Raft log there and
- // replication will fault and send the snapshot.
- term := r.getCurrentTerm()
- lastIndex := r.getLastIndex()
- if meta.Index > lastIndex {
- lastIndex = meta.Index
- }
- lastIndex++
-
- // Dump the snapshot. Note that we use the latest configuration,
- // not the one that came with the snapshot.
- sink, err := r.snapshots.Create(version, lastIndex, term,
- r.configurations.latest, r.configurations.latestIndex, r.trans)
- if err != nil {
- return fmt.Errorf("failed to create snapshot: %v", err)
- }
- n, err := io.Copy(sink, reader)
- if err != nil {
- sink.Cancel()
- return fmt.Errorf("failed to write snapshot: %v", err)
- }
- if n != meta.Size {
- sink.Cancel()
- return fmt.Errorf("failed to write snapshot, size didn't match (%d != %d)", n, meta.Size)
- }
- if err := sink.Close(); err != nil {
- return fmt.Errorf("failed to close snapshot: %v", err)
- }
- r.logger.Info(fmt.Sprintf("Copied %d bytes to local snapshot", n))
-
- // Restore the snapshot into the FSM. If this fails we are in a
- // bad state so we panic to take ourselves out.
- fsm := &restoreFuture{ID: sink.ID()}
- fsm.init()
- select {
- case r.fsmMutateCh <- fsm:
- case <-r.shutdownCh:
- return ErrRaftShutdown
- }
- if err := fsm.Error(); err != nil {
- panic(fmt.Errorf("failed to restore snapshot: %v", err))
- }
-
- // We set the last log so it looks like we've stored the empty
- // index we burned. The last applied is set because we made the
- // FSM take the snapshot state, and we store the last snapshot
- // in the stable store since we created a snapshot as part of
- // this process.
- r.setLastLog(lastIndex, term)
- r.setLastApplied(lastIndex)
- r.setLastSnapshot(lastIndex, term)
-
- r.logger.Info(fmt.Sprintf("Restored user snapshot (index %d)", lastIndex))
- return nil
-}
-
-// appendConfigurationEntry changes the configuration and adds a new
-// configuration entry to the log. This must only be called from the
-// main thread.
-func (r *Raft) appendConfigurationEntry(future *configurationChangeFuture) {
- configuration, err := nextConfiguration(r.configurations.latest, r.configurations.latestIndex, future.req)
- if err != nil {
- future.respond(err)
- return
- }
-
- r.logger.Info(fmt.Sprintf("Updating configuration with %s (%v, %v) to %+v",
- future.req.command, future.req.serverID, future.req.serverAddress, configuration.Servers))
-
- // In pre-ID compatibility mode we translate all configuration changes
- // in to an old remove peer message, which can handle all supported
- // cases for peer changes in the pre-ID world (adding and removing
- // voters). Both add peer and remove peer log entries are handled
- // similarly on old Raft servers, but remove peer does extra checks to
- // see if a leader needs to step down. Since they both assert the full
- // configuration, then we can safely call remove peer for everything.
- if r.protocolVersion < 2 {
- future.log = Log{
- Type: LogRemovePeerDeprecated,
- Data: encodePeers(configuration, r.trans),
- }
- } else {
- future.log = Log{
- Type: LogConfiguration,
- Data: encodeConfiguration(configuration),
- }
- }
-
- r.dispatchLogs([]*logFuture{&future.logFuture})
- index := future.Index()
- r.configurations.latest = configuration
- r.configurations.latestIndex = index
- r.leaderState.commitment.setConfiguration(configuration)
- r.startStopReplication()
-}
-
-// dispatchLog is called on the leader to push a log to disk, mark it
-// as inflight and begin replication of it.
-func (r *Raft) dispatchLogs(applyLogs []*logFuture) {
- now := time.Now()
- defer metrics.MeasureSince([]string{"raft", "leader", "dispatchLog"}, now)
-
- term := r.getCurrentTerm()
- lastIndex := r.getLastIndex()
-
- n := len(applyLogs)
- logs := make([]*Log, n)
- metrics.SetGauge([]string{"raft", "leader", "dispatchNumLogs"}, float32(n))
-
- for idx, applyLog := range applyLogs {
- applyLog.dispatch = now
- lastIndex++
- applyLog.log.Index = lastIndex
- applyLog.log.Term = term
- logs[idx] = &applyLog.log
- r.leaderState.inflight.PushBack(applyLog)
- }
-
- // Write the log entry locally
- if err := r.logs.StoreLogs(logs); err != nil {
- r.logger.Error(fmt.Sprintf("Failed to commit logs: %v", err))
- for _, applyLog := range applyLogs {
- applyLog.respond(err)
- }
- r.setState(Follower)
- return
- }
- r.leaderState.commitment.match(r.localID, lastIndex)
-
- // Update the last log since it's on disk now
- r.setLastLog(lastIndex, term)
-
- // Notify the replicators of the new log
- for _, f := range r.leaderState.replState {
- asyncNotifyCh(f.triggerCh)
- }
-}
-
-// processLogs is used to apply all the committed entries that haven't been
-// applied up to the given index limit.
-// This can be called from both leaders and followers.
-// Followers call this from AppendEntries, for n entries at a time, and always
-// pass future=nil.
-// Leaders call this once per inflight when entries are committed. They pass
-// the future from inflights.
-func (r *Raft) processLogs(index uint64, future *logFuture) {
- // Reject logs we've applied already
- lastApplied := r.getLastApplied()
- if index <= lastApplied {
- r.logger.Warn(fmt.Sprintf("Skipping application of old log: %d", index))
- return
- }
-
- // Apply all the preceding logs
- for idx := r.getLastApplied() + 1; idx <= index; idx++ {
- // Get the log, either from the future or from our log store
- if future != nil && future.log.Index == idx {
- r.processLog(&future.log, future)
- } else {
- l := new(Log)
- if err := r.logs.GetLog(idx, l); err != nil {
- r.logger.Error(fmt.Sprintf("Failed to get log at %d: %v", idx, err))
- panic(err)
- }
- r.processLog(l, nil)
- }
-
- // Update the lastApplied index and term
- r.setLastApplied(idx)
- }
-}
-
-// processLog is invoked to process the application of a single committed log entry.
-func (r *Raft) processLog(l *Log, future *logFuture) {
- switch l.Type {
- case LogBarrier:
- // Barrier is handled by the FSM
- fallthrough
-
- case LogCommand:
- // Forward to the fsm handler
- select {
- case r.fsmMutateCh <- &commitTuple{l, future}:
- case <-r.shutdownCh:
- if future != nil {
- future.respond(ErrRaftShutdown)
- }
- }
-
- // Return so that the future is only responded to
- // by the FSM handler when the application is done
- return
-
- case LogConfiguration:
- case LogAddPeerDeprecated:
- case LogRemovePeerDeprecated:
- case LogNoop:
- // Ignore the no-op
-
- default:
- panic(fmt.Errorf("unrecognized log type: %#v", l))
- }
-
- // Invoke the future if given
- if future != nil {
- future.respond(nil)
- }
-}
-
-// processRPC is called to handle an incoming RPC request. This must only be
-// called from the main thread.
-func (r *Raft) processRPC(rpc RPC) {
- if err := r.checkRPCHeader(rpc); err != nil {
- rpc.Respond(nil, err)
- return
- }
-
- switch cmd := rpc.Command.(type) {
- case *AppendEntriesRequest:
- r.appendEntries(rpc, cmd)
- case *RequestVoteRequest:
- r.requestVote(rpc, cmd)
- case *InstallSnapshotRequest:
- r.installSnapshot(rpc, cmd)
- default:
- r.logger.Error(fmt.Sprintf("Got unexpected command: %#v", rpc.Command))
- rpc.Respond(nil, fmt.Errorf("unexpected command"))
- }
-}
-
-// processHeartbeat is a special handler used just for heartbeat requests
-// so that they can be fast-pathed if a transport supports it. This must only
-// be called from the main thread.
-func (r *Raft) processHeartbeat(rpc RPC) {
- defer metrics.MeasureSince([]string{"raft", "rpc", "processHeartbeat"}, time.Now())
-
- // Check if we are shutdown, just ignore the RPC
- select {
- case <-r.shutdownCh:
- return
- default:
- }
-
- // Ensure we are only handling a heartbeat
- switch cmd := rpc.Command.(type) {
- case *AppendEntriesRequest:
- r.appendEntries(rpc, cmd)
- default:
- r.logger.Error(fmt.Sprintf("Expected heartbeat, got command: %#v", rpc.Command))
- rpc.Respond(nil, fmt.Errorf("unexpected command"))
- }
-}
-
-// appendEntries is invoked when we get an append entries RPC call. This must
-// only be called from the main thread.
-func (r *Raft) appendEntries(rpc RPC, a *AppendEntriesRequest) {
- defer metrics.MeasureSince([]string{"raft", "rpc", "appendEntries"}, time.Now())
- // Setup a response
- resp := &AppendEntriesResponse{
- RPCHeader: r.getRPCHeader(),
- Term: r.getCurrentTerm(),
- LastLog: r.getLastIndex(),
- Success: false,
- NoRetryBackoff: false,
- }
- var rpcErr error
- defer func() {
- rpc.Respond(resp, rpcErr)
- }()
-
- // Ignore an older term
- if a.Term < r.getCurrentTerm() {
- return
- }
-
- // Increase the term if we see a newer one, also transition to follower
- // if we ever get an appendEntries call
- if a.Term > r.getCurrentTerm() || r.getState() != Follower {
- // Ensure transition to follower
- r.setState(Follower)
- r.setCurrentTerm(a.Term)
- resp.Term = a.Term
- }
-
- // Save the current leader
- r.setLeader(ServerAddress(r.trans.DecodePeer(a.Leader)))
-
- // Verify the last log entry
- if a.PrevLogEntry > 0 {
- lastIdx, lastTerm := r.getLastEntry()
-
- var prevLogTerm uint64
- if a.PrevLogEntry == lastIdx {
- prevLogTerm = lastTerm
-
- } else {
- var prevLog Log
- if err := r.logs.GetLog(a.PrevLogEntry, &prevLog); err != nil {
- r.logger.Warn(fmt.Sprintf("Failed to get previous log: %d %v (last: %d)",
- a.PrevLogEntry, err, lastIdx))
- resp.NoRetryBackoff = true
- return
- }
- prevLogTerm = prevLog.Term
- }
-
- if a.PrevLogTerm != prevLogTerm {
- r.logger.Warn(fmt.Sprintf("Previous log term mis-match: ours: %d remote: %d",
- prevLogTerm, a.PrevLogTerm))
- resp.NoRetryBackoff = true
- return
- }
- }
-
- // Process any new entries
- if len(a.Entries) > 0 {
- start := time.Now()
-
- // Delete any conflicting entries, skip any duplicates
- lastLogIdx, _ := r.getLastLog()
- var newEntries []*Log
- for i, entry := range a.Entries {
- if entry.Index > lastLogIdx {
- newEntries = a.Entries[i:]
- break
- }
- var storeEntry Log
- if err := r.logs.GetLog(entry.Index, &storeEntry); err != nil {
- r.logger.Warn(fmt.Sprintf("Failed to get log entry %d: %v",
- entry.Index, err))
- return
- }
- if entry.Term != storeEntry.Term {
- r.logger.Warn(fmt.Sprintf("Clearing log suffix from %d to %d", entry.Index, lastLogIdx))
- if err := r.logs.DeleteRange(entry.Index, lastLogIdx); err != nil {
- r.logger.Error(fmt.Sprintf("Failed to clear log suffix: %v", err))
- return
- }
- if entry.Index <= r.configurations.latestIndex {
- r.configurations.latest = r.configurations.committed
- r.configurations.latestIndex = r.configurations.committedIndex
- }
- newEntries = a.Entries[i:]
- break
- }
- }
-
- if n := len(newEntries); n > 0 {
- // Append the new entries
- if err := r.logs.StoreLogs(newEntries); err != nil {
- r.logger.Error(fmt.Sprintf("Failed to append to logs: %v", err))
- // TODO: leaving r.getLastLog() in the wrong
- // state if there was a truncation above
- return
- }
-
- // Handle any new configuration changes
- for _, newEntry := range newEntries {
- r.processConfigurationLogEntry(newEntry)
- }
-
- // Update the lastLog
- last := newEntries[n-1]
- r.setLastLog(last.Index, last.Term)
- }
-
- metrics.MeasureSince([]string{"raft", "rpc", "appendEntries", "storeLogs"}, start)
- }
-
- // Update the commit index
- if a.LeaderCommitIndex > 0 && a.LeaderCommitIndex > r.getCommitIndex() {
- start := time.Now()
- idx := min(a.LeaderCommitIndex, r.getLastIndex())
- r.setCommitIndex(idx)
- if r.configurations.latestIndex <= idx {
- r.configurations.committed = r.configurations.latest
- r.configurations.committedIndex = r.configurations.latestIndex
- }
- r.processLogs(idx, nil)
- metrics.MeasureSince([]string{"raft", "rpc", "appendEntries", "processLogs"}, start)
- }
-
- // Everything went well, set success
- resp.Success = true
- r.setLastContact()
- return
-}
-
-// processConfigurationLogEntry takes a log entry and updates the latest
-// configuration if the entry results in a new configuration. This must only be
-// called from the main thread, or from NewRaft() before any threads have begun.
-func (r *Raft) processConfigurationLogEntry(entry *Log) {
- if entry.Type == LogConfiguration {
- r.configurations.committed = r.configurations.latest
- r.configurations.committedIndex = r.configurations.latestIndex
- r.configurations.latest = decodeConfiguration(entry.Data)
- r.configurations.latestIndex = entry.Index
- } else if entry.Type == LogAddPeerDeprecated || entry.Type == LogRemovePeerDeprecated {
- r.configurations.committed = r.configurations.latest
- r.configurations.committedIndex = r.configurations.latestIndex
- r.configurations.latest = decodePeers(entry.Data, r.trans)
- r.configurations.latestIndex = entry.Index
- }
-}
-
-// requestVote is invoked when we get an request vote RPC call.
-func (r *Raft) requestVote(rpc RPC, req *RequestVoteRequest) {
- defer metrics.MeasureSince([]string{"raft", "rpc", "requestVote"}, time.Now())
- r.observe(*req)
-
- // Setup a response
- resp := &RequestVoteResponse{
- RPCHeader: r.getRPCHeader(),
- Term: r.getCurrentTerm(),
- Granted: false,
- }
- var rpcErr error
- defer func() {
- rpc.Respond(resp, rpcErr)
- }()
-
- // Version 0 servers will panic unless the peers is present. It's only
- // used on them to produce a warning message.
- if r.protocolVersion < 2 {
- resp.Peers = encodePeers(r.configurations.latest, r.trans)
- }
-
- // Check if we have an existing leader [who's not the candidate]
- candidate := r.trans.DecodePeer(req.Candidate)
- if leader := r.Leader(); leader != "" && leader != candidate {
- r.logger.Warn(fmt.Sprintf("Rejecting vote request from %v since we have a leader: %v",
- candidate, leader))
- return
- }
-
- // Ignore an older term
- if req.Term < r.getCurrentTerm() {
- return
- }
-
- // Increase the term if we see a newer one
- if req.Term > r.getCurrentTerm() {
- // Ensure transition to follower
- r.setState(Follower)
- r.setCurrentTerm(req.Term)
- resp.Term = req.Term
- }
-
- // Check if we have voted yet
- lastVoteTerm, err := r.stable.GetUint64(keyLastVoteTerm)
- if err != nil && err.Error() != "not found" {
- r.logger.Error(fmt.Sprintf("Failed to get last vote term: %v", err))
- return
- }
- lastVoteCandBytes, err := r.stable.Get(keyLastVoteCand)
- if err != nil && err.Error() != "not found" {
- r.logger.Error(fmt.Sprintf("Failed to get last vote candidate: %v", err))
- return
- }
-
- // Check if we've voted in this election before
- if lastVoteTerm == req.Term && lastVoteCandBytes != nil {
- r.logger.Info(fmt.Sprintf("Duplicate RequestVote for same term: %d", req.Term))
- if bytes.Compare(lastVoteCandBytes, req.Candidate) == 0 {
- r.logger.Warn(fmt.Sprintf("Duplicate RequestVote from candidate: %s", req.Candidate))
- resp.Granted = true
- }
- return
- }
-
- // Reject if their term is older
- lastIdx, lastTerm := r.getLastEntry()
- if lastTerm > req.LastLogTerm {
- r.logger.Warn(fmt.Sprintf("Rejecting vote request from %v since our last term is greater (%d, %d)",
- candidate, lastTerm, req.LastLogTerm))
- return
- }
-
- if lastTerm == req.LastLogTerm && lastIdx > req.LastLogIndex {
- r.logger.Warn(fmt.Sprintf("Rejecting vote request from %v since our last index is greater (%d, %d)",
- candidate, lastIdx, req.LastLogIndex))
- return
- }
-
- // Persist a vote for safety
- if err := r.persistVote(req.Term, req.Candidate); err != nil {
- r.logger.Error(fmt.Sprintf("Failed to persist vote: %v", err))
- return
- }
-
- resp.Granted = true
- r.setLastContact()
- return
-}
-
-// installSnapshot is invoked when we get a InstallSnapshot RPC call.
-// We must be in the follower state for this, since it means we are
-// too far behind a leader for log replay. This must only be called
-// from the main thread.
-func (r *Raft) installSnapshot(rpc RPC, req *InstallSnapshotRequest) {
- defer metrics.MeasureSince([]string{"raft", "rpc", "installSnapshot"}, time.Now())
- // Setup a response
- resp := &InstallSnapshotResponse{
- Term: r.getCurrentTerm(),
- Success: false,
- }
- var rpcErr error
- defer func() {
- io.Copy(ioutil.Discard, rpc.Reader) // ensure we always consume all the snapshot data from the stream [see issue #212]
- rpc.Respond(resp, rpcErr)
- }()
-
- // Sanity check the version
- if req.SnapshotVersion < SnapshotVersionMin ||
- req.SnapshotVersion > SnapshotVersionMax {
- rpcErr = fmt.Errorf("unsupported snapshot version %d", req.SnapshotVersion)
- return
- }
-
- // Ignore an older term
- if req.Term < r.getCurrentTerm() {
- r.logger.Info(fmt.Sprintf("Ignoring installSnapshot request with older term of %d vs currentTerm %d",
- req.Term, r.getCurrentTerm()))
- return
- }
-
- // Increase the term if we see a newer one
- if req.Term > r.getCurrentTerm() {
- // Ensure transition to follower
- r.setState(Follower)
- r.setCurrentTerm(req.Term)
- resp.Term = req.Term
- }
-
- // Save the current leader
- r.setLeader(ServerAddress(r.trans.DecodePeer(req.Leader)))
-
- // Create a new snapshot
- var reqConfiguration Configuration
- var reqConfigurationIndex uint64
- if req.SnapshotVersion > 0 {
- reqConfiguration = decodeConfiguration(req.Configuration)
- reqConfigurationIndex = req.ConfigurationIndex
- } else {
- reqConfiguration = decodePeers(req.Peers, r.trans)
- reqConfigurationIndex = req.LastLogIndex
- }
- version := getSnapshotVersion(r.protocolVersion)
- sink, err := r.snapshots.Create(version, req.LastLogIndex, req.LastLogTerm,
- reqConfiguration, reqConfigurationIndex, r.trans)
- if err != nil {
- r.logger.Error(fmt.Sprintf("Failed to create snapshot to install: %v", err))
- rpcErr = fmt.Errorf("failed to create snapshot: %v", err)
- return
- }
-
- // Spill the remote snapshot to disk
- n, err := io.Copy(sink, rpc.Reader)
- if err != nil {
- sink.Cancel()
- r.logger.Error(fmt.Sprintf("Failed to copy snapshot: %v", err))
- rpcErr = err
- return
- }
-
- // Check that we received it all
- if n != req.Size {
- sink.Cancel()
- r.logger.Error(fmt.Sprintf("Failed to receive whole snapshot: %d / %d", n, req.Size))
- rpcErr = fmt.Errorf("short read")
- return
- }
-
- // Finalize the snapshot
- if err := sink.Close(); err != nil {
- r.logger.Error(fmt.Sprintf("Failed to finalize snapshot: %v", err))
- rpcErr = err
- return
- }
- r.logger.Info(fmt.Sprintf("Copied %d bytes to local snapshot", n))
-
- // Restore snapshot
- future := &restoreFuture{ID: sink.ID()}
- future.init()
- select {
- case r.fsmMutateCh <- future:
- case <-r.shutdownCh:
- future.respond(ErrRaftShutdown)
- return
- }
-
- // Wait for the restore to happen
- if err := future.Error(); err != nil {
- r.logger.Error(fmt.Sprintf("Failed to restore snapshot: %v", err))
- rpcErr = err
- return
- }
-
- // Update the lastApplied so we don't replay old logs
- r.setLastApplied(req.LastLogIndex)
-
- // Update the last stable snapshot info
- r.setLastSnapshot(req.LastLogIndex, req.LastLogTerm)
-
- // Restore the peer set
- r.configurations.latest = reqConfiguration
- r.configurations.latestIndex = reqConfigurationIndex
- r.configurations.committed = reqConfiguration
- r.configurations.committedIndex = reqConfigurationIndex
-
- // Compact logs, continue even if this fails
- if err := r.compactLogs(req.LastLogIndex); err != nil {
- r.logger.Error(fmt.Sprintf("Failed to compact logs: %v", err))
- }
-
- r.logger.Info("Installed remote snapshot")
- resp.Success = true
- r.setLastContact()
- return
-}
-
-// setLastContact is used to set the last contact time to now
-func (r *Raft) setLastContact() {
- r.lastContactLock.Lock()
- r.lastContact = time.Now()
- r.lastContactLock.Unlock()
-}
-
-type voteResult struct {
- RequestVoteResponse
- voterID ServerID
-}
-
-// electSelf is used to send a RequestVote RPC to all peers, and vote for
-// ourself. This has the side affecting of incrementing the current term. The
-// response channel returned is used to wait for all the responses (including a
-// vote for ourself). This must only be called from the main thread.
-func (r *Raft) electSelf() <-chan *voteResult {
- // Create a response channel
- respCh := make(chan *voteResult, len(r.configurations.latest.Servers))
-
- // Increment the term
- r.setCurrentTerm(r.getCurrentTerm() + 1)
-
- // Construct the request
- lastIdx, lastTerm := r.getLastEntry()
- req := &RequestVoteRequest{
- RPCHeader: r.getRPCHeader(),
- Term: r.getCurrentTerm(),
- Candidate: r.trans.EncodePeer(r.localID, r.localAddr),
- LastLogIndex: lastIdx,
- LastLogTerm: lastTerm,
- }
-
- // Construct a function to ask for a vote
- askPeer := func(peer Server) {
- r.goFunc(func() {
- defer metrics.MeasureSince([]string{"raft", "candidate", "electSelf"}, time.Now())
- resp := &voteResult{voterID: peer.ID}
- err := r.trans.RequestVote(peer.ID, peer.Address, req, &resp.RequestVoteResponse)
- if err != nil {
- r.logger.Error(fmt.Sprintf("Failed to make RequestVote RPC to %v: %v", peer, err))
- resp.Term = req.Term
- resp.Granted = false
- }
- respCh <- resp
- })
- }
-
- // For each peer, request a vote
- for _, server := range r.configurations.latest.Servers {
- if server.Suffrage == Voter {
- if server.ID == r.localID {
- // Persist a vote for ourselves
- if err := r.persistVote(req.Term, req.Candidate); err != nil {
- r.logger.Error(fmt.Sprintf("Failed to persist vote : %v", err))
- return nil
- }
- // Include our own vote
- respCh <- &voteResult{
- RequestVoteResponse: RequestVoteResponse{
- RPCHeader: r.getRPCHeader(),
- Term: req.Term,
- Granted: true,
- },
- voterID: r.localID,
- }
- } else {
- askPeer(server)
- }
- }
- }
-
- return respCh
-}
-
-// persistVote is used to persist our vote for safety.
-func (r *Raft) persistVote(term uint64, candidate []byte) error {
- if err := r.stable.SetUint64(keyLastVoteTerm, term); err != nil {
- return err
- }
- if err := r.stable.Set(keyLastVoteCand, candidate); err != nil {
- return err
- }
- return nil
-}
-
-// setCurrentTerm is used to set the current term in a durable manner.
-func (r *Raft) setCurrentTerm(t uint64) {
- // Persist to disk first
- if err := r.stable.SetUint64(keyCurrentTerm, t); err != nil {
- panic(fmt.Errorf("failed to save current term: %v", err))
- }
- r.raftState.setCurrentTerm(t)
-}
-
-// setState is used to update the current state. Any state
-// transition causes the known leader to be cleared. This means
-// that leader should be set only after updating the state.
-func (r *Raft) setState(state RaftState) {
- r.setLeader("")
- oldState := r.raftState.getState()
- r.raftState.setState(state)
- if oldState != state {
- r.observe(state)
- }
-}
diff --git a/vendor/github.com/hashicorp/raft/replication.go b/vendor/github.com/hashicorp/raft/replication.go
deleted file mode 100644
index 1f5f1007f5..0000000000
--- a/vendor/github.com/hashicorp/raft/replication.go
+++ /dev/null
@@ -1,572 +0,0 @@
-package raft
-
-import (
- "errors"
- "fmt"
- "sync"
- "time"
-
- "github.com/armon/go-metrics"
-)
-
-const (
- maxFailureScale = 12
- failureWait = 10 * time.Millisecond
-)
-
-var (
- // ErrLogNotFound indicates a given log entry is not available.
- ErrLogNotFound = errors.New("log not found")
-
- // ErrPipelineReplicationNotSupported can be returned by the transport to
- // signal that pipeline replication is not supported in general, and that
- // no error message should be produced.
- ErrPipelineReplicationNotSupported = errors.New("pipeline replication not supported")
-)
-
-// followerReplication is in charge of sending snapshots and log entries from
-// this leader during this particular term to a remote follower.
-type followerReplication struct {
- // peer contains the network address and ID of the remote follower.
- peer Server
-
- // commitment tracks the entries acknowledged by followers so that the
- // leader's commit index can advance. It is updated on successful
- // AppendEntries responses.
- commitment *commitment
-
- // stopCh is notified/closed when this leader steps down or the follower is
- // removed from the cluster. In the follower removed case, it carries a log
- // index; replication should be attempted with a best effort up through that
- // index, before exiting.
- stopCh chan uint64
- // triggerCh is notified every time new entries are appended to the log.
- triggerCh chan struct{}
-
- // currentTerm is the term of this leader, to be included in AppendEntries
- // requests.
- currentTerm uint64
- // nextIndex is the index of the next log entry to send to the follower,
- // which may fall past the end of the log.
- nextIndex uint64
-
- // lastContact is updated to the current time whenever any response is
- // received from the follower (successful or not). This is used to check
- // whether the leader should step down (Raft.checkLeaderLease()).
- lastContact time.Time
- // lastContactLock protects 'lastContact'.
- lastContactLock sync.RWMutex
-
- // failures counts the number of failed RPCs since the last success, which is
- // used to apply backoff.
- failures uint64
-
- // notifyCh is notified to send out a heartbeat, which is used to check that
- // this server is still leader.
- notifyCh chan struct{}
- // notify is a map of futures to be resolved upon receipt of an
- // acknowledgement, then cleared from this map.
- notify map[*verifyFuture]struct{}
- // notifyLock protects 'notify'.
- notifyLock sync.Mutex
-
- // stepDown is used to indicate to the leader that we
- // should step down based on information from a follower.
- stepDown chan struct{}
-
- // allowPipeline is used to determine when to pipeline the AppendEntries RPCs.
- // It is private to this replication goroutine.
- allowPipeline bool
-}
-
-// notifyAll is used to notify all the waiting verify futures
-// if the follower believes we are still the leader.
-func (s *followerReplication) notifyAll(leader bool) {
- // Clear the waiting notifies minimizing lock time
- s.notifyLock.Lock()
- n := s.notify
- s.notify = make(map[*verifyFuture]struct{})
- s.notifyLock.Unlock()
-
- // Submit our votes
- for v, _ := range n {
- v.vote(leader)
- }
-}
-
-// cleanNotify is used to delete notify, .
-func (s *followerReplication) cleanNotify(v *verifyFuture) {
- s.notifyLock.Lock()
- delete(s.notify, v)
- s.notifyLock.Unlock()
-}
-
-// LastContact returns the time of last contact.
-func (s *followerReplication) LastContact() time.Time {
- s.lastContactLock.RLock()
- last := s.lastContact
- s.lastContactLock.RUnlock()
- return last
-}
-
-// setLastContact sets the last contact to the current time.
-func (s *followerReplication) setLastContact() {
- s.lastContactLock.Lock()
- s.lastContact = time.Now()
- s.lastContactLock.Unlock()
-}
-
-// replicate is a long running routine that replicates log entries to a single
-// follower.
-func (r *Raft) replicate(s *followerReplication) {
- // Start an async heartbeating routing
- stopHeartbeat := make(chan struct{})
- defer close(stopHeartbeat)
- r.goFunc(func() { r.heartbeat(s, stopHeartbeat) })
-
-RPC:
- shouldStop := false
- for !shouldStop {
- select {
- case maxIndex := <-s.stopCh:
- // Make a best effort to replicate up to this index
- if maxIndex > 0 {
- r.replicateTo(s, maxIndex)
- }
- return
- case <-s.triggerCh:
- lastLogIdx, _ := r.getLastLog()
- shouldStop = r.replicateTo(s, lastLogIdx)
- // This is _not_ our heartbeat mechanism but is to ensure
- // followers quickly learn the leader's commit index when
- // raft commits stop flowing naturally. The actual heartbeats
- // can't do this to keep them unblocked by disk IO on the
- // follower. See https://github.com/hashicorp/raft/issues/282.
- case <-randomTimeout(r.conf.CommitTimeout):
- lastLogIdx, _ := r.getLastLog()
- shouldStop = r.replicateTo(s, lastLogIdx)
- }
-
- // If things looks healthy, switch to pipeline mode
- if !shouldStop && s.allowPipeline {
- goto PIPELINE
- }
- }
- return
-
-PIPELINE:
- // Disable until re-enabled
- s.allowPipeline = false
-
- // Replicates using a pipeline for high performance. This method
- // is not able to gracefully recover from errors, and so we fall back
- // to standard mode on failure.
- if err := r.pipelineReplicate(s); err != nil {
- if err != ErrPipelineReplicationNotSupported {
- r.logger.Error(fmt.Sprintf("Failed to start pipeline replication to %s: %s", s.peer, err))
- }
- }
- goto RPC
-}
-
-// replicateTo is a helper to replicate(), used to replicate the logs up to a
-// given last index.
-// If the follower log is behind, we take care to bring them up to date.
-func (r *Raft) replicateTo(s *followerReplication, lastIndex uint64) (shouldStop bool) {
- // Create the base request
- var req AppendEntriesRequest
- var resp AppendEntriesResponse
- var start time.Time
-START:
- // Prevent an excessive retry rate on errors
- if s.failures > 0 {
- select {
- case <-time.After(backoff(failureWait, s.failures, maxFailureScale)):
- case <-r.shutdownCh:
- }
- }
-
- // Setup the request
- if err := r.setupAppendEntries(s, &req, s.nextIndex, lastIndex); err == ErrLogNotFound {
- goto SEND_SNAP
- } else if err != nil {
- return
- }
-
- // Make the RPC call
- start = time.Now()
- if err := r.trans.AppendEntries(s.peer.ID, s.peer.Address, &req, &resp); err != nil {
- r.logger.Error(fmt.Sprintf("Failed to AppendEntries to %v: %v", s.peer, err))
- s.failures++
- return
- }
- appendStats(string(s.peer.ID), start, float32(len(req.Entries)))
-
- // Check for a newer term, stop running
- if resp.Term > req.Term {
- r.handleStaleTerm(s)
- return true
- }
-
- // Update the last contact
- s.setLastContact()
-
- // Update s based on success
- if resp.Success {
- // Update our replication state
- updateLastAppended(s, &req)
-
- // Clear any failures, allow pipelining
- s.failures = 0
- s.allowPipeline = true
- } else {
- s.nextIndex = max(min(s.nextIndex-1, resp.LastLog+1), 1)
- if resp.NoRetryBackoff {
- s.failures = 0
- } else {
- s.failures++
- }
- r.logger.Warn(fmt.Sprintf("AppendEntries to %v rejected, sending older logs (next: %d)", s.peer, s.nextIndex))
- }
-
-CHECK_MORE:
- // Poll the stop channel here in case we are looping and have been asked
- // to stop, or have stepped down as leader. Even for the best effort case
- // where we are asked to replicate to a given index and then shutdown,
- // it's better to not loop in here to send lots of entries to a straggler
- // that's leaving the cluster anyways.
- select {
- case <-s.stopCh:
- return true
- default:
- }
-
- // Check if there are more logs to replicate
- if s.nextIndex <= lastIndex {
- goto START
- }
- return
-
- // SEND_SNAP is used when we fail to get a log, usually because the follower
- // is too far behind, and we must ship a snapshot down instead
-SEND_SNAP:
- if stop, err := r.sendLatestSnapshot(s); stop {
- return true
- } else if err != nil {
- r.logger.Error(fmt.Sprintf("Failed to send snapshot to %v: %v", s.peer, err))
- return
- }
-
- // Check if there is more to replicate
- goto CHECK_MORE
-}
-
-// sendLatestSnapshot is used to send the latest snapshot we have
-// down to our follower.
-func (r *Raft) sendLatestSnapshot(s *followerReplication) (bool, error) {
- // Get the snapshots
- snapshots, err := r.snapshots.List()
- if err != nil {
- r.logger.Error(fmt.Sprintf("Failed to list snapshots: %v", err))
- return false, err
- }
-
- // Check we have at least a single snapshot
- if len(snapshots) == 0 {
- return false, fmt.Errorf("no snapshots found")
- }
-
- // Open the most recent snapshot
- snapID := snapshots[0].ID
- meta, snapshot, err := r.snapshots.Open(snapID)
- if err != nil {
- r.logger.Error(fmt.Sprintf("Failed to open snapshot %v: %v", snapID, err))
- return false, err
- }
- defer snapshot.Close()
-
- // Setup the request
- req := InstallSnapshotRequest{
- RPCHeader: r.getRPCHeader(),
- SnapshotVersion: meta.Version,
- Term: s.currentTerm,
- Leader: r.trans.EncodePeer(r.localID, r.localAddr),
- LastLogIndex: meta.Index,
- LastLogTerm: meta.Term,
- Peers: meta.Peers,
- Size: meta.Size,
- Configuration: encodeConfiguration(meta.Configuration),
- ConfigurationIndex: meta.ConfigurationIndex,
- }
-
- // Make the call
- start := time.Now()
- var resp InstallSnapshotResponse
- if err := r.trans.InstallSnapshot(s.peer.ID, s.peer.Address, &req, &resp, snapshot); err != nil {
- r.logger.Error(fmt.Sprintf("Failed to install snapshot %v: %v", snapID, err))
- s.failures++
- return false, err
- }
- metrics.MeasureSince([]string{"raft", "replication", "installSnapshot", string(s.peer.ID)}, start)
-
- // Check for a newer term, stop running
- if resp.Term > req.Term {
- r.handleStaleTerm(s)
- return true, nil
- }
-
- // Update the last contact
- s.setLastContact()
-
- // Check for success
- if resp.Success {
- // Update the indexes
- s.nextIndex = meta.Index + 1
- s.commitment.match(s.peer.ID, meta.Index)
-
- // Clear any failures
- s.failures = 0
-
- // Notify we are still leader
- s.notifyAll(true)
- } else {
- s.failures++
- r.logger.Warn(fmt.Sprintf("InstallSnapshot to %v rejected", s.peer))
- }
- return false, nil
-}
-
-// heartbeat is used to periodically invoke AppendEntries on a peer
-// to ensure they don't time out. This is done async of replicate(),
-// since that routine could potentially be blocked on disk IO.
-func (r *Raft) heartbeat(s *followerReplication, stopCh chan struct{}) {
- var failures uint64
- req := AppendEntriesRequest{
- RPCHeader: r.getRPCHeader(),
- Term: s.currentTerm,
- Leader: r.trans.EncodePeer(r.localID, r.localAddr),
- }
- var resp AppendEntriesResponse
- for {
- // Wait for the next heartbeat interval or forced notify
- select {
- case <-s.notifyCh:
- case <-randomTimeout(r.conf.HeartbeatTimeout / 10):
- case <-stopCh:
- return
- }
-
- start := time.Now()
- if err := r.trans.AppendEntries(s.peer.ID, s.peer.Address, &req, &resp); err != nil {
- r.logger.Error(fmt.Sprintf("Failed to heartbeat to %v: %v", s.peer.Address, err))
- failures++
- select {
- case <-time.After(backoff(failureWait, failures, maxFailureScale)):
- case <-stopCh:
- }
- } else {
- s.setLastContact()
- failures = 0
- metrics.MeasureSince([]string{"raft", "replication", "heartbeat", string(s.peer.ID)}, start)
- s.notifyAll(resp.Success)
- }
- }
-}
-
-// pipelineReplicate is used when we have synchronized our state with the follower,
-// and want to switch to a higher performance pipeline mode of replication.
-// We only pipeline AppendEntries commands, and if we ever hit an error, we fall
-// back to the standard replication which can handle more complex situations.
-func (r *Raft) pipelineReplicate(s *followerReplication) error {
- // Create a new pipeline
- pipeline, err := r.trans.AppendEntriesPipeline(s.peer.ID, s.peer.Address)
- if err != nil {
- return err
- }
- defer pipeline.Close()
-
- // Log start and stop of pipeline
- r.logger.Info(fmt.Sprintf("pipelining replication to peer %v", s.peer))
- defer r.logger.Info(fmt.Sprintf("aborting pipeline replication to peer %v", s.peer))
-
- // Create a shutdown and finish channel
- stopCh := make(chan struct{})
- finishCh := make(chan struct{})
-
- // Start a dedicated decoder
- r.goFunc(func() { r.pipelineDecode(s, pipeline, stopCh, finishCh) })
-
- // Start pipeline sends at the last good nextIndex
- nextIndex := s.nextIndex
-
- shouldStop := false
-SEND:
- for !shouldStop {
- select {
- case <-finishCh:
- break SEND
- case maxIndex := <-s.stopCh:
- // Make a best effort to replicate up to this index
- if maxIndex > 0 {
- r.pipelineSend(s, pipeline, &nextIndex, maxIndex)
- }
- break SEND
- case <-s.triggerCh:
- lastLogIdx, _ := r.getLastLog()
- shouldStop = r.pipelineSend(s, pipeline, &nextIndex, lastLogIdx)
- case <-randomTimeout(r.conf.CommitTimeout):
- lastLogIdx, _ := r.getLastLog()
- shouldStop = r.pipelineSend(s, pipeline, &nextIndex, lastLogIdx)
- }
- }
-
- // Stop our decoder, and wait for it to finish
- close(stopCh)
- select {
- case <-finishCh:
- case <-r.shutdownCh:
- }
- return nil
-}
-
-// pipelineSend is used to send data over a pipeline. It is a helper to
-// pipelineReplicate.
-func (r *Raft) pipelineSend(s *followerReplication, p AppendPipeline, nextIdx *uint64, lastIndex uint64) (shouldStop bool) {
- // Create a new append request
- req := new(AppendEntriesRequest)
- if err := r.setupAppendEntries(s, req, *nextIdx, lastIndex); err != nil {
- return true
- }
-
- // Pipeline the append entries
- if _, err := p.AppendEntries(req, new(AppendEntriesResponse)); err != nil {
- r.logger.Error(fmt.Sprintf("Failed to pipeline AppendEntries to %v: %v", s.peer, err))
- return true
- }
-
- // Increase the next send log to avoid re-sending old logs
- if n := len(req.Entries); n > 0 {
- last := req.Entries[n-1]
- *nextIdx = last.Index + 1
- }
- return false
-}
-
-// pipelineDecode is used to decode the responses of pipelined requests.
-func (r *Raft) pipelineDecode(s *followerReplication, p AppendPipeline, stopCh, finishCh chan struct{}) {
- defer close(finishCh)
- respCh := p.Consumer()
- for {
- select {
- case ready := <-respCh:
- req, resp := ready.Request(), ready.Response()
- appendStats(string(s.peer.ID), ready.Start(), float32(len(req.Entries)))
-
- // Check for a newer term, stop running
- if resp.Term > req.Term {
- r.handleStaleTerm(s)
- return
- }
-
- // Update the last contact
- s.setLastContact()
-
- // Abort pipeline if not successful
- if !resp.Success {
- return
- }
-
- // Update our replication state
- updateLastAppended(s, req)
- case <-stopCh:
- return
- }
- }
-}
-
-// setupAppendEntries is used to setup an append entries request.
-func (r *Raft) setupAppendEntries(s *followerReplication, req *AppendEntriesRequest, nextIndex, lastIndex uint64) error {
- req.RPCHeader = r.getRPCHeader()
- req.Term = s.currentTerm
- req.Leader = r.trans.EncodePeer(r.localID, r.localAddr)
- req.LeaderCommitIndex = r.getCommitIndex()
- if err := r.setPreviousLog(req, nextIndex); err != nil {
- return err
- }
- if err := r.setNewLogs(req, nextIndex, lastIndex); err != nil {
- return err
- }
- return nil
-}
-
-// setPreviousLog is used to setup the PrevLogEntry and PrevLogTerm for an
-// AppendEntriesRequest given the next index to replicate.
-func (r *Raft) setPreviousLog(req *AppendEntriesRequest, nextIndex uint64) error {
- // Guard for the first index, since there is no 0 log entry
- // Guard against the previous index being a snapshot as well
- lastSnapIdx, lastSnapTerm := r.getLastSnapshot()
- if nextIndex == 1 {
- req.PrevLogEntry = 0
- req.PrevLogTerm = 0
-
- } else if (nextIndex - 1) == lastSnapIdx {
- req.PrevLogEntry = lastSnapIdx
- req.PrevLogTerm = lastSnapTerm
-
- } else {
- var l Log
- if err := r.logs.GetLog(nextIndex-1, &l); err != nil {
- r.logger.Error(fmt.Sprintf("Failed to get log at index %d: %v", nextIndex-1, err))
- return err
- }
-
- // Set the previous index and term (0 if nextIndex is 1)
- req.PrevLogEntry = l.Index
- req.PrevLogTerm = l.Term
- }
- return nil
-}
-
-// setNewLogs is used to setup the logs which should be appended for a request.
-func (r *Raft) setNewLogs(req *AppendEntriesRequest, nextIndex, lastIndex uint64) error {
- // Append up to MaxAppendEntries or up to the lastIndex
- req.Entries = make([]*Log, 0, r.conf.MaxAppendEntries)
- maxIndex := min(nextIndex+uint64(r.conf.MaxAppendEntries)-1, lastIndex)
- for i := nextIndex; i <= maxIndex; i++ {
- oldLog := new(Log)
- if err := r.logs.GetLog(i, oldLog); err != nil {
- r.logger.Error(fmt.Sprintf("Failed to get log at index %d: %v", i, err))
- return err
- }
- req.Entries = append(req.Entries, oldLog)
- }
- return nil
-}
-
-// appendStats is used to emit stats about an AppendEntries invocation.
-func appendStats(peer string, start time.Time, logs float32) {
- metrics.MeasureSince([]string{"raft", "replication", "appendEntries", "rpc", peer}, start)
- metrics.IncrCounter([]string{"raft", "replication", "appendEntries", "logs", peer}, logs)
-}
-
-// handleStaleTerm is used when a follower indicates that we have a stale term.
-func (r *Raft) handleStaleTerm(s *followerReplication) {
- r.logger.Error(fmt.Sprintf("peer %v has newer term, stopping replication", s.peer))
- s.notifyAll(false) // No longer leader
- asyncNotifyCh(s.stepDown)
-}
-
-// updateLastAppended is used to update follower replication state after a
-// successful AppendEntries RPC.
-// TODO: This isn't used during InstallSnapshot, but the code there is similar.
-func updateLastAppended(s *followerReplication, req *AppendEntriesRequest) {
- // Mark any inflight logs as committed
- if logs := req.Entries; len(logs) > 0 {
- last := logs[len(logs)-1]
- s.nextIndex = last.Index + 1
- s.commitment.match(s.peer.ID, last.Index)
- }
-
- // Notify still leader
- s.notifyAll(true)
-}
diff --git a/vendor/github.com/hashicorp/raft/snapshot.go b/vendor/github.com/hashicorp/raft/snapshot.go
deleted file mode 100644
index 2e0f77a5dd..0000000000
--- a/vendor/github.com/hashicorp/raft/snapshot.go
+++ /dev/null
@@ -1,239 +0,0 @@
-package raft
-
-import (
- "fmt"
- "io"
- "time"
-
- "github.com/armon/go-metrics"
-)
-
-// SnapshotMeta is for metadata of a snapshot.
-type SnapshotMeta struct {
- // Version is the version number of the snapshot metadata. This does not cover
- // the application's data in the snapshot, that should be versioned
- // separately.
- Version SnapshotVersion
-
- // ID is opaque to the store, and is used for opening.
- ID string
-
- // Index and Term store when the snapshot was taken.
- Index uint64
- Term uint64
-
- // Peers is deprecated and used to support version 0 snapshots, but will
- // be populated in version 1 snapshots as well to help with upgrades.
- Peers []byte
-
- // Configuration and ConfigurationIndex are present in version 1
- // snapshots and later.
- Configuration Configuration
- ConfigurationIndex uint64
-
- // Size is the size of the snapshot in bytes.
- Size int64
-}
-
-// SnapshotStore interface is used to allow for flexible implementations
-// of snapshot storage and retrieval. For example, a client could implement
-// a shared state store such as S3, allowing new nodes to restore snapshots
-// without streaming from the leader.
-type SnapshotStore interface {
- // Create is used to begin a snapshot at a given index and term, and with
- // the given committed configuration. The version parameter controls
- // which snapshot version to create.
- Create(version SnapshotVersion, index, term uint64, configuration Configuration,
- configurationIndex uint64, trans Transport) (SnapshotSink, error)
-
- // List is used to list the available snapshots in the store.
- // It should return then in descending order, with the highest index first.
- List() ([]*SnapshotMeta, error)
-
- // Open takes a snapshot ID and provides a ReadCloser. Once close is
- // called it is assumed the snapshot is no longer needed.
- Open(id string) (*SnapshotMeta, io.ReadCloser, error)
-}
-
-// SnapshotSink is returned by StartSnapshot. The FSM will Write state
-// to the sink and call Close on completion. On error, Cancel will be invoked.
-type SnapshotSink interface {
- io.WriteCloser
- ID() string
- Cancel() error
-}
-
-// runSnapshots is a long running goroutine used to manage taking
-// new snapshots of the FSM. It runs in parallel to the FSM and
-// main goroutines, so that snapshots do not block normal operation.
-func (r *Raft) runSnapshots() {
- for {
- select {
- case <-randomTimeout(r.conf.SnapshotInterval):
- // Check if we should snapshot
- if !r.shouldSnapshot() {
- continue
- }
-
- // Trigger a snapshot
- if _, err := r.takeSnapshot(); err != nil {
- r.logger.Error(fmt.Sprintf("Failed to take snapshot: %v", err))
- }
-
- case future := <-r.userSnapshotCh:
- // User-triggered, run immediately
- id, err := r.takeSnapshot()
- if err != nil {
- r.logger.Error(fmt.Sprintf("Failed to take snapshot: %v", err))
- } else {
- future.opener = func() (*SnapshotMeta, io.ReadCloser, error) {
- return r.snapshots.Open(id)
- }
- }
- future.respond(err)
-
- case <-r.shutdownCh:
- return
- }
- }
-}
-
-// shouldSnapshot checks if we meet the conditions to take
-// a new snapshot.
-func (r *Raft) shouldSnapshot() bool {
- // Check the last snapshot index
- lastSnap, _ := r.getLastSnapshot()
-
- // Check the last log index
- lastIdx, err := r.logs.LastIndex()
- if err != nil {
- r.logger.Error(fmt.Sprintf("Failed to get last log index: %v", err))
- return false
- }
-
- // Compare the delta to the threshold
- delta := lastIdx - lastSnap
- return delta >= r.conf.SnapshotThreshold
-}
-
-// takeSnapshot is used to take a new snapshot. This must only be called from
-// the snapshot thread, never the main thread. This returns the ID of the new
-// snapshot, along with an error.
-func (r *Raft) takeSnapshot() (string, error) {
- defer metrics.MeasureSince([]string{"raft", "snapshot", "takeSnapshot"}, time.Now())
-
- // Create a request for the FSM to perform a snapshot.
- snapReq := &reqSnapshotFuture{}
- snapReq.init()
-
- // Wait for dispatch or shutdown.
- select {
- case r.fsmSnapshotCh <- snapReq:
- case <-r.shutdownCh:
- return "", ErrRaftShutdown
- }
-
- // Wait until we get a response
- if err := snapReq.Error(); err != nil {
- if err != ErrNothingNewToSnapshot {
- err = fmt.Errorf("failed to start snapshot: %v", err)
- }
- return "", err
- }
- defer snapReq.snapshot.Release()
-
- // Make a request for the configurations and extract the committed info.
- // We have to use the future here to safely get this information since
- // it is owned by the main thread.
- configReq := &configurationsFuture{}
- configReq.init()
- select {
- case r.configurationsCh <- configReq:
- case <-r.shutdownCh:
- return "", ErrRaftShutdown
- }
- if err := configReq.Error(); err != nil {
- return "", err
- }
- committed := configReq.configurations.committed
- committedIndex := configReq.configurations.committedIndex
-
- // We don't support snapshots while there's a config change outstanding
- // since the snapshot doesn't have a means to represent this state. This
- // is a little weird because we need the FSM to apply an index that's
- // past the configuration change, even though the FSM itself doesn't see
- // the configuration changes. It should be ok in practice with normal
- // application traffic flowing through the FSM. If there's none of that
- // then it's not crucial that we snapshot, since there's not much going
- // on Raft-wise.
- if snapReq.index < committedIndex {
- return "", fmt.Errorf("cannot take snapshot now, wait until the configuration entry at %v has been applied (have applied %v)",
- committedIndex, snapReq.index)
- }
-
- // Create a new snapshot.
- r.logger.Info(fmt.Sprintf("Starting snapshot up to %d", snapReq.index))
- start := time.Now()
- version := getSnapshotVersion(r.protocolVersion)
- sink, err := r.snapshots.Create(version, snapReq.index, snapReq.term, committed, committedIndex, r.trans)
- if err != nil {
- return "", fmt.Errorf("failed to create snapshot: %v", err)
- }
- metrics.MeasureSince([]string{"raft", "snapshot", "create"}, start)
-
- // Try to persist the snapshot.
- start = time.Now()
- if err := snapReq.snapshot.Persist(sink); err != nil {
- sink.Cancel()
- return "", fmt.Errorf("failed to persist snapshot: %v", err)
- }
- metrics.MeasureSince([]string{"raft", "snapshot", "persist"}, start)
-
- // Close and check for error.
- if err := sink.Close(); err != nil {
- return "", fmt.Errorf("failed to close snapshot: %v", err)
- }
-
- // Update the last stable snapshot info.
- r.setLastSnapshot(snapReq.index, snapReq.term)
-
- // Compact the logs.
- if err := r.compactLogs(snapReq.index); err != nil {
- return "", err
- }
-
- r.logger.Info(fmt.Sprintf("Snapshot to %d complete", snapReq.index))
- return sink.ID(), nil
-}
-
-// compactLogs takes the last inclusive index of a snapshot
-// and trims the logs that are no longer needed.
-func (r *Raft) compactLogs(snapIdx uint64) error {
- defer metrics.MeasureSince([]string{"raft", "compactLogs"}, time.Now())
- // Determine log ranges to compact
- minLog, err := r.logs.FirstIndex()
- if err != nil {
- return fmt.Errorf("failed to get first log index: %v", err)
- }
-
- // Check if we have enough logs to truncate
- lastLogIdx, _ := r.getLastLog()
- if lastLogIdx <= r.conf.TrailingLogs {
- return nil
- }
-
- // Truncate up to the end of the snapshot, or `TrailingLogs`
- // back from the head, which ever is further back. This ensures
- // at least `TrailingLogs` entries, but does not allow logs
- // after the snapshot to be removed.
- maxLog := min(snapIdx, lastLogIdx-r.conf.TrailingLogs)
-
- // Log this
- r.logger.Info(fmt.Sprintf("Compacting logs from %d to %d", minLog, maxLog))
-
- // Compact the logs
- if err := r.logs.DeleteRange(minLog, maxLog); err != nil {
- return fmt.Errorf("log compaction failed: %v", err)
- }
- return nil
-}
diff --git a/vendor/github.com/hashicorp/raft/stable.go b/vendor/github.com/hashicorp/raft/stable.go
deleted file mode 100644
index ff59a8c570..0000000000
--- a/vendor/github.com/hashicorp/raft/stable.go
+++ /dev/null
@@ -1,15 +0,0 @@
-package raft
-
-// StableStore is used to provide stable storage
-// of key configurations to ensure safety.
-type StableStore interface {
- Set(key []byte, val []byte) error
-
- // Get returns the value for key, or an empty byte slice if key was not found.
- Get(key []byte) ([]byte, error)
-
- SetUint64(key []byte, val uint64) error
-
- // GetUint64 returns the uint64 value for key, or 0 if key was not found.
- GetUint64(key []byte) (uint64, error)
-}
diff --git a/vendor/github.com/hashicorp/raft/state.go b/vendor/github.com/hashicorp/raft/state.go
deleted file mode 100644
index a58cd0d19e..0000000000
--- a/vendor/github.com/hashicorp/raft/state.go
+++ /dev/null
@@ -1,171 +0,0 @@
-package raft
-
-import (
- "sync"
- "sync/atomic"
-)
-
-// RaftState captures the state of a Raft node: Follower, Candidate, Leader,
-// or Shutdown.
-type RaftState uint32
-
-const (
- // Follower is the initial state of a Raft node.
- Follower RaftState = iota
-
- // Candidate is one of the valid states of a Raft node.
- Candidate
-
- // Leader is one of the valid states of a Raft node.
- Leader
-
- // Shutdown is the terminal state of a Raft node.
- Shutdown
-)
-
-func (s RaftState) String() string {
- switch s {
- case Follower:
- return "Follower"
- case Candidate:
- return "Candidate"
- case Leader:
- return "Leader"
- case Shutdown:
- return "Shutdown"
- default:
- return "Unknown"
- }
-}
-
-// raftState is used to maintain various state variables
-// and provides an interface to set/get the variables in a
-// thread safe manner.
-type raftState struct {
- // currentTerm commitIndex, lastApplied, must be kept at the top of
- // the struct so they're 64 bit aligned which is a requirement for
- // atomic ops on 32 bit platforms.
-
- // The current term, cache of StableStore
- currentTerm uint64
-
- // Highest committed log entry
- commitIndex uint64
-
- // Last applied log to the FSM
- lastApplied uint64
-
- // protects 4 next fields
- lastLock sync.Mutex
-
- // Cache the latest snapshot index/term
- lastSnapshotIndex uint64
- lastSnapshotTerm uint64
-
- // Cache the latest log from LogStore
- lastLogIndex uint64
- lastLogTerm uint64
-
- // Tracks running goroutines
- routinesGroup sync.WaitGroup
-
- // The current state
- state RaftState
-}
-
-func (r *raftState) getState() RaftState {
- stateAddr := (*uint32)(&r.state)
- return RaftState(atomic.LoadUint32(stateAddr))
-}
-
-func (r *raftState) setState(s RaftState) {
- stateAddr := (*uint32)(&r.state)
- atomic.StoreUint32(stateAddr, uint32(s))
-}
-
-func (r *raftState) getCurrentTerm() uint64 {
- return atomic.LoadUint64(&r.currentTerm)
-}
-
-func (r *raftState) setCurrentTerm(term uint64) {
- atomic.StoreUint64(&r.currentTerm, term)
-}
-
-func (r *raftState) getLastLog() (index, term uint64) {
- r.lastLock.Lock()
- index = r.lastLogIndex
- term = r.lastLogTerm
- r.lastLock.Unlock()
- return
-}
-
-func (r *raftState) setLastLog(index, term uint64) {
- r.lastLock.Lock()
- r.lastLogIndex = index
- r.lastLogTerm = term
- r.lastLock.Unlock()
-}
-
-func (r *raftState) getLastSnapshot() (index, term uint64) {
- r.lastLock.Lock()
- index = r.lastSnapshotIndex
- term = r.lastSnapshotTerm
- r.lastLock.Unlock()
- return
-}
-
-func (r *raftState) setLastSnapshot(index, term uint64) {
- r.lastLock.Lock()
- r.lastSnapshotIndex = index
- r.lastSnapshotTerm = term
- r.lastLock.Unlock()
-}
-
-func (r *raftState) getCommitIndex() uint64 {
- return atomic.LoadUint64(&r.commitIndex)
-}
-
-func (r *raftState) setCommitIndex(index uint64) {
- atomic.StoreUint64(&r.commitIndex, index)
-}
-
-func (r *raftState) getLastApplied() uint64 {
- return atomic.LoadUint64(&r.lastApplied)
-}
-
-func (r *raftState) setLastApplied(index uint64) {
- atomic.StoreUint64(&r.lastApplied, index)
-}
-
-// Start a goroutine and properly handle the race between a routine
-// starting and incrementing, and exiting and decrementing.
-func (r *raftState) goFunc(f func()) {
- r.routinesGroup.Add(1)
- go func() {
- defer r.routinesGroup.Done()
- f()
- }()
-}
-
-func (r *raftState) waitShutdown() {
- r.routinesGroup.Wait()
-}
-
-// getLastIndex returns the last index in stable storage.
-// Either from the last log or from the last snapshot.
-func (r *raftState) getLastIndex() uint64 {
- r.lastLock.Lock()
- defer r.lastLock.Unlock()
- return max(r.lastLogIndex, r.lastSnapshotIndex)
-}
-
-// getLastEntry returns the last index and term in stable storage.
-// Either from the last log or from the last snapshot.
-func (r *raftState) getLastEntry() (uint64, uint64) {
- r.lastLock.Lock()
- defer r.lastLock.Unlock()
- if r.lastLogIndex >= r.lastSnapshotIndex {
- return r.lastLogIndex, r.lastLogTerm
- }
- return r.lastSnapshotIndex, r.lastSnapshotTerm
-}
diff --git a/vendor/github.com/hashicorp/raft/tag.sh b/vendor/github.com/hashicorp/raft/tag.sh
deleted file mode 100755
index cd16623a70..0000000000
--- a/vendor/github.com/hashicorp/raft/tag.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-# The version must be supplied from the environment. Do not include the
-# leading "v".
-if [ -z $VERSION ]; then
- echo "Please specify a version."
- exit 1
-fi
-
-# Generate the tag.
-echo "==> Tagging version $VERSION..."
-git commit --allow-empty -a --gpg-sign=348FFC4C -m "Release v$VERSION"
-git tag -a -m "Version $VERSION" -s -u 348FFC4C "v${VERSION}" master
-
-exit 0
diff --git a/vendor/github.com/hashicorp/raft/tcp_transport.go b/vendor/github.com/hashicorp/raft/tcp_transport.go
deleted file mode 100644
index 69c928ed92..0000000000
--- a/vendor/github.com/hashicorp/raft/tcp_transport.go
+++ /dev/null
@@ -1,116 +0,0 @@
-package raft
-
-import (
- "errors"
- "io"
- "log"
- "net"
- "time"
-)
-
-var (
- errNotAdvertisable = errors.New("local bind address is not advertisable")
- errNotTCP = errors.New("local address is not a TCP address")
-)
-
-// TCPStreamLayer implements StreamLayer interface for plain TCP.
-type TCPStreamLayer struct {
- advertise net.Addr
- listener *net.TCPListener
-}
-
-// NewTCPTransport returns a NetworkTransport that is built on top of
-// a TCP streaming transport layer.
-func NewTCPTransport(
- bindAddr string,
- advertise net.Addr,
- maxPool int,
- timeout time.Duration,
- logOutput io.Writer,
-) (*NetworkTransport, error) {
- return newTCPTransport(bindAddr, advertise, func(stream StreamLayer) *NetworkTransport {
- return NewNetworkTransport(stream, maxPool, timeout, logOutput)
- })
-}
-
-// NewTCPTransportWithLogger returns a NetworkTransport that is built on top of
-// a TCP streaming transport layer, with log output going to the supplied Logger
-func NewTCPTransportWithLogger(
- bindAddr string,
- advertise net.Addr,
- maxPool int,
- timeout time.Duration,
- logger *log.Logger,
-) (*NetworkTransport, error) {
- return newTCPTransport(bindAddr, advertise, func(stream StreamLayer) *NetworkTransport {
- return NewNetworkTransportWithLogger(stream, maxPool, timeout, logger)
- })
-}
-
-// NewTCPTransportWithConfig returns a NetworkTransport that is built on top of
-// a TCP streaming transport layer, using the given config struct.
-func NewTCPTransportWithConfig(
- bindAddr string,
- advertise net.Addr,
- config *NetworkTransportConfig,
-) (*NetworkTransport, error) {
- return newTCPTransport(bindAddr, advertise, func(stream StreamLayer) *NetworkTransport {
- config.Stream = stream
- return NewNetworkTransportWithConfig(config)
- })
-}
-
-func newTCPTransport(bindAddr string,
- advertise net.Addr,
- transportCreator func(stream StreamLayer) *NetworkTransport) (*NetworkTransport, error) {
- // Try to bind
- list, err := net.Listen("tcp", bindAddr)
- if err != nil {
- return nil, err
- }
-
- // Create stream
- stream := &TCPStreamLayer{
- advertise: advertise,
- listener: list.(*net.TCPListener),
- }
-
- // Verify that we have a usable advertise address
- addr, ok := stream.Addr().(*net.TCPAddr)
- if !ok {
- list.Close()
- return nil, errNotTCP
- }
- if addr.IP.IsUnspecified() {
- list.Close()
- return nil, errNotAdvertisable
- }
-
- // Create the network transport
- trans := transportCreator(stream)
- return trans, nil
-}
-
-// Dial implements the StreamLayer interface.
-func (t *TCPStreamLayer) Dial(address ServerAddress, timeout time.Duration) (net.Conn, error) {
- return net.DialTimeout("tcp", string(address), timeout)
-}
-
-// Accept implements the net.Listener interface.
-func (t *TCPStreamLayer) Accept() (c net.Conn, err error) {
- return t.listener.Accept()
-}
-
-// Close implements the net.Listener interface.
-func (t *TCPStreamLayer) Close() (err error) {
- return t.listener.Close()
-}
-
-// Addr implements the net.Listener interface.
-func (t *TCPStreamLayer) Addr() net.Addr {
- // Use an advertise addr if provided
- if t.advertise != nil {
- return t.advertise
- }
- return t.listener.Addr()
-}
diff --git a/vendor/github.com/hashicorp/raft/transport.go b/vendor/github.com/hashicorp/raft/transport.go
deleted file mode 100644
index 85459b221d..0000000000
--- a/vendor/github.com/hashicorp/raft/transport.go
+++ /dev/null
@@ -1,124 +0,0 @@
-package raft
-
-import (
- "io"
- "time"
-)
-
-// RPCResponse captures both a response and a potential error.
-type RPCResponse struct {
- Response interface{}
- Error error
-}
-
-// RPC has a command, and provides a response mechanism.
-type RPC struct {
- Command interface{}
- Reader io.Reader // Set only for InstallSnapshot
- RespChan chan<- RPCResponse
-}
-
-// Respond is used to respond with a response, error or both
-func (r *RPC) Respond(resp interface{}, err error) {
- r.RespChan <- RPCResponse{resp, err}
-}
-
-// Transport provides an interface for network transports
-// to allow Raft to communicate with other nodes.
-type Transport interface {
- // Consumer returns a channel that can be used to
- // consume and respond to RPC requests.
- Consumer() <-chan RPC
-
- // LocalAddr is used to return our local address to distinguish from our peers.
- LocalAddr() ServerAddress
-
- // AppendEntriesPipeline returns an interface that can be used to pipeline
- // AppendEntries requests.
- AppendEntriesPipeline(id ServerID, target ServerAddress) (AppendPipeline, error)
-
- // AppendEntries sends the appropriate RPC to the target node.
- AppendEntries(id ServerID, target ServerAddress, args *AppendEntriesRequest, resp *AppendEntriesResponse) error
-
- // RequestVote sends the appropriate RPC to the target node.
- RequestVote(id ServerID, target ServerAddress, args *RequestVoteRequest, resp *RequestVoteResponse) error
-
- // InstallSnapshot is used to push a snapshot down to a follower. The data is read from
- // the ReadCloser and streamed to the client.
- InstallSnapshot(id ServerID, target ServerAddress, args *InstallSnapshotRequest, resp *InstallSnapshotResponse, data io.Reader) error
-
- // EncodePeer is used to serialize a peer's address.
- EncodePeer(id ServerID, addr ServerAddress) []byte
-
- // DecodePeer is used to deserialize a peer's address.
- DecodePeer([]byte) ServerAddress
-
- // SetHeartbeatHandler is used to setup a heartbeat handler
- // as a fast-pass. This is to avoid head-of-line blocking from
- // disk IO. If a Transport does not support this, it can simply
- // ignore the call, and push the heartbeat onto the Consumer channel.
- SetHeartbeatHandler(cb func(rpc RPC))
-}
-
-// WithClose is an interface that a transport may provide which
-// allows a transport to be shut down cleanly when a Raft instance
-// shuts down.
-//
-// It is defined separately from Transport as unfortunately it wasn't in the
-// original interface specification.
-type WithClose interface {
- // Close permanently closes a transport, stopping
- // any associated goroutines and freeing other resources.
- Close() error
-}
-
-// LoopbackTransport is an interface that provides a loopback transport suitable for testing
-// e.g. InmemTransport. It's there so we don't have to rewrite tests.
-type LoopbackTransport interface {
- Transport // Embedded transport reference
- WithPeers // Embedded peer management
- WithClose // with a close routine
-}
-
-// WithPeers is an interface that a transport may provide which allows for connection and
-// disconnection. Unless the transport is a loopback transport, the transport specified to
-// "Connect" is likely to be nil.
-type WithPeers interface {
- Connect(peer ServerAddress, t Transport) // Connect a peer
- Disconnect(peer ServerAddress) // Disconnect a given peer
- DisconnectAll() // Disconnect all peers, possibly to reconnect them later
-}
-
-// AppendPipeline is used for pipelining AppendEntries requests. It is used
-// to increase the replication throughput by masking latency and better
-// utilizing bandwidth.
-type AppendPipeline interface {
- // AppendEntries is used to add another request to the pipeline.
- // The send may block which is an effective form of back-pressure.
- AppendEntries(args *AppendEntriesRequest, resp *AppendEntriesResponse) (AppendFuture, error)
-
- // Consumer returns a channel that can be used to consume
- // response futures when they are ready.
- Consumer() <-chan AppendFuture
-
- // Close closes the pipeline and cancels all inflight RPCs
- Close() error
-}
-
-// AppendFuture is used to return information about a pipelined AppendEntries request.
-type AppendFuture interface {
- Future
-
- // Start returns the time that the append request was started.
- // It is always OK to call this method.
- Start() time.Time
-
- // Request holds the parameters of the AppendEntries call.
- // It is always OK to call this method.
- Request() *AppendEntriesRequest
-
- // Response holds the results of the AppendEntries call.
- // This method must only be called after the Error
- // method returns, and will only be valid on success.
- Response() *AppendEntriesResponse
-}
diff --git a/vendor/github.com/hashicorp/raft/util.go b/vendor/github.com/hashicorp/raft/util.go
deleted file mode 100644
index 90428d7437..0000000000
--- a/vendor/github.com/hashicorp/raft/util.go
+++ /dev/null
@@ -1,133 +0,0 @@
-package raft
-
-import (
- "bytes"
- crand "crypto/rand"
- "fmt"
- "math"
- "math/big"
- "math/rand"
- "time"
-
- "github.com/hashicorp/go-msgpack/codec"
-)
-
-func init() {
- // Ensure we use a high-entropy seed for the psuedo-random generator
- rand.Seed(newSeed())
-}
-
-// returns an int64 from a crypto random source
-// can be used to seed a source for a math/rand.
-func newSeed() int64 {
- r, err := crand.Int(crand.Reader, big.NewInt(math.MaxInt64))
- if err != nil {
- panic(fmt.Errorf("failed to read random bytes: %v", err))
- }
- return r.Int64()
-}
-
-// randomTimeout returns a value that is between the minVal and 2x minVal.
-func randomTimeout(minVal time.Duration) <-chan time.Time {
- if minVal == 0 {
- return nil
- }
- extra := (time.Duration(rand.Int63()) % minVal)
- return time.After(minVal + extra)
-}
-
-// min returns the minimum.
-func min(a, b uint64) uint64 {
- if a <= b {
- return a
- }
- return b
-}
-
-// max returns the maximum.
-func max(a, b uint64) uint64 {
- if a >= b {
- return a
- }
- return b
-}
-
-// generateUUID is used to generate a random UUID.
-func generateUUID() string {
- buf := make([]byte, 16)
- if _, err := crand.Read(buf); err != nil {
- panic(fmt.Errorf("failed to read random bytes: %v", err))
- }
-
- return fmt.Sprintf("%08x-%04x-%04x-%04x-%12x",
- buf[0:4],
- buf[4:6],
- buf[6:8],
- buf[8:10],
- buf[10:16])
-}
-
-// asyncNotifyCh is used to do an async channel send
-// to a single channel without blocking.
-func asyncNotifyCh(ch chan struct{}) {
- select {
- case ch <- struct{}{}:
- default:
- }
-}
-
-// drainNotifyCh empties out a single-item notification channel without
-// blocking, and returns whether it received anything.
-func drainNotifyCh(ch chan struct{}) bool {
- select {
- case <-ch:
- return true
- default:
- return false
- }
-}
-
-// asyncNotifyBool is used to do an async notification
-// on a bool channel.
-func asyncNotifyBool(ch chan bool, v bool) {
- select {
- case ch <- v:
- default:
- }
-}
-
-// Decode reverses the encode operation on a byte slice input.
-func decodeMsgPack(buf []byte, out interface{}) error {
- r := bytes.NewBuffer(buf)
- hd := codec.MsgpackHandle{}
- dec := codec.NewDecoder(r, &hd)
- return dec.Decode(out)
-}
-
-// Encode writes an encoded object to a new bytes buffer.
-func encodeMsgPack(in interface{}) (*bytes.Buffer, error) {
- buf := bytes.NewBuffer(nil)
- hd := codec.MsgpackHandle{}
- enc := codec.NewEncoder(buf, &hd)
- err := enc.Encode(in)
- return buf, err
-}
-
-// backoff is used to compute an exponential backoff
-// duration. Base time is scaled by the current round,
-// up to some maximum scale factor.
-func backoff(base time.Duration, round, limit uint64) time.Duration {
- power := min(round, limit)
- for power > 2 {
- base *= 2
- power--
- }
- return base
-}
-
-// Needed for sorting []uint64, used to determine commitment
-type uint64Slice []uint64
-
-func (p uint64Slice) Len() int { return len(p) }
-func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
-func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
diff --git a/vendor/vendor.json b/vendor/vendor.json
deleted file mode 100644
index 42f68b2267..0000000000
--- a/vendor/vendor.json
+++ /dev/null
@@ -1,45 +0,0 @@
-{
- "comment": "",
- "ignore": "test",
- "package": [
- {
- "checksumSHA1": "HF3V9ieTLnqjlDcqyGmHxYojZXE=",
- "path": "github.com/CanonicalLtd/go-dqlite",
- "revision": "3eab944668d7af5d0fc69ddb387ffda76300541c",
- "revisionTime": "2019-03-22T09:57:25Z",
- "tree": true
- },
- {
- "checksumSHA1": "5UAXxv+O1Oxx8kQAUvR94zCVy+Q=",
- "path": "github.com/CanonicalLtd/raft-http",
- "revision": "4c2dd679d3b46c11b250d63ae43467d4c4ab0962",
- "revisionTime": "2018-04-14T15:56:53Z"
- },
- {
- "checksumSHA1": "nflIYP3tDRTgp2g4I1qoK8fDgmc=",
- "path": "github.com/CanonicalLtd/raft-membership",
- "revision": "3846634b0164affd0b3dfba1fdd7f9da6387e501",
- "revisionTime": "2018-04-13T13:33:40Z"
- },
- {
- "checksumSHA1": "nbblYWwQstB9B+OhB1zoDFLhYWQ=",
- "path": "github.com/CanonicalLtd/raft-test",
- "revision": "586f073e84d2c7bbf01340756979db76179c7a7a",
- "revisionTime": "2019-04-30T22:51:17Z",
- "tree": true
- },
- {
- "checksumSHA1": "RMI9XuADcv+6w3jS5FpqzjDKuhI=",
- "path": "github.com/hashicorp/raft",
- "revision": "2c551690b5c0eb05ef7f4ad72ed01f7f6ce3fcb6",
- "revisionTime": "2019-05-11T03:54:14Z"
- },
- {
- "checksumSHA1": "Y2PM65le0fGtiD12RaKknBscFys=",
- "path": "github.com/hashicorp/raft-boltdb",
- "revision": "6e5ba93211eaf8d9a2ad7e41ffad8c6f160f9fe3",
- "revisionTime": "2017-10-10T15:18:10Z"
- }
- ],
- "rootPath": "github.com/lxc/lxd"
-}
From 5182d1a95bc0725e0b31e7de8b6ae443df0f7112 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 9 May 2019 10:03:31 +0200
Subject: [PATCH 39/39] Fix typo in test/suites/clustering.sh
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
test/suites/clustering.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/test/suites/clustering.sh b/test/suites/clustering.sh
index 6ddfea0bd9..625a9b8c2d 100644
--- a/test/suites/clustering.sh
+++ b/test/suites/clustering.sh
@@ -969,7 +969,7 @@ test_clustering_shutdown_nodes() {
ns3="${prefix}3"
spawn_lxd_and_join_cluster "${ns3}" "${bridge}" "${cert}" 3 1 "${LXD_THREE_DIR}"
- # Init a container on node2, using a client connected to node1
+ # Init a container on node1, using a client connected to node1
LXD_DIR="${LXD_ONE_DIR}" ensure_import_testimage
LXD_DIR="${LXD_ONE_DIR}" lxc launch --target node1 testimage foo
More information about the lxc-devel
mailing list