diff --git a/api/v1/heimdall.pb.go b/api/v1/heimdall.pb.go index e22ca8e..b1c32a0 100644 --- a/api/v1/heimdall.pb.go +++ b/api/v1/heimdall.pb.go @@ -81,6 +81,7 @@ func (m *Master) GetRedisURL() string { type ConnectRequest struct { ID string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + ClusterKey string `protobuf:"bytes,2,opt,name=cluster_key,json=clusterKey,proto3" json:"cluster_key,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -117,6 +118,13 @@ func (m *ConnectRequest) GetID() string { return "" } +func (m *ConnectRequest) GetClusterKey() string { + if m != nil { + return m.ClusterKey + } + return "" +} + type ConnectResponse struct { Master *Master `protobuf:"bytes,1,opt,name=master,proto3" json:"master,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -283,30 +291,31 @@ func init() { } var fileDescriptor_b6184fc395da86b1 = []byte{ - // 354 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x92, 0xc1, 0x6b, 0xc2, 0x30, - 0x14, 0xc6, 0xb1, 0xba, 0x4e, 0x5f, 0x9d, 0x42, 0x18, 0xa3, 0x78, 0xd1, 0xf5, 0x32, 0x85, 0x91, - 0xd2, 0xee, 0x32, 0x18, 0x3b, 0xa8, 0x83, 0x6d, 0xb0, 0xc9, 0x08, 0xf3, 0xb2, 0x8b, 0xd4, 0x36, - 0xd4, 0x8c, 0xda, 0x74, 0x49, 0xf5, 0x34, 0xd8, 0x7f, 0xea, 0xc1, 0xbf, 0x64, 0x34, 0x0d, 0x82, - 0x07, 0x99, 0xbb, 0xe5, 0xbd, 0x7c, 0xbf, 0xc7, 0xf7, 0x3d, 0x1e, 0xdc, 0xc6, 0x2c, 0x5f, 0xac, - 0xe6, 0x38, 0xe4, 0x4b, 0x57, 0xe6, 0x34, 0x49, 0x02, 0x91, 0x09, 0xfe, 0x49, 0xc3, 0xdc, 0x5d, - 0x50, 0xb6, 0x8c, 0x82, 0x24, 0x71, 0x83, 0x8c, 0xb9, 0x6b, 0x6f, 0x57, 0xe3, 0x4c, 0xf0, 0x9c, - 0xa3, 0x4b, 0xc6, 0xf1, 0x3e, 0x81, 0x77, 0x8a, 0x20, 0x63, 0x78, 0xed, 0x75, 0xce, 0x63, 0x1e, - 0x73, 0xa5, 0x76, 0x8b, 0x57, 0x09, 0x3a, 0x3f, 0x60, 0xbe, 0x06, 0x32, 0xa7, 0x02, 0x5d, 0x80, - 0xc1, 0x22, 0xbb, 0xd2, 0xab, 0xf4, 0x1b, 0x23, 0x73, 0xbb, 0xe9, 0x1a, 0xcf, 0x0f, 0xc4, 0x60, - 0x11, 0xf2, 0xa1, 0x19, 0x8b, 0x2c, 0x9c, 0x05, 0x51, 0x24, 0xa8, 0x94, 0xb6, 0xa1, 0x14, 0xed, - 0xed, 0xa6, 0x6b, 0x3d, 0x92, 0xb7, 0xf1, 0xb0, 0x6c, 0x13, 0xab, 0x10, 0xe9, 0x02, 0x0d, 0xa0, - 0x21, 0x68, 0xc4, 0xe4, 0x6c, 0x25, 0x12, 0xbb, 0xaa, 0x80, 0xe6, 0x76, 0xd3, 0xad, 0x93, 0xa2, - 0x39, 0x25, 0x2f, 0xa4, 0xae, 0xbe, 0xa7, 0x22, 0x71, 0xfa, 0xd0, 0x1a, 0xf3, 0x34, 0xa5, 0x61, - 0x4e, 0xe8, 0xd7, 0x8a, 0xca, 0xfc, 0x90, 0x11, 0xe7, 0x1d, 0xda, 0x3b, 0xa5, 0xcc, 0x78, 0x2a, - 0x29, 0x1a, 0x82, 0xb9, 0x54, 0xee, 0x95, 0xdc, 0xf2, 0x07, 0xf8, 0xcf, 0x3d, 0xe0, 0x32, 0x2e, - 0xd1, 0xa0, 0xe3, 0x43, 0x6d, 0xc2, 0x23, 0x7a, 0x30, 0x3e, 0x82, 0x5a, 0x91, 0xbc, 0x8c, 0x4d, - 0xd4, 0xdb, 0x69, 0x41, 0xb3, 0x60, 0xa4, 0x76, 0xec, 0x4c, 0xe0, 0x4c, 0xd7, 0xda, 0xd7, 0x3d, - 0x9c, 0xa4, 0x45, 0xc3, 0xae, 0xf4, 0xaa, 0x7d, 0xcb, 0xbf, 0x3a, 0xc2, 0x56, 0x31, 0x80, 0x94, - 0x94, 0xff, 0x0d, 0xf5, 0x27, 0xfd, 0x8d, 0x32, 0x38, 0xd5, 0xa9, 0x91, 0x77, 0xc4, 0x98, 0xfd, - 0x5d, 0x76, 0xfc, 0xff, 0x20, 0xa5, 0xf9, 0x11, 0xfe, 0xb8, 0x3e, 0xfa, 0x0e, 0xef, 0xd6, 0xde, - 0xdc, 0x54, 0x97, 0x74, 0xf3, 0x1b, 0x00, 0x00, 0xff, 0xff, 0x2b, 0x3d, 0x45, 0x9b, 0xbe, 0x02, - 0x00, 0x00, + // 375 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x92, 0xc1, 0xab, 0xda, 0x40, + 0x10, 0xc6, 0x31, 0xda, 0x54, 0x27, 0x56, 0x61, 0x29, 0x25, 0x78, 0x89, 0xcd, 0xa5, 0x0a, 0x65, + 0x43, 0xd2, 0x4b, 0xa1, 0xf4, 0xa0, 0x16, 0x5a, 0x69, 0x2b, 0x65, 0xa9, 0x97, 0x5e, 0x24, 0x26, + 0x4b, 0xdc, 0x36, 0x66, 0xd3, 0xdd, 0x44, 0x10, 0x0a, 0xfd, 0x4f, 0x3d, 0xf8, 0x97, 0x3c, 0x76, + 0xb3, 0x08, 0x1e, 0xe4, 0xf9, 0x6e, 0x3b, 0xb3, 0xf3, 0x7d, 0xfc, 0xe6, 0x63, 0xe0, 0x7d, 0xc6, + 0xaa, 0x5d, 0xbd, 0xc5, 0x09, 0xdf, 0x07, 0xb2, 0xa2, 0x79, 0x1e, 0x8b, 0x52, 0xf0, 0xdf, 0x34, + 0xa9, 0x82, 0x1d, 0x65, 0xfb, 0x34, 0xce, 0xf3, 0x20, 0x2e, 0x59, 0x70, 0x08, 0x2f, 0x35, 0x2e, + 0x05, 0xaf, 0x38, 0x7a, 0xcd, 0x38, 0xbe, 0x56, 0xe0, 0xcb, 0x44, 0x5c, 0x32, 0x7c, 0x08, 0x47, + 0x2f, 0x33, 0x9e, 0x71, 0x3d, 0x1d, 0xa8, 0x57, 0x23, 0xf4, 0xff, 0x83, 0xfd, 0x3d, 0x96, 0x15, + 0x15, 0xe8, 0x15, 0x58, 0x2c, 0x75, 0x5b, 0xe3, 0xd6, 0xa4, 0x37, 0xb7, 0xcf, 0x27, 0xcf, 0x5a, + 0x7e, 0x22, 0x16, 0x4b, 0x51, 0x04, 0xfd, 0x4c, 0x94, 0xc9, 0x26, 0x4e, 0x53, 0x41, 0xa5, 0x74, + 0x2d, 0x3d, 0x31, 0x3c, 0x9f, 0x3c, 0xe7, 0x33, 0xf9, 0xb1, 0x98, 0x35, 0x6d, 0xe2, 0xa8, 0x21, + 0x53, 0xa0, 0x29, 0xf4, 0x04, 0x4d, 0x99, 0xdc, 0xd4, 0x22, 0x77, 0xdb, 0x5a, 0xd0, 0x3f, 0x9f, + 0xbc, 0x2e, 0x51, 0xcd, 0x35, 0xf9, 0x46, 0xba, 0xfa, 0x7b, 0x2d, 0x72, 0x7f, 0x09, 0x83, 0x05, + 0x2f, 0x0a, 0x9a, 0x54, 0x84, 0xfe, 0xad, 0xa9, 0xac, 0x6e, 0x82, 0x78, 0xe0, 0x24, 0x79, 0xad, + 0x58, 0x37, 0x7f, 0xe8, 0xb1, 0xe1, 0x20, 0x60, 0x5a, 0x5f, 0xe9, 0xd1, 0xff, 0x09, 0xc3, 0x8b, + 0x95, 0x2c, 0x79, 0x21, 0x29, 0x9a, 0x81, 0xbd, 0xd7, 0xeb, 0x69, 0x3f, 0x27, 0x9a, 0xe2, 0x47, + 0x83, 0xc2, 0x4d, 0x1e, 0xc4, 0x08, 0xfd, 0x08, 0x3a, 0x2b, 0x9e, 0xd2, 0x9b, 0x58, 0x08, 0x3a, + 0x2a, 0x1a, 0xc3, 0xa3, 0xdf, 0xfe, 0x00, 0xfa, 0x4a, 0x23, 0xcd, 0x4a, 0xfe, 0x0a, 0x5e, 0x98, + 0xda, 0x70, 0x7d, 0x84, 0x67, 0x85, 0x6a, 0xb8, 0xad, 0x71, 0x7b, 0xe2, 0x44, 0x6f, 0xee, 0xc0, + 0x52, 0x06, 0xa4, 0x51, 0x45, 0xff, 0xa0, 0xfb, 0xc5, 0x7c, 0xa3, 0x12, 0x9e, 0x9b, 0xad, 0x51, + 0x78, 0x87, 0xcd, 0x75, 0xd8, 0xa3, 0xe8, 0x29, 0x92, 0x06, 0x7e, 0x8e, 0x7f, 0xbd, 0xbd, 0xfb, + 0x50, 0x3f, 0x1c, 0xc2, 0xad, 0xad, 0x4f, 0xed, 0xdd, 0x43, 0x00, 0x00, 0x00, 0xff, 0xff, 0x57, + 0x3c, 0x78, 0x51, 0xdf, 0x02, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/api/v1/heimdall.proto b/api/v1/heimdall.proto index c2fd728..d2be522 100644 --- a/api/v1/heimdall.proto +++ b/api/v1/heimdall.proto @@ -22,6 +22,7 @@ message Master { message ConnectRequest { string id = 1 [(gogoproto.customname) = "ID"]; + string cluster_key = 2; } message ConnectResponse { diff --git a/client/connect.go b/client/connect.go index 72d5904..6ecd53d 100644 --- a/client/connect.go +++ b/client/connect.go @@ -28,10 +28,11 @@ import ( ) // Connect attempts to connect to the peer and returns the master info -func (c *Client) Connect() (*v1.Master, error) { +func (c *Client) Connect(key string) (*v1.Master, error) { ctx := context.Background() resp, err := c.heimdallClient.Connect(ctx, &v1.ConnectRequest{ - ID: c.id, + ID: c.id, + ClusterKey: key, }) if err != nil { return nil, err diff --git a/cmd/heimdall/main.go b/cmd/heimdall/main.go index ab13255..8355f31 100644 --- a/cmd/heimdall/main.go +++ b/cmd/heimdall/main.go @@ -25,6 +25,7 @@ import ( "fmt" "os" + "github.com/google/uuid" log "github.com/sirupsen/logrus" "github.com/stellarproject/heimdall" "github.com/stellarproject/heimdall/version" @@ -76,6 +77,12 @@ func main() { Usage: "grpc address to join a peer", EnvVar: "HEIMDALL_PEER", }, + cli.StringFlag{ + Name: "cluster-key", + Usage: "preshared key for cluster peer joins", + Value: generateKey(), + EnvVar: "HEIMDALL_CLUSTER_KEY", + }, } app.Before = func(c *cli.Context) error { if c.Bool("debug") { @@ -90,3 +97,7 @@ func main() { log.Fatal(err) } } + +func generateKey() string { + return uuid.New().String() +} diff --git a/cmd/heimdall/run.go b/cmd/heimdall/run.go index 8027beb..a491a1f 100644 --- a/cmd/heimdall/run.go +++ b/cmd/heimdall/run.go @@ -44,6 +44,7 @@ func runServer(cx *cli.Context) error { ID: cx.String("id"), GRPCAddress: cx.String("addr"), GRPCPeerAddress: cx.String("peer"), + ClusterKey: cx.String("cluster-key"), RedisURL: cx.String("redis-url"), AdvertiseRedisURL: cx.String("advertise-redis-url"), } diff --git a/config.go b/config.go index 88d50d8..cfe2a06 100644 --- a/config.go +++ b/config.go @@ -29,6 +29,8 @@ type Config struct { GRPCAddress string // GRPCPeerAddress is the peer address to join GRPCPeerAddress string + // ClusterKey is a preshared key for cluster peers + ClusterKey string // RedisURL is the uri to the redis backend RedisURL string // AdvertiseRedisURL is the uri to the public redis backend diff --git a/server/connect.go b/server/connect.go index 4e40202..46489f8 100644 --- a/server/connect.go +++ b/server/connect.go @@ -23,6 +23,7 @@ package server import ( "context" + "errors" "github.com/gogo/protobuf/proto" "github.com/gomodule/redigo/redis" @@ -30,9 +31,21 @@ import ( v1 "github.com/stellarproject/heimdall/api/v1" ) +var ( + // ErrInvalidAuth is returned when an invalid cluster key is specified upon connect + ErrInvalidAuth = errors.New("invalid cluster key specified") +) + // Connect is called when a peer wants to connect to the node func (s *Server) Connect(ctx context.Context, req *v1.ConnectRequest) (*v1.ConnectResponse, error) { logrus.Debugf("connect request from %s", req.ID) + key, err := s.getClusterKey(ctx) + if err != nil { + return nil, err + } + if req.ClusterKey != key { + return nil, ErrInvalidAuth + } data, err := redis.Bytes(s.local(ctx, "GET", masterKey)) if err != nil { return nil, err diff --git a/server/node.go b/server/node.go index fb41a5d..2e82ba9 100644 --- a/server/node.go +++ b/server/node.go @@ -60,7 +60,7 @@ func (s *Server) configureNode() error { logrus.Warn(err) continue } - m, err := c.Connect() + m, err := c.Connect(s.cfg.ClusterKey) if err != nil { c.Close() logrus.Warn(err) @@ -149,13 +149,17 @@ func (s *Server) replicaMonitor() { func (s *Server) masterHeartbeat() { logrus.Debugf("starting master heartbeat: ttl=%s", heartbeatInterval) // initial update - if err := s.updateMasterInfo(); err != nil { + ctx, cancel := context.WithTimeout(context.Background(), heartbeatInterval) + defer cancel() + + logrus.Infof("cluster master key=%s", s.cfg.ClusterKey) + if err := s.updateMasterInfo(ctx); err != nil { logrus.Error(err) } t := time.NewTicker(heartbeatInterval) for range t.C { - if err := s.updateMasterInfo(); err != nil { + if err := s.updateMasterInfo(ctx); err != nil { logrus.Error(err) continue } @@ -193,8 +197,11 @@ func (s *Server) joinMaster(m *v1.Master) error { return nil } -func (s *Server) updateMasterInfo() error { +func (s *Server) updateMasterInfo(ctx context.Context) error { // update master info + if _, err := s.master(ctx, "SET", clusterKey, s.cfg.ClusterKey); err != nil { + return err + } m := &v1.Master{ ID: s.cfg.ID, GRPCAddress: s.cfg.GRPCAddress, @@ -205,27 +212,28 @@ func (s *Server) updateMasterInfo() error { return errors.Wrap(err, "error marshalling master info") } - if _, err := s.master(context.Background(), "SET", masterKey, data); err != nil { + if _, err := s.master(ctx, "SET", masterKey, data); err != nil { return errors.Wrap(err, "error setting master info") } - if _, err := s.master(context.Background(), "EXPIRE", masterKey, int(heartbeatInterval.Seconds())); err != nil { + if _, err := s.master(ctx, "EXPIRE", masterKey, int(heartbeatInterval.Seconds())); err != nil { return errors.Wrap(err, "error setting expire for master info") } return nil } func (s *Server) nodeHeartbeat() { - logrus.Debugf("starting node heartbeat: ttl=%s", heartbeatInterval) - t := time.NewTicker(heartbeatInterval) + logrus.Debugf("starting node heartbeat: ttl=%s", nodeHeartbeatInterval) + ctx := context.Background() + t := time.NewTicker(nodeHeartbeatInterval) key := fmt.Sprintf("%s:%s", nodesKey, s.cfg.ID) for range t.C { - if _, err := s.master(context.Background(), "SET", key, s.cfg.GRPCAddress); err != nil { + if _, err := s.master(ctx, "SET", key, s.cfg.GRPCAddress); err != nil { logrus.Error(err) continue } - if _, err := s.master(context.Background(), "EXPIRE", key, nodeHeartbeatExpiry); err != nil { + if _, err := s.master(ctx, "EXPIRE", key, nodeHeartbeatExpiry); err != nil { logrus.Error(err) continue } diff --git a/server/server.go b/server/server.go index aeea7b8..4af4023 100644 --- a/server/server.go +++ b/server/server.go @@ -40,14 +40,16 @@ import ( const ( masterKey = "heimdall:master" + clusterKey = "heimdall:key" nodesKey = "heimdall:nodes" nodeJoinKey = "heimdall:join" ) var ( - empty = &ptypes.Empty{} - heartbeatInterval = time.Second * 15 - nodeHeartbeatExpiry = 86400 + empty = &ptypes.Empty{} + heartbeatInterval = time.Second * 5 + nodeHeartbeatInterval = time.Second * 60 + nodeHeartbeatExpiry = 86400 ) type Server struct { @@ -97,7 +99,7 @@ func (s *Server) Run() error { } defer c.Close() - master, err := c.Connect() + master, err := c.Connect(s.cfg.ClusterKey) if err != nil { return err } @@ -163,6 +165,10 @@ func (s *Server) getClient(addr string) (*client.Client, error) { return client.NewClient(s.cfg.ID, addr) } +func (s *Server) getClusterKey(ctx context.Context) (string, error) { + return redis.String(s.local(ctx, "GET", clusterKey)) +} + func (s *Server) local(ctx context.Context, cmd string, args ...interface{}) (interface{}, error) { return s.do(ctx, s.rpool, cmd, args...) }