traefik/vendor/github.com/mesos/mesos-go/detector/zoo/detect.go

370 lines
11 KiB
Go
Raw Normal View History

2017-02-07 21:33:23 +00:00
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package zoo
import (
"encoding/json"
"fmt"
"math"
"net/url"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/gogo/protobuf/proto"
log "github.com/golang/glog"
"github.com/mesos/mesos-go/detector"
mesos "github.com/mesos/mesos-go/mesosproto"
)
const (
// prefix for nodes listed at the ZK URL path
nodePrefix = "info_"
nodeJSONPrefix = "json.info_"
defaultMinDetectorCyclePeriod = 1 * time.Second
)
// reasonable default for a noop change listener
var ignoreChanged = detector.OnMasterChanged(func(*mesos.MasterInfo) {})
type zkInterface interface {
stopped() <-chan struct{}
stop()
data(string) ([]byte, error)
watchChildren(string) (string, <-chan []string, <-chan error)
}
type infoCodec func(path, node string) (*mesos.MasterInfo, error)
// Detector uses ZooKeeper to detect new leading master.
type MasterDetector struct {
client zkInterface
leaderNode string
bootstrapLock sync.RWMutex // guard against concurrent invocations of bootstrapFunc
bootstrapFunc func() error // for one-time zk client initiation
// latch: only install, at most, one ignoreChanged listener; see MasterDetector.Detect
ignoreInstalled int32
// detection should not signal master change listeners more frequently than this
minDetectorCyclePeriod time.Duration
done chan struct{}
cancel func()
}
// Internal constructor function
func NewMasterDetector(zkurls string) (*MasterDetector, error) {
zkHosts, zkPath, err := parseZk(zkurls)
if err != nil {
log.Fatalln("Failed to parse url", err)
return nil, err
}
detector := &MasterDetector{
minDetectorCyclePeriod: defaultMinDetectorCyclePeriod,
done: make(chan struct{}),
cancel: func() {},
}
detector.bootstrapFunc = func() (err error) {
if detector.client == nil {
detector.client, err = connect2(zkHosts, zkPath)
}
return
}
log.V(2).Infoln("Created new detector to watch", zkHosts, zkPath)
return detector, nil
}
func parseZk(zkurls string) ([]string, string, error) {
u, err := url.Parse(zkurls)
if err != nil {
log.V(1).Infof("failed to parse url: %v", err)
return nil, "", err
}
if u.Scheme != "zk" {
return nil, "", fmt.Errorf("invalid url scheme for zk url: '%v'", u.Scheme)
}
return strings.Split(u.Host, ","), u.Path, nil
}
// returns a chan that, when closed, indicates termination of the detector
func (md *MasterDetector) Done() <-chan struct{} {
return md.done
}
func (md *MasterDetector) Cancel() {
md.bootstrapLock.RLock()
defer md.bootstrapLock.RUnlock()
md.cancel()
}
func (md *MasterDetector) childrenChanged(path string, list []string, obs detector.MasterChanged) {
md.notifyMasterChanged(path, list, obs)
md.notifyAllMasters(path, list, obs)
}
func (md *MasterDetector) notifyMasterChanged(path string, list []string, obs detector.MasterChanged) {
// mesos v0.24 writes JSON only, v0.23 writes json and protobuf, v0.22 and prior only write protobuf
topNode, codec := md.selectTopNode(list)
if md.leaderNode == topNode {
log.V(2).Infof("ignoring children-changed event, leader has not changed: %v", path)
return
}
log.V(2).Infof("changing leader node from %q -> %q", md.leaderNode, topNode)
md.leaderNode = topNode
var masterInfo *mesos.MasterInfo
if md.leaderNode != "" {
var err error
if masterInfo, err = codec(path, topNode); err != nil {
log.Errorln(err.Error())
}
}
log.V(2).Infof("detected master info: %+v", masterInfo)
logPanic(func() { obs.OnMasterChanged(masterInfo) })
}
// logPanic safely executes the given func, recovering from and logging a panic if one occurs.
func logPanic(f func()) {
defer func() {
if r := recover(); r != nil {
log.Errorf("recovered from client panic: %v", r)
}
}()
f()
}
func (md *MasterDetector) pullMasterInfo(path, node string) (*mesos.MasterInfo, error) {
data, err := md.client.data(fmt.Sprintf("%s/%s", path, node))
if err != nil {
return nil, fmt.Errorf("failed to retrieve leader data: %v", err)
}
masterInfo := &mesos.MasterInfo{}
err = proto.Unmarshal(data, masterInfo)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal protobuf MasterInfo data from zookeeper: %v", err)
}
return masterInfo, nil
}
func (md *MasterDetector) pullMasterJsonInfo(path, node string) (*mesos.MasterInfo, error) {
data, err := md.client.data(fmt.Sprintf("%s/%s", path, node))
if err != nil {
return nil, fmt.Errorf("failed to retrieve leader data: %v", err)
}
masterInfo := &mesos.MasterInfo{}
err = json.Unmarshal(data, masterInfo)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal json MasterInfo data from zookeeper: %v", err)
}
return masterInfo, nil
}
func (md *MasterDetector) notifyAllMasters(path string, list []string, obs detector.MasterChanged) {
all, ok := obs.(detector.AllMasters)
if !ok {
// not interested in entire master list
return
}
// mesos v0.24 writes JSON only, v0.23 writes json and protobuf, v0.22 and prior only write protobuf
masters := map[string]*mesos.MasterInfo{}
tryStore := func(node string, codec infoCodec) {
info, err := codec(path, node)
if err != nil {
log.Errorln(err.Error())
} else {
masters[info.GetId()] = info
}
}
for _, node := range list {
// compare https://github.com/apache/mesos/blob/0.23.0/src/master/detector.cpp#L437
if strings.HasPrefix(node, nodePrefix) {
tryStore(node, md.pullMasterInfo)
} else if strings.HasPrefix(node, nodeJSONPrefix) {
tryStore(node, md.pullMasterJsonInfo)
} else {
continue
}
}
masterList := make([]*mesos.MasterInfo, 0, len(masters))
for _, v := range masters {
masterList = append(masterList, v)
}
log.V(2).Infof("notifying of master membership change: %+v", masterList)
logPanic(func() { all.UpdatedMasters(masterList) })
}
func (md *MasterDetector) callBootstrap() (e error) {
log.V(2).Infoln("invoking detector boostrap")
md.bootstrapLock.Lock()
defer md.bootstrapLock.Unlock()
clientConfigured := md.client != nil
if e = md.bootstrapFunc(); e == nil && !clientConfigured && md.client != nil {
// chain the lifetime of this detector to that of the newly created client impl
client := md.client
md.cancel = client.stop
go func() {
defer close(md.done)
<-client.stopped()
}()
}
return
}
// the first call to Detect will kickstart a connection to zookeeper. a nil change listener may
// be spec'd, result of which is a detector that will still listen for master changes and record
// leaderhip changes internally but no listener would be notified. Detect may be called more than
// once, and each time the spec'd listener will be added to the list of those receiving notifications.
func (md *MasterDetector) Detect(f detector.MasterChanged) (err error) {
// kickstart zk client connectivity
if err := md.callBootstrap(); err != nil {
log.V(3).Infoln("failed to execute bootstrap function", err.Error())
return err
}
if f == nil {
// only ever install, at most, one ignoreChanged listener. multiple instances of it
// just consume resources and generate misleading log messages.
if !atomic.CompareAndSwapInt32(&md.ignoreInstalled, 0, 1) {
log.V(3).Infoln("ignoreChanged listener already installed")
return
}
f = ignoreChanged
}
log.V(3).Infoln("spawning detect()")
go md.detect(f)
return nil
}
func (md *MasterDetector) detect(f detector.MasterChanged) {
log.V(3).Infoln("detecting children at", currentPath)
detectLoop:
for {
select {
case <-md.Done():
return
default:
}
log.V(3).Infoln("watching children at", currentPath)
path, childrenCh, errCh := md.client.watchChildren(currentPath)
rewatch := false
for {
started := time.Now()
select {
case children := <-childrenCh:
md.childrenChanged(path, children, f)
case err, ok := <-errCh:
// check for a tie first (required for predictability (tests)); the downside of
// doing this is that a listener might get two callbacks back-to-back ("new leader",
// followed by "no leader").
select {
case children := <-childrenCh:
md.childrenChanged(path, children, f)
default:
}
if ok {
log.V(1).Infoln("child watch ended with error, master lost; error was:", err.Error())
} else {
// detector shutdown likely...
log.V(1).Infoln("child watch ended, master lost")
}
select {
case <-md.Done():
return
default:
if md.leaderNode != "" {
log.V(2).Infof("changing leader node from %q -> \"\"", md.leaderNode)
md.leaderNode = ""
f.OnMasterChanged(nil)
}
}
rewatch = true
}
// rate-limit master changes
if elapsed := time.Now().Sub(started); elapsed > 0 {
log.V(2).Infoln("resting before next detection cycle")
select {
case <-md.Done():
return
case <-time.After(md.minDetectorCyclePeriod - elapsed): // noop
}
}
if rewatch {
continue detectLoop
}
}
}
}
func (md *MasterDetector) selectTopNode(list []string) (topNode string, codec infoCodec) {
// mesos v0.24 writes JSON only, v0.23 writes json and protobuf, v0.22 and prior only write protobuf
topNode = selectTopNodePrefix(list, nodeJSONPrefix)
codec = md.pullMasterJsonInfo
if topNode == "" {
topNode = selectTopNodePrefix(list, nodePrefix)
codec = md.pullMasterInfo
if topNode != "" {
log.Warningf("Leading master is using a Protobuf binary format when registering "+
"with Zookeeper (%s): this will be deprecated as of Mesos 0.24 (see MESOS-2340).",
topNode)
}
}
return
}
func selectTopNodePrefix(list []string, pre string) (node string) {
var leaderSeq uint64 = math.MaxUint64
for _, v := range list {
if !strings.HasPrefix(v, pre) {
continue // only care about participants
}
seqStr := strings.TrimPrefix(v, pre)
seq, err := strconv.ParseUint(seqStr, 10, 64)
if err != nil {
log.Warningf("unexpected zk node format '%s': %v", seqStr, err)
continue
}
if seq < leaderSeq {
leaderSeq = seq
node = v
}
}
if node == "" {
log.V(3).Infoln("No top node found.")
} else {
log.V(3).Infof("Top node selected: '%s'", node)
}
return node
}