Resign Leadership (#414)

* Added resign leadership job to operator. (Version check missing)
* Added version check.
* Added missing deep copy of new field.
* New go-driver.
* Take current version from member.

Author: Lars Maier

go.mod

@@ -23,7 +23,7 @@ require (
 	github.com/aktau/github-release v0.7.2
 	github.com/arangodb-helper/go-certificates v0.0.0-20180821055445-9fca24fc2680
 	github.com/arangodb/arangosync-client v0.6.3
-	github.com/arangodb/go-driver v0.0.0-20190514134119-4cd2eb482d16
+	github.com/arangodb/go-driver v0.0.0-20190802095550-7a2c11a3ff12
 	github.com/arangodb/go-upgrade-rules v0.0.0-20180809110947-031b4774ff21
 	github.com/arangodb/go-velocypack v0.0.0-20190129082528-7896a965b4ad
 	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a

go.sum

@@ -63,6 +63,10 @@ github.com/arangodb/go-driver v0.0.0-20190430103524-b14f41496c3d h1:9w1hB1yqZ8B1
 github.com/arangodb/go-driver v0.0.0-20190430103524-b14f41496c3d/go.mod h1:5nMxlcbN6PHA05uGUw2EH6PIQHvdRVTrg/S/GM8jG4w=
 github.com/arangodb/go-driver v0.0.0-20190514134119-4cd2eb482d16 h1:D0+YgwcHnePwiKuV1lOzJLPZb6Tw8hHK2qKSQGqN28s=
 github.com/arangodb/go-driver v0.0.0-20190514134119-4cd2eb482d16/go.mod h1:5nMxlcbN6PHA05uGUw2EH6PIQHvdRVTrg/S/GM8jG4w=
+github.com/arangodb/go-driver v0.0.0-20190704144722-b504aec1c4d1 h1:tozWjsO7b891qPZJTZoC4HnMmlV3wrBFo/y9DnKMOVA=
+github.com/arangodb/go-driver v0.0.0-20190704144722-b504aec1c4d1/go.mod h1:5nMxlcbN6PHA05uGUw2EH6PIQHvdRVTrg/S/GM8jG4w=
+github.com/arangodb/go-driver v0.0.0-20190802095550-7a2c11a3ff12 h1:SZZFNsy/6c3CAv9HmhPukMtkbpdt5u1kZ1AaZElDGqE=
+github.com/arangodb/go-driver v0.0.0-20190802095550-7a2c11a3ff12/go.mod h1:5nMxlcbN6PHA05uGUw2EH6PIQHvdRVTrg/S/GM8jG4w=
 github.com/arangodb/go-upgrade-rules v0.0.0-20180809110947-031b4774ff21 h1:+W7D5ttxi/Ygh/39vialtypE23p9KI7P0J2qtoqUV4w=
 github.com/arangodb/go-upgrade-rules v0.0.0-20180809110947-031b4774ff21/go.mod h1:RkPIG6JJ2pcJUoymc18NxAJGraZd+iAEVnOTDjZey/w=
 github.com/arangodb/go-velocypack v0.0.0-20190129082528-7896a965b4ad h1:Ah+VRYUWLuqgbfnDyuC8IrIe8XFzpt9tBVVnPGFNTJ8=

pkg/apis/deployment/v1alpha/deployment_spec.go

@@ -54,6 +54,7 @@ type DeploymentSpec struct {
 	ImagePullPolicy *v1.PullPolicy  `json:"imagePullPolicy,omitempty"`
 	DowntimeAllowed *bool           `json:"downtimeAllowed,omitempty"`
 	DisableIPv6     *bool           `json:"disableIPv6,omitempty"`
+	LocallyAttachedVolumes *bool 	`json:"locallyAttachedVolumes,omitempty"`
 
 	ExternalAccess ExternalAccessSpec `json:"externalAccess"`
 	RocksDB        RocksDBSpec        `json:"rocksdb"`
@@ -123,6 +124,11 @@ func (s DeploymentSpec) IsDisableIPv6() bool {
 	return util.BoolOrDefault(s.DisableIPv6)
 }
 
+// IsLocallyAttachedVolumes returns the value of locallyAttachedVolumes, default true
+func (s DeploymentSpec) IsLocallyAttachedVolumes() bool {
+	return util.BoolOrDefault(s.LocallyAttachedVolumes, true)
+}
+
 // GetListenAddr returns "[::]" or "0.0.0.0" depending on IsDisableIPv6
 func (s DeploymentSpec) GetListenAddr() string {
 	if s.IsDisableIPv6() {

pkg/apis/deployment/v1alpha/member_phase.go

@@ -36,6 +36,8 @@ const (
 	MemberPhaseCleanOut MemberPhase = "CleanOut"
 	// MemberPhaseDrain indicates that a dbserver is in the process of being cleaned out as result of draining a node
 	MemberPhaseDrain MemberPhase = "Drain"
+	// MemberPhaseResign indicates that a dbserver is in the process of resigning for a shutdown
+	MemberPhaseResign MemberPhase = "Resign"
 	// MemberPhaseShuttingDown indicates that a member is shutting down
 	MemberPhaseShuttingDown MemberPhase = "ShuttingDown"
 	// MemberPhaseRotating indicates that a member is being rotated

pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go

@@ -28,14 +28,14 @@ import (
 	"time"
 
 	"github.com/rs/zerolog"
-	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	driver "github.com/arangodb/go-driver"
 	"github.com/arangodb/go-driver/agency"
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
 	"github.com/arangodb/kube-arangodb/pkg/util/arangod"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+	v1 "k8s.io/api/core/v1"
 )
 
 // prepareAgencyPodTermination checks if the given agency pod is allowed to terminate
@@ -137,6 +137,16 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol
 		return nil
 	}
 
+	resignJobAvailable := false
+	currentVersion := memberStatus.ArangoVersion
+	if currentVersion != "" {
+		if currentVersion.CompareTo("3.4.7") > 0 && currentVersion.CompareTo("3.5") < 0 {
+		  resignJobAvailable = true
+	  } else if currentVersion.CompareTo("3.5.0") > 0 {
+			resignJobAvailable = true
+		}
+	}
+
 	// Check node the pod is scheduled on
 	dbserverDataWillBeGone := false
 	if p.Spec.NodeName != "" {
@@ -147,7 +157,9 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol
 			log.Warn().Err(err).Msg("Failed to get node for member")
 			return maskAny(err)
 		} else if node.Spec.Unschedulable {
-			dbserverDataWillBeGone = true
+			if r.context.GetSpec().IsLocallyAttachedVolumes() || !resignJobAvailable {
+				dbserverDataWillBeGone = true
+			}
 		}
 	}
 
@@ -168,13 +180,12 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol
 	}
 
 	// Is this a simple pod restart?
-	if !dbserverDataWillBeGone {
+	if !dbserverDataWillBeGone && !resignJobAvailable {
 		log.Debug().Msg("Pod is just being restarted, safe to remove dbserver pod")
 		return nil
 	}
 
 	// Inspect cleaned out state
-	log.Debug().Msg("DBServer data is being deleted, so we will cleanout the dbserver first")
 	c, err := r.context.GetDatabaseClient(ctx)
 	if err != nil {
 		log.Debug().Err(err).Msg("Failed to create member client")
@@ -201,7 +212,7 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol
 	}
 	// Not cleaned out yet, check member status
 	if memberStatus.Conditions.IsTrue(api.ConditionTypeTerminated) {
-		log.Warn().Msg("Member is already terminated before it could be cleaned out. Not good, but removing dbserver pod because we cannot do anything further")
+		log.Warn().Msg("Member is already terminated before it could resign or be cleaned out. Not good, but removing dbserver pod because we cannot do anything further")
 		// At this point we have to set CleanedOut to true,
 		// because we can no longer reason about the state in the agency and
 		// bringing back the dbserver again may result in an cleaned out server without us knowing
@@ -220,13 +231,24 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol
 		var jobID string
 		ctx = driver.WithJobIDResponse(ctx, &jobID)
 		// Ensure the cleanout is triggered
-		log.Debug().Msg("Server is not yet clean out. Triggering a clean out now")
-		if err := cluster.CleanOutServer(ctx, memberStatus.ID); err != nil {
-			log.Debug().Err(err).Msg("Failed to clean out server")
-			return maskAny(err)
+		if dbserverDataWillBeGone {
+			log.Debug().Msg("Server is not yet cleaned out. Triggering a clean out now")
+			if err := cluster.CleanOutServer(ctx, memberStatus.ID); err != nil {
+				log.Debug().Err(err).Msg("Failed to clean out server")
+				return maskAny(err)
+			}
+			memberStatus.Phase = api.MemberPhaseDrain
+		} else {
+			log.Debug().Msg("Temporary shutdown, resign leadership")
+			if err := cluster.ResignServer(ctx, memberStatus.ID); err != nil {
+				log.Debug().Err(err).Msg("Failed to resign server")
+				return maskAny(err)
+			}
+			memberStatus.Phase = api.MemberPhaseResign
 		}
+
 		memberStatus.CleanoutJobID = jobID
-		memberStatus.Phase = api.MemberPhaseDrain
+
 		if err := updateMember(memberStatus); err != nil {
 			return maskAny(err)
 		}
@@ -239,18 +261,54 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol
 		}
 		jobStatus, err := arangod.CleanoutServerJobStatus(ctx, memberStatus.CleanoutJobID, c, agency)
 		if err != nil {
-			log.Debug().Err(err).Msg("Failed to fetch cleanout job status")
+			log.Debug().Err(err).Msg("Failed to fetch job status")
+			return maskAny(err)
+		}
+		if jobStatus.IsFailed() {
+			log.Warn().Str("reason", jobStatus.Reason()).Msg("Job failed")
+			// Revert cleanout state
+			memberStatus.Phase = api.MemberPhaseCreated
+			memberStatus.CleanoutJobID = ""
+			if err := updateMember(memberStatus); err != nil {
+				return maskAny(err)
+			}
+			log.Error().Msg("Cleanout/Resign server job failed, continue anyway")
+			return nil
+		}
+		if jobStatus.IsFinished() {
+			memberStatus.CleanoutJobID = ""
+			memberStatus.Phase = api.MemberPhaseCreated
+		}
+	} else if memberStatus.Phase == api.MemberPhaseResign {
+		// Check the job progress
+		agency, err := r.context.GetAgency(ctx)
+		if err != nil {
+			log.Debug().Err(err).Msg("Failed to create agency client")
+			return maskAny(err)
+		}
+		jobStatus, err := arangod.CleanoutServerJobStatus(ctx, memberStatus.CleanoutJobID, c, agency)
+		if err != nil {
+			log.Debug().Err(err).Msg("Failed to fetch job status")
 			return maskAny(err)
 		}
 		if jobStatus.IsFailed() {
-			log.Warn().Str("reason", jobStatus.Reason()).Msg("Cleanout Job failed")
+			log.Warn().Str("reason", jobStatus.Reason()).Msg("Resign Job failed")
 			// Revert cleanout state
 			memberStatus.Phase = api.MemberPhaseCreated
 			memberStatus.CleanoutJobID = ""
 			if err := updateMember(memberStatus); err != nil {
 				return maskAny(err)
 			}
-			log.Error().Msg("Cleanout server job failed, continue anyway")
+			log.Error().Msg("Cleanout/Resign server job failed, continue anyway")
+			return nil
+		}
+		if jobStatus.IsFinished() {
+			log.Debug().Str("reason", jobStatus.Reason()).Msg("Resign Job finished")
+			memberStatus.CleanoutJobID = ""
+			memberStatus.Phase = api.MemberPhaseCreated
+			if err := updateMember(memberStatus); err != nil {
+				return maskAny(err)
+			}
 			return nil
 		}
 	}