a941bcb679
* Changing incorrect license in some source files. * Changing CreateService to mimic CreateJob by setting the batch size to the instance count. * Changing Getcerts to GetCerts to match the style of the rest of the codebase. * Overhauled error handling. Backoff now recognizes temporary errors and continues to retry if it finds one. * Changed thrift function call wrapper to be more explicitly named and to perform more safety checks. * Moved Jitter function from realis to retry. * API code is now more uniform and follows a certain template. * Lock added whenever a thrift call is made or when a modification is done to the connection. Note that calling ReestablishConn externally may result in some race conditions. We will move to make this function private in the near future. * Added test for Realis session thread safety. Tested ScheduleStatus monitor. Tested monitor timing out. * Returning nil whenever there is an error return so that there are no ambiguities. * Using defer with unlock so that the lock is still released if a panic is invoked.
82 lines
2.3 KiB
Go
82 lines
2.3 KiB
Go
/*
|
|
Copyright 2014 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package realis
|
|
|
|
import (
|
|
"errors"
|
|
"time"
|
|
|
|
"math/rand"
|
|
)
|
|
|
|
// Jitter returns a time.Duration between duration and duration + maxFactor *
|
|
// duration.
|
|
//
|
|
// This allows clients to avoid converging on periodic behavior. If maxFactor
|
|
// is 0.0, a suggested default value will be chosen.
|
|
func Jitter(duration time.Duration, maxFactor float64) time.Duration {
|
|
if maxFactor <= 0.0 {
|
|
maxFactor = 1.0
|
|
}
|
|
wait := duration + time.Duration(rand.Float64()*maxFactor*float64(duration))
|
|
return wait
|
|
}
|
|
|
|
// ConditionFunc returns true if the condition is satisfied, or an error
|
|
// if the loop should be aborted.
|
|
type ConditionFunc func() (done bool, err error)
|
|
|
|
// Modified version of the Kubernetes exponential-backoff code.
|
|
// ExponentialBackoff repeats a condition check with exponential backoff.
|
|
//
|
|
// It checks the condition up to Steps times, increasing the wait by multiplying
|
|
// the previous duration by Factor.
|
|
//
|
|
// If Jitter is greater than zero, a random amount of each duration is added
|
|
// (between duration and duration*(1+jitter)).
|
|
//
|
|
// If the condition never returns true, ErrWaitTimeout is returned. All other
|
|
// errors terminate immediately.
|
|
func ExponentialBackoff(backoff Backoff, condition ConditionFunc) error {
|
|
duration := backoff.Duration
|
|
for i := 0; i < backoff.Steps; i++ {
|
|
if i != 0 {
|
|
adjusted := duration
|
|
if backoff.Jitter > 0.0 {
|
|
adjusted = Jitter(duration, backoff.Jitter)
|
|
}
|
|
time.Sleep(adjusted)
|
|
duration = time.Duration(float64(duration) * backoff.Factor)
|
|
}
|
|
|
|
ok, err := condition()
|
|
|
|
// If the function executed says it succeeded, stop retrying
|
|
if ok {
|
|
return nil
|
|
}
|
|
|
|
// Stop retrying if the error is NOT temporary.
|
|
if err != nil {
|
|
if !IsTemporary(err) {
|
|
return err
|
|
}
|
|
}
|
|
|
|
}
|
|
return NewTimeoutError(errors.New("Timed out while retrying"))
|
|
}
|