Add verification to retry mechanism (#131)
CreateJob, CreateService, and StartJobUpdate now include a rudimentary verification function to check if the call made it to the Aurora Scheduler when the client experiences a timeout.
This commit is contained in:
Renán I. Del Valle
GitHub
parent
a9d99067ee
commit
82b40a53f0
7 changed files with 286 additions and 108 deletions
163
retry.go
163
retry.go
|
|
@ -114,10 +114,19 @@ func ExponentialBackoff(backoff Backoff, logger logger, condition ConditionFunc)
|
|||
|
||||
type auroraThriftCall func() (resp *aurora.Response, err error)
|
||||
|
||||
// verifyOntimeout defines the type of function that will be used to verify whether a Thirft call to the Scheduler
|
||||
// made it to the scheduler or not. In general, these types of functions will have to interact with the scheduler
|
||||
// through the very same Thrift API which previously encountered a time out from the client.
|
||||
// This means that the functions themselves should be kept to a minimum number of Thrift calls.
|
||||
// It should also be noted that this is a best effort mechanism and
|
||||
// is likely to fail for the same reasons that the original call failed.
|
||||
type verifyOnTimeout func() (*aurora.Response, bool)
|
||||
|
||||
// Duplicates the functionality of ExponentialBackoff but is specifically targeted towards ThriftCalls.
|
||||
func (r *realisClient) thriftCallWithRetries(
|
||||
returnOnTimeout bool,
|
||||
thriftCall auroraThriftCall) (*aurora.Response, error) {
|
||||
thriftCall auroraThriftCall,
|
||||
verifyOnTimeout verifyOnTimeout) (*aurora.Response, error) {
|
||||
|
||||
var resp *aurora.Response
|
||||
var clientErr error
|
||||
|
|
@ -157,42 +166,22 @@ func (r *realisClient) thriftCallWithRetries(
|
|||
r.logger.tracePrintf("Aurora Thrift Call ended resp: %v clientErr: %v", resp, clientErr)
|
||||
}()
|
||||
|
||||
// Check if our thrift call is returning an error. This is a retryable event as we don't know
|
||||
// if it was caused by network issues.
|
||||
// Check if our thrift call is returning an error.
|
||||
if clientErr != nil {
|
||||
|
||||
// Print out the error to the user
|
||||
r.logger.Printf("Client Error: %v", clientErr)
|
||||
|
||||
// Determine if error is a temporary URL error by going up the stack
|
||||
e, ok := clientErr.(thrift.TTransportException)
|
||||
if ok {
|
||||
r.logger.debugPrint("Encountered a transport exception")
|
||||
temporary, timedout := isConnectionError(clientErr)
|
||||
if !temporary && r.RealisConfig().failOnPermanentErrors {
|
||||
return nil, errors.Wrap(clientErr, "permanent connection error")
|
||||
}
|
||||
|
||||
e, ok := e.Err().(*url.Error)
|
||||
if ok {
|
||||
|
||||
// EOF error occurs when the server closes the read buffer of the client. This is common
|
||||
// when the server is overloaded and should be retried. All other errors that are permanent
|
||||
// will not be retried.
|
||||
if e.Err != io.EOF && !e.Temporary() && r.RealisConfig().failOnPermanentErrors {
|
||||
return nil, errors.Wrap(clientErr, "permanent connection error")
|
||||
}
|
||||
|
||||
// Corner case where thrift payload was received by Aurora but connection timed out before Aurora was
|
||||
// able to reply. In this case we will return whatever response was received and a TimedOut behaving
|
||||
// error. Users can take special action on a timeout by using IsTimedout and reacting accordingly.
|
||||
if e.Timeout() {
|
||||
timeouts++
|
||||
r.logger.debugPrintf(
|
||||
"Client closed connection (timedout) %d times before server responded, "+
|
||||
"consider increasing connection timeout",
|
||||
timeouts)
|
||||
if returnOnTimeout {
|
||||
return resp, newTimedoutError(errors.New("client connection closed before server answer"))
|
||||
}
|
||||
}
|
||||
}
|
||||
// There exists a corner case where thrift payload was received by Aurora but
|
||||
// connection timed out before Aurora was able to reply.
|
||||
// Users can take special action on a timeout by using IsTimedout and reacting accordingly
|
||||
// if they have configured the client to return on a timeout.
|
||||
if timedout && returnOnTimeout {
|
||||
return resp, newTimedoutError(errors.New("client connection closed before server answer"))
|
||||
}
|
||||
|
||||
// In the future, reestablish connection should be able to check if it is actually possible
|
||||
|
|
@ -202,48 +191,71 @@ func (r *realisClient) thriftCallWithRetries(
|
|||
if reestablishErr != nil {
|
||||
r.logger.debugPrintf("error re-establishing connection ", reestablishErr)
|
||||
}
|
||||
} else {
|
||||
|
||||
// If there was no client error, but the response is nil, something went wrong.
|
||||
// Ideally, we'll never encounter this but we're placing a safeguard here.
|
||||
if resp == nil {
|
||||
return nil, errors.New("response from aurora is nil")
|
||||
// If users did not opt for a return on timeout in order to react to a timedout error,
|
||||
// attempt to verify that the call made it to the scheduler after the connection was re-established.
|
||||
if timedout {
|
||||
timeouts++
|
||||
r.logger.debugPrintf(
|
||||
"Client closed connection %d times before server responded, "+
|
||||
"consider increasing connection timeout",
|
||||
timeouts)
|
||||
|
||||
// Allow caller to provide a function which checks if the original call was successful before
|
||||
// it timed out.
|
||||
if verifyOnTimeout != nil {
|
||||
if verifyResp, ok := verifyOnTimeout(); ok {
|
||||
r.logger.Print("verified that the call went through successfully after a client timeout")
|
||||
// Response here might be different than the original as it is no longer constructed
|
||||
// by the scheduler but mimicked.
|
||||
// This is OK since the scheduler is very unlikely to change responses at this point in its
|
||||
// development cycle but we must be careful to not return an incorrectly constructed response.
|
||||
return verifyResp, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check Response Code from thrift and make a decision to continue retrying or not.
|
||||
switch responseCode := resp.GetResponseCode(); responseCode {
|
||||
// Retry the thrift payload
|
||||
continue
|
||||
}
|
||||
|
||||
// If the thrift call succeeded, stop retrying
|
||||
case aurora.ResponseCode_OK:
|
||||
return resp, nil
|
||||
// If there was no client error, but the response is nil, something went wrong.
|
||||
// Ideally, we'll never encounter this but we're placing a safeguard here.
|
||||
if resp == nil {
|
||||
return nil, errors.New("response from aurora is nil")
|
||||
}
|
||||
|
||||
// If the response code is transient, continue retrying
|
||||
case aurora.ResponseCode_ERROR_TRANSIENT:
|
||||
r.logger.Println("Aurora replied with Transient error code, retrying")
|
||||
continue
|
||||
// Check Response Code from thrift and make a decision to continue retrying or not.
|
||||
switch responseCode := resp.GetResponseCode(); responseCode {
|
||||
|
||||
// Failure scenarios, these indicate a bad payload or a bad config. Stop retrying.
|
||||
case aurora.ResponseCode_INVALID_REQUEST,
|
||||
aurora.ResponseCode_ERROR,
|
||||
aurora.ResponseCode_AUTH_FAILED,
|
||||
aurora.ResponseCode_JOB_UPDATING_ERROR:
|
||||
r.logger.Printf("Terminal Response Code %v from Aurora, won't retry\n", resp.GetResponseCode().String())
|
||||
return resp, errors.New(response.CombineMessage(resp))
|
||||
// If the thrift call succeeded, stop retrying
|
||||
case aurora.ResponseCode_OK:
|
||||
return resp, nil
|
||||
|
||||
// The only case that should fall down to here is a WARNING response code.
|
||||
// It is currently not used as a response in the scheduler so it is unknown how to handle it.
|
||||
default:
|
||||
r.logger.debugPrintf("unhandled response code %v received from Aurora\n", responseCode)
|
||||
return nil, errors.Errorf("unhandled response code from Aurora %v", responseCode.String())
|
||||
}
|
||||
// If the response code is transient, continue retrying
|
||||
case aurora.ResponseCode_ERROR_TRANSIENT:
|
||||
r.logger.Println("Aurora replied with Transient error code, retrying")
|
||||
continue
|
||||
|
||||
// Failure scenarios, these indicate a bad payload or a bad config. Stop retrying.
|
||||
case aurora.ResponseCode_INVALID_REQUEST,
|
||||
aurora.ResponseCode_ERROR,
|
||||
aurora.ResponseCode_AUTH_FAILED,
|
||||
aurora.ResponseCode_JOB_UPDATING_ERROR:
|
||||
r.logger.Printf("Terminal Response Code %v from Aurora, won't retry\n", resp.GetResponseCode().String())
|
||||
return resp, errors.New(response.CombineMessage(resp))
|
||||
|
||||
// The only case that should fall down to here is a WARNING response code.
|
||||
// It is currently not used as a response in the scheduler so it is unknown how to handle it.
|
||||
default:
|
||||
r.logger.debugPrintf("unhandled response code %v received from Aurora\n", responseCode)
|
||||
return nil, errors.Errorf("unhandled response code from Aurora %v", responseCode.String())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
r.logger.debugPrintf("it took %v retries to complete this operation\n", curStep)
|
||||
|
||||
if curStep > 1 {
|
||||
r.config.logger.Printf("retried this thrift call %d time(s)", curStep)
|
||||
r.config.logger.Printf("this thrift call was retried %d time(s)", curStep)
|
||||
}
|
||||
|
||||
// Provide more information to the user wherever possible.
|
||||
|
|
@ -253,3 +265,30 @@ func (r *realisClient) thriftCallWithRetries(
|
|||
|
||||
return nil, newRetryError(errors.New("ran out of retries"), curStep)
|
||||
}
|
||||
|
||||
// isConnectionError processes the error received by the client.
|
||||
// The return values indicate weather this was determined to be a temporary error
|
||||
// and weather it was determined to be a timeout error
|
||||
func isConnectionError(err error) (bool, bool) {
|
||||
|
||||
// Determine if error is a temporary URL error by going up the stack
|
||||
transportException, ok := err.(thrift.TTransportException)
|
||||
if !ok {
|
||||
return false, false
|
||||
}
|
||||
|
||||
urlError, ok := transportException.Err().(*url.Error)
|
||||
if !ok {
|
||||
return false, false
|
||||
}
|
||||
|
||||
// EOF error occurs when the server closes the read buffer of the client. This is common
|
||||
// when the server is overloaded and we consider it temporary.
|
||||
// All other which are not temporary as per the member function Temporary(),
|
||||
// are considered not temporary (permanent).
|
||||
if urlError.Err != io.EOF && !urlError.Temporary() {
|
||||
return false, false
|
||||
}
|
||||
|
||||
return true, urlError.Timeout()
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue