Thread safety, misc fixes, and refactoring (#51)

* Changing incorrect license in some source files.

* Changing CreateService to mimic CreateJob by setting the batch size to the instance count.

* Changing Getcerts to GetCerts to match the style of the rest of the codebase.

* Overhauled error handling. Backoff now recognizes temporary errors and continues to retry if it finds one.

* Changed thrift function call wrapper to be more explicitly named and to perform more safety checks.

* Moved Jitter function from realis to retry.

* API code is now more uniform and follows a certain template.

* Lock added whenever a thrift call is made or when a modification is done to the connection. Note that calling ReestablishConn externally may result in some race conditions. We will move to make this function private in the near future.

* Added test for Realis session thread safety. Tested ScheduleStatus monitor. Tested monitor timing out.

* Returning nil whenever there is an error return so that there are no ambiguities.

* Using defer with unlock so that the lock is still released if a panic is invoked.

realis_e2e_test.go

@@ -21,6 +21,8 @@ import (
 	"testing"
 	"time"
 
+	"sync"
+
 	"github.com/paypal/gorealis"
 	"github.com/paypal/gorealis/gen-go/apache/aurora"
 	"github.com/stretchr/testify/assert"
@@ -72,8 +74,8 @@ func TestRealisClient_ReestablishConn(t *testing.T) {
 	assert.NoError(t, err)
 }
 
-func TestGetCacerts(t *testing.T) {
-	certs, err := realis.Getcerts("./examples/certs")
+func TestGetCACerts(t *testing.T) {
+	certs, err := realis.GetCerts("./examples/certs")
 	assert.NoError(t, err)
 	assert.Equal(t, len(certs.Subjects()), 2)
 
@@ -167,6 +169,7 @@ func TestRealisClient_ScheduleCronJob_Thermos(t *testing.T) {
 		fmt.Printf("Deschedule cron call took %d ns\n", (end.UnixNano() - start.UnixNano()))
 	})
 }
+
 func TestRealisClient_DrainHosts(t *testing.T) {
 	hosts := []string{"192.168.33.7"}
 	_, _, err := r.DrainHosts(hosts...)
@@ -214,3 +217,48 @@ func TestRealisClient_DrainHosts(t *testing.T) {
 	})
 
 }
+
+// Test multiple go routines using a single connection
+func TestRealisClient_SessionThreadSafety(t *testing.T) {
+
+	// Create a single job
+	job := realis.NewJob().
+		Environment("prod").
+		Role("vagrant").
+		Name("create_thermos_job_test_multi").
+		ExecutorName(aurora.AURORA_EXECUTOR_NAME).
+		ExecutorData(string(thermosPayload)).
+		CPU(.25).
+		RAM(4).
+		Disk(10).
+		InstanceCount(100) // Impossible amount to go live in the current vagrant default settings
+
+	resp, err := r.CreateJob(job)
+	assert.NoError(t, err)
+
+	assert.Equal(t, aurora.ResponseCode_OK, resp.ResponseCode)
+
+	wg := sync.WaitGroup{}
+	for i := 0; i < 20; i++ {
+
+		wg.Add(1)
+
+		// Launch multiple monitors that will poll every second
+		go func() {
+			defer wg.Done()
+
+			// Test Schedule status monitor for terminal state and timing out after 30 seconds
+			success, err := monitor.ScheduleStatus(job.JobKey(), job.GetInstanceCount(), aurora.LIVE_STATES, 1, 30)
+			assert.False(t, success)
+			assert.Error(t, err)
+
+			resp, err := r.KillJob(job.JobKey())
+			assert.NoError(t, err)
+
+			assert.Equal(t, aurora.ResponseCode_OK, resp.ResponseCode)
+
+		}()
+	}
+
+	wg.Wait()
+}