V1 CreateService and StartJobUpdate Timeout signal and cleanup (#105)

* Bumped up version to 1.21.1

* Moving admin functions to a new file. They are still part of the same pointer receiver type.

* Removing dead code and fixing some comments to add space between backslash and comment.

* Adding set up and tear down to run tests script. It sets up a pod, runs all tests, and then tears down the pod.

* Added `--rm` to run tests Mac script.

* Removing cookie jar from transport layer as it's not needed.

* Changing all error messages to start with a lower case letter. Changing some messages around to be more descriptive.

* Adding an argument to allow the retry mechanism to stop if a timeout has been encountered. This is useful for mutating API calls. Only StartUpdate and CreateService have enabled by default stop at timeout.

* Added 2 tests for when a call goes through despite the client timing out. One is with a good payload, one is with a bad payload.

* Updating changelog with information about the error type returned.

* Adding test for duplicate metadata.

* Refactored JobUpdateStatus monitor to use a new monitor called JobUpdateQuery. Update monitor will now still continue if it does not find an update to monitor. Furthermore, it has been optimized to reduce returning payloads from the scheduler as much as possible. This is through using the GetJobUpdateSummaries API instead of JobUpdateDetails and by including a the statuses we're searching for as part of the query.


* Added documentation as to how to handle a timeout on an API request.

* Optimized GetInstancesIds to create a copy of the JobKey being passed down in order to avoid unexpected behavior. Instead of setting every variable name separately, now a JobKey array is being created.

realis_e2e_test.go

@@ -35,11 +35,13 @@ var r realis.Realis
 var monitor *realis.Monitor
 var thermosPayload []byte
 
+const auroraURL = "http://192.168.33.7:8081"
+
 func TestMain(m *testing.M) {
 	var err error
 
 	// New configuration to connect to docker container
-	r, err = realis.NewRealisClient(realis.SchedulerUrl("http://192.168.33.7:8081"),
+	r, err = realis.NewRealisClient(realis.SchedulerUrl(auroraURL),
 		realis.BasicAuth("aurora", "secret"),
 		realis.TimeoutMS(20000))
 
@@ -93,7 +95,7 @@ func TestNonExistentEndpoint(t *testing.T) {
 }
 
 func TestThriftBinary(t *testing.T) {
-	r, err := realis.NewRealisClient(realis.SchedulerUrl("http://192.168.33.7:8081"),
+	r, err := realis.NewRealisClient(realis.SchedulerUrl(auroraURL),
 		realis.BasicAuth("aurora", "secret"),
 		realis.TimeoutMS(20000),
 		realis.ThriftBinary())
@@ -115,7 +117,7 @@ func TestThriftBinary(t *testing.T) {
 }
 
 func TestThriftJSON(t *testing.T) {
-	r, err := realis.NewRealisClient(realis.SchedulerUrl("http://192.168.33.7:8081"),
+	r, err := realis.NewRealisClient(realis.SchedulerUrl(auroraURL),
 		realis.BasicAuth("aurora", "secret"),
 		realis.TimeoutMS(20000),
 		realis.ThriftJSON())
@@ -137,7 +139,7 @@ func TestThriftJSON(t *testing.T) {
 }
 
 func TestNoopLogger(t *testing.T) {
-	r, err := realis.NewRealisClient(realis.SchedulerUrl("http://192.168.33.7:8081"),
+	r, err := realis.NewRealisClient(realis.SchedulerUrl(auroraURL),
 		realis.BasicAuth("aurora", "secret"),
 		realis.SetLogger(realis.NoopLogger{}))
 
@@ -161,6 +163,8 @@ func TestLeaderFromZK(t *testing.T) {
 	url, err := realis.LeaderFromZK(*cluster)
 
 	assert.NoError(t, err)
+
+	// Address stored inside of ZK might be different than the one we connect to in our tests.
 	assert.Equal(t, "http://192.168.33.7:8081", url)
 }
 
@@ -271,6 +275,25 @@ func TestRealisClient_CreateJob_Thermos(t *testing.T) {
 		assert.True(t, success)
 		assert.NoError(t, err)
 	})
+
+	t.Run("Duplicate_Metadata", func(t *testing.T) {
+		job.Name("thermos_duplicate_metadata").
+			AddLabel("hostname", "cookie").
+			AddLabel("hostname", "candy").
+			AddLabel("hostname", "popcorn").
+			AddLabel("hostname", "chips").
+			AddLabel("chips", "chips")
+
+		_, err := r.CreateJob(job)
+		assert.NoError(t, err)
+
+		success, err := monitor.Instances(job.JobKey(), 2, 1, 50)
+		assert.True(t, success)
+		assert.NoError(t, err)
+
+		_, err = r.KillJob(job.JobKey())
+		assert.NoError(t, err)
+	})
 }
 
 // Test configuring an executor that doesn't exist for CreateJob API
@@ -461,6 +484,64 @@ func TestRealisClient_CreateService(t *testing.T) {
 	// Kill task test task after confirming it came up fine
 	_, err = r.KillJob(job.JobKey())
 	assert.NoError(t, err)
+
+	success, err := monitor.Instances(job.JobKey(), 0, 1, 50)
+	assert.True(t, success)
+
+	// Create a client which will timeout and close the connection before receiving an answer
+	timeoutClient, err := realis.NewRealisClient(realis.SchedulerUrl(auroraURL),
+		realis.BasicAuth("aurora", "secret"),
+		realis.TimeoutMS(10))
+	assert.NoError(t, err)
+	defer timeoutClient.Close()
+
+	// Test case where http connection timeouts out.
+	t.Run("TimeoutError", func(t *testing.T) {
+		job.Name("createService_timeout")
+
+		// Make sure a timedout error was returned
+		_, _, err = timeoutClient.CreateService(job, settings)
+		assert.Error(t, err)
+		assert.True(t, realis.IsTimeout(err))
+
+		updateReceivedQuery := aurora.JobUpdateQuery{
+			Role:           &job.JobKey().Role,
+			JobKey:         job.JobKey(),
+			UpdateStatuses: aurora.ACTIVE_JOB_UPDATE_STATES,
+			Limit:          1}
+
+		updateSummaries, err := monitor.JobUpdateQuery(updateReceivedQuery, time.Second*1, time.Second*50)
+		assert.NoError(t, err)
+
+		assert.Len(t, updateSummaries, 1)
+
+		_, err = r.AbortJobUpdate(*updateSummaries[0].Key, "Cleaning up")
+		_, err = r.KillJob(job.JobKey())
+		assert.NoError(t, err)
+
+	})
+
+	// Test case where http connection timeouts out.
+	t.Run("TimeoutError_BadPayload", func(t *testing.T) {
+		// Illegal payload
+		job.InstanceCount(-1)
+		job.Name("createService_timeout_bad_payload")
+
+		// Make sure a timedout error was returned
+		_, _, err = timeoutClient.CreateService(job, settings)
+		assert.Error(t, err)
+		assert.True(t, realis.IsTimeout(err))
+
+		summary, err := r.GetJobUpdateSummaries(
+			&aurora.JobUpdateQuery{
+				Role:           &job.JobKey().Role,
+				JobKey:         job.JobKey(),
+				UpdateStatuses: aurora.ACTIVE_JOB_UPDATE_STATES})
+		assert.NoError(t, err)
+
+		// Payload should have been rejected, no update should exist
+		assert.Len(t, summary.GetResult_().GetGetJobUpdateSummariesResult_().GetUpdateSummaries(), 0)
+	})
 }
 
 // Test configuring an executor that doesn't exist for CreateJob API