From 45a44168309db67898ce2fb240e8012e97bedcaa Mon Sep 17 00:00:00 2001 From: Renan DelValle Date: Mon, 3 Dec 2018 16:09:46 -0800 Subject: [PATCH 1/7] Adding .gitattributes to ignore generated files. --- .gitattributes | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..c2f77d7 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +gen-go/ linguist-generated=true +vendor/ linguist-generated=true +Gopkg.lock linguist-generated=true From b1001580806cdbb83b6b32d9a2875201eb40ae05 Mon Sep 17 00:00:00 2001 From: Renan DelValle Date: Wed, 19 Dec 2018 15:30:22 -0800 Subject: [PATCH 2/7] Updating Travis CI config file to include running CI on master-v2.0 branch --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index f9daf34..7b9b7b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,11 @@ sudo: required language: go +branches: + only: + - master + - master-v2.0 + go: - "1.10.x" From 9a835631b2cf0169c58c674fe0776946e3a00e2b Mon Sep 17 00:00:00 2001 From: Renan DelValle Date: Wed, 19 Dec 2018 15:33:35 -0800 Subject: [PATCH 3/7] Running goimports on all repository to conform to newest goimports. --- clusters_test.go | 2 +- examples/client.go | 2 +- examples/jsonClient.go | 2 +- realis_e2e_test.go | 2 +- zk_test.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/clusters_test.go b/clusters_test.go index 4d34038..0663cdc 100644 --- a/clusters_test.go +++ b/clusters_test.go @@ -18,7 +18,7 @@ import ( "fmt" "testing" - "github.com/paypal/gorealis" + realis "github.com/paypal/gorealis" "github.com/stretchr/testify/assert" ) diff --git a/examples/client.go b/examples/client.go index 4d62630..dd5ce2a 100644 --- a/examples/client.go +++ b/examples/client.go @@ -22,7 +22,7 @@ import ( "strings" "time" - "github.com/paypal/gorealis" + realis "github.com/paypal/gorealis" "github.com/paypal/gorealis/gen-go/apache/aurora" "github.com/paypal/gorealis/response" ) diff --git a/examples/jsonClient.go b/examples/jsonClient.go index 0989671..c138a63 100644 --- a/examples/jsonClient.go +++ b/examples/jsonClient.go @@ -23,7 +23,7 @@ import ( "os" "time" - "github.com/paypal/gorealis" + realis "github.com/paypal/gorealis" "github.com/paypal/gorealis/gen-go/apache/aurora" "github.com/pkg/errors" ) diff --git a/realis_e2e_test.go b/realis_e2e_test.go index 9108903..688a797 100644 --- a/realis_e2e_test.go +++ b/realis_e2e_test.go @@ -22,7 +22,7 @@ import ( "testing" "time" - "github.com/paypal/gorealis" + realis "github.com/paypal/gorealis" "github.com/paypal/gorealis/gen-go/apache/aurora" "github.com/paypal/gorealis/response" "github.com/pkg/errors" diff --git a/zk_test.go b/zk_test.go index 4f05b46..c23ff75 100644 --- a/zk_test.go +++ b/zk_test.go @@ -20,7 +20,7 @@ import ( "testing" "time" - "github.com/paypal/gorealis" + realis "github.com/paypal/gorealis" "github.com/stretchr/testify/assert" ) From 296af622d1217e28f9ee8573d540c2fc7f836627 Mon Sep 17 00:00:00 2001 From: Robert Allen Date: Thu, 20 Dec 2018 16:38:06 -0600 Subject: [PATCH 4/7] This adds the following function to the PartitionPolicy configuration to the Job interface (#91) * Adding Partition Policy API --- job.go | 8 ++++++++ realis_e2e_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/job.go b/job.go index 0ff2aac..eb7eaa0 100644 --- a/job.go +++ b/job.go @@ -55,6 +55,7 @@ type Job interface { GetInstanceCount() int32 MaxFailure(maxFail int32) Job Container(container Container) Job + PartitionPolicy(policy *aurora.PartitionPolicy) Job } // Structure to collect all information pertaining to an Aurora job. @@ -315,3 +316,10 @@ func (j *AuroraJob) Container(container Container) Job { return j } + +// Set a partition policy for the job configuration to implement. +func (j *AuroraJob) PartitionPolicy(policy *aurora.PartitionPolicy) Job { + j.jobConfig.TaskConfig.PartitionPolicy = policy + + return j +} diff --git a/realis_e2e_test.go b/realis_e2e_test.go index 688a797..7fb3504 100644 --- a/realis_e2e_test.go +++ b/realis_e2e_test.go @@ -692,3 +692,41 @@ func TestRealisClient_ForceExplicitTaskReconciliation(t *testing.T) { err = r.ForceExplicitTaskReconciliation(&batchSize) assert.NoError(t, err) } + +func TestRealisClient_PartitionPolicy(t *testing.T) { + + role := "vagrant" + var partitionDelay int64 = 30 + job := realis.NewJob(). + Environment("prod"). + Role(role). + Name("create_thermos_job_partition_policy_test"). + ExecutorName(aurora.AURORA_EXECUTOR_NAME). + ExecutorData(string(thermosPayload)). + CPU(.5). + RAM(64). + Disk(100). + IsService(true). + InstanceCount(2). + PartitionPolicy(&aurora.PartitionPolicy{Reschedule: true, DelaySecs: &partitionDelay}) + + settings := realis.NewUpdateSettings() + settings.UpdateGroupSize = 2 + resp, result, err := r.CreateService(job, settings) + assert.NoError(t, err) + + assert.Equal(t, aurora.ResponseCode_OK, resp.ResponseCode) + + var ok bool + var mErr error + + if ok, mErr = monitor.JobUpdate(*result.GetKey(), 5, 180); !ok || mErr != nil { + // Update may already be in a terminal state so don't check for error + _, err := r.AbortJobUpdate(*result.GetKey(), "Monitor timed out.") + + _, err = r.KillJob(job.JobKey()) + + assert.NoError(t, err) + } + +} From 2f7015571cec92d09d9a55f6ac87a9c293c8807d Mon Sep 17 00:00:00 2001 From: Renan DelValle Date: Tue, 8 Jan 2019 15:11:52 -0800 Subject: [PATCH 5/7] Adding support for setting GPU as a resource. (#93) * Adding support for setting GPU as a resource. * Refactoring pulse update test. --- job.go | 40 +++++++++++++-------- realis_e2e_test.go | 88 +++++++++++++++++++++++++--------------------- updatejob.go | 17 +++++---- 3 files changed, 83 insertions(+), 62 deletions(-) diff --git a/job.go b/job.go index eb7eaa0..86849a2 100644 --- a/job.go +++ b/job.go @@ -58,10 +58,19 @@ type Job interface { PartitionPolicy(policy *aurora.PartitionPolicy) Job } +type ResourceType int + +const ( + CPU ResourceType = iota + RAM + DISK + GPU +) + // Structure to collect all information pertaining to an Aurora job. type AuroraJob struct { jobConfig *aurora.JobConfiguration - resources map[string]*aurora.Resource + resources map[ResourceType]*aurora.Resource portCount int } @@ -88,15 +97,8 @@ func NewJob() Job { ramMb := aurora.NewResource() diskMb := aurora.NewResource() - resources := make(map[string]*aurora.Resource) - resources["cpu"] = numCpus - resources["ram"] = ramMb - resources["disk"] = diskMb - - taskConfig.Resources = make(map[*aurora.Resource]bool) - taskConfig.Resources[numCpus] = true - taskConfig.Resources[ramMb] = true - taskConfig.Resources[diskMb] = true + resources := map[ResourceType]*aurora.Resource{CPU: numCpus, RAM: ramMb, DISK: diskMb} + taskConfig.Resources = map[*aurora.Resource]bool{numCpus: true, ramMb: true, diskMb: true} numCpus.NumCpus = new(float64) ramMb.RamMb = new(int64) @@ -155,20 +157,28 @@ func (j *AuroraJob) ExecutorData(data string) Job { } func (j *AuroraJob) CPU(cpus float64) Job { - *j.resources["cpu"].NumCpus = cpus - + *j.resources[CPU].NumCpus = cpus return j } func (j *AuroraJob) RAM(ram int64) Job { - *j.resources["ram"].RamMb = ram - + *j.resources[RAM].RamMb = ram return j } func (j *AuroraJob) Disk(disk int64) Job { - *j.resources["disk"].DiskMb = disk + *j.resources[DISK].DiskMb = disk + return j +} +func (j *AuroraJob) GPU(gpus int64) Job { + if _, ok := j.resources[GPU]; !ok { + numGPUs := &aurora.Resource{NumGpus: new(int64)} + j.resources[GPU] = numGPUs + j.TaskConfig().Resources[numGPUs] = true + } + + *j.resources[GPU].NumGpus = gpus return j } diff --git a/realis_e2e_test.go b/realis_e2e_test.go index 7fb3504..163e67c 100644 --- a/realis_e2e_test.go +++ b/realis_e2e_test.go @@ -22,11 +22,13 @@ import ( "testing" "time" + "git.apache.org/thrift.git/lib/go/thrift" realis "github.com/paypal/gorealis" "github.com/paypal/gorealis/gen-go/apache/aurora" "github.com/paypal/gorealis/response" "github.com/pkg/errors" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var r realis.Realis @@ -309,10 +311,8 @@ func TestRealisClient_CreateService_WithPulse_Thermos(t *testing.T) { AddPorts(1). AddLabel("currentTime", time.Now().String()) - pulse := int32(30) - timeout := 300 settings := realis.NewUpdateSettings() - settings.BlockIfNoPulsesAfterMs = &pulse + settings.BlockIfNoPulsesAfterMs = thrift.Int32Ptr(30) settings.UpdateGroupSize = 1 settings.WaitForBatchCompletion = true job.InstanceCount(2) @@ -327,52 +327,58 @@ func TestRealisClient_CreateService_WithPulse_Thermos(t *testing.T) { Limit: 1, } - start := time.Now() - for i := 0; i*int(pulse) <= timeout; i++ { + var updateDetails []*aurora.JobUpdateDetails - fmt.Println("sending PulseJobUpdate....") - resp, err = r.PulseJobUpdate(result.GetKey()) - assert.NotNil(t, resp) - assert.Nil(t, err) + ticker := time.NewTicker(time.Second * 3) + timer := time.NewTimer(time.Minute * 6) + defer ticker.Stop() + defer timer.Stop() - respDetail, err := r.JobUpdateDetails(updateQ) - assert.Nil(t, err) +pulseLoop: + for { + select { + case <-ticker.C: - updateDetail := response.JobUpdateDetails(respDetail) - if len(updateDetail) == 0 { - fmt.Println("No update found") - assert.NotEqual(t, len(updateDetail), 0) - } - status := updateDetail[0].Update.Summary.State.Status + fmt.Println("sending PulseJobUpdate....") + resp, err = r.PulseJobUpdate(result.GetKey()) + require.NotNil(t, resp, "received a nil response from Aurora") + assert.Nil(t, err) - if _, ok := aurora.ACTIVE_JOB_UPDATE_STATES[status]; !ok { + respDetail, err := r.JobUpdateDetails(updateQ) + assert.Nil(t, err) - // Rolled forward is the only state in which an update has been successfully updated - // if we encounter an inactive state and it is not at rolled forward, update failed - if status == aurora.JobUpdateStatus_ROLLED_FORWARD { - fmt.Println("Update succeded") - break - } else { - fmt.Println("Update failed") - break + updateDetails = response.JobUpdateDetails(respDetail) + if len(updateDetails) == 0 { + fmt.Println("No update found") + assert.NotEqual(t, len(updateDetails), 0) } + status := updateDetails[0].Update.Summary.State.Status + + if _, ok := aurora.ACTIVE_JOB_UPDATE_STATES[status]; !ok { + + // Rolled forward is the only state in which an update has been successfully updated + // if we encounter an inactive state and it is not at rolled forward, update failed + if status == aurora.JobUpdateStatus_ROLLED_FORWARD { + fmt.Println("Update succeded") + break pulseLoop + } else { + fmt.Println("Update failed") + break pulseLoop + } + } + + fmt.Println("Polling, update still active...") + case <-timer.C: + _, err := r.AbortJobUpdate(*updateDetails[0].GetUpdate().GetSummary().GetKey(), "") + assert.NoError(t, err) + _, err = r.KillJob(job.JobKey()) + require.NoError(t, err, "timed out during pulse update test") } - - fmt.Println("Polling, update still active...") - time.Sleep(time.Duration(pulse) * time.Second) } - end := time.Now() - fmt.Printf("Update call took %d ns\n", (end.UnixNano() - start.UnixNano())) - - t.Run("TestRealisClient_KillJob_Thermos", func(t *testing.T) { - start := time.Now() - resp, err := r.KillJob(job.JobKey()) - end := time.Now() - assert.NoError(t, err) - assert.Equal(t, aurora.ResponseCode_OK, resp.ResponseCode) - fmt.Printf("Kill call took %d ns\n", (end.UnixNano() - start.UnixNano())) - }) + resp, err = r.KillJob(job.JobKey()) + assert.NoError(t, err) + assert.Equal(t, aurora.ResponseCode_OK, resp.ResponseCode) } // Test configuring an executor that doesn't exist for CreateJob API @@ -403,7 +409,7 @@ func TestRealisClient_CreateService(t *testing.T) { var ok bool var mErr error - if ok, mErr = monitor.JobUpdate(*result.GetKey(), 5, 180); !ok || mErr != nil { + if ok, mErr = monitor.JobUpdate(*result.GetKey(), 5, 240); !ok || mErr != nil { // Update may already be in a terminal state so don't check for error _, err := r.AbortJobUpdate(*result.GetKey(), "Monitor timed out.") diff --git a/updatejob.go b/updatejob.go index 4ee4f14..f7ef2fb 100644 --- a/updatejob.go +++ b/updatejob.go @@ -37,17 +37,17 @@ func NewDefaultUpdateJob(config *aurora.TaskConfig) *UpdateJob { // Rebuild resource map from TaskConfig for ptr := range config.Resources { if ptr.NumCpus != nil { - job.resources["cpu"].NumCpus = ptr.NumCpus + job.resources[CPU].NumCpus = ptr.NumCpus continue // Guard against Union violations that Go won't enforce } if ptr.RamMb != nil { - job.resources["ram"].RamMb = ptr.RamMb + job.resources[RAM].RamMb = ptr.RamMb continue } if ptr.DiskMb != nil { - job.resources["disk"].DiskMb = ptr.DiskMb + job.resources[DISK].DiskMb = ptr.DiskMb continue } } @@ -77,19 +77,24 @@ func NewUpdateJob(config *aurora.TaskConfig, settings *aurora.JobUpdateSettings) // Rebuild resource map from TaskConfig for ptr := range config.Resources { if ptr.NumCpus != nil { - job.resources["cpu"].NumCpus = ptr.NumCpus + job.resources[CPU].NumCpus = ptr.NumCpus continue // Guard against Union violations that Go won't enforce } if ptr.RamMb != nil { - job.resources["ram"].RamMb = ptr.RamMb + job.resources[RAM].RamMb = ptr.RamMb continue } if ptr.DiskMb != nil { - job.resources["disk"].DiskMb = ptr.DiskMb + job.resources[DISK].DiskMb = ptr.DiskMb continue } + + if ptr.NumGpus != nil { + job.resources[GPU].NumGpus = ptr.NumGpus + continue // Guard against Union violations that Go won't enforce + } } //TODO(rdelvalle): Deep copy job struct to avoid unexpected behavior From 22b1d82d88fde5ea9e78efcaabfb1886be821cda Mon Sep 17 00:00:00 2001 From: Renan DelValle Date: Tue, 8 Jan 2019 15:37:25 -0800 Subject: [PATCH 6/7] Bug fix for logger interface. Varidic arguments need to be unrolled when passed to print functions. --- logger.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/logger.go b/logger.go index 05a3e85..54a901f 100644 --- a/logger.go +++ b/logger.go @@ -40,20 +40,20 @@ func (l *LevelLogger) EnableDebug(enable bool) { func (l LevelLogger) DebugPrintf(format string, a ...interface{}) { if l.debug { l.Print("[DEBUG] ") - l.Printf(format, a) + l.Printf(format, a...) } } func (l LevelLogger) DebugPrint(a ...interface{}) { if l.debug { l.Print("[DEBUG] ") - l.Print(a) + l.Print(a...) } } func (l LevelLogger) DebugPrintln(a ...interface{}) { if l.debug { l.Print("[DEBUG] ") - l.Println(a) + l.Println(a...) } } From 73e7ab267129064f0f0c4685de20342db5c44b30 Mon Sep 17 00:00:00 2001 From: Renan DelValle Date: Tue, 8 Jan 2019 15:57:19 -0800 Subject: [PATCH 7/7] Releasing version 1.3.1 --- realis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/realis.go b/realis.go index 8878b45..2597738 100644 --- a/realis.go +++ b/realis.go @@ -36,7 +36,7 @@ import ( "github.com/pkg/errors" ) -const VERSION = "1.3.0" +const VERSION = "1.3.1" // TODO(rdelvalle): Move documentation to interface in order to make godoc look better/more accessible type Realis interface {