From 7286d2fc9f9b63ec41feadd367157d816106951f Mon Sep 17 00:00:00 2001 From: Renan DelValle Date: Thu, 21 Mar 2019 12:26:14 -0700 Subject: [PATCH] Adding rudimentary support for creating jobs on Aurora. --- cmd/create.go | 151 ++++++++++++++++++++++++++++++++++++++++++++--- go.mod | 2 +- go.sum | 10 ---- hello_world.yaml | 13 ++++ 4 files changed, 158 insertions(+), 18 deletions(-) create mode 100644 hello_world.yaml diff --git a/cmd/create.go b/cmd/create.go index 9cf3d9f..ca763f4 100644 --- a/cmd/create.go +++ b/cmd/create.go @@ -1,25 +1,162 @@ package cmd import ( + "os" + + yaml "gopkg.in/yaml.v2" + + realis "github.com/paypal/gorealis/v2" + "github.com/pkg/errors" "github.com/spf13/cobra" ) func init() { rootCmd.AddCommand(createCmd) - createCmd.Flags().StringVarP(env, "environment", "e", "", "Aurora Environment") - createCmd.Flags().StringVarP(role, "role", "r", "", "Aurora Role") - createCmd.Flags().StringVarP(name, "name", "n", "", "Aurora Name") - createCmd.MarkFlagRequired("environment") - createCmd.MarkFlagRequired("role") - createCmd.MarkFlagRequired("name") } var createCmd = &cobra.Command{ Use: "create", Short: "Create an Aurora Job", Run: createJob, + Args: cobra.ExactArgs(1), +} + +type URI struct { + URI string `yaml:"uri"` + Extract bool `yaml:"extract"` + Cache bool `yaml:"cache"` +} + +type Executor struct { + Name string `yaml:"name"` + Data string `yaml:"data"` +} + +type ThermosProcess struct { + Name string `yaml:"name"` + Cmd string `yaml:"cmd"` +} + +type Job struct { + Environment string `yaml:"environment"` + Role string `yaml:"role"` + Name string `yaml:"name"` + CPU float64 `yaml:"cpu"` + RAM int64 `yaml:"ram"` + Disk int64 `yaml:"disk"` + Executor Executor `yaml:"executor"` + Instances int32 `yaml:"instances"` + URIs []URI `yaml:"uris"` + Metadata map[string]string `yaml:"labels"` + Service bool `yaml:"service"` + Thermos []ThermosProcess `yaml:",flow"` +} + +func (j *Job) Validate() bool { + if j.Name == "" { + return false + } + + if j.Role == "" { + return false + } + + if j.Environment == "" { + return false + } + + if j.Instances <= 0 { + return false + } + + if j.CPU <= 0.0 { + return false + } + + if j.RAM <= 0 { + return false + } + + if j.Disk <= 0 { + return false + } + + return true +} + +func unmarshalJob(filename string) (Job, error) { + + job := Job{} + + if jobsFile, err := os.Open(filename); err != nil { + return job, errors.Wrap(err, "unable to read the job config file") + } else { + if err := yaml.NewDecoder(jobsFile).Decode(&job); err != nil { + return job, errors.Wrap(err, "unable to parse job config file") + } + + if !job.Validate() { + return job, errors.New("invalid job config") + } + } + + return job, nil } func createJob(cmd *cobra.Command, args []string) { - log.Println("Not implemented yet.") + + job, err := unmarshalJob(args[0]) + + if err != nil { + log.Fatalln(err) + } + + auroraJob := realis.NewJob(). + Environment(job.Environment). + Role(job.Role). + Name(job.Name). + CPU(job.CPU). + RAM(job.RAM). + Disk(job.Disk). + IsService(job.Service). + InstanceCount(job.Instances) + + // Adding URIs. + for _, uri := range job.URIs { + auroraJob.AddURIs(uri.Extract, uri.Cache, uri.URI) + } + + // Adding Metadata. + for key, value := range job.Metadata { + auroraJob.AddLabel(key, value) + } + + // If thermos jobs processes are provided, use them + if len(job.Thermos) > 0 { + thermosExec := realis.ThermosExecutor{} + for _, process := range job.Thermos { + thermosExec.AddProcess(realis.NewThermosProcess(process.Name, process.Cmd)) + } + auroraJob.ThermosExecutor(thermosExec) + } else { + // Non-Thermos executor + if job.Executor.Name == "" { + log.Fatal("no executor provided") + } + + auroraJob.ExecutorName(job.Executor.Name) + auroraJob.ExecutorData(job.Executor.Data) + } + + if err := client.CreateJob(auroraJob); err != nil { + log.Fatal("unable to create Aurora job: ", err) + } else { + if ok, monitorErr := client.InstancesMonitor(auroraJob.JobKey(), auroraJob.GetInstanceCount(), 5, 50); !ok || monitorErr != nil { + if err := client.KillJob(auroraJob.JobKey()); err != nil { + log.Fatal(monitorErr, err) + } + log.Fatal(monitorErr) + } + } + } diff --git a/go.mod b/go.mod index 65b580b..3faaea9 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,11 @@ module github.com/rdelval/australis require ( github.com/BurntSushi/toml v0.3.1 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/paypal/gorealis v0.0.0-20181219011448-e4e8a1c0b333 github.com/paypal/gorealis/v2 v2.0.1 github.com/pkg/errors v0.0.0-20171216070316-e881fd58d78e github.com/sirupsen/logrus v1.2.0 github.com/spf13/cobra v0.0.0-20180115160933-0c34d16c3123 github.com/spf13/pflag v1.0.3 github.com/spf13/viper v1.3.1 + gopkg.in/yaml.v2 v2.2.2 ) diff --git a/go.sum b/go.sum index d42b27d..2890159 100644 --- a/go.sum +++ b/go.sum @@ -1,36 +1,28 @@ -git.apache.org/thrift.git v0.0.0-20181016064013-5c1ecb67cde4/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= git.apache.org/thrift.git v0.12.0 h1:692K1/SsOcQvkvMRMdt60FCq2AvKpuQNM6sIeH3mN4s= git.apache.org/thrift.git v0.12.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/paypal/gorealis v0.0.0-20181219011448-e4e8a1c0b333 h1:eR+G8xTWhJ6k3bs+x0lrHmmp41CjNKRdGu7S6hYFs4k= -github.com/paypal/gorealis v0.0.0-20181219011448-e4e8a1c0b333/go.mod h1:trnpzEqR5XeSaZNKE+QPtJaoIncPmbhGvEqgAP34+DA= github.com/paypal/gorealis/v2 v2.0.1 h1:7V9jjoeRWeVWaTm9Fgc08gWq1/k/sebVDe2WeFWAVGA= github.com/paypal/gorealis/v2 v2.0.1/go.mod h1:mJz1e40v9vHyw8dDvzGvblHcCtFuJ+eo7yPjFYnowhE= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.0.0-20171216070316-e881fd58d78e h1:+RHxT/gm0O3UF7nLJbdNzAmULvCFt4XfXHWzh3XI/zs= github.com/pkg/errors v0.0.0-20171216070316-e881fd58d78e/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/samuel/go-zookeeper v0.0.0-20171117190445-471cd4e61d7a h1:EYL2xz/Zdo0hyqdZMXR4lmT2O11jDLTPCEqIe/FR6W4= github.com/samuel/go-zookeeper v0.0.0-20171117190445-471cd4e61d7a/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= @@ -50,7 +42,6 @@ github.com/spf13/viper v1.3.1 h1:5+8j8FTpnFV4nEImW/ofkzEt8VoOiLXxdYIDsB73T38= github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -62,7 +53,6 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/hello_world.yaml b/hello_world.yaml new file mode 100644 index 0000000..35a76d8 --- /dev/null +++ b/hello_world.yaml @@ -0,0 +1,13 @@ +--- +environment: "prod" +role: "vagrant" +name: "hello_world" +cpu: 0.09 +ram: 64 +disk: 128 +instances: 1 +thermos: + - name: "bootstrap" + cmd: "echo bootstrapping" + - name: "hello_gorealis" + cmd: "while true; do echo hello world from gorealis; sleep 10; done"