diff --git a/README.md b/README.md index 4a3456d..ff9f4a0 100644 --- a/README.md +++ b/README.md @@ -5,18 +5,18 @@ _Elektron_ is a [Mesos](mesos.apache.org) framework that behaves as a playground However, in addition to being a scheduler, Elektron also takes advantage of tools such as [Performance Co-Pilot](http://pcp.io/) and [RAPL](https://01.org/blogs/2014/running-average-power-limit--rapl) to help contain the power envelope within defined thresholds, reduce peak power consumption, and also reduce total energy consumption. Elektron is able to leverage the Mesos-provided resource abstraction to allow different algorithms to decide how to consume resource offers made by a Mesos Master. -#Architecture +# Architecture ![](docs/ElekArch.png) -#Usage +# Usage * [Pluggable Scheduling Policies](docs/SchedulingPolicies.md) * [Pluggable Power-Capping strategies](docs/PowerCappingStrategies.md) * [Scheduling Policy Switching](docs/SchedulingPolicySwitching.md) -#Logging +# Logging Please go through the [logging library doc](docs/Logging.md) to understand how the logging library has been setup. There are also instructions on how one can add additional loggers. -#Data +# Data * [Cluster Resource Consumption](docs/data/ClusterResourceConsumption.md) * [Schedule Trace](docs/data/ScheduledTrace.md) * [Degree of Collocation](docs/data/DegreeOfCollocation.md) @@ -25,7 +25,7 @@ Please go through the [logging library doc](docs/Logging.md) to understand how t - [Scheduling Policy Switch Trace](docs/data/withSpsEnabled/SchedulingPolicySwitchTrace.md) - [Scheduling Window](docs/data/withSpsEnabled/SchedulingWindow.md) -#Software Requirements +# Software Requirements **Requires [Performance Co-Pilot](http://pcp.io/) tool pmdumptext to be installed on the machine on which electron is launched for logging to work and PCP collector agents installed on the Mesos Agents** @@ -35,7 +35,7 @@ Compatible with the following versions: * Mesos 1.5.0 * Go 1.9.7 -#Build and Run +# Build and Run Compile the source code using the `go build` tool as shown below. ```commandline go build -o elektron diff --git a/def/task.go b/def/task.go index a99f74f..9ed74e8 100644 --- a/def/task.go +++ b/def/task.go @@ -1,12 +1,13 @@ package def import ( - "gitlab.com/spdf/elektron/constants" - "gitlab.com/spdf/elektron/utilities/offerUtils" "encoding/json" + "os" + mesos "github.com/mesos/mesos-go/api/v0/mesosproto" "github.com/pkg/errors" - "os" + "gitlab.com/spdf/elektron/constants" + "gitlab.com/spdf/elektron/utilities/offerUtils" ) type Task struct { diff --git a/def/taskUtils.go b/def/taskUtils.go index f753c26..171153f 100644 --- a/def/taskUtils.go +++ b/def/taskUtils.go @@ -3,10 +3,11 @@ package def import ( "errors" "fmt" - "github.com/mash/gokmeans" - "github.com/montanaflynn/stats" "log" "sort" + + "github.com/mash/gokmeans" + "github.com/montanaflynn/stats" ) // Information about a cluster of tasks. diff --git a/logging/def/loggerFactory.go b/logging/def/loggerFactory.go index 8592db2..321128e 100644 --- a/logging/def/loggerFactory.go +++ b/logging/def/loggerFactory.go @@ -1,9 +1,10 @@ package logging import ( - logUtils "gitlab.com/spdf/elektron/logging/utils" "strings" "time" + + logUtils "gitlab.com/spdf/elektron/logging/utils" ) // Names of different loggers diff --git a/pcp/pcp.go b/pcp/pcp.go index 94ecfef..e5c3126 100644 --- a/pcp/pcp.go +++ b/pcp/pcp.go @@ -1,16 +1,17 @@ package pcp import ( - elecLogDef "gitlab.com/spdf/elektron/logging/def" - "gitlab.com/spdf/elektron/schedulers" "bufio" "fmt" - "github.com/mesos/mesos-go/api/v0/scheduler" - "github.com/montanaflynn/stats" "log" "os/exec" "syscall" "time" + + "github.com/mesos/mesos-go/api/v0/scheduler" + "github.com/montanaflynn/stats" + elecLogDef "gitlab.com/spdf/elektron/logging/def" + "gitlab.com/spdf/elektron/schedulers" ) func Start(quit chan struct{}, logging *bool, logMType chan elecLogDef.LogMessageType, logMsg chan string, s scheduler.Scheduler) { diff --git a/power-capping/extrema.go b/power-capping/extrema.go index 82e849a..5c0bd45 100644 --- a/power-capping/extrema.go +++ b/power-capping/extrema.go @@ -1,9 +1,6 @@ package pcp import ( - elecLogDef "gitlab.com/spdf/elektron/logging/def" - "gitlab.com/spdf/elektron/pcp" - "gitlab.com/spdf/elektron/rapl" "bufio" "container/ring" "fmt" @@ -14,6 +11,10 @@ import ( "strings" "syscall" "time" + + elecLogDef "gitlab.com/spdf/elektron/logging/def" + "gitlab.com/spdf/elektron/pcp" + "gitlab.com/spdf/elektron/rapl" ) func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThreshold, loThreshold float64, diff --git a/power-capping/progressive-extrema.go b/power-capping/progressive-extrema.go index 7b9a188..b5f1f6b 100644 --- a/power-capping/progressive-extrema.go +++ b/power-capping/progressive-extrema.go @@ -1,11 +1,6 @@ package pcp import ( - "gitlab.com/spdf/elektron/constants" - elecLogDef "gitlab.com/spdf/elektron/logging/def" - "gitlab.com/spdf/elektron/pcp" - "gitlab.com/spdf/elektron/rapl" - "gitlab.com/spdf/elektron/utilities" "bufio" "container/ring" "fmt" @@ -17,6 +12,12 @@ import ( "strings" "syscall" "time" + + "gitlab.com/spdf/elektron/constants" + elecLogDef "gitlab.com/spdf/elektron/logging/def" + "gitlab.com/spdf/elektron/pcp" + "gitlab.com/spdf/elektron/rapl" + "gitlab.com/spdf/elektron/utilities" ) func round(num float64) int { diff --git a/rapl/cap.go b/rapl/cap.go index 29c6844..c630f75 100644 --- a/rapl/cap.go +++ b/rapl/cap.go @@ -1,11 +1,12 @@ package rapl import ( - elekEnv "gitlab.com/spdf/elektron/environment" - "github.com/pkg/errors" - "golang.org/x/crypto/ssh" "os" "strconv" + + "github.com/pkg/errors" + elekEnv "gitlab.com/spdf/elektron/environment" + "golang.org/x/crypto/ssh" ) func Cap(host, username string, percentage float64) error { diff --git a/scheduler.go b/scheduler.go index feec741..dd57c15 100644 --- a/scheduler.go +++ b/scheduler.go @@ -1,20 +1,21 @@ package main import ( - "gitlab.com/spdf/elektron/def" - elecLogDef "gitlab.com/spdf/elektron/logging/def" - "gitlab.com/spdf/elektron/pcp" - "gitlab.com/spdf/elektron/schedulers" "flag" "fmt" - "github.com/golang/protobuf/proto" - mesos "github.com/mesos/mesos-go/api/v0/mesosproto" - sched "github.com/mesos/mesos-go/api/v0/scheduler" "log" "os" "os/signal" "strings" "time" + + "github.com/golang/protobuf/proto" + mesos "github.com/mesos/mesos-go/api/v0/mesosproto" + sched "github.com/mesos/mesos-go/api/v0/scheduler" + "gitlab.com/spdf/elektron/def" + elecLogDef "gitlab.com/spdf/elektron/logging/def" + "gitlab.com/spdf/elektron/pcp" + "gitlab.com/spdf/elektron/schedulers" ) var master = flag.String("master", "", "Location of leading Mesos master -- :") diff --git a/schedulers/MaxGreedyMins.go b/schedulers/MaxGreedyMins.go index 057bc97..505cfb9 100644 --- a/schedulers/MaxGreedyMins.go +++ b/schedulers/MaxGreedyMins.go @@ -1,12 +1,13 @@ package schedulers import ( + "log" + + mesos "github.com/mesos/mesos-go/api/v0/mesosproto" + sched "github.com/mesos/mesos-go/api/v0/scheduler" "gitlab.com/spdf/elektron/def" "gitlab.com/spdf/elektron/utilities/mesosUtils" "gitlab.com/spdf/elektron/utilities/offerUtils" - mesos "github.com/mesos/mesos-go/api/v0/mesosproto" - sched "github.com/mesos/mesos-go/api/v0/scheduler" - "log" ) // Decides if to take an offer or not diff --git a/schedulers/MaxMin.go b/schedulers/MaxMin.go index 39a0a2a..867bdb5 100644 --- a/schedulers/MaxMin.go +++ b/schedulers/MaxMin.go @@ -1,12 +1,13 @@ package schedulers import ( + "log" + + mesos "github.com/mesos/mesos-go/api/v0/mesosproto" + sched "github.com/mesos/mesos-go/api/v0/scheduler" "gitlab.com/spdf/elektron/def" "gitlab.com/spdf/elektron/utilities/mesosUtils" "gitlab.com/spdf/elektron/utilities/offerUtils" - mesos "github.com/mesos/mesos-go/api/v0/mesosproto" - sched "github.com/mesos/mesos-go/api/v0/scheduler" - "log" ) // Decides if to take an offer or not diff --git a/schedulers/base.go b/schedulers/base.go index d646656..397d669 100644 --- a/schedulers/base.go +++ b/schedulers/base.go @@ -1,19 +1,20 @@ package schedulers import ( - "gitlab.com/spdf/elektron/def" - elecLogDef "gitlab.com/spdf/elektron/logging/def" - "gitlab.com/spdf/elektron/utilities" - "gitlab.com/spdf/elektron/utilities/schedUtils" "bytes" "fmt" + "log" + "sync" + "time" + "github.com/golang/protobuf/proto" mesos "github.com/mesos/mesos-go/api/v0/mesosproto" "github.com/mesos/mesos-go/api/v0/mesosutil" sched "github.com/mesos/mesos-go/api/v0/scheduler" - "log" - "sync" - "time" + "gitlab.com/spdf/elektron/def" + elecLogDef "gitlab.com/spdf/elektron/logging/def" + "gitlab.com/spdf/elektron/utilities" + "gitlab.com/spdf/elektron/utilities/schedUtils" ) type BaseScheduler struct { diff --git a/schedulers/bin-packing.go b/schedulers/bin-packing.go index da53642..4fb1850 100644 --- a/schedulers/bin-packing.go +++ b/schedulers/bin-packing.go @@ -1,12 +1,13 @@ package schedulers import ( + "log" + + mesos "github.com/mesos/mesos-go/api/v0/mesosproto" + sched "github.com/mesos/mesos-go/api/v0/scheduler" "gitlab.com/spdf/elektron/def" "gitlab.com/spdf/elektron/utilities/mesosUtils" "gitlab.com/spdf/elektron/utilities/offerUtils" - mesos "github.com/mesos/mesos-go/api/v0/mesosproto" - sched "github.com/mesos/mesos-go/api/v0/scheduler" - "log" ) // Decides if to take an offer or not diff --git a/schedulers/electronScheduler.go b/schedulers/electronScheduler.go index 85c3297..9fd295e 100644 --- a/schedulers/electronScheduler.go +++ b/schedulers/electronScheduler.go @@ -1,11 +1,12 @@ package schedulers import ( - "gitlab.com/spdf/elektron/def" - elecLogDef "gitlab.com/spdf/elektron/logging/def" + "time" + mesos "github.com/mesos/mesos-go/api/v0/mesosproto" sched "github.com/mesos/mesos-go/api/v0/scheduler" - "time" + "gitlab.com/spdf/elektron/def" + elecLogDef "gitlab.com/spdf/elektron/logging/def" ) // Implements mesos scheduler. diff --git a/schedulers/first-fit.go b/schedulers/first-fit.go index eb67566..bb7a3e1 100644 --- a/schedulers/first-fit.go +++ b/schedulers/first-fit.go @@ -1,11 +1,11 @@ package schedulers import ( + mesos "github.com/mesos/mesos-go/api/v0/mesosproto" + sched "github.com/mesos/mesos-go/api/v0/scheduler" "gitlab.com/spdf/elektron/def" "gitlab.com/spdf/elektron/utilities/mesosUtils" "gitlab.com/spdf/elektron/utilities/offerUtils" - mesos "github.com/mesos/mesos-go/api/v0/mesosproto" - sched "github.com/mesos/mesos-go/api/v0/scheduler" ) // Decides if to take an offer or not diff --git a/schedulers/helpers.go b/schedulers/helpers.go index 500a88c..c0fa22f 100644 --- a/schedulers/helpers.go +++ b/schedulers/helpers.go @@ -1,14 +1,14 @@ package schedulers import ( + mesos "github.com/mesos/mesos-go/api/v0/mesosproto" + sched "github.com/mesos/mesos-go/api/v0/scheduler" + "github.com/pkg/errors" "gitlab.com/spdf/elektron/constants" "gitlab.com/spdf/elektron/def" elecLogDef "gitlab.com/spdf/elektron/logging/def" "gitlab.com/spdf/elektron/utilities" "gitlab.com/spdf/elektron/utilities/mesosUtils" - mesos "github.com/mesos/mesos-go/api/v0/mesosproto" - sched "github.com/mesos/mesos-go/api/v0/scheduler" - "github.com/pkg/errors" ) func coLocated(tasks map[string]bool, s BaseScheduler) { diff --git a/schedulers/schedPolicy.go b/schedulers/schedPolicy.go index 31645bd..8fabd29 100644 --- a/schedulers/schedPolicy.go +++ b/schedulers/schedPolicy.go @@ -1,13 +1,14 @@ package schedulers import ( - "gitlab.com/spdf/elektron/def" - elecLogDef "gitlab.com/spdf/elektron/logging/def" "fmt" - mesos "github.com/mesos/mesos-go/api/v0/mesosproto" - sched "github.com/mesos/mesos-go/api/v0/scheduler" "log" "time" + + mesos "github.com/mesos/mesos-go/api/v0/mesosproto" + sched "github.com/mesos/mesos-go/api/v0/scheduler" + "gitlab.com/spdf/elektron/def" + elecLogDef "gitlab.com/spdf/elektron/logging/def" ) type SchedPolicyContext interface { diff --git a/schedulers/states.go b/schedulers/states.go index 6b5f4ed..a0a5d19 100644 --- a/schedulers/states.go +++ b/schedulers/states.go @@ -2,6 +2,7 @@ package schedulers import ( "fmt" + mesos "github.com/mesos/mesos-go/api/v0/mesosproto" ) diff --git a/schedulers/store.go b/schedulers/store.go index 574cb11..b7c9133 100644 --- a/schedulers/store.go +++ b/schedulers/store.go @@ -1,12 +1,13 @@ package schedulers import ( - "gitlab.com/spdf/elektron/utilities" "encoding/json" - sched "github.com/mesos/mesos-go/api/v0/scheduler" - "github.com/pkg/errors" "os" "sort" + + sched "github.com/mesos/mesos-go/api/v0/scheduler" + "github.com/pkg/errors" + "gitlab.com/spdf/elektron/utilities" ) // Names of different scheduling policies. diff --git a/utilities/offerUtils/offerUtils.go b/utilities/offerUtils/offerUtils.go index 450e6ec..27997fc 100644 --- a/utilities/offerUtils/offerUtils.go +++ b/utilities/offerUtils/offerUtils.go @@ -1,10 +1,11 @@ package offerUtils import ( - "gitlab.com/spdf/elektron/constants" - mesos "github.com/mesos/mesos-go/api/v0/mesosproto" "log" "strings" + + mesos "github.com/mesos/mesos-go/api/v0/mesosproto" + "gitlab.com/spdf/elektron/constants" ) func OfferAgg(offer *mesos.Offer) (float64, float64, float64) { diff --git a/utilities/schedUtils/schedUtils.go b/utilities/schedUtils/schedUtils.go index 7950f53..7bea147 100644 --- a/utilities/schedUtils/schedUtils.go +++ b/utilities/schedUtils/schedUtils.go @@ -1,9 +1,10 @@ package schedUtils import ( + "log" + "gitlab.com/spdf/elektron/def" "gitlab.com/spdf/elektron/utilities" - "log" ) // Criteria for resizing the scheduling window. diff --git a/utilities/trackResourceUsage.go b/utilities/trackResourceUsage.go index 7951a4f..7f82f8c 100644 --- a/utilities/trackResourceUsage.go +++ b/utilities/trackResourceUsage.go @@ -1,11 +1,12 @@ package utilities import ( + "errors" + "sync" + + mesos "github.com/mesos/mesos-go/api/v0/mesosproto" "gitlab.com/spdf/elektron/def" "gitlab.com/spdf/elektron/utilities/offerUtils" - "errors" - mesos "github.com/mesos/mesos-go/api/v0/mesosproto" - "sync" ) type TrackResourceUsage struct {