diff --git a/def/taskUtils.go b/def/taskUtils.go index c358a27..0259de2 100644 --- a/def/taskUtils.go +++ b/def/taskUtils.go @@ -52,7 +52,7 @@ func (tc TasksToClassify) taskObservationCalculator(task Task) []float64 { } else if task.Watts != 0.0 { return []float64{task.Watts} } else { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, log.FatalLevel, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.FatalLevel, log.Fields{}, "Unable to classify tasks. Missing Watts or ClassToWatts attribute in workload") return []float64{0.0} // Won't reach here. } @@ -108,7 +108,7 @@ func clusterSizeAvgMMMPU(tasks []Task, taskObservation func(task Task) []float64 } else { // skip this value // there is an error in the task config. - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, log.ErrorLevel, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.ErrorLevel, log.Fields{}, err.Error()) } } else { diff --git a/def/task_test.go b/def/task_test.go index 80ff2d2..28b77f4 100644 --- a/def/task_test.go +++ b/def/task_test.go @@ -20,11 +20,12 @@ package def import ( "fmt" + "reflect" + "testing" + mesos "github.com/mesos/mesos-go/api/v0/mesosproto" "github.com/spdfg/elektron/constants" "github.com/stretchr/testify/assert" - "reflect" - "testing" ) func TestTasksFromJSON(t *testing.T) { diff --git a/elektronLogging/ElektronFormatter.go b/elektronLogging/ElektronFormatter.go index 066cb95..97d609e 100644 --- a/elektronLogging/ElektronFormatter.go +++ b/elektronLogging/ElektronFormatter.go @@ -3,8 +3,9 @@ package elektronLogging import ( "bytes" "github.com/fatih/color" - log "github.com/sirupsen/logrus" "strings" + + log "github.com/sirupsen/logrus" ) type ElektronFormatter struct { diff --git a/elektronLogging/clsfnTaskDistOverheadLogger.go b/elektronLogging/clsfnTaskDistOverheadLogger.go index 2b70b62..db1dbee 100644 --- a/elektronLogging/clsfnTaskDistOverheadLogger.go +++ b/elektronLogging/clsfnTaskDistOverheadLogger.go @@ -1,10 +1,11 @@ package elektronLogging import ( - log "github.com/sirupsen/logrus" "os" "path/filepath" "strings" + + log "github.com/sirupsen/logrus" ) type ClsfnTaskDistrOverheadLogger struct { diff --git a/elektronLogging/consoleLogger.go b/elektronLogging/consoleLogger.go index fd2d079..52a62a6 100644 --- a/elektronLogging/consoleLogger.go +++ b/elektronLogging/consoleLogger.go @@ -1,10 +1,11 @@ package elektronLogging import ( - log "github.com/sirupsen/logrus" "os" "path/filepath" "strings" + + log "github.com/sirupsen/logrus" ) type ConsoleLogger struct { diff --git a/elektronLogging/createLogDir.go b/elektronLogging/createLogDir.go index 8d82024..c40abbc 100644 --- a/elektronLogging/createLogDir.go +++ b/elektronLogging/createLogDir.go @@ -1,11 +1,12 @@ package elektronLogging import ( - log "github.com/sirupsen/logrus" "os" "strconv" "strings" "time" + + log "github.com/sirupsen/logrus" ) type logDirectory struct { @@ -19,8 +20,7 @@ func (logD *logDirectory) getDirName() string { func (logD *logDirectory) createLogDir(prefix string, startTime time.Time) { if logD.name == "" { - // Creating directory to store all logs for this run - // directory name format - 2019-November-21_14-33-0 + // Creating directory to store all logs for this run. Directory name format - 2019-November-21_14-33-0. logDirName := strings.Join([]string{"./", prefix, strconv.Itoa(startTime.Year())}, "") logDirName = strings.Join([]string{logDirName, startTime.Month().String(), strconv.Itoa(startTime.Day())}, "-") logDirName = strings.Join([]string{logDirName, strconv.Itoa(startTime.Hour())}, "_") diff --git a/elektronLogging/logger.go b/elektronLogging/logger.go index 191cde6..7eae26e 100644 --- a/elektronLogging/logger.go +++ b/elektronLogging/logger.go @@ -1,30 +1,32 @@ package elektronLogging import ( - log "github.com/sirupsen/logrus" - . "github.com/spdfg/elektron/elektronLogging/types" "os" "strings" "time" + + log "github.com/sirupsen/logrus" + . "github.com/spdfg/elektron/elektronLogging/types" ) var config LoggerConfig var logger *log.Logger var formatter ElektronFormatter -var ElektronLog *ConsoleLogger +var ElektronLogger *ConsoleLogger var logDir logDirectory func BuildLogger(prefix string, logConfigFilename string) { - // read configuration from yaml + // Read configuration from yaml. config.GetConfig(logConfigFilename) - // create the log directory + // Create the log directory. startTime := time.Now() formatter.TimestampFormat = "2006-01-02 15:04:05" formattedStartTime := startTime.Format("20060102150405") - logDir.createLogDir(prefix, startTime) + + // Instantiate the logrus instance. prefix = strings.Join([]string{prefix, formattedStartTime}, "_") logger = &log.Logger{ Out: os.Stderr, @@ -32,8 +34,7 @@ func BuildLogger(prefix string, logConfigFilename string) { Formatter: &formatter, } - // create a chain of loggers - //head := &LoggerImpl{} + // Create a chain of loggers. cLog := NewConsoleLogger(CONSOLE, prefix) pLog := NewPCPLogger(PCP, prefix) schedTraceLog := NewSchedTraceLogger(SCHED_TRACE, prefix) @@ -41,12 +42,11 @@ func BuildLogger(prefix string, logConfigFilename string) { schedWindowLog := NewSchedWindowLogger(SCHED_WINDOW, prefix) tskDistLog := NewClsfnTaskDistrOverheadLogger(CLSFN_TASKDIST_OVERHEAD, prefix) - //head.SetNext(cLog) cLog.SetNext(pLog) pLog.SetNext(schedTraceLog) schedTraceLog.SetNext(spsLog) spsLog.SetNext(schedWindowLog) schedWindowLog.SetNext(tskDistLog) - ElektronLog = cLog + ElektronLogger = cLog } diff --git a/elektronLogging/pcpLogger.go b/elektronLogging/pcpLogger.go index db1e432..fd9c39c 100644 --- a/elektronLogging/pcpLogger.go +++ b/elektronLogging/pcpLogger.go @@ -1,10 +1,11 @@ package elektronLogging import ( - log "github.com/sirupsen/logrus" "os" "path/filepath" "strings" + + log "github.com/sirupsen/logrus" ) type PCPLogger struct { diff --git a/elektronLogging/schedPolicySwitchLogger.go b/elektronLogging/schedPolicySwitchLogger.go index bae2c3e..8e7b887 100644 --- a/elektronLogging/schedPolicySwitchLogger.go +++ b/elektronLogging/schedPolicySwitchLogger.go @@ -1,10 +1,11 @@ package elektronLogging import ( - log "github.com/sirupsen/logrus" "os" "path/filepath" "strings" + + log "github.com/sirupsen/logrus" ) type SchedPolicySwitchLogger struct { diff --git a/elektronLogging/schedTraceLogger.go b/elektronLogging/schedTraceLogger.go index 8a4c5ba..9b692b9 100644 --- a/elektronLogging/schedTraceLogger.go +++ b/elektronLogging/schedTraceLogger.go @@ -1,10 +1,11 @@ package elektronLogging import ( - log "github.com/sirupsen/logrus" "os" "path/filepath" "strings" + + log "github.com/sirupsen/logrus" ) type SchedTraceLogger struct { diff --git a/elektronLogging/schedWindowLogger.go b/elektronLogging/schedWindowLogger.go index 09fc633..b5e94da 100644 --- a/elektronLogging/schedWindowLogger.go +++ b/elektronLogging/schedWindowLogger.go @@ -1,10 +1,11 @@ package elektronLogging import ( - log "github.com/sirupsen/logrus" "os" "path/filepath" "strings" + + log "github.com/sirupsen/logrus" ) type SchedWindowLogger struct { diff --git a/pcp/pcp.go b/pcp/pcp.go index cbe8170..8781651 100644 --- a/pcp/pcp.go +++ b/pcp/pcp.go @@ -47,7 +47,7 @@ func Start(quit chan struct{}, logging *bool, pcpConfigFile string) { scanner.Scan() // Write to logfile - elekLog.ElektronLog.Log(elekLogTypes.PCP, + elekLog.ElektronLogger.Log(elekLogTypes.PCP, log.InfoLevel, log.Fields{}, scanner.Text()) @@ -60,7 +60,7 @@ func Start(quit chan struct{}, logging *bool, pcpConfigFile string) { text := scanner.Text() if *logging { - elekLog.ElektronLog.Log(elekLogTypes.PCP, + elekLog.ElektronLogger.Log(elekLogTypes.PCP, log.InfoLevel, log.Fields{}, text) } @@ -69,7 +69,7 @@ func Start(quit chan struct{}, logging *bool, pcpConfigFile string) { } }(logging) - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "PCP logging started") @@ -81,7 +81,7 @@ func Start(quit chan struct{}, logging *bool, pcpConfigFile string) { select { case <-quit: - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "Stopping PCP logging in 5 seconds") time.Sleep(5 * time.Second) diff --git a/powerCap/extrema.go b/powerCap/extrema.go index be8b158..67b00b9 100644 --- a/powerCap/extrema.go +++ b/powerCap/extrema.go @@ -43,7 +43,7 @@ func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThresh cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} if hiThreshold < loThreshold { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "High threshold is lower than low threshold!") } @@ -61,7 +61,7 @@ func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThresh scanner.Scan() // Write to logfile - elekLog.ElektronLog.Log(elekLogTypes.PCP, + elekLog.ElektronLogger.Log(elekLogTypes.PCP, log.InfoLevel, log.Fields{}, scanner.Text()) @@ -99,14 +99,14 @@ func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThresh if *logging { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "Logging PCP...") text := scanner.Text() split := strings.Split(text, ",") - elekLog.ElektronLog.Log(elekLogTypes.PCP, + elekLog.ElektronLogger.Log(elekLogTypes.PCP, log.InfoLevel, log.Fields{}, text) @@ -119,7 +119,7 @@ func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThresh powerHistories[host].Value = power powerHistories[host] = powerHistories[host].Next() - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"Host": fmt.Sprintf("%s", indexToHost[powerIndex]), "Power": fmt.Sprintf("%f", (power * pcp.RAPLUnits))}, "") @@ -133,14 +133,14 @@ func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThresh clusterMean := pcp.AverageClusterPowerHistory(clusterPowerHist) - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"Total power": fmt.Sprintf("%f %d", clusterPower, clusterPowerHist.Len()), "Sec Avg": fmt.Sprintf("%f", clusterMean)}, "") if clusterMean > hiThreshold { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "Need to cap a node") // Create statics for all victims and choose one to cap @@ -163,12 +163,12 @@ func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThresh if !cappedHosts[victim.Host] { cappedHosts[victim.Host] = true orderCapped = append(orderCapped, victim.Host) - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"Capping Victim": fmt.Sprintf("%s", victim.Host), "Avg. Wattage": fmt.Sprintf("%f", victim.Watts*pcp.RAPLUnits)}, "") if err := rapl.Cap(victim.Host, "rapl", 50); err != nil { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.ErrorLevel, log.Fields{}, "Error capping host") } @@ -184,11 +184,11 @@ func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThresh cappedHosts[host] = false // User RAPL package to send uncap. log.Printf("Uncapping host %s", host) - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"Uncapped host": host}, "") if err := rapl.Cap(host, "rapl", 100); err != nil { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.ErrorLevel, log.Fields{}, "Error capping host") } @@ -200,7 +200,7 @@ func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThresh } }(logging, hiThreshold, loThreshold) - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "PCP logging started") @@ -212,7 +212,7 @@ func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThresh select { case <-quit: - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "Stopping PCP logging in 5 seconds") time.Sleep(5 * time.Second) diff --git a/powerCap/progressiveExtrema.go b/powerCap/progressiveExtrema.go index 6cd234f..1362f7a 100644 --- a/powerCap/progressiveExtrema.go +++ b/powerCap/progressiveExtrema.go @@ -56,7 +56,7 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} if hiThreshold < loThreshold { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "High threshold is lower than low threshold!") } @@ -74,7 +74,7 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh scanner.Scan() // Write to logfile - elekLog.ElektronLog.Log(elekLogTypes.PCP, + elekLog.ElektronLogger.Log(elekLogTypes.PCP, log.InfoLevel, log.Fields{}, scanner.Text()) @@ -115,13 +115,13 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh for scanner.Scan() { if *logging { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "Logging PCP...") split := strings.Split(scanner.Text(), ",") text := scanner.Text() - elekLog.ElektronLog.Log(elekLogTypes.PCP, + elekLog.ElektronLogger.Log(elekLogTypes.PCP, log.InfoLevel, log.Fields{}, text) @@ -134,7 +134,7 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh powerHistories[host].Value = power powerHistories[host] = powerHistories[host].Next() - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"Host": fmt.Sprintf("%s", indexToHost[powerIndex]), "Power": fmt.Sprintf("%f", (power * pcp.RAPLUnits))}, "") @@ -147,22 +147,22 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh clusterMean := pcp.AverageClusterPowerHistory(clusterPowerHist) - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"Total power": fmt.Sprintf("%f %d", clusterPower, clusterPowerHist.Len()), "Sec Avg": fmt.Sprintf("%f", clusterMean)}, "") if clusterMean >= hiThreshold { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "Need to cap a node") - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"Cap values of capped victims": fmt.Sprintf("%v", cappedVictims)}, "") - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"Cap values of victims to uncap": fmt.Sprintf("%v", orderCappedVictims)}, "") // Create statics for all victims and choose one to cap @@ -192,12 +192,12 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh // Need to cap this victim. if err := rapl.Cap(victims[i].Host, "rapl", 50.0); err != nil { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.ErrorLevel, log.Fields{"Error capping host": fmt.Sprintf("%s", victims[i].Host)}, "") } else { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, fmt.Sprintf("Capped host[%s] at %f", victims[i].Host, 50.0)) // Keeping track of this victim and it's cap value @@ -223,12 +223,12 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh newCapValue := getNextCapValue(capValue, 2) if err := rapl.Cap(alreadyCappedHosts[i], "rapl", newCapValue); err != nil { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.ErrorLevel, log.Fields{"Error capping host": fmt.Sprintf("%s", alreadyCappedHosts[i])}, "") } else { // Successful cap - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, fmt.Sprintf("Capped host[%s] at %f", alreadyCappedHosts[i], newCapValue)) // Checking whether this victim can be capped further @@ -253,7 +253,7 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh } } if !canCapAlreadyCappedVictim { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "No Victim left to cap") } @@ -261,13 +261,13 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh } else if clusterMean < loThreshold { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "Need to uncap a node") - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"Cap values of capped victims": fmt.Sprintf("%v", cappedVictims)}, "") - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"Cap values of victims to uncap": fmt.Sprintf("%v", orderCappedVictims)}, "") if len(orderCapped) > 0 { @@ -280,12 +280,12 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh newUncapValue := orderCappedVictims[hostToUncap] * 2.0 if err := rapl.Cap(hostToUncap, "rapl", newUncapValue); err != nil { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.ErrorLevel, log.Fields{"Error uncapping host": fmt.Sprintf("%s", hostToUncap)}, "") } else { // Successful uncap - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, fmt.Sprintf("Uncapped host[%s] to %f", hostToUncap, newUncapValue)) // Can we uncap this host further. If not, then we remove its entry from orderCapped @@ -308,7 +308,7 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh } } } else { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "No host staged for Uncapped") } @@ -319,7 +319,7 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh }(logging, hiThreshold, loThreshold) - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "PCP logging started") if err := cmd.Start(); err != nil { @@ -330,7 +330,7 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh select { case <-quit: - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "Stopping PCP logging in 5 seconds") time.Sleep(5 * time.Second) diff --git a/rapl/cap.go b/rapl/cap.go index 42f94f6..2b36e9d 100644 --- a/rapl/cap.go +++ b/rapl/cap.go @@ -21,11 +21,11 @@ package rapl import ( "os" "strconv" + "strings" "github.com/pkg/errors" elekEnv "github.com/spdfg/elektron/environment" "golang.org/x/crypto/ssh" - "strings" ) func Cap(host, username string, percentage float64) error { diff --git a/scheduler.go b/scheduler.go index 299dfca..d171870 100644 --- a/scheduler.go +++ b/scheduler.go @@ -21,6 +21,11 @@ package main // import github.com/spdfg/elektron import ( "flag" "fmt" + "os" + "os/signal" + "strings" + "time" + "github.com/golang/protobuf/proto" mesos "github.com/mesos/mesos-go/api/v0/mesosproto" sched "github.com/mesos/mesos-go/api/v0/scheduler" @@ -31,10 +36,6 @@ import ( "github.com/spdfg/elektron/pcp" "github.com/spdfg/elektron/powerCap" "github.com/spdfg/elektron/schedulers" - "os" - "os/signal" - "strings" - "time" ) var master = flag.String("master", "", "Location of leading Mesos master -- :") @@ -225,6 +226,7 @@ func main() { if strings.Contains(*pcplogPrefix, "/") { log.Fatal("log file prefix should not contain '/'.") } + // Build Logger for elektron. elekLog.BuildLogger(*pcplogPrefix, *logConfigFilename) // Starting PCP logging. @@ -280,11 +282,11 @@ func main() { // Starting the scheduler driver. if status, err := driver.Run(); err != nil { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.ErrorLevel, log.Fields{"status": status.String(), "error": err.Error()}, "Framework stopped ") } - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "Exiting...") } diff --git a/schedulers/MaxGreedyMins.go b/schedulers/MaxGreedyMins.go index b4b250d..12adfcb 100644 --- a/schedulers/MaxGreedyMins.go +++ b/schedulers/MaxGreedyMins.go @@ -19,10 +19,9 @@ package schedulers import ( - log "github.com/sirupsen/logrus" - mesos "github.com/mesos/mesos-go/api/v0/mesosproto" sched "github.com/mesos/mesos-go/api/v0/scheduler" + log "github.com/sirupsen/logrus" "github.com/spdfg/elektron/def" "github.com/spdfg/elektron/utilities/mesosUtils" "github.com/spdfg/elektron/utilities/offerUtils" diff --git a/schedulers/MaxMin.go b/schedulers/MaxMin.go index a7ba29e..7901698 100644 --- a/schedulers/MaxMin.go +++ b/schedulers/MaxMin.go @@ -19,10 +19,9 @@ package schedulers import ( - log "github.com/sirupsen/logrus" - mesos "github.com/mesos/mesos-go/api/v0/mesosproto" sched "github.com/mesos/mesos-go/api/v0/scheduler" + log "github.com/sirupsen/logrus" "github.com/spdfg/elektron/def" "github.com/spdfg/elektron/utilities/mesosUtils" "github.com/spdfg/elektron/utilities/offerUtils" diff --git a/schedulers/base.go b/schedulers/base.go index 31c0d34..5844281 100644 --- a/schedulers/base.go +++ b/schedulers/base.go @@ -250,10 +250,10 @@ func (s *BaseScheduler) StatusUpdate(driver sched.SchedulerDriver, status *mesos func (s *BaseScheduler) LogTaskStarting(ts *def.Task, offer *mesos.Offer) { if ts == nil { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, log.InfoLevel, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"host": fmt.Sprintf("%s", offer.GetHostname())}, "TASKS STARTING...") } else { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"task": fmt.Sprintf("%s", ts.Name), "Instance": fmt.Sprintf("%d", *ts.Instances), "host": fmt.Sprintf("%s", offer.GetHostname())}, @@ -262,25 +262,25 @@ func (s *BaseScheduler) LogTaskStarting(ts *def.Task, offer *mesos.Offer) { } func (s *BaseScheduler) LogTaskWattsConsideration(ts def.Task, host string, wattsToConsider float64) { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"task": ts.Name, "host": host, "Watts": fmt.Sprintf("%f", wattsToConsider)}, "Watts considered for ") } func (s *BaseScheduler) LogOffersReceived(offers []*mesos.Offer) { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"Resource offers received": fmt.Sprintf("%d", len(offers))}, "") } func (s *BaseScheduler) LogNoPendingTasksDeclineOffers(offer *mesos.Offer) { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.WarnLevel, log.Fields{"DECLINING OFFER for host": fmt.Sprintf("%s", offer.GetHostname())}, "No tasks left to schedule ") } func (s *BaseScheduler) LogNumberOfRunningTasks() { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"Number of tasks still Running": fmt.Sprintf("%d", s.tasksRunning)}, "") } @@ -292,81 +292,81 @@ func (s *BaseScheduler) LogCoLocatedTasks(slaveID string) { buffer.WriteString(fmt.Sprintln(taskName)) } s.TasksRunningMutex.Unlock() - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"Colocated with": fmt.Sprintf("%s", buffer.String())}, "") } func (s *BaseScheduler) LogSchedTrace(taskToSchedule *mesos.TaskInfo, offer *mesos.Offer) { - elekLog.ElektronLog.Log(elekLogTypes.SCHED_TRACE, + elekLog.ElektronLogger.Log(elekLogTypes.SCHED_TRACE, log.InfoLevel, log.Fields{offer.GetHostname(): fmt.Sprintf("%s", taskToSchedule.GetTaskId().GetValue())}, "") } func (s *BaseScheduler) LogTerminateScheduler() { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "Done scheduling all tasks!") + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "Done scheduling all tasks!") } func (s *BaseScheduler) LogInsufficientResourcesDeclineOffer(offer *mesos.Offer, offerResources ...interface{}) { buffer := bytes.Buffer{} buffer.WriteString(fmt.Sprintf("", offerResources...)) - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.WarnLevel, log.Fields{"Offer Resources": fmt.Sprintf("%s", buffer.String())}, "DECLINING OFFER... Offer has insufficient resources to launch a task") } func (s *BaseScheduler) LogOfferRescinded(offerID *mesos.OfferID) { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.ErrorLevel, log.Fields{"OfferID": fmt.Sprintf("%s", offerID)}, "OFFER RESCINDED") } func (s *BaseScheduler) LogSlaveLost(slaveID *mesos.SlaveID) { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.ErrorLevel, log.Fields{"SlaveID": fmt.Sprintf("%s", slaveID)}, "SLAVE LOST") } func (s *BaseScheduler) LogExecutorLost(executorID *mesos.ExecutorID, slaveID *mesos.SlaveID) { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.ErrorLevel, log.Fields{"ExecutorID": fmt.Sprintf("%s", executorID), "SlaveID": fmt.Sprintf("%s", slaveID)}, "EXECUTOR LOST") } func (s *BaseScheduler) LogFrameworkMessage(executorID *mesos.ExecutorID, slaveID *mesos.SlaveID, message string) { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"Received Framework message from executor": executorID}, message) } func (s *BaseScheduler) LogMesosError(err string) { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.ErrorLevel, log.Fields{"MESOS CONSOLE": fmt.Sprintf("%v", err)}, "") } func (s *BaseScheduler) LogElectronError(err error) { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.ErrorLevel, log.Fields{"ELECTRON CONSOLE": fmt.Sprintf("%v", err)}, "") } func (s *BaseScheduler) LogFrameworkRegistered(frameworkID *mesos.FrameworkID, masterInfo *mesos.MasterInfo) { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"frameworkID": fmt.Sprintf("%s", frameworkID), "master": fmt.Sprintf("%v", masterInfo)}, "FRAMEWORK REGISTERED!") } func (s *BaseScheduler) LogFrameworkReregistered(masterInfo *mesos.MasterInfo) { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"master": fmt.Sprintf("%v", masterInfo)}, "Framework re-registered") } func (s *BaseScheduler) LogDisconnected() { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, log.WarnLevel, log.Fields{}, "Framework disconnected with master") + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.WarnLevel, log.Fields{}, "Framework disconnected with master") } func (s *BaseScheduler) LogTaskStatusUpdate(status *mesos.TaskStatus) { @@ -378,13 +378,13 @@ func (s *BaseScheduler) LogTaskStatusUpdate(status *mesos.TaskStatus) { default: level = log.InfoLevel } - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, level, log.Fields{"task": fmt.Sprintf("%s", *status.TaskId.Value), "state": NameFor(status.State)}, "Task Status received") } func (s *BaseScheduler) LogSchedPolicySwitch(name string, nextPolicy SchedPolicyState) { logSPS := func() { - elekLog.ElektronLog.Log(elekLogTypes.SPS, log.InfoLevel, log.Fields{"Name": name}, "") + elekLog.ElektronLogger.Log(elekLogTypes.SPS, log.InfoLevel, log.Fields{"Name": name}, "") } if s.hasReceivedResourceOffers && (s.curSchedPolicy != nextPolicy) { logSPS() @@ -393,14 +393,14 @@ func (s *BaseScheduler) LogSchedPolicySwitch(name string, nextPolicy SchedPolicy } // Logging the size of the scheduling window and the scheduling policy // that is going to schedule the tasks in the scheduling window. - elekLog.ElektronLog.Log(elekLogTypes.SCHED_WINDOW, + elekLog.ElektronLogger.Log(elekLogTypes.SCHED_WINDOW, log.InfoLevel, log.Fields{"Window size": fmt.Sprintf("%d", s.schedWindowSize), "Name": name}, "") } func (s *BaseScheduler) LogClsfnAndTaskDistOverhead(overhead time.Duration) { // Logging the overhead in microseconds. - elekLog.ElektronLog.Log(elekLogTypes.CLSFN_TASKDIST_OVERHEAD, + elekLog.ElektronLogger.Log(elekLogTypes.CLSFN_TASKDIST_OVERHEAD, log.InfoLevel, log.Fields{"Overhead in microseconds": fmt.Sprintf("%f", float64(overhead.Nanoseconds())/1000.0)}, "") } diff --git a/schedulers/bin-packing.go b/schedulers/bin-packing.go index 8bd90fb..edd40e3 100644 --- a/schedulers/bin-packing.go +++ b/schedulers/bin-packing.go @@ -19,10 +19,9 @@ package schedulers import ( - log "github.com/sirupsen/logrus" - mesos "github.com/mesos/mesos-go/api/v0/mesosproto" sched "github.com/mesos/mesos-go/api/v0/scheduler" + log "github.com/sirupsen/logrus" "github.com/spdfg/elektron/def" "github.com/spdfg/elektron/utilities/mesosUtils" "github.com/spdfg/elektron/utilities/offerUtils" diff --git a/schedulers/helpers.go b/schedulers/helpers.go index 3cb81b3..2bf0cd3 100644 --- a/schedulers/helpers.go +++ b/schedulers/helpers.go @@ -20,6 +20,7 @@ package schedulers import ( "fmt" + mesos "github.com/mesos/mesos-go/api/v0/mesosproto" sched "github.com/mesos/mesos-go/api/v0/scheduler" "github.com/pkg/errors" @@ -35,10 +36,10 @@ import ( func coLocated(tasks map[string]bool, s BaseScheduler) { for _, task := range tasks { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"Task": task}, "") + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"Task": task}, "") } - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "---------------------") + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "---------------------") } // Get the powerClass of the given hostname. diff --git a/schedulers/schedPolicy.go b/schedulers/schedPolicy.go index eb98176..dd0c58f 100644 --- a/schedulers/schedPolicy.go +++ b/schedulers/schedPolicy.go @@ -90,7 +90,7 @@ func switchTaskDistBased(baseSchedRef *BaseScheduler) string { // Determine the distribution of tasks in the new scheduling window. taskDist, err := def.GetTaskDistributionInWindow(baseSchedRef.schedWindowSize, baseSchedRef.tasks) baseSchedRef.LogClsfnAndTaskDistOverhead(time.Now().Sub(startTime)) - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"Task Distribution": fmt.Sprintf("%f", taskDist)}, "Switching... ") if err != nil { diff --git a/utilities/offerUtils/offerUtils.go b/utilities/offerUtils/offerUtils.go index 05f1465..7aaf30d 100644 --- a/utilities/offerUtils/offerUtils.go +++ b/utilities/offerUtils/offerUtils.go @@ -90,13 +90,13 @@ func UpdateEnvironment(offer *mesos.Offer) { var host = offer.GetHostname() // If this host is not present in the set of hosts. if _, ok := constants.Hosts[host]; !ok { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, log.InfoLevel, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"Adding host": host}, "New host detected") // Add this host. constants.Hosts[host] = struct{}{} // Get the power class of this host. class := PowerClass(offer) - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, log.InfoLevel, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"host": host, "PowerClass": class}, "Registering the power class...") // If new power class, register the power class. if _, ok := constants.PowerClasses[class]; !ok { diff --git a/utilities/schedUtils/schedUtils.go b/utilities/schedUtils/schedUtils.go index 98631ca..6f50334 100644 --- a/utilities/schedUtils/schedUtils.go +++ b/utilities/schedUtils/schedUtils.go @@ -20,6 +20,7 @@ package schedUtils import ( "fmt" + log "github.com/sirupsen/logrus" "github.com/spdfg/elektron/def" elekLog "github.com/spdfg/elektron/elektronLogging" @@ -79,7 +80,7 @@ func (s *fillNextOfferCycle) apply(taskQueue []def.Task) (int, int) { for _, task := range taskQueue { numberOfTasksTraversed++ for i := *task.Instances; i > 0; i-- { - elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, log.InfoLevel, + elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, fmt.Sprintf("Checking if Instance #%d of Task[%s] can be scheduled "+ "during the next offer cycle...", i, task.Name)) if canSchedule(task) {