spdf/elektron
Archived
1
0
Fork 0
Code Issues 4 Pull requests 1 Projects Releases Packages Wiki Activity

Code Refactoring

Browse source 
Modified comments. Organized imports.
This commit is contained in:
balandi1 2019-11-26 14:24:20 -05:00
parent 2fa3bebaa6
commit 0c0361e209
24 changed files with 114 additions and 105 deletions
Download patch file Download diff file Expand all files Collapse all files

4
def/taskUtils.go
View file

@ -52,7 +52,7 @@ func (tc TasksToClassify) taskObservationCalculator(task Task) []float64 {
} else if task.Watts != 0.0 {
return []float64{task.Watts}
} else {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, log.FatalLevel,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.FatalLevel,
log.Fields{}, "Unable to classify tasks. Missing Watts or ClassToWatts attribute in workload")
return []float64{0.0} // Won't reach here.
}
@ -108,7 +108,7 @@ func clusterSizeAvgMMMPU(tasks []Task, taskObservation func(task Task) []float64
} else {
// skip this value
// there is an error in the task config.
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, log.ErrorLevel,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.ErrorLevel,
log.Fields{}, err.Error())
}
} else {

5
def/task_test.go
View file

@ -20,11 +20,12 @@ package def
import (
"fmt"
"reflect"
"testing"
mesos "github.com/mesos/mesos-go/api/v0/mesosproto"
"github.com/spdfg/elektron/constants"
"github.com/stretchr/testify/assert"
"reflect"
"testing"
)
func TestTasksFromJSON(t *testing.T) {

3
elektronLogging/ElektronFormatter.go
View file

@ -3,8 +3,9 @@ package elektronLogging
import (
"bytes"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"strings"
log "github.com/sirupsen/logrus"
)
type ElektronFormatter struct {

3
elektronLogging/clsfnTaskDistOverheadLogger.go
View file

@ -1,10 +1,11 @@
package elektronLogging
import (
log "github.com/sirupsen/logrus"
"os"
"path/filepath"
"strings"
log "github.com/sirupsen/logrus"
)
type ClsfnTaskDistrOverheadLogger struct {

3
elektronLogging/consoleLogger.go
View file

@ -1,10 +1,11 @@
package elektronLogging
import (
log "github.com/sirupsen/logrus"
"os"
"path/filepath"
"strings"
log "github.com/sirupsen/logrus"
)
type ConsoleLogger struct {

6
elektronLogging/createLogDir.go
View file

@ -1,11 +1,12 @@
package elektronLogging
import (
log "github.com/sirupsen/logrus"
"os"
"strconv"
"strings"
"time"
log "github.com/sirupsen/logrus"
)
type logDirectory struct {
@ -19,8 +20,7 @@ func (logD *logDirectory) getDirName() string {
func (logD *logDirectory) createLogDir(prefix string, startTime time.Time) {
if logD.name == "" {
// Creating directory to store all logs for this run
// directory name format - 2019-November-21_14-33-0
// Creating directory to store all logs for this run. Directory name format - 2019-November-21_14-33-0.
logDirName := strings.Join([]string{"./", prefix, strconv.Itoa(startTime.Year())}, "")
logDirName = strings.Join([]string{logDirName, startTime.Month().String(), strconv.Itoa(startTime.Day())}, "-")
logDirName = strings.Join([]string{logDirName, strconv.Itoa(startTime.Hour())}, "_")

20
elektronLogging/logger.go
View file

@ -1,30 +1,32 @@
package elektronLogging
import (
log "github.com/sirupsen/logrus"
. "github.com/spdfg/elektron/elektronLogging/types"
"os"
"strings"
"time"
log "github.com/sirupsen/logrus"
. "github.com/spdfg/elektron/elektronLogging/types"
)
var config LoggerConfig
var logger *log.Logger
var formatter ElektronFormatter
var ElektronLog *ConsoleLogger
var ElektronLogger *ConsoleLogger
var logDir logDirectory
func BuildLogger(prefix string, logConfigFilename string) {
// read configuration from yaml
// Read configuration from yaml.
config.GetConfig(logConfigFilename)
// create the log directory
// Create the log directory.
startTime := time.Now()
formatter.TimestampFormat = "2006-01-02 15:04:05"
formattedStartTime := startTime.Format("20060102150405")
logDir.createLogDir(prefix, startTime)
// Instantiate the logrus instance.
prefix = strings.Join([]string{prefix, formattedStartTime}, "_")
logger = &log.Logger{
Out: os.Stderr,
@ -32,8 +34,7 @@ func BuildLogger(prefix string, logConfigFilename string) {
Formatter: &formatter,
}
// create a chain of loggers
//head := &LoggerImpl{}
// Create a chain of loggers.
cLog := NewConsoleLogger(CONSOLE, prefix)
pLog := NewPCPLogger(PCP, prefix)
schedTraceLog := NewSchedTraceLogger(SCHED_TRACE, prefix)
@ -41,12 +42,11 @@ func BuildLogger(prefix string, logConfigFilename string) {
schedWindowLog := NewSchedWindowLogger(SCHED_WINDOW, prefix)
tskDistLog := NewClsfnTaskDistrOverheadLogger(CLSFN_TASKDIST_OVERHEAD, prefix)
//head.SetNext(cLog)
cLog.SetNext(pLog)
pLog.SetNext(schedTraceLog)
schedTraceLog.SetNext(spsLog)
spsLog.SetNext(schedWindowLog)
schedWindowLog.SetNext(tskDistLog)
ElektronLog = cLog
ElektronLogger = cLog
}

3
elektronLogging/pcpLogger.go
View file

@ -1,10 +1,11 @@
package elektronLogging
import (
log "github.com/sirupsen/logrus"
"os"
"path/filepath"
"strings"
log "github.com/sirupsen/logrus"
)
type PCPLogger struct {

3
elektronLogging/schedPolicySwitchLogger.go
View file

@ -1,10 +1,11 @@
package elektronLogging
import (
log "github.com/sirupsen/logrus"
"os"
"path/filepath"
"strings"
log "github.com/sirupsen/logrus"
)
type SchedPolicySwitchLogger struct {

3
elektronLogging/schedTraceLogger.go
View file

@ -1,10 +1,11 @@
package elektronLogging
import (
log "github.com/sirupsen/logrus"
"os"
"path/filepath"
"strings"
log "github.com/sirupsen/logrus"
)
type SchedTraceLogger struct {

3
elektronLogging/schedWindowLogger.go
View file

@ -1,10 +1,11 @@
package elektronLogging
import (
log "github.com/sirupsen/logrus"
"os"
"path/filepath"
"strings"
log "github.com/sirupsen/logrus"
)
type SchedWindowLogger struct {

8
pcp/pcp.go
View file

@ -47,7 +47,7 @@ func Start(quit chan struct{}, logging *bool, pcpConfigFile string) {
scanner.Scan()
// Write to logfile
elekLog.ElektronLog.Log(elekLogTypes.PCP,
elekLog.ElektronLogger.Log(elekLogTypes.PCP,
log.InfoLevel,
log.Fields{}, scanner.Text())
@ -60,7 +60,7 @@ func Start(quit chan struct{}, logging *bool, pcpConfigFile string) {
text := scanner.Text()
if *logging {
elekLog.ElektronLog.Log(elekLogTypes.PCP,
elekLog.ElektronLogger.Log(elekLogTypes.PCP,
log.InfoLevel,
log.Fields{}, text)
}
@ -69,7 +69,7 @@ func Start(quit chan struct{}, logging *bool, pcpConfigFile string) {
}
}(logging)
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{}, "PCP logging started")
@ -81,7 +81,7 @@ func Start(quit chan struct{}, logging *bool, pcpConfigFile string) {
select {
case <-quit:
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{}, "Stopping PCP logging in 5 seconds")
time.Sleep(5 * time.Second)

26
powerCap/extrema.go
View file

@ -43,7 +43,7 @@ func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThresh
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
if hiThreshold < loThreshold {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{}, "High threshold is lower than low threshold!")
}
@ -61,7 +61,7 @@ func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThresh
scanner.Scan()
// Write to logfile
elekLog.ElektronLog.Log(elekLogTypes.PCP,
elekLog.ElektronLogger.Log(elekLogTypes.PCP,
log.InfoLevel,
log.Fields{}, scanner.Text())
@ -99,14 +99,14 @@ func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThresh
if *logging {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{}, "Logging PCP...")
text := scanner.Text()
split := strings.Split(text, ",")
elekLog.ElektronLog.Log(elekLogTypes.PCP,
elekLog.ElektronLogger.Log(elekLogTypes.PCP,
log.InfoLevel,
log.Fields{}, text)
@ -119,7 +119,7 @@ func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThresh
powerHistories[host].Value = power
powerHistories[host] = powerHistories[host].Next()
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{"Host": fmt.Sprintf("%s", indexToHost[powerIndex]), "Power": fmt.Sprintf("%f", (power * pcp.RAPLUnits))},
"")
@ -133,14 +133,14 @@ func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThresh
clusterMean := pcp.AverageClusterPowerHistory(clusterPowerHist)
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{"Total power": fmt.Sprintf("%f %d", clusterPower, clusterPowerHist.Len()),
"Sec Avg": fmt.Sprintf("%f", clusterMean)},
"")
if clusterMean > hiThreshold {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{}, "Need to cap a node")
// Create statics for all victims and choose one to cap
@ -163,12 +163,12 @@ func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThresh
if !cappedHosts[victim.Host] {
cappedHosts[victim.Host] = true
orderCapped = append(orderCapped, victim.Host)
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{"Capping Victim": fmt.Sprintf("%s", victim.Host),
"Avg. Wattage": fmt.Sprintf("%f", victim.Watts*pcp.RAPLUnits)}, "")
if err := rapl.Cap(victim.Host, "rapl", 50); err != nil {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.ErrorLevel,
log.Fields{}, "Error capping host")
}
@ -184,11 +184,11 @@ func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThresh
cappedHosts[host] = false
// User RAPL package to send uncap.
log.Printf("Uncapping host %s", host)
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{"Uncapped host": host}, "")
if err := rapl.Cap(host, "rapl", 100); err != nil {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.ErrorLevel,
log.Fields{}, "Error capping host")
}
@ -200,7 +200,7 @@ func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThresh
}
}(logging, hiThreshold, loThreshold)
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{}, "PCP logging started")
@ -212,7 +212,7 @@ func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThresh
select {
case <-quit:
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{}, "Stopping PCP logging in 5 seconds")
time.Sleep(5 * time.Second)

44
powerCap/progressiveExtrema.go
View file

@ -56,7 +56,7 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
if hiThreshold < loThreshold {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{}, "High threshold is lower than low threshold!")
}
@ -74,7 +74,7 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh
scanner.Scan()
// Write to logfile
elekLog.ElektronLog.Log(elekLogTypes.PCP,
elekLog.ElektronLogger.Log(elekLogTypes.PCP,
log.InfoLevel,
log.Fields{}, scanner.Text())
@ -115,13 +115,13 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh
for scanner.Scan() {
if *logging {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{}, "Logging PCP...")
split := strings.Split(scanner.Text(), ",")
text := scanner.Text()
elekLog.ElektronLog.Log(elekLogTypes.PCP,
elekLog.ElektronLogger.Log(elekLogTypes.PCP,
log.InfoLevel,
log.Fields{}, text)
@ -134,7 +134,7 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh
powerHistories[host].Value = power
powerHistories[host] = powerHistories[host].Next()
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{"Host": fmt.Sprintf("%s", indexToHost[powerIndex]), "Power": fmt.Sprintf("%f", (power * pcp.RAPLUnits))},
"")
@ -147,22 +147,22 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh
clusterMean := pcp.AverageClusterPowerHistory(clusterPowerHist)
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{"Total power": fmt.Sprintf("%f %d", clusterPower, clusterPowerHist.Len()),
"Sec Avg": fmt.Sprintf("%f", clusterMean)},
"")
if clusterMean >= hiThreshold {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{}, "Need to cap a node")
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{"Cap values of capped victims": fmt.Sprintf("%v", cappedVictims)}, "")
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{"Cap values of victims to uncap": fmt.Sprintf("%v", orderCappedVictims)}, "")
// Create statics for all victims and choose one to cap
@ -192,12 +192,12 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh
// Need to cap this victim.
if err := rapl.Cap(victims[i].Host, "rapl", 50.0); err != nil {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.ErrorLevel,
log.Fields{"Error capping host": fmt.Sprintf("%s", victims[i].Host)}, "")
} else {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{}, fmt.Sprintf("Capped host[%s] at %f", victims[i].Host, 50.0))
// Keeping track of this victim and it's cap value
@ -223,12 +223,12 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh
newCapValue := getNextCapValue(capValue, 2)
if err := rapl.Cap(alreadyCappedHosts[i], "rapl", newCapValue); err != nil {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.ErrorLevel,
log.Fields{"Error capping host": fmt.Sprintf("%s", alreadyCappedHosts[i])}, "")
} else {
// Successful cap
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{}, fmt.Sprintf("Capped host[%s] at %f", alreadyCappedHosts[i], newCapValue))
// Checking whether this victim can be capped further
@ -253,7 +253,7 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh
}
}
if !canCapAlreadyCappedVictim {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{}, "No Victim left to cap")
}
@ -261,13 +261,13 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh
} else if clusterMean < loThreshold {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{}, "Need to uncap a node")
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{"Cap values of capped victims": fmt.Sprintf("%v", cappedVictims)}, "")
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{"Cap values of victims to uncap": fmt.Sprintf("%v", orderCappedVictims)}, "")
if len(orderCapped) > 0 {
@ -280,12 +280,12 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh
newUncapValue := orderCappedVictims[hostToUncap] * 2.0
if err := rapl.Cap(hostToUncap, "rapl", newUncapValue); err != nil {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.ErrorLevel,
log.Fields{"Error uncapping host": fmt.Sprintf("%s", hostToUncap)}, "")
} else {
// Successful uncap
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{}, fmt.Sprintf("Uncapped host[%s] to %f", hostToUncap, newUncapValue))
// Can we uncap this host further. If not, then we remove its entry from orderCapped
@ -308,7 +308,7 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh
}
}
} else {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{}, "No host staged for Uncapped")
}
@ -319,7 +319,7 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh
}(logging, hiThreshold, loThreshold)
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{}, "PCP logging started")
if err := cmd.Start(); err != nil {
@ -330,7 +330,7 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh
select {
case <-quit:
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{}, "Stopping PCP logging in 5 seconds")
time.Sleep(5 * time.Second)

2
rapl/cap.go
View file

@ -21,11 +21,11 @@ package rapl
import (
"os"
"strconv"
"strings"
"github.com/pkg/errors"
elekEnv "github.com/spdfg/elektron/environment"
"golang.org/x/crypto/ssh"
"strings"
)
func Cap(host, username string, percentage float64) error {

14
scheduler.go
View file

@ -21,6 +21,11 @@ package main // import github.com/spdfg/elektron
import (
"flag"
"fmt"
"os"
"os/signal"
"strings"
"time"
"github.com/golang/protobuf/proto"
mesos "github.com/mesos/mesos-go/api/v0/mesosproto"
sched "github.com/mesos/mesos-go/api/v0/scheduler"
@ -31,10 +36,6 @@ import (
"github.com/spdfg/elektron/pcp"
"github.com/spdfg/elektron/powerCap"
"github.com/spdfg/elektron/schedulers"
"os"
"os/signal"
"strings"
"time"
)
var master = flag.String("master", "", "Location of leading Mesos master -- <mesos-master>:<port>")
@ -225,6 +226,7 @@ func main() {
if strings.Contains(*pcplogPrefix, "/") {
log.Fatal("log file prefix should not contain '/'.")
}
// Build Logger for elektron.
elekLog.BuildLogger(*pcplogPrefix, *logConfigFilename)
// Starting PCP logging.
@ -280,11 +282,11 @@ func main() {
// Starting the scheduler driver.
if status, err := driver.Run(); err != nil {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.ErrorLevel,
log.Fields{"status": status.String(), "error": err.Error()}, "Framework stopped ")
}
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{}, "Exiting...")
}

3
schedulers/MaxGreedyMins.go
View file

@ -19,10 +19,9 @@
package schedulers
import (
log "github.com/sirupsen/logrus"
mesos "github.com/mesos/mesos-go/api/v0/mesosproto"
sched "github.com/mesos/mesos-go/api/v0/scheduler"
log "github.com/sirupsen/logrus"
"github.com/spdfg/elektron/def"
"github.com/spdfg/elektron/utilities/mesosUtils"
"github.com/spdfg/elektron/utilities/offerUtils"

3
schedulers/MaxMin.go
View file

@ -19,10 +19,9 @@
package schedulers
import (
log "github.com/sirupsen/logrus"
mesos "github.com/mesos/mesos-go/api/v0/mesosproto"
sched "github.com/mesos/mesos-go/api/v0/scheduler"
log "github.com/sirupsen/logrus"
"github.com/spdfg/elektron/def"
"github.com/spdfg/elektron/utilities/mesosUtils"
"github.com/spdfg/elektron/utilities/offerUtils"

46
schedulers/base.go
View file

@ -250,10 +250,10 @@ func (s *BaseScheduler) StatusUpdate(driver sched.SchedulerDriver, status *mesos
func (s *BaseScheduler) LogTaskStarting(ts *def.Task, offer *mesos.Offer) {
if ts == nil {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, log.InfoLevel,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel,
log.Fields{"host": fmt.Sprintf("%s", offer.GetHostname())}, "TASKS STARTING...")
} else {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{"task": fmt.Sprintf("%s", ts.Name),
"Instance": fmt.Sprintf("%d", *ts.Instances), "host": fmt.Sprintf("%s", offer.GetHostname())},
@ -262,25 +262,25 @@ func (s *BaseScheduler) LogTaskStarting(ts *def.Task, offer *mesos.Offer) {
}
func (s *BaseScheduler) LogTaskWattsConsideration(ts def.Task, host string, wattsToConsider float64) {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{"task": ts.Name, "host": host, "Watts": fmt.Sprintf("%f", wattsToConsider)}, "Watts considered for ")
}
func (s *BaseScheduler) LogOffersReceived(offers []*mesos.Offer) {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{"Resource offers received": fmt.Sprintf("%d", len(offers))}, "")
}
func (s *BaseScheduler) LogNoPendingTasksDeclineOffers(offer *mesos.Offer) {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.WarnLevel,
log.Fields{"DECLINING OFFER for host": fmt.Sprintf("%s", offer.GetHostname())}, "No tasks left to schedule ")
}
func (s *BaseScheduler) LogNumberOfRunningTasks() {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{"Number of tasks still Running": fmt.Sprintf("%d", s.tasksRunning)}, "")
}
@ -292,81 +292,81 @@ func (s *BaseScheduler) LogCoLocatedTasks(slaveID string) {
buffer.WriteString(fmt.Sprintln(taskName))
}
s.TasksRunningMutex.Unlock()
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{"Colocated with": fmt.Sprintf("%s", buffer.String())}, "")
}
func (s *BaseScheduler) LogSchedTrace(taskToSchedule *mesos.TaskInfo, offer *mesos.Offer) {
elekLog.ElektronLog.Log(elekLogTypes.SCHED_TRACE,
elekLog.ElektronLogger.Log(elekLogTypes.SCHED_TRACE,
log.InfoLevel,
log.Fields{offer.GetHostname(): fmt.Sprintf("%s", taskToSchedule.GetTaskId().GetValue())}, "")
}
func (s *BaseScheduler) LogTerminateScheduler() {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "Done scheduling all tasks!")
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "Done scheduling all tasks!")
}
func (s *BaseScheduler) LogInsufficientResourcesDeclineOffer(offer *mesos.Offer,
offerResources ...interface{}) {
buffer := bytes.Buffer{}
buffer.WriteString(fmt.Sprintf("<CPU: %f, RAM: %f, Watts: %f>", offerResources...))
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.WarnLevel,
log.Fields{"Offer Resources": fmt.Sprintf("%s", buffer.String())}, "DECLINING OFFER... Offer has insufficient resources to launch a task")
}
func (s *BaseScheduler) LogOfferRescinded(offerID *mesos.OfferID) {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.ErrorLevel,
log.Fields{"OfferID": fmt.Sprintf("%s", offerID)}, "OFFER RESCINDED")
}
func (s *BaseScheduler) LogSlaveLost(slaveID *mesos.SlaveID) {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.ErrorLevel,
log.Fields{"SlaveID": fmt.Sprintf("%s", slaveID)}, "SLAVE LOST")
}
func (s *BaseScheduler) LogExecutorLost(executorID *mesos.ExecutorID, slaveID *mesos.SlaveID) {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.ErrorLevel,
log.Fields{"ExecutorID": fmt.Sprintf("%s", executorID), "SlaveID": fmt.Sprintf("%s", slaveID)}, "EXECUTOR LOST")
}
func (s *BaseScheduler) LogFrameworkMessage(executorID *mesos.ExecutorID,
slaveID *mesos.SlaveID, message string) {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{"Received Framework message from executor": executorID}, message)
}
func (s *BaseScheduler) LogMesosError(err string) {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.ErrorLevel,
log.Fields{"MESOS CONSOLE": fmt.Sprintf("%v", err)}, "")
}
func (s *BaseScheduler) LogElectronError(err error) {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.ErrorLevel, log.Fields{"ELECTRON CONSOLE": fmt.Sprintf("%v", err)}, "")
}
func (s *BaseScheduler) LogFrameworkRegistered(frameworkID *mesos.FrameworkID,
masterInfo *mesos.MasterInfo) {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{"frameworkID": fmt.Sprintf("%s", frameworkID), "master": fmt.Sprintf("%v", masterInfo)}, "FRAMEWORK REGISTERED!")
}
func (s *BaseScheduler) LogFrameworkReregistered(masterInfo *mesos.MasterInfo) {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{"master": fmt.Sprintf("%v", masterInfo)}, "Framework re-registered")
}
func (s *BaseScheduler) LogDisconnected() {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, log.WarnLevel, log.Fields{}, "Framework disconnected with master")
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.WarnLevel, log.Fields{}, "Framework disconnected with master")
}
func (s *BaseScheduler) LogTaskStatusUpdate(status *mesos.TaskStatus) {
@ -378,13 +378,13 @@ func (s *BaseScheduler) LogTaskStatusUpdate(status *mesos.TaskStatus) {
default:
level = log.InfoLevel
}
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
level, log.Fields{"task": fmt.Sprintf("%s", *status.TaskId.Value), "state": NameFor(status.State)}, "Task Status received")
}
func (s *BaseScheduler) LogSchedPolicySwitch(name string, nextPolicy SchedPolicyState) {
logSPS := func() {
elekLog.ElektronLog.Log(elekLogTypes.SPS, log.InfoLevel, log.Fields{"Name": name}, "")
elekLog.ElektronLogger.Log(elekLogTypes.SPS, log.InfoLevel, log.Fields{"Name": name}, "")
}
if s.hasReceivedResourceOffers && (s.curSchedPolicy != nextPolicy) {
logSPS()
@ -393,14 +393,14 @@ func (s *BaseScheduler) LogSchedPolicySwitch(name string, nextPolicy SchedPolicy
}
// Logging the size of the scheduling window and the scheduling policy
// that is going to schedule the tasks in the scheduling window.
elekLog.ElektronLog.Log(elekLogTypes.SCHED_WINDOW,
elekLog.ElektronLogger.Log(elekLogTypes.SCHED_WINDOW,
log.InfoLevel,
log.Fields{"Window size": fmt.Sprintf("%d", s.schedWindowSize), "Name": name}, "")
}
func (s *BaseScheduler) LogClsfnAndTaskDistOverhead(overhead time.Duration) {
// Logging the overhead in microseconds.
elekLog.ElektronLog.Log(elekLogTypes.CLSFN_TASKDIST_OVERHEAD,
elekLog.ElektronLogger.Log(elekLogTypes.CLSFN_TASKDIST_OVERHEAD,
log.InfoLevel,
log.Fields{"Overhead in microseconds": fmt.Sprintf("%f", float64(overhead.Nanoseconds())/1000.0)}, "")
}

3
schedulers/bin-packing.go
View file

@ -19,10 +19,9 @@
package schedulers
import (
log "github.com/sirupsen/logrus"
mesos "github.com/mesos/mesos-go/api/v0/mesosproto"
sched "github.com/mesos/mesos-go/api/v0/scheduler"
log "github.com/sirupsen/logrus"
"github.com/spdfg/elektron/def"
"github.com/spdfg/elektron/utilities/mesosUtils"
"github.com/spdfg/elektron/utilities/offerUtils"

5
schedulers/helpers.go
View file

@ -20,6 +20,7 @@ package schedulers
import (
"fmt"
mesos "github.com/mesos/mesos-go/api/v0/mesosproto"
sched "github.com/mesos/mesos-go/api/v0/scheduler"
"github.com/pkg/errors"
@ -35,10 +36,10 @@ import (
func coLocated(tasks map[string]bool, s BaseScheduler) {
for _, task := range tasks {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"Task": task}, "")
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{"Task": task}, "")
}
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "---------------------")
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel, log.Fields{}, "---------------------")
}
// Get the powerClass of the given hostname.

2
schedulers/schedPolicy.go
View file

@ -90,7 +90,7 @@ func switchTaskDistBased(baseSchedRef *BaseScheduler) string {
// Determine the distribution of tasks in the new scheduling window.
taskDist, err := def.GetTaskDistributionInWindow(baseSchedRef.schedWindowSize, baseSchedRef.tasks)
baseSchedRef.LogClsfnAndTaskDistOverhead(time.Now().Sub(startTime))
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE,
log.InfoLevel,
log.Fields{"Task Distribution": fmt.Sprintf("%f", taskDist)}, "Switching... ")
if err != nil {

4
utilities/offerUtils/offerUtils.go
View file

@ -90,13 +90,13 @@ func UpdateEnvironment(offer *mesos.Offer) {
var host = offer.GetHostname()
// If this host is not present in the set of hosts.
if _, ok := constants.Hosts[host]; !ok {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, log.InfoLevel,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel,
log.Fields{"Adding host": host}, "New host detected")
// Add this host.
constants.Hosts[host] = struct{}{}
// Get the power class of this host.
class := PowerClass(offer)
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, log.InfoLevel,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel,
log.Fields{"host": host, "PowerClass": class}, "Registering the power class...")
// If new power class, register the power class.
if _, ok := constants.PowerClasses[class]; !ok {

3
utilities/schedUtils/schedUtils.go
View file

@ -20,6 +20,7 @@ package schedUtils
import (
"fmt"
log "github.com/sirupsen/logrus"
"github.com/spdfg/elektron/def"
elekLog "github.com/spdfg/elektron/elektronLogging"
@ -79,7 +80,7 @@ func (s *fillNextOfferCycle) apply(taskQueue []def.Task) (int, int) {
for _, task := range taskQueue {
numberOfTasksTraversed++
for i := *task.Instances; i > 0; i-- {
elekLog.ElektronLog.Log(elekLogTypes.CONSOLE, log.InfoLevel,
elekLog.ElektronLogger.Log(elekLogTypes.CONSOLE, log.InfoLevel,
log.Fields{}, fmt.Sprintf("Checking if Instance #%d of Task[%s] can be scheduled "+
"during the next offer cycle...", i, task.Name))
if canSchedule(task) {