Merged in experimentation/schedPolicySwitcher (pull request #1)

Experimentation/schedPolicySwitcher
1. Initial commit for consolidated loggers using observer pattern.
2. class factory for schedulers.
3. Using the scheduling policy class factory in schedulers/store.go and the scheduler builder helpers in schedulers/helpers.go, feature to be able to be able to plug a scheduling policy of your choice from the command line (right now only first-fit and bin-packing are possible. Will be updating the class factory to include other scheduling policies as well.
4. Removed TODO for using generic task sorters. Modified TODO for a config file input to run electron.
5. Added other schedulers to the factory
6. Partially retrofitted the other scheduling policies to use the logging library.
7. Retrofitted extrema and progressive to use the consolidated logging library. Fixed parameter issue with s.base.Disconnected(). Formatted project
8. Move statusUpdate(...) into base.go to remove redundant code.
9. Converted the baseScheduler into a state machine where the state is a scheduling policy that defines an approach to consume resource offers.
10. Added another command line argument to be used to enable switching of scheduling policies. Retrofitted scheduling policies to switch only if the particular feature has been enabled.

changed argument to coLocated(...) to take base type rather than ElectronScheduler type. Also, prepended the prefix to the directory of the logs so that it would be easier to determine what the files in a directory correspond to without viewing the contents of the directory.
Defined methods in ElectronScheduler. Each of these methods corresponds to a type of log that an ElectronScheduler would make. Each of these methods would need to be implemented by the scheduling policy.

Electron has only one scheduler that implements the mesos scheduler interface. All the scheduling policies are just different implementations of ways to consume mesos resource offers. Retrofitted scheduling policies to now embed SchedPolicyState instead of baseScheduler.

Approved-by: Pradyumna Kaushik <pkaushi1@binghamton.edu>

logging/def/consoleLogger.go

@@ -0,0 +1,18 @@
+package logging
+
+import (
+	"log"
+)
+
+type ConsoleLogger struct {
+	loggerObserverImpl
+}
+
+func (cl ConsoleLogger) Log(message string) {
+	// We need to log to console only if the message is not empty
+	if message != "" {
+		log.Println(message)
+		// Also logging the message to the console log file
+		cl.logObserverSpecifics[conLogger].logFile.Println(message)
+	}
+}

logging/def/logType.go

@@ -0,0 +1,41 @@
+package logging
+
+import "github.com/fatih/color"
+
+// Defining enums of log message types
+var logMessageNames []string
+
+// Possible log message types
+var (
+	ERROR       = messageNametoMessageType("ERROR")
+	WARNING     = messageNametoMessageType("WARNING")
+	GENERAL     = messageNametoMessageType("GENERAL")
+	SUCCESS     = messageNametoMessageType("SUCCESS")
+	SCHED_TRACE = messageNametoMessageType("SCHED_TRACE")
+	PCP         = messageNametoMessageType("PCP")
+)
+
+// Text colors for the different types of log messages.
+var LogMessageColors map[LogMessageType]*color.Color = map[LogMessageType]*color.Color{
+	ERROR:   color.New(color.FgRed, color.Bold),
+	WARNING: color.New(color.FgYellow, color.Bold),
+	GENERAL: color.New(color.FgWhite, color.Bold),
+	SUCCESS: color.New(color.FgGreen, color.Bold),
+}
+
+type LogMessageType int
+
+func (lmt LogMessageType) String() string {
+	return logMessageNames[lmt]
+}
+
+func GetLogMessageTypes() []string {
+	return logMessageNames
+}
+
+func messageNametoMessageType(messageName string) LogMessageType {
+	// Appending messageName to LogMessageNames
+	logMessageNames = append(logMessageNames, messageName)
+	// Mapping messageName to int
+	return LogMessageType(len(logMessageNames) - 1)
+}

logging/def/logger.go

@@ -0,0 +1,56 @@
+package logging
+
+import (
+	"time"
+)
+
+type LoggerDriver struct {
+	loggerSubject
+	allowedMessageTypes map[LogMessageType]bool
+}
+
+func newLogger() *LoggerDriver {
+	logger := &LoggerDriver{
+		allowedMessageTypes: map[LogMessageType]bool{
+			ERROR:       true,
+			GENERAL:     true,
+			WARNING:     true,
+			SCHED_TRACE: true,
+			SUCCESS:     true,
+			PCP:         true,
+		},
+	}
+	return logger
+}
+
+func BuildLogger(startTime time.Time, prefix string) *LoggerDriver {
+	// building logger
+	l := newLogger()
+	attachAllLoggers(l, startTime, prefix)
+	return l
+}
+
+func (log *LoggerDriver) EnabledLogging(messageType LogMessageType) {
+	log.allowedMessageTypes[messageType] = true
+}
+
+func (log *LoggerDriver) DisableLogging(messageType LogMessageType) {
+	log.allowedMessageTypes[messageType] = false
+}
+
+func (log *LoggerDriver) WriteLog(messageType LogMessageType, message string) {
+	// checking to see if logging for given messageType is disabled
+	if log.allowedMessageTypes[messageType] {
+		log.setMessage(message)
+		// notify registered loggers to log
+		log.notify(messageType)
+	}
+}
+
+func (log *LoggerDriver) Listen(logMType <-chan LogMessageType, logMsg <-chan string) {
+	for {
+		mType := <-logMType
+		msg := <-logMsg
+		log.WriteLog(mType, msg)
+	}
+}

logging/def/loggerFactory.go

@@ -0,0 +1,78 @@
+package logging
+
+import (
+	logUtils "bitbucket.org/sunybingcloud/elektron/logging/utils"
+	"strings"
+	"time"
+)
+
+// Names of different loggers
+const (
+	conLogger        = "console-logger"
+	schedTraceLogger = "schedTrace-logger"
+	pcpLogger        = "pcp-logger"
+)
+
+// Logger class factory
+var Loggers map[string]loggerObserver = map[string]loggerObserver{
+	conLogger:        nil,
+	schedTraceLogger: nil,
+	pcpLogger:        nil,
+}
+
+// Logger options to help initialize loggers
+type loggerOption func(l loggerObserver) error
+
+func withLogDirectory(startTime time.Time, prefix string) loggerOption {
+	return func(l loggerObserver) error {
+		l.(*loggerObserverImpl).setLogDirectory(logUtils.GetLogDir(startTime, prefix))
+		return nil
+	}
+}
+
+// This loggerOption initializes the specifics for each loggerObserver
+func withLoggerSpecifics(prefix string) loggerOption {
+	return func(l loggerObserver) error {
+		l.(*loggerObserverImpl).logObserverSpecifics = map[string]*specifics{
+			conLogger:        &specifics{},
+			schedTraceLogger: &specifics{},
+			pcpLogger:        &specifics{},
+		}
+		l.(*loggerObserverImpl).setLogFilePrefix(prefix)
+		l.(*loggerObserverImpl).setLogFile()
+		return nil
+	}
+}
+
+// Build and assign all loggers
+func attachAllLoggers(lg *LoggerDriver, startTime time.Time, prefix string) {
+	loi := &loggerObserverImpl{}
+	loi.init(withLogDirectory(startTime, strings.Split(prefix, startTime.Format("20060102150405"))[0]),
+		withLoggerSpecifics(prefix))
+	Loggers[conLogger] = &ConsoleLogger{
+		loggerObserverImpl: *loi,
+	}
+	Loggers[schedTraceLogger] = &SchedTraceLogger{
+		loggerObserverImpl: *loi,
+	}
+	Loggers[pcpLogger] = &PCPLogger{
+		loggerObserverImpl: *loi,
+	}
+
+	for _, lmt := range GetLogMessageTypes() {
+		switch lmt {
+		case SCHED_TRACE.String():
+			lg.attach(SCHED_TRACE, Loggers[schedTraceLogger])
+		case GENERAL.String():
+			lg.attach(GENERAL, Loggers[conLogger])
+		case WARNING.String():
+			lg.attach(WARNING, Loggers[conLogger])
+		case ERROR.String():
+			lg.attach(ERROR, Loggers[conLogger])
+		case SUCCESS.String():
+			lg.attach(SUCCESS, Loggers[conLogger])
+		case PCP.String():
+			lg.attach(PCP, Loggers[pcpLogger])
+		}
+	}
+}

logging/def/loggerObservers.go

@@ -0,0 +1,77 @@
+package logging
+
+import (
+	"fmt"
+	"log"
+	"os"
+)
+
+// Logging platform
+type loggerObserver interface {
+	Log(message string)
+	setLogFile()
+	setLogFilePrefix(prefix string)
+	setLogDirectory(dirName string)
+	init(opts ...loggerOption)
+}
+
+type specifics struct {
+	logFilePrefix string
+	logFile       *log.Logger
+}
+
+type loggerObserverImpl struct {
+	logFile              *log.Logger
+	logObserverSpecifics map[string]*specifics
+	logDirectory         string
+}
+
+func (loi *loggerObserverImpl) init(opts ...loggerOption) {
+	for _, opt := range opts {
+		// applying logger options
+		if err := opt(loi); err != nil {
+			log.Fatal(err)
+		}
+	}
+}
+
+func (loi loggerObserverImpl) Log(message string) {}
+
+// Requires logFilePrefix to have already been set
+func (loi *loggerObserverImpl) setLogFile() {
+	for prefix, ls := range loi.logObserverSpecifics {
+		if logFile, err := os.Create(ls.logFilePrefix); err != nil {
+			log.Fatal("Unable to create logFile: ", err)
+		} else {
+			fmt.Printf("Creating logFile with pathname: %s, and prefix: %s\n", ls.logFilePrefix, prefix)
+			ls.logFile = log.New(logFile, "", log.LstdFlags)
+		}
+	}
+}
+
+func (loi *loggerObserverImpl) setLogFilePrefix(prefix string) {
+	// Setting logFilePrefix for pcp logger
+	pcpLogFilePrefix := prefix + ".pcplog"
+	if loi.logDirectory != "" {
+		pcpLogFilePrefix = loi.logDirectory + "/" + pcpLogFilePrefix
+	}
+	loi.logObserverSpecifics[pcpLogger].logFilePrefix = pcpLogFilePrefix
+
+	// Setting logFilePrefix for console logger
+	consoleLogFilePrefix := prefix + "_console.log"
+	if loi.logDirectory != "" {
+		consoleLogFilePrefix = loi.logDirectory + "/" + consoleLogFilePrefix
+	}
+	loi.logObserverSpecifics[conLogger].logFilePrefix = consoleLogFilePrefix
+
+	// Setting logFilePrefix for schedTrace logger
+	schedTraceLogFilePrefix := prefix + "_schedTrace.log"
+	if loi.logDirectory != "" {
+		schedTraceLogFilePrefix = loi.logDirectory + "/" + schedTraceLogFilePrefix
+	}
+	loi.logObserverSpecifics[schedTraceLogger].logFilePrefix = schedTraceLogFilePrefix
+}
+
+func (loi *loggerObserverImpl) setLogDirectory(dirName string) {
+	loi.logDirectory = dirName
+}

logging/def/loggerSubject.go

@@ -0,0 +1,23 @@
+package logging
+
+type loggerSubject struct {
+	Registry map[LogMessageType][]loggerObserver
+	message  string
+}
+
+func (ls *loggerSubject) setMessage(message string) {
+	ls.message = message
+}
+
+func (ls *loggerSubject) attach(messageType LogMessageType, lo loggerObserver) {
+	if ls.Registry == nil {
+		ls.Registry = make(map[LogMessageType][]loggerObserver)
+	}
+	ls.Registry[messageType] = append(ls.Registry[messageType], lo)
+}
+
+func (ls *loggerSubject) notify(messageType LogMessageType) {
+	for _, logObserver := range ls.Registry[messageType] {
+		logObserver.Log(ls.message)
+	}
+}

logging/def/pcpLogger.go

@@ -0,0 +1,9 @@
+package logging
+
+type PCPLogger struct {
+	loggerObserverImpl
+}
+
+func (pl *PCPLogger) Log(message string) {
+	pl.logObserverSpecifics[pcpLogger].logFile.Println(message)
+}

logging/def/schedTraceLogger.go

@@ -0,0 +1,10 @@
+package logging
+
+type SchedTraceLogger struct {
+	loggerObserverImpl
+}
+
+func (stl SchedTraceLogger) Log(message string) {
+	// Logging schedule trace to mentioned file
+	stl.logObserverSpecifics[schedTraceLogger].logFile.Println(message)
+}

logging/utils/createLogDir.go

@@ -0,0 +1,39 @@
+package logging
+
+import (
+	"log"
+	"os"
+	"strconv"
+	"time"
+)
+
+var LogDir string
+
+func GetLogDir(startTime time.Time, prefix string) string {
+	if LogDir == "" {
+		LogDir = createLogDir(prefix, startTime)
+	}
+	return LogDir
+}
+
+func createLogDir(prefix string, startTime time.Time) string {
+	// Creating directory to store all logs for this run
+	logDirName := "./" + prefix + strconv.Itoa(startTime.Year())
+	logDirName += "-"
+	logDirName += startTime.Month().String()
+	logDirName += "-"
+	logDirName += strconv.Itoa(startTime.Day())
+	logDirName += "_"
+	logDirName += strconv.Itoa(startTime.Hour())
+	logDirName += "-"
+	logDirName += strconv.Itoa(startTime.Minute())
+	logDirName += "-"
+	logDirName += strconv.Itoa(startTime.Second())
+	if _, err := os.Stat(logDirName); os.IsNotExist(err) {
+		os.Mkdir(logDirName, 0700)
+	} else {
+		log.Println("Unable to create log directory: ", err)
+		logDirName = ""
+	}
+	return logDirName
+}