spdf/elektron
Archived
1
0
Fork 0
Code Issues 4 Pull requests 1 Projects Releases Packages Wiki Activity

Added degcol logger to extrema and prog-extrema.

Browse source 
Degree of collocation logger (or task share variance logger) was
only present in pcp.go. Copied that code into extrema and prog-extrema.
Also, exposed visibility of pcp/utils.go#cpuUtilsPerNode(...) and
pcp/utils.go#memUtilsPerNode(...) to public.
This commit is contained in:
Pradyumna Kaushik 2018-10-05 00:38:24 -04:00
parent d9780a6a9f
commit aca9dcd185
4 changed files with 92 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

4
pcp/pcp.go
View file

@ -55,10 +55,10 @@ func Start(quit chan struct{}, logging *bool, logMType chan elekLogDef.LogMessag
seconds++
memUtils := memUtilPerNode(text)
memUtils := MemUtilPerNode(text)
memTaskShares := make([]float64, len(memUtils))
cpuUtils := cpuUtilPerNode(text)
cpuUtils := CpuUtilPerNode(text)
cpuTaskShares := make([]float64, len(cpuUtils))
for i := 0; i < 8; i++ {

4
pcp/utils.go
View file

@ -56,7 +56,7 @@ func utilization(used string, free string) float64 {
return u / (u + f)
}
func cpuUtilPerNode(text string) []float64 {
func CpuUtilPerNode(text string) []float64 {
tokenSlice := strings.Split(text, ",")
cpuUtil := make([]float64, 8)
for i := 0; i < 8; i++ {
@ -65,7 +65,7 @@ func cpuUtilPerNode(text string) []float64 {
return cpuUtil
}
func memUtilPerNode(text string) []float64 {
func MemUtilPerNode(text string) []float64 {
tokenSlice := strings.Split(text, ",")
memUtil := make([]float64, 8)
for i := 0; i < 8; i++ {

47
powerCap/extrema.go
View file

@ -12,14 +12,18 @@ import (
"syscall"
"time"
"github.com/mesos/mesos-go/api/v0/scheduler"
"github.com/montanaflynn/stats"
elekLogDef "gitlab.com/spdf/elektron/logging/def"
"gitlab.com/spdf/elektron/pcp"
"gitlab.com/spdf/elektron/rapl"
"gitlab.com/spdf/elektron/schedulers"
)
func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThreshold, loThreshold float64,
logMType chan elekLogDef.LogMessageType, logMsg chan string, pcpConfigFile string) {
logMType chan elekLogDef.LogMessageType, logMsg chan string, pcpConfigFile string, s scheduler.Scheduler) {
baseSchedRef := s.(*schedulers.BaseScheduler)
var pcpCommand string = "pmdumptext -m -l -f '' -t 1.0 -d , -c " + pcpConfigFile
cmd := exec.Command("sh", "-c", pcpCommand, pcpConfigFile)
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
@ -46,6 +50,9 @@ func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThresh
headers := strings.Split(scanner.Text(), ",")
logMType <- elekLogDef.DEG_COL
logMsg <- "CPU Variance, CPU Task Share Variance, Memory Variance, Memory Task Share Variance"
powerIndexes := make([]int, 0, 0)
powerHistories := make(map[string]*ring.Ring)
indexToHost := make(map[int]string)
@ -79,9 +86,43 @@ func StartPCPLogAndExtremaDynamicCap(quit chan struct{}, logging *bool, hiThresh
if *logging {
logMType <- elekLogDef.GENERAL
logMsg <- "Logging PCP..."
split := strings.Split(scanner.Text(), ",")
text := scanner.Text()
split := strings.Split(text, ",")
logMType <- elekLogDef.PCP
logMsg <- scanner.Text()
logMsg <- text
memUtils := pcp.MemUtilPerNode(text)
memTaskShares := make([]float64, len(memUtils))
cpuUtils := pcp.CpuUtilPerNode(text)
cpuTaskShares := make([]float64, len(cpuUtils))
for i := 0; i < 8; i++ {
host := fmt.Sprintf("stratos-00%d.cs.binghamton.edu", i+1)
if slaveID, ok := baseSchedRef.HostNameToSlaveID[host]; ok {
baseSchedRef.TasksRunningMutex.Lock()
tasksRunning := len(baseSchedRef.Running[slaveID])
baseSchedRef.TasksRunningMutex.Unlock()
if tasksRunning > 0 {
cpuTaskShares[i] = cpuUtils[i] / float64(tasksRunning)
memTaskShares[i] = memUtils[i] / float64(tasksRunning)
}
}
}
// Variance in resource utilization shows how the current workload has been distributed.
// However, if the number of tasks running are not equally distributed, utilization variance figures become
// less relevant as they do not express the distribution of CPU intensive tasks.
// We thus also calculate `task share variance`, which basically signifies how the workload is distributed
// across each node per share.
cpuVariance, _ := stats.Variance(cpuUtils)
cpuTaskSharesVariance, _ := stats.Variance(cpuTaskShares)
memVariance, _ := stats.Variance(memUtils)
memTaskSharesVariance, _ := stats.Variance(memTaskShares)
logMType <- elekLogDef.DEG_COL
logMsg <- fmt.Sprintf("%f, %f, %f, %f", cpuVariance, cpuTaskSharesVariance, memVariance, memTaskSharesVariance)
totalPower := 0.0
for _, powerIndex := range powerIndexes {

46
powerCap/progressiveExtrema.go
View file

@ -13,10 +13,13 @@ import (
"syscall"
"time"
"github.com/mesos/mesos-go/api/v0/scheduler"
"github.com/montanaflynn/stats"
"gitlab.com/spdf/elektron/constants"
elekLogDef "gitlab.com/spdf/elektron/logging/def"
"gitlab.com/spdf/elektron/pcp"
"gitlab.com/spdf/elektron/rapl"
"gitlab.com/spdf/elektron/schedulers"
"gitlab.com/spdf/elektron/utilities"
)
@ -31,8 +34,9 @@ func getNextCapValue(curCapValue float64, precision int) float64 {
}
func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiThreshold, loThreshold float64,
logMType chan elekLogDef.LogMessageType, logMsg chan string, pcpConfigFile string) {
logMType chan elekLogDef.LogMessageType, logMsg chan string, pcpConfigFile string, s scheduler.Scheduler) {
baseSchedRef := s.(*schedulers.BaseScheduler)
var pcpCommand string = "pmdumptext -m -l -f '' -t 1.0 -d , -c " + pcpConfigFile
cmd := exec.Command("sh", "-c", pcpCommand, pcpConfigFile)
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
@ -60,6 +64,9 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh
headers := strings.Split(scanner.Text(), ",")
logMType <- elekLogDef.DEG_COL
logMsg <- "CPU Variance, CPU Task Share Variance, Memory Variance, Memory Task Share Variance"
powerIndexes := make([]int, 0, 0)
powerHistories := make(map[string]*ring.Ring)
indexToHost := make(map[int]string)
@ -98,8 +105,43 @@ func StartPCPLogAndProgressiveExtremaCap(quit chan struct{}, logging *bool, hiTh
logMType <- elekLogDef.GENERAL
logMsg <- "Logging PCP..."
split := strings.Split(scanner.Text(), ",")
text := scanner.Text()
logMType <- elekLogDef.PCP
logMsg <- scanner.Text()
logMsg <- text
memUtils := pcp.MemUtilPerNode(text)
memTaskShares := make([]float64, len(memUtils))
cpuUtils := pcp.CpuUtilPerNode(text)
cpuTaskShares := make([]float64, len(cpuUtils))
for i := 0; i < 8; i++ {
host := fmt.Sprintf("stratos-00%d.cs.binghamton.edu", i+1)
if slaveID, ok := baseSchedRef.HostNameToSlaveID[host]; ok {
baseSchedRef.TasksRunningMutex.Lock()
tasksRunning := len(baseSchedRef.Running[slaveID])
baseSchedRef.TasksRunningMutex.Unlock()
if tasksRunning > 0 {
cpuTaskShares[i] = cpuUtils[i] / float64(tasksRunning)
memTaskShares[i] = memUtils[i] / float64(tasksRunning)
}
}
}
// Variance in resource utilization shows how the current workload has been distributed.
// However, if the number of tasks running are not equally distributed, utilization variance figures become
// less relevant as they do not express the distribution of CPU intensive tasks.
// We thus also calculate `task share variance`, which basically signifies how the workload is distributed
// across each node per share.
cpuVariance, _ := stats.Variance(cpuUtils)
cpuTaskSharesVariance, _ := stats.Variance(cpuTaskShares)
memVariance, _ := stats.Variance(memUtils)
memTaskSharesVariance, _ := stats.Variance(memTaskShares)
logMType <- elekLogDef.DEG_COL
logMsg <- fmt.Sprintf("%f, %f, %f, %f", cpuVariance, cpuTaskSharesVariance, memVariance, memTaskSharesVariance)
totalPower := 0.0
for _, powerIndex := range powerIndexes {