From 215026bd195085d2922484f7ab0d8314a7c036df Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 31 Oct 2016 21:38:40 -0400 Subject: [PATCH 01/94] Added proactive dynamic capping (FCFS and Rank based). Also added a primitive readme file. --- .../proactive_dynamic_capping/README.md | 10 + schedulers/proactive_dynamic_capping/main.go | 99 ++++++++ .../src/constants/constants.go | 39 +++ .../src/github.com/montanaflynn/stats | 1 + .../src/proactive_dynamic_capping/capper.go | 235 ++++++++++++++++++ .../src/task/task.go | 73 ++++++ .../src/utilities/utils.go | 9 + 7 files changed, 466 insertions(+) create mode 100644 schedulers/proactive_dynamic_capping/README.md create mode 100644 schedulers/proactive_dynamic_capping/main.go create mode 100644 schedulers/proactive_dynamic_capping/src/constants/constants.go create mode 160000 schedulers/proactive_dynamic_capping/src/github.com/montanaflynn/stats create mode 100644 schedulers/proactive_dynamic_capping/src/proactive_dynamic_capping/capper.go create mode 100644 schedulers/proactive_dynamic_capping/src/task/task.go create mode 100644 schedulers/proactive_dynamic_capping/src/utilities/utils.go diff --git a/schedulers/proactive_dynamic_capping/README.md b/schedulers/proactive_dynamic_capping/README.md new file mode 100644 index 0000000..60f4431 --- /dev/null +++ b/schedulers/proactive_dynamic_capping/README.md @@ -0,0 +1,10 @@ +##Proactive Dynamic Capping + +Perform Cluster wide dynamic capping. + +Offer 2 methods: + 1. First Come First Serve -- For each task that needs to be scheduled, in the order in which it arrives, compute the cluster wide cap. + 2. Rank based cluster wide capping -- Sort a given set of tasks to be scheduled, in ascending order of requested watts, and then compute the cluster wide cap for each of the tasks in the ordered set. + +#Note + The github.com folder contains a library that is required to compute the median of a given set of values. diff --git a/schedulers/proactive_dynamic_capping/main.go b/schedulers/proactive_dynamic_capping/main.go new file mode 100644 index 0000000..d705017 --- /dev/null +++ b/schedulers/proactive_dynamic_capping/main.go @@ -0,0 +1,99 @@ +package main + +import ( + "constants" + "fmt" + "math/rand" + "task" + "proactive_dynamic_capping" + ) + +func sample_available_power() map[string]float64{ + return map[string]float64{ + "stratos-001":100.0, + "stratos-002":150.0, + "stratos-003":80.0, + "stratos-004":90.0, + } +} + +func get_random_power(min, max int) int { + return rand.Intn(max - min) + min +} + +func cap_value_one_task_fcfs(capper *proactive_dynamic_capping.Capper) { + fmt.Println("==== FCFS, Number of tasks: 1 ====") + available_power := sample_available_power() + tsk := task.NewTask("gouravr/minife:v5", "minife:v5", "stratos-001", + "minife_command", 4.0, 10, 50, 1) + if cap_value, err := capper.Fcfs_determine_cap(available_power, tsk); err == nil { + fmt.Println("task = " + tsk.String()) + fmt.Printf("cap value = %f\n", cap_value) + } +} + +func cap_value_window_size_tasks_fcfs(capper *proactive_dynamic_capping.Capper) { + fmt.Println() + fmt.Println("==== FCFS, Number of tasks: 3 (window size) ====") + available_power := sample_available_power() + for i := 0; i < constants.Window_size; i++ { + tsk := task.NewTask("gouravr/minife:v5", "minife:v5", "stratos-001", + "minife_command", 4.0, 10, get_random_power(30, 150), 1) + fmt.Printf("task%d = %s\n", i, tsk.String()) + if cap_value, err := capper.Fcfs_determine_cap(available_power, tsk); err == nil { + fmt.Printf("CAP: %f\n", cap_value) + } + } +} + +func cap_value_more_than_window_size_tasks_fcfs(capper *proactive_dynamic_capping.Capper) { + fmt.Println() + fmt.Println("==== FCFS, Number of tasks: >3 (> window_size) ====") + available_power := sample_available_power() + for i := 0; i < constants.Window_size + 2; i++ { + tsk := task.NewTask("gouravr/minife:v5", "minife:v5", "stratos-001", + "minife_command", 4.0, 10, get_random_power(30, 150), 1) + fmt.Printf("task%d = %s\n", i, tsk.String()) + if cap_value, err := capper.Fcfs_determine_cap(available_power, tsk); err == nil { + fmt.Printf("CAP: %f\n", cap_value) + } + } +} + +func cap_values_for_ranked_tasks(capper *proactive_dynamic_capping.Capper) { + fmt.Println() + fmt.Println("==== Ranked, Number of tasks: 5 (window size + 2) ====") + available_power := sample_available_power() + var tasks_to_schedule []*task.Task + for i := 0; i < constants.Window_size + 2; i++ { + tasks_to_schedule = append(tasks_to_schedule, + task.NewTask("gouravr/minife:v5", "minife:v5", "stratos-001", + "minife_command", 4.0, 10, get_random_power(30, 150), 1)) + } + // Printing the tasks that need to be scheduled. + index := 0 + for _, tsk := range tasks_to_schedule { + fmt.Printf("task%d = %s\n", index, tsk.String()) + index++ + } + if sorted_tasks_to_be_scheduled, cwcv, err := capper.Ranked_determine_cap(available_power, tasks_to_schedule); err == nil { + fmt.Printf("The cap values are: ") + fmt.Println(cwcv) + fmt.Println("The order of tasks to be scheduled :-") + for _, tsk := range sorted_tasks_to_be_scheduled { + fmt.Println(tsk.String()) + } + } +} + +func main() { + capper := proactive_dynamic_capping.GetInstance() + cap_value_one_task_fcfs(capper) + capper.Clear() + cap_value_window_size_tasks_fcfs(capper) + capper.Clear() + cap_value_more_than_window_size_tasks_fcfs(capper) + capper.Clear() + cap_values_for_ranked_tasks(capper) + capper.Clear() +} diff --git a/schedulers/proactive_dynamic_capping/src/constants/constants.go b/schedulers/proactive_dynamic_capping/src/constants/constants.go new file mode 100644 index 0000000..0b1a0cc --- /dev/null +++ b/schedulers/proactive_dynamic_capping/src/constants/constants.go @@ -0,0 +1,39 @@ +/* +Constants that are used across scripts +1. The available hosts = stratos-00x (x varies from 1 to 8) +2. cap_margin = percentage of the requested power to allocate +3. power_threshold = overloading factor +4. total_power = total power per node +5. window_size = number of tasks to consider for computation of the dynamic cap. +*/ +package constants + +var Hosts = []string{"stratos-001", "stratos-002", + "stratos-003", "stratos-004", + "stratos-005", "stratos-006", + "stratos-007", "stratos-008"} + +/* + Margin with respect to the required power for a job. + So, if power required = 10W, the node would be capped to 75%*10W. + This value can be changed upon convenience. +*/ +var Cap_margin = 0.75 + +// Lower bound of the power threshold for a tasks +var Power_threshold = 0.6 + +// Total power per node +var Total_power = map[string]float64 { + "stratos-001": 100.0, + "stratos-002": 150.0, + "stratos-003": 80.0, + "stratos-004": 90.0, + "stratos-005": 200.0, + "stratos-006": 100.0, + "stratos-007": 175.0, + "stratos-008": 175.0, +} + +// Window size for running average +var Window_size = 3 diff --git a/schedulers/proactive_dynamic_capping/src/github.com/montanaflynn/stats b/schedulers/proactive_dynamic_capping/src/github.com/montanaflynn/stats new file mode 160000 index 0000000..60dcacf --- /dev/null +++ b/schedulers/proactive_dynamic_capping/src/github.com/montanaflynn/stats @@ -0,0 +1 @@ +Subproject commit 60dcacf48f43d6dd654d0ed94120ff5806c5ca5c diff --git a/schedulers/proactive_dynamic_capping/src/proactive_dynamic_capping/capper.go b/schedulers/proactive_dynamic_capping/src/proactive_dynamic_capping/capper.go new file mode 100644 index 0000000..4e183f3 --- /dev/null +++ b/schedulers/proactive_dynamic_capping/src/proactive_dynamic_capping/capper.go @@ -0,0 +1,235 @@ +/* +Cluster wide dynamic capping +Step1. Compute running average of tasks in window. +Step2. Compute what percentage of available power of each node, is the running average. +Step3. Compute the median of the percentages and this is the percentage that the cluster needs to be cpaped at. + +1. First Fit Scheduling -- Perform the above steps for each task that needs to be scheduled. +2. Rank based Scheduling -- Sort a set of tasks to be scheduled, in ascending order of power, and then perform the above steps for each of them in the sorted order. +*/ + +package proactive_dynamic_capping + +import ( + "constants" + "container/list" + "errors" + "github.com/montanaflynn/stats" + "task" + "sort" + "sync" +) + +// Structure containing utility data structures used to compute cluster wide dyanmic cap. +type Capper struct { + // window of tasks. + window_of_tasks list.List + // The current sum of requested powers of the tasks in the window. + current_sum float64 + // The current number of tasks in the window. + number_of_tasks_in_window int +} + +// Defining constructor for Capper. +func NewCapper() *Capper { + return &Capper{current_sum: 0.0, number_of_tasks_in_window: 0} +} + +// For locking on operations that may result in race conditions. +var mutex sync.Mutex + +// Singleton instance of Capper +var singleton_capper *Capper +// Retrieve the singleton instance of Capper. +func GetInstance() *Capper { + if singleton_capper == nil { + mutex.Lock() + singleton_capper = NewCapper() + mutex.Unlock() + } else { + // Do nothing + } + return singleton_capper +} + +// Clear and initialize all the members of Capper. +func (capper Capper) Clear() { + capper.window_of_tasks.Init() + capper.current_sum = 0 + capper.number_of_tasks_in_window = 0 +} + +// Compute the average of watts of all the tasks in the window. +func (capper Capper) average() float64 { + return capper.current_sum / float64(capper.window_of_tasks.Len()) +} + +/* + Compute the running average + + Using Capper#window_of_tasks to store the tasks in the window. Task at position 0 (oldest task) removed when window is full and new task arrives. +*/ +func (capper Capper) running_average_of_watts(tsk *task.Task) float64 { + var average float64 + if capper.number_of_tasks_in_window < constants.Window_size { + capper.window_of_tasks.PushBack(tsk) + capper.number_of_tasks_in_window++ + capper.current_sum += float64(tsk.Watts) + } else { + task_to_remove_element := capper.window_of_tasks.Front() + if task_to_remove, ok := task_to_remove_element.Value.(*task.Task); ok { + capper.current_sum -= float64(task_to_remove.Watts) + capper.window_of_tasks.Remove(task_to_remove_element) + } + capper.window_of_tasks.PushBack(tsk) + capper.current_sum += float64(tsk.Watts) + } + average = capper.average() + return average +} + +/* + Calculating cap value + + 1. Sorting the values of running_average_available_power_percentage in ascending order. + 2. Computing the median of the above sorted values. + 3. The median is now the cap value. +*/ +func (capper Capper) get_cap(running_average_available_power_percentage map[string]float64) float64 { + var values []float64 + // Validation + if running_average_available_power_percentage == nil { + return 100.0 + } + for _, apower := range running_average_available_power_percentage { + values = append(values, apower) + } + // sorting the values in ascending order + sort.Float64s(values) + // Calculating the median + if median, err := stats.Median(values); err == nil { + return median + } + // should never reach here. If here, then just setting the cap value to be 100 + return 100.0 +} + +// In place sorting of tasks to be scheduled based on the requested watts. +func qsort_tasks(low int, high int, tasks_to_sort []*task.Task) { + i := low + j := high + // calculating the pivot + pivot_index := low + (high - low)/2 + pivot := tasks_to_sort[pivot_index] + for i <= j { + for tasks_to_sort[i].Watts < pivot.Watts { + i++ + } + for tasks_to_sort[j].Watts > pivot.Watts { + j-- + } + if i <= j { + temp := tasks_to_sort[i] + tasks_to_sort[i] = tasks_to_sort[j] + tasks_to_sort[j] = temp + i++ + j-- + } + } + if low < j { + qsort_tasks(low, j, tasks_to_sort) + } + if i < high { + qsort_tasks(i, high, tasks_to_sort) + } +} + +// Sorting tasks in ascending order of requested watts. +func (capper Capper) sort_tasks(tasks_to_sort []*task.Task) { + qsort_tasks(0, len(tasks_to_sort)-1, tasks_to_sort) +} + +/* +Remove entry for finished task. +Electron needs to call this whenever a task completes so that the finished task no longer contributes to the computation of the cluster wide cap. +*/ +func (capper Capper) Task_finished(finished_task *task.Task) { + // If the window is empty then just return. Should not be entering this condition as it would mean that there is a bug. + if capper.window_of_tasks.Len() == 0 { + return + } + + // Checking whether the finished task is currently present in the window of tasks. + var task_element_to_remove *list.Element + for task_element := capper.window_of_tasks.Front(); task_element != nil; task_element = task_element.Next() { + if tsk, ok := task_element.Value.(*task.Task); ok { + if task.Compare(tsk, finished_task) { + task_element_to_remove = task_element + } + } + } + + // If finished task is there in the window of tasks, then we need to remove the task from the same and modify the members of Capper accordingly. + if task_to_remove, ok := task_element_to_remove.Value.(*task.Task); ok { + capper.window_of_tasks.Remove(task_element_to_remove) + capper.number_of_tasks_in_window -= 1 + capper.current_sum -= float64(task_to_remove.Watts) + } +} + +// Ranked based scheduling +func (capper Capper) Ranked_determine_cap(available_power map[string]float64, tasks_to_schedule []*task.Task) ([]*task.Task, map[int]float64, error) { + // Validation + if available_power == nil || len(tasks_to_schedule) == 0 { + return nil, nil, errors.New("No available power and no tasks to schedule.") + } else { + // Need to sort the tasks in ascending order of requested power + capper.sort_tasks(tasks_to_schedule) + + // Now, for each task in the sorted set of tasks, we need to use the Fcfs_determine_cap logic. + cluster_wide_cap_values := make(map[int]float64) + index := 0 + for _, tsk := range tasks_to_schedule { + /* + Note that even though Fcfs_determine_cap is called, we have sorted the tasks aprior and thus, the tasks are scheduled in the sorted fashion. + Calling Fcfs_determine_cap(...) just to avoid redundant code. + */ + if cap, err := capper.Fcfs_determine_cap(available_power, tsk); err == nil { + cluster_wide_cap_values[index] = cap + } else { + return nil, nil, err + } + index++ + } + // Now returning the sorted set of tasks and the cluster wide cap values for each task that is launched. + return tasks_to_schedule, cluster_wide_cap_values, nil + } +} + +// First come first serve scheduling. +func (capper Capper) Fcfs_determine_cap(available_power map[string]float64, new_task *task.Task) (float64, error) { + // Validation + if available_power == nil { + // If no power available power, then capping the cluster at 100%. Electron might choose to queue the task. + return 100.0, errors.New("No available power.") + } else { + mutex.Lock() + // Need to calcualte the running average + running_average := capper.running_average_of_watts(new_task) + // What percent of available power for each node is the running average + running_average_available_power_percentage := make(map[string]float64) + for node, apower := range available_power { + if apower >= running_average { + running_average_available_power_percentage[node] = (running_average/apower) * 100 + } else { + // We don't consider this node in the offers + } + } + + // Determine the cluster wide cap value. + cap_value := capper.get_cap(running_average_available_power_percentage) + // Electron has to now cap the cluster to this value before launching the next task. + mutex.Unlock() + return cap_value, nil + } +} diff --git a/schedulers/proactive_dynamic_capping/src/task/task.go b/schedulers/proactive_dynamic_capping/src/task/task.go new file mode 100644 index 0000000..47d8aa5 --- /dev/null +++ b/schedulers/proactive_dynamic_capping/src/task/task.go @@ -0,0 +1,73 @@ +package task + +import ( + "constants" + "encoding/json" + "reflect" + "strconv" + "utilities" +) + +/* + Blueprint for the task. + Members: + image: + name: + host: + cmd: + cpu: + ram: + watts: + inst: +*/ +type Task struct { + Image string + Name string + Host string + CMD string + CPU float64 + RAM int + Watts int + Inst int +} + +// Defining a constructor for Task +func NewTask(image string, name string, host string, + cmd string, cpu float64, ram int, watts int, inst int) *Task { + return &Task{Image: image, Name: name, Host: host, CPU: cpu, + RAM: ram, Watts: watts, Inst: inst} +} + +// Update the host on which the task needs to be scheduled. +func (task Task) Update_host(new_host string) { + // Validation + if _, ok := constants.Total_power[new_host]; ok { + task.Host = new_host + } +} + +// Stringify task instance +func (task Task) String() string { + task_map := make(map[string]string) + task_map["image"] = task.Image + task_map["name"] = task.Name + task_map["host"] = task.Host + task_map["cmd"] = task.CMD + task_map["cpu"] = utils.FloatToString(task.CPU) + task_map["ram"] = strconv.Itoa(task.RAM) + task_map["watts"] = strconv.Itoa(task.Watts) + task_map["inst"] = strconv.Itoa(task.Inst) + + json_string, _ := json.Marshal(task_map) + return string(json_string) +} + +// Compare one task to another. 2 tasks are the same if all the corresponding members are the same. +func Compare(task *Task, other_task *Task) bool { + // If comparing the same pointers (checking the addresses). + if task == other_task { + return true + } + // Checking member equality + return reflect.DeepEqual(*task, *other_task) +} diff --git a/schedulers/proactive_dynamic_capping/src/utilities/utils.go b/schedulers/proactive_dynamic_capping/src/utilities/utils.go new file mode 100644 index 0000000..5f2e341 --- /dev/null +++ b/schedulers/proactive_dynamic_capping/src/utilities/utils.go @@ -0,0 +1,9 @@ +package utils + +import "strconv" + +// Convert float64 to string +func FloatToString(input float64) string { + // Precision is 2, Base is 64 + return strconv.FormatFloat(input, 'f', 2, 64) +} From 91d0c6f341c790e0987c0ee09b7c022eda0f69c9 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 31 Oct 2016 21:41:59 -0400 Subject: [PATCH 02/94] Modified readme to include commands to build and run the program and also added documentation to mention what main.go contains --- schedulers/proactive_dynamic_capping/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/schedulers/proactive_dynamic_capping/README.md b/schedulers/proactive_dynamic_capping/README.md index 60f4431..4066b73 100644 --- a/schedulers/proactive_dynamic_capping/README.md +++ b/schedulers/proactive_dynamic_capping/README.md @@ -6,5 +6,14 @@ Offer 2 methods: 1. First Come First Serve -- For each task that needs to be scheduled, in the order in which it arrives, compute the cluster wide cap. 2. Rank based cluster wide capping -- Sort a given set of tasks to be scheduled, in ascending order of requested watts, and then compute the cluster wide cap for each of the tasks in the ordered set. + +main.go contains a set of test functions for the above algorithm. + +#Please run the following commands to install dependencies and run the test code. +''' + go build + go run main.go +''' + #Note The github.com folder contains a library that is required to compute the median of a given set of values. From 4761de1399649ead0f4fe1e68da632e633a196d6 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 31 Oct 2016 21:43:18 -0400 Subject: [PATCH 03/94] fixed an error in the readme file --- schedulers/proactive_dynamic_capping/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schedulers/proactive_dynamic_capping/README.md b/schedulers/proactive_dynamic_capping/README.md index 4066b73..e9f7332 100644 --- a/schedulers/proactive_dynamic_capping/README.md +++ b/schedulers/proactive_dynamic_capping/README.md @@ -10,10 +10,10 @@ Offer 2 methods: main.go contains a set of test functions for the above algorithm. #Please run the following commands to install dependencies and run the test code. -''' +``` go build go run main.go -''' +``` #Note The github.com folder contains a library that is required to compute the median of a given set of values. From 172d49df7ab41ba8a0af45f771802bcd6767591f Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 31 Oct 2016 23:20:17 -0400 Subject: [PATCH 04/94] fixed an error in the readme.md --- schedulers/proactive_dynamic_capping/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/schedulers/proactive_dynamic_capping/README.md b/schedulers/proactive_dynamic_capping/README.md index e9f7332..5effb82 100644 --- a/schedulers/proactive_dynamic_capping/README.md +++ b/schedulers/proactive_dynamic_capping/README.md @@ -3,6 +3,7 @@ Perform Cluster wide dynamic capping. Offer 2 methods: + 1. First Come First Serve -- For each task that needs to be scheduled, in the order in which it arrives, compute the cluster wide cap. 2. Rank based cluster wide capping -- Sort a given set of tasks to be scheduled, in ascending order of requested watts, and then compute the cluster wide cap for each of the tasks in the ordered set. From 0e8db53dee26ab9aedd8733faec162b9f811edb8 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 31 Oct 2016 23:55:25 -0400 Subject: [PATCH 05/94] added description of the .go files. --- .../proactive_dynamic_capping/README.md | 47 +++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/schedulers/proactive_dynamic_capping/README.md b/schedulers/proactive_dynamic_capping/README.md index 5effb82..cf6e3dc 100644 --- a/schedulers/proactive_dynamic_capping/README.md +++ b/schedulers/proactive_dynamic_capping/README.md @@ -7,14 +7,55 @@ Offer 2 methods: 1. First Come First Serve -- For each task that needs to be scheduled, in the order in which it arrives, compute the cluster wide cap. 2. Rank based cluster wide capping -- Sort a given set of tasks to be scheduled, in ascending order of requested watts, and then compute the cluster wide cap for each of the tasks in the ordered set. - main.go contains a set of test functions for the above algorithm. -#Please run the following commands to install dependencies and run the test code. +***.go Files*** +*main.go* +``` +Contains functions that simulate FCFS and Ranked based scheduling. +``` + +*task.go* +``` +Contains the blue print for a task. +A task contains the following information, + 1. Image -- The image tag of the benchmark. + 2. Name -- The name of the benchmark. + 3. Host -- The host on which the task is to be scheduled. + 4. CMD -- Comamnd to execute the benchmark. + 5. CPU -- CPU shares to be allocated to the task. + 6. RAM -- Amount of RAM to be given to the task. + 7. Watts -- Requested amount of power, in watts. + 8. Inst -- Number of instances. +``` + +*constants.go* +``` +Contains constants that are used by all the subroutines. +Defines the following constants, + 1. Hosts -- The possible hosts on which tasks can be scheduled. + 2. Cap margin -- Margin of the requested power to be given to the task. + 3. Power threshold -- Lower bound of power threshold for a task. + 4. Total power -- Total power (including the static power) per node. + 5. Window size -- size of the window of tasks. +``` + +*utils.go* +``` +Contains functions that are used by all other Go routines. +``` + +###Please run the following commands to install dependencies and run the test code. ``` go build go run main.go ``` -#Note +###Note The github.com folder contains a library that is required to compute the median of a given set of values. + +###Things to do + + 1. Need to improve the test cases in main.go. + 2. Need to add more test cases to main.go. + 3. Add better exception handling to capper.go. From 5e96fc61581a08e8057c2c5c79d1c6824d19e0f5 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 31 Oct 2016 23:56:30 -0400 Subject: [PATCH 06/94] fixed an error in the readme --- schedulers/proactive_dynamic_capping/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/schedulers/proactive_dynamic_capping/README.md b/schedulers/proactive_dynamic_capping/README.md index cf6e3dc..a895174 100644 --- a/schedulers/proactive_dynamic_capping/README.md +++ b/schedulers/proactive_dynamic_capping/README.md @@ -10,6 +10,7 @@ Offer 2 methods: main.go contains a set of test functions for the above algorithm. ***.go Files*** + *main.go* ``` Contains functions that simulate FCFS and Ranked based scheduling. From 5bad32e15b0fc4a6e265b33bb3b63cced736e8e5 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 31 Oct 2016 23:58:59 -0400 Subject: [PATCH 07/94] changed the .go files heading in readme --- schedulers/proactive_dynamic_capping/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schedulers/proactive_dynamic_capping/README.md b/schedulers/proactive_dynamic_capping/README.md index a895174..c798555 100644 --- a/schedulers/proactive_dynamic_capping/README.md +++ b/schedulers/proactive_dynamic_capping/README.md @@ -9,7 +9,7 @@ Offer 2 methods: main.go contains a set of test functions for the above algorithm. -***.go Files*** +###**.go Files** *main.go* ``` From aa84b10a58ff31bfac4228102b77378b99c16146 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 10 Nov 2016 19:47:03 -0500 Subject: [PATCH 08/94] Added TaskID as a field. Added a function UpdateHost() to update the host on which the task runs. Added a setter for TaskID. Added a comparator called Compare() that compares to instances of Task --- def/task.go | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/def/task.go b/def/task.go index 5e0f8de..19ed98c 100644 --- a/def/task.go +++ b/def/task.go @@ -1,9 +1,11 @@ package def import ( + "bitbucket.org/sunybingcloud/electron/constants" "encoding/json" "github.com/pkg/errors" "os" + "reflect" ) type Task struct { @@ -15,6 +17,7 @@ type Task struct { CMD string `json:"cmd"` Instances *int `json:"inst"` Host string `json:"host"` + TaskID string `json:"taskID"` } func TasksFromJSON(uri string) ([]Task, error) { @@ -34,6 +37,34 @@ func TasksFromJSON(uri string) ([]Task, error) { return tasks, nil } +// Update the host on which the task needs to be scheduled. +func (tsk *Task) UpdateHost(new_host string) bool { + // Validation + is_correct_host := false + for _, existing_host := range constants.Hosts { + if host == existing_host { + is_correct_host = true + } + } + if !is_correct_host { + return false + } else { + tsk.Host = new_host + return true + } +} + +// Set the taskID of the task. +func (tsk *Task) SetTaskID(taskID string) bool { + // Validation + if taskID == "" { + return false + } else { + tsk.TaskID = taskID + return true + } +} + type WattsSorter []Task func (slice WattsSorter) Len() int { @@ -47,3 +78,22 @@ func (slice WattsSorter) Less(i, j int) bool { func (slice WattsSorter) Swap(i, j int) { slice[i], slice[j] = slice[j], slice[i] } + +// Compare two tasks. +func Compare(task1 *Task, task2 *Task) bool { + // If comparing the same pointers (checking the addresses). + if task1 == task2 { + return true + } + // Checking member equality + if reflect.DeepEqual(*task1, *task2) { + // Need to check for the task ID + if task1.TaskID == task2.TaskID { + return true + } else { + return false + } + } else { + return false + } +} From 383f088b6aa449b8bcb657fe43079cfa0c6fdc6a Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 10 Nov 2016 19:48:41 -0500 Subject: [PATCH 09/94] changed bingcloud to sunybingcloud in the import statements. --- pcp/loganddynamiccap.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcp/loganddynamiccap.go b/pcp/loganddynamiccap.go index cc2776f..4b17fb5 100644 --- a/pcp/loganddynamiccap.go +++ b/pcp/loganddynamiccap.go @@ -1,7 +1,7 @@ package pcp import ( - "bitbucket.org/bingcloud/electron/rapl" + "bitbucket.org/sunybingcloud/electron/rapl" "bufio" "container/ring" "log" From a35e43da51a435d1dd390e427c6c115728292a55 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 10 Nov 2016 19:49:39 -0500 Subject: [PATCH 10/94] scheduler is now an instance of ProactiveClusterwideCapFCFS --- scheduler.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scheduler.go b/scheduler.go index 93bbdf4..168b72e 100644 --- a/scheduler.go +++ b/scheduler.go @@ -1,9 +1,9 @@ package main import ( - "bitbucket.org/bingcloud/electron/def" - "bitbucket.org/bingcloud/electron/pcp" - "bitbucket.org/bingcloud/electron/schedulers" + "bitbucket.org/sunybingcloud/electron/def" + "bitbucket.org/sunybingcloud/electron/pcp" + "bitbucket.org/sunybingcloud/electron/schedulers" "flag" "fmt" "github.com/golang/protobuf/proto" @@ -56,7 +56,7 @@ func main() { fmt.Println(task) } - scheduler := schedulers.NewFirstFit(tasks, *ignoreWatts) + scheduler := schedulers.NewProactiveClusterwideCapFCFS(tasks, *ignoreWatts) driver, err := sched.NewMesosSchedulerDriver(sched.DriverConfig{ Master: *master, Framework: &mesos.FrameworkInfo{ From df553193fd8ea0c9acc3218049746501436746ff Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 10 Nov 2016 19:50:00 -0500 Subject: [PATCH 11/94] changed bingcloud to sunybingcloud in the import statements. --- schedulers/binpacksortedwatts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schedulers/binpacksortedwatts.go b/schedulers/binpacksortedwatts.go index d73b64c..b05a3e3 100644 --- a/schedulers/binpacksortedwatts.go +++ b/schedulers/binpacksortedwatts.go @@ -1,7 +1,7 @@ package schedulers import ( - "bitbucket.org/bingcloud/electron/def" + "bitbucket.org/sunybingcloud/electron/def" "fmt" "github.com/golang/protobuf/proto" mesos "github.com/mesos/mesos-go/mesosproto" From 42a15ec8547dfd68f7ff535c874592f2639275f7 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 10 Nov 2016 19:50:21 -0500 Subject: [PATCH 12/94] changed bingcloud to sunybingcloud in the import statements. --- schedulers/firstfit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schedulers/firstfit.go b/schedulers/firstfit.go index bdfad7e..e426ab1 100644 --- a/schedulers/firstfit.go +++ b/schedulers/firstfit.go @@ -1,7 +1,7 @@ package schedulers import ( - "bitbucket.org/bingcloud/electron/def" + "bitbucket.org/sunybingcloud/electron/def" "fmt" "github.com/golang/protobuf/proto" mesos "github.com/mesos/mesos-go/mesosproto" From 6c3f48e324328782676a3bde97e3bd3ff91dc96e Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 10 Nov 2016 19:50:40 -0500 Subject: [PATCH 13/94] changed bingcloud to sunybingcloud in the import statements. --- schedulers/firstfitsortedwatts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schedulers/firstfitsortedwatts.go b/schedulers/firstfitsortedwatts.go index faab082..9067e1c 100644 --- a/schedulers/firstfitsortedwatts.go +++ b/schedulers/firstfitsortedwatts.go @@ -1,7 +1,7 @@ package schedulers import ( - "bitbucket.org/bingcloud/electron/def" + "bitbucket.org/sunybingcloud/electron/def" "fmt" "github.com/golang/protobuf/proto" mesos "github.com/mesos/mesos-go/mesosproto" From c578f9904d17353be3a4c23a540310d6dad4c6f5 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 10 Nov 2016 19:50:53 -0500 Subject: [PATCH 14/94] changed bingcloud to sunybingcloud in the import statements. --- schedulers/firstfitwattsonly.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schedulers/firstfitwattsonly.go b/schedulers/firstfitwattsonly.go index b19ce3b..e5962a7 100644 --- a/schedulers/firstfitwattsonly.go +++ b/schedulers/firstfitwattsonly.go @@ -1,7 +1,7 @@ package schedulers import ( - "bitbucket.org/bingcloud/electron/def" + "bitbucket.org/sunybingcloud/electron/def" "fmt" "github.com/golang/protobuf/proto" mesos "github.com/mesos/mesos-go/mesosproto" From 10824a8520d558176382db441db7468300b94060 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 10 Nov 2016 19:55:28 -0500 Subject: [PATCH 15/94] Defined constants that help in scheduling of tasks. --- constants/constants.go | 103 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 constants/constants.go diff --git a/constants/constants.go b/constants/constants.go new file mode 100644 index 0000000..50ca468 --- /dev/null +++ b/constants/constants.go @@ -0,0 +1,103 @@ +/* +Constants that are used across scripts +1. The available hosts = stratos-00x (x varies from 1 to 8) +2. cap_margin = percentage of the requested power to allocate +3. power_threshold = overloading factor +4. total_power = total power per node +5. window_size = number of tasks to consider for computation of the dynamic cap. + +Also, exposing functions to update or initialize some of the constants. +*/ +package constants + +var Hosts = []string{"stratos-001", "stratos-002", + "stratos-003", "stratos-004", + "stratos-005", "stratos-006", + "stratos-007", "stratos-008"} + +// Add a new host to the slice of hosts. +func AddNewHost(new_host string) bool { + // Validation + if new_host == "" { + return false + } else { + Hosts = append(Hosts, new_host) + return true + } +} + +// Lower bound of the percentage of requested power, that can be allocated to a task. +var Power_threshold = 0.6 // Right now saying that a task will never be given lesser than 60% of the power it requested. + +/* + Margin with respect to the required power for a job. + So, if power required = 10W, the node would be capped to 75%*10W. + This value can be changed upon convenience. +*/ +var Cap_margin = 0.75 + +// Modify the cap margin. +func UpdateCapMargin(new_cap_margin float64) bool { + // Checking if the new_cap_margin is less than the power threshold. + if new_cap_margin < Starvation_factor { + return false + } else { + Cap_margin = new_cap_margin + return true + } +} + + +// Threshold factor that would make (Cap_margin * task.Watts) equal to (60/100 * task.Watts). +var Starvation_factor = 0.8 + +// Total power per node. +var Total_power map[string]float64 + +// Initialize the total power per node. This should be done before accepting any set of tasks for scheduling. +func AddTotalPowerForHost(host string, total_power float64) bool { + // Validation + is_correct_host := false + for _, existing_host := range Hosts { + if host == existing_host { + is_correct_host = true + } + } + + if !is_correct_host { + return false + } else { + Total_power[host] = total_power + return true + } +} + +// Window size for running average +var Window_size = 10 + +// Update the window size. +func UpdateWindowSize(new_window_size int) bool { + // Validation + if new_window_size == 0 { + return false + } else{ + Window_size = new_window_size + return true + } +} + +// Time duration between successive cluster wide capping. +var Clusterwide_cap_interval = 10.0 // Right now capping the cluster at 10 second intervals. + +// Modify the cluster wide capping interval. We can update the interval depending on the workload. +// TODO: If the workload is heavy then we can set a longer interval, while on the other hand, +// if the workload is light then a smaller interval is sufficient. +func UpdateClusterwideCapInterval(new_interval float64) bool { + // Validation + if new_interval == 0.0 { + return false + } else { + Clusterwide_cap_interval = new_interval + return true + } +} From d8710385becb38e2cda1be613f0c5f6e11bd72b0 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 10 Nov 2016 19:57:36 -0500 Subject: [PATCH 16/94] Proactive cluster wide capper that defines two types of schedulers. First fit scheduling, that uses running average of task.Watts to calculate the cluster wide cap, and Ranked based cluster wide capper that ranks the tasks (sorts) based on the watts required and then performs fcfs in the sorted order. --- schedulers/proactiveclusterwidecappers.go | 244 ++++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 schedulers/proactiveclusterwidecappers.go diff --git a/schedulers/proactiveclusterwidecappers.go b/schedulers/proactiveclusterwidecappers.go new file mode 100644 index 0000000..7b94982 --- /dev/null +++ b/schedulers/proactiveclusterwidecappers.go @@ -0,0 +1,244 @@ +/* +Cluster wide dynamic capping +Step1. Compute running average of tasks in window. +Step2. Compute what percentage of available power of each node, is the running average. +Step3. Compute the median of the percentages and this is the percentage that the cluster needs to be cpaped at. + +1. First Fit Scheduling -- Perform the above steps for each task that needs to be scheduled. +2. Rank based Scheduling -- Sort a set of tasks to be scheduled, in ascending order of power, and then perform the above steps for each of them in the sorted order. + +This is not a scheduler but a scheduling scheme that schedulers can use. +*/ +package schedulers + +import ( + "bitbucket.org/sunybingcloud/electron/constants" + "bitbucket.org/sunybingcloud/electron/def" + "container/list" + "errors" + "github.com/montanaflynn/stats" + "sort" + "sync" +) + +// Structure containing utility data structures used to compute cluster-wide dynamic cap. +type clusterwideCapper struct { + // window of tasks. + window_of_tasks list.list + // The current sum of requested powers of the tasks in the window. + current_sum float64 + // The current number of tasks in the window. + number_of_tasks_in_window int +} + +// Defining constructor for clusterwideCapper. Please don't call this directly and instead use getClusterwideCapperInstance(). +func newClusterwideCapper() *clusterwideCapper { + return &clusterwideCapper{current_sum: 0.0, number_of_tasks_in_window: 0} +} + +// For locking on operations that may result in race conditions. +var clusterwide_capper_mutex sync.Mutex + +// Singleton instance of clusterwideCapper +var singleton_capper *clusterwideCapper +// Retrieve the singleton instance of clusterwideCapper. +func getClusterwideCapperInstance() *clusterwideCapper { + if singleton_capper == nil { + clusterwide_capper_mutex.Lock() + singleton_capper = newClusterwideCapper() + clusterwide_capper_mutex.Unlock() + } else { + // Do nothing + } + return singleton_capper +} + +// Clear and initialize all the members of clusterwideCapper. +func (capper clusterwideCapper) clear() { + capper.window_of_tasks.Init() + capper.current_sum = 0 + capper.number_of_tasks_in_window = 0 +} + +// Compute the average of watts of all the tasks in the window. +func (capper clusterwideCapper) average() float64 { + return capper.current_sum / float64(capper.window_of_tasks.Len()) +} + +/* +Compute the running average. + +Using clusterwideCapper#window_of_tasks to store the tasks. +Task at position 0 (oldest task) is removed when the window is full and new task arrives. +*/ +func (capper clusterwideCapper) running_average_of_watts(tsk *def.Task) float64 { + var average float64 + if capper.number_of_tasks_in_window < constants.Window_size { + capper.window_of_tasks.PushBack(tsk) + capper.number_of_tasks_in_window++ + capper.current_sum += float64(tsk.Watts) + } else { + task_to_remove_element := capper.window_of_tasks.Front() + if task_to_remove, ok := task_to_remove_element.Value.(*def.Task); ok { + capper.current_sum -= float64(task_to_remove.Watts) + capper.window_of_tasks.Remove(task_to_remove_element) + } + capper.window_of_tasks.PushBack(tsk) + capper.current_sum += float64(tsk.Watts) + } + average = capper.average() + return average +} + +/* +Calculating cap value. + +1. Sorting the values of running_average_available_power_percentage in ascending order. +2. Computing the median of the above sorted values. +3. The median is now the cap value. +*/ +func (capper clusterwideCapper) get_cap(running_average_available_power_percentage map[string]float64) float64 { + var values []float64 + // Validation + if running_average_available_power_percentage == nil { + return 100.0 + } + for _, apower := range running_average_available_power_percentage { + values = append(values, apower) + } + // sorting the values in ascending order + sort.Float64s(values) + // Calculating the median + if median, err := stats.Median(values); err == nil { + return median + } + // should never reach here. If here, then just setting the cap value to be 100 + return 100.0 +} + +/* Quick sort algorithm to sort tasks, in place, +in ascending order of power.*/ +func (capper clusterwideCapper) quick_sort(low int, high int, tasks_to_sort []*def.Task) { + i := low + j := high + // calculating the pivot + pivot_index := low + (high - low)/2 + pivot := tasks_to_sort[pivot_index] + for i <= j { + for tasks_to_sort[i].Watts < pivot.Watts { + i++ + } + for tasks_to_sort[j].Watts > pivot.Watts { + j-- + } + if i <= j { + temp := tasks_to_sort[i] + tasks_to_sort[i] = tasks_to_sort[j] + tasks_to_sort[j] = temp + i++ + j-- + } + } + if low < j { + capper.quick_sort(low, j, tasks_to_sort) + } + if i < high { + capper.quick_sort(i, high, tasks_to_sort) + } +} + +// Sorting tasks in ascending order of requested watts. +func (capper clusterwideCapper) sort_tasks(tasks_to_sort []*def.Task) { + capper.quick_sort(0, len(tasks_to_sort)-1, tasks_to_sort) +} + +/* +Remove entry for finished task. +This function is called when a task completes. This completed task needs to be removed from the window of tasks (if it is still present) + so that it doesn't contribute to the computation of the cap value. +*/ +func (capper clusterwideCapper) taskFinished(taskID string) { + // If the window is empty the just return. This condition should technically return false. + if capper.window_of_tasks.Len() == 0 { + return + } + + // Checking whether the task with the given taskID is currently present in the window of tasks. + var task_element_to_remove *list.Element + for task_element := capper.window_of_tasks.Front(); task_element != nil; task_element = task_element.Next() { + if tsk, ok := task_element.Value.(*def.Task); ok { + if task.TaskID == taskID { + task_element_to_remove = task_element + } + } + } + + // If finished task is there in the window of tasks, then we need to remove the task from the same and modify the members of clusterwideCapper accordingly. + if task_to_remove, ok := task_element_to_remove.Value.(*def.Task); ok { + capper.window_of_tasks.Remove(task_element_to_remove) + capper.number_of_tasks_in_window -= 1 + capper.current_sum -= float64(task_to_remove.Watts) + } +} + +// Ranked based scheduling. +func (capper clusterwideCapper) rankedDetermineCap(available_power map[string]float64, + tasks_to_schedule []*def.Task) ([]*def.Task, map[string]float64, error) { + // Validation + if available_power == nil || len(tasks_to_schedule) == 0 { + return nil, nil, errors.New("Invalid argument: available_power, tasks_to_schedule") + } else { + // Need to sort the tasks in ascending order of requested power. + capper.sort_tasks(tasks_to_schedule) + + // Now, for each task in the sorted set of tasks, we need to use the Fcfs_determine_cap logic. + cluster_wide_cap_values := make(map[int]float64) + index := 0 + for _, tsk := range tasks_to_schedule { + /* + Note that even though Fcfs_determine_cap is called, we have sorted the tasks aprior and thus, the tasks are scheduled in the sorted fashion. + Calling Fcfs_determine_cap(...) just to avoid redundant code. + */ + if cap, err := capper.fcfsDetermineCap(available_power, tsk); err == nil { + cluster_wide_cap_values[index] = cap + } else { + return nil, nil, err + } + index++ + } + // Now returning the sorted set of tasks and the cluster wide cap values for each task that is launched. + return tasks_to_schedule, cluster_wide_cap_values, nil + } +} + +// First come first serve shceduling. +func (capper clusterwideCapper) fcfsDetermineCap(available_power map[string]float64, new_task *def.Task) (float64, error) { + // Validation + if available_power == nil { + return 100, errors.New("Invalid argument: available_power") + } else { + clusterwide_capper_mutex.Lock() + // Need to calculate the running average + running_average := capper.running_average_of_watts(new_task) + // What percent of available_power for each node is the running average. + running_average_available_power_percentage := make(map[string]float64) + for host, apower := range available_power { + if apower >= running_average { + running_average_available_power_percentage[host] = (running_average/apower) * 100 + } else { + // We don't consider this host in the offers. + } + } + + // Determine the cluster wide cap value. + cap_value := capper.get_cap(running_average_available_power_percentage) + // Need to cap the cluster to this value before launching the next task. + clusterwide_capper_mutex.Unlock() + return cap_value, nil + } +} + +// Stringer for an instance of clusterwideCapper +func (capper clusterwideCapper) string() string { + return "Clusterwide Capper -- Proactively cap the entire cluster." +} From 8d032f9547f19fa10851225480a9a7930a31bd1c Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 10 Nov 2016 19:59:20 -0500 Subject: [PATCH 17/94] Proactive cluster wide first come first server scheduler. This schedules tasks based on the fcfs cluster wide capping policy defined in proactiveclusterwidecappers.go --- schedulers/proactiveclusterwidecappingfcfs.go | 269 ++++++++++++++++++ 1 file changed, 269 insertions(+) create mode 100644 schedulers/proactiveclusterwidecappingfcfs.go diff --git a/schedulers/proactiveclusterwidecappingfcfs.go b/schedulers/proactiveclusterwidecappingfcfs.go new file mode 100644 index 0000000..ac77dee --- /dev/null +++ b/schedulers/proactiveclusterwidecappingfcfs.go @@ -0,0 +1,269 @@ +package schedulers + +import ( + "bitbucket.org/sunybingcloud/electron/def" + "bitbucket.org/sunybingcloud/electron/constants" + "bitbucket.org/sunybingcloud/electron/rapl" + "errors" + "fmt" + "github.com/golang/protobuf/proto" + mesos "github.com/mesos/mesos-go/mesosproto" + "github.com/mesos/mesos-go/mesosutil" + sched "github.com/mesos/mesos-go/scheduler" + "log" + "sort" + "strings" + "sync" + "time" +) + +// electronScheduler implements the Scheduler interface. +type ProactiveClusterwideCapFCFS struct { + tasksCreated int + tasksRunning int + tasks []def.Task + metrics map[string]def.Metric + running map[string]map[string]bool + ignoreWatts bool + capper *clusterwideCapper + ticker *time.Ticker + isCapping bool + + // First set of PCP values are garbage values, signal to logger to start recording when we're + // about to schedule the new task. + RecordPCP bool + + // This channel is closed when the program receives an interrupt, + // signalling that the program should shut down. + Shutdown chan struct{} + + // This channel is closed after shutdown is closed, and only when all + // outstanding tasks have been cleaned up. + Done chan struct{} + + // Controls when to shutdown pcp logging. + PCPLog chan struct{} +} + +// New electron scheduler. +func NewProactiveClusterwideCapFCFS(tasks []def.Task, ignoreWatts bool) *ProactiveClusterwideCapFCFS { + s := &ProactiveClusterwideCapFCFS { + tasks: tasks, + ignoreWatts: ignoreWatts, + Shutdown: make(chan struct{}), + Done: make(chan struct{}), + PCPLog: make(chan struct{}), + running: make(mapp[string]map[string]bool), + RecordPCP: false, + capper: getClusterwideCapperInstance(), + ticker: time.NewTicker(constants.Clusterwide_cap_interval * time.Second), + isCapping: false + } + return s +} + +func (s *ProactiveClusterwideCapFCFS) newTask(offer *mesos.Offer, task def.Task) *mesos.TaskInfo { + taskName := fmt.Sprintf("%s-%d", task.Name, *task.Instances) + s.tasksCreated++ + + if !s.RecordPCP { + // Turn on logging. + s.RecordPCP = true + time.Sleep(1 * time.Second) // Make sure we're recording by the time the first task starts + } + + // If this is our first time running into this Agent + if _, ok := s.running[offer.GetSlaveId().GoString()]; !ok { + s.running[offer.GetSlaveId().GoString()] = make(map[string]bool) + } + + // Setting the task ID to the task. This is done so that we can consider each task to be different, + // even though they have the same parameters. + task.SetTaskID(proto.String(taskName)) + // Add task to the list of tasks running on the node. + s.running[offer.GetSlaveId().GoString()][taskName] = true + + resources := []*mesos.Resource{ + mesosutil.NewScalarResource("cpus", task.CPU), + mesosutil.NewScalarResource("mem", task.RAM), + } + + if !s.ignoreWatts { + resources = append(resources, mesosutil.NewScalarResource("watts", task.Watts)) + } + + return &mesos.TaskInfo{ + Name: proto.String(taskName), + TaskId: &mesos.TaskID{ + Value: proto.String("electron-" + taskName), + }, + SlaveId: offer.SlaveId, + Resources: resources, + Command: &mesos.CommandInfo{ + Value: proto.String(task.CMD), + }, + Container: &mesos.ContainerInfo{ + Type: mesos.ContainerInfo_DOCKER.Enum(), + Docker: &mesos.ContainerInfo_DockerInfo{ + Image: proto.String(task.Image), + Network: mesos.ContainerInfo_DockerInfo_BRIDGE.Enum(), // Run everything isolated + }, + }, + } +} + +func (s *ProactiveClusterwideCapFCFS) Registered( + _ sched.SchedulerDriver, + framewordID *mesos.FrameworkID, + masterInfo *mesos.MasterInfo) { + log.Printf("Framework %s registered with master %s", frameworkID, masterInfo) +} + +func (s *ProactiveClusterwideCapFCFS) Reregistered(_ sched.SchedulerDriver, masterInfo *mesos.MasterInfo) { + log.Printf("Framework re-registered with master %s", masterInfo) +} + +func (s *ProactiveClusterwideCapFCFS) Disconnected(sched.SchedulerDriver) { + log.Println("Framework disconnected with master") +} + +// go routine to cap the entire cluster in regular intervals of time. +func (s *ProactiveClusterwideCapFCFS) startCapping(currentCapValue float64, mutex sync.Mutex) { + go func() { + for tick := range s.ticker.C { + // Need to cap the cluster to the currentCapValue. + if currentCapValue > 0.0 { + mutex.Lock() + for _, host := range constants.Hosts { + if err := rapl.Cap(host, int(math.Floor(currentCapValue + 0.5))); err != nil { + fmt.Println(err) + } else { + fmt.Println("Successfully capped %s to %d\\%", host, currentCapValue) + } + } + mutex.Unlock() + } + } + } +} + +// TODO: Need to reduce the time complexity: looping over offers twice (Possible to do it just once?). +func (s *ProactiveClusterwideCapFCFS) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { + log.Printf("Received %d resource offers", len(offers)) + + // retrieving the available power for all the hosts in the offers. + available_power := make(map[string]float64) + for _, offer := range offers { + _, _, offer_watts := OfferAgg(offer) + available_power[offer.Hostname] = offer_watts + } + + for _, offer := range offers { + select { + case <-s.Shutdown; + log.Println("Done scheduling tasks: declining offerf on [", offer.GetHostname(), "]") + driver.DeclineOffer(offer.Id, longFilter) + + log.Println("Number og tasks still running: ", s.tasksRunning) + continue + default: + } + + /* + Clusterwide Capping strategy + + For each task in s.tasks, + 1. I need to check whether the mesos offer can be taken or not (based on CPU and RAM). + 2. If the tasks fits the offer then I need to detemrine the cluster wide cap. + 3. First need to cap the cluster to the determine cap value and then launch the task on the host corresponding to the offer. + + Capping the cluster for every task would create a lot of overhead. Hence, clusterwide capping is performed at regular intervals. + TODO: We can choose to cap the cluster only if the clusterwide cap varies more than the current clusterwide cap. + Although this sounds like a better approach, it only works when the resource requirements of neighbouring tasks are similar. + */ + offer_cpu, offer_ram, _ := OfferAgg(offer) + + taken := false + currentCapValue := 0.0 // initial value to indicate that we haven't capped the cluster yet. + var mutex sync.Mutex + + for _, task := range s.tasks { + // Don't take offer if it doesn't match our task's host requirement. + if !strings.HasPrefix(*offer.Hostname, task.Host) { + continue + } + + // Does the task fit. + if (s.ignoreWatts || offer_cpu >= task.CPU ||| offer_ram >= task.RAM) { + taken = true + mutex.Lock() + tempCap, err = s.capper.fcfsDetermineCap(available_power, task) + if err == nil { + currentCapValue = tempCap + } else { + fmt.Println("Failed to determine cluster wide cap: " + err.String()) + } + mutex.Unlock() + fmt.Printf("Starting on [%s]\n", offer.GetHostname()) + driver.LaunchTasks([]*mesos.OfferID{offer.Id}, [s.newTask(offer, task)], defaultFilter) + } else { + // Task doesn't fit the offer. Move onto the next offer. + } + } + + // If no task fit the offer, then declining the offer. + if !taken { + fmt.Println("There is not enough resources to launch a task:") + cpus, mem, watts := OfferAgg(offer) + + log.Printf("\n", cpus, mem, watts) + driver.DeclineOffer(offer.Id, defaultFilter) + } + } +} + +func (s *ProactiveClusterwideCapFCFS) StatusUpdate(driver sched.SchedulerDriver, status *mesos.TaskStatus) { + log.Printf("Received task status [%s] for task [%s]", NameFor(status.State), *status.TaskId.Value) + + if *status.State == mesos.TaskState_TASK_RUNNING { + s.tasksRunning++ + } else if IsTerminal(status.State) { + delete(s.running[status.GetSlaveId().GoString()], *status.TaskId.Value) + // Need to remove the task from the window of tasks. + s.capper.taskFinished(status.TaskId.Value) + s.tasksRunning-- + if s.tasksRunning == 0 { + select { + case <-s.Shutdown: + close(s.Done) + default: + } + } + } + log.Printf("DONE: Task status [%s] for task [%s]", NameFor(status.State), *status.TaskId.Value) +} + +func (s *ProactiveClusterwideCapFCFS) FrameworkMessage(driver sched.SchedulerDriver, + executorID *mesos.ExecutorID, + slaveID *mesos.SlaveID, + message string) { + + log.Println("Getting a framework message: ", message) + log.Printf("Received a framework message from some unknown source: %s", *executorID.Value) +} + +func (s *ProactiveClusterwideCapFCFS) OfferRescinded(_ sched.SchedulerDriver, offerID *mesos.OfferID) { + log.Printf("Offer %s rescinded", offerID) +} + +func (s *ProactiveClusterwideCapFCFS) SlaveLost(_ sched.SchedulerDriver, slaveID *mesos.SlaveID) { + log.Printf("Slave %s lost", slaveID) +} + +func (s *ProactiveClusterwideCapFCFS) ExecutorLost(_ sched.SchedulerDriver, executorID *mesos.ExecutorID, slaveID *mesos.SlaveID, status int) { + log.Printf("Executor %s on slave %s was lost", executorID, slaveID) +} + +func (s *ProactiveClusterwideCapFCFS) Error(_ sched.SchedulerDriver, err string) { + log.Printf("Receiving an error: %s", err) +} From 0d6b714e1d5ee3ecaf2aa88a9f8cf85740d55d08 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 10 Nov 2016 20:01:22 -0500 Subject: [PATCH 18/94] Utility data structures and functions. --- utilities/utils.go | 54 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 utilities/utils.go diff --git a/utilities/utils.go b/utilities/utils.go new file mode 100644 index 0000000..d6406d6 --- /dev/null +++ b/utilities/utils.go @@ -0,0 +1,54 @@ +package utilities + +import "errors" + +/* +The Pair and PairList have been taken from google groups forum, +https://groups.google.com/forum/#!topic/golang-nuts/FT7cjmcL7gw +*/ + +// Utility struct that helps in sorting the available power by value. +type Pair struct { + Key string + Value float64 +} + +// A slice of pairs that implements the sort.Interface to sort by value. +type PairList []Pair + +// Swap pairs in the PairList +func (plist PairList) Swap(i, j int) { + plist[i], plist[j] = plist[j], plist[i] +} + +// function to return the length of the pairlist. +func (plist PairList) Len() int { + return len(plist) +} + +// function to compare two elements in pairlist. +func (plist PairList) Less(i, j int) bool { + return plist[i].Value < plist[j].Value +} + +// convert a PairList to a map[string]float64 +func OrderedKeys(plist PairList) ([]string, error) { + // Validation + if plist == nil { + return nil, errors.New("Invalid argument: plist") + } + ordered_keys := make([]string, len(plist)) + for _, pair := range plist { + ordered_keys = append(ordered_keys, pair.Key) + } + return ordered_keys, nil +} + +// determine the max value +func Max(a, b float64) float64 { + if a > b { + return a + } else { + return b + } +} From 46afab1be0ce0d3d4437def9a03ef84e5b37e443 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 10 Nov 2016 20:05:48 -0500 Subject: [PATCH 19/94] Removed these files and integrated them with electron --- .../proactive_dynamic_capping/README.md | 62 ----- schedulers/proactive_dynamic_capping/main.go | 99 -------- .../src/constants/constants.go | 39 --- .../src/github.com/montanaflynn/stats | 1 - .../src/proactive_dynamic_capping/capper.go | 235 ------------------ .../src/task/task.go | 73 ------ .../src/utilities/utils.go | 9 - 7 files changed, 518 deletions(-) delete mode 100644 schedulers/proactive_dynamic_capping/README.md delete mode 100644 schedulers/proactive_dynamic_capping/main.go delete mode 100644 schedulers/proactive_dynamic_capping/src/constants/constants.go delete mode 160000 schedulers/proactive_dynamic_capping/src/github.com/montanaflynn/stats delete mode 100644 schedulers/proactive_dynamic_capping/src/proactive_dynamic_capping/capper.go delete mode 100644 schedulers/proactive_dynamic_capping/src/task/task.go delete mode 100644 schedulers/proactive_dynamic_capping/src/utilities/utils.go diff --git a/schedulers/proactive_dynamic_capping/README.md b/schedulers/proactive_dynamic_capping/README.md deleted file mode 100644 index c798555..0000000 --- a/schedulers/proactive_dynamic_capping/README.md +++ /dev/null @@ -1,62 +0,0 @@ -##Proactive Dynamic Capping - -Perform Cluster wide dynamic capping. - -Offer 2 methods: - - 1. First Come First Serve -- For each task that needs to be scheduled, in the order in which it arrives, compute the cluster wide cap. - 2. Rank based cluster wide capping -- Sort a given set of tasks to be scheduled, in ascending order of requested watts, and then compute the cluster wide cap for each of the tasks in the ordered set. - -main.go contains a set of test functions for the above algorithm. - -###**.go Files** - -*main.go* -``` -Contains functions that simulate FCFS and Ranked based scheduling. -``` - -*task.go* -``` -Contains the blue print for a task. -A task contains the following information, - 1. Image -- The image tag of the benchmark. - 2. Name -- The name of the benchmark. - 3. Host -- The host on which the task is to be scheduled. - 4. CMD -- Comamnd to execute the benchmark. - 5. CPU -- CPU shares to be allocated to the task. - 6. RAM -- Amount of RAM to be given to the task. - 7. Watts -- Requested amount of power, in watts. - 8. Inst -- Number of instances. -``` - -*constants.go* -``` -Contains constants that are used by all the subroutines. -Defines the following constants, - 1. Hosts -- The possible hosts on which tasks can be scheduled. - 2. Cap margin -- Margin of the requested power to be given to the task. - 3. Power threshold -- Lower bound of power threshold for a task. - 4. Total power -- Total power (including the static power) per node. - 5. Window size -- size of the window of tasks. -``` - -*utils.go* -``` -Contains functions that are used by all other Go routines. -``` - -###Please run the following commands to install dependencies and run the test code. -``` - go build - go run main.go -``` - -###Note - The github.com folder contains a library that is required to compute the median of a given set of values. - -###Things to do - - 1. Need to improve the test cases in main.go. - 2. Need to add more test cases to main.go. - 3. Add better exception handling to capper.go. diff --git a/schedulers/proactive_dynamic_capping/main.go b/schedulers/proactive_dynamic_capping/main.go deleted file mode 100644 index d705017..0000000 --- a/schedulers/proactive_dynamic_capping/main.go +++ /dev/null @@ -1,99 +0,0 @@ -package main - -import ( - "constants" - "fmt" - "math/rand" - "task" - "proactive_dynamic_capping" - ) - -func sample_available_power() map[string]float64{ - return map[string]float64{ - "stratos-001":100.0, - "stratos-002":150.0, - "stratos-003":80.0, - "stratos-004":90.0, - } -} - -func get_random_power(min, max int) int { - return rand.Intn(max - min) + min -} - -func cap_value_one_task_fcfs(capper *proactive_dynamic_capping.Capper) { - fmt.Println("==== FCFS, Number of tasks: 1 ====") - available_power := sample_available_power() - tsk := task.NewTask("gouravr/minife:v5", "minife:v5", "stratos-001", - "minife_command", 4.0, 10, 50, 1) - if cap_value, err := capper.Fcfs_determine_cap(available_power, tsk); err == nil { - fmt.Println("task = " + tsk.String()) - fmt.Printf("cap value = %f\n", cap_value) - } -} - -func cap_value_window_size_tasks_fcfs(capper *proactive_dynamic_capping.Capper) { - fmt.Println() - fmt.Println("==== FCFS, Number of tasks: 3 (window size) ====") - available_power := sample_available_power() - for i := 0; i < constants.Window_size; i++ { - tsk := task.NewTask("gouravr/minife:v5", "minife:v5", "stratos-001", - "minife_command", 4.0, 10, get_random_power(30, 150), 1) - fmt.Printf("task%d = %s\n", i, tsk.String()) - if cap_value, err := capper.Fcfs_determine_cap(available_power, tsk); err == nil { - fmt.Printf("CAP: %f\n", cap_value) - } - } -} - -func cap_value_more_than_window_size_tasks_fcfs(capper *proactive_dynamic_capping.Capper) { - fmt.Println() - fmt.Println("==== FCFS, Number of tasks: >3 (> window_size) ====") - available_power := sample_available_power() - for i := 0; i < constants.Window_size + 2; i++ { - tsk := task.NewTask("gouravr/minife:v5", "minife:v5", "stratos-001", - "minife_command", 4.0, 10, get_random_power(30, 150), 1) - fmt.Printf("task%d = %s\n", i, tsk.String()) - if cap_value, err := capper.Fcfs_determine_cap(available_power, tsk); err == nil { - fmt.Printf("CAP: %f\n", cap_value) - } - } -} - -func cap_values_for_ranked_tasks(capper *proactive_dynamic_capping.Capper) { - fmt.Println() - fmt.Println("==== Ranked, Number of tasks: 5 (window size + 2) ====") - available_power := sample_available_power() - var tasks_to_schedule []*task.Task - for i := 0; i < constants.Window_size + 2; i++ { - tasks_to_schedule = append(tasks_to_schedule, - task.NewTask("gouravr/minife:v5", "minife:v5", "stratos-001", - "minife_command", 4.0, 10, get_random_power(30, 150), 1)) - } - // Printing the tasks that need to be scheduled. - index := 0 - for _, tsk := range tasks_to_schedule { - fmt.Printf("task%d = %s\n", index, tsk.String()) - index++ - } - if sorted_tasks_to_be_scheduled, cwcv, err := capper.Ranked_determine_cap(available_power, tasks_to_schedule); err == nil { - fmt.Printf("The cap values are: ") - fmt.Println(cwcv) - fmt.Println("The order of tasks to be scheduled :-") - for _, tsk := range sorted_tasks_to_be_scheduled { - fmt.Println(tsk.String()) - } - } -} - -func main() { - capper := proactive_dynamic_capping.GetInstance() - cap_value_one_task_fcfs(capper) - capper.Clear() - cap_value_window_size_tasks_fcfs(capper) - capper.Clear() - cap_value_more_than_window_size_tasks_fcfs(capper) - capper.Clear() - cap_values_for_ranked_tasks(capper) - capper.Clear() -} diff --git a/schedulers/proactive_dynamic_capping/src/constants/constants.go b/schedulers/proactive_dynamic_capping/src/constants/constants.go deleted file mode 100644 index 0b1a0cc..0000000 --- a/schedulers/proactive_dynamic_capping/src/constants/constants.go +++ /dev/null @@ -1,39 +0,0 @@ -/* -Constants that are used across scripts -1. The available hosts = stratos-00x (x varies from 1 to 8) -2. cap_margin = percentage of the requested power to allocate -3. power_threshold = overloading factor -4. total_power = total power per node -5. window_size = number of tasks to consider for computation of the dynamic cap. -*/ -package constants - -var Hosts = []string{"stratos-001", "stratos-002", - "stratos-003", "stratos-004", - "stratos-005", "stratos-006", - "stratos-007", "stratos-008"} - -/* - Margin with respect to the required power for a job. - So, if power required = 10W, the node would be capped to 75%*10W. - This value can be changed upon convenience. -*/ -var Cap_margin = 0.75 - -// Lower bound of the power threshold for a tasks -var Power_threshold = 0.6 - -// Total power per node -var Total_power = map[string]float64 { - "stratos-001": 100.0, - "stratos-002": 150.0, - "stratos-003": 80.0, - "stratos-004": 90.0, - "stratos-005": 200.0, - "stratos-006": 100.0, - "stratos-007": 175.0, - "stratos-008": 175.0, -} - -// Window size for running average -var Window_size = 3 diff --git a/schedulers/proactive_dynamic_capping/src/github.com/montanaflynn/stats b/schedulers/proactive_dynamic_capping/src/github.com/montanaflynn/stats deleted file mode 160000 index 60dcacf..0000000 --- a/schedulers/proactive_dynamic_capping/src/github.com/montanaflynn/stats +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 60dcacf48f43d6dd654d0ed94120ff5806c5ca5c diff --git a/schedulers/proactive_dynamic_capping/src/proactive_dynamic_capping/capper.go b/schedulers/proactive_dynamic_capping/src/proactive_dynamic_capping/capper.go deleted file mode 100644 index 4e183f3..0000000 --- a/schedulers/proactive_dynamic_capping/src/proactive_dynamic_capping/capper.go +++ /dev/null @@ -1,235 +0,0 @@ -/* -Cluster wide dynamic capping -Step1. Compute running average of tasks in window. -Step2. Compute what percentage of available power of each node, is the running average. -Step3. Compute the median of the percentages and this is the percentage that the cluster needs to be cpaped at. - -1. First Fit Scheduling -- Perform the above steps for each task that needs to be scheduled. -2. Rank based Scheduling -- Sort a set of tasks to be scheduled, in ascending order of power, and then perform the above steps for each of them in the sorted order. -*/ - -package proactive_dynamic_capping - -import ( - "constants" - "container/list" - "errors" - "github.com/montanaflynn/stats" - "task" - "sort" - "sync" -) - -// Structure containing utility data structures used to compute cluster wide dyanmic cap. -type Capper struct { - // window of tasks. - window_of_tasks list.List - // The current sum of requested powers of the tasks in the window. - current_sum float64 - // The current number of tasks in the window. - number_of_tasks_in_window int -} - -// Defining constructor for Capper. -func NewCapper() *Capper { - return &Capper{current_sum: 0.0, number_of_tasks_in_window: 0} -} - -// For locking on operations that may result in race conditions. -var mutex sync.Mutex - -// Singleton instance of Capper -var singleton_capper *Capper -// Retrieve the singleton instance of Capper. -func GetInstance() *Capper { - if singleton_capper == nil { - mutex.Lock() - singleton_capper = NewCapper() - mutex.Unlock() - } else { - // Do nothing - } - return singleton_capper -} - -// Clear and initialize all the members of Capper. -func (capper Capper) Clear() { - capper.window_of_tasks.Init() - capper.current_sum = 0 - capper.number_of_tasks_in_window = 0 -} - -// Compute the average of watts of all the tasks in the window. -func (capper Capper) average() float64 { - return capper.current_sum / float64(capper.window_of_tasks.Len()) -} - -/* - Compute the running average - - Using Capper#window_of_tasks to store the tasks in the window. Task at position 0 (oldest task) removed when window is full and new task arrives. -*/ -func (capper Capper) running_average_of_watts(tsk *task.Task) float64 { - var average float64 - if capper.number_of_tasks_in_window < constants.Window_size { - capper.window_of_tasks.PushBack(tsk) - capper.number_of_tasks_in_window++ - capper.current_sum += float64(tsk.Watts) - } else { - task_to_remove_element := capper.window_of_tasks.Front() - if task_to_remove, ok := task_to_remove_element.Value.(*task.Task); ok { - capper.current_sum -= float64(task_to_remove.Watts) - capper.window_of_tasks.Remove(task_to_remove_element) - } - capper.window_of_tasks.PushBack(tsk) - capper.current_sum += float64(tsk.Watts) - } - average = capper.average() - return average -} - -/* - Calculating cap value - - 1. Sorting the values of running_average_available_power_percentage in ascending order. - 2. Computing the median of the above sorted values. - 3. The median is now the cap value. -*/ -func (capper Capper) get_cap(running_average_available_power_percentage map[string]float64) float64 { - var values []float64 - // Validation - if running_average_available_power_percentage == nil { - return 100.0 - } - for _, apower := range running_average_available_power_percentage { - values = append(values, apower) - } - // sorting the values in ascending order - sort.Float64s(values) - // Calculating the median - if median, err := stats.Median(values); err == nil { - return median - } - // should never reach here. If here, then just setting the cap value to be 100 - return 100.0 -} - -// In place sorting of tasks to be scheduled based on the requested watts. -func qsort_tasks(low int, high int, tasks_to_sort []*task.Task) { - i := low - j := high - // calculating the pivot - pivot_index := low + (high - low)/2 - pivot := tasks_to_sort[pivot_index] - for i <= j { - for tasks_to_sort[i].Watts < pivot.Watts { - i++ - } - for tasks_to_sort[j].Watts > pivot.Watts { - j-- - } - if i <= j { - temp := tasks_to_sort[i] - tasks_to_sort[i] = tasks_to_sort[j] - tasks_to_sort[j] = temp - i++ - j-- - } - } - if low < j { - qsort_tasks(low, j, tasks_to_sort) - } - if i < high { - qsort_tasks(i, high, tasks_to_sort) - } -} - -// Sorting tasks in ascending order of requested watts. -func (capper Capper) sort_tasks(tasks_to_sort []*task.Task) { - qsort_tasks(0, len(tasks_to_sort)-1, tasks_to_sort) -} - -/* -Remove entry for finished task. -Electron needs to call this whenever a task completes so that the finished task no longer contributes to the computation of the cluster wide cap. -*/ -func (capper Capper) Task_finished(finished_task *task.Task) { - // If the window is empty then just return. Should not be entering this condition as it would mean that there is a bug. - if capper.window_of_tasks.Len() == 0 { - return - } - - // Checking whether the finished task is currently present in the window of tasks. - var task_element_to_remove *list.Element - for task_element := capper.window_of_tasks.Front(); task_element != nil; task_element = task_element.Next() { - if tsk, ok := task_element.Value.(*task.Task); ok { - if task.Compare(tsk, finished_task) { - task_element_to_remove = task_element - } - } - } - - // If finished task is there in the window of tasks, then we need to remove the task from the same and modify the members of Capper accordingly. - if task_to_remove, ok := task_element_to_remove.Value.(*task.Task); ok { - capper.window_of_tasks.Remove(task_element_to_remove) - capper.number_of_tasks_in_window -= 1 - capper.current_sum -= float64(task_to_remove.Watts) - } -} - -// Ranked based scheduling -func (capper Capper) Ranked_determine_cap(available_power map[string]float64, tasks_to_schedule []*task.Task) ([]*task.Task, map[int]float64, error) { - // Validation - if available_power == nil || len(tasks_to_schedule) == 0 { - return nil, nil, errors.New("No available power and no tasks to schedule.") - } else { - // Need to sort the tasks in ascending order of requested power - capper.sort_tasks(tasks_to_schedule) - - // Now, for each task in the sorted set of tasks, we need to use the Fcfs_determine_cap logic. - cluster_wide_cap_values := make(map[int]float64) - index := 0 - for _, tsk := range tasks_to_schedule { - /* - Note that even though Fcfs_determine_cap is called, we have sorted the tasks aprior and thus, the tasks are scheduled in the sorted fashion. - Calling Fcfs_determine_cap(...) just to avoid redundant code. - */ - if cap, err := capper.Fcfs_determine_cap(available_power, tsk); err == nil { - cluster_wide_cap_values[index] = cap - } else { - return nil, nil, err - } - index++ - } - // Now returning the sorted set of tasks and the cluster wide cap values for each task that is launched. - return tasks_to_schedule, cluster_wide_cap_values, nil - } -} - -// First come first serve scheduling. -func (capper Capper) Fcfs_determine_cap(available_power map[string]float64, new_task *task.Task) (float64, error) { - // Validation - if available_power == nil { - // If no power available power, then capping the cluster at 100%. Electron might choose to queue the task. - return 100.0, errors.New("No available power.") - } else { - mutex.Lock() - // Need to calcualte the running average - running_average := capper.running_average_of_watts(new_task) - // What percent of available power for each node is the running average - running_average_available_power_percentage := make(map[string]float64) - for node, apower := range available_power { - if apower >= running_average { - running_average_available_power_percentage[node] = (running_average/apower) * 100 - } else { - // We don't consider this node in the offers - } - } - - // Determine the cluster wide cap value. - cap_value := capper.get_cap(running_average_available_power_percentage) - // Electron has to now cap the cluster to this value before launching the next task. - mutex.Unlock() - return cap_value, nil - } -} diff --git a/schedulers/proactive_dynamic_capping/src/task/task.go b/schedulers/proactive_dynamic_capping/src/task/task.go deleted file mode 100644 index 47d8aa5..0000000 --- a/schedulers/proactive_dynamic_capping/src/task/task.go +++ /dev/null @@ -1,73 +0,0 @@ -package task - -import ( - "constants" - "encoding/json" - "reflect" - "strconv" - "utilities" -) - -/* - Blueprint for the task. - Members: - image: - name: - host: - cmd: - cpu: - ram: - watts: - inst: -*/ -type Task struct { - Image string - Name string - Host string - CMD string - CPU float64 - RAM int - Watts int - Inst int -} - -// Defining a constructor for Task -func NewTask(image string, name string, host string, - cmd string, cpu float64, ram int, watts int, inst int) *Task { - return &Task{Image: image, Name: name, Host: host, CPU: cpu, - RAM: ram, Watts: watts, Inst: inst} -} - -// Update the host on which the task needs to be scheduled. -func (task Task) Update_host(new_host string) { - // Validation - if _, ok := constants.Total_power[new_host]; ok { - task.Host = new_host - } -} - -// Stringify task instance -func (task Task) String() string { - task_map := make(map[string]string) - task_map["image"] = task.Image - task_map["name"] = task.Name - task_map["host"] = task.Host - task_map["cmd"] = task.CMD - task_map["cpu"] = utils.FloatToString(task.CPU) - task_map["ram"] = strconv.Itoa(task.RAM) - task_map["watts"] = strconv.Itoa(task.Watts) - task_map["inst"] = strconv.Itoa(task.Inst) - - json_string, _ := json.Marshal(task_map) - return string(json_string) -} - -// Compare one task to another. 2 tasks are the same if all the corresponding members are the same. -func Compare(task *Task, other_task *Task) bool { - // If comparing the same pointers (checking the addresses). - if task == other_task { - return true - } - // Checking member equality - return reflect.DeepEqual(*task, *other_task) -} diff --git a/schedulers/proactive_dynamic_capping/src/utilities/utils.go b/schedulers/proactive_dynamic_capping/src/utilities/utils.go deleted file mode 100644 index 5f2e341..0000000 --- a/schedulers/proactive_dynamic_capping/src/utilities/utils.go +++ /dev/null @@ -1,9 +0,0 @@ -package utils - -import "strconv" - -// Convert float64 to string -func FloatToString(input float64) string { - // Precision is 2, Base is 64 - return strconv.FormatFloat(input, 'f', 2, 64) -} From 04864ca174f8334b779228169d08552707e57f31 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 10 Nov 2016 20:07:04 -0500 Subject: [PATCH 20/94] No change made. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f7e8a33..ae50170 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,10 @@ To Do: * Create metrics for each task launched [Time to schedule, run time, power used] * Have calibration phase? - * Add ability to use constraints + * Add ability to use constraints * Running average calculations https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average - + **Requires Performance-Copilot tool pmdumptext to be installed on the machine on which electron is launched for logging to work** @@ -43,4 +43,4 @@ Workload schema: "inst": 9 } ] -``` \ No newline at end of file +``` From 2382b66e29a3b6d4c6496a53b1ea69ac99b72be2 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 10 Nov 2016 21:16:06 -0500 Subject: [PATCH 21/94] commented out the constant Clusterwide_cap_interval and its setter function. Instead hardcoding this in proactiveclusterwidecappingfcfs.go --- constants/constants.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/constants/constants.go b/constants/constants.go index 50ca468..133d61f 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -86,18 +86,18 @@ func UpdateWindowSize(new_window_size int) bool { } } -// Time duration between successive cluster wide capping. -var Clusterwide_cap_interval = 10.0 // Right now capping the cluster at 10 second intervals. - -// Modify the cluster wide capping interval. We can update the interval depending on the workload. -// TODO: If the workload is heavy then we can set a longer interval, while on the other hand, -// if the workload is light then a smaller interval is sufficient. -func UpdateClusterwideCapInterval(new_interval float64) bool { - // Validation - if new_interval == 0.0 { - return false - } else { - Clusterwide_cap_interval = new_interval - return true - } -} +// // Time duration between successive cluster wide capping. +// var Clusterwide_cap_interval = 10 // Right now capping the cluster at 10 second intervals. +// +// // Modify the cluster wide capping interval. We can update the interval depending on the workload. +// // TODO: If the workload is heavy then we can set a longer interval, while on the other hand, +// // if the workload is light then a smaller interval is sufficient. +// func UpdateClusterwideCapInterval(new_interval int) bool { +// // Validation +// if new_interval == 0.0 { +// return false +// } else { +// Clusterwide_cap_interval = new_interval +// return true +// } +// } From c44088f48b7e4b7ae9d25e59c82f9877d4b47f25 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 10 Nov 2016 21:16:39 -0500 Subject: [PATCH 22/94] fixed a an error. --- def/task.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/def/task.go b/def/task.go index 19ed98c..c7326c6 100644 --- a/def/task.go +++ b/def/task.go @@ -42,7 +42,7 @@ func (tsk *Task) UpdateHost(new_host string) bool { // Validation is_correct_host := false for _, existing_host := range constants.Hosts { - if host == existing_host { + if new_host == existing_host { is_correct_host = true } } From 8a6ad00e2172592d07c5bc931b505f48bb81cf4d Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 10 Nov 2016 21:16:57 -0500 Subject: [PATCH 23/94] fixed an error --- schedulers/proactiveclusterwidecappers.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/schedulers/proactiveclusterwidecappers.go b/schedulers/proactiveclusterwidecappers.go index 7b94982..65bdf2e 100644 --- a/schedulers/proactiveclusterwidecappers.go +++ b/schedulers/proactiveclusterwidecappers.go @@ -24,7 +24,7 @@ import ( // Structure containing utility data structures used to compute cluster-wide dynamic cap. type clusterwideCapper struct { // window of tasks. - window_of_tasks list.list + window_of_tasks list.List // The current sum of requested powers of the tasks in the window. current_sum float64 // The current number of tasks in the window. @@ -167,7 +167,7 @@ func (capper clusterwideCapper) taskFinished(taskID string) { var task_element_to_remove *list.Element for task_element := capper.window_of_tasks.Front(); task_element != nil; task_element = task_element.Next() { if tsk, ok := task_element.Value.(*def.Task); ok { - if task.TaskID == taskID { + if tsk.TaskID == taskID { task_element_to_remove = task_element } } @@ -183,7 +183,7 @@ func (capper clusterwideCapper) taskFinished(taskID string) { // Ranked based scheduling. func (capper clusterwideCapper) rankedDetermineCap(available_power map[string]float64, - tasks_to_schedule []*def.Task) ([]*def.Task, map[string]float64, error) { + tasks_to_schedule []*def.Task) ([]*def.Task, map[int]float64, error) { // Validation if available_power == nil || len(tasks_to_schedule) == 0 { return nil, nil, errors.New("Invalid argument: available_power, tasks_to_schedule") From a9f7ca5c917b30395781e7f8a7b31e97628ab4fd Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 10 Nov 2016 21:18:05 -0500 Subject: [PATCH 24/94] Made a check to see whether cluster wide capping has started and if not then starting the go routine that performs the cluster wide capping at regular intervals. --- schedulers/proactiveclusterwidecappingfcfs.go | 66 +++++++++++-------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/schedulers/proactiveclusterwidecappingfcfs.go b/schedulers/proactiveclusterwidecappingfcfs.go index ac77dee..ae116f8 100644 --- a/schedulers/proactiveclusterwidecappingfcfs.go +++ b/schedulers/proactiveclusterwidecappingfcfs.go @@ -4,17 +4,16 @@ import ( "bitbucket.org/sunybingcloud/electron/def" "bitbucket.org/sunybingcloud/electron/constants" "bitbucket.org/sunybingcloud/electron/rapl" - "errors" "fmt" "github.com/golang/protobuf/proto" mesos "github.com/mesos/mesos-go/mesosproto" "github.com/mesos/mesos-go/mesosutil" sched "github.com/mesos/mesos-go/scheduler" "log" - "sort" "strings" "sync" "time" + "math" ) // electronScheduler implements the Scheduler interface. @@ -53,11 +52,11 @@ func NewProactiveClusterwideCapFCFS(tasks []def.Task, ignoreWatts bool) *Proacti Shutdown: make(chan struct{}), Done: make(chan struct{}), PCPLog: make(chan struct{}), - running: make(mapp[string]map[string]bool), + running: make(map[string]map[string]bool), RecordPCP: false, capper: getClusterwideCapperInstance(), - ticker: time.NewTicker(constants.Clusterwide_cap_interval * time.Second), - isCapping: false + ticker: time.NewTicker(10 * time.Second), + isCapping: false, } return s } @@ -79,7 +78,7 @@ func (s *ProactiveClusterwideCapFCFS) newTask(offer *mesos.Offer, task def.Task) // Setting the task ID to the task. This is done so that we can consider each task to be different, // even though they have the same parameters. - task.SetTaskID(proto.String(taskName)) + task.SetTaskID(*proto.String(taskName)) // Add task to the list of tasks running on the node. s.running[offer.GetSlaveId().GoString()][taskName] = true @@ -114,7 +113,7 @@ func (s *ProactiveClusterwideCapFCFS) newTask(offer *mesos.Offer, task def.Task) func (s *ProactiveClusterwideCapFCFS) Registered( _ sched.SchedulerDriver, - framewordID *mesos.FrameworkID, + frameworkID *mesos.FrameworkID, masterInfo *mesos.MasterInfo) { log.Printf("Framework %s registered with master %s", frameworkID, masterInfo) } @@ -128,23 +127,27 @@ func (s *ProactiveClusterwideCapFCFS) Disconnected(sched.SchedulerDriver) { } // go routine to cap the entire cluster in regular intervals of time. -func (s *ProactiveClusterwideCapFCFS) startCapping(currentCapValue float64, mutex sync.Mutex) { +var currentCapValue = 0.0 // initial value to indicate that we haven't capped the cluster yet. +func (s *ProactiveClusterwideCapFCFS) startCapping(mutex sync.Mutex) { go func() { - for tick := range s.ticker.C { - // Need to cap the cluster to the currentCapValue. - if currentCapValue > 0.0 { - mutex.Lock() - for _, host := range constants.Hosts { - if err := rapl.Cap(host, int(math.Floor(currentCapValue + 0.5))); err != nil { - fmt.Println(err) - } else { - fmt.Println("Successfully capped %s to %d\\%", host, currentCapValue) + for { + select { + case <- s.ticker.C: + // Need to cap the cluster to the currentCapValue. + if currentCapValue > 0.0 { + mutex.Lock() + for _, host := range constants.Hosts { + if err := rapl.Cap(host, "rapl", int(math.Floor(currentCapValue + 0.5))); err != nil { + fmt.Println(err) + } else { + fmt.Printf("Successfully capped %s to %d\\% at %\n", host, currentCapValue) + } + } + mutex.Unlock() } - } - mutex.Unlock() } } - } + }() } // TODO: Need to reduce the time complexity: looping over offers twice (Possible to do it just once?). @@ -155,12 +158,12 @@ func (s *ProactiveClusterwideCapFCFS) ResourceOffers(driver sched.SchedulerDrive available_power := make(map[string]float64) for _, offer := range offers { _, _, offer_watts := OfferAgg(offer) - available_power[offer.Hostname] = offer_watts + available_power[*offer.Hostname] = offer_watts } for _, offer := range offers { select { - case <-s.Shutdown; + case <-s.Shutdown: log.Println("Done scheduling tasks: declining offerf on [", offer.GetHostname(), "]") driver.DeclineOffer(offer.Id, longFilter) @@ -184,9 +187,14 @@ func (s *ProactiveClusterwideCapFCFS) ResourceOffers(driver sched.SchedulerDrive offer_cpu, offer_ram, _ := OfferAgg(offer) taken := false - currentCapValue := 0.0 // initial value to indicate that we haven't capped the cluster yet. var mutex sync.Mutex + // If haven't started cluster wide capping then doing so, + if !s.isCapping { + s.startCapping(mutex) + s.isCapping = true + } + for _, task := range s.tasks { // Don't take offer if it doesn't match our task's host requirement. if !strings.HasPrefix(*offer.Hostname, task.Host) { @@ -194,18 +202,20 @@ func (s *ProactiveClusterwideCapFCFS) ResourceOffers(driver sched.SchedulerDrive } // Does the task fit. - if (s.ignoreWatts || offer_cpu >= task.CPU ||| offer_ram >= task.RAM) { + if (s.ignoreWatts || offer_cpu >= task.CPU || offer_ram >= task.RAM) { taken = true mutex.Lock() - tempCap, err = s.capper.fcfsDetermineCap(available_power, task) + tempCap, err := s.capper.fcfsDetermineCap(available_power, &task) if err == nil { currentCapValue = tempCap } else { - fmt.Println("Failed to determine cluster wide cap: " + err.String()) + fmt.Printf("Failed to determine cluster wide cap: ") + fmt.Println(err) } mutex.Unlock() fmt.Printf("Starting on [%s]\n", offer.GetHostname()) - driver.LaunchTasks([]*mesos.OfferID{offer.Id}, [s.newTask(offer, task)], defaultFilter) + to_schedule := []*mesos.TaskInfo{s.newTask(offer, task)} + driver.LaunchTasks([]*mesos.OfferID{offer.Id}, to_schedule, defaultFilter) } else { // Task doesn't fit the offer. Move onto the next offer. } @@ -230,7 +240,7 @@ func (s *ProactiveClusterwideCapFCFS) StatusUpdate(driver sched.SchedulerDriver, } else if IsTerminal(status.State) { delete(s.running[status.GetSlaveId().GoString()], *status.TaskId.Value) // Need to remove the task from the window of tasks. - s.capper.taskFinished(status.TaskId.Value) + s.capper.taskFinished(*status.TaskId.Value) s.tasksRunning-- if s.tasksRunning == 0 { select { From 79a5dca76b195b3e9fa8e219dcc8c081d8edfe78 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 14 Nov 2016 22:40:31 -0500 Subject: [PATCH 25/94] Added another log message to log the name of the file to which the pcplogs are getting written to. --- pcp/pcp.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pcp/pcp.go b/pcp/pcp.go index 8a1d46d..3e65a70 100644 --- a/pcp/pcp.go +++ b/pcp/pcp.go @@ -19,6 +19,7 @@ func Start(quit chan struct{}, logging *bool, prefix string) { if err != nil { log.Fatal(err) } + log.Println("Writing pcp logs to file: " + logFile.Name()) defer logFile.Close() From a1c8319b8133aa0163f2c47647f75418a2203706 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 14 Nov 2016 22:42:22 -0500 Subject: [PATCH 26/94] changed extrema to non-extrema. This was done so that proactive cluster wide capping scheme doesn't conflict with the extrema capping scheme. --- scheduler.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scheduler.go b/scheduler.go index 168b72e..356587f 100644 --- a/scheduler.go +++ b/scheduler.go @@ -2,8 +2,8 @@ package main import ( "bitbucket.org/sunybingcloud/electron/def" - "bitbucket.org/sunybingcloud/electron/pcp" "bitbucket.org/sunybingcloud/electron/schedulers" + "bitbucket.org/sunybingcloud/electron/pcp" "flag" "fmt" "github.com/golang/protobuf/proto" @@ -70,8 +70,8 @@ func main() { return } - //go pcp.Start(scheduler.PCPLog, &scheduler.RecordPCP, *pcplogPrefix) - go pcp.StartLogAndDynamicCap(scheduler.PCPLog, &scheduler.RecordPCP, *pcplogPrefix, *hiThreshold, *loThreshold) + go pcp.Start(scheduler.PCPLog, &scheduler.RecordPCP, *pcplogPrefix) + //go pcp.StartLogAndDynamicCap(scheduler.PCPLog, &scheduler.RecordPCP, *pcplogPrefix, *hiThreshold, *loThreshold) time.Sleep(1 * time.Second) // Attempt to handle signint to not leave pmdumptext running From 7a69aff8d7504f8a0b2eb34217973f16a697c6f3 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 14 Nov 2016 22:43:05 -0500 Subject: [PATCH 27/94] The computation of the cluster wide cap now considers total power per node rather than the available power per node. Also, added function recap(...) that is called to compute the cluster wide cap once a task completes. This value is used to change the cluster wide cap once a task completes." --- schedulers/proactiveclusterwidecappers.go | 130 ++++++++++++++-------- 1 file changed, 83 insertions(+), 47 deletions(-) diff --git a/schedulers/proactiveclusterwidecappers.go b/schedulers/proactiveclusterwidecappers.go index 65bdf2e..8d7a55e 100644 --- a/schedulers/proactiveclusterwidecappers.go +++ b/schedulers/proactiveclusterwidecappers.go @@ -1,24 +1,22 @@ /* Cluster wide dynamic capping -Step1. Compute running average of tasks in window. -Step2. Compute what percentage of available power of each node, is the running average. -Step3. Compute the median of the percentages and this is the percentage that the cluster needs to be cpaped at. +Step1. Compute the running average of watts of tasks in window. +Step2. Compute what percentage of total power of each node, is the running average. +Step3. Compute the median of the percetages and this is the percentage that the cluster needs to be capped at. -1. First Fit Scheduling -- Perform the above steps for each task that needs to be scheduled. -2. Rank based Scheduling -- Sort a set of tasks to be scheduled, in ascending order of power, and then perform the above steps for each of them in the sorted order. +1. First fit scheduling -- Perform the above steps for each task that needs to be scheduled. This is not a scheduler but a scheduling scheme that schedulers can use. */ package schedulers import ( - "bitbucket.org/sunybingcloud/electron/constants" - "bitbucket.org/sunybingcloud/electron/def" - "container/list" - "errors" - "github.com/montanaflynn/stats" - "sort" - "sync" + "bitbucket.org/sunybingcloud/electron/constants" + "bitbucket.org/sunybingcloud/electron/def" + "container/list" + "errors" + "github.com/montanaflynn/stats" + "sort" ) // Structure containing utility data structures used to compute cluster-wide dynamic cap. @@ -36,17 +34,12 @@ func newClusterwideCapper() *clusterwideCapper { return &clusterwideCapper{current_sum: 0.0, number_of_tasks_in_window: 0} } -// For locking on operations that may result in race conditions. -var clusterwide_capper_mutex sync.Mutex - // Singleton instance of clusterwideCapper var singleton_capper *clusterwideCapper // Retrieve the singleton instance of clusterwideCapper. func getClusterwideCapperInstance() *clusterwideCapper { if singleton_capper == nil { - clusterwide_capper_mutex.Lock() singleton_capper = newClusterwideCapper() - clusterwide_capper_mutex.Unlock() } else { // Do nothing } @@ -76,15 +69,15 @@ func (capper clusterwideCapper) running_average_of_watts(tsk *def.Task) float64 if capper.number_of_tasks_in_window < constants.Window_size { capper.window_of_tasks.PushBack(tsk) capper.number_of_tasks_in_window++ - capper.current_sum += float64(tsk.Watts) + capper.current_sum += float64(tsk.Watts) * constants.Cap_margin } else { task_to_remove_element := capper.window_of_tasks.Front() if task_to_remove, ok := task_to_remove_element.Value.(*def.Task); ok { - capper.current_sum -= float64(task_to_remove.Watts) + capper.current_sum -= float64(task_to_remove.Watts) * constants.Cap_margin capper.window_of_tasks.Remove(task_to_remove_element) } capper.window_of_tasks.PushBack(tsk) - capper.current_sum += float64(tsk.Watts) + capper.current_sum += float64(tsk.Watts) * constants.Cap_margin } average = capper.average() return average @@ -93,20 +86,20 @@ func (capper clusterwideCapper) running_average_of_watts(tsk *def.Task) float64 /* Calculating cap value. -1. Sorting the values of running_average_available_power_percentage in ascending order. -2. Computing the median of the above sorted values. -3. The median is now the cap value. +1. Sorting the values of running_average_to_total_power_percentage in ascending order. +2. Computing the median of above sorted values. +3. The median is now the cap. */ -func (capper clusterwideCapper) get_cap(running_average_available_power_percentage map[string]float64) float64 { +func (capper clusterwideCapper) get_cap(running_average_to_total_power_percentage map[string]float64) float64 { var values []float64 // Validation - if running_average_available_power_percentage == nil { + if running_average_to_total_power_percentage == nil { return 100.0 } - for _, apower := range running_average_available_power_percentage { + for _, apower := range running_average_to_total_power_percentage { values = append(values, apower) } - // sorting the values in ascending order + // sorting the values in ascending order. sort.Float64s(values) // Calculating the median if median, err := stats.Median(values); err == nil { @@ -116,8 +109,51 @@ func (capper clusterwideCapper) get_cap(running_average_available_power_percenta return 100.0 } -/* Quick sort algorithm to sort tasks, in place, -in ascending order of power.*/ +/* +Recapping the entire cluster. + +1. Remove the task that finished from the list of running tasks. +2. Compute the average allocated power of each of the tasks that are currently running. +3. For each host, determine the ratio of the average to the total power. +4. Determine the median of the ratios and this would be the new cluster wide cap. + +This needs to be called whenever a task finishes execution. +*/ +func (capper clusterwideCapper) recap(total_power map[string]float64, + task_monitor map[string][]def.Task, finished_taskId string) (float64, error) { + // Validation + if total_power == nil || task_monitor == nil { + return 100.0, errors.New("Invalid argument: total_power, task_monitor") + } + total_allocated_power := 0.0 + total_running_tasks := 0 + for _, tasks := range task_monitor { + index := 0 + for i, task := range tasks { + if task.TaskID == finished_taskId { + index = i + continue + } + total_allocated_power += float64(task.Watts) * constants.Cap_margin + total_running_tasks++ + } + tasks = append(tasks[:index], tasks[index+1:]...) + } + average := total_allocated_power / float64(total_running_tasks) + ratios := []float64{} + for _, tpower := range total_power { + ratios = append(ratios, (average/tpower) * 100) + } + sort.Float64s(ratios) + median, err := stats.Median(ratios) + if err == nil { + return median, nil + } else { + return 100, err + } +} + +/* Quick sort algorithm to sort tasks, in place, in ascending order of power.*/ func (capper clusterwideCapper) quick_sort(low int, high int, tasks_to_sort []*def.Task) { i := low j := high @@ -154,7 +190,8 @@ func (capper clusterwideCapper) sort_tasks(tasks_to_sort []*def.Task) { /* Remove entry for finished task. -This function is called when a task completes. This completed task needs to be removed from the window of tasks (if it is still present) +This function is called when a task completes. +This completed task needs to be removed from the window of tasks (if it is still present) so that it doesn't contribute to the computation of the cap value. */ func (capper clusterwideCapper) taskFinished(taskID string) { @@ -173,11 +210,11 @@ func (capper clusterwideCapper) taskFinished(taskID string) { } } - // If finished task is there in the window of tasks, then we need to remove the task from the same and modify the members of clusterwideCapper accordingly. + // Ee need to remove the task from the window. if task_to_remove, ok := task_element_to_remove.Value.(*def.Task); ok { capper.window_of_tasks.Remove(task_element_to_remove) capper.number_of_tasks_in_window -= 1 - capper.current_sum -= float64(task_to_remove.Watts) + capper.current_sum -= float64(task_to_remove.Watts) * constants.Cap_margin } } @@ -211,34 +248,33 @@ func (capper clusterwideCapper) rankedDetermineCap(available_power map[string]fl } } -// First come first serve shceduling. -func (capper clusterwideCapper) fcfsDetermineCap(available_power map[string]float64, new_task *def.Task) (float64, error) { +// First come first serve scheduling. +func (capper clusterwideCapper) fcfsDetermineCap(total_power map[string]float64, + new_task *def.Task) (float64, error) { // Validation - if available_power == nil { - return 100, errors.New("Invalid argument: available_power") + if total_power == nil { + return 100, errors.New("Invalid argument: total_power") } else { - clusterwide_capper_mutex.Lock() // Need to calculate the running average running_average := capper.running_average_of_watts(new_task) - // What percent of available_power for each node is the running average. - running_average_available_power_percentage := make(map[string]float64) - for host, apower := range available_power { - if apower >= running_average { - running_average_available_power_percentage[host] = (running_average/apower) * 100 + // For each node, calculate the percentage of the running average to the total power. + running_average_to_total_power_percentage := make(map[string]float64) + for host, tpower := range total_power { + if tpower >= running_average { + running_average_to_total_power_percentage[host] = (running_average/tpower) * 100 } else { - // We don't consider this host in the offers. + // We don't consider this host for the computation of the cluster wide cap. } } // Determine the cluster wide cap value. - cap_value := capper.get_cap(running_average_available_power_percentage) - // Need to cap the cluster to this value before launching the next task. - clusterwide_capper_mutex.Unlock() + cap_value := capper.get_cap(running_average_to_total_power_percentage) + // Need to cap the cluster to this value. return cap_value, nil } } // Stringer for an instance of clusterwideCapper func (capper clusterwideCapper) string() string { - return "Clusterwide Capper -- Proactively cap the entire cluster." + return "Cluster Capper -- Proactively cap the entire cluster." } From 4cc1dd8e63ab00db7470c210690da011593c47bf Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 14 Nov 2016 22:46:38 -0500 Subject: [PATCH 28/94] Kept track of totalPower per node. The watts resource for the first offer corresponds to the total power per node. Removed tasks, that had all their instances scheduled, from the list of tasks to schedule. Also, calling recap(...) every time a task completes to determine the new cluster wide cap." --- schedulers/proactiveclusterwidecappingfcfs.go | 132 +++++++++++++----- 1 file changed, 100 insertions(+), 32 deletions(-) diff --git a/schedulers/proactiveclusterwidecappingfcfs.go b/schedulers/proactiveclusterwidecappingfcfs.go index ae116f8..679686a 100644 --- a/schedulers/proactiveclusterwidecappingfcfs.go +++ b/schedulers/proactiveclusterwidecappingfcfs.go @@ -10,12 +10,21 @@ import ( "github.com/mesos/mesos-go/mesosutil" sched "github.com/mesos/mesos-go/scheduler" "log" - "strings" - "sync" - "time" "math" + "strings" + "time" ) +// Decides if to take an offer or not +func (_ *ProactiveClusterwideCapFCFS) takeOffer(offer *mesos.Offer, task def.Task) bool { + offer_cpu, offer_mem, _ := OfferAgg(offer) + + if offer_cpu >= task.CPU && offer_mem >= task.RAM { + return true + } + return false +} + // electronScheduler implements the Scheduler interface. type ProactiveClusterwideCapFCFS struct { tasksCreated int @@ -23,10 +32,14 @@ type ProactiveClusterwideCapFCFS struct { tasks []def.Task metrics map[string]def.Metric running map[string]map[string]bool + taskMonitor map[string][]def.Task // store tasks that are currently running. + availablePower map[string]float64 // available power for each node in the cluster. + totalPower map[string]float64 // total power for each node in the cluster. ignoreWatts bool capper *clusterwideCapper ticker *time.Ticker - isCapping bool + isCapping bool // indicate whether we are currently performing cluster wide capping. + //lock *sync.Mutex // First set of PCP values are garbage values, signal to logger to start recording when we're // about to schedule the new task. @@ -53,10 +66,14 @@ func NewProactiveClusterwideCapFCFS(tasks []def.Task, ignoreWatts bool) *Proacti Done: make(chan struct{}), PCPLog: make(chan struct{}), running: make(map[string]map[string]bool), + taskMonitor: make(map[string][]def.Task), + availablePower: make(map[string]float64), + totalPower: make(map[string]float64), RecordPCP: false, capper: getClusterwideCapperInstance(), - ticker: time.NewTicker(10 * time.Second), + ticker: time.NewTicker(5 * time.Second), isCapping: false, + //lock: new(sync.Mutex), } return s } @@ -81,6 +98,7 @@ func (s *ProactiveClusterwideCapFCFS) newTask(offer *mesos.Offer, task def.Task) task.SetTaskID(*proto.String(taskName)) // Add task to the list of tasks running on the node. s.running[offer.GetSlaveId().GoString()][taskName] = true + s.taskMonitor[offer.GetSlaveId().GoString()] = []def.Task{task} resources := []*mesos.Resource{ mesosutil.NewScalarResource("cpus", task.CPU), @@ -123,51 +141,76 @@ func (s *ProactiveClusterwideCapFCFS) Reregistered(_ sched.SchedulerDriver, mast } func (s *ProactiveClusterwideCapFCFS) Disconnected(sched.SchedulerDriver) { + // Need to stop the capping process. + s.ticker.Stop() + s.isCapping = false log.Println("Framework disconnected with master") } // go routine to cap the entire cluster in regular intervals of time. var currentCapValue = 0.0 // initial value to indicate that we haven't capped the cluster yet. -func (s *ProactiveClusterwideCapFCFS) startCapping(mutex sync.Mutex) { +func (s *ProactiveClusterwideCapFCFS) startCapping() { go func() { for { select { case <- s.ticker.C: // Need to cap the cluster to the currentCapValue. if currentCapValue > 0.0 { - mutex.Lock() + //mutex.Lock() + //s.lock.Lock() for _, host := range constants.Hosts { + // Rounding curreCapValue to the nearest int. if err := rapl.Cap(host, "rapl", int(math.Floor(currentCapValue + 0.5))); err != nil { fmt.Println(err) } else { - fmt.Printf("Successfully capped %s to %d\\% at %\n", host, currentCapValue) + fmt.Printf("Successfully capped %s to %f%\n", host, currentCapValue) } } - mutex.Unlock() + //mutex.Unlock() + //s.lock.Unlock() } } } }() } +// Stop cluster wide capping +func (s *ProactiveClusterwideCapFCFS) stopCapping() { + if s.isCapping { + log.Println("Stopping the cluster wide capping.") + s.ticker.Stop() + s.isCapping = false + } +} + // TODO: Need to reduce the time complexity: looping over offers twice (Possible to do it just once?). func (s *ProactiveClusterwideCapFCFS) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { log.Printf("Received %d resource offers", len(offers)) // retrieving the available power for all the hosts in the offers. - available_power := make(map[string]float64) for _, offer := range offers { _, _, offer_watts := OfferAgg(offer) - available_power[*offer.Hostname] = offer_watts + s.availablePower[*offer.Hostname] = offer_watts + // setting total power if the first time. + if _, ok := s.totalPower[*offer.Hostname]; !ok { + s.totalPower[*offer.Hostname] = offer_watts + } + } + + for host, tpower := range s.totalPower { + fmt.Printf("TotalPower[%s] = %f\n", host, tpower) + } + for host, apower := range s.availablePower { + fmt.Printf("AvailablePower[%s] = %f\n", host, apower) } for _, offer := range offers { select { case <-s.Shutdown: - log.Println("Done scheduling tasks: declining offerf on [", offer.GetHostname(), "]") + log.Println("Done scheduling tasks: declining offer on [", offer.GetHostname(), "]") driver.DeclineOffer(offer.Id, longFilter) - log.Println("Number og tasks still running: ", s.tasksRunning) + log.Println("Number of tasks still running: ", s.tasksRunning) continue default: } @@ -176,46 +219,64 @@ func (s *ProactiveClusterwideCapFCFS) ResourceOffers(driver sched.SchedulerDrive Clusterwide Capping strategy For each task in s.tasks, - 1. I need to check whether the mesos offer can be taken or not (based on CPU and RAM). - 2. If the tasks fits the offer then I need to detemrine the cluster wide cap. - 3. First need to cap the cluster to the determine cap value and then launch the task on the host corresponding to the offer. + 1. Need to check whether the offer can be taken or not (based on CPU and RAM requirements). + 2. If the tasks fits the offer, then I need to detemrine the cluster wide cap. + 3. currentCapValue is updated with the determined cluster wide cap. - Capping the cluster for every task would create a lot of overhead. Hence, clusterwide capping is performed at regular intervals. + Cluster wide capping is currently performed at regular intervals of time. TODO: We can choose to cap the cluster only if the clusterwide cap varies more than the current clusterwide cap. Although this sounds like a better approach, it only works when the resource requirements of neighbouring tasks are similar. */ - offer_cpu, offer_ram, _ := OfferAgg(offer) + //offer_cpu, offer_ram, _ := OfferAgg(offer) taken := false - var mutex sync.Mutex + //var mutex sync.Mutex - // If haven't started cluster wide capping then doing so, - if !s.isCapping { - s.startCapping(mutex) - s.isCapping = true - } - - for _, task := range s.tasks { + for i, task := range s.tasks { // Don't take offer if it doesn't match our task's host requirement. if !strings.HasPrefix(*offer.Hostname, task.Host) { continue } // Does the task fit. - if (s.ignoreWatts || offer_cpu >= task.CPU || offer_ram >= task.RAM) { + if s.takeOffer(offer, task) { + // Capping the cluster if haven't yet started, + if !s.isCapping { + s.startCapping() + s.isCapping = true + } taken = true - mutex.Lock() - tempCap, err := s.capper.fcfsDetermineCap(available_power, &task) + //mutex.Lock() + //s.lock.Lock() + //tempCap, err := s.capper.fcfsDetermineCap(s.availablePower, &task) + tempCap, err := s.capper.fcfsDetermineCap(s.totalPower, &task) + if err == nil { currentCapValue = tempCap } else { - fmt.Printf("Failed to determine cluster wide cap: ") + fmt.Printf("Failed to determine new cluster wide cap: ") fmt.Println(err) } - mutex.Unlock() + //mutex.Unlock() + //s.lock.Unlock() fmt.Printf("Starting on [%s]\n", offer.GetHostname()) to_schedule := []*mesos.TaskInfo{s.newTask(offer, task)} driver.LaunchTasks([]*mesos.OfferID{offer.Id}, to_schedule, defaultFilter) + fmt.Printf("Inst: %d", *task.Instances) + *task.Instances-- + if *task.Instances <= 0 { + // All instances of the task have been scheduled. Need to remove it from the list of tasks to schedule. + s.tasks[i] = s.tasks[len(s.tasks)-1] + s.tasks = s.tasks[:len(s.tasks)-1] + + if len(s.tasks) <= 0 { + log.Println("Done scheduling all tasks") + // Need to stop the cluster wide capping as there aren't any more tasks to schedule. + s.stopCapping() + close(s.Shutdown) + } + } + break // Offer taken, move on. } else { // Task doesn't fit the offer. Move onto the next offer. } @@ -223,7 +284,7 @@ func (s *ProactiveClusterwideCapFCFS) ResourceOffers(driver sched.SchedulerDrive // If no task fit the offer, then declining the offer. if !taken { - fmt.Println("There is not enough resources to launch a task:") + fmt.Printf("There is not enough resources to launch a task on Host: %s\n", offer.GetHostname()) cpus, mem, watts := OfferAgg(offer) log.Printf("\n", cpus, mem, watts) @@ -241,10 +302,17 @@ func (s *ProactiveClusterwideCapFCFS) StatusUpdate(driver sched.SchedulerDriver, delete(s.running[status.GetSlaveId().GoString()], *status.TaskId.Value) // Need to remove the task from the window of tasks. s.capper.taskFinished(*status.TaskId.Value) + //currentCapValue, _ = s.capper.recap(s.availablePower, s.taskMonitor, *status.TaskId.Value) + // Determining the new cluster wide cap. + currentCapValue, _ = s.capper.recap(s.totalPower, s.taskMonitor, *status.TaskId.Value) + log.Printf("Recapping the cluster to %f\n", currentCapValue) + s.tasksRunning-- if s.tasksRunning == 0 { select { case <-s.Shutdown: + // Need to stop the capping process. + s.stopCapping() close(s.Done) default: } From 7522cf987944f8c0a534c50fa683c142e0de1921 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 14 Nov 2016 22:53:06 -0500 Subject: [PATCH 29/94] formatted the code --- schedulers/proactiveclusterwidecappers.go | 359 +++++++-------- schedulers/proactiveclusterwidecappingfcfs.go | 434 +++++++++--------- 2 files changed, 397 insertions(+), 396 deletions(-) diff --git a/schedulers/proactiveclusterwidecappers.go b/schedulers/proactiveclusterwidecappers.go index 8d7a55e..aa3eafa 100644 --- a/schedulers/proactiveclusterwidecappers.go +++ b/schedulers/proactiveclusterwidecappers.go @@ -11,51 +11,52 @@ This is not a scheduler but a scheduling scheme that schedulers can use. package schedulers import ( - "bitbucket.org/sunybingcloud/electron/constants" - "bitbucket.org/sunybingcloud/electron/def" - "container/list" - "errors" - "github.com/montanaflynn/stats" - "sort" + "bitbucket.org/sunybingcloud/electron/constants" + "bitbucket.org/sunybingcloud/electron/def" + "container/list" + "errors" + "github.com/montanaflynn/stats" + "sort" ) // Structure containing utility data structures used to compute cluster-wide dynamic cap. type clusterwideCapper struct { - // window of tasks. - window_of_tasks list.List - // The current sum of requested powers of the tasks in the window. - current_sum float64 - // The current number of tasks in the window. - number_of_tasks_in_window int + // window of tasks. + window_of_tasks list.List + // The current sum of requested powers of the tasks in the window. + current_sum float64 + // The current number of tasks in the window. + number_of_tasks_in_window int } // Defining constructor for clusterwideCapper. Please don't call this directly and instead use getClusterwideCapperInstance(). func newClusterwideCapper() *clusterwideCapper { - return &clusterwideCapper{current_sum: 0.0, number_of_tasks_in_window: 0} + return &clusterwideCapper{current_sum: 0.0, number_of_tasks_in_window: 0} } // Singleton instance of clusterwideCapper var singleton_capper *clusterwideCapper + // Retrieve the singleton instance of clusterwideCapper. func getClusterwideCapperInstance() *clusterwideCapper { - if singleton_capper == nil { - singleton_capper = newClusterwideCapper() - } else { - // Do nothing - } - return singleton_capper + if singleton_capper == nil { + singleton_capper = newClusterwideCapper() + } else { + // Do nothing + } + return singleton_capper } // Clear and initialize all the members of clusterwideCapper. func (capper clusterwideCapper) clear() { - capper.window_of_tasks.Init() - capper.current_sum = 0 - capper.number_of_tasks_in_window = 0 + capper.window_of_tasks.Init() + capper.current_sum = 0 + capper.number_of_tasks_in_window = 0 } // Compute the average of watts of all the tasks in the window. func (capper clusterwideCapper) average() float64 { - return capper.current_sum / float64(capper.window_of_tasks.Len()) + return capper.current_sum / float64(capper.window_of_tasks.Len()) } /* @@ -65,22 +66,22 @@ Using clusterwideCapper#window_of_tasks to store the tasks. Task at position 0 (oldest task) is removed when the window is full and new task arrives. */ func (capper clusterwideCapper) running_average_of_watts(tsk *def.Task) float64 { - var average float64 - if capper.number_of_tasks_in_window < constants.Window_size { - capper.window_of_tasks.PushBack(tsk) - capper.number_of_tasks_in_window++ - capper.current_sum += float64(tsk.Watts) * constants.Cap_margin - } else { - task_to_remove_element := capper.window_of_tasks.Front() - if task_to_remove, ok := task_to_remove_element.Value.(*def.Task); ok { - capper.current_sum -= float64(task_to_remove.Watts) * constants.Cap_margin - capper.window_of_tasks.Remove(task_to_remove_element) - } - capper.window_of_tasks.PushBack(tsk) - capper.current_sum += float64(tsk.Watts) * constants.Cap_margin - } - average = capper.average() - return average + var average float64 + if capper.number_of_tasks_in_window < constants.Window_size { + capper.window_of_tasks.PushBack(tsk) + capper.number_of_tasks_in_window++ + capper.current_sum += float64(tsk.Watts) * constants.Cap_margin + } else { + task_to_remove_element := capper.window_of_tasks.Front() + if task_to_remove, ok := task_to_remove_element.Value.(*def.Task); ok { + capper.current_sum -= float64(task_to_remove.Watts) * constants.Cap_margin + capper.window_of_tasks.Remove(task_to_remove_element) + } + capper.window_of_tasks.PushBack(tsk) + capper.current_sum += float64(tsk.Watts) * constants.Cap_margin + } + average = capper.average() + return average } /* @@ -91,22 +92,22 @@ Calculating cap value. 3. The median is now the cap. */ func (capper clusterwideCapper) get_cap(running_average_to_total_power_percentage map[string]float64) float64 { - var values []float64 - // Validation - if running_average_to_total_power_percentage == nil { - return 100.0 - } - for _, apower := range running_average_to_total_power_percentage { - values = append(values, apower) - } - // sorting the values in ascending order. - sort.Float64s(values) - // Calculating the median - if median, err := stats.Median(values); err == nil { - return median - } - // should never reach here. If here, then just setting the cap value to be 100 - return 100.0 + var values []float64 + // Validation + if running_average_to_total_power_percentage == nil { + return 100.0 + } + for _, apower := range running_average_to_total_power_percentage { + values = append(values, apower) + } + // sorting the values in ascending order. + sort.Float64s(values) + // Calculating the median + if median, err := stats.Median(values); err == nil { + return median + } + // should never reach here. If here, then just setting the cap value to be 100 + return 100.0 } /* @@ -120,72 +121,72 @@ Recapping the entire cluster. This needs to be called whenever a task finishes execution. */ func (capper clusterwideCapper) recap(total_power map[string]float64, - task_monitor map[string][]def.Task, finished_taskId string) (float64, error) { - // Validation - if total_power == nil || task_monitor == nil { - return 100.0, errors.New("Invalid argument: total_power, task_monitor") - } - total_allocated_power := 0.0 - total_running_tasks := 0 - for _, tasks := range task_monitor { - index := 0 - for i, task := range tasks { - if task.TaskID == finished_taskId { - index = i - continue - } - total_allocated_power += float64(task.Watts) * constants.Cap_margin - total_running_tasks++ - } - tasks = append(tasks[:index], tasks[index+1:]...) - } - average := total_allocated_power / float64(total_running_tasks) - ratios := []float64{} - for _, tpower := range total_power { - ratios = append(ratios, (average/tpower) * 100) - } - sort.Float64s(ratios) - median, err := stats.Median(ratios) - if err == nil { - return median, nil - } else { - return 100, err - } + task_monitor map[string][]def.Task, finished_taskId string) (float64, error) { + // Validation + if total_power == nil || task_monitor == nil { + return 100.0, errors.New("Invalid argument: total_power, task_monitor") + } + total_allocated_power := 0.0 + total_running_tasks := 0 + for _, tasks := range task_monitor { + index := 0 + for i, task := range tasks { + if task.TaskID == finished_taskId { + index = i + continue + } + total_allocated_power += float64(task.Watts) * constants.Cap_margin + total_running_tasks++ + } + tasks = append(tasks[:index], tasks[index+1:]...) + } + average := total_allocated_power / float64(total_running_tasks) + ratios := []float64{} + for _, tpower := range total_power { + ratios = append(ratios, (average/tpower)*100) + } + sort.Float64s(ratios) + median, err := stats.Median(ratios) + if err == nil { + return median, nil + } else { + return 100, err + } } /* Quick sort algorithm to sort tasks, in place, in ascending order of power.*/ func (capper clusterwideCapper) quick_sort(low int, high int, tasks_to_sort []*def.Task) { - i := low - j := high - // calculating the pivot - pivot_index := low + (high - low)/2 - pivot := tasks_to_sort[pivot_index] - for i <= j { - for tasks_to_sort[i].Watts < pivot.Watts { - i++ - } - for tasks_to_sort[j].Watts > pivot.Watts { - j-- - } - if i <= j { - temp := tasks_to_sort[i] - tasks_to_sort[i] = tasks_to_sort[j] - tasks_to_sort[j] = temp - i++ - j-- - } - } - if low < j { - capper.quick_sort(low, j, tasks_to_sort) - } - if i < high { - capper.quick_sort(i, high, tasks_to_sort) - } + i := low + j := high + // calculating the pivot + pivot_index := low + (high-low)/2 + pivot := tasks_to_sort[pivot_index] + for i <= j { + for tasks_to_sort[i].Watts < pivot.Watts { + i++ + } + for tasks_to_sort[j].Watts > pivot.Watts { + j-- + } + if i <= j { + temp := tasks_to_sort[i] + tasks_to_sort[i] = tasks_to_sort[j] + tasks_to_sort[j] = temp + i++ + j-- + } + } + if low < j { + capper.quick_sort(low, j, tasks_to_sort) + } + if i < high { + capper.quick_sort(i, high, tasks_to_sort) + } } // Sorting tasks in ascending order of requested watts. func (capper clusterwideCapper) sort_tasks(tasks_to_sort []*def.Task) { - capper.quick_sort(0, len(tasks_to_sort)-1, tasks_to_sort) + capper.quick_sort(0, len(tasks_to_sort)-1, tasks_to_sort) } /* @@ -195,86 +196,86 @@ This completed task needs to be removed from the window of tasks (if it is still so that it doesn't contribute to the computation of the cap value. */ func (capper clusterwideCapper) taskFinished(taskID string) { - // If the window is empty the just return. This condition should technically return false. - if capper.window_of_tasks.Len() == 0 { - return - } + // If the window is empty the just return. This condition should technically return false. + if capper.window_of_tasks.Len() == 0 { + return + } - // Checking whether the task with the given taskID is currently present in the window of tasks. - var task_element_to_remove *list.Element - for task_element := capper.window_of_tasks.Front(); task_element != nil; task_element = task_element.Next() { - if tsk, ok := task_element.Value.(*def.Task); ok { - if tsk.TaskID == taskID { - task_element_to_remove = task_element - } - } - } + // Checking whether the task with the given taskID is currently present in the window of tasks. + var task_element_to_remove *list.Element + for task_element := capper.window_of_tasks.Front(); task_element != nil; task_element = task_element.Next() { + if tsk, ok := task_element.Value.(*def.Task); ok { + if tsk.TaskID == taskID { + task_element_to_remove = task_element + } + } + } - // Ee need to remove the task from the window. - if task_to_remove, ok := task_element_to_remove.Value.(*def.Task); ok { - capper.window_of_tasks.Remove(task_element_to_remove) - capper.number_of_tasks_in_window -= 1 - capper.current_sum -= float64(task_to_remove.Watts) * constants.Cap_margin - } + // Ee need to remove the task from the window. + if task_to_remove, ok := task_element_to_remove.Value.(*def.Task); ok { + capper.window_of_tasks.Remove(task_element_to_remove) + capper.number_of_tasks_in_window -= 1 + capper.current_sum -= float64(task_to_remove.Watts) * constants.Cap_margin + } } // Ranked based scheduling. func (capper clusterwideCapper) rankedDetermineCap(available_power map[string]float64, - tasks_to_schedule []*def.Task) ([]*def.Task, map[int]float64, error) { - // Validation - if available_power == nil || len(tasks_to_schedule) == 0 { - return nil, nil, errors.New("Invalid argument: available_power, tasks_to_schedule") - } else { - // Need to sort the tasks in ascending order of requested power. - capper.sort_tasks(tasks_to_schedule) + tasks_to_schedule []*def.Task) ([]*def.Task, map[int]float64, error) { + // Validation + if available_power == nil || len(tasks_to_schedule) == 0 { + return nil, nil, errors.New("Invalid argument: available_power, tasks_to_schedule") + } else { + // Need to sort the tasks in ascending order of requested power. + capper.sort_tasks(tasks_to_schedule) - // Now, for each task in the sorted set of tasks, we need to use the Fcfs_determine_cap logic. - cluster_wide_cap_values := make(map[int]float64) - index := 0 - for _, tsk := range tasks_to_schedule { - /* - Note that even though Fcfs_determine_cap is called, we have sorted the tasks aprior and thus, the tasks are scheduled in the sorted fashion. - Calling Fcfs_determine_cap(...) just to avoid redundant code. - */ - if cap, err := capper.fcfsDetermineCap(available_power, tsk); err == nil { - cluster_wide_cap_values[index] = cap - } else { - return nil, nil, err - } - index++ - } - // Now returning the sorted set of tasks and the cluster wide cap values for each task that is launched. - return tasks_to_schedule, cluster_wide_cap_values, nil - } + // Now, for each task in the sorted set of tasks, we need to use the Fcfs_determine_cap logic. + cluster_wide_cap_values := make(map[int]float64) + index := 0 + for _, tsk := range tasks_to_schedule { + /* + Note that even though Fcfs_determine_cap is called, we have sorted the tasks aprior and thus, the tasks are scheduled in the sorted fashion. + Calling Fcfs_determine_cap(...) just to avoid redundant code. + */ + if cap, err := capper.fcfsDetermineCap(available_power, tsk); err == nil { + cluster_wide_cap_values[index] = cap + } else { + return nil, nil, err + } + index++ + } + // Now returning the sorted set of tasks and the cluster wide cap values for each task that is launched. + return tasks_to_schedule, cluster_wide_cap_values, nil + } } // First come first serve scheduling. func (capper clusterwideCapper) fcfsDetermineCap(total_power map[string]float64, - new_task *def.Task) (float64, error) { - // Validation - if total_power == nil { - return 100, errors.New("Invalid argument: total_power") - } else { - // Need to calculate the running average - running_average := capper.running_average_of_watts(new_task) - // For each node, calculate the percentage of the running average to the total power. - running_average_to_total_power_percentage := make(map[string]float64) - for host, tpower := range total_power { - if tpower >= running_average { - running_average_to_total_power_percentage[host] = (running_average/tpower) * 100 - } else { - // We don't consider this host for the computation of the cluster wide cap. - } - } + new_task *def.Task) (float64, error) { + // Validation + if total_power == nil { + return 100, errors.New("Invalid argument: total_power") + } else { + // Need to calculate the running average + running_average := capper.running_average_of_watts(new_task) + // For each node, calculate the percentage of the running average to the total power. + running_average_to_total_power_percentage := make(map[string]float64) + for host, tpower := range total_power { + if tpower >= running_average { + running_average_to_total_power_percentage[host] = (running_average / tpower) * 100 + } else { + // We don't consider this host for the computation of the cluster wide cap. + } + } - // Determine the cluster wide cap value. - cap_value := capper.get_cap(running_average_to_total_power_percentage) - // Need to cap the cluster to this value. - return cap_value, nil - } + // Determine the cluster wide cap value. + cap_value := capper.get_cap(running_average_to_total_power_percentage) + // Need to cap the cluster to this value. + return cap_value, nil + } } // Stringer for an instance of clusterwideCapper func (capper clusterwideCapper) string() string { - return "Cluster Capper -- Proactively cap the entire cluster." + return "Cluster Capper -- Proactively cap the entire cluster." } diff --git a/schedulers/proactiveclusterwidecappingfcfs.go b/schedulers/proactiveclusterwidecappingfcfs.go index 679686a..b12cb7c 100644 --- a/schedulers/proactiveclusterwidecappingfcfs.go +++ b/schedulers/proactiveclusterwidecappingfcfs.go @@ -1,111 +1,111 @@ package schedulers import ( - "bitbucket.org/sunybingcloud/electron/def" - "bitbucket.org/sunybingcloud/electron/constants" - "bitbucket.org/sunybingcloud/electron/rapl" - "fmt" - "github.com/golang/protobuf/proto" - mesos "github.com/mesos/mesos-go/mesosproto" - "github.com/mesos/mesos-go/mesosutil" - sched "github.com/mesos/mesos-go/scheduler" - "log" - "math" - "strings" - "time" + "bitbucket.org/sunybingcloud/electron/constants" + "bitbucket.org/sunybingcloud/electron/def" + "bitbucket.org/sunybingcloud/electron/rapl" + "fmt" + "github.com/golang/protobuf/proto" + mesos "github.com/mesos/mesos-go/mesosproto" + "github.com/mesos/mesos-go/mesosutil" + sched "github.com/mesos/mesos-go/scheduler" + "log" + "math" + "strings" + "time" ) // Decides if to take an offer or not func (_ *ProactiveClusterwideCapFCFS) takeOffer(offer *mesos.Offer, task def.Task) bool { - offer_cpu, offer_mem, _ := OfferAgg(offer) + offer_cpu, offer_mem, _ := OfferAgg(offer) - if offer_cpu >= task.CPU && offer_mem >= task.RAM { - return true - } - return false + if offer_cpu >= task.CPU && offer_mem >= task.RAM { + return true + } + return false } // electronScheduler implements the Scheduler interface. type ProactiveClusterwideCapFCFS struct { - tasksCreated int - tasksRunning int - tasks []def.Task - metrics map[string]def.Metric - running map[string]map[string]bool - taskMonitor map[string][]def.Task // store tasks that are currently running. - availablePower map[string]float64 // available power for each node in the cluster. - totalPower map[string]float64 // total power for each node in the cluster. - ignoreWatts bool - capper *clusterwideCapper - ticker *time.Ticker - isCapping bool // indicate whether we are currently performing cluster wide capping. - //lock *sync.Mutex + tasksCreated int + tasksRunning int + tasks []def.Task + metrics map[string]def.Metric + running map[string]map[string]bool + taskMonitor map[string][]def.Task // store tasks that are currently running. + availablePower map[string]float64 // available power for each node in the cluster. + totalPower map[string]float64 // total power for each node in the cluster. + ignoreWatts bool + capper *clusterwideCapper + ticker *time.Ticker + isCapping bool // indicate whether we are currently performing cluster wide capping. + //lock *sync.Mutex - // First set of PCP values are garbage values, signal to logger to start recording when we're - // about to schedule the new task. - RecordPCP bool + // First set of PCP values are garbage values, signal to logger to start recording when we're + // about to schedule the new task. + RecordPCP bool - // This channel is closed when the program receives an interrupt, - // signalling that the program should shut down. - Shutdown chan struct{} + // This channel is closed when the program receives an interrupt, + // signalling that the program should shut down. + Shutdown chan struct{} - // This channel is closed after shutdown is closed, and only when all - // outstanding tasks have been cleaned up. - Done chan struct{} + // This channel is closed after shutdown is closed, and only when all + // outstanding tasks have been cleaned up. + Done chan struct{} - // Controls when to shutdown pcp logging. - PCPLog chan struct{} + // Controls when to shutdown pcp logging. + PCPLog chan struct{} } // New electron scheduler. func NewProactiveClusterwideCapFCFS(tasks []def.Task, ignoreWatts bool) *ProactiveClusterwideCapFCFS { - s := &ProactiveClusterwideCapFCFS { - tasks: tasks, - ignoreWatts: ignoreWatts, - Shutdown: make(chan struct{}), - Done: make(chan struct{}), - PCPLog: make(chan struct{}), - running: make(map[string]map[string]bool), - taskMonitor: make(map[string][]def.Task), - availablePower: make(map[string]float64), - totalPower: make(map[string]float64), - RecordPCP: false, - capper: getClusterwideCapperInstance(), - ticker: time.NewTicker(5 * time.Second), - isCapping: false, - //lock: new(sync.Mutex), - } - return s + s := &ProactiveClusterwideCapFCFS{ + tasks: tasks, + ignoreWatts: ignoreWatts, + Shutdown: make(chan struct{}), + Done: make(chan struct{}), + PCPLog: make(chan struct{}), + running: make(map[string]map[string]bool), + taskMonitor: make(map[string][]def.Task), + availablePower: make(map[string]float64), + totalPower: make(map[string]float64), + RecordPCP: false, + capper: getClusterwideCapperInstance(), + ticker: time.NewTicker(5 * time.Second), + isCapping: false, + //lock: new(sync.Mutex), + } + return s } func (s *ProactiveClusterwideCapFCFS) newTask(offer *mesos.Offer, task def.Task) *mesos.TaskInfo { - taskName := fmt.Sprintf("%s-%d", task.Name, *task.Instances) - s.tasksCreated++ + taskName := fmt.Sprintf("%s-%d", task.Name, *task.Instances) + s.tasksCreated++ - if !s.RecordPCP { - // Turn on logging. - s.RecordPCP = true - time.Sleep(1 * time.Second) // Make sure we're recording by the time the first task starts - } + if !s.RecordPCP { + // Turn on logging. + s.RecordPCP = true + time.Sleep(1 * time.Second) // Make sure we're recording by the time the first task starts + } - // If this is our first time running into this Agent - if _, ok := s.running[offer.GetSlaveId().GoString()]; !ok { - s.running[offer.GetSlaveId().GoString()] = make(map[string]bool) - } + // If this is our first time running into this Agent + if _, ok := s.running[offer.GetSlaveId().GoString()]; !ok { + s.running[offer.GetSlaveId().GoString()] = make(map[string]bool) + } - // Setting the task ID to the task. This is done so that we can consider each task to be different, - // even though they have the same parameters. - task.SetTaskID(*proto.String(taskName)) - // Add task to the list of tasks running on the node. - s.running[offer.GetSlaveId().GoString()][taskName] = true - s.taskMonitor[offer.GetSlaveId().GoString()] = []def.Task{task} + // Setting the task ID to the task. This is done so that we can consider each task to be different, + // even though they have the same parameters. + task.SetTaskID(*proto.String(taskName)) + // Add task to the list of tasks running on the node. + s.running[offer.GetSlaveId().GoString()][taskName] = true + s.taskMonitor[offer.GetSlaveId().GoString()] = []def.Task{task} - resources := []*mesos.Resource{ - mesosutil.NewScalarResource("cpus", task.CPU), - mesosutil.NewScalarResource("mem", task.RAM), - } + resources := []*mesos.Resource{ + mesosutil.NewScalarResource("cpus", task.CPU), + mesosutil.NewScalarResource("mem", task.RAM), + } - if !s.ignoreWatts { + if !s.ignoreWatts { resources = append(resources, mesosutil.NewScalarResource("watts", task.Watts)) } @@ -130,189 +130,189 @@ func (s *ProactiveClusterwideCapFCFS) newTask(offer *mesos.Offer, task def.Task) } func (s *ProactiveClusterwideCapFCFS) Registered( - _ sched.SchedulerDriver, - frameworkID *mesos.FrameworkID, - masterInfo *mesos.MasterInfo) { - log.Printf("Framework %s registered with master %s", frameworkID, masterInfo) + _ sched.SchedulerDriver, + frameworkID *mesos.FrameworkID, + masterInfo *mesos.MasterInfo) { + log.Printf("Framework %s registered with master %s", frameworkID, masterInfo) } func (s *ProactiveClusterwideCapFCFS) Reregistered(_ sched.SchedulerDriver, masterInfo *mesos.MasterInfo) { - log.Printf("Framework re-registered with master %s", masterInfo) + log.Printf("Framework re-registered with master %s", masterInfo) } func (s *ProactiveClusterwideCapFCFS) Disconnected(sched.SchedulerDriver) { - // Need to stop the capping process. - s.ticker.Stop() - s.isCapping = false - log.Println("Framework disconnected with master") + // Need to stop the capping process. + s.ticker.Stop() + s.isCapping = false + log.Println("Framework disconnected with master") } // go routine to cap the entire cluster in regular intervals of time. var currentCapValue = 0.0 // initial value to indicate that we haven't capped the cluster yet. func (s *ProactiveClusterwideCapFCFS) startCapping() { - go func() { - for { - select { - case <- s.ticker.C: - // Need to cap the cluster to the currentCapValue. - if currentCapValue > 0.0 { - //mutex.Lock() - //s.lock.Lock() - for _, host := range constants.Hosts { - // Rounding curreCapValue to the nearest int. - if err := rapl.Cap(host, "rapl", int(math.Floor(currentCapValue + 0.5))); err != nil { - fmt.Println(err) - } else { - fmt.Printf("Successfully capped %s to %f%\n", host, currentCapValue) - } - } - //mutex.Unlock() - //s.lock.Unlock() - } - } - } - }() + go func() { + for { + select { + case <-s.ticker.C: + // Need to cap the cluster to the currentCapValue. + if currentCapValue > 0.0 { + //mutex.Lock() + //s.lock.Lock() + for _, host := range constants.Hosts { + // Rounding curreCapValue to the nearest int. + if err := rapl.Cap(host, "rapl", int(math.Floor(currentCapValue+0.5))); err != nil { + fmt.Println(err) + } else { + fmt.Printf("Successfully capped %s to %f%\n", host, currentCapValue) + } + } + //mutex.Unlock() + //s.lock.Unlock() + } + } + } + }() } // Stop cluster wide capping func (s *ProactiveClusterwideCapFCFS) stopCapping() { - if s.isCapping { - log.Println("Stopping the cluster wide capping.") - s.ticker.Stop() - s.isCapping = false - } + if s.isCapping { + log.Println("Stopping the cluster wide capping.") + s.ticker.Stop() + s.isCapping = false + } } // TODO: Need to reduce the time complexity: looping over offers twice (Possible to do it just once?). func (s *ProactiveClusterwideCapFCFS) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { - log.Printf("Received %d resource offers", len(offers)) + log.Printf("Received %d resource offers", len(offers)) - // retrieving the available power for all the hosts in the offers. - for _, offer := range offers { - _, _, offer_watts := OfferAgg(offer) - s.availablePower[*offer.Hostname] = offer_watts - // setting total power if the first time. - if _, ok := s.totalPower[*offer.Hostname]; !ok { - s.totalPower[*offer.Hostname] = offer_watts - } - } + // retrieving the available power for all the hosts in the offers. + for _, offer := range offers { + _, _, offer_watts := OfferAgg(offer) + s.availablePower[*offer.Hostname] = offer_watts + // setting total power if the first time. + if _, ok := s.totalPower[*offer.Hostname]; !ok { + s.totalPower[*offer.Hostname] = offer_watts + } + } - for host, tpower := range s.totalPower { - fmt.Printf("TotalPower[%s] = %f\n", host, tpower) - } - for host, apower := range s.availablePower { - fmt.Printf("AvailablePower[%s] = %f\n", host, apower) - } + for host, tpower := range s.totalPower { + fmt.Printf("TotalPower[%s] = %f\n", host, tpower) + } + for host, apower := range s.availablePower { + fmt.Printf("AvailablePower[%s] = %f\n", host, apower) + } - for _, offer := range offers { - select { - case <-s.Shutdown: - log.Println("Done scheduling tasks: declining offer on [", offer.GetHostname(), "]") - driver.DeclineOffer(offer.Id, longFilter) + for _, offer := range offers { + select { + case <-s.Shutdown: + log.Println("Done scheduling tasks: declining offer on [", offer.GetHostname(), "]") + driver.DeclineOffer(offer.Id, longFilter) - log.Println("Number of tasks still running: ", s.tasksRunning) - continue - default: - } + log.Println("Number of tasks still running: ", s.tasksRunning) + continue + default: + } - /* - Clusterwide Capping strategy + /* + Clusterwide Capping strategy - For each task in s.tasks, - 1. Need to check whether the offer can be taken or not (based on CPU and RAM requirements). - 2. If the tasks fits the offer, then I need to detemrine the cluster wide cap. - 3. currentCapValue is updated with the determined cluster wide cap. + For each task in s.tasks, + 1. Need to check whether the offer can be taken or not (based on CPU and RAM requirements). + 2. If the tasks fits the offer, then I need to detemrine the cluster wide cap. + 3. currentCapValue is updated with the determined cluster wide cap. - Cluster wide capping is currently performed at regular intervals of time. - TODO: We can choose to cap the cluster only if the clusterwide cap varies more than the current clusterwide cap. - Although this sounds like a better approach, it only works when the resource requirements of neighbouring tasks are similar. - */ - //offer_cpu, offer_ram, _ := OfferAgg(offer) + Cluster wide capping is currently performed at regular intervals of time. + TODO: We can choose to cap the cluster only if the clusterwide cap varies more than the current clusterwide cap. + Although this sounds like a better approach, it only works when the resource requirements of neighbouring tasks are similar. + */ + //offer_cpu, offer_ram, _ := OfferAgg(offer) - taken := false - //var mutex sync.Mutex + taken := false + //var mutex sync.Mutex - for i, task := range s.tasks { - // Don't take offer if it doesn't match our task's host requirement. - if !strings.HasPrefix(*offer.Hostname, task.Host) { - continue - } + for i, task := range s.tasks { + // Don't take offer if it doesn't match our task's host requirement. + if !strings.HasPrefix(*offer.Hostname, task.Host) { + continue + } - // Does the task fit. - if s.takeOffer(offer, task) { - // Capping the cluster if haven't yet started, - if !s.isCapping { - s.startCapping() - s.isCapping = true - } - taken = true - //mutex.Lock() - //s.lock.Lock() - //tempCap, err := s.capper.fcfsDetermineCap(s.availablePower, &task) - tempCap, err := s.capper.fcfsDetermineCap(s.totalPower, &task) + // Does the task fit. + if s.takeOffer(offer, task) { + // Capping the cluster if haven't yet started, + if !s.isCapping { + s.startCapping() + s.isCapping = true + } + taken = true + //mutex.Lock() + //s.lock.Lock() + //tempCap, err := s.capper.fcfsDetermineCap(s.availablePower, &task) + tempCap, err := s.capper.fcfsDetermineCap(s.totalPower, &task) - if err == nil { - currentCapValue = tempCap - } else { - fmt.Printf("Failed to determine new cluster wide cap: ") - fmt.Println(err) - } - //mutex.Unlock() - //s.lock.Unlock() - fmt.Printf("Starting on [%s]\n", offer.GetHostname()) - to_schedule := []*mesos.TaskInfo{s.newTask(offer, task)} - driver.LaunchTasks([]*mesos.OfferID{offer.Id}, to_schedule, defaultFilter) - fmt.Printf("Inst: %d", *task.Instances) - *task.Instances-- - if *task.Instances <= 0 { - // All instances of the task have been scheduled. Need to remove it from the list of tasks to schedule. - s.tasks[i] = s.tasks[len(s.tasks)-1] + if err == nil { + currentCapValue = tempCap + } else { + fmt.Printf("Failed to determine new cluster wide cap: ") + fmt.Println(err) + } + //mutex.Unlock() + //s.lock.Unlock() + fmt.Printf("Starting on [%s]\n", offer.GetHostname()) + to_schedule := []*mesos.TaskInfo{s.newTask(offer, task)} + driver.LaunchTasks([]*mesos.OfferID{offer.Id}, to_schedule, defaultFilter) + fmt.Printf("Inst: %d", *task.Instances) + *task.Instances-- + if *task.Instances <= 0 { + // All instances of the task have been scheduled. Need to remove it from the list of tasks to schedule. + s.tasks[i] = s.tasks[len(s.tasks)-1] s.tasks = s.tasks[:len(s.tasks)-1] if len(s.tasks) <= 0 { log.Println("Done scheduling all tasks") - // Need to stop the cluster wide capping as there aren't any more tasks to schedule. - s.stopCapping() + // Need to stop the cluster wide capping as there aren't any more tasks to schedule. + s.stopCapping() close(s.Shutdown) } - } - break // Offer taken, move on. - } else { - // Task doesn't fit the offer. Move onto the next offer. - } - } + } + break // Offer taken, move on. + } else { + // Task doesn't fit the offer. Move onto the next offer. + } + } - // If no task fit the offer, then declining the offer. - if !taken { - fmt.Printf("There is not enough resources to launch a task on Host: %s\n", offer.GetHostname()) - cpus, mem, watts := OfferAgg(offer) + // If no task fit the offer, then declining the offer. + if !taken { + fmt.Printf("There is not enough resources to launch a task on Host: %s\n", offer.GetHostname()) + cpus, mem, watts := OfferAgg(offer) - log.Printf("\n", cpus, mem, watts) - driver.DeclineOffer(offer.Id, defaultFilter) - } - } + log.Printf("\n", cpus, mem, watts) + driver.DeclineOffer(offer.Id, defaultFilter) + } + } } func (s *ProactiveClusterwideCapFCFS) StatusUpdate(driver sched.SchedulerDriver, status *mesos.TaskStatus) { - log.Printf("Received task status [%s] for task [%s]", NameFor(status.State), *status.TaskId.Value) + log.Printf("Received task status [%s] for task [%s]", NameFor(status.State), *status.TaskId.Value) if *status.State == mesos.TaskState_TASK_RUNNING { s.tasksRunning++ } else if IsTerminal(status.State) { delete(s.running[status.GetSlaveId().GoString()], *status.TaskId.Value) - // Need to remove the task from the window of tasks. - s.capper.taskFinished(*status.TaskId.Value) - //currentCapValue, _ = s.capper.recap(s.availablePower, s.taskMonitor, *status.TaskId.Value) - // Determining the new cluster wide cap. - currentCapValue, _ = s.capper.recap(s.totalPower, s.taskMonitor, *status.TaskId.Value) - log.Printf("Recapping the cluster to %f\n", currentCapValue) + // Need to remove the task from the window of tasks. + s.capper.taskFinished(*status.TaskId.Value) + //currentCapValue, _ = s.capper.recap(s.availablePower, s.taskMonitor, *status.TaskId.Value) + // Determining the new cluster wide cap. + currentCapValue, _ = s.capper.recap(s.totalPower, s.taskMonitor, *status.TaskId.Value) + log.Printf("Recapping the cluster to %f\n", currentCapValue) s.tasksRunning-- if s.tasksRunning == 0 { select { case <-s.Shutdown: - // Need to stop the capping process. - s.stopCapping() + // Need to stop the capping process. + s.stopCapping() close(s.Done) default: } @@ -322,20 +322,20 @@ func (s *ProactiveClusterwideCapFCFS) StatusUpdate(driver sched.SchedulerDriver, } func (s *ProactiveClusterwideCapFCFS) FrameworkMessage(driver sched.SchedulerDriver, - executorID *mesos.ExecutorID, - slaveID *mesos.SlaveID, - message string) { + executorID *mesos.ExecutorID, + slaveID *mesos.SlaveID, + message string) { log.Println("Getting a framework message: ", message) log.Printf("Received a framework message from some unknown source: %s", *executorID.Value) } func (s *ProactiveClusterwideCapFCFS) OfferRescinded(_ sched.SchedulerDriver, offerID *mesos.OfferID) { - log.Printf("Offer %s rescinded", offerID) + log.Printf("Offer %s rescinded", offerID) } func (s *ProactiveClusterwideCapFCFS) SlaveLost(_ sched.SchedulerDriver, slaveID *mesos.SlaveID) { - log.Printf("Slave %s lost", slaveID) + log.Printf("Slave %s lost", slaveID) } func (s *ProactiveClusterwideCapFCFS) ExecutorLost(_ sched.SchedulerDriver, executorID *mesos.ExecutorID, slaveID *mesos.SlaveID, status int) { From e562df0f5c20aff34300faf7b76a0d8eca74b8a4 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Tue, 15 Nov 2016 15:11:00 -0500 Subject: [PATCH 30/94] Formatted the code --- def/task.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/def/task.go b/def/task.go index c7326c6..63668ad 100644 --- a/def/task.go +++ b/def/task.go @@ -17,7 +17,7 @@ type Task struct { CMD string `json:"cmd"` Instances *int `json:"inst"` Host string `json:"host"` - TaskID string `json:"taskID"` + TaskID string `json:"taskID"` } func TasksFromJSON(uri string) ([]Task, error) { @@ -42,10 +42,10 @@ func (tsk *Task) UpdateHost(new_host string) bool { // Validation is_correct_host := false for _, existing_host := range constants.Hosts { - if new_host == existing_host { - is_correct_host = true - } - } + if new_host == existing_host { + is_correct_host = true + } + } if !is_correct_host { return false } else { From 3f90ccfe74ae8db1d1bad5bd6d54d2563e373dd6 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 17 Nov 2016 21:51:02 -0500 Subject: [PATCH 31/94] Sycnrhonized operations that change the value of the cluster wide cap. Added cleverRecap(...) that determines the recap value of the cluster at a much finer level, taking into account the average load on each node in the cluster. Bug fix in cap.go -- closed the session once capping had been done. This prevented from running out of file descriptors. --- constants/constants.go | 20 +-- rapl/cap.go | 1 + schedulers/proactiveclusterwidecappers.go | 106 ++++++++++++-- schedulers/proactiveclusterwidecappingfcfs.go | 135 +++++++++++++----- 4 files changed, 194 insertions(+), 68 deletions(-) diff --git a/constants/constants.go b/constants/constants.go index 133d61f..cc6d705 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -34,7 +34,7 @@ var Power_threshold = 0.6 // Right now saying that a task will never be given le So, if power required = 10W, the node would be capped to 75%*10W. This value can be changed upon convenience. */ -var Cap_margin = 0.75 +var Cap_margin = 0.70 // Modify the cap margin. func UpdateCapMargin(new_cap_margin float64) bool { @@ -84,20 +84,4 @@ func UpdateWindowSize(new_window_size int) bool { Window_size = new_window_size return true } -} - -// // Time duration between successive cluster wide capping. -// var Clusterwide_cap_interval = 10 // Right now capping the cluster at 10 second intervals. -// -// // Modify the cluster wide capping interval. We can update the interval depending on the workload. -// // TODO: If the workload is heavy then we can set a longer interval, while on the other hand, -// // if the workload is light then a smaller interval is sufficient. -// func UpdateClusterwideCapInterval(new_interval int) bool { -// // Validation -// if new_interval == 0.0 { -// return false -// } else { -// Clusterwide_cap_interval = new_interval -// return true -// } -// } +} \ No newline at end of file diff --git a/rapl/cap.go b/rapl/cap.go index 20cd945..b15d352 100644 --- a/rapl/cap.go +++ b/rapl/cap.go @@ -26,6 +26,7 @@ func Cap(host, username string, percentage int) error { } session, err := connection.NewSession() + defer session.Close() if err != nil { return errors.Wrap(err, "Failed to create session") } diff --git a/schedulers/proactiveclusterwidecappers.go b/schedulers/proactiveclusterwidecappers.go index aa3eafa..38aaca0 100644 --- a/schedulers/proactiveclusterwidecappers.go +++ b/schedulers/proactiveclusterwidecappers.go @@ -16,6 +16,7 @@ import ( "container/list" "errors" "github.com/montanaflynn/stats" + "log" "sort" ) @@ -110,6 +111,68 @@ func (capper clusterwideCapper) get_cap(running_average_to_total_power_percentag return 100.0 } +/* +Recapping the entire cluster. Also, removing the finished task from the list of running tasks. + +We would, at this point, have a better knowledge about the state of the cluster. + +1. Calculate the total allocated watts per node in the cluster. +2. Compute the ratio of the total watts usage per node to the total power for that node. + This would give us the load on that node. +3. Now, compute the average load across all the nodes in the cluster. + This would be the cap value. +*/ +func (capper clusterwideCapper) cleverRecap(total_power map[string]float64, + task_monitor map[string][]def.Task, finished_taskId string) (float64, error) { + // Validation + if total_power == nil || task_monitor == nil { + return 100.0, errors.New("Invalid argument: total_power, task_monitor") + } + // watts usage on each node in the cluster. + watts_usages := make(map[string][]float64) + host_of_finished_task := "" + index_of_finished_task := -1 + for _, host := range constants.Hosts { + watts_usages[host] = []float64{0.0} + } + for host, tasks := range task_monitor { + for i, task := range tasks { + if task.TaskID == finished_taskId { + host_of_finished_task = host + index_of_finished_task = i + // Not considering this task + continue + } + watts_usages[host] = append(watts_usages[host], float64(task.Watts) * constants.Cap_margin) + } + } + + // Updating task monitor + if host_of_finished_task != "" && index_of_finished_task != -1 { + log.Printf("Removing task with task [%s] from the list of running tasks\n", + task_monitor[host_of_finished_task][index_of_finished_task].TaskID) + task_monitor[host_of_finished_task] = append(task_monitor[host_of_finished_task][:index_of_finished_task], + task_monitor[host_of_finished_task][index_of_finished_task+1:]...) + } + + // load on each node in the cluster. + loads := []float64{} + for host, usages := range watts_usages { + total_usage := 0.0 + for _, usage := range usages { + total_usage += usage + } + loads = append(loads, total_usage / total_power[host]) + } + // Now need to compute the average load. + total_load := 0.0 + for _, load := range loads { + total_load += load + } + average_load := total_load / float64(len(loads)) // this would be the cap value. + return average_load, nil +} + /* Recapping the entire cluster. @@ -128,18 +191,35 @@ func (capper clusterwideCapper) recap(total_power map[string]float64, } total_allocated_power := 0.0 total_running_tasks := 0 - for _, tasks := range task_monitor { - index := 0 - for i, task := range tasks { - if task.TaskID == finished_taskId { - index = i - continue - } - total_allocated_power += float64(task.Watts) * constants.Cap_margin - total_running_tasks++ - } - tasks = append(tasks[:index], tasks[index+1:]...) - } + + host_of_finished_task := "" + index_of_finished_task := -1 + for host, tasks := range task_monitor { + for i, task := range tasks { + if task.TaskID == finished_taskId { + host_of_finished_task = host + index_of_finished_task = i + // Not considering this task for the computation of total_allocated_power and total_running_tasks + continue + } + total_allocated_power += (float64(task.Watts) * constants.Cap_margin) + total_running_tasks++ + } + } + + // Updating task monitor + if host_of_finished_task != "" && index_of_finished_task != -1 { + log.Printf("Removing task with task [%s] from the list of running tasks\n", + task_monitor[host_of_finished_task][index_of_finished_task].TaskID) + task_monitor[host_of_finished_task] = append(task_monitor[host_of_finished_task][:index_of_finished_task], + task_monitor[host_of_finished_task][index_of_finished_task+1:]...) + } + + // For the last task, total_allocated_power and total_running_tasks would be 0 + if total_allocated_power == 0 && total_running_tasks == 0 { + return 100, errors.New("No task running on the cluster.") + } + average := total_allocated_power / float64(total_running_tasks) ratios := []float64{} for _, tpower := range total_power { @@ -211,7 +291,7 @@ func (capper clusterwideCapper) taskFinished(taskID string) { } } - // Ee need to remove the task from the window. + // we need to remove the task from the window. if task_to_remove, ok := task_element_to_remove.Value.(*def.Task); ok { capper.window_of_tasks.Remove(task_element_to_remove) capper.number_of_tasks_in_window -= 1 diff --git a/schedulers/proactiveclusterwidecappingfcfs.go b/schedulers/proactiveclusterwidecappingfcfs.go index b12cb7c..59c3ac5 100644 --- a/schedulers/proactiveclusterwidecappingfcfs.go +++ b/schedulers/proactiveclusterwidecappingfcfs.go @@ -12,14 +12,15 @@ import ( "log" "math" "strings" + "sync" "time" ) // Decides if to take an offer or not func (_ *ProactiveClusterwideCapFCFS) takeOffer(offer *mesos.Offer, task def.Task) bool { - offer_cpu, offer_mem, _ := OfferAgg(offer) + offer_cpu, offer_mem, offer_watts := OfferAgg(offer) - if offer_cpu >= task.CPU && offer_mem >= task.RAM { + if offer_cpu >= task.CPU && offer_mem >= task.RAM && offer_watts >= task.Watts { return true } return false @@ -38,8 +39,9 @@ type ProactiveClusterwideCapFCFS struct { ignoreWatts bool capper *clusterwideCapper ticker *time.Ticker + recapTicker *time.Ticker isCapping bool // indicate whether we are currently performing cluster wide capping. - //lock *sync.Mutex + isRecapping bool // indicate whether we are currently performing cluster wide re-capping. // First set of PCP values are garbage values, signal to logger to start recording when we're // about to schedule the new task. @@ -71,13 +73,17 @@ func NewProactiveClusterwideCapFCFS(tasks []def.Task, ignoreWatts bool) *Proacti totalPower: make(map[string]float64), RecordPCP: false, capper: getClusterwideCapperInstance(), - ticker: time.NewTicker(5 * time.Second), + ticker: time.NewTicker(10 * time.Second), + recapTicker: time.NewTicker(20 * time.Second), isCapping: false, - //lock: new(sync.Mutex), + isRecapping: false, } return s } +// mutex +var mutex sync.Mutex + func (s *ProactiveClusterwideCapFCFS) newTask(offer *mesos.Offer, task def.Task) *mesos.TaskInfo { taskName := fmt.Sprintf("%s-%d", task.Name, *task.Instances) s.tasksCreated++ @@ -95,10 +101,14 @@ func (s *ProactiveClusterwideCapFCFS) newTask(offer *mesos.Offer, task def.Task) // Setting the task ID to the task. This is done so that we can consider each task to be different, // even though they have the same parameters. - task.SetTaskID(*proto.String(taskName)) + task.SetTaskID(*proto.String("electron-" + taskName)) // Add task to the list of tasks running on the node. s.running[offer.GetSlaveId().GoString()][taskName] = true - s.taskMonitor[offer.GetSlaveId().GoString()] = []def.Task{task} + if len(s.taskMonitor[offer.GetSlaveId().GoString()]) == 0 { + s.taskMonitor[offer.GetSlaveId().GoString()] = []def.Task{task} + } else { + s.taskMonitor[offer.GetSlaveId().GoString()] = append(s.taskMonitor[offer.GetSlaveId().GoString()], task) + } resources := []*mesos.Resource{ mesosutil.NewScalarResource("cpus", task.CPU), @@ -143,7 +153,10 @@ func (s *ProactiveClusterwideCapFCFS) Reregistered(_ sched.SchedulerDriver, mast func (s *ProactiveClusterwideCapFCFS) Disconnected(sched.SchedulerDriver) { // Need to stop the capping process. s.ticker.Stop() + s.recapTicker.Stop() + mutex.Lock() s.isCapping = false + mutex.Unlock() log.Println("Framework disconnected with master") } @@ -155,20 +168,44 @@ func (s *ProactiveClusterwideCapFCFS) startCapping() { select { case <-s.ticker.C: // Need to cap the cluster to the currentCapValue. + mutex.Lock() if currentCapValue > 0.0 { - //mutex.Lock() - //s.lock.Lock() for _, host := range constants.Hosts { // Rounding curreCapValue to the nearest int. if err := rapl.Cap(host, "rapl", int(math.Floor(currentCapValue+0.5))); err != nil { - fmt.Println(err) - } else { - fmt.Printf("Successfully capped %s to %f%\n", host, currentCapValue) + log.Println(err) } } - //mutex.Unlock() - //s.lock.Unlock() + log.Printf("Capped the cluster to %d", int(math.Floor(currentCapValue+0.5))) } + mutex.Unlock() + } + } + }() +} + +// go routine to cap the entire cluster in regular intervals of time. +var recapValue = 0.0 // The cluster wide cap value when recapping. +func (s *ProactiveClusterwideCapFCFS) startRecapping() { + go func() { + for { + select { + case <-s.recapTicker.C: + mutex.Lock() + // If stopped performing cluster wide capping then we need to explicitly cap the entire cluster. + //if !s.isCapping && s.isRecapping && recapValue > 0.0 { + if s.isRecapping && recapValue > 0.0 { + for _, host := range constants.Hosts { + // Rounding curreCapValue to the nearest int. + if err := rapl.Cap(host, "rapl", int(math.Floor(recapValue+0.5))); err != nil { + log.Println(err) + } + } + log.Printf("Recapped the cluster to %d", int(math.Floor(recapValue+0.5))) + } + // setting recapping to false + s.isRecapping = false + mutex.Unlock() } } }() @@ -179,7 +216,22 @@ func (s *ProactiveClusterwideCapFCFS) stopCapping() { if s.isCapping { log.Println("Stopping the cluster wide capping.") s.ticker.Stop() + mutex.Lock() s.isCapping = false + s.isRecapping = true + mutex.Unlock() + } +} + +// Stop cluster wide Recapping +func (s *ProactiveClusterwideCapFCFS) stopRecapping() { + // If not capping, then definitely recapping. + if !s.isCapping && s.isRecapping { + log.Println("Stopping the cluster wide re-capping.") + s.recapTicker.Stop() + mutex.Lock() + s.isRecapping = false + mutex.Unlock() } } @@ -198,10 +250,7 @@ func (s *ProactiveClusterwideCapFCFS) ResourceOffers(driver sched.SchedulerDrive } for host, tpower := range s.totalPower { - fmt.Printf("TotalPower[%s] = %f\n", host, tpower) - } - for host, apower := range s.availablePower { - fmt.Printf("AvailablePower[%s] = %f\n", host, apower) + log.Printf("TotalPower[%s] = %f", host, tpower) } for _, offer := range offers { @@ -227,10 +276,7 @@ func (s *ProactiveClusterwideCapFCFS) ResourceOffers(driver sched.SchedulerDrive TODO: We can choose to cap the cluster only if the clusterwide cap varies more than the current clusterwide cap. Although this sounds like a better approach, it only works when the resource requirements of neighbouring tasks are similar. */ - //offer_cpu, offer_ram, _ := OfferAgg(offer) - taken := false - //var mutex sync.Mutex for i, task := range s.tasks { // Don't take offer if it doesn't match our task's host requirement. @@ -242,27 +288,26 @@ func (s *ProactiveClusterwideCapFCFS) ResourceOffers(driver sched.SchedulerDrive if s.takeOffer(offer, task) { // Capping the cluster if haven't yet started, if !s.isCapping { - s.startCapping() + mutex.Lock() s.isCapping = true + mutex.Unlock() + s.startCapping() } taken = true - //mutex.Lock() - //s.lock.Lock() - //tempCap, err := s.capper.fcfsDetermineCap(s.availablePower, &task) tempCap, err := s.capper.fcfsDetermineCap(s.totalPower, &task) if err == nil { + mutex.Lock() currentCapValue = tempCap + mutex.Unlock() } else { - fmt.Printf("Failed to determine new cluster wide cap: ") - fmt.Println(err) + log.Printf("Failed to determine new cluster wide cap: ") + log.Println(err) } - //mutex.Unlock() - //s.lock.Unlock() - fmt.Printf("Starting on [%s]\n", offer.GetHostname()) + log.Printf("Starting on [%s]\n", offer.GetHostname()) to_schedule := []*mesos.TaskInfo{s.newTask(offer, task)} driver.LaunchTasks([]*mesos.OfferID{offer.Id}, to_schedule, defaultFilter) - fmt.Printf("Inst: %d", *task.Instances) + log.Printf("Inst: %d", *task.Instances) *task.Instances-- if *task.Instances <= 0 { // All instances of the task have been scheduled. Need to remove it from the list of tasks to schedule. @@ -273,6 +318,7 @@ func (s *ProactiveClusterwideCapFCFS) ResourceOffers(driver sched.SchedulerDrive log.Println("Done scheduling all tasks") // Need to stop the cluster wide capping as there aren't any more tasks to schedule. s.stopCapping() + s.startRecapping() // Load changes after every task finishes and hence we need to change the capping of the cluster. close(s.Shutdown) } } @@ -284,7 +330,7 @@ func (s *ProactiveClusterwideCapFCFS) ResourceOffers(driver sched.SchedulerDrive // If no task fit the offer, then declining the offer. if !taken { - fmt.Printf("There is not enough resources to launch a task on Host: %s\n", offer.GetHostname()) + log.Printf("There is not enough resources to launch a task on Host: %s\n", offer.GetHostname()) cpus, mem, watts := OfferAgg(offer) log.Printf("\n", cpus, mem, watts) @@ -294,7 +340,7 @@ func (s *ProactiveClusterwideCapFCFS) ResourceOffers(driver sched.SchedulerDrive } func (s *ProactiveClusterwideCapFCFS) StatusUpdate(driver sched.SchedulerDriver, status *mesos.TaskStatus) { - log.Printf("Received task status [%s] for task [%s]", NameFor(status.State), *status.TaskId.Value) + log.Printf("Received task status [%s] for task [%s]\n", NameFor(status.State), *status.TaskId.Value) if *status.State == mesos.TaskState_TASK_RUNNING { s.tasksRunning++ @@ -302,17 +348,32 @@ func (s *ProactiveClusterwideCapFCFS) StatusUpdate(driver sched.SchedulerDriver, delete(s.running[status.GetSlaveId().GoString()], *status.TaskId.Value) // Need to remove the task from the window of tasks. s.capper.taskFinished(*status.TaskId.Value) - //currentCapValue, _ = s.capper.recap(s.availablePower, s.taskMonitor, *status.TaskId.Value) // Determining the new cluster wide cap. - currentCapValue, _ = s.capper.recap(s.totalPower, s.taskMonitor, *status.TaskId.Value) - log.Printf("Recapping the cluster to %f\n", currentCapValue) + tempCap, err := s.capper.recap(s.totalPower, s.taskMonitor, *status.TaskId.Value) + if err == nil { + // if new determined cap value is different from the current recap value then we need to recap. + if int(math.Floor(tempCap+0.5)) != int(math.Floor(recapValue+0.5)) { + recapValue = tempCap + mutex.Lock() + s.isRecapping = true + mutex.Unlock() + log.Printf("Determined re-cap value: %f\n", recapValue) + } else { + mutex.Lock() + s.isRecapping = false + mutex.Unlock() + } + } else { + // Not updating currentCapValue + log.Println(err) + } s.tasksRunning-- if s.tasksRunning == 0 { select { case <-s.Shutdown: // Need to stop the capping process. - s.stopCapping() + s.stopRecapping() close(s.Done) default: } From a158f4a341c0bb7bd639d24e98501dda211b52bf Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Tue, 22 Nov 2016 17:02:58 -0500 Subject: [PATCH 32/94] removed rankedDetermineCap(...) as it was not needed. This algorithm has been integrated into proactiveclusterwidecappingranked.go --- schedulers/proactiveclusterwidecappers.go | 52 ++++++----------------- 1 file changed, 13 insertions(+), 39 deletions(-) diff --git a/schedulers/proactiveclusterwidecappers.go b/schedulers/proactiveclusterwidecappers.go index 38aaca0..6da6873 100644 --- a/schedulers/proactiveclusterwidecappers.go +++ b/schedulers/proactiveclusterwidecappers.go @@ -5,6 +5,7 @@ Step2. Compute what percentage of total power of each node, is the running avera Step3. Compute the median of the percetages and this is the percentage that the cluster needs to be capped at. 1. First fit scheduling -- Perform the above steps for each task that needs to be scheduled. +2. Ranked based scheduling -- Sort the tasks to be scheduled, in ascending order, and then determine the cluster wide cap. This is not a scheduler but a scheduling scheme that schedulers can use. */ @@ -121,6 +122,9 @@ We would, at this point, have a better knowledge about the state of the cluster. This would give us the load on that node. 3. Now, compute the average load across all the nodes in the cluster. This would be the cap value. + +Note: Although this would ensure lesser power usage, it might increase makespan if there is a heavy workload on just one node. +TODO: return a map[string]float64 that contains the recap value per node. This way, we can provide the right amount of power per node. */ func (capper clusterwideCapper) cleverRecap(total_power map[string]float64, task_monitor map[string][]def.Task, finished_taskId string) (float64, error) { @@ -235,23 +239,23 @@ func (capper clusterwideCapper) recap(total_power map[string]float64, } /* Quick sort algorithm to sort tasks, in place, in ascending order of power.*/ -func (capper clusterwideCapper) quick_sort(low int, high int, tasks_to_sort []*def.Task) { +func (capper clusterwideCapper) quick_sort(low int, high int, tasks_to_sort *[]def.Task) { i := low j := high // calculating the pivot pivot_index := low + (high-low)/2 - pivot := tasks_to_sort[pivot_index] + pivot := (*tasks_to_sort)[pivot_index] for i <= j { - for tasks_to_sort[i].Watts < pivot.Watts { + for (*tasks_to_sort)[i].Watts < pivot.Watts { i++ } - for tasks_to_sort[j].Watts > pivot.Watts { + for (*tasks_to_sort)[j].Watts > pivot.Watts { j-- } if i <= j { - temp := tasks_to_sort[i] - tasks_to_sort[i] = tasks_to_sort[j] - tasks_to_sort[j] = temp + temp := (*tasks_to_sort)[i] + (*tasks_to_sort)[i] = (*tasks_to_sort)[j] + (*tasks_to_sort)[j] = temp i++ j-- } @@ -265,8 +269,8 @@ func (capper clusterwideCapper) quick_sort(low int, high int, tasks_to_sort []*d } // Sorting tasks in ascending order of requested watts. -func (capper clusterwideCapper) sort_tasks(tasks_to_sort []*def.Task) { - capper.quick_sort(0, len(tasks_to_sort)-1, tasks_to_sort) +func (capper clusterwideCapper) sort_tasks(tasks_to_sort *[]def.Task) { + capper.quick_sort(0, len(*tasks_to_sort)-1, tasks_to_sort) } /* @@ -299,36 +303,6 @@ func (capper clusterwideCapper) taskFinished(taskID string) { } } -// Ranked based scheduling. -func (capper clusterwideCapper) rankedDetermineCap(available_power map[string]float64, - tasks_to_schedule []*def.Task) ([]*def.Task, map[int]float64, error) { - // Validation - if available_power == nil || len(tasks_to_schedule) == 0 { - return nil, nil, errors.New("Invalid argument: available_power, tasks_to_schedule") - } else { - // Need to sort the tasks in ascending order of requested power. - capper.sort_tasks(tasks_to_schedule) - - // Now, for each task in the sorted set of tasks, we need to use the Fcfs_determine_cap logic. - cluster_wide_cap_values := make(map[int]float64) - index := 0 - for _, tsk := range tasks_to_schedule { - /* - Note that even though Fcfs_determine_cap is called, we have sorted the tasks aprior and thus, the tasks are scheduled in the sorted fashion. - Calling Fcfs_determine_cap(...) just to avoid redundant code. - */ - if cap, err := capper.fcfsDetermineCap(available_power, tsk); err == nil { - cluster_wide_cap_values[index] = cap - } else { - return nil, nil, err - } - index++ - } - // Now returning the sorted set of tasks and the cluster wide cap values for each task that is launched. - return tasks_to_schedule, cluster_wide_cap_values, nil - } -} - // First come first serve scheduling. func (capper clusterwideCapper) fcfsDetermineCap(total_power map[string]float64, new_task *def.Task) (float64, error) { From 6c858c4e881ae3ac0b7f9a671346a34ac0994725 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Tue, 22 Nov 2016 17:04:30 -0500 Subject: [PATCH 33/94] Added another line that needs to be uncommented to choose cleverRecap. --- schedulers/proactiveclusterwidecappingfcfs.go | 73 ++++++++++--------- 1 file changed, 37 insertions(+), 36 deletions(-) diff --git a/schedulers/proactiveclusterwidecappingfcfs.go b/schedulers/proactiveclusterwidecappingfcfs.go index 59c3ac5..c352cb1 100644 --- a/schedulers/proactiveclusterwidecappingfcfs.go +++ b/schedulers/proactiveclusterwidecappingfcfs.go @@ -82,7 +82,7 @@ func NewProactiveClusterwideCapFCFS(tasks []def.Task, ignoreWatts bool) *Proacti } // mutex -var mutex sync.Mutex +var fcfsMutex sync.Mutex func (s *ProactiveClusterwideCapFCFS) newTask(offer *mesos.Offer, task def.Task) *mesos.TaskInfo { taskName := fmt.Sprintf("%s-%d", task.Name, *task.Instances) @@ -154,58 +154,58 @@ func (s *ProactiveClusterwideCapFCFS) Disconnected(sched.SchedulerDriver) { // Need to stop the capping process. s.ticker.Stop() s.recapTicker.Stop() - mutex.Lock() + fcfsMutex.Lock() s.isCapping = false - mutex.Unlock() + fcfsMutex.Unlock() log.Println("Framework disconnected with master") } // go routine to cap the entire cluster in regular intervals of time. -var currentCapValue = 0.0 // initial value to indicate that we haven't capped the cluster yet. +var fcfsCurrentCapValue = 0.0 // initial value to indicate that we haven't capped the cluster yet. func (s *ProactiveClusterwideCapFCFS) startCapping() { go func() { for { select { case <-s.ticker.C: - // Need to cap the cluster to the currentCapValue. - mutex.Lock() - if currentCapValue > 0.0 { + // Need to cap the cluster to the fcfsCurrentCapValue. + fcfsMutex.Lock() + if fcfsCurrentCapValue > 0.0 { for _, host := range constants.Hosts { // Rounding curreCapValue to the nearest int. - if err := rapl.Cap(host, "rapl", int(math.Floor(currentCapValue+0.5))); err != nil { + if err := rapl.Cap(host, "rapl", int(math.Floor(fcfsCurrentCapValue+0.5))); err != nil { log.Println(err) } } - log.Printf("Capped the cluster to %d", int(math.Floor(currentCapValue+0.5))) + log.Printf("Capped the cluster to %d", int(math.Floor(fcfsCurrentCapValue+0.5))) } - mutex.Unlock() + fcfsMutex.Unlock() } } }() } // go routine to cap the entire cluster in regular intervals of time. -var recapValue = 0.0 // The cluster wide cap value when recapping. +var fcfsRecapValue = 0.0 // The cluster wide cap value when recapping. func (s *ProactiveClusterwideCapFCFS) startRecapping() { go func() { for { select { case <-s.recapTicker.C: - mutex.Lock() + fcfsMutex.Lock() // If stopped performing cluster wide capping then we need to explicitly cap the entire cluster. - //if !s.isCapping && s.isRecapping && recapValue > 0.0 { - if s.isRecapping && recapValue > 0.0 { + //if !s.isCapping && s.isRecapping && fcfsRecapValue > 0.0 { + if s.isRecapping && fcfsRecapValue > 0.0 { for _, host := range constants.Hosts { // Rounding curreCapValue to the nearest int. - if err := rapl.Cap(host, "rapl", int(math.Floor(recapValue+0.5))); err != nil { + if err := rapl.Cap(host, "rapl", int(math.Floor(fcfsRecapValue+0.5))); err != nil { log.Println(err) } } - log.Printf("Recapped the cluster to %d", int(math.Floor(recapValue+0.5))) + log.Printf("Recapped the cluster to %d", int(math.Floor(fcfsRecapValue+0.5))) } // setting recapping to false s.isRecapping = false - mutex.Unlock() + fcfsMutex.Unlock() } } }() @@ -216,10 +216,10 @@ func (s *ProactiveClusterwideCapFCFS) stopCapping() { if s.isCapping { log.Println("Stopping the cluster wide capping.") s.ticker.Stop() - mutex.Lock() + fcfsMutex.Lock() s.isCapping = false s.isRecapping = true - mutex.Unlock() + fcfsMutex.Unlock() } } @@ -229,9 +229,9 @@ func (s *ProactiveClusterwideCapFCFS) stopRecapping() { if !s.isCapping && s.isRecapping { log.Println("Stopping the cluster wide re-capping.") s.recapTicker.Stop() - mutex.Lock() + fcfsMutex.Lock() s.isRecapping = false - mutex.Unlock() + fcfsMutex.Unlock() } } @@ -270,7 +270,7 @@ func (s *ProactiveClusterwideCapFCFS) ResourceOffers(driver sched.SchedulerDrive For each task in s.tasks, 1. Need to check whether the offer can be taken or not (based on CPU and RAM requirements). 2. If the tasks fits the offer, then I need to detemrine the cluster wide cap. - 3. currentCapValue is updated with the determined cluster wide cap. + 3. fcfsCurrentCapValue is updated with the determined cluster wide cap. Cluster wide capping is currently performed at regular intervals of time. TODO: We can choose to cap the cluster only if the clusterwide cap varies more than the current clusterwide cap. @@ -288,18 +288,18 @@ func (s *ProactiveClusterwideCapFCFS) ResourceOffers(driver sched.SchedulerDrive if s.takeOffer(offer, task) { // Capping the cluster if haven't yet started, if !s.isCapping { - mutex.Lock() + fcfsMutex.Lock() s.isCapping = true - mutex.Unlock() + fcfsMutex.Unlock() s.startCapping() } taken = true tempCap, err := s.capper.fcfsDetermineCap(s.totalPower, &task) if err == nil { - mutex.Lock() - currentCapValue = tempCap - mutex.Unlock() + fcfsMutex.Lock() + fcfsCurrentCapValue = tempCap + fcfsMutex.Unlock() } else { log.Printf("Failed to determine new cluster wide cap: ") log.Println(err) @@ -350,21 +350,22 @@ func (s *ProactiveClusterwideCapFCFS) StatusUpdate(driver sched.SchedulerDriver, s.capper.taskFinished(*status.TaskId.Value) // Determining the new cluster wide cap. tempCap, err := s.capper.recap(s.totalPower, s.taskMonitor, *status.TaskId.Value) + //tempCap, err := s.capper.cleverRecap(s.totalPower, s.taskMonitor, *status.TaskId.Value) if err == nil { // if new determined cap value is different from the current recap value then we need to recap. - if int(math.Floor(tempCap+0.5)) != int(math.Floor(recapValue+0.5)) { - recapValue = tempCap - mutex.Lock() + if int(math.Floor(tempCap+0.5)) != int(math.Floor(fcfsRecapValue+0.5)) { + fcfsRecapValue = tempCap + fcfsMutex.Lock() s.isRecapping = true - mutex.Unlock() - log.Printf("Determined re-cap value: %f\n", recapValue) + fcfsMutex.Unlock() + log.Printf("Determined re-cap value: %f\n", fcfsRecapValue) } else { - mutex.Lock() + fcfsMutex.Lock() s.isRecapping = false - mutex.Unlock() + fcfsMutex.Unlock() } } else { - // Not updating currentCapValue + // Not updating fcfsCurrentCapValue log.Println(err) } @@ -372,7 +373,7 @@ func (s *ProactiveClusterwideCapFCFS) StatusUpdate(driver sched.SchedulerDriver, if s.tasksRunning == 0 { select { case <-s.Shutdown: - // Need to stop the capping process. + // Need to stop the recapping process. s.stopRecapping() close(s.Done) default: From d7a5b1005753922cfb0f694e7965bb5fd32e3e7c Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Tue, 22 Nov 2016 17:07:08 -0500 Subject: [PATCH 34/94] Praoctive cluster wide capping after ranking the tasks based on the requested watts --- .../proactiveclusterwidecappingranked.go | 423 ++++++++++++++++++ 1 file changed, 423 insertions(+) create mode 100644 schedulers/proactiveclusterwidecappingranked.go diff --git a/schedulers/proactiveclusterwidecappingranked.go b/schedulers/proactiveclusterwidecappingranked.go new file mode 100644 index 0000000..0ef4d77 --- /dev/null +++ b/schedulers/proactiveclusterwidecappingranked.go @@ -0,0 +1,423 @@ +/* +Ranked based cluster wide capping. + +Note: Sorting the tasks right in the beginning, in ascending order of watts. + You are hence certain that the tasks that didn't fit are the ones that require more resources, + and hence, you can find a way to address that issue. + On the other hand, if you use first fit to fit the tasks and then sort them to determine the cap, + you are never certain as which tasks are the ones that don't fit and hence, it becomes much harder + to address this issue. +*/ +package schedulers + +import ( + "bitbucket.org/sunybingcloud/electron/constants" + "bitbucket.org/sunybingcloud/electron/def" + "bitbucket.org/sunybingcloud/electron/rapl" + "fmt" + "github.com/golang/protobuf/proto" + mesos "github.com/mesos/mesos-go/mesosproto" + "github.com/mesos/mesos-go/mesosutil" + sched "github.com/mesos/mesos-go/scheduler" + "log" + "math" + "strings" + "sync" + "time" +) + +// Decides if to taken an offer or not +func (_ *ProactiveClusterwideCapRanked) takeOffer(offer *mesos.Offer, task def.Task) bool { + offer_cpu, offer_mem, offer_watts := OfferAgg(offer) + + if offer_cpu >= task.CPU && offer_mem >= task.RAM && offer_watts >= task.Watts { + return true + } + return false +} + +// electronScheduler implements the Scheduler interface +type ProactiveClusterwideCapRanked struct { + tasksCreated int + tasksRunning int + tasks []def.Task + metrics map[string]def.Metric + running map[string]map[string]bool + taskMonitor map[string][]def.Task // store tasks that are currently running. + availablePower map[string]float64 // available power for each node in the cluster. + totalPower map[string]float64 // total power for each node in the cluster. + ignoreWatts bool + capper *clusterwideCapper + ticker *time.Ticker + recapTicker *time.Ticker + isCapping bool // indicate whether we are currently performing cluster wide capping. + isRecapping bool // indicate whether we are currently performing cluster wide re-capping. + + // First set of PCP values are garbage values, signal to logger to start recording when we're + // about to schedule the new task. + RecordPCP bool + + // This channel is closed when the program receives an interrupt, + // signalling that the program should shut down. + Shutdown chan struct{} + + // This channel is closed after shutdown is closed, and only when all + // outstanding tasks have been cleaned up. + Done chan struct{} + + // Controls when to shutdown pcp logging. + PCPLog chan struct{} +} + +// New electron scheduler. +func NewProactiveClusterwideCapRanked(tasks []def.Task, ignoreWatts bool) *ProactiveClusterwideCapRanked { + s := &ProactiveClusterwideCapRanked{ + tasks: tasks, + ignoreWatts: ignoreWatts, + Shutdown: make(chan struct{}), + Done: make(chan struct{}), + PCPLog: make(chan struct{}), + running: make(map[string]map[string]bool), + taskMonitor: make(map[string][]def.Task), + availablePower: make(map[string]float64), + totalPower: make(map[string]float64), + RecordPCP: false, + capper: getClusterwideCapperInstance(), + ticker: time.NewTicker(10 * time.Second), + recapTicker: time.NewTicker(20 * time.Second), + isCapping: false, + isRecapping: false, + } + return s +} + +// mutex +var rankedMutex sync.Mutex + +func (s *ProactiveClusterwideCapRanked) newTask(offer *mesos.Offer, task def.Task) *mesos.TaskInfo { + taskName := fmt.Sprintf("%s-%d", task.Name, *task.Instances) + s.tasksCreated++ + + if !s.RecordPCP { + // Turn on logging. + s.RecordPCP = true + time.Sleep(1 * time.Second) // Make sure we're recording by the time the first task starts + } + + // If this is our first time running into this Agent + if _, ok := s.running[offer.GetSlaveId().GoString()]; !ok { + s.running[offer.GetSlaveId().GoString()] = make(map[string]bool) + } + + // Setting the task ID to the task. This is done so that we can consider each task to be different, + // even though they have the same parameters. + task.SetTaskID(*proto.String("electron-" + taskName)) + // Add task to the list of tasks running on the node. + s.running[offer.GetSlaveId().GoString()][taskName] = true + if len(s.taskMonitor[offer.GetSlaveId().GoString()]) == 0 { + s.taskMonitor[offer.GetSlaveId().GoString()] = []def.Task{task} + } else { + s.taskMonitor[offer.GetSlaveId().GoString()] = append(s.taskMonitor[offer.GetSlaveId().GoString()], task) + } + + resources := []*mesos.Resource{ + mesosutil.NewScalarResource("cpus", task.CPU), + mesosutil.NewScalarResource("mem", task.RAM), + } + + if !s.ignoreWatts { + resources = append(resources, mesosutil.NewScalarResource("watts", task.Watts)) + } + + return &mesos.TaskInfo{ + Name: proto.String(taskName), + TaskId: &mesos.TaskID{ + Value: proto.String("electron-" + taskName), + }, + SlaveId: offer.SlaveId, + Resources: resources, + Command: &mesos.CommandInfo{ + Value: proto.String(task.CMD), + }, + Container: &mesos.ContainerInfo{ + Type: mesos.ContainerInfo_DOCKER.Enum(), + Docker: &mesos.ContainerInfo_DockerInfo{ + Image: proto.String(task.Image), + Network: mesos.ContainerInfo_DockerInfo_BRIDGE.Enum(), // Run everything isolated + }, + }, + } +} + +func (s *ProactiveClusterwideCapRanked) Registered( + _ sched.SchedulerDriver, + frameworkID *mesos.FrameworkID, + masterInfo *mesos.MasterInfo) { + log.Printf("Framework %s registered with master %s", frameworkID, masterInfo) +} + +func (s *ProactiveClusterwideCapRanked) Reregistered(_ sched.SchedulerDriver, masterInfo *mesos.MasterInfo) { + log.Printf("Framework re-registered with master %s", masterInfo) +} + +func (s *ProactiveClusterwideCapRanked) Disconnected(sched.SchedulerDriver) { + // Need to stop the capping process. + s.ticker.Stop() + s.recapTicker.Stop() + rankedMutex.Lock() + s.isCapping = false + rankedMutex.Unlock() + log.Println("Framework disconnected with master") +} + +// go routine to cap the entire cluster in regular intervals of time. +var rankedCurrentCapValue = 0.0 // initial value to indicate that we haven't capped the cluster yet. +func (s *ProactiveClusterwideCapRanked) startCapping() { + go func() { + for { + select { + case <-s.ticker.C: + // Need to cap the cluster to the rankedCurrentCapValue. + rankedMutex.Lock() + if rankedCurrentCapValue > 0.0 { + for _, host := range constants.Hosts { + // Rounding curreCapValue to the nearest int. + if err := rapl.Cap(host, "rapl", int(math.Floor(rankedCurrentCapValue+0.5))); err != nil { + log.Println(err) + } + } + log.Printf("Capped the cluster to %d", int(math.Floor(rankedCurrentCapValue+0.5))) + } + rankedMutex.Unlock() + } + } + }() +} + +// go routine to cap the entire cluster in regular intervals of time. +var rankedRecapValue = 0.0 // The cluster wide cap value when recapping. +func (s *ProactiveClusterwideCapRanked) startRecapping() { + go func() { + for { + select { + case <-s.recapTicker.C: + rankedMutex.Lock() + // If stopped performing cluster wide capping then we need to explicitly cap the entire cluster. + //if !s.isCapping && s.isRecapping && rankedRecapValue > 0.0 { + if s.isRecapping && rankedRecapValue > 0.0 { + for _, host := range constants.Hosts { + // Rounding curreCapValue to the nearest int. + if err := rapl.Cap(host, "rapl", int(math.Floor(rankedRecapValue+0.5))); err != nil { + log.Println(err) + } + } + log.Printf("Recapped the cluster to %d", int(math.Floor(rankedRecapValue+0.5))) + } + // setting recapping to false + s.isRecapping = false + rankedMutex.Unlock() + } + } + }() +} + +// Stop cluster wide capping +func (s *ProactiveClusterwideCapRanked) stopCapping() { + log.Println("Stopping the cluster wide capping.") + s.ticker.Stop() + rankedMutex.Lock() + s.isCapping = false + s.isRecapping = true + rankedMutex.Unlock() +} + +// Stop cluster wide Recapping +func (s *ProactiveClusterwideCapRanked) stopRecapping() { + log.Println("Stopping the cluster wide re-capping.") + s.recapTicker.Stop() + rankedMutex.Lock() + s.isRecapping = false + rankedMutex.Unlock() +} + +func (s *ProactiveClusterwideCapRanked) ResouceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { + log.Printf("Received %d resource offers", len(offers)) + + // retrieving the available power for all the hosts in the offers. + for _, offer := range offers { + _, _, offer_watts := OfferAgg(offer) + s.availablePower[*offer.Hostname] = offer_watts + // setting total power if the first time. + if _, ok := s.totalPower[*offer.Hostname]; !ok { + s.totalPower[*offer.Hostname] = offer_watts + } + } + + for host, tpower := range s.totalPower { + log.Printf("TotalPower[%s] = %f", host, tpower) + } + + // sorting the tasks in ascending order of watts. + s.capper.sort_tasks(&s.tasks) + // displaying the ranked tasks. + log.Println("The ranked tasks are:\n---------------------\n\t[") + for rank, task := range s.tasks { + log.Printf("\t\t%d: %s\n", rank+1, task.TaskID) + } + log.Println("\t]") + + for _, offer := range offers { + select { + case <-s.Shutdown: + log.Println("Done scheduling tasks: declining offer on [", offer.GetHostname(), "]") + driver.DeclineOffer(offer.Id, longFilter) + + log.Println("Number of tasks still running: ", s.tasksRunning) + continue + default: + } + + /* + Ranked cluster wide capping strategy + + For each task in the sorted tasks, + 1. Need to check whether the offer can be taken or not (based on CPU, RAM and WATTS requirements). + 2. If the task fits the offer, then need to determine the cluster wide cap.' + 3. rankedCurrentCapValue is updated with the determined cluster wide cap. + + Once we are done scheduling all the tasks, + we start recalculating the cluster wide cap each time a task finishes. + + Cluster wide capping is currently performed at regular intervals of time. + */ + taken := false + + for i, task := range s.tasks { + // Don't take offer if it doesn't match our task's host requirement. + if !strings.HasPrefix(*offer.Hostname, task.Host) { + continue + } + + // Does the task fit. + if s.takeOffer(offer, task) { + // Capping the cluster if haven't yet started + if !s.isCapping { + rankedMutex.Lock() + s.isCapping = true + rankedMutex.Unlock() + s.startCapping() + } + taken = true + tempCap, err := s.capper.fcfsDetermineCap(s.totalPower, &task) + + if err == nil { + rankedMutex.Lock() + rankedCurrentCapValue = tempCap + rankedMutex.Unlock() + } else { + log.Println("Failed to determine the new cluster wide cap: ", err) + } + log.Printf("Starting on [%s]\n", offer.GetHostname()) + to_schedule := []*mesos.TaskInfo{s.newTask(offer, task)} + driver.LaunchTasks([]*mesos.OfferID{offer.Id}, to_schedule, defaultFilter) + log.Printf("Inst: %d", *task.Instances) + *task.Instances-- + if *task.Instances <= 0 { + // All instances of the task have been scheduled. Need to remove it from the list of tasks to schedule. + s.tasks[i] = s.tasks[len(s.tasks)-1] + s.tasks = s.tasks[:len(s.tasks)-1] + + if len(s.tasks) <= 0 { + log.Println("Done scheduling all tasks") + // Need to stop the cluster wide capping as there aren't any more tasks to schedule. + s.stopCapping() + s.startRecapping() + close(s.Shutdown) + } + } + break // Offer taken, move on. + } else { + // Task doesn't fit the offer. Move onto the next offer. + } + } + + // If no tasks fit the offer, then declining the offer. + if !taken { + log.Printf("There is not enough resources to launch a task on Host: %s\n", offer.GetHostname()) + cpus, mem, watts := OfferAgg(offer) + + log.Printf("\n", cpus, mem, watts) + driver.DeclineOffer(offer.Id, defaultFilter) + } + } +} + +func (s *ProactiveClusterwideCapRanked) StatusUpdate(driver sched.SchedulerDriver, status *mesos.TaskStatus) { + log.Printf("Received task status [%s] for task [%s]\n", NameFor(status.State), *status.TaskId.Value) + + if *status.State == mesos.TaskState_TASK_RUNNING { + s.tasksRunning++ + } else if IsTerminal(status.State) { + delete(s.running[status.GetSlaveId().GoString()], *status.TaskId.Value) + s.tasksRunning-- + if s.tasksRunning == 0 { + select { + case <-s.Shutdown: + // Need to stop the recapping process. + s.stopRecapping() + close(s.Done) + default: + } + } else { + // Need to remove the task from the window + s.capper.taskFinished(*status.TaskId.Value) + // Determining the new cluster wide cap. + tempCap, err := s.capper.recap(s.totalPower, s.taskMonitor, *status.TaskId.Value) + // tempCap, err := s.capper.cleverRecap(s.totalPower, s.taskMonitor, *status.TaskId.Value) + + if err == nil { + // If new determined cap value is different from the current recap value then we need to recap. + if int(math.Floor(tempCap+0.5)) != int(math.Floor(rankedRecapValue+0.5)) { + rankedRecapValue = tempCap + rankedMutex.Lock() + s.isRecapping = true + rankedMutex.Unlock() + log.Printf("Determined re-cap value: %f\n", rankedRecapValue) + } else { + rankedMutex.Lock() + s.isRecapping = false + rankedMutex.Unlock() + } + } else { + // Not updating rankedCurrentCapValue + log.Println(err) + } + } + } + log.Printf("DONE: Task status [%s] for task [%s]", NameFor(status.State), *status.TaskId.Value) +} + +func (s *ProactiveClusterwideCapRanked) FrameworkMessage(driver sched.SchedulerDriver, + executorID *mesos.ExecutorID, + slaveID *mesos.SlaveID, + message string) { + + log.Println("Getting a framework message: ", message) + log.Printf("Received a framework message from some unknown source: %s", *executorID.Value) +} + +func (s *ProactiveClusterwideCapRanked) OfferRescinded(_ sched.SchedulerDriver, offerID *mesos.OfferID) { + log.Printf("Offer %s rescinded", offerID) +} + +func (s *ProactiveClusterwideCapRanked) SlaveLost(_ sched.SchedulerDriver, slaveID *mesos.SlaveID) { + log.Printf("Slave %s lost", slaveID) +} + +func (s *ProactiveClusterwideCapRanked) ExecutorLost(_ sched.SchedulerDriver, executorID *mesos.ExecutorID, slaveID *mesos.SlaveID, status int) { + log.Printf("Executor %s on slave %s was lost", executorID, slaveID) +} + +func (s *ProactiveClusterwideCapRanked) Error(_ sched.SchedulerDriver, err string) { + log.Printf("Receiving an error: %s", err) +} From e8db16282095111d9e831c15ac64bf2266a82108 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Tue, 22 Nov 2016 17:08:27 -0500 Subject: [PATCH 35/94] changed the window size and capmargin to create differernt configurations. No chnage made to the code. --- constants/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/constants/constants.go b/constants/constants.go index cc6d705..6563715 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -34,7 +34,7 @@ var Power_threshold = 0.6 // Right now saying that a task will never be given le So, if power required = 10W, the node would be capped to 75%*10W. This value can be changed upon convenience. */ -var Cap_margin = 0.70 +var Cap_margin = 0.7 // Modify the cap margin. func UpdateCapMargin(new_cap_margin float64) bool { From cceedad1f0c3636a69dd70c7f3026e942e9701c3 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Wed, 23 Nov 2016 19:18:19 -0500 Subject: [PATCH 36/94] Made a mention to use --help option to get more information about the other command line options --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ae50170..aa625e4 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ machine on which electron is launched for logging to work** -How to run: +How to run (Use the --help option to get information about other command-line options): `./electron -workload -ignoreWatts ` From abdfef2044ba00a65e729b9d54891fbe8b49e36f Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Wed, 23 Nov 2016 19:18:56 -0500 Subject: [PATCH 37/94] README to list the different scheduling algorithms defined --- schedulers/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 schedulers/README.md diff --git a/schedulers/README.md b/schedulers/README.md new file mode 100644 index 0000000..7d725ae --- /dev/null +++ b/schedulers/README.md @@ -0,0 +1,11 @@ +Electron: Scheduling Algorithms +================================ + +To Do: + * Design changes -- Possible to have one scheduler with different scheduling schemes? + +Scheduling Algorithms: + * Bin-packing with sorted watts + * FCFS Proactive Cluster-wide Capping + * First Fit + * First Fit with sorted watts From 5f32074c4c915070e008aaef9010ebfbdd32e086 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Wed, 23 Nov 2016 19:20:22 -0500 Subject: [PATCH 38/94] Fixed an error in markdown --- schedulers/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/schedulers/README.md b/schedulers/README.md index 7d725ae..58b9f04 100644 --- a/schedulers/README.md +++ b/schedulers/README.md @@ -5,6 +5,7 @@ To Do: * Design changes -- Possible to have one scheduler with different scheduling schemes? Scheduling Algorithms: + * Bin-packing with sorted watts * FCFS Proactive Cluster-wide Capping * First Fit From 44d355523bf304afaacb7b76ef179312715c143c Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Wed, 23 Nov 2016 19:24:27 -0500 Subject: [PATCH 39/94] Markdown fix in README --- schedulers/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/schedulers/README.md b/schedulers/README.md index 58b9f04..15c0d74 100644 --- a/schedulers/README.md +++ b/schedulers/README.md @@ -2,6 +2,7 @@ Electron: Scheduling Algorithms ================================ To Do: + * Design changes -- Possible to have one scheduler with different scheduling schemes? Scheduling Algorithms: From 43c173c60b8c0e5e78b769ad552d6a67431bf20d Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Fri, 25 Nov 2016 16:04:11 -0500 Subject: [PATCH 40/94] changed the hosts from stratos-00x to stratos-00x.cs.binghamton.edu --- constants/constants.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/constants/constants.go b/constants/constants.go index 6563715..5dc98ea 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -10,10 +10,10 @@ Also, exposing functions to update or initialize some of the constants. */ package constants -var Hosts = []string{"stratos-001", "stratos-002", - "stratos-003", "stratos-004", - "stratos-005", "stratos-006", - "stratos-007", "stratos-008"} +var Hosts = []string{"stratos-001.cs.binghamton.edu", "stratos-002.cs.binghamton.edu", + "stratos-003.cs.binghamton.edu", "stratos-004.cs.binghamton.edu", + "stratos-005.cs.binghamton.edu", "stratos-006.cs.binghamton.edu", + "stratos-007.cs.binghamton.edu", "stratos-008.cs.binghamton.edu"} // Add a new host to the slice of hosts. func AddNewHost(new_host string) bool { @@ -34,7 +34,7 @@ var Power_threshold = 0.6 // Right now saying that a task will never be given le So, if power required = 10W, the node would be capped to 75%*10W. This value can be changed upon convenience. */ -var Cap_margin = 0.7 +var Cap_margin = 0.50 // Modify the cap margin. func UpdateCapMargin(new_cap_margin float64) bool { @@ -73,7 +73,7 @@ func AddTotalPowerForHost(host string, total_power float64) bool { } // Window size for running average -var Window_size = 10 +var Window_size = 160 // Update the window size. func UpdateWindowSize(new_window_size int) bool { From 7bea56206b0ef2d733b9717118151257c828dab2 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Fri, 25 Nov 2016 16:05:55 -0500 Subject: [PATCH 41/94] fixed bug in cleverRecap(...). Now we switch from the primitive recap to the clever recap as the cap determined by the later would be lesser when the cluster is relatively idle. --- schedulers/proactiveclusterwidecappers.go | 91 ++++++++++++++--------- 1 file changed, 57 insertions(+), 34 deletions(-) diff --git a/schedulers/proactiveclusterwidecappers.go b/schedulers/proactiveclusterwidecappers.go index 6da6873..505ac76 100644 --- a/schedulers/proactiveclusterwidecappers.go +++ b/schedulers/proactiveclusterwidecappers.go @@ -113,18 +113,11 @@ func (capper clusterwideCapper) get_cap(running_average_to_total_power_percentag } /* -Recapping the entire cluster. Also, removing the finished task from the list of running tasks. +A recapping strategy which decides between 2 different recapping schemes. +1. the regular scheme based on the average power usage across the cluster. +2. A scheme based on the average of the loads on each node in the cluster. -We would, at this point, have a better knowledge about the state of the cluster. - -1. Calculate the total allocated watts per node in the cluster. -2. Compute the ratio of the total watts usage per node to the total power for that node. - This would give us the load on that node. -3. Now, compute the average load across all the nodes in the cluster. - This would be the cap value. - -Note: Although this would ensure lesser power usage, it might increase makespan if there is a heavy workload on just one node. -TODO: return a map[string]float64 that contains the recap value per node. This way, we can provide the right amount of power per node. +The recap value picked the least among the two. */ func (capper clusterwideCapper) cleverRecap(total_power map[string]float64, task_monitor map[string][]def.Task, finished_taskId string) (float64, error) { @@ -132,49 +125,79 @@ func (capper clusterwideCapper) cleverRecap(total_power map[string]float64, if total_power == nil || task_monitor == nil { return 100.0, errors.New("Invalid argument: total_power, task_monitor") } + + // determining the recap value by calling the regular recap(...) + toggle := false + recapValue, err := capper.recap(total_power, task_monitor, finished_taskId) + if err == nil { + toggle = true + } + // watts usage on each node in the cluster. watts_usages := make(map[string][]float64) host_of_finished_task := "" - index_of_finished_task := -1 + index_of_finished_task := -1 for _, host := range constants.Hosts { watts_usages[host] = []float64{0.0} } for host, tasks := range task_monitor { for i, task := range tasks { if task.TaskID == finished_taskId { - host_of_finished_task = host - index_of_finished_task = i - // Not considering this task - continue - } + host_of_finished_task = host + index_of_finished_task = i + // Not considering this task for the computation of total_allocated_power and total_running_tasks + continue + } watts_usages[host] = append(watts_usages[host], float64(task.Watts) * constants.Cap_margin) } } - // Updating task monitor + // Updating task monitor. If recap(...) has deleted the finished task from the taskMonitor, + // then this will be ignored. if host_of_finished_task != "" && index_of_finished_task != -1 { log.Printf("Removing task with task [%s] from the list of running tasks\n", - task_monitor[host_of_finished_task][index_of_finished_task].TaskID) + task_monitor[host_of_finished_task][index_of_finished_task].TaskID) task_monitor[host_of_finished_task] = append(task_monitor[host_of_finished_task][:index_of_finished_task], - task_monitor[host_of_finished_task][index_of_finished_task+1:]...) + task_monitor[host_of_finished_task][index_of_finished_task+1:]...) } - // load on each node in the cluster. - loads := []float64{} - for host, usages := range watts_usages { - total_usage := 0.0 - for _, usage := range usages { - total_usage += usage + // Need to check whether there are still tasks running on the cluster. If not then we return an error. + clusterIdle := true + for _, tasks := range task_monitor { + if len(tasks) > 0 { + clusterIdle = false + } + } + + if !clusterIdle { + // load on each node in the cluster. + loads := []float64{0.0} + for host, usages := range watts_usages { + total_usage := 0.0 + for _, usage := range usages { + total_usage += usage + } + loads = append(loads, total_usage / total_power[host]) + } + + // Now need to compute the average load. + total_load := 0.0 + for _, load := range loads { + total_load += load + } + average_load := (total_load / float64(len(loads)) * 100.0) // this would be the cap value. + // If toggle is true, then we need to return the least recap value. + if toggle { + if average_load <= recapValue { + return average_load, nil + } else { + return recapValue, nil + } + } else { + return average_load, nil } - loads = append(loads, total_usage / total_power[host]) } - // Now need to compute the average load. - total_load := 0.0 - for _, load := range loads { - total_load += load - } - average_load := total_load / float64(len(loads)) // this would be the cap value. - return average_load, nil + return 100.0, errors.New("No task running on the cluster.") } /* From 948e00bfeb8e8cb75af686c0057e5d58ed0e8e86 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Fri, 25 Nov 2016 16:19:45 -0500 Subject: [PATCH 42/94] changed the keys in taskMonitor from offer.SlaveId() to offer.Hostname. --- schedulers/proactiveclusterwidecappingfcfs.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/schedulers/proactiveclusterwidecappingfcfs.go b/schedulers/proactiveclusterwidecappingfcfs.go index c352cb1..c749071 100644 --- a/schedulers/proactiveclusterwidecappingfcfs.go +++ b/schedulers/proactiveclusterwidecappingfcfs.go @@ -104,10 +104,10 @@ func (s *ProactiveClusterwideCapFCFS) newTask(offer *mesos.Offer, task def.Task) task.SetTaskID(*proto.String("electron-" + taskName)) // Add task to the list of tasks running on the node. s.running[offer.GetSlaveId().GoString()][taskName] = true - if len(s.taskMonitor[offer.GetSlaveId().GoString()]) == 0 { - s.taskMonitor[offer.GetSlaveId().GoString()] = []def.Task{task} + if len(s.taskMonitor[*offer.Hostname]) == 0 { + s.taskMonitor[*offer.Hostname] = []def.Task{task} } else { - s.taskMonitor[offer.GetSlaveId().GoString()] = append(s.taskMonitor[offer.GetSlaveId().GoString()], task) + s.taskMonitor[*offer.Hostname] = append(s.taskMonitor[*offer.Hostname], task) } resources := []*mesos.Resource{ @@ -349,8 +349,8 @@ func (s *ProactiveClusterwideCapFCFS) StatusUpdate(driver sched.SchedulerDriver, // Need to remove the task from the window of tasks. s.capper.taskFinished(*status.TaskId.Value) // Determining the new cluster wide cap. - tempCap, err := s.capper.recap(s.totalPower, s.taskMonitor, *status.TaskId.Value) - //tempCap, err := s.capper.cleverRecap(s.totalPower, s.taskMonitor, *status.TaskId.Value) + //tempCap, err := s.capper.recap(s.totalPower, s.taskMonitor, *status.TaskId.Value) + tempCap, err := s.capper.cleverRecap(s.totalPower, s.taskMonitor, *status.TaskId.Value) if err == nil { // if new determined cap value is different from the current recap value then we need to recap. if int(math.Floor(tempCap+0.5)) != int(math.Floor(fcfsRecapValue+0.5)) { From 892d9b86dc19f946846081c2bf11f1f232666aad Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Fri, 25 Nov 2016 17:42:08 -0500 Subject: [PATCH 43/94] formatted the code --- constants/constants.go | 79 +++++++------ schedulers/proactiveclusterwidecappers.go | 105 +++++++++--------- schedulers/proactiveclusterwidecappingfcfs.go | 2 +- utilities/utils.go | 38 +++---- 4 files changed, 113 insertions(+), 111 deletions(-) diff --git a/constants/constants.go b/constants/constants.go index 5dc98ea..2d9b516 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -11,19 +11,19 @@ Also, exposing functions to update or initialize some of the constants. package constants var Hosts = []string{"stratos-001.cs.binghamton.edu", "stratos-002.cs.binghamton.edu", - "stratos-003.cs.binghamton.edu", "stratos-004.cs.binghamton.edu", - "stratos-005.cs.binghamton.edu", "stratos-006.cs.binghamton.edu", - "stratos-007.cs.binghamton.edu", "stratos-008.cs.binghamton.edu"} + "stratos-003.cs.binghamton.edu", "stratos-004.cs.binghamton.edu", + "stratos-005.cs.binghamton.edu", "stratos-006.cs.binghamton.edu", + "stratos-007.cs.binghamton.edu", "stratos-008.cs.binghamton.edu"} // Add a new host to the slice of hosts. func AddNewHost(new_host string) bool { - // Validation - if new_host == "" { - return false - } else { - Hosts = append(Hosts, new_host) - return true - } + // Validation + if new_host == "" { + return false + } else { + Hosts = append(Hosts, new_host) + return true + } } // Lower bound of the percentage of requested power, that can be allocated to a task. @@ -38,16 +38,15 @@ var Cap_margin = 0.50 // Modify the cap margin. func UpdateCapMargin(new_cap_margin float64) bool { - // Checking if the new_cap_margin is less than the power threshold. - if new_cap_margin < Starvation_factor { - return false - } else { - Cap_margin = new_cap_margin - return true - } + // Checking if the new_cap_margin is less than the power threshold. + if new_cap_margin < Starvation_factor { + return false + } else { + Cap_margin = new_cap_margin + return true + } } - // Threshold factor that would make (Cap_margin * task.Watts) equal to (60/100 * task.Watts). var Starvation_factor = 0.8 @@ -56,32 +55,32 @@ var Total_power map[string]float64 // Initialize the total power per node. This should be done before accepting any set of tasks for scheduling. func AddTotalPowerForHost(host string, total_power float64) bool { - // Validation - is_correct_host := false - for _, existing_host := range Hosts { - if host == existing_host { - is_correct_host = true - } - } + // Validation + is_correct_host := false + for _, existing_host := range Hosts { + if host == existing_host { + is_correct_host = true + } + } - if !is_correct_host { - return false - } else { - Total_power[host] = total_power - return true - } + if !is_correct_host { + return false + } else { + Total_power[host] = total_power + return true + } } // Window size for running average -var Window_size = 160 +var Window_size = 10 // Update the window size. func UpdateWindowSize(new_window_size int) bool { - // Validation - if new_window_size == 0 { - return false - } else{ - Window_size = new_window_size - return true - } -} \ No newline at end of file + // Validation + if new_window_size == 0 { + return false + } else { + Window_size = new_window_size + return true + } +} diff --git a/schedulers/proactiveclusterwidecappers.go b/schedulers/proactiveclusterwidecappers.go index 505ac76..5de8a47 100644 --- a/schedulers/proactiveclusterwidecappers.go +++ b/schedulers/proactiveclusterwidecappers.go @@ -17,7 +17,7 @@ import ( "container/list" "errors" "github.com/montanaflynn/stats" - "log" + "log" "sort" ) @@ -118,8 +118,11 @@ A recapping strategy which decides between 2 different recapping schemes. 2. A scheme based on the average of the loads on each node in the cluster. The recap value picked the least among the two. + +The cleverRecap scheme works well when the cluster is relatively idle and until then, + the primitive recapping scheme works better. */ -func (capper clusterwideCapper) cleverRecap(total_power map[string]float64, +func (capper clusterwideCapper) cleverRecap(total_power map[string]float64, task_monitor map[string][]def.Task, finished_taskId string) (float64, error) { // Validation if total_power == nil || task_monitor == nil { @@ -136,48 +139,48 @@ func (capper clusterwideCapper) cleverRecap(total_power map[string]float64, // watts usage on each node in the cluster. watts_usages := make(map[string][]float64) host_of_finished_task := "" - index_of_finished_task := -1 + index_of_finished_task := -1 for _, host := range constants.Hosts { watts_usages[host] = []float64{0.0} } for host, tasks := range task_monitor { for i, task := range tasks { if task.TaskID == finished_taskId { - host_of_finished_task = host - index_of_finished_task = i - // Not considering this task for the computation of total_allocated_power and total_running_tasks - continue - } - watts_usages[host] = append(watts_usages[host], float64(task.Watts) * constants.Cap_margin) + host_of_finished_task = host + index_of_finished_task = i + // Not considering this task for the computation of total_allocated_power and total_running_tasks + continue + } + watts_usages[host] = append(watts_usages[host], float64(task.Watts)*constants.Cap_margin) } } // Updating task monitor. If recap(...) has deleted the finished task from the taskMonitor, - // then this will be ignored. - if host_of_finished_task != "" && index_of_finished_task != -1 { - log.Printf("Removing task with task [%s] from the list of running tasks\n", - task_monitor[host_of_finished_task][index_of_finished_task].TaskID) - task_monitor[host_of_finished_task] = append(task_monitor[host_of_finished_task][:index_of_finished_task], - task_monitor[host_of_finished_task][index_of_finished_task+1:]...) - } + // then this will be ignored. Else (this is only when an error occured with recap(...)), we remove it here. + if host_of_finished_task != "" && index_of_finished_task != -1 { + log.Printf("Removing task with task [%s] from the list of running tasks\n", + task_monitor[host_of_finished_task][index_of_finished_task].TaskID) + task_monitor[host_of_finished_task] = append(task_monitor[host_of_finished_task][:index_of_finished_task], + task_monitor[host_of_finished_task][index_of_finished_task+1:]...) + } - // Need to check whether there are still tasks running on the cluster. If not then we return an error. - clusterIdle := true - for _, tasks := range task_monitor { - if len(tasks) > 0 { - clusterIdle = false - } - } + // Need to check whether there are still tasks running on the cluster. If not then we return an error. + clusterIdle := true + for _, tasks := range task_monitor { + if len(tasks) > 0 { + clusterIdle = false + } + } - if !clusterIdle { - // load on each node in the cluster. + if !clusterIdle { + // load on each node in the cluster. loads := []float64{0.0} for host, usages := range watts_usages { total_usage := 0.0 for _, usage := range usages { total_usage += usage } - loads = append(loads, total_usage / total_power[host]) + loads = append(loads, total_usage/total_power[host]) } // Now need to compute the average load. @@ -219,33 +222,33 @@ func (capper clusterwideCapper) recap(total_power map[string]float64, total_allocated_power := 0.0 total_running_tasks := 0 - host_of_finished_task := "" - index_of_finished_task := -1 - for host, tasks := range task_monitor { - for i, task := range tasks { - if task.TaskID == finished_taskId { - host_of_finished_task = host - index_of_finished_task = i - // Not considering this task for the computation of total_allocated_power and total_running_tasks - continue - } - total_allocated_power += (float64(task.Watts) * constants.Cap_margin) - total_running_tasks++ - } - } + host_of_finished_task := "" + index_of_finished_task := -1 + for host, tasks := range task_monitor { + for i, task := range tasks { + if task.TaskID == finished_taskId { + host_of_finished_task = host + index_of_finished_task = i + // Not considering this task for the computation of total_allocated_power and total_running_tasks + continue + } + total_allocated_power += (float64(task.Watts) * constants.Cap_margin) + total_running_tasks++ + } + } - // Updating task monitor - if host_of_finished_task != "" && index_of_finished_task != -1 { - log.Printf("Removing task with task [%s] from the list of running tasks\n", - task_monitor[host_of_finished_task][index_of_finished_task].TaskID) - task_monitor[host_of_finished_task] = append(task_monitor[host_of_finished_task][:index_of_finished_task], - task_monitor[host_of_finished_task][index_of_finished_task+1:]...) - } + // Updating task monitor + if host_of_finished_task != "" && index_of_finished_task != -1 { + log.Printf("Removing task with task [%s] from the list of running tasks\n", + task_monitor[host_of_finished_task][index_of_finished_task].TaskID) + task_monitor[host_of_finished_task] = append(task_monitor[host_of_finished_task][:index_of_finished_task], + task_monitor[host_of_finished_task][index_of_finished_task+1:]...) + } - // For the last task, total_allocated_power and total_running_tasks would be 0 - if total_allocated_power == 0 && total_running_tasks == 0 { - return 100, errors.New("No task running on the cluster.") - } + // For the last task, total_allocated_power and total_running_tasks would be 0 + if total_allocated_power == 0 && total_running_tasks == 0 { + return 100, errors.New("No task running on the cluster.") + } average := total_allocated_power / float64(total_running_tasks) ratios := []float64{} diff --git a/schedulers/proactiveclusterwidecappingfcfs.go b/schedulers/proactiveclusterwidecappingfcfs.go index c749071..4a13574 100644 --- a/schedulers/proactiveclusterwidecappingfcfs.go +++ b/schedulers/proactiveclusterwidecappingfcfs.go @@ -76,7 +76,7 @@ func NewProactiveClusterwideCapFCFS(tasks []def.Task, ignoreWatts bool) *Proacti ticker: time.NewTicker(10 * time.Second), recapTicker: time.NewTicker(20 * time.Second), isCapping: false, - isRecapping: false, + isRecapping: false, } return s } diff --git a/utilities/utils.go b/utilities/utils.go index d6406d6..ede4f64 100644 --- a/utilities/utils.go +++ b/utilities/utils.go @@ -9,8 +9,8 @@ https://groups.google.com/forum/#!topic/golang-nuts/FT7cjmcL7gw // Utility struct that helps in sorting the available power by value. type Pair struct { - Key string - Value float64 + Key string + Value float64 } // A slice of pairs that implements the sort.Interface to sort by value. @@ -18,37 +18,37 @@ type PairList []Pair // Swap pairs in the PairList func (plist PairList) Swap(i, j int) { - plist[i], plist[j] = plist[j], plist[i] + plist[i], plist[j] = plist[j], plist[i] } // function to return the length of the pairlist. func (plist PairList) Len() int { - return len(plist) + return len(plist) } // function to compare two elements in pairlist. func (plist PairList) Less(i, j int) bool { - return plist[i].Value < plist[j].Value + return plist[i].Value < plist[j].Value } // convert a PairList to a map[string]float64 func OrderedKeys(plist PairList) ([]string, error) { - // Validation - if plist == nil { - return nil, errors.New("Invalid argument: plist") - } - ordered_keys := make([]string, len(plist)) - for _, pair := range plist { - ordered_keys = append(ordered_keys, pair.Key) - } - return ordered_keys, nil + // Validation + if plist == nil { + return nil, errors.New("Invalid argument: plist") + } + ordered_keys := make([]string, len(plist)) + for _, pair := range plist { + ordered_keys = append(ordered_keys, pair.Key) + } + return ordered_keys, nil } // determine the max value func Max(a, b float64) float64 { - if a > b { - return a - } else { - return b - } + if a > b { + return a + } else { + return b + } } From f67773dcde031aaec34af53e612c15fe8bf6257e Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Fri, 25 Nov 2016 18:00:55 -0500 Subject: [PATCH 44/94] Added proactive cluster wide capping with ranked tasks as another scheduler. --- .../proactiveclusterwidecappingranked.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/schedulers/proactiveclusterwidecappingranked.go b/schedulers/proactiveclusterwidecappingranked.go index 0ef4d77..a6f8aa8 100644 --- a/schedulers/proactiveclusterwidecappingranked.go +++ b/schedulers/proactiveclusterwidecappingranked.go @@ -38,11 +38,11 @@ func (_ *ProactiveClusterwideCapRanked) takeOffer(offer *mesos.Offer, task def.T // electronScheduler implements the Scheduler interface type ProactiveClusterwideCapRanked struct { - tasksCreated int - tasksRunning int - tasks []def.Task - metrics map[string]def.Metric - running map[string]map[string]bool + tasksCreated int + tasksRunning int + tasks []def.Task + metrics map[string]def.Metric + running map[string]map[string]bool taskMonitor map[string][]def.Task // store tasks that are currently running. availablePower map[string]float64 // available power for each node in the cluster. totalPower map[string]float64 // total power for each node in the cluster. @@ -86,7 +86,7 @@ func NewProactiveClusterwideCapRanked(tasks []def.Task, ignoreWatts bool) *Proac ticker: time.NewTicker(10 * time.Second), recapTicker: time.NewTicker(20 * time.Second), isCapping: false, - isRecapping: false, + isRecapping: false, } return s } @@ -114,10 +114,10 @@ func (s *ProactiveClusterwideCapRanked) newTask(offer *mesos.Offer, task def.Tas task.SetTaskID(*proto.String("electron-" + taskName)) // Add task to the list of tasks running on the node. s.running[offer.GetSlaveId().GoString()][taskName] = true - if len(s.taskMonitor[offer.GetSlaveId().GoString()]) == 0 { - s.taskMonitor[offer.GetSlaveId().GoString()] = []def.Task{task} + if len(s.taskMonitor[*offer.Hostname]) == 0 { + s.taskMonitor[*offer.Hostname] = []def.Task{task} } else { - s.taskMonitor[offer.GetSlaveId().GoString()] = append(s.taskMonitor[offer.GetSlaveId().GoString()], task) + s.taskMonitor[*offer.Hostname] = append(s.taskMonitor[*offer.Hostname], task) } resources := []*mesos.Resource{ From f60661023261582ddb97b89a31aee1d8ff991777 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Fri, 25 Nov 2016 18:02:21 -0500 Subject: [PATCH 45/94] Added the proactive cluster wide capping with ranked tasks scheduler to the list of schedulers. --- schedulers/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/schedulers/README.md b/schedulers/README.md index 15c0d74..a61559b 100644 --- a/schedulers/README.md +++ b/schedulers/README.md @@ -9,5 +9,6 @@ Scheduling Algorithms: * Bin-packing with sorted watts * FCFS Proactive Cluster-wide Capping + * Ranked Proactive Cluster-wide Capping * First Fit * First Fit with sorted watts From 7ef10e63109c4745644ef8184b7494045fafbb00 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Fri, 25 Nov 2016 19:32:50 -0500 Subject: [PATCH 46/94] Removed a comment --- utilities/utils.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/utilities/utils.go b/utilities/utils.go index ede4f64..b953446 100644 --- a/utilities/utils.go +++ b/utilities/utils.go @@ -2,11 +2,6 @@ package utilities import "errors" -/* -The Pair and PairList have been taken from google groups forum, -https://groups.google.com/forum/#!topic/golang-nuts/FT7cjmcL7gw -*/ - // Utility struct that helps in sorting the available power by value. type Pair struct { Key string From 262b13e6b7ff22a8b974517c2fe1ed7a8c24b66f Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Fri, 25 Nov 2016 20:21:01 -0500 Subject: [PATCH 47/94] Revert "Removed a comment" This reverts commit fcdffb5c1006d7938d8be4aacad7ec5a7b78f20e. --- utilities/utils.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/utilities/utils.go b/utilities/utils.go index b953446..ede4f64 100644 --- a/utilities/utils.go +++ b/utilities/utils.go @@ -2,6 +2,11 @@ package utilities import "errors" +/* +The Pair and PairList have been taken from google groups forum, +https://groups.google.com/forum/#!topic/golang-nuts/FT7cjmcL7gw +*/ + // Utility struct that helps in sorting the available power by value. type Pair struct { Key string From 978d51135bbece258c99d556cf435ea083c23150 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 28 Nov 2016 16:29:12 -0500 Subject: [PATCH 48/94] Removed a commented line. --- schedulers/proactiveclusterwidecappingfcfs.go | 1 - schedulers/proactiveclusterwidecappingranked.go | 1 - 2 files changed, 2 deletions(-) diff --git a/schedulers/proactiveclusterwidecappingfcfs.go b/schedulers/proactiveclusterwidecappingfcfs.go index 4a13574..5ff439f 100644 --- a/schedulers/proactiveclusterwidecappingfcfs.go +++ b/schedulers/proactiveclusterwidecappingfcfs.go @@ -193,7 +193,6 @@ func (s *ProactiveClusterwideCapFCFS) startRecapping() { case <-s.recapTicker.C: fcfsMutex.Lock() // If stopped performing cluster wide capping then we need to explicitly cap the entire cluster. - //if !s.isCapping && s.isRecapping && fcfsRecapValue > 0.0 { if s.isRecapping && fcfsRecapValue > 0.0 { for _, host := range constants.Hosts { // Rounding curreCapValue to the nearest int. diff --git a/schedulers/proactiveclusterwidecappingranked.go b/schedulers/proactiveclusterwidecappingranked.go index a6f8aa8..3c9ef81 100644 --- a/schedulers/proactiveclusterwidecappingranked.go +++ b/schedulers/proactiveclusterwidecappingranked.go @@ -203,7 +203,6 @@ func (s *ProactiveClusterwideCapRanked) startRecapping() { case <-s.recapTicker.C: rankedMutex.Lock() // If stopped performing cluster wide capping then we need to explicitly cap the entire cluster. - //if !s.isCapping && s.isRecapping && rankedRecapValue > 0.0 { if s.isRecapping && rankedRecapValue > 0.0 { for _, host := range constants.Hosts { // Rounding curreCapValue to the nearest int. From 6aa849bbe7b90c74cf7710d2ad2a8b062929756d Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 28 Nov 2016 17:18:33 -0500 Subject: [PATCH 49/94] fixed naming convensions to be camel cased. Reformatted the code. --- constants/constants.go | 42 ++-- def/task.go | 14 +- schedulers/proactiveclusterwidecappers.go | 226 +++++++++--------- schedulers/proactiveclusterwidecappingfcfs.go | 4 +- .../proactiveclusterwidecappingranked.go | 2 +- utilities/utils.go | 6 +- 6 files changed, 147 insertions(+), 147 deletions(-) diff --git a/constants/constants.go b/constants/constants.go index 2d9b516..fb06a9d 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -16,71 +16,71 @@ var Hosts = []string{"stratos-001.cs.binghamton.edu", "stratos-002.cs.binghamton "stratos-007.cs.binghamton.edu", "stratos-008.cs.binghamton.edu"} // Add a new host to the slice of hosts. -func AddNewHost(new_host string) bool { +func AddNewHost(newHost string) bool { // Validation - if new_host == "" { + if newHost == "" { return false } else { - Hosts = append(Hosts, new_host) + Hosts = append(Hosts, newHost) return true } } // Lower bound of the percentage of requested power, that can be allocated to a task. -var Power_threshold = 0.6 // Right now saying that a task will never be given lesser than 60% of the power it requested. +var PowerThreshold = 0.6 // Right now saying that a task will never be given lesser than 60% of the power it requested. /* Margin with respect to the required power for a job. So, if power required = 10W, the node would be capped to 75%*10W. This value can be changed upon convenience. */ -var Cap_margin = 0.50 +var CapMargin = 0.70 // Modify the cap margin. -func UpdateCapMargin(new_cap_margin float64) bool { +func UpdateCapMargin(newCapMargin float64) bool { // Checking if the new_cap_margin is less than the power threshold. - if new_cap_margin < Starvation_factor { + if newCapMargin < StarvationFactor { return false } else { - Cap_margin = new_cap_margin + CapMargin = newCapMargin return true } } // Threshold factor that would make (Cap_margin * task.Watts) equal to (60/100 * task.Watts). -var Starvation_factor = 0.8 +var StarvationFactor = 0.8 // Total power per node. -var Total_power map[string]float64 +var TotalPower map[string]float64 // Initialize the total power per node. This should be done before accepting any set of tasks for scheduling. -func AddTotalPowerForHost(host string, total_power float64) bool { +func AddTotalPowerForHost(host string, totalPower float64) bool { // Validation - is_correct_host := false - for _, existing_host := range Hosts { - if host == existing_host { - is_correct_host = true + isCorrectHost := false + for _, existingHost := range Hosts { + if host == existingHost { + isCorrectHost = true } } - if !is_correct_host { + if !isCorrectHost { return false } else { - Total_power[host] = total_power + TotalPower[host] = totalPower return true } } // Window size for running average -var Window_size = 10 +var WindowSize = 160 // Update the window size. -func UpdateWindowSize(new_window_size int) bool { +func UpdateWindowSize(newWindowSize int) bool { // Validation - if new_window_size == 0 { + if newWindowSize == 0 { return false } else { - Window_size = new_window_size + WindowSize = newWindowSize return true } } diff --git a/def/task.go b/def/task.go index 63668ad..9699812 100644 --- a/def/task.go +++ b/def/task.go @@ -38,18 +38,18 @@ func TasksFromJSON(uri string) ([]Task, error) { } // Update the host on which the task needs to be scheduled. -func (tsk *Task) UpdateHost(new_host string) bool { +func (tsk *Task) UpdateHost(newHost string) bool { // Validation - is_correct_host := false - for _, existing_host := range constants.Hosts { - if new_host == existing_host { - is_correct_host = true + isCorrectHost := false + for _, existingHost := range constants.Hosts { + if newHost == existingHost { + isCorrectHost = true } } - if !is_correct_host { + if !isCorrectHost { return false } else { - tsk.Host = new_host + tsk.Host = newHost return true } } diff --git a/schedulers/proactiveclusterwidecappers.go b/schedulers/proactiveclusterwidecappers.go index 5de8a47..e943d37 100644 --- a/schedulers/proactiveclusterwidecappers.go +++ b/schedulers/proactiveclusterwidecappers.go @@ -24,63 +24,63 @@ import ( // Structure containing utility data structures used to compute cluster-wide dynamic cap. type clusterwideCapper struct { // window of tasks. - window_of_tasks list.List + windowOfTasks list.List // The current sum of requested powers of the tasks in the window. - current_sum float64 + currentSum float64 // The current number of tasks in the window. - number_of_tasks_in_window int + numberOfTasksInWindow int } // Defining constructor for clusterwideCapper. Please don't call this directly and instead use getClusterwideCapperInstance(). func newClusterwideCapper() *clusterwideCapper { - return &clusterwideCapper{current_sum: 0.0, number_of_tasks_in_window: 0} + return &clusterwideCapper{currentSum: 0.0, numberOfTasksInWindow: 0} } // Singleton instance of clusterwideCapper -var singleton_capper *clusterwideCapper +var singletonCapper *clusterwideCapper // Retrieve the singleton instance of clusterwideCapper. func getClusterwideCapperInstance() *clusterwideCapper { - if singleton_capper == nil { - singleton_capper = newClusterwideCapper() + if singletonCapper == nil { + singletonCapper = newClusterwideCapper() } else { // Do nothing } - return singleton_capper + return singletonCapper } // Clear and initialize all the members of clusterwideCapper. func (capper clusterwideCapper) clear() { - capper.window_of_tasks.Init() - capper.current_sum = 0 - capper.number_of_tasks_in_window = 0 + capper.windowOfTasks.Init() + capper.currentSum = 0 + capper.numberOfTasksInWindow = 0 } // Compute the average of watts of all the tasks in the window. func (capper clusterwideCapper) average() float64 { - return capper.current_sum / float64(capper.window_of_tasks.Len()) + return capper.currentSum / float64(capper.windowOfTasks.Len()) } /* Compute the running average. -Using clusterwideCapper#window_of_tasks to store the tasks. +Using clusterwideCapper#windowOfTasks to store the tasks. Task at position 0 (oldest task) is removed when the window is full and new task arrives. */ -func (capper clusterwideCapper) running_average_of_watts(tsk *def.Task) float64 { +func (capper clusterwideCapper) runningAverageOfWatts(tsk *def.Task) float64 { var average float64 - if capper.number_of_tasks_in_window < constants.Window_size { - capper.window_of_tasks.PushBack(tsk) - capper.number_of_tasks_in_window++ - capper.current_sum += float64(tsk.Watts) * constants.Cap_margin + if capper.numberOfTasksInWindow < constants.WindowSize { + capper.windowOfTasks.PushBack(tsk) + capper.numberOfTasksInWindow++ + capper.currentSum += float64(tsk.Watts) * constants.CapMargin } else { - task_to_remove_element := capper.window_of_tasks.Front() - if task_to_remove, ok := task_to_remove_element.Value.(*def.Task); ok { - capper.current_sum -= float64(task_to_remove.Watts) * constants.Cap_margin - capper.window_of_tasks.Remove(task_to_remove_element) + taskToRemoveElement := capper.windowOfTasks.Front() + if taskToRemove, ok := taskToRemoveElement.Value.(*def.Task); ok { + capper.currentSum -= float64(taskToRemove.Watts) * constants.CapMargin + capper.windowOfTasks.Remove(taskToRemoveElement) } - capper.window_of_tasks.PushBack(tsk) - capper.current_sum += float64(tsk.Watts) * constants.Cap_margin + capper.windowOfTasks.PushBack(tsk) + capper.currentSum += float64(tsk.Watts) * constants.CapMargin } average = capper.average() return average @@ -89,17 +89,17 @@ func (capper clusterwideCapper) running_average_of_watts(tsk *def.Task) float64 /* Calculating cap value. -1. Sorting the values of running_average_to_total_power_percentage in ascending order. +1. Sorting the values of runningAverageToTotalPowerPercentage in ascending order. 2. Computing the median of above sorted values. 3. The median is now the cap. */ -func (capper clusterwideCapper) get_cap(running_average_to_total_power_percentage map[string]float64) float64 { +func (capper clusterwideCapper) getCap(runningAverageToTotalPowerPercentage map[string]float64) float64 { var values []float64 // Validation - if running_average_to_total_power_percentage == nil { + if runningAverageToTotalPowerPercentage == nil { return 100.0 } - for _, apower := range running_average_to_total_power_percentage { + for _, apower := range runningAverageToTotalPowerPercentage { values = append(values, apower) } // sorting the values in ascending order. @@ -122,51 +122,51 @@ The recap value picked the least among the two. The cleverRecap scheme works well when the cluster is relatively idle and until then, the primitive recapping scheme works better. */ -func (capper clusterwideCapper) cleverRecap(total_power map[string]float64, - task_monitor map[string][]def.Task, finished_taskId string) (float64, error) { +func (capper clusterwideCapper) cleverRecap(totalPower map[string]float64, + taskMonitor map[string][]def.Task, finishedTaskId string) (float64, error) { // Validation - if total_power == nil || task_monitor == nil { - return 100.0, errors.New("Invalid argument: total_power, task_monitor") + if totalPower == nil || taskMonitor == nil { + return 100.0, errors.New("Invalid argument: totalPower, taskMonitor") } // determining the recap value by calling the regular recap(...) toggle := false - recapValue, err := capper.recap(total_power, task_monitor, finished_taskId) + recapValue, err := capper.recap(totalPower, taskMonitor, finishedTaskId) if err == nil { toggle = true } // watts usage on each node in the cluster. - watts_usages := make(map[string][]float64) - host_of_finished_task := "" - index_of_finished_task := -1 + wattsUsages := make(map[string][]float64) + hostOfFinishedTask := "" + indexOfFinishedTask := -1 for _, host := range constants.Hosts { - watts_usages[host] = []float64{0.0} + wattsUsages[host] = []float64{0.0} } - for host, tasks := range task_monitor { + for host, tasks := range taskMonitor { for i, task := range tasks { - if task.TaskID == finished_taskId { - host_of_finished_task = host - index_of_finished_task = i - // Not considering this task for the computation of total_allocated_power and total_running_tasks + if task.TaskID == finishedTaskId { + hostOfFinishedTask = host + indexOfFinishedTask = i + // Not considering this task for the computation of totalAllocatedPower and totalRunningTasks continue } - watts_usages[host] = append(watts_usages[host], float64(task.Watts)*constants.Cap_margin) + wattsUsages[host] = append(wattsUsages[host], float64(task.Watts)*constants.CapMargin) } } // Updating task monitor. If recap(...) has deleted the finished task from the taskMonitor, // then this will be ignored. Else (this is only when an error occured with recap(...)), we remove it here. - if host_of_finished_task != "" && index_of_finished_task != -1 { + if hostOfFinishedTask != "" && indexOfFinishedTask != -1 { log.Printf("Removing task with task [%s] from the list of running tasks\n", - task_monitor[host_of_finished_task][index_of_finished_task].TaskID) - task_monitor[host_of_finished_task] = append(task_monitor[host_of_finished_task][:index_of_finished_task], - task_monitor[host_of_finished_task][index_of_finished_task+1:]...) + taskMonitor[hostOfFinishedTask][indexOfFinishedTask].TaskID) + taskMonitor[hostOfFinishedTask] = append(taskMonitor[hostOfFinishedTask][:indexOfFinishedTask], + taskMonitor[hostOfFinishedTask][indexOfFinishedTask+1:]...) } // Need to check whether there are still tasks running on the cluster. If not then we return an error. clusterIdle := true - for _, tasks := range task_monitor { + for _, tasks := range taskMonitor { if len(tasks) > 0 { clusterIdle = false } @@ -175,29 +175,29 @@ func (capper clusterwideCapper) cleverRecap(total_power map[string]float64, if !clusterIdle { // load on each node in the cluster. loads := []float64{0.0} - for host, usages := range watts_usages { - total_usage := 0.0 + for host, usages := range wattsUsages { + totalUsage := 0.0 for _, usage := range usages { - total_usage += usage + totalUsage += usage } - loads = append(loads, total_usage/total_power[host]) + loads = append(loads, totalUsage/totalPower[host]) } // Now need to compute the average load. - total_load := 0.0 + totalLoad := 0.0 for _, load := range loads { - total_load += load + totalLoad += load } - average_load := (total_load / float64(len(loads)) * 100.0) // this would be the cap value. + averageLoad := (totalLoad / float64(len(loads)) * 100.0) // this would be the cap value. // If toggle is true, then we need to return the least recap value. if toggle { - if average_load <= recapValue { - return average_load, nil + if averageLoad <= recapValue { + return averageLoad, nil } else { return recapValue, nil } } else { - return average_load, nil + return averageLoad, nil } } return 100.0, errors.New("No task running on the cluster.") @@ -213,46 +213,46 @@ Recapping the entire cluster. This needs to be called whenever a task finishes execution. */ -func (capper clusterwideCapper) recap(total_power map[string]float64, - task_monitor map[string][]def.Task, finished_taskId string) (float64, error) { +func (capper clusterwideCapper) recap(totalPower map[string]float64, + taskMonitor map[string][]def.Task, finishedTaskId string) (float64, error) { // Validation - if total_power == nil || task_monitor == nil { - return 100.0, errors.New("Invalid argument: total_power, task_monitor") + if totalPower == nil || taskMonitor == nil { + return 100.0, errors.New("Invalid argument: totalPower, taskMonitor") } - total_allocated_power := 0.0 - total_running_tasks := 0 + totalAllocatedPower := 0.0 + totalRunningTasks := 0 - host_of_finished_task := "" - index_of_finished_task := -1 - for host, tasks := range task_monitor { + hostOfFinishedTask := "" + indexOfFinishedTask := -1 + for host, tasks := range taskMonitor { for i, task := range tasks { - if task.TaskID == finished_taskId { - host_of_finished_task = host - index_of_finished_task = i - // Not considering this task for the computation of total_allocated_power and total_running_tasks + if task.TaskID == finishedTaskId { + hostOfFinishedTask = host + indexOfFinishedTask = i + // Not considering this task for the computation of totalAllocatedPower and totalRunningTasks continue } - total_allocated_power += (float64(task.Watts) * constants.Cap_margin) - total_running_tasks++ + totalAllocatedPower += (float64(task.Watts) * constants.CapMargin) + totalRunningTasks++ } } // Updating task monitor - if host_of_finished_task != "" && index_of_finished_task != -1 { + if hostOfFinishedTask != "" && indexOfFinishedTask != -1 { log.Printf("Removing task with task [%s] from the list of running tasks\n", - task_monitor[host_of_finished_task][index_of_finished_task].TaskID) - task_monitor[host_of_finished_task] = append(task_monitor[host_of_finished_task][:index_of_finished_task], - task_monitor[host_of_finished_task][index_of_finished_task+1:]...) + taskMonitor[hostOfFinishedTask][indexOfFinishedTask].TaskID) + taskMonitor[hostOfFinishedTask] = append(taskMonitor[hostOfFinishedTask][:indexOfFinishedTask], + taskMonitor[hostOfFinishedTask][indexOfFinishedTask+1:]...) } - // For the last task, total_allocated_power and total_running_tasks would be 0 - if total_allocated_power == 0 && total_running_tasks == 0 { + // For the last task, totalAllocatedPower and totalRunningTasks would be 0 + if totalAllocatedPower == 0 && totalRunningTasks == 0 { return 100, errors.New("No task running on the cluster.") } - average := total_allocated_power / float64(total_running_tasks) + average := totalAllocatedPower / float64(totalRunningTasks) ratios := []float64{} - for _, tpower := range total_power { + for _, tpower := range totalPower { ratios = append(ratios, (average/tpower)*100) } sort.Float64s(ratios) @@ -265,38 +265,38 @@ func (capper clusterwideCapper) recap(total_power map[string]float64, } /* Quick sort algorithm to sort tasks, in place, in ascending order of power.*/ -func (capper clusterwideCapper) quick_sort(low int, high int, tasks_to_sort *[]def.Task) { +func (capper clusterwideCapper) quickSort(low int, high int, tasksToSort *[]def.Task) { i := low j := high // calculating the pivot - pivot_index := low + (high-low)/2 - pivot := (*tasks_to_sort)[pivot_index] + pivotIndex := low + (high-low)/2 + pivot := (*tasksToSort)[pivotIndex] for i <= j { - for (*tasks_to_sort)[i].Watts < pivot.Watts { + for (*tasksToSort)[i].Watts < pivot.Watts { i++ } - for (*tasks_to_sort)[j].Watts > pivot.Watts { + for (*tasksToSort)[j].Watts > pivot.Watts { j-- } if i <= j { - temp := (*tasks_to_sort)[i] - (*tasks_to_sort)[i] = (*tasks_to_sort)[j] - (*tasks_to_sort)[j] = temp + temp := (*tasksToSort)[i] + (*tasksToSort)[i] = (*tasksToSort)[j] + (*tasksToSort)[j] = temp i++ j-- } } if low < j { - capper.quick_sort(low, j, tasks_to_sort) + capper.quickSort(low, j, tasksToSort) } if i < high { - capper.quick_sort(i, high, tasks_to_sort) + capper.quickSort(i, high, tasksToSort) } } // Sorting tasks in ascending order of requested watts. -func (capper clusterwideCapper) sort_tasks(tasks_to_sort *[]def.Task) { - capper.quick_sort(0, len(*tasks_to_sort)-1, tasks_to_sort) +func (capper clusterwideCapper) sortTasks(tasksToSort *[]def.Task) { + capper.quickSort(0, len(*tasksToSort)-1, tasksToSort) } /* @@ -307,51 +307,51 @@ This completed task needs to be removed from the window of tasks (if it is still */ func (capper clusterwideCapper) taskFinished(taskID string) { // If the window is empty the just return. This condition should technically return false. - if capper.window_of_tasks.Len() == 0 { + if capper.windowOfTasks.Len() == 0 { return } // Checking whether the task with the given taskID is currently present in the window of tasks. - var task_element_to_remove *list.Element - for task_element := capper.window_of_tasks.Front(); task_element != nil; task_element = task_element.Next() { - if tsk, ok := task_element.Value.(*def.Task); ok { + var taskElementToRemove *list.Element + for taskElement := capper.windowOfTasks.Front(); taskElement != nil; taskElement = taskElement.Next() { + if tsk, ok := taskElement.Value.(*def.Task); ok { if tsk.TaskID == taskID { - task_element_to_remove = task_element + taskElementToRemove = taskElement } } } // we need to remove the task from the window. - if task_to_remove, ok := task_element_to_remove.Value.(*def.Task); ok { - capper.window_of_tasks.Remove(task_element_to_remove) - capper.number_of_tasks_in_window -= 1 - capper.current_sum -= float64(task_to_remove.Watts) * constants.Cap_margin + if taskToRemove, ok := taskElementToRemove.Value.(*def.Task); ok { + capper.windowOfTasks.Remove(taskElementToRemove) + capper.numberOfTasksInWindow -= 1 + capper.currentSum -= float64(taskToRemove.Watts) * constants.CapMargin } } // First come first serve scheduling. -func (capper clusterwideCapper) fcfsDetermineCap(total_power map[string]float64, - new_task *def.Task) (float64, error) { +func (capper clusterwideCapper) fcfsDetermineCap(totalPower map[string]float64, + newTask *def.Task) (float64, error) { // Validation - if total_power == nil { - return 100, errors.New("Invalid argument: total_power") + if totalPower == nil { + return 100, errors.New("Invalid argument: totalPower") } else { // Need to calculate the running average - running_average := capper.running_average_of_watts(new_task) + runningAverage := capper.runningAverageOfWatts(newTask) // For each node, calculate the percentage of the running average to the total power. - running_average_to_total_power_percentage := make(map[string]float64) - for host, tpower := range total_power { - if tpower >= running_average { - running_average_to_total_power_percentage[host] = (running_average / tpower) * 100 + runningAverageToTotalPowerPercentage := make(map[string]float64) + for host, tpower := range totalPower { + if tpower >= runningAverage { + runningAverageToTotalPowerPercentage[host] = (runningAverage / tpower) * 100 } else { // We don't consider this host for the computation of the cluster wide cap. } } // Determine the cluster wide cap value. - cap_value := capper.get_cap(running_average_to_total_power_percentage) + capValue := capper.getCap(runningAverageToTotalPowerPercentage) // Need to cap the cluster to this value. - return cap_value, nil + return capValue, nil } } diff --git a/schedulers/proactiveclusterwidecappingfcfs.go b/schedulers/proactiveclusterwidecappingfcfs.go index 5ff439f..49094cd 100644 --- a/schedulers/proactiveclusterwidecappingfcfs.go +++ b/schedulers/proactiveclusterwidecappingfcfs.go @@ -304,8 +304,8 @@ func (s *ProactiveClusterwideCapFCFS) ResourceOffers(driver sched.SchedulerDrive log.Println(err) } log.Printf("Starting on [%s]\n", offer.GetHostname()) - to_schedule := []*mesos.TaskInfo{s.newTask(offer, task)} - driver.LaunchTasks([]*mesos.OfferID{offer.Id}, to_schedule, defaultFilter) + toSchedule := []*mesos.TaskInfo{s.newTask(offer, task)} + driver.LaunchTasks([]*mesos.OfferID{offer.Id}, toSchedule, defaultFilter) log.Printf("Inst: %d", *task.Instances) *task.Instances-- if *task.Instances <= 0 { diff --git a/schedulers/proactiveclusterwidecappingranked.go b/schedulers/proactiveclusterwidecappingranked.go index 3c9ef81..f6ea425 100644 --- a/schedulers/proactiveclusterwidecappingranked.go +++ b/schedulers/proactiveclusterwidecappingranked.go @@ -257,7 +257,7 @@ func (s *ProactiveClusterwideCapRanked) ResouceOffers(driver sched.SchedulerDriv } // sorting the tasks in ascending order of watts. - s.capper.sort_tasks(&s.tasks) + s.capper.sortTasks(&s.tasks) // displaying the ranked tasks. log.Println("The ranked tasks are:\n---------------------\n\t[") for rank, task := range s.tasks { diff --git a/utilities/utils.go b/utilities/utils.go index ede4f64..c53df74 100644 --- a/utilities/utils.go +++ b/utilities/utils.go @@ -37,11 +37,11 @@ func OrderedKeys(plist PairList) ([]string, error) { if plist == nil { return nil, errors.New("Invalid argument: plist") } - ordered_keys := make([]string, len(plist)) + orderedKeys := make([]string, len(plist)) for _, pair := range plist { - ordered_keys = append(ordered_keys, pair.Key) + orderedKeys = append(orderedKeys, pair.Key) } - return ordered_keys, nil + return orderedKeys, nil } // determine the max value From 785de5bafb9fda4045009b05c5c9a8d0478f7253 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 28 Nov 2016 20:00:12 -0500 Subject: [PATCH 50/94] Added explanation for StarvationThreshold. Removed TotalPower as it was embedded inside the schedulers. --- constants/constants.go | 43 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/constants/constants.go b/constants/constants.go index fb06a9d..c9d7b36 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -26,7 +26,11 @@ func AddNewHost(newHost string) bool { } } -// Lower bound of the percentage of requested power, that can be allocated to a task. +/* + Lower bound of the percentage of requested power, that can be allocated to a task. + + Note: This constant is not used for the proactive cluster wide capping schemes. +*/ var PowerThreshold = 0.6 // Right now saying that a task will never be given lesser than 60% of the power it requested. /* @@ -47,29 +51,22 @@ func UpdateCapMargin(newCapMargin float64) bool { } } -// Threshold factor that would make (Cap_margin * task.Watts) equal to (60/100 * task.Watts). -var StarvationFactor = 0.8 +/* + The factor, that when multiplied with (task.Watts * CapMargin) results in (task.Watts * PowerThreshold). + This is used to check whether available power, for a host in an offer, is not less than (PowerThreshold * task.Watts), + which is assumed to result in starvation of the task. + Here is an example, + Suppose a task requires 100W of power. Assuming CapMargin = 0.75 and PowerThreshold = 0.6. + So, the assumed allocated watts is 75W. + Now, when we get an offer, we need to check whether the available power, for the host in that offer, is + not less than 60% (the PowerTreshold) of the requested power (100W). + To put it in other words, + availablePower >= 100W * 0.75 * X + where X is the StarvationFactor (80% in this case) -// Total power per node. -var TotalPower map[string]float64 - -// Initialize the total power per node. This should be done before accepting any set of tasks for scheduling. -func AddTotalPowerForHost(host string, totalPower float64) bool { - // Validation - isCorrectHost := false - for _, existingHost := range Hosts { - if host == existingHost { - isCorrectHost = true - } - } - - if !isCorrectHost { - return false - } else { - TotalPower[host] = totalPower - return true - } -} + Note: This constant is not used for the proactive cluster wide capping schemes. +*/ +var StarvationFactor = PowerThreshold / CapMargin // Window size for running average var WindowSize = 160 From f99fce77cb8474317742f87da9e615d511531cf9 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 28 Nov 2016 20:01:25 -0500 Subject: [PATCH 51/94] Updated comments --- constants/constants.go | 1 - 1 file changed, 1 deletion(-) diff --git a/constants/constants.go b/constants/constants.go index c9d7b36..08ad28a 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -3,7 +3,6 @@ Constants that are used across scripts 1. The available hosts = stratos-00x (x varies from 1 to 8) 2. cap_margin = percentage of the requested power to allocate 3. power_threshold = overloading factor -4. total_power = total power per node 5. window_size = number of tasks to consider for computation of the dynamic cap. Also, exposing functions to update or initialize some of the constants. From e4bf4d37275748d9d4dc52de1955ea151d5368f4 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 28 Nov 2016 20:02:03 -0500 Subject: [PATCH 52/94] Prevented the use of reflect and instead compared the TaskIDs of the tasks. --- def/task.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/def/task.go b/def/task.go index 9699812..e52acb3 100644 --- a/def/task.go +++ b/def/task.go @@ -5,7 +5,6 @@ import ( "encoding/json" "github.com/pkg/errors" "os" - "reflect" ) type Task struct { @@ -85,15 +84,9 @@ func Compare(task1 *Task, task2 *Task) bool { if task1 == task2 { return true } - // Checking member equality - if reflect.DeepEqual(*task1, *task2) { - // Need to check for the task ID - if task1.TaskID == task2.TaskID { - return true - } else { - return false - } - } else { + if task1.TaskID != task2.TaskID { return false + } else { + return true } } From 979b0ab9a9e73410a54347f0d023b48fea1bbeff Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 28 Nov 2016 22:28:44 -0500 Subject: [PATCH 53/94] Added to TODO (config template per scheduler). --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index aa625e4..636c6d2 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ To Do: * Have calibration phase? * Add ability to use constraints * Running average calculations https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average + * Make parameters corresponding to each scheduler configurable (possible to have a config template for each scheduler?) From 7f5d9c58a33f9690e61fc3016bfba80d3a7c38a4 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 28 Nov 2016 22:29:13 -0500 Subject: [PATCH 54/94] Added to TODO (generic running average computation). --- schedulers/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/schedulers/README.md b/schedulers/README.md index a61559b..11f25a0 100644 --- a/schedulers/README.md +++ b/schedulers/README.md @@ -4,6 +4,7 @@ Electron: Scheduling Algorithms To Do: * Design changes -- Possible to have one scheduler with different scheduling schemes? + * Make the running average calculation generic, so that schedulers in the future can use it and not implement their own. Scheduling Algorithms: From 8eda0c68b1a4089d637fc020eb7a737c951798b5 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 28 Nov 2016 22:38:02 -0500 Subject: [PATCH 55/94] Removed TODO comment in ResourceOffers(...), that wasn't necessary anymore. --- schedulers/proactiveclusterwidecappingfcfs.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/schedulers/proactiveclusterwidecappingfcfs.go b/schedulers/proactiveclusterwidecappingfcfs.go index 49094cd..37fa19a 100644 --- a/schedulers/proactiveclusterwidecappingfcfs.go +++ b/schedulers/proactiveclusterwidecappingfcfs.go @@ -234,7 +234,6 @@ func (s *ProactiveClusterwideCapFCFS) stopRecapping() { } } -// TODO: Need to reduce the time complexity: looping over offers twice (Possible to do it just once?). func (s *ProactiveClusterwideCapFCFS) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { log.Printf("Received %d resource offers", len(offers)) @@ -272,8 +271,6 @@ func (s *ProactiveClusterwideCapFCFS) ResourceOffers(driver sched.SchedulerDrive 3. fcfsCurrentCapValue is updated with the determined cluster wide cap. Cluster wide capping is currently performed at regular intervals of time. - TODO: We can choose to cap the cluster only if the clusterwide cap varies more than the current clusterwide cap. - Although this sounds like a better approach, it only works when the resource requirements of neighbouring tasks are similar. */ taken := false From b8c33b25b4524c27d478953c04838acecdc0010f Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Tue, 29 Nov 2016 15:22:01 -0500 Subject: [PATCH 56/94] Added TODO to fix the -p option. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 636c6d2..a46ec9f 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ To Do: * Add ability to use constraints * Running average calculations https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average * Make parameters corresponding to each scheduler configurable (possible to have a config template for each scheduler?) + * Fix the -p option. From 395e8e3e3dc1d48c1ba4cc31bc06f1fc4bf2d47b Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Tue, 29 Nov 2016 22:26:55 -0500 Subject: [PATCH 57/94] synchronized operations on tasksRunning and hence prevented the previously occuring race condition. --- schedulers/proactiveclusterwidecappingfcfs.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/schedulers/proactiveclusterwidecappingfcfs.go b/schedulers/proactiveclusterwidecappingfcfs.go index 37fa19a..a96d496 100644 --- a/schedulers/proactiveclusterwidecappingfcfs.go +++ b/schedulers/proactiveclusterwidecappingfcfs.go @@ -339,7 +339,9 @@ func (s *ProactiveClusterwideCapFCFS) StatusUpdate(driver sched.SchedulerDriver, log.Printf("Received task status [%s] for task [%s]\n", NameFor(status.State), *status.TaskId.Value) if *status.State == mesos.TaskState_TASK_RUNNING { + fcfsMutex.Lock() s.tasksRunning++ + fcfsMutex.Unlock() } else if IsTerminal(status.State) { delete(s.running[status.GetSlaveId().GoString()], *status.TaskId.Value) // Need to remove the task from the window of tasks. @@ -365,7 +367,9 @@ func (s *ProactiveClusterwideCapFCFS) StatusUpdate(driver sched.SchedulerDriver, log.Println(err) } + fcfsMutex.Lock() s.tasksRunning-- + fcfsMutex.Unlock() if s.tasksRunning == 0 { select { case <-s.Shutdown: From 6c29f2c5a44374eb962c5cd30f250adba71f33f6 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Tue, 29 Nov 2016 23:00:03 -0500 Subject: [PATCH 58/94] synchronized operations on tasksRunning and hence, prevented previously occuring race condition. --- .../proactiveclusterwidecappingranked.go | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/schedulers/proactiveclusterwidecappingranked.go b/schedulers/proactiveclusterwidecappingranked.go index f6ea425..d2565f3 100644 --- a/schedulers/proactiveclusterwidecappingranked.go +++ b/schedulers/proactiveclusterwidecappingranked.go @@ -222,21 +222,26 @@ func (s *ProactiveClusterwideCapRanked) startRecapping() { // Stop cluster wide capping func (s *ProactiveClusterwideCapRanked) stopCapping() { - log.Println("Stopping the cluster wide capping.") - s.ticker.Stop() - rankedMutex.Lock() - s.isCapping = false - s.isRecapping = true - rankedMutex.Unlock() + if s.isCapping { + log.Println("Stopping the cluster wide capping.") + s.ticker.Stop() + fcfsMutex.Lock() + s.isCapping = false + s.isRecapping = true + fcfsMutex.Unlock() + } } // Stop cluster wide Recapping func (s *ProactiveClusterwideCapRanked) stopRecapping() { - log.Println("Stopping the cluster wide re-capping.") - s.recapTicker.Stop() - rankedMutex.Lock() - s.isRecapping = false - rankedMutex.Unlock() + // If not capping, then definitely recapping. + if !s.isCapping && s.isRecapping { + log.Println("Stopping the cluster wide re-capping.") + s.recapTicker.Stop() + fcfsMutex.Lock() + s.isRecapping = false + fcfsMutex.Unlock() + } } func (s *ProactiveClusterwideCapRanked) ResouceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { @@ -355,10 +360,14 @@ func (s *ProactiveClusterwideCapRanked) StatusUpdate(driver sched.SchedulerDrive log.Printf("Received task status [%s] for task [%s]\n", NameFor(status.State), *status.TaskId.Value) if *status.State == mesos.TaskState_TASK_RUNNING { + rankedMutex.Lock() s.tasksRunning++ + rankedMutex.Unlock() } else if IsTerminal(status.State) { delete(s.running[status.GetSlaveId().GoString()], *status.TaskId.Value) + rankedMutex.Lock() s.tasksRunning-- + rankedMutex.Unlock() if s.tasksRunning == 0 { select { case <-s.Shutdown: From 7f59bc4b6859be439ae8abc5312db8a8919b7894 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Wed, 30 Nov 2016 21:51:55 -0500 Subject: [PATCH 59/94] Fixed the -p option. Changed the README for the same. --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index a46ec9f..23a0801 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,6 @@ To Do: * Add ability to use constraints * Running average calculations https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average * Make parameters corresponding to each scheduler configurable (possible to have a config template for each scheduler?) - * Fix the -p option. @@ -19,7 +18,7 @@ machine on which electron is launched for logging to work** How to run (Use the --help option to get information about other command-line options): -`./electron -workload -ignoreWatts ` +`./electron -workload ` Workload schema: From 3e1fe71459df6e7c3b34cb5a3b941b424d296ec5 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Wed, 30 Nov 2016 22:05:42 -0500 Subject: [PATCH 60/94] Updated readme to include run command with and without ignoreWatts --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 23a0801..0443c66 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,10 @@ How to run (Use the --help option to get information about other command-line op `./electron -workload ` +To run electron with ignoreWatts, run the following command, + +`./electron -workload -ignoreWatts` + Workload schema: From ef839c530d41f8c016c7c7e7e9d6370e187a53bb Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 1 Dec 2016 18:31:49 -0500 Subject: [PATCH 61/94] Fixed corner case bug with sorting of tasks. Not sorting if there are no more tasks to schedule. --- schedulers/proactiveclusterwidecappingranked.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/schedulers/proactiveclusterwidecappingranked.go b/schedulers/proactiveclusterwidecappingranked.go index d2565f3..963de40 100644 --- a/schedulers/proactiveclusterwidecappingranked.go +++ b/schedulers/proactiveclusterwidecappingranked.go @@ -244,7 +244,7 @@ func (s *ProactiveClusterwideCapRanked) stopRecapping() { } } -func (s *ProactiveClusterwideCapRanked) ResouceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { +func (s *ProactiveClusterwideCapRanked) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { log.Printf("Received %d resource offers", len(offers)) // retrieving the available power for all the hosts in the offers. @@ -262,14 +262,15 @@ func (s *ProactiveClusterwideCapRanked) ResouceOffers(driver sched.SchedulerDriv } // sorting the tasks in ascending order of watts. - s.capper.sortTasks(&s.tasks) - // displaying the ranked tasks. - log.Println("The ranked tasks are:\n---------------------\n\t[") - for rank, task := range s.tasks { - log.Printf("\t\t%d: %s\n", rank+1, task.TaskID) + if (len(s.tasks) > 0) { + s.capper.sortTasks(&s.tasks) + // calculating the total number of tasks ranked. + numberOfRankedTasks := 0 + for _, task := range s.tasks { + numberOfRankedTasks += *task.Instances + } + log.Printf("Ranked %d tasks in ascending order of tasks.", numberOfRankedTasks) } - log.Println("\t]") - for _, offer := range offers { select { case <-s.Shutdown: From b0140a8b9366b3b6d17c4fe1a0f412833b9dacc7 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Wed, 7 Dec 2016 01:11:30 -0500 Subject: [PATCH 62/94] Using ranked proactive cluster wide capper with clever recapping --- scheduler.go | 2 +- schedulers/proactiveclusterwidecappingranked.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scheduler.go b/scheduler.go index 356587f..280d788 100644 --- a/scheduler.go +++ b/scheduler.go @@ -56,7 +56,7 @@ func main() { fmt.Println(task) } - scheduler := schedulers.NewProactiveClusterwideCapFCFS(tasks, *ignoreWatts) + scheduler := schedulers.NewProactiveClusterwideCapRanked(tasks, *ignoreWatts) driver, err := sched.NewMesosSchedulerDriver(sched.DriverConfig{ Master: *master, Framework: &mesos.FrameworkInfo{ diff --git a/schedulers/proactiveclusterwidecappingranked.go b/schedulers/proactiveclusterwidecappingranked.go index 963de40..69ae26f 100644 --- a/schedulers/proactiveclusterwidecappingranked.go +++ b/schedulers/proactiveclusterwidecappingranked.go @@ -381,8 +381,8 @@ func (s *ProactiveClusterwideCapRanked) StatusUpdate(driver sched.SchedulerDrive // Need to remove the task from the window s.capper.taskFinished(*status.TaskId.Value) // Determining the new cluster wide cap. - tempCap, err := s.capper.recap(s.totalPower, s.taskMonitor, *status.TaskId.Value) - // tempCap, err := s.capper.cleverRecap(s.totalPower, s.taskMonitor, *status.TaskId.Value) + //tempCap, err := s.capper.recap(s.totalPower, s.taskMonitor, *status.TaskId.Value) + tempCap, err := s.capper.cleverRecap(s.totalPower, s.taskMonitor, *status.TaskId.Value) if err == nil { // If new determined cap value is different from the current recap value then we need to recap. From 54a55ec523421f4176ef3666498bb17aee83ee90 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Wed, 7 Dec 2016 18:47:37 -0500 Subject: [PATCH 63/94] Nit: Changed variable name 'runningAverageToTotalPowerPercentage' to 'ratios' --- schedulers/proactiveclusterwidecappers.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/schedulers/proactiveclusterwidecappers.go b/schedulers/proactiveclusterwidecappers.go index e943d37..98adc53 100644 --- a/schedulers/proactiveclusterwidecappers.go +++ b/schedulers/proactiveclusterwidecappers.go @@ -89,17 +89,17 @@ func (capper clusterwideCapper) runningAverageOfWatts(tsk *def.Task) float64 { /* Calculating cap value. -1. Sorting the values of runningAverageToTotalPowerPercentage in ascending order. +1. Sorting the values of ratios ((running average/totalPower) per node) in ascending order. 2. Computing the median of above sorted values. 3. The median is now the cap. */ -func (capper clusterwideCapper) getCap(runningAverageToTotalPowerPercentage map[string]float64) float64 { +func (capper clusterwideCapper) getCap(ratios map[string]float64) float64 { var values []float64 // Validation - if runningAverageToTotalPowerPercentage == nil { + if ratios == nil { return 100.0 } - for _, apower := range runningAverageToTotalPowerPercentage { + for _, apower := range ratios { values = append(values, apower) } // sorting the values in ascending order. @@ -339,17 +339,17 @@ func (capper clusterwideCapper) fcfsDetermineCap(totalPower map[string]float64, // Need to calculate the running average runningAverage := capper.runningAverageOfWatts(newTask) // For each node, calculate the percentage of the running average to the total power. - runningAverageToTotalPowerPercentage := make(map[string]float64) + ratios := make(map[string]float64) for host, tpower := range totalPower { if tpower >= runningAverage { - runningAverageToTotalPowerPercentage[host] = (runningAverage / tpower) * 100 + ratios[host] = (runningAverage / tpower) * 100 } else { // We don't consider this host for the computation of the cluster wide cap. } } // Determine the cluster wide cap value. - capValue := capper.getCap(runningAverageToTotalPowerPercentage) + capValue := capper.getCap(ratios) // Need to cap the cluster to this value. return capValue, nil } From de4df8ba7bb7b7a9c2603088d1739877dc1211af Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 15 Dec 2016 14:30:43 -0500 Subject: [PATCH 64/94] switched scheduler to now be an instance of PistonCapper. --- scheduler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scheduler.go b/scheduler.go index 280d788..bd7c943 100644 --- a/scheduler.go +++ b/scheduler.go @@ -56,7 +56,7 @@ func main() { fmt.Println(task) } - scheduler := schedulers.NewProactiveClusterwideCapRanked(tasks, *ignoreWatts) + scheduler := schedulers.NewPistonCapper(tasks, *ignoreWatts) driver, err := sched.NewMesosSchedulerDriver(sched.DriverConfig{ Master: *master, Framework: &mesos.FrameworkInfo{ From a6dcda04c87e431a747cbb232a8ab6d81bcb4c38 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 15 Dec 2016 14:31:35 -0500 Subject: [PATCH 65/94] Panic when few tasks are present and hence added a TODO to test each scheduler for corner cases --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0443c66..320e4e5 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ To Do: * Add ability to use constraints * Running average calculations https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average * Make parameters corresponding to each scheduler configurable (possible to have a config template for each scheduler?) + * Write test code for each scheduler (This should be after the design change) From d9357f59e76403a22aa9331890b62ea63bdd95e0 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 15 Dec 2016 14:32:34 -0500 Subject: [PATCH 66/94] New scheduler. BinPacking with capping each node to different values (piston capping). --- schedulers/pistoncapper.go | 409 +++++++++++++++++++++++++++++++++++++ 1 file changed, 409 insertions(+) create mode 100644 schedulers/pistoncapper.go diff --git a/schedulers/pistoncapper.go b/schedulers/pistoncapper.go new file mode 100644 index 0000000..60f2a56 --- /dev/null +++ b/schedulers/pistoncapper.go @@ -0,0 +1,409 @@ +package schedulers + +import ( + "bitbucket.org/sunybingcloud/electron/constants" + "bitbucket.org/sunybingcloud/electron/def" + "bitbucket.org/sunybingcloud/electron/rapl" + "fmt" + "errors" + "github.com/golang/protobuf/proto" + mesos "github.com/mesos/mesos-go/mesosproto" + "github.com/mesos/mesos-go/mesosutil" + sched "github.com/mesos/mesos-go/scheduler" + "log" + "math" + "sync" + "strings" + "time" +) + +/* + Piston Capper implements the Scheduler interface + + This is basically extending the BinPacking algorithm to also cap each node at a different values, + corresponding to the load on that node. +*/ +type PistonCapper struct { + tasksCreated int + tasksRunning int + tasks []def.Task + metrics map[string]def.Metric + running map[string]map[string]bool + taskMonitor map[string][]def.Task + clusterLoad map[string]float64 + totalPower map[string]float64 + ignoreWatts bool + ticker *time.Ticker + isCapping bool + + // First set of PCP values are garbage values, signal to logger to start recording when we're + // about to schedule the new task. + RecordPCP bool + + // This channel is closed when the program receives an interrupt, + // signalling that the program should shut down. + Shutdown chan struct{} + + // This channel is closed after shutdown is closed, and only when all + // outstanding tasks have been cleaned up. + Done chan struct{} + + // Controls when to shutdown pcp logging. + PCPLog chan struct{} +} + +// New electron scheduler. +func NewPistonCapper(tasks []def.Task, ignoreWatts bool) *PistonCapper { + s := &PistonCapper{ + tasks: tasks, + ignoreWatts: ignoreWatts, + Shutdown: make(chan struct{}), + Done: make(chan struct{}), + PCPLog: make(chan struct{}), + running: make(map[string]map[string]bool), + taskMonitor: make(map[string][]def.Task), + clusterLoad: make(map[string]float64), + totalPower: make(map[string]float64), + RecordPCP: false, + ticker: time.NewTicker(10 * time.Second), + isCapping: false, + } + return s +} + +// mutex +var mutex sync.Mutex + +func (s *PistonCapper) newTask(offer *mesos.Offer, task def.Task) *mesos.TaskInfo { + taskName := fmt.Sprintf("%s-%d", task.Name, *task.Instances) + s.tasksCreated++ + + if !s.RecordPCP { + // Turn on logging + s.RecordPCP = true + time.Sleep(1 * time.Second) // Make sure we're recording by the time the first task starts + } + + // If this is our first time running into this Agent + if _, ok := s.running[offer.GetSlaveId().GoString()]; !ok { + s.running[offer.GetSlaveId().GoString()] = make(map[string]bool) + } + + // Setting the task ID to the task. This is done so that we can consider each task to be different, + // even though they have the same parameters. + task.SetTaskID(*proto.String("electron-" + taskName)) + // Add task to list of tasks running on node + s.running[offer.GetSlaveId().GoString()][taskName] = true + // Adding the task to the taskMonitor + if len(s.taskMonitor[*offer.Hostname]) == 0 { + s.taskMonitor[*offer.Hostname] = []def.Task{task} + } else { + s.taskMonitor[*offer.Hostname] = append(s.taskMonitor[*offer.Hostname], task) + } + + resources := []*mesos.Resource{ + mesosutil.NewScalarResource("cpus", task.CPU), + mesosutil.NewScalarResource("mem", task.RAM), + } + + if !s.ignoreWatts { + resources = append(resources, mesosutil.NewScalarResource("watts", task.Watts)) + } + + return &mesos.TaskInfo{ + Name: proto.String(taskName), + TaskId: &mesos.TaskID{ + Value: proto.String("electron-" + taskName), + }, + SlaveId: offer.SlaveId, + Resources: resources, + Command: &mesos.CommandInfo{ + Value: proto.String(task.CMD), + }, + Container: &mesos.ContainerInfo{ + Type: mesos.ContainerInfo_DOCKER.Enum(), + Docker: &mesos.ContainerInfo_DockerInfo{ + Image: proto.String(task.Image), + Network: mesos.ContainerInfo_DockerInfo_BRIDGE.Enum(), // Run everything isolated + }, + }, + } +} + + + +func (s *PistonCapper) Registered( + _ sched.SchedulerDriver, + frameworkID *mesos.FrameworkID, + masterInfo *mesos.MasterInfo) { + log.Printf("Framework %s registered with master %s", frameworkID, masterInfo) +} + +func (s *PistonCapper) Reregistered(_ sched.SchedulerDriver, masterInfo *mesos.MasterInfo) { + log.Printf("Framework re-registered with master %s", masterInfo) +} + +func (s *PistonCapper) Disconnected(sched.SchedulerDriver) { + log.Println("Framework disconnected with master") +} + +// go routine to cap the each node in the cluster at regular intervals of time. +var capValues = make(map[string]float64) +// Storing the previous cap value for each host so as to not repeatedly cap the nodes to the same value. (reduces overhead) +var previousRoundedCapValues = make(map[string]int) +func (s *PistonCapper) startCapping() { + go func() { + for { + select { + case <-s.ticker.C: + // Need to cap each node + mutex.Lock() + for host, capValue := range capValues { + roundedCapValue := int(math.Floor(capValue + 0.5)) + // has the cap value changed + if prevRoundedCap, ok := previousRoundedCapValues[host]; ok { + if prevRoundedCap != roundedCapValue { + if err := rapl.Cap(host, "rapl", roundedCapValue); err != nil { + log.Println(err) + } + log.Printf("Capped [%s] at %d", host, int(math.Floor(capValue + 0.5))) + previousRoundedCapValues[host] = roundedCapValue + } + } else { + if err := rapl.Cap(host, "rapl", roundedCapValue); err != nil { + log.Println(err) + } + log.Printf("Capped [%s] at %d", host, int(math.Floor(capValue + 0.5))) + previousRoundedCapValues[host] = roundedCapValue + } + } + mutex.Unlock() + } + } + }() +} + +// Stop the capping +func (s *PistonCapper) stopCapping() { + if s.isCapping { + log.Println("Stopping the capping.") + s.ticker.Stop() + mutex.Lock() + s.isCapping = false + mutex.Unlock() + } +} + +// Check whether we are overloading the host (from watts perspective) +func wattsOverload(task def.Task, offerWatts float64, totalPower float64) bool { + if offerWatts >= (totalPower + (task.Watts * constants.CapMargin)) { + return false + } else { + return true + } +} + +func (s *PistonCapper) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { + log.Printf("Received %d resource offers", len(offers)) + + // retrieving the total power for each host in the offers + for _, offer := range offers { + if _, ok := s.totalPower[*offer.Hostname]; !ok { + _, _, offer_watts := OfferAgg(offer) + s.totalPower[*offer.Hostname] = offer_watts + } + } + + // Displaying the totalPower + for host, tpower := range s.totalPower { + log.Printf("TotalPower[%s] = %f", host, tpower) + } + + /* + Piston capping strategy + + Perform bin-packing of tasks on nodes in the cluster, making sure that no task is given less hard-limit resources than requested. + For each set of tasks that are scheduled, compute the new cap values for each host in the cluster. + At regular intervals of time, cap each node in the cluster. + */ + log.Printf("Number of tasks yet to be scheduled: %d", len(s.tasks)) + + + for _, offer := range offers { + select { + case <-s.Shutdown: + log.Println("Done scheduling tasks: declining offer on [", offer.GetHostname(), "]") + driver.DeclineOffer(offer.Id, longFilter) + + log.Println("Number of tasks still running: ", s.tasksRunning) + continue + default: + } + + fitTasks := []*mesos.TaskInfo{} + offerCPU, offerRAM, offerWatts := OfferAgg(offer) + taken := false + totalWatts := 0.0 + totalCPU := 0.0 + totalRAM := 0.0 + // Store the partialLoad for host corresponding to this offer. + // Once we can't fit any more tasks, we update capValue for this host with partialLoad and then launch the fit tasks. + partialLoad := 0.0 + for i, task := range s.tasks { + // Check host if it exists + if task.Host != "" { + // Don't take offer if it doens't match our task's host requirement. + if !strings.HasPrefix(*offer.Hostname, task.Host) { + continue + } + } + + for *task.Instances > 0 { + // Does the task fit + if (s.ignoreWatts || !wattsOverload(task, offerWatts, totalWatts)) && + (offerCPU >= (totalCPU + task.CPU)) && + (offerRAM >= (totalRAM + task.RAM)) { + + // Start piston capping if haven't started yet + if !s.isCapping { + s.isCapping = true + s.startCapping() + } + + taken = true + totalWatts += (task.Watts * constants.CapMargin) + totalCPU += task.CPU + totalRAM += task.RAM + log.Println("Co-Located with: ") + coLocated(s.running[offer.GetSlaveId().GoString()]) + fitTasks = append(fitTasks, s.newTask(offer, task)) + + log.Println("Inst: ", *task.Instances) + *task.Instances-- + // updating the cap value for offer.Hostname + partialLoad += ((task.Watts * constants.CapMargin) / s.totalPower[*offer.Hostname]) * 100 + + if *task.Instances <= 0 { + // All instances of task have been scheduled. Remove it + s.tasks = append(s.tasks[:i], s.tasks[i+1:]...) + if len(s.tasks) <= 0 { + log.Println("Done scheduling all tasks") + close(s.Shutdown) + } + } + } else { + break // Continue on to next task + } + } + } + + if taken { + // Updating the cap value for offer.Hostname + mutex.Lock() + capValues[*offer.Hostname] += partialLoad + mutex.Unlock() + log.Printf("Starting on [%s]\n", offer.GetHostname()) + driver.LaunchTasks([]*mesos.OfferID{offer.Id}, fitTasks, defaultFilter) + } else { + // If there was no match for task + log.Println("There is not enough resources to launch task: ") + cpus, mem, watts := OfferAgg(offer) + + log.Printf("\n", cpus, mem, watts) + driver.DeclineOffer(offer.Id, defaultFilter) + } + } +} + +// Remove finished task from the taskMonitor +func (s *PistonCapper) deleteFromTaskMonitor(finishedTaskID string) (def.Task, string, error) { + hostOfFinishedTask := "" + indexOfFinishedTask := -1 + found := false + var finishedTask def.Task + + for host, tasks := range s.taskMonitor { + for i, task := range tasks { + if task.TaskID == finishedTaskID { + hostOfFinishedTask = host + indexOfFinishedTask = i + found = true + } + } + if found { + break + } + } + + if hostOfFinishedTask != "" && indexOfFinishedTask != -1 { + finishedTask = s.taskMonitor[hostOfFinishedTask][indexOfFinishedTask] + log.Printf("Removing task with TaskID [%s] from the list of running tasks\n", + s.taskMonitor[hostOfFinishedTask][indexOfFinishedTask].TaskID) + s.taskMonitor[hostOfFinishedTask] = append(s.taskMonitor[hostOfFinishedTask][:indexOfFinishedTask], + s.taskMonitor[hostOfFinishedTask][indexOfFinishedTask+1:]...) + } else { + return finishedTask, hostOfFinishedTask, errors.New("Finished Task not present in TaskMonitor") + } + return finishedTask, hostOfFinishedTask, nil +} + +func (s *PistonCapper) StatusUpdate(driver sched.SchedulerDriver, status *mesos.TaskStatus) { + log.Printf("Received task status [%s] for task [%s]\n", NameFor(status.State), *status.TaskId.Value) + + if *status.State == mesos.TaskState_TASK_RUNNING { + mutex.Lock() + s.tasksRunning++ + mutex.Unlock() + } else if IsTerminal(status.State) { + delete(s.running[status.GetSlaveId().GoString()], *status.TaskId.Value) + // Deleting the task from the taskMonitor + finishedTask, hostOfFinishedTask, err := s.deleteFromTaskMonitor(*status.TaskId.Value) + if err != nil { + log.Println(err) + } + + // Need to update the cap values for host of the finishedTask + mutex.Lock() + capValues[hostOfFinishedTask] -= ((finishedTask.Watts * constants.CapMargin) / s.totalPower[hostOfFinishedTask]) * 100 + // Checking to see if the cap value has become 0, in which case we uncap the host. + if int(math.Floor(capValues[hostOfFinishedTask] + 0.5)) == 0 { + capValues[hostOfFinishedTask] = 100 + } + s.tasksRunning-- + mutex.Unlock() + + if s.tasksRunning == 0 { + select { + case <-s.Shutdown: + s.stopCapping() + close(s.Done) + default: + } + } + } + log.Printf("DONE: Task status [%s] for task [%s]", NameFor(status.State), *status.TaskId.Value) +} + +func (s *PistonCapper) FrameworkMessage( + driver sched.SchedulerDriver, + executorID *mesos.ExecutorID, + slaveID *mesos.SlaveID, + message string) { + + log.Println("Getting a framework message: ", message) + log.Printf("Received a framework message from some unknown source: %s", *executorID.Value) +} + +func (s *PistonCapper) OfferRescinded(_ sched.SchedulerDriver, offerID *mesos.OfferID) { + log.Printf("Offer %s rescinded", offerID) +} +func (s *PistonCapper) SlaveLost(_ sched.SchedulerDriver, slaveID *mesos.SlaveID) { + log.Printf("Slave %s lost", slaveID) +} +func (s *PistonCapper) ExecutorLost(_ sched.SchedulerDriver, executorID *mesos.ExecutorID, slaveID *mesos.SlaveID, status int) { + log.Printf("Executor %s on slave %s was lost", executorID, slaveID) +} + +func (s *PistonCapper) Error(_ sched.SchedulerDriver, err string) { + log.Printf("Receiving an error: %s", err) +} From 236be222cdcedc360afd841b76a0a79f19b852b6 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 15 Dec 2016 14:39:06 -0500 Subject: [PATCH 67/94] Logging of capping happens only when there is no error with rapl.Cap. --- schedulers/pistoncapper.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/schedulers/pistoncapper.go b/schedulers/pistoncapper.go index 60f2a56..eea64cf 100644 --- a/schedulers/pistoncapper.go +++ b/schedulers/pistoncapper.go @@ -165,15 +165,17 @@ func (s *PistonCapper) startCapping() { if prevRoundedCap != roundedCapValue { if err := rapl.Cap(host, "rapl", roundedCapValue); err != nil { log.Println(err) + } else { + log.Printf("Capped [%s] at %d", host, int(math.Floor(capValue + 0.5))) } - log.Printf("Capped [%s] at %d", host, int(math.Floor(capValue + 0.5))) previousRoundedCapValues[host] = roundedCapValue } } else { if err := rapl.Cap(host, "rapl", roundedCapValue); err != nil { log.Println(err) + } else { + log.Printf("Capped [%s] at %d", host, int(math.Floor(capValue + 0.5))) } - log.Printf("Capped [%s] at %d", host, int(math.Floor(capValue + 0.5))) previousRoundedCapValues[host] = roundedCapValue } } From a07806f570e544d5cb477776133a39964b2a038c Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 15 Dec 2016 15:33:47 -0500 Subject: [PATCH 68/94] Checked for task fitting watts requirement considering watts to be a hard limit. --- schedulers/pistoncapper.go | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/schedulers/pistoncapper.go b/schedulers/pistoncapper.go index eea64cf..002a5f2 100644 --- a/schedulers/pistoncapper.go +++ b/schedulers/pistoncapper.go @@ -196,15 +196,6 @@ func (s *PistonCapper) stopCapping() { } } -// Check whether we are overloading the host (from watts perspective) -func wattsOverload(task def.Task, offerWatts float64, totalPower float64) bool { - if offerWatts >= (totalPower + (task.Watts * constants.CapMargin)) { - return false - } else { - return true - } -} - func (s *PistonCapper) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) { log.Printf("Received %d resource offers", len(offers)) @@ -228,9 +219,6 @@ func (s *PistonCapper) ResourceOffers(driver sched.SchedulerDriver, offers []*me For each set of tasks that are scheduled, compute the new cap values for each host in the cluster. At regular intervals of time, cap each node in the cluster. */ - log.Printf("Number of tasks yet to be scheduled: %d", len(s.tasks)) - - for _, offer := range offers { select { case <-s.Shutdown: @@ -262,8 +250,8 @@ func (s *PistonCapper) ResourceOffers(driver sched.SchedulerDriver, offers []*me for *task.Instances > 0 { // Does the task fit - if (s.ignoreWatts || !wattsOverload(task, offerWatts, totalWatts)) && - (offerCPU >= (totalCPU + task.CPU)) && + if (s.ignoreWatts || (offerWatts >= (totalWatts + task.Watts))) && + (offerCPU >= (totalCPU + task.CPU)) && (offerRAM >= (totalRAM + task.RAM)) { // Start piston capping if haven't started yet @@ -273,7 +261,7 @@ func (s *PistonCapper) ResourceOffers(driver sched.SchedulerDriver, offers []*me } taken = true - totalWatts += (task.Watts * constants.CapMargin) + totalWatts += task.Watts totalCPU += task.CPU totalRAM += task.RAM log.Println("Co-Located with: ") From b5566c6719cc0bb0dca618190a1eaea001b30742 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 15 Dec 2016 15:37:31 -0500 Subject: [PATCH 69/94] Added a note to the command to run without ignoreWatts, to mention that Watts becomes a hard limit when not ignored. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 320e4e5..ca3922e 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ machine on which electron is launched for logging to work** How to run (Use the --help option to get information about other command-line options): `./electron -workload ` +*Here, watts would be considered a hard limit when fitting tasks with offers.* To run electron with ignoreWatts, run the following command, From 7c488d451befbb0b8883825e99d527d568a1ae3a Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 15 Dec 2016 15:38:07 -0500 Subject: [PATCH 70/94] formatted --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ca3922e..7b50a8a 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,8 @@ machine on which electron is launched for logging to work** How to run (Use the --help option to get information about other command-line options): -`./electron -workload ` +`./electron -workload ` + *Here, watts would be considered a hard limit when fitting tasks with offers.* To run electron with ignoreWatts, run the following command, From aeabaa7ce0564aba8fcdcd5197aaff93f911e9cb Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 15 Dec 2016 15:38:51 -0500 Subject: [PATCH 71/94] formatted --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b50a8a..4f8321a 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,8 @@ How to run (Use the --help option to get information about other command-line op `./electron -workload ` -*Here, watts would be considered a hard limit when fitting tasks with offers.* +*Here, watts would be considered a hard limit when fitting tasks with offers.* + To run electron with ignoreWatts, run the following command, From 16e25cea0f511c5fce51efc0e30733470fa826bb Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 15 Dec 2016 15:40:23 -0500 Subject: [PATCH 72/94] removed the explanation of ignoreWatts --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 4f8321a..9d5001e 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,6 @@ How to run (Use the --help option to get information about other command-line op `./electron -workload ` -*Here, watts would be considered a hard limit when fitting tasks with offers.* - - To run electron with ignoreWatts, run the following command, `./electron -workload -ignoreWatts` From bfcb254f23f9084e2db0a83383e484e6c54d7a97 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Fri, 16 Dec 2016 15:49:30 -0500 Subject: [PATCH 73/94] formatted the code --- schedulers/pistoncapper.go | 65 +++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/schedulers/pistoncapper.go b/schedulers/pistoncapper.go index 002a5f2..488a1cc 100644 --- a/schedulers/pistoncapper.go +++ b/schedulers/pistoncapper.go @@ -4,16 +4,17 @@ import ( "bitbucket.org/sunybingcloud/electron/constants" "bitbucket.org/sunybingcloud/electron/def" "bitbucket.org/sunybingcloud/electron/rapl" - "fmt" "errors" + "fmt" "github.com/golang/protobuf/proto" mesos "github.com/mesos/mesos-go/mesosproto" "github.com/mesos/mesos-go/mesosutil" sched "github.com/mesos/mesos-go/scheduler" "log" "math" - "sync" + "sort" "strings" + "sync" "time" ) @@ -26,15 +27,14 @@ import ( type PistonCapper struct { tasksCreated int tasksRunning int - tasks []def.Task - metrics map[string]def.Metric - running map[string]map[string]bool - taskMonitor map[string][]def.Task - clusterLoad map[string]float64 - totalPower map[string]float64 - ignoreWatts bool - ticker *time.Ticker - isCapping bool + tasks []def.Task + metrics map[string]def.Metric + running map[string]map[string]bool + taskMonitor map[string][]def.Task + totalPower map[string]float64 + ignoreWatts bool + ticker *time.Ticker + isCapping bool // First set of PCP values are garbage values, signal to logger to start recording when we're // about to schedule the new task. @@ -55,18 +55,17 @@ type PistonCapper struct { // New electron scheduler. func NewPistonCapper(tasks []def.Task, ignoreWatts bool) *PistonCapper { s := &PistonCapper{ - tasks: tasks, - ignoreWatts: ignoreWatts, - Shutdown: make(chan struct{}), - Done: make(chan struct{}), - PCPLog: make(chan struct{}), - running: make(map[string]map[string]bool), - taskMonitor: make(map[string][]def.Task), - clusterLoad: make(map[string]float64), - totalPower: make(map[string]float64), - RecordPCP: false, - ticker: time.NewTicker(10 * time.Second), - isCapping: false, + tasks: tasks, + ignoreWatts: ignoreWatts, + Shutdown: make(chan struct{}), + Done: make(chan struct{}), + PCPLog: make(chan struct{}), + running: make(map[string]map[string]bool), + taskMonitor: make(map[string][]def.Task), + totalPower: make(map[string]float64), + RecordPCP: false, + ticker: time.NewTicker(5 * time.Second), + isCapping: false, } return s } @@ -130,8 +129,6 @@ func (s *PistonCapper) newTask(offer *mesos.Offer, task def.Task) *mesos.TaskInf } } - - func (s *PistonCapper) Registered( _ sched.SchedulerDriver, frameworkID *mesos.FrameworkID, @@ -149,8 +146,10 @@ func (s *PistonCapper) Disconnected(sched.SchedulerDriver) { // go routine to cap the each node in the cluster at regular intervals of time. var capValues = make(map[string]float64) + // Storing the previous cap value for each host so as to not repeatedly cap the nodes to the same value. (reduces overhead) var previousRoundedCapValues = make(map[string]int) + func (s *PistonCapper) startCapping() { go func() { for { @@ -166,7 +165,7 @@ func (s *PistonCapper) startCapping() { if err := rapl.Cap(host, "rapl", roundedCapValue); err != nil { log.Println(err) } else { - log.Printf("Capped [%s] at %d", host, int(math.Floor(capValue + 0.5))) + log.Printf("Capped [%s] at %d", host, int(math.Floor(capValue+0.5))) } previousRoundedCapValues[host] = roundedCapValue } @@ -174,7 +173,7 @@ func (s *PistonCapper) startCapping() { if err := rapl.Cap(host, "rapl", roundedCapValue); err != nil { log.Println(err) } else { - log.Printf("Capped [%s] at %d", host, int(math.Floor(capValue + 0.5))) + log.Printf("Capped [%s] at %d", host, int(math.Floor(capValue+0.5))) } previousRoundedCapValues[host] = roundedCapValue } @@ -213,11 +212,11 @@ func (s *PistonCapper) ResourceOffers(driver sched.SchedulerDriver, offers []*me } /* - Piston capping strategy + Piston capping strategy - Perform bin-packing of tasks on nodes in the cluster, making sure that no task is given less hard-limit resources than requested. - For each set of tasks that are scheduled, compute the new cap values for each host in the cluster. - At regular intervals of time, cap each node in the cluster. + Perform bin-packing of tasks on nodes in the cluster, making sure that no task is given less hard-limit resources than requested. + For each set of tasks that are scheduled, compute the new cap values for each host in the cluster. + At regular intervals of time, cap each node in the cluster. */ for _, offer := range offers { select { @@ -251,7 +250,7 @@ func (s *PistonCapper) ResourceOffers(driver sched.SchedulerDriver, offers []*me for *task.Instances > 0 { // Does the task fit if (s.ignoreWatts || (offerWatts >= (totalWatts + task.Watts))) && - (offerCPU >= (totalCPU + task.CPU)) && + (offerCPU >= (totalCPU + task.CPU)) && (offerRAM >= (totalRAM + task.RAM)) { // Start piston capping if haven't started yet @@ -356,7 +355,7 @@ func (s *PistonCapper) StatusUpdate(driver sched.SchedulerDriver, status *mesos. mutex.Lock() capValues[hostOfFinishedTask] -= ((finishedTask.Watts * constants.CapMargin) / s.totalPower[hostOfFinishedTask]) * 100 // Checking to see if the cap value has become 0, in which case we uncap the host. - if int(math.Floor(capValues[hostOfFinishedTask] + 0.5)) == 0 { + if int(math.Floor(capValues[hostOfFinishedTask]+0.5)) == 0 { capValues[hostOfFinishedTask] = 100 } s.tasksRunning-- From 6ae5cd0cdd75097eaa8e95270a54d364f6671e01 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Fri, 16 Dec 2016 15:51:34 -0500 Subject: [PATCH 74/94] removed unnecessary import --- schedulers/pistoncapper.go | 1 - 1 file changed, 1 deletion(-) diff --git a/schedulers/pistoncapper.go b/schedulers/pistoncapper.go index 488a1cc..699bc0f 100644 --- a/schedulers/pistoncapper.go +++ b/schedulers/pistoncapper.go @@ -12,7 +12,6 @@ import ( sched "github.com/mesos/mesos-go/scheduler" "log" "math" - "sort" "strings" "sync" "time" From d69c5006ed74ace86cc9bbd8cf80100bd104f545 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Fri, 16 Dec 2016 15:52:14 -0500 Subject: [PATCH 75/94] removed quickSort algorithm to sort tasks. We can do this using the sort interface --- schedulers/proactiveclusterwidecappers.go | 35 ----------------------- 1 file changed, 35 deletions(-) diff --git a/schedulers/proactiveclusterwidecappers.go b/schedulers/proactiveclusterwidecappers.go index 98adc53..418980e 100644 --- a/schedulers/proactiveclusterwidecappers.go +++ b/schedulers/proactiveclusterwidecappers.go @@ -264,41 +264,6 @@ func (capper clusterwideCapper) recap(totalPower map[string]float64, } } -/* Quick sort algorithm to sort tasks, in place, in ascending order of power.*/ -func (capper clusterwideCapper) quickSort(low int, high int, tasksToSort *[]def.Task) { - i := low - j := high - // calculating the pivot - pivotIndex := low + (high-low)/2 - pivot := (*tasksToSort)[pivotIndex] - for i <= j { - for (*tasksToSort)[i].Watts < pivot.Watts { - i++ - } - for (*tasksToSort)[j].Watts > pivot.Watts { - j-- - } - if i <= j { - temp := (*tasksToSort)[i] - (*tasksToSort)[i] = (*tasksToSort)[j] - (*tasksToSort)[j] = temp - i++ - j-- - } - } - if low < j { - capper.quickSort(low, j, tasksToSort) - } - if i < high { - capper.quickSort(i, high, tasksToSort) - } -} - -// Sorting tasks in ascending order of requested watts. -func (capper clusterwideCapper) sortTasks(tasksToSort *[]def.Task) { - capper.quickSort(0, len(*tasksToSort)-1, tasksToSort) -} - /* Remove entry for finished task. This function is called when a task completes. From c493bd066eedf3a62039d9d0308424ac0832d3a3 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Fri, 16 Dec 2016 15:52:47 -0500 Subject: [PATCH 76/94] used sort interface instead of the quickSort algorithm to sort tasks in ascending order of watts. --- schedulers/proactiveclusterwidecappingranked.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schedulers/proactiveclusterwidecappingranked.go b/schedulers/proactiveclusterwidecappingranked.go index 69ae26f..b1c9a87 100644 --- a/schedulers/proactiveclusterwidecappingranked.go +++ b/schedulers/proactiveclusterwidecappingranked.go @@ -21,6 +21,7 @@ import ( sched "github.com/mesos/mesos-go/scheduler" "log" "math" + "sort" "strings" "sync" "time" @@ -263,7 +264,7 @@ func (s *ProactiveClusterwideCapRanked) ResourceOffers(driver sched.SchedulerDri // sorting the tasks in ascending order of watts. if (len(s.tasks) > 0) { - s.capper.sortTasks(&s.tasks) + sort.Sort(def.WattsSorter(s.tasks)) // calculating the total number of tasks ranked. numberOfRankedTasks := 0 for _, task := range s.tasks { From 2e0ec0cc99463d05d8763aae87bc4ce4272a3792 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Sun, 18 Dec 2016 14:30:04 -0500 Subject: [PATCH 77/94] Added Piston Capper to the list of schedulers --- schedulers/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/schedulers/README.md b/schedulers/README.md index 11f25a0..b0af70f 100644 --- a/schedulers/README.md +++ b/schedulers/README.md @@ -8,8 +8,9 @@ To Do: Scheduling Algorithms: + * First Fit + * First Fit with sorted watts * Bin-packing with sorted watts * FCFS Proactive Cluster-wide Capping * Ranked Proactive Cluster-wide Capping - * First Fit - * First Fit with sorted watts + * Piston Capping -- Works when scheduler is run with WAR From 63a7f0acb51874744c4d64291ca249fbd9b1bb94 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Sun, 18 Dec 2016 14:30:29 -0500 Subject: [PATCH 78/94] Moved the check for fitting tasks into a different function. --- schedulers/pistoncapper.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/schedulers/pistoncapper.go b/schedulers/pistoncapper.go index 699bc0f..9590553 100644 --- a/schedulers/pistoncapper.go +++ b/schedulers/pistoncapper.go @@ -69,6 +69,18 @@ func NewPistonCapper(tasks []def.Task, ignoreWatts bool) *PistonCapper { return s } +// check whether task fits the offer or not. +func (s *PistonCapper) takeOffer(offerWatts float64, offerCPU float64, offerRAM float64, totalWatts float64, totalCPU float64, + totalRAM float64, task def.Task) bool { + if (s.ignoreWatts || (offerWatts >= (totalWatts + task.Watts))) && + (offerCPU >= (totalCPU + task.CPU)) && + (offerRAM >= (totalRAM + task.RAM)) { + return true + } else { + return false + } +} + // mutex var mutex sync.Mutex @@ -248,9 +260,7 @@ func (s *PistonCapper) ResourceOffers(driver sched.SchedulerDriver, offers []*me for *task.Instances > 0 { // Does the task fit - if (s.ignoreWatts || (offerWatts >= (totalWatts + task.Watts))) && - (offerCPU >= (totalCPU + task.CPU)) && - (offerRAM >= (totalRAM + task.RAM)) { + if s.takeOffer(offerWatts, offerCPU, offerRAM, totalWatts, totalCPU, totalRAM, task) { // Start piston capping if haven't started yet if !s.isCapping { From 1321010d6f2996abd47802f29b24d1ab73d288c8 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Sun, 18 Dec 2016 14:31:37 -0500 Subject: [PATCH 79/94] Changed comment. The utility can be used to sort any map[string]float64 by value, and not just availablePower. --- utilities/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utilities/utils.go b/utilities/utils.go index c53df74..0421c4a 100644 --- a/utilities/utils.go +++ b/utilities/utils.go @@ -7,7 +7,7 @@ The Pair and PairList have been taken from google groups forum, https://groups.google.com/forum/#!topic/golang-nuts/FT7cjmcL7gw */ -// Utility struct that helps in sorting the available power by value. +// Utility that helps in sorting a map[string]float64 by value. type Pair struct { Key string Value float64 From 59b8303aaff1d4362503491e95ffd5769857d9f4 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 19 Dec 2016 16:30:03 -0500 Subject: [PATCH 80/94] Generic running average calculator. --- utilities/runAvg/runAvg.go | 78 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 utilities/runAvg/runAvg.go diff --git a/utilities/runAvg/runAvg.go b/utilities/runAvg/runAvg.go new file mode 100644 index 0000000..b0de3fd --- /dev/null +++ b/utilities/runAvg/runAvg.go @@ -0,0 +1,78 @@ +/* +A utility to calculate the running average. + +One should implement Val() to be able to use this utility. +*/ + +package runAvg + +import "container/list" + +type Interface interface { + // Value to use for running average calculation. + Val() float64 +} + +type runningAverageCalculator struct { + window list.List + windowSize int + currentSum float64 + numberOfElementsInWindow int +} + +// singleton instance +var racSingleton *runningAverageCalculator + +// return single instance +func getInstance(curSum float64, n int, wSize int) *runningAverageCalculator { + if racSingleton == nil { + racSingleton = &runningAverageCalculator { + windowSize: wSize, + currentSum: curSum, + numberOfElementsInWindow: n, + } + return racSingleton + } else { + // Updating window size if a new window size is given. + if wSize != racSingleton.windowSize { + racSingleton.windowSize = wSize + } + return racSingleton + } +} + +// Compute the running average by adding 'data' to the window. +// Updating currentSum to get constant time complexity for every running average computation. +func (rac *runningAverageCalculator) calculate(data Interface) float64 { + if rac.numberOfElementsInWindow < rac.windowSize { + rac.window.PushBack(data) + rac.numberOfElementsInWindow++ + rac.currentSum += data.Val() + } else { + // removing the element at the front of the window. + elementToRemove := rac.window.Front() + rac.currentSum -= elementToRemove.Value.(Interface).Val() + rac.window.Remove(elementToRemove) + + // adding new element to the window + rac.window.PushBack(data) + rac.currentSum += data.Val() + } + return rac.currentSum / float64(rac.window.Len()) +} + +// Taking windowSize as a parameter to allow for sliding window implementation. +func Calc(data Interface, windowSize int) float64 { + rac := getInstance(0.0, 0, windowSize) + return rac.calculate(data) +} + +// initialize the parameters of the running average calculator +func Init() { + // Setting parameters to default values. Could also set racSingleton to nil but this leads to unnecessary overhead of creating + // another instance when Calc is called. + racSingleton.window.Init() + racSingleton.windowSize = 0 + racSingleton.currentSum = 0.0 + racSingleton.numberOfElementsInWindow = 0 +} \ No newline at end of file From 822ef0067ce39bbde99f777442c639770ff12d34 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 19 Dec 2016 16:31:55 -0500 Subject: [PATCH 81/94] Removed TODO for creating generic running average calculator. Added TODO to use the generic running average calculator in proactiveclusterwidecappers.go --- schedulers/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schedulers/README.md b/schedulers/README.md index b0af70f..480816c 100644 --- a/schedulers/README.md +++ b/schedulers/README.md @@ -4,7 +4,7 @@ Electron: Scheduling Algorithms To Do: * Design changes -- Possible to have one scheduler with different scheduling schemes? - * Make the running average calculation generic, so that schedulers in the future can use it and not implement their own. + * Use the generic running average calculator in utilities/runAvg in schedulers/proactiveclusterwidecappers.go Scheduling Algorithms: From 7aea52991b7dd23cdabce46a9e8fb9995e94d0b8 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 19 Dec 2016 16:46:02 -0500 Subject: [PATCH 82/94] TODO to add another functionality to the runAvg interface to be able to remove element from the window based on a criteria. --- schedulers/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/schedulers/README.md b/schedulers/README.md index 480816c..8ad8e09 100644 --- a/schedulers/README.md +++ b/schedulers/README.md @@ -5,6 +5,7 @@ To Do: * Design changes -- Possible to have one scheduler with different scheduling schemes? * Use the generic running average calculator in utilities/runAvg in schedulers/proactiveclusterwidecappers.go + * Added functionality to the runAvg interface to be able to remove element in the window based on a criteria. Scheduling Algorithms: From bd3802ddfdc6fb3fe25d4b3e687b9bbde24ad2f3 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 19 Dec 2016 16:46:52 -0500 Subject: [PATCH 83/94] Made change to TODO --- schedulers/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schedulers/README.md b/schedulers/README.md index 8ad8e09..72c2f34 100644 --- a/schedulers/README.md +++ b/schedulers/README.md @@ -5,7 +5,7 @@ To Do: * Design changes -- Possible to have one scheduler with different scheduling schemes? * Use the generic running average calculator in utilities/runAvg in schedulers/proactiveclusterwidecappers.go - * Added functionality to the runAvg interface to be able to remove element in the window based on a criteria. + * Add functionality to the runAvg interface to be able to remove element in the window based on a criteria. Scheduling Algorithms: From e0a16da97ae0c51f27021a17d2399f95c2b44f1b Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Mon, 19 Dec 2016 20:43:25 -0500 Subject: [PATCH 84/94] Added functionality to be able to remove elements from the window by providing a unique ID. Also, added another function to the interface that needs to be implemented by all the structs implementing this interface. This function should return a unique ID identifying the object so as to be able to distinguish between multiple instances of the same type. --- utilities/runAvg/runAvg.go | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/utilities/runAvg/runAvg.go b/utilities/runAvg/runAvg.go index b0de3fd..f832732 100644 --- a/utilities/runAvg/runAvg.go +++ b/utilities/runAvg/runAvg.go @@ -6,11 +6,16 @@ One should implement Val() to be able to use this utility. package runAvg -import "container/list" +import ( + "errors" + "container/list" +) type Interface interface { // Value to use for running average calculation. Val() float64 + // Unique ID + ID() string } type runningAverageCalculator struct { @@ -61,14 +66,42 @@ func (rac *runningAverageCalculator) calculate(data Interface) float64 { return rac.currentSum / float64(rac.window.Len()) } +/* +If element with given ID present in the window, then remove it and return (removeElement, nil). +Else, return (nil, error) +*/ +func (rac *runningAverageCalculator) removeFromWindow(id string) (interface{}, error) { + for element := rac.window.Front(); element != nil; element = element.Next() { + if elementToRemove := element.Value.(Interface); elementToRemove.ID() == id { + rac.window.Remove(element) + return elementToRemove, nil + } + } + return nil, errors.New("Error: Element not found in the window.") +} + // Taking windowSize as a parameter to allow for sliding window implementation. func Calc(data Interface, windowSize int) float64 { rac := getInstance(0.0, 0, windowSize) return rac.calculate(data) } +// Remove element from the window if it is present. +func Remove(id string) (interface{}, error) { + // checking if racSingleton has been instantiated + if racSingleton == nil { + return nil, errors.New("Error: Not instantiated. Please call Init() to instantiate.") + } else { + return racSingleton.removeFromWindow(id) + } +} + // initialize the parameters of the running average calculator func Init() { + // checking to see if racSingleton needs top be instantiated + if racSingleton == nil { + racSingleton = getInstance(0.0, 0, 0) + } // Setting parameters to default values. Could also set racSingleton to nil but this leads to unnecessary overhead of creating // another instance when Calc is called. racSingleton.window.Init() From 1b821735ed6663d9d533c449a54b072a6fb83cb1 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Tue, 20 Dec 2016 13:53:28 -0500 Subject: [PATCH 85/94] fixed bug in Remove() where the currentSum and numberOfTasksInWindow weren't getting updated. --- utilities/runAvg/runAvg.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utilities/runAvg/runAvg.go b/utilities/runAvg/runAvg.go index f832732..993577f 100644 --- a/utilities/runAvg/runAvg.go +++ b/utilities/runAvg/runAvg.go @@ -74,6 +74,8 @@ func (rac *runningAverageCalculator) removeFromWindow(id string) (interface{}, e for element := rac.window.Front(); element != nil; element = element.Next() { if elementToRemove := element.Value.(Interface); elementToRemove.ID() == id { rac.window.Remove(element) + rac.currentSum -= elementToRemove.Val() + rac.numberOfElementsInWindow-- return elementToRemove, nil } } From f304cd295afc13df4e723897f491e4147687dc93 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Tue, 20 Dec 2016 14:55:46 -0500 Subject: [PATCH 86/94] Completed couple of TODOs and added two more. --- schedulers/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schedulers/README.md b/schedulers/README.md index 72c2f34..7492e56 100644 --- a/schedulers/README.md +++ b/schedulers/README.md @@ -4,8 +4,8 @@ Electron: Scheduling Algorithms To Do: * Design changes -- Possible to have one scheduler with different scheduling schemes? - * Use the generic running average calculator in utilities/runAvg in schedulers/proactiveclusterwidecappers.go - * Add functionality to the runAvg interface to be able to remove element in the window based on a criteria. + * Fix the race condition on 'tasksRunning' in proactiveclusterwidecappingfcfs.go and proactiveclusterwidecappingranked.go + * Separate the capping strategies from the scheduling algorithms and make it possible to use any capping strategy with any scheduler. Scheduling Algorithms: From bf6c5eded9d9169707c23234af8e7ff375f33dce Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Tue, 20 Dec 2016 14:56:07 -0500 Subject: [PATCH 87/94] used the generic running average calculator --- schedulers/proactiveclusterwidecappers.go | 102 ++++++---------------- 1 file changed, 25 insertions(+), 77 deletions(-) diff --git a/schedulers/proactiveclusterwidecappers.go b/schedulers/proactiveclusterwidecappers.go index 418980e..275aaa8 100644 --- a/schedulers/proactiveclusterwidecappers.go +++ b/schedulers/proactiveclusterwidecappers.go @@ -1,12 +1,5 @@ /* Cluster wide dynamic capping -Step1. Compute the running average of watts of tasks in window. -Step2. Compute what percentage of total power of each node, is the running average. -Step3. Compute the median of the percetages and this is the percentage that the cluster needs to be capped at. - -1. First fit scheduling -- Perform the above steps for each task that needs to be scheduled. -2. Ranked based scheduling -- Sort the tasks to be scheduled, in ascending order, and then determine the cluster wide cap. - This is not a scheduler but a scheduling scheme that schedulers can use. */ package schedulers @@ -14,26 +7,32 @@ package schedulers import ( "bitbucket.org/sunybingcloud/electron/constants" "bitbucket.org/sunybingcloud/electron/def" - "container/list" + "bitbucket.org/sunybingcloud/electron/utilities/runAvg" "errors" "github.com/montanaflynn/stats" "log" "sort" ) -// Structure containing utility data structures used to compute cluster-wide dynamic cap. -type clusterwideCapper struct { - // window of tasks. - windowOfTasks list.List - // The current sum of requested powers of the tasks in the window. - currentSum float64 - // The current number of tasks in the window. - numberOfTasksInWindow int +// wrapper around def.Task that implements runAvg.Interface +type taskWrapper struct { + task def.Task } -// Defining constructor for clusterwideCapper. Please don't call this directly and instead use getClusterwideCapperInstance(). +func (tw taskWrapper) Val() float64 { + return tw.task.Watts * constants.CapMargin +} + +func (tw taskWrapper) ID() string { + return tw.task.TaskID +} + +// Cluster wide capper. Contains a type that implements runAvg.Interface +type clusterwideCapper struct {} + +// Defining constructor for clusterwideCapper. Please don't call this directly and instead use getClusterwideCapperInstance() func newClusterwideCapper() *clusterwideCapper { - return &clusterwideCapper{currentSum: 0.0, numberOfTasksInWindow: 0} + return &clusterwideCapper{} } // Singleton instance of clusterwideCapper @@ -49,41 +48,9 @@ func getClusterwideCapperInstance() *clusterwideCapper { return singletonCapper } -// Clear and initialize all the members of clusterwideCapper. +// Clear and initialize the runAvg calculator func (capper clusterwideCapper) clear() { - capper.windowOfTasks.Init() - capper.currentSum = 0 - capper.numberOfTasksInWindow = 0 -} - -// Compute the average of watts of all the tasks in the window. -func (capper clusterwideCapper) average() float64 { - return capper.currentSum / float64(capper.windowOfTasks.Len()) -} - -/* -Compute the running average. - -Using clusterwideCapper#windowOfTasks to store the tasks. -Task at position 0 (oldest task) is removed when the window is full and new task arrives. -*/ -func (capper clusterwideCapper) runningAverageOfWatts(tsk *def.Task) float64 { - var average float64 - if capper.numberOfTasksInWindow < constants.WindowSize { - capper.windowOfTasks.PushBack(tsk) - capper.numberOfTasksInWindow++ - capper.currentSum += float64(tsk.Watts) * constants.CapMargin - } else { - taskToRemoveElement := capper.windowOfTasks.Front() - if taskToRemove, ok := taskToRemoveElement.Value.(*def.Task); ok { - capper.currentSum -= float64(taskToRemove.Watts) * constants.CapMargin - capper.windowOfTasks.Remove(taskToRemoveElement) - } - capper.windowOfTasks.PushBack(tsk) - capper.currentSum += float64(tsk.Watts) * constants.CapMargin - } - average = capper.average() - return average + runAvg.Init() } /* @@ -265,33 +232,14 @@ func (capper clusterwideCapper) recap(totalPower map[string]float64, } /* -Remove entry for finished task. -This function is called when a task completes. +Remove entry for finished task from the window + +This function is called when a task completes. This completed task needs to be removed from the window of tasks (if it is still present) - so that it doesn't contribute to the computation of the cap value. + so that it doesn't contribute to the computation of the next cap value. */ func (capper clusterwideCapper) taskFinished(taskID string) { - // If the window is empty the just return. This condition should technically return false. - if capper.windowOfTasks.Len() == 0 { - return - } - - // Checking whether the task with the given taskID is currently present in the window of tasks. - var taskElementToRemove *list.Element - for taskElement := capper.windowOfTasks.Front(); taskElement != nil; taskElement = taskElement.Next() { - if tsk, ok := taskElement.Value.(*def.Task); ok { - if tsk.TaskID == taskID { - taskElementToRemove = taskElement - } - } - } - - // we need to remove the task from the window. - if taskToRemove, ok := taskElementToRemove.Value.(*def.Task); ok { - capper.windowOfTasks.Remove(taskElementToRemove) - capper.numberOfTasksInWindow -= 1 - capper.currentSum -= float64(taskToRemove.Watts) * constants.CapMargin - } + runAvg.Remove(taskID) } // First come first serve scheduling. @@ -302,7 +250,7 @@ func (capper clusterwideCapper) fcfsDetermineCap(totalPower map[string]float64, return 100, errors.New("Invalid argument: totalPower") } else { // Need to calculate the running average - runningAverage := capper.runningAverageOfWatts(newTask) + runningAverage := runAvg.Calc(taskWrapper{task: *newTask}, constants.WindowSize) // For each node, calculate the percentage of the running average to the total power. ratios := make(map[string]float64) for host, tpower := range totalPower { From 9b5ac0bfa843ae7661d521dbc4041e6918ff56e7 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Tue, 20 Dec 2016 15:03:43 -0500 Subject: [PATCH 88/94] removed unnecessary variable 'numberOfElementsInWindow' and just used window.Len() instead --- utilities/runAvg/runAvg.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/utilities/runAvg/runAvg.go b/utilities/runAvg/runAvg.go index 993577f..f022715 100644 --- a/utilities/runAvg/runAvg.go +++ b/utilities/runAvg/runAvg.go @@ -22,19 +22,17 @@ type runningAverageCalculator struct { window list.List windowSize int currentSum float64 - numberOfElementsInWindow int } // singleton instance var racSingleton *runningAverageCalculator // return single instance -func getInstance(curSum float64, n int, wSize int) *runningAverageCalculator { +func getInstance(curSum float64, wSize int) *runningAverageCalculator { if racSingleton == nil { racSingleton = &runningAverageCalculator { windowSize: wSize, currentSum: curSum, - numberOfElementsInWindow: n, } return racSingleton } else { @@ -49,9 +47,8 @@ func getInstance(curSum float64, n int, wSize int) *runningAverageCalculator { // Compute the running average by adding 'data' to the window. // Updating currentSum to get constant time complexity for every running average computation. func (rac *runningAverageCalculator) calculate(data Interface) float64 { - if rac.numberOfElementsInWindow < rac.windowSize { + if rac.window.Len() < rac.windowSize { rac.window.PushBack(data) - rac.numberOfElementsInWindow++ rac.currentSum += data.Val() } else { // removing the element at the front of the window. @@ -75,7 +72,6 @@ func (rac *runningAverageCalculator) removeFromWindow(id string) (interface{}, e if elementToRemove := element.Value.(Interface); elementToRemove.ID() == id { rac.window.Remove(element) rac.currentSum -= elementToRemove.Val() - rac.numberOfElementsInWindow-- return elementToRemove, nil } } @@ -84,7 +80,7 @@ func (rac *runningAverageCalculator) removeFromWindow(id string) (interface{}, e // Taking windowSize as a parameter to allow for sliding window implementation. func Calc(data Interface, windowSize int) float64 { - rac := getInstance(0.0, 0, windowSize) + rac := getInstance(0.0, windowSize) return rac.calculate(data) } @@ -102,12 +98,11 @@ func Remove(id string) (interface{}, error) { func Init() { // checking to see if racSingleton needs top be instantiated if racSingleton == nil { - racSingleton = getInstance(0.0, 0, 0) + racSingleton = getInstance(0.0, 0) } // Setting parameters to default values. Could also set racSingleton to nil but this leads to unnecessary overhead of creating // another instance when Calc is called. racSingleton.window.Init() racSingleton.windowSize = 0 racSingleton.currentSum = 0.0 - racSingleton.numberOfElementsInWindow = 0 } \ No newline at end of file From 4ef038f22f79f79831e0b4855c5849378e40ab7f Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Tue, 20 Dec 2016 15:07:31 -0500 Subject: [PATCH 89/94] updated comment --- schedulers/proactiveclusterwidecappers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schedulers/proactiveclusterwidecappers.go b/schedulers/proactiveclusterwidecappers.go index 275aaa8..98bdc08 100644 --- a/schedulers/proactiveclusterwidecappers.go +++ b/schedulers/proactiveclusterwidecappers.go @@ -27,7 +27,7 @@ func (tw taskWrapper) ID() string { return tw.task.TaskID } -// Cluster wide capper. Contains a type that implements runAvg.Interface +// Cluster wide capper type clusterwideCapper struct {} // Defining constructor for clusterwideCapper. Please don't call this directly and instead use getClusterwideCapperInstance() From 373a437bae8e0b0a150508d32a134eff6e88a77c Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Tue, 20 Dec 2016 16:27:44 -0500 Subject: [PATCH 90/94] Added TODO to setup the constants at runtime. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9d5001e..65c77f6 100644 --- a/README.md +++ b/README.md @@ -9,14 +9,14 @@ To Do: * Running average calculations https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average * Make parameters corresponding to each scheduler configurable (possible to have a config template for each scheduler?) * Write test code for each scheduler (This should be after the design change) - + * Some of the constants in constants/constants.go can vary based on the environment. + Possible to setup the constants at runtime based on the environment? **Requires Performance-Copilot tool pmdumptext to be installed on the machine on which electron is launched for logging to work** - How to run (Use the --help option to get information about other command-line options): `./electron -workload ` From 45d84f94851e7814b91d917399c4300ca573163a Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Tue, 20 Dec 2016 16:28:03 -0500 Subject: [PATCH 91/94] formatted code --- schedulers/pistoncapper.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schedulers/pistoncapper.go b/schedulers/pistoncapper.go index 9590553..d5a22d1 100644 --- a/schedulers/pistoncapper.go +++ b/schedulers/pistoncapper.go @@ -70,8 +70,8 @@ func NewPistonCapper(tasks []def.Task, ignoreWatts bool) *PistonCapper { } // check whether task fits the offer or not. -func (s *PistonCapper) takeOffer(offerWatts float64, offerCPU float64, offerRAM float64, totalWatts float64, totalCPU float64, - totalRAM float64, task def.Task) bool { +func (s *PistonCapper) takeOffer(offerWatts float64, offerCPU float64, offerRAM float64, + totalWatts float64, totalCPU float64, totalRAM float64, task def.Task) bool { if (s.ignoreWatts || (offerWatts >= (totalWatts + task.Watts))) && (offerCPU >= (totalCPU + task.CPU)) && (offerRAM >= (totalRAM + task.RAM)) { From f829cfea66e3f45d32f0e1f3a1cfc62c352ce0de Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 22 Dec 2016 13:49:43 -0500 Subject: [PATCH 92/94] Moved it to pcp/ as it is not a scheduler and is a capping strategy --- .../proactiveclusterwidecappers.go | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) rename {schedulers => pcp}/proactiveclusterwidecappers.go (84%) diff --git a/schedulers/proactiveclusterwidecappers.go b/pcp/proactiveclusterwidecappers.go similarity index 84% rename from schedulers/proactiveclusterwidecappers.go rename to pcp/proactiveclusterwidecappers.go index cc4da2b..fe94022 100644 --- a/schedulers/proactiveclusterwidecappers.go +++ b/pcp/proactiveclusterwidecappers.go @@ -2,7 +2,7 @@ Cluster wide dynamic capping this is not a scheduler but a scheduling scheme that schedulers can use. */ -package schedulers +package pcp import ( "bitbucket.org/sunybingcloud/electron/constants" @@ -28,18 +28,18 @@ func (tw taskWrapper) ID() string { } // Cluster wide capper -type clusterwideCapper struct {} +type ClusterwideCapper struct {} -// Defining constructor for clusterwideCapper. Please don't call this directly and instead use getClusterwideCapperInstance() -func newClusterwideCapper() *clusterwideCapper { - return &clusterwideCapper{} +// Defining constructor for clusterwideCapper. Please don't call this directly and instead use GetClusterwideCapperInstance() +func newClusterwideCapper() *ClusterwideCapper { + return &ClusterwideCapper{} } // Singleton instance of clusterwideCapper -var singletonCapper *clusterwideCapper +var singletonCapper *ClusterwideCapper // Retrieve the singleton instance of clusterwideCapper. -func getClusterwideCapperInstance() *clusterwideCapper { +func GetClusterwideCapperInstance() *ClusterwideCapper { if singletonCapper == nil { singletonCapper = newClusterwideCapper() } else { @@ -49,7 +49,7 @@ func getClusterwideCapperInstance() *clusterwideCapper { } // Clear and initialize the runAvg calculator -func (capper clusterwideCapper) clear() { +func (capper ClusterwideCapper) clear() { runAvg.Init() } @@ -60,7 +60,7 @@ Calculating cap value. 2. Computing the median of above sorted values. 3. The median is now the cap. */ -func (capper clusterwideCapper) getCap(ratios map[string]float64) float64 { +func (capper ClusterwideCapper) getCap(ratios map[string]float64) float64 { var values []float64 // Validation if ratios == nil { @@ -80,25 +80,25 @@ func (capper clusterwideCapper) getCap(ratios map[string]float64) float64 { } /* -A recapping strategy which decides between 2 different recapping schemes. +A Recapping strategy which decides between 2 different Recapping schemes. 1. the regular scheme based on the average power usage across the cluster. 2. A scheme based on the average of the loads on each node in the cluster. -The recap value picked the least among the two. +The Recap value picked the least among the two. -The cleverRecap scheme works well when the cluster is relatively idle and until then, - the primitive recapping scheme works better. +The CleverRecap scheme works well when the cluster is relatively idle and until then, + the primitive Recapping scheme works better. */ -func (capper clusterwideCapper) cleverRecap(totalPower map[string]float64, +func (capper ClusterwideCapper) CleverRecap(totalPower map[string]float64, taskMonitor map[string][]def.Task, finishedTaskId string) (float64, error) { // Validation if totalPower == nil || taskMonitor == nil { return 100.0, errors.New("Invalid argument: totalPower, taskMonitor") } - // determining the recap value by calling the regular recap(...) + // determining the Recap value by calling the regular Recap(...) toggle := false - recapValue, err := capper.recap(totalPower, taskMonitor, finishedTaskId) + RecapValue, err := capper.Recap(totalPower, taskMonitor, finishedTaskId) if err == nil { toggle = true } @@ -122,8 +122,8 @@ func (capper clusterwideCapper) cleverRecap(totalPower map[string]float64, } } - // Updating task monitor. If recap(...) has deleted the finished task from the taskMonitor, - // then this will be ignored. Else (this is only when an error occured with recap(...)), we remove it here. + // Updating task monitor. If Recap(...) has deleted the finished task from the taskMonitor, + // then this will be ignored. Else (this is only when an error occured with Recap(...)), we remove it here. if hostOfFinishedTask != "" && indexOfFinishedTask != -1 { log.Printf("Removing task with task [%s] from the list of running tasks\n", taskMonitor[hostOfFinishedTask][indexOfFinishedTask].TaskID) @@ -156,12 +156,12 @@ func (capper clusterwideCapper) cleverRecap(totalPower map[string]float64, totalLoad += load } averageLoad := (totalLoad / float64(len(loads)) * 100.0) // this would be the cap value. - // If toggle is true, then we need to return the least recap value. + // If toggle is true, then we need to return the least Recap value. if toggle { - if averageLoad <= recapValue { + if averageLoad <= RecapValue { return averageLoad, nil } else { - return recapValue, nil + return RecapValue, nil } } else { return averageLoad, nil @@ -180,7 +180,7 @@ Recapping the entire cluster. This needs to be called whenever a task finishes execution. */ -func (capper clusterwideCapper) recap(totalPower map[string]float64, +func (capper ClusterwideCapper) Recap(totalPower map[string]float64, taskMonitor map[string][]def.Task, finishedTaskId string) (float64, error) { // Validation if totalPower == nil || taskMonitor == nil { @@ -238,12 +238,12 @@ This function is called when a task completes. This completed task needs to be removed from the window of tasks (if it is still present) so that it doesn't contribute to the computation of the next cap value. */ -func (capper clusterwideCapper) taskFinished(taskID string) { +func (capper ClusterwideCapper) TaskFinished(taskID string) { runAvg.Remove(taskID) } // First come first serve scheduling. -func (capper clusterwideCapper) fcfsDetermineCap(totalPower map[string]float64, +func (capper ClusterwideCapper) FCFSDeterminedCap(totalPower map[string]float64, newTask *def.Task) (float64, error) { // Validation if totalPower == nil { @@ -269,6 +269,6 @@ func (capper clusterwideCapper) fcfsDetermineCap(totalPower map[string]float64, } // Stringer for an instance of clusterwideCapper -func (capper clusterwideCapper) string() string { +func (capper ClusterwideCapper) String() string { return "Cluster Capper -- Proactively cap the entire cluster." } From 7d0a90277358c8e924d4c6549de9cf8f90a73331 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 22 Dec 2016 13:50:37 -0500 Subject: [PATCH 93/94] Used the capping strategy 'proactiveclusterwidecappers' from pcp/ instead of from the same package as it was moved --- schedulers/proactiveclusterwidecappingfcfs.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/schedulers/proactiveclusterwidecappingfcfs.go b/schedulers/proactiveclusterwidecappingfcfs.go index a96d496..68b096e 100644 --- a/schedulers/proactiveclusterwidecappingfcfs.go +++ b/schedulers/proactiveclusterwidecappingfcfs.go @@ -3,6 +3,7 @@ package schedulers import ( "bitbucket.org/sunybingcloud/electron/constants" "bitbucket.org/sunybingcloud/electron/def" + "bitbucket.org/sunybingcloud/electron/pcp" "bitbucket.org/sunybingcloud/electron/rapl" "fmt" "github.com/golang/protobuf/proto" @@ -37,7 +38,7 @@ type ProactiveClusterwideCapFCFS struct { availablePower map[string]float64 // available power for each node in the cluster. totalPower map[string]float64 // total power for each node in the cluster. ignoreWatts bool - capper *clusterwideCapper + capper *pcp.ClusterwideCapper ticker *time.Ticker recapTicker *time.Ticker isCapping bool // indicate whether we are currently performing cluster wide capping. @@ -72,7 +73,7 @@ func NewProactiveClusterwideCapFCFS(tasks []def.Task, ignoreWatts bool) *Proacti availablePower: make(map[string]float64), totalPower: make(map[string]float64), RecordPCP: false, - capper: getClusterwideCapperInstance(), + capper: pcp.GetClusterwideCapperInstance(), ticker: time.NewTicker(10 * time.Second), recapTicker: time.NewTicker(20 * time.Second), isCapping: false, @@ -290,7 +291,7 @@ func (s *ProactiveClusterwideCapFCFS) ResourceOffers(driver sched.SchedulerDrive s.startCapping() } taken = true - tempCap, err := s.capper.fcfsDetermineCap(s.totalPower, &task) + tempCap, err := s.capper.FCFSDeterminedCap(s.totalPower, &task) if err == nil { fcfsMutex.Lock() @@ -345,10 +346,10 @@ func (s *ProactiveClusterwideCapFCFS) StatusUpdate(driver sched.SchedulerDriver, } else if IsTerminal(status.State) { delete(s.running[status.GetSlaveId().GoString()], *status.TaskId.Value) // Need to remove the task from the window of tasks. - s.capper.taskFinished(*status.TaskId.Value) + s.capper.TaskFinished(*status.TaskId.Value) // Determining the new cluster wide cap. - //tempCap, err := s.capper.recap(s.totalPower, s.taskMonitor, *status.TaskId.Value) - tempCap, err := s.capper.cleverRecap(s.totalPower, s.taskMonitor, *status.TaskId.Value) + //tempCap, err := s.capper.Recap(s.totalPower, s.taskMonitor, *status.TaskId.Value) + tempCap, err := s.capper.CleverRecap(s.totalPower, s.taskMonitor, *status.TaskId.Value) if err == nil { // if new determined cap value is different from the current recap value then we need to recap. if int(math.Floor(tempCap+0.5)) != int(math.Floor(fcfsRecapValue+0.5)) { From b25158336d3c297b59abc75ee7008ce4ee0f2941 Mon Sep 17 00:00:00 2001 From: Pradyumna Kaushik Date: Thu, 22 Dec 2016 13:51:08 -0500 Subject: [PATCH 94/94] Used the capping strategy 'proactiveclusterwidecappers' from pcp/ instead of from the same package as it was moved. --- schedulers/proactiveclusterwidecappingranked.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/schedulers/proactiveclusterwidecappingranked.go b/schedulers/proactiveclusterwidecappingranked.go index b1c9a87..85e8537 100644 --- a/schedulers/proactiveclusterwidecappingranked.go +++ b/schedulers/proactiveclusterwidecappingranked.go @@ -13,6 +13,7 @@ package schedulers import ( "bitbucket.org/sunybingcloud/electron/constants" "bitbucket.org/sunybingcloud/electron/def" + "bitbucket.org/sunybingcloud/electron/pcp" "bitbucket.org/sunybingcloud/electron/rapl" "fmt" "github.com/golang/protobuf/proto" @@ -48,7 +49,7 @@ type ProactiveClusterwideCapRanked struct { availablePower map[string]float64 // available power for each node in the cluster. totalPower map[string]float64 // total power for each node in the cluster. ignoreWatts bool - capper *clusterwideCapper + capper *pcp.ClusterwideCapper ticker *time.Ticker recapTicker *time.Ticker isCapping bool // indicate whether we are currently performing cluster wide capping. @@ -83,7 +84,7 @@ func NewProactiveClusterwideCapRanked(tasks []def.Task, ignoreWatts bool) *Proac availablePower: make(map[string]float64), totalPower: make(map[string]float64), RecordPCP: false, - capper: getClusterwideCapperInstance(), + capper: pcp.GetClusterwideCapperInstance(), ticker: time.NewTicker(10 * time.Second), recapTicker: time.NewTicker(20 * time.Second), isCapping: false, @@ -314,7 +315,7 @@ func (s *ProactiveClusterwideCapRanked) ResourceOffers(driver sched.SchedulerDri s.startCapping() } taken = true - tempCap, err := s.capper.fcfsDetermineCap(s.totalPower, &task) + tempCap, err := s.capper.FCFSDeterminedCap(s.totalPower, &task) if err == nil { rankedMutex.Lock() @@ -380,10 +381,10 @@ func (s *ProactiveClusterwideCapRanked) StatusUpdate(driver sched.SchedulerDrive } } else { // Need to remove the task from the window - s.capper.taskFinished(*status.TaskId.Value) + s.capper.TaskFinished(*status.TaskId.Value) // Determining the new cluster wide cap. - //tempCap, err := s.capper.recap(s.totalPower, s.taskMonitor, *status.TaskId.Value) - tempCap, err := s.capper.cleverRecap(s.totalPower, s.taskMonitor, *status.TaskId.Value) + //tempCap, err := s.capper.Recap(s.totalPower, s.taskMonitor, *status.TaskId.Value) + tempCap, err := s.capper.CleverRecap(s.totalPower, s.taskMonitor, *status.TaskId.Value) if err == nil { // If new determined cap value is different from the current recap value then we need to recap.