diff --git a/constants/constants.go b/constants/constants.go index 834d65f..e1045d5 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -1,8 +1,18 @@ /* Constants that are used across scripts 1. The available hosts = stratos-00x (x varies from 1 to 8) +<<<<<<< HEAD 2. CapMargin = percentage of the requested power to allocate 3. ConsiderationWindowSize = number of tasks to consider for computation of the dynamic cap. +======= +2. cap_margin = percentage of the requested power to allocate +3. power_threshold = overloading factor +5. window_size = number of tasks to consider for computation of the dynamic cap. + +Also, exposing functions to update or initialize some of the constants. + +TODO: Clean this up and use Mesos Attributes instead. +>>>>>>> a0a3e78041067e5e2f9dc9b5d1e7b6dd001ce1e9 */ package constants diff --git a/schedulers/binPackSortedWattsSortedOffers.go b/schedulers/binPackSortedWattsSortedOffers.go index 9c27aad..809e2e6 100644 --- a/schedulers/binPackSortedWattsSortedOffers.go +++ b/schedulers/binPackSortedWattsSortedOffers.go @@ -12,26 +12,23 @@ import ( "log" "os" "sort" - "strings" "time" ) // Decides if to take an offer or not func (s *BinPackSortedWattsSortedOffers) takeOffer(offer *mesos.Offer, task def.Task) bool { - cpus, mem, watts := offerUtils.OfferAgg(offer) + offerCPU, offerRAM, offerWatts := offerUtils.OfferAgg(offer) //TODO: Insert watts calculation here instead of taking them as a parameter - wattsConsideration, err := def.WattsToConsider(task, s.classMapWatts, offer) if err != nil { // Error in determining wattsConsideration log.Fatal(err) } - if cpus >= task.CPU && mem >= task.RAM && (!s.wattsAsAResource || (watts >= wattsConsideration)) { + if offerCPU >= task.CPU && offerRAM >= task.RAM && (!s.wattsAsAResource || (offerWatts >= wattsConsideration)) { return true } - return false } @@ -166,8 +163,6 @@ func (s *BinPackSortedWattsSortedOffers) ResourceOffers(driver sched.SchedulerDr tasks := []*mesos.TaskInfo{} - offer_cpu, offer_ram, offer_watts := offerUtils.OfferAgg(offer) - offerTaken := false totalWatts := 0.0 totalCPU := 0.0 @@ -180,19 +175,14 @@ func (s *BinPackSortedWattsSortedOffers) ResourceOffers(driver sched.SchedulerDr log.Fatal(err) } - // Check host if it exists - if task.Host != "" { - // Don't take offer if it doesn't match our task's host requirement - if !strings.HasPrefix(*offer.Hostname, task.Host) { - continue - } + // Don't take offer if it doesn't match our task's host requirement + if offerUtils.HostMismatch(*offer.Hostname, task.Host) { + continue } for *task.Instances > 0 { // Does the task fit - if (!s.wattsAsAResource || (offer_watts >= (totalWatts + wattsConsideration))) && - (offer_cpu >= (totalCPU + task.CPU)) && - (offer_ram >= (totalRAM + task.RAM)) { + if s.takeOffer(offer, task) { offerTaken = true totalWatts += wattsConsideration diff --git a/schedulers/binpackedpistoncapping.go b/schedulers/binpackedpistoncapping.go index ca5ab5a..2b24b8a 100644 --- a/schedulers/binpackedpistoncapping.go +++ b/schedulers/binpackedpistoncapping.go @@ -15,7 +15,6 @@ import ( "log" "math" "os" - "strings" "sync" "time" ) @@ -279,13 +278,8 @@ func (s *BinPackedPistonCapper) ResourceOffers(driver sched.SchedulerDriver, off log.Fatal(err) } - // Check host if it exists - if task.Host != "" { - // Don't take offer if it doens't match our task's host requirement. - if !strings.HasPrefix(*offer.Hostname, task.Host) { - continue - } - } + // Don't take offer if it doesn't match our task's host requirement + if offerUtils.HostMismatch(*offer.Hostname, task.Host) {continue} for *task.Instances > 0 { // Does the task fit diff --git a/schedulers/binpacksortedwatts.go b/schedulers/binpacksortedwatts.go index 936f7f6..3a46235 100644 --- a/schedulers/binpacksortedwatts.go +++ b/schedulers/binpacksortedwatts.go @@ -12,7 +12,6 @@ import ( "log" "os" "sort" - "strings" "time" ) @@ -31,7 +30,6 @@ func (s *BinPackSortedWatts) takeOffer(offer *mesos.Offer, task def.Task) bool { if cpus >= task.CPU && mem >= task.RAM && (!s.wattsAsAResource || (watts >= wattsConsideration)) { return true } - return false } @@ -154,8 +152,6 @@ func (s *BinPackSortedWatts) ResourceOffers(driver sched.SchedulerDriver, offers tasks := []*mesos.TaskInfo{} - offer_cpu, offer_ram, offer_watts := offerUtils.OfferAgg(offer) - offerTaken := false totalWatts := 0.0 totalCPU := 0.0 @@ -168,19 +164,14 @@ func (s *BinPackSortedWatts) ResourceOffers(driver sched.SchedulerDriver, offers log.Fatal(err) } - // Check host if it exists - if task.Host != "" { - // Don't take offer if it doesn't match our task's host requirement - if !strings.HasPrefix(*offer.Hostname, task.Host) { - continue - } + // Don't take offer if it doesn't match our task's host requirement + if offerUtils.HostMismatch(*offer.Hostname, task.Host) { + continue } for *task.Instances > 0 { // Does the task fit - if (!s.wattsAsAResource || (offer_watts >= (totalWatts + wattsConsideration))) && - (offer_cpu >= (totalCPU + task.CPU)) && - (offer_ram >= (totalRAM + task.RAM)) { + if s.takeOffer(offer, task) { offerTaken = true totalWatts += wattsConsideration diff --git a/schedulers/bottomHeavy.go b/schedulers/bottomHeavy.go index 7ee8fca..6ec542d 100644 --- a/schedulers/bottomHeavy.go +++ b/schedulers/bottomHeavy.go @@ -26,6 +26,20 @@ BinPacking has the most effect when co-scheduling of tasks is increased. Large t co-scheduling them has a great impact on the total power utilization. */ +func (s *BottomHeavy) takeOffer(offer *mesos.Offer, totalCPU, totalRAM, totalWatts, + wattsToConsider float64, task def.Task) bool { + offerCPU, offerRAM, offerWatts := offerUtils.OfferAgg(offer) + + //TODO: Insert watts calculation here instead of taking them as a parameter + if (!s.wattsAsAResource || (offerWatts >= (totalWatts + wattsToConsider))) && + (offerCPU >= (totalCPU + task.CPU)) && + (offerRAM >= (totalRAM + task.RAM)) { + return true + } + return false + +} + // electronScheduler implements the Scheduler interface type BottomHeavy struct { base // Type embedded to inherit common functions @@ -169,7 +183,6 @@ func (s *BottomHeavy) pack(offers []*mesos.Offer, driver sched.SchedulerDriver) } tasks := []*mesos.TaskInfo{} - offerCPU, offerRAM, offerWatts := offerUtils.OfferAgg(offer) totalWatts := 0.0 totalCPU := 0.0 totalRAM := 0.0 @@ -186,9 +199,7 @@ func (s *BottomHeavy) pack(offers []*mesos.Offer, driver sched.SchedulerDriver) // Does the task fit // OR lazy evaluation. If ignore watts is set to true, second statement won't // be evaluated. - if (!s.wattsAsAResource || (offerWatts >= (totalWatts + wattsConsideration))) && - (offerCPU >= (totalCPU + task.CPU)) && - (offerRAM >= (totalRAM + task.RAM)) { + if s.takeOffer(offer, totalCPU, totalRAM, totalWatts, wattsConsideration, task) { offerTaken = true totalWatts += wattsConsideration totalCPU += task.CPU diff --git a/schedulers/bpswMaxMin.go b/schedulers/bpswMaxMin.go index ae47645..bdb8f45 100644 --- a/schedulers/bpswMaxMin.go +++ b/schedulers/bpswMaxMin.go @@ -12,7 +12,6 @@ import ( "log" "os" "sort" - "strings" "time" ) @@ -31,7 +30,6 @@ func (s *BPSWMaxMinWatts) takeOffer(offer *mesos.Offer, task def.Task) bool { if cpus >= task.CPU && mem >= task.RAM && (!s.wattsAsAResource || (watts >= wattsConsideration)) { return true } - return false } @@ -149,12 +147,8 @@ func (s *BPSWMaxMinWatts) CheckFit(i int, totalRAM *float64, totalWatts *float64) (bool, *mesos.TaskInfo) { - offerCPU, offerRAM, offerWatts := offerUtils.OfferAgg(offer) - // Does the task fit - if (!s.wattsAsAResource || (offerWatts >= (*totalWatts + wattsConsideration))) && - (offerCPU >= (*totalCPU + task.CPU)) && - (offerRAM >= (*totalRAM + task.RAM)) { + if s.takeOffer(offer, task) { *totalWatts += wattsConsideration *totalCPU += task.CPU @@ -218,12 +212,9 @@ func (s *BPSWMaxMinWatts) ResourceOffers(driver sched.SchedulerDriver, offers [] log.Fatal(err) } - // Check host if it exists - if task.Host != "" { - // Don't take offer if it doesn't match our task's host requirement - if !strings.HasPrefix(*offer.Hostname, task.Host) { - continue - } + // Don't take offer if it doesn't match our task's host requirement + if offerUtils.HostMismatch(*offer.Hostname, task.Host) { + continue } // TODO: Fix this so index doesn't need to be passed @@ -246,12 +237,9 @@ func (s *BPSWMaxMinWatts) ResourceOffers(driver sched.SchedulerDriver, offers [] log.Fatal(err) } - // Check host if it exists - if task.Host != "" { - // Don't take offer if it doesn't match our task's host requirement - if !strings.HasPrefix(*offer.Hostname, task.Host) { - continue - } + // Don't take offer if it doesn't match our task's host requirement + if offerUtils.HostMismatch(*offer.Hostname, task.Host) { + continue } for *task.Instances > 0 { diff --git a/schedulers/bpswMaxMinPistonCapping.go b/schedulers/bpswMaxMinPistonCapping.go index d2f63e1..3bf48df 100644 --- a/schedulers/bpswMaxMinPistonCapping.go +++ b/schedulers/bpswMaxMinPistonCapping.go @@ -16,7 +16,6 @@ import ( "math" "os" "sort" - "strings" "sync" "time" ) @@ -36,7 +35,6 @@ func (s *BPSWMaxMinPistonCapping) takeOffer(offer *mesos.Offer, task def.Task) b if cpus >= task.CPU && mem >= task.RAM && (!s.wattsAsAResource || (watts >= wattsConsideration)) { return true } - return false } @@ -239,12 +237,8 @@ func (s *BPSWMaxMinPistonCapping) CheckFit(i int, totalWatts *float64, partialLoad *float64) (bool, *mesos.TaskInfo) { - offerCPU, offerRAM, offerWatts := offerUtils.OfferAgg(offer) - // Does the task fit - if (!s.wattsAsAResource || (offerWatts >= (*totalWatts + wattsConsideration))) && - (offerCPU >= (*totalCPU + task.CPU)) && - (offerRAM >= (*totalRAM + task.RAM)) { + if s.takeOffer(offer, task) { // Start piston capping if haven't started yet if !s.isCapping { @@ -318,12 +312,9 @@ func (s *BPSWMaxMinPistonCapping) ResourceOffers(driver sched.SchedulerDriver, o log.Fatal(err) } - // Check host if it exists - if task.Host != "" { - // Don't take offer if it doesn't match our task's host requirement - if !strings.HasPrefix(*offer.Hostname, task.Host) { - continue - } + // Don't take offer if it doesn't match our task's host requirement + if offerUtils.HostMismatch(*offer.Hostname, task.Host) { + continue } // TODO: Fix this so index doesn't need to be passed @@ -346,12 +337,9 @@ func (s *BPSWMaxMinPistonCapping) ResourceOffers(driver sched.SchedulerDriver, o log.Fatal(err) } - // Check host if it exists - if task.Host != "" { - // Don't take offer if it doesn't match our task's host requirement - if !strings.HasPrefix(*offer.Hostname, task.Host) { - continue - } + // Don't take offer if it doesn't match our task's host requirement + if offerUtils.HostMismatch(*offer.Hostname, task.Host) { + continue } for *task.Instances > 0 { diff --git a/schedulers/bpswMaxMinProacCC.go b/schedulers/bpswMaxMinProacCC.go index 129b030..dc6912a 100644 --- a/schedulers/bpswMaxMinProacCC.go +++ b/schedulers/bpswMaxMinProacCC.go @@ -16,7 +16,6 @@ import ( "math" "os" "sort" - "strings" "sync" "time" ) @@ -35,7 +34,6 @@ func (s *BPSWMaxMinProacCC) takeOffer(offer *mesos.Offer, task def.Task) bool { if cpus >= task.CPU && mem >= task.RAM && (!s.wattsAsAResource || (watts >= wattsConsideration)) { return true } - return false } @@ -262,12 +260,8 @@ func (s *BPSWMaxMinProacCC) CheckFit(i int, totalRAM *float64, totalWatts *float64) (bool, *mesos.TaskInfo) { - offerCPU, offerRAM, offerWatts := offerUtils.OfferAgg(offer) - // Does the task fit - if (!s.wattsAsAResource || (offerWatts >= (*totalWatts + wattsConsideration))) && - (offerCPU >= (*totalCPU + task.CPU)) && - (offerRAM >= (*totalRAM + task.RAM)) { + if s.takeOffer(offer, task) { // Capping the cluster if haven't yet started if !s.isCapping { @@ -366,12 +360,9 @@ func (s *BPSWMaxMinProacCC) ResourceOffers(driver sched.SchedulerDriver, offers // Error in determining wattsConsideration log.Fatal(err) } - // Check host if it exists - if task.Host != "" { - // Don't take offer if it doesn't match our task's host requirement - if !strings.HasPrefix(*offer.Hostname, task.Host) { - continue - } + // Don't take offer if it doesn't match our task's host requirement + if offerUtils.HostMismatch(*offer.Hostname, task.Host) { + continue } // TODO: Fix this so index doesn't need to be passed @@ -394,12 +385,9 @@ func (s *BPSWMaxMinProacCC) ResourceOffers(driver sched.SchedulerDriver, offers log.Fatal(err) } - // Check host if it exists - if task.Host != "" { - // Don't take offer if it doesn't match our task's host requirement - if !strings.HasPrefix(*offer.Hostname, task.Host) { - continue - } + // Don't take offer if it doesn't match our task's host requirement + if offerUtils.HostMismatch(*offer.Hostname, task.Host) { + continue } for *task.Instances > 0 { diff --git a/schedulers/firstfit.go b/schedulers/firstfit.go index 9992721..fae4320 100644 --- a/schedulers/firstfit.go +++ b/schedulers/firstfit.go @@ -11,7 +11,6 @@ import ( sched "github.com/mesos/mesos-go/scheduler" "log" "os" - "strings" "time" ) @@ -159,12 +158,9 @@ func (s *FirstFit) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos. for i := 0; i < len(s.tasks); i++ { task := s.tasks[i] - // Check host if it exists - if task.Host != "" { - // Don't take offer if it doesn't match our task's host requirement - if !strings.HasPrefix(*offer.Hostname, task.Host) { - continue - } + // Don't take offer if it doesn't match our task's host requirement + if offerUtils.HostMismatch(*offer.Hostname, task.Host) { + continue } // Decision to take the offer or not diff --git a/schedulers/firstfitProacCC.go b/schedulers/firstfitProacCC.go index ba06be6..782c426 100644 --- a/schedulers/firstfitProacCC.go +++ b/schedulers/firstfitProacCC.go @@ -15,7 +15,6 @@ import ( "log" "math" "os" - "strings" "sync" "time" ) @@ -293,8 +292,8 @@ func (s *FirstFitProacCC) ResourceOffers(driver sched.SchedulerDriver, offers [] for i := 0; i < len(s.tasks); i++ { task := s.tasks[i] - // Don't take offer if it doesn't match our task's host requirement. - if !strings.HasPrefix(*offer.Hostname, task.Host) { + // Don't take offer if it doesn't match our task's host requirement + if offerUtils.HostMismatch(*offer.Hostname, task.Host) { continue } diff --git a/schedulers/firstfitSortedOffers.go b/schedulers/firstfitSortedOffers.go index 3e4fabe..192638c 100644 --- a/schedulers/firstfitSortedOffers.go +++ b/schedulers/firstfitSortedOffers.go @@ -12,7 +12,6 @@ import ( "log" "os" "sort" - "strings" "time" ) @@ -171,12 +170,9 @@ func (s *FirstFitSortedOffers) ResourceOffers(driver sched.SchedulerDriver, offe for i := 0; i < len(s.tasks); i++ { task := s.tasks[i] - // Check host if it exists - if task.Host != "" { - // Don't take offer if it doesn't match our task's host requirement - if !strings.HasPrefix(*offer.Hostname, task.Host) { - continue - } + // Don't take offer if it doesn't match our task's host requirement + if offerUtils.HostMismatch(*offer.Hostname, task.Host) { + continue } // Decision to take the offer or not diff --git a/schedulers/firstfitSortedWattsProacCC.go b/schedulers/firstfitSortedWattsProacCC.go index bf4964e..34767c6 100644 --- a/schedulers/firstfitSortedWattsProacCC.go +++ b/schedulers/firstfitSortedWattsProacCC.go @@ -26,7 +26,6 @@ import ( "math" "os" "sort" - "strings" "sync" "time" ) @@ -309,9 +308,8 @@ func (s *FirstFitSortedWattsProacCC) ResourceOffers(driver sched.SchedulerDriver for i := 0; i < len(s.tasks); i++ { task := s.tasks[i] - - // Don't take offer if it doesn't match our task's host requirement. - if !strings.HasPrefix(*offer.Hostname, task.Host) { + // Don't take offer if it doesn't match our task's host requirement + if offerUtils.HostMismatch(*offer.Hostname, task.Host) { continue } diff --git a/schedulers/firstfitSortedWattsSortedOffers.go b/schedulers/firstfitSortedWattsSortedOffers.go index 3742db2..5c5a78b 100644 --- a/schedulers/firstfitSortedWattsSortedOffers.go +++ b/schedulers/firstfitSortedWattsSortedOffers.go @@ -12,7 +12,6 @@ import ( "log" "os" "sort" - "strings" "time" ) @@ -175,12 +174,9 @@ func (s *FirstFitSortedWattsSortedOffers) ResourceOffers(driver sched.SchedulerD for i := 0; i < len(s.tasks); i++ { task := s.tasks[i] - // Check host if it exists - if task.Host != "" { - // Don't take offer if it doesn't match our task's host requirement - if !strings.HasPrefix(*offer.Hostname, task.Host) { - continue - } + // Don't take offer if it doesn't match our task's host requirement + if offerUtils.HostMismatch(*offer.Hostname, task.Host) { + continue } // Decision to take the offer or not diff --git a/schedulers/firstfitsortedwatts.go b/schedulers/firstfitsortedwatts.go index 5d624cf..9d63037 100644 --- a/schedulers/firstfitsortedwatts.go +++ b/schedulers/firstfitsortedwatts.go @@ -12,7 +12,6 @@ import ( "log" "os" "sort" - "strings" "time" ) @@ -162,12 +161,9 @@ func (s *FirstFitSortedWatts) ResourceOffers(driver sched.SchedulerDriver, offer for i := 0; i < len(s.tasks); i++ { task := s.tasks[i] - // Check host if it exists - if task.Host != "" { - // Don't take offer if it doesn't match our task's host requirement - if !strings.HasPrefix(*offer.Hostname, task.Host) { - continue - } + // Don't take offer if it doesn't match our task's host requirement + if offerUtils.HostMismatch(*offer.Hostname, task.Host) { + continue } // Decision to take the offer or not diff --git a/schedulers/firstfitwattsonly.go b/schedulers/firstfitwattsonly.go index 2d531c9..8f771bf 100644 --- a/schedulers/firstfitwattsonly.go +++ b/schedulers/firstfitwattsonly.go @@ -11,7 +11,6 @@ import ( sched "github.com/mesos/mesos-go/scheduler" "log" "os" - "strings" "time" ) @@ -152,12 +151,9 @@ func (s *FirstFitWattsOnly) ResourceOffers(driver sched.SchedulerDriver, offers for i := 0; i < len(s.tasks); i++ { task := s.tasks[i] - // Check host if it exists - if task.Host != "" { - // Don't take offer if it doesn't match our task's host requirement - if !strings.HasPrefix(*offer.Hostname, task.Host) { - continue - } + // Don't take offer if it doesn't match our task's host requirement + if offerUtils.HostMismatch(*offer.Hostname, task.Host) { + continue } // Decision to take the offer or not diff --git a/utilities/offerUtils/offerUtils.go b/utilities/offerUtils/offerUtils.go index e1f6817..6f5dc81 100644 --- a/utilities/offerUtils/offerUtils.go +++ b/utilities/offerUtils/offerUtils.go @@ -33,6 +33,8 @@ func PowerClass(offer *mesos.Offer) string { return powerClass } +// Implements the sort.Sort interface to sort Offers based on CPU. +// TODO: Have a generic sorter that sorts based on a defined requirement (CPU, RAM, DISK or Watts) type OffersSorter []*mesos.Offer func (offersSorter OffersSorter) Len() int {