From b4f9a989cb3c0bab68b0855bb5a5ca9de1b8eaed Mon Sep 17 00:00:00 2001 From: Abhishek Jain Date: Fri, 24 Mar 2017 16:28:49 -0400 Subject: [PATCH] Retrofitted all the schedulers to now call the offerutils.AddHostIfNew(..) utility function which will dynamically populate the constants.Hosts and constants.PowerClasses. --- schedulers/binPackSortedWattsSortedOffers.go | 6 +----- schedulers/binpackedpistoncapping.go | 5 +---- schedulers/binpacksortedwatts.go | 6 +----- schedulers/bottomHeavy.go | 21 +++++++++++-------- schedulers/bpswMaxMin.go | 6 +----- schedulers/bpswMaxMinPistonCapping.go | 5 +---- schedulers/bpswMaxMinProacCC.go | 5 +---- schedulers/firstfit.go | 6 +----- schedulers/firstfitProacCC.go | 5 +---- schedulers/firstfitSortedOffers.go | 6 +----- schedulers/firstfitSortedWattsProacCC.go | 5 +---- schedulers/firstfitSortedWattsSortedOffers.go | 6 +----- schedulers/firstfitsortedwatts.go | 6 +----- schedulers/firstfitwattsonly.go | 6 +----- schedulers/helpers.go | 2 +- schedulers/topHeavy.go | 16 +++++++++----- 16 files changed, 37 insertions(+), 75 deletions(-) diff --git a/schedulers/binPackSortedWattsSortedOffers.go b/schedulers/binPackSortedWattsSortedOffers.go index 93e564a..e89236f 100644 --- a/schedulers/binPackSortedWattsSortedOffers.go +++ b/schedulers/binPackSortedWattsSortedOffers.go @@ -1,7 +1,6 @@ package schedulers import ( - "bitbucket.org/sunybingcloud/electron/constants" "bitbucket.org/sunybingcloud/electron/def" "bitbucket.org/sunybingcloud/electron/utilities/mesosUtils" "bitbucket.org/sunybingcloud/electron/utilities/offerUtils" @@ -128,10 +127,7 @@ func (s *BinPackSortedWattsSortedOffers) ResourceOffers(driver sched.SchedulerDr log.Println("Sorted Offers:") for i := 0; i < len(offers); i++ { offer := offers[i] - if _, ok := constants.Hosts[offer.GetHostname()]; !ok { - log.Printf("New host found. Adding host [%s]", offer.GetHostname()) - constants.Hosts[offer.GetHostname()] = struct{}{} - } + offerUtils.AddHostIfNew(offer) offerCPU, _, _ := offerUtils.OfferAgg(offer) log.Printf("Offer[%s].CPU = %f\n", offer.GetHostname(), offerCPU) } diff --git a/schedulers/binpackedpistoncapping.go b/schedulers/binpackedpistoncapping.go index 3022d6c..a14e18f 100644 --- a/schedulers/binpackedpistoncapping.go +++ b/schedulers/binpackedpistoncapping.go @@ -210,10 +210,7 @@ func (s *BinPackedPistonCapper) ResourceOffers(driver sched.SchedulerDriver, off // retrieving the total power for each host in the offers for _, offer := range offers { - if _, ok := constants.Hosts[offer.GetHostname()]; !ok { - log.Printf("New host found. Adding host [%s]", offer.GetHostname()) - constants.Hosts[offer.GetHostname()] = struct{}{} - } + offerUtils.AddHostIfNew(offer) if _, ok := s.totalPower[*offer.Hostname]; !ok { _, _, offerWatts := offerUtils.OfferAgg(offer) s.totalPower[*offer.Hostname] = offerWatts diff --git a/schedulers/binpacksortedwatts.go b/schedulers/binpacksortedwatts.go index 45d2488..2dd6f5f 100644 --- a/schedulers/binpacksortedwatts.go +++ b/schedulers/binpacksortedwatts.go @@ -1,7 +1,6 @@ package schedulers import ( - "bitbucket.org/sunybingcloud/electron/constants" "bitbucket.org/sunybingcloud/electron/def" "bitbucket.org/sunybingcloud/electron/utilities/mesosUtils" "bitbucket.org/sunybingcloud/electron/utilities/offerUtils" @@ -121,10 +120,7 @@ func (s *BinPackSortedWatts) ResourceOffers(driver sched.SchedulerDriver, offers log.Printf("Received %d resource offers", len(offers)) for _, offer := range offers { - if _, ok := constants.Hosts[offer.GetHostname()]; !ok { - log.Printf("New host found. Adding host [%s]", offer.GetHostname()) - constants.Hosts[offer.GetHostname()] = struct{}{} - } + offerUtils.AddHostIfNew(offer) select { case <-s.Shutdown: log.Println("Done scheduling tasks: declining offer on [", offer.GetHostname(), "]") diff --git a/schedulers/bottomHeavy.go b/schedulers/bottomHeavy.go index eef8bd6..49487fc 100644 --- a/schedulers/bottomHeavy.go +++ b/schedulers/bottomHeavy.go @@ -282,10 +282,7 @@ func (s *BottomHeavy) ResourceOffers(driver sched.SchedulerDriver, offers []*mes offersLightPowerClasses := []*mesos.Offer{} for _, offer := range offers { - if _, ok := constants.Hosts[offer.GetHostname()]; !ok { - log.Printf("New host found. Adding host [%s]", offer.GetHostname()) - constants.Hosts[offer.GetHostname()] = struct{}{} - } + offerUtils.AddHostIfNew(offer) select { case <-s.Shutdown: log.Println("Done scheduling tasks: declining offer on [", offer.GetHostname(), "]") @@ -296,13 +293,19 @@ func (s *BottomHeavy) ResourceOffers(driver sched.SchedulerDriver, offers []*mes default: } - if constants.PowerClasses["A"][*offer.Hostname] || - constants.PowerClasses["B"][*offer.Hostname] { + if _, ok := constants.PowerClasses["A"][*offer.Hostname]; ok{ offersHeavyPowerClasses = append(offersHeavyPowerClasses, offer) - } else if constants.PowerClasses["C"][*offer.Hostname] || - constants.PowerClasses["D"][*offer.Hostname] { - offersLightPowerClasses = append(offersLightPowerClasses, offer) } + if _, ok := constants.PowerClasses["B"][*offer.Hostname]; ok{ + offersHeavyPowerClasses = append(offersHeavyPowerClasses, offer) + } + if _, ok := constants.PowerClasses["C"][*offer.Hostname]; ok{ + offersHeavyPowerClasses = append(offersLightPowerClasses, offer) + } + if _, ok := constants.PowerClasses["D"][*offer.Hostname]; ok{ + offersHeavyPowerClasses = append(offersLightPowerClasses, offer) + } + } log.Println("Packing Large tasks into ClassAB offers:") diff --git a/schedulers/bpswMaxMin.go b/schedulers/bpswMaxMin.go index 940439a..a176501 100644 --- a/schedulers/bpswMaxMin.go +++ b/schedulers/bpswMaxMin.go @@ -1,7 +1,6 @@ package schedulers import ( - "bitbucket.org/sunybingcloud/electron/constants" "bitbucket.org/sunybingcloud/electron/def" "bitbucket.org/sunybingcloud/electron/utilities/mesosUtils" "bitbucket.org/sunybingcloud/electron/utilities/offerUtils" @@ -165,10 +164,7 @@ func (s *BPSWMaxMinWatts) ResourceOffers(driver sched.SchedulerDriver, offers [] log.Printf("Received %d resource offers", len(offers)) for _, offer := range offers { - if _, ok := constants.Hosts[offer.GetHostname()]; !ok { - log.Printf("New host found. Adding host [%s]", offer.GetHostname()) - constants.Hosts[offer.GetHostname()] = struct{}{} - } + offerUtils.AddHostIfNew(offer) select { case <-s.Shutdown: log.Println("Done scheduling tasks: declining offer on [", offer.GetHostname(), "]") diff --git a/schedulers/bpswMaxMinPistonCapping.go b/schedulers/bpswMaxMinPistonCapping.go index 4031e88..41c1559 100644 --- a/schedulers/bpswMaxMinPistonCapping.go +++ b/schedulers/bpswMaxMinPistonCapping.go @@ -261,10 +261,7 @@ func (s *BPSWMaxMinPistonCapping) ResourceOffers(driver sched.SchedulerDriver, o log.Printf("Received %d resource offers", len(offers)) for _, offer := range offers { - if _, ok := constants.Hosts[offer.GetHostname()]; !ok { - log.Printf("New host found. Adding host [%s]", offer.GetHostname()) - constants.Hosts[offer.GetHostname()] = struct{}{} - } + offerUtils.AddHostIfNew(offer) select { case <-s.Shutdown: log.Println("Done scheduling tasks: declining offer on [", offer.GetHostname(), "]") diff --git a/schedulers/bpswMaxMinProacCC.go b/schedulers/bpswMaxMinProacCC.go index d80f05f..c3f2e90 100644 --- a/schedulers/bpswMaxMinProacCC.go +++ b/schedulers/bpswMaxMinProacCC.go @@ -300,10 +300,7 @@ func (s *BPSWMaxMinProacCC) ResourceOffers(driver sched.SchedulerDriver, offers // retrieving the available power for all the hosts in the offers. for _, offer := range offers { - if _, ok := constants.Hosts[offer.GetHostname()]; !ok { - log.Printf("New host found. Adding host [%s]", offer.GetHostname()) - constants.Hosts[offer.GetHostname()] = struct{}{} - } + offerUtils.AddHostIfNew(offer) _, _, offerWatts := offerUtils.OfferAgg(offer) s.availablePower[*offer.Hostname] = offerWatts // setting total power if the first time diff --git a/schedulers/firstfit.go b/schedulers/firstfit.go index e07839c..e0f114d 100644 --- a/schedulers/firstfit.go +++ b/schedulers/firstfit.go @@ -1,7 +1,6 @@ package schedulers import ( - "bitbucket.org/sunybingcloud/electron/constants" "bitbucket.org/sunybingcloud/electron/def" "bitbucket.org/sunybingcloud/electron/utilities/mesosUtils" "bitbucket.org/sunybingcloud/electron/utilities/offerUtils" @@ -120,10 +119,7 @@ func (s *FirstFit) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos. log.Printf("Received %d resource offers", len(offers)) for _, offer := range offers { - if _, ok := constants.Hosts[offer.GetHostname()]; !ok { - log.Printf("New host found. Adding host [%s]", offer.GetHostname()) - constants.Hosts[offer.GetHostname()] = struct{}{} - } + offerUtils.AddHostIfNew(offer) select { case <-s.Shutdown: log.Println("Done scheduling tasks: declining offer on [", offer.GetHostname(), "]") diff --git a/schedulers/firstfitProacCC.go b/schedulers/firstfitProacCC.go index 51011e5..0d4c3d4 100644 --- a/schedulers/firstfitProacCC.go +++ b/schedulers/firstfitProacCC.go @@ -233,10 +233,7 @@ func (s *FirstFitProacCC) ResourceOffers(driver sched.SchedulerDriver, offers [] // retrieving the available power for all the hosts in the offers. for _, offer := range offers { - if _, ok := constants.Hosts[offer.GetHostname()]; !ok { - log.Printf("New host found. Adding host [%s]", offer.GetHostname()) - constants.Hosts[offer.GetHostname()] = struct{}{} - } + offerUtils.AddHostIfNew(offer) _, _, offer_watts := offerUtils.OfferAgg(offer) s.availablePower[*offer.Hostname] = offer_watts // setting total power if the first time. diff --git a/schedulers/firstfitSortedOffers.go b/schedulers/firstfitSortedOffers.go index a9ab1d8..0bcd47a 100644 --- a/schedulers/firstfitSortedOffers.go +++ b/schedulers/firstfitSortedOffers.go @@ -1,7 +1,6 @@ package schedulers import ( - "bitbucket.org/sunybingcloud/electron/constants" "bitbucket.org/sunybingcloud/electron/def" "bitbucket.org/sunybingcloud/electron/utilities/mesosUtils" "bitbucket.org/sunybingcloud/electron/utilities/offerUtils" @@ -127,10 +126,7 @@ func (s *FirstFitSortedOffers) ResourceOffers(driver sched.SchedulerDriver, offe log.Println("Sorted Offers:") for i := 0; i < len(offers); i++ { offer := offers[i] - if _, ok := constants.Hosts[offer.GetHostname()]; !ok { - log.Printf("New host found. Adding host [%s]", offer.GetHostname()) - constants.Hosts[offer.GetHostname()] = struct{}{} - } + offerUtils.AddHostIfNew(offer) offerCPU, _, _ := offerUtils.OfferAgg(offer) log.Printf("Offer[%s].CPU = %f\n", offer.GetHostname(), offerCPU) } diff --git a/schedulers/firstfitSortedWattsProacCC.go b/schedulers/firstfitSortedWattsProacCC.go index 4173ae6..99524b2 100644 --- a/schedulers/firstfitSortedWattsProacCC.go +++ b/schedulers/firstfitSortedWattsProacCC.go @@ -246,10 +246,7 @@ func (s *FirstFitSortedWattsProacCC) ResourceOffers(driver sched.SchedulerDriver // retrieving the available power for all the hosts in the offers. for _, offer := range offers { - if _, ok := constants.Hosts[offer.GetHostname()]; !ok { - log.Printf("New host found. Adding host [%s]", offer.GetHostname()) - constants.Hosts[offer.GetHostname()] = struct{}{} - } + offerUtils.AddHostIfNew(offer) _, _, offer_watts := offerUtils.OfferAgg(offer) s.availablePower[*offer.Hostname] = offer_watts // setting total power if the first time. diff --git a/schedulers/firstfitSortedWattsSortedOffers.go b/schedulers/firstfitSortedWattsSortedOffers.go index 5745ec6..dd552de 100644 --- a/schedulers/firstfitSortedWattsSortedOffers.go +++ b/schedulers/firstfitSortedWattsSortedOffers.go @@ -1,7 +1,6 @@ package schedulers import ( - "bitbucket.org/sunybingcloud/electron/constants" "bitbucket.org/sunybingcloud/electron/def" "bitbucket.org/sunybingcloud/electron/utilities/mesosUtils" "bitbucket.org/sunybingcloud/electron/utilities/offerUtils" @@ -129,10 +128,7 @@ func (s *FirstFitSortedWattsSortedOffers) ResourceOffers(driver sched.SchedulerD log.Println("Sorted Offers:") for i := 0; i < len(offers); i++ { offer := offers[i] - if _, ok := constants.Hosts[offer.GetHostname()]; !ok { - log.Printf("New host found. Adding host [%s]", offer.GetHostname()) - constants.Hosts[offer.GetHostname()] = struct{}{} - } + offerUtils.AddHostIfNew(offer) offerCPU, _, _ := offerUtils.OfferAgg(offer) log.Printf("Offer[%s].CPU = %f\n", offer.GetHostname(), offerCPU) } diff --git a/schedulers/firstfitsortedwatts.go b/schedulers/firstfitsortedwatts.go index f19bec0..6c6cc24 100644 --- a/schedulers/firstfitsortedwatts.go +++ b/schedulers/firstfitsortedwatts.go @@ -1,7 +1,6 @@ package schedulers import ( - "bitbucket.org/sunybingcloud/electron/constants" "bitbucket.org/sunybingcloud/electron/def" "bitbucket.org/sunybingcloud/electron/utilities/mesosUtils" "bitbucket.org/sunybingcloud/electron/utilities/offerUtils" @@ -123,10 +122,7 @@ func (s *FirstFitSortedWatts) ResourceOffers(driver sched.SchedulerDriver, offer log.Printf("Received %d resource offers", len(offers)) for _, offer := range offers { - if _, ok := constants.Hosts[offer.GetHostname()]; !ok { - log.Printf("New host found. Adding host [%s]", offer.GetHostname()) - constants.Hosts[offer.GetHostname()] = struct{}{} - } + offerUtils.AddHostIfNew(offer) select { case <-s.Shutdown: log.Println("Done scheduling tasks: declining offer on [", offer.GetHostname(), "]") diff --git a/schedulers/firstfitwattsonly.go b/schedulers/firstfitwattsonly.go index f7833b1..5bef674 100644 --- a/schedulers/firstfitwattsonly.go +++ b/schedulers/firstfitwattsonly.go @@ -1,7 +1,6 @@ package schedulers import ( - "bitbucket.org/sunybingcloud/electron/constants" "bitbucket.org/sunybingcloud/electron/def" "bitbucket.org/sunybingcloud/electron/utilities/mesosUtils" "bitbucket.org/sunybingcloud/electron/utilities/offerUtils" @@ -113,10 +112,7 @@ func (s *FirstFitWattsOnly) ResourceOffers(driver sched.SchedulerDriver, offers log.Printf("Received %d resource offers", len(offers)) for _, offer := range offers { - if _, ok := constants.Hosts[offer.GetHostname()]; !ok { - log.Printf("New host found. Adding host [%s]", offer.GetHostname()) - constants.Hosts[offer.GetHostname()] = struct{}{} - } + offerUtils.AddHostIfNew(offer) select { case <-s.Shutdown: log.Println("Done scheduling tasks: declining offer on [", offer.GetHostname(), "]") diff --git a/schedulers/helpers.go b/schedulers/helpers.go index e6ba7fb..1e39c20 100644 --- a/schedulers/helpers.go +++ b/schedulers/helpers.go @@ -18,7 +18,7 @@ func coLocated(tasks map[string]bool) { // Get the powerClass of the given hostname func hostToPowerClass(hostName string) string { for powerClass, hosts := range constants.PowerClasses { - if ok := hosts[hostName]; ok { + if _, ok := hosts[hostName]; ok { return powerClass } } diff --git a/schedulers/topHeavy.go b/schedulers/topHeavy.go index 63c11e0..fbcd093 100644 --- a/schedulers/topHeavy.go +++ b/schedulers/topHeavy.go @@ -281,6 +281,7 @@ func (s *TopHeavy) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos. offersLightPowerClasses := []*mesos.Offer{} for _, offer := range offers { + offerUtils.AddHostIfNew(offer) select { case <-s.Shutdown: log.Println("Done scheduling tasks: declining offer on [", offer.GetHostname(), "]") @@ -291,12 +292,17 @@ func (s *TopHeavy) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos. default: } - if constants.PowerClasses["A"][*offer.Hostname] || - constants.PowerClasses["B"][*offer.Hostname] { + if _, ok := constants.PowerClasses["A"][*offer.Hostname]; ok{ offersHeavyPowerClasses = append(offersHeavyPowerClasses, offer) - } else if constants.PowerClasses["C"][*offer.Hostname] || - constants.PowerClasses["D"][*offer.Hostname] { - offersLightPowerClasses = append(offersLightPowerClasses, offer) + } + if _, ok := constants.PowerClasses["B"][*offer.Hostname]; ok{ + offersHeavyPowerClasses = append(offersHeavyPowerClasses, offer) + } + if _, ok := constants.PowerClasses["C"][*offer.Hostname]; ok{ + offersHeavyPowerClasses = append(offersLightPowerClasses, offer) + } + if _, ok := constants.PowerClasses["D"][*offer.Hostname]; ok{ + offersHeavyPowerClasses = append(offersLightPowerClasses, offer) } }