Track resource usage across the cluster. Created utility in utilities/ to track the total and the unused resources for each host in the cluster. Added utility to def/taskUtils.go to retrieve the resource requirement for a given taskID. Decoupled the code, to launch a list of tasks on a set of offerIDs, to schedulers/helpers.go and updated all the scheduling policies to call this function instead of directly calling mesos.SchedulerDriver#LaunchTasks. The resource availability of the cluster is updated at 2 stages -- 1. When the tasks are about to be launched (in schedulers/helpers.go#LaunchTasks), the scheduling policy switching logic will be able to adhere to the update in the resource availability due to the JUST launched tasks and 2. when a terminal status update is received for a task (in schedulers/base.go#statusUpdate).

schedulers/MaxGreedyMins.go

@@ -160,7 +160,9 @@ func (s *MaxGreedyMins) ConsumeOffers(spc SchedPolicyContext, driver sched.Sched
 
 		if offerTaken {
 			baseSchedRef.LogTaskStarting(nil, offer)
-			driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, mesosUtils.DefaultFilter)
+			if err := LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, driver); err != nil {
+				baseSchedRef.LogElectronError(err)
+			}
 		} else {
 
 			// If there was no match for the task

schedulers/MaxMin.go

@@ -155,7 +155,9 @@ func (s *MaxMin) ConsumeOffers(spc SchedPolicyContext, driver sched.SchedulerDri
 
 		if offerTaken {
 			baseSchedRef.LogTaskStarting(nil, offer)
-			driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, mesosUtils.DefaultFilter)
+			if err := LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, driver); err != nil {
+				baseSchedRef.LogElectronError(err)
+			}
 		} else {
 			// If there was no match for the task
 			cpus, mem, watts := offerUtils.OfferAgg(offer)

schedulers/base.go

@@ -12,6 +12,7 @@ import (
 	"log"
 	"sync"
 	"time"
+	"bitbucket.org/sunybingcloud/elektron/utilities"
 )
 
 type baseScheduler struct {
@@ -27,6 +28,7 @@ type baseScheduler struct {
 	running          map[string]map[string]bool
 	wattsAsAResource bool
 	classMapWatts    bool
+	totalResourceAvailabilityRecorded bool
 
 	// First set of PCP values are garbage values, signal to logger to start recording when we're
 	// about to schedule a new task
@@ -162,6 +164,7 @@ func (s *baseScheduler) Disconnected(sched.SchedulerDriver) {
 }
 
 func (s *baseScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
+        utilities.RecordTotalResourceAvailability(offers)
 	s.curSchedPolicy.ConsumeOffers(s, driver, offers)
 }
 
@@ -170,6 +173,9 @@ func (s *baseScheduler) StatusUpdate(driver sched.SchedulerDriver, status *mesos
 	if *status.State == mesos.TaskState_TASK_RUNNING {
 		s.tasksRunning++
 	} else if IsTerminal(status.State) {
+		// Update resource availability.
+		utilities.ResourceAvailabilityUpdate("ON_TASK_TERMINAL_STATE",
+			*status.TaskId, *status.SlaveId)
 		delete(s.running[status.GetSlaveId().GoString()], *status.TaskId.Value)
 		s.tasksRunning--
 		if s.tasksRunning == 0 {

schedulers/bin-packing.go

@@ -104,7 +104,9 @@ func (s *BinPackSortedWatts) ConsumeOffers(spc SchedPolicyContext, driver sched.
 
 		if offerTaken {
 			baseSchedRef.LogTaskStarting(nil, offer)
-			driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, mesosUtils.DefaultFilter)
+			if err := LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, driver); err != nil {
+				baseSchedRef.LogElectronError(err)
+			}
 		} else {
 
 			// If there was no match for the task

schedulers/first-fit.go

@@ -71,8 +71,9 @@ func (s *FirstFit) ConsumeOffers(spc SchedPolicyContext, driver sched.SchedulerD
 				tasks = append(tasks, taskToSchedule)
 
 				baseSchedRef.LogTaskStarting(&task, offer)
-				driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, mesosUtils.DefaultFilter)
-
+				if err := LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, driver); err != nil {
+					baseSchedRef.LogElectronError(err)
+				}
 				offerTaken = true
 
 				baseSchedRef.LogSchedTrace(taskToSchedule, offer)

schedulers/helpers.go

@@ -5,6 +5,10 @@ import (
 	"bitbucket.org/sunybingcloud/elektron/def"
 	"errors"
 	elecLogDef "bitbucket.org/sunybingcloud/elektron/logging/def"
+	"bitbucket.org/sunybingcloud/elektron/utilities"
+	"bitbucket.org/sunybingcloud/elektron/utilities/mesosUtils"
+	mesos "github.com/mesos/mesos-go/api/v0/mesosproto"
+	sched "github.com/mesos/mesos-go/api/v0/scheduler"
 )
 
 func coLocated(tasks map[string]bool, s baseScheduler) {
@@ -119,3 +123,14 @@ func WithSchedPolSwitchEnabled(enableSchedPolicySwitch bool) schedPolicyOption {
 		return nil
 	}
 }
+
+// Launch tasks.
+func LaunchTasks(offerIDs []*mesos.OfferID, tasksToLaunch []*mesos.TaskInfo, driver sched.SchedulerDriver) error {
+	driver.LaunchTasks(offerIDs, tasksToLaunch, mesosUtils.DefaultFilter)
+	// Update resource availability
+	var err error
+        for _, task := range tasksToLaunch {
+		err = utilities.ResourceAvailabilityUpdate("ON_TASK_ACTIVE_STATE", *task.TaskId, *task.SlaveId)
+	}
+	return err
+}