Workaround for intervals and timeout being dependent on the init execution order.
This commit is contained in:
Renan DelValle
parent
ddc9bc408a
commit
3c817a7ffc
6 changed files with 103 additions and 101 deletions
130
cmd/start.go
130
cmd/start.go
|
|
@ -11,38 +11,42 @@ func init() {
|
|||
rootCmd.AddCommand(startCmd)
|
||||
|
||||
// Sub-commands
|
||||
startCmd.AddCommand(startDrainCmd)
|
||||
startCmd.AddCommand(startDrainCmd.cmd)
|
||||
startDrainCmd.cmd.Run = drain
|
||||
|
||||
// Maintenance specific flags
|
||||
startDrainCmd.Flags().DurationVar(&monitorInterval, "interval", time.Second*5, "Interval at which to poll scheduler.")
|
||||
startDrainCmd.Flags().DurationVar(&monitorTimeout, "timeout", time.Minute*10, "Time after which the monitor will stop polling and throw an error.")
|
||||
startDrainCmd.cmd.Flags().DurationVar(&startDrainCmd.monitorInterval, "interval", time.Second*5, "Interval at which to poll scheduler.")
|
||||
startDrainCmd.cmd.Flags().DurationVar(&startDrainCmd.monitorTimeout, "timeout", time.Minute*10, "Time after which the monitor will stop polling and throw an error.")
|
||||
|
||||
startCmd.AddCommand(startSLADrainCmd)
|
||||
|
||||
/* SLA Aware commands */
|
||||
startSLADrainCmd.AddCommand(startSLACountDrainCmd)
|
||||
startSLADrainCmd.AddCommand(startSLACountDrainCmd.cmd)
|
||||
startSLACountDrainCmd.cmd.Run = SLACountDrain
|
||||
|
||||
// SLA Maintenance specific flags
|
||||
startSLACountDrainCmd.Flags().DurationVar(&monitorInterval, "interval", time.Second*10, "Interval at which to poll scheduler.")
|
||||
startSLACountDrainCmd.Flags().DurationVar(&monitorTimeout, "timeout", time.Minute*20, "Time after which the monitor will stop polling and throw an error.")
|
||||
startSLACountDrainCmd.Flags().Int64Var(&count, "count", 5, "Instances count that should be running to meet SLA.")
|
||||
startSLACountDrainCmd.Flags().DurationVar(&duration, "duration", time.Second*45, "Minimum time duration a task needs to be `RUNNING` to be treated as active.")
|
||||
startSLACountDrainCmd.Flags().DurationVar(&forceDrainTimeout, "sla-limit", time.Minute*60, "Time limit after which SLA-Aware drain sheds SLA Awareness.")
|
||||
startSLACountDrainCmd.cmd.Flags().DurationVar(&startSLACountDrainCmd.monitorInterval, "interval", time.Second*10, "Interval at which to poll scheduler.")
|
||||
startSLACountDrainCmd.cmd.Flags().DurationVar(&startSLACountDrainCmd.monitorTimeout, "timeout", time.Minute*20, "Time after which the monitor will stop polling and throw an error.")
|
||||
startSLACountDrainCmd.cmd.Flags().Int64Var(&count, "count", 5, "Instances count that should be running to meet SLA.")
|
||||
startSLACountDrainCmd.cmd.Flags().DurationVar(&duration, "duration", time.Second*45, "Minimum time duration a task needs to be `RUNNING` to be treated as active.")
|
||||
startSLACountDrainCmd.cmd.Flags().DurationVar(&forceDrainTimeout, "sla-limit", time.Minute*60, "Time limit after which SLA-Aware drain sheds SLA Awareness.")
|
||||
|
||||
startSLADrainCmd.AddCommand(startSLAPercentageDrainCmd)
|
||||
startSLADrainCmd.AddCommand(startSLAPercentageDrainCmd.cmd)
|
||||
startSLAPercentageDrainCmd.cmd.Run = SLAPercentageDrain
|
||||
|
||||
// SLA Maintenance specific flags
|
||||
startSLAPercentageDrainCmd.Flags().DurationVar(&monitorInterval, "interval", time.Second*10, "Interval at which to poll scheduler.")
|
||||
startSLAPercentageDrainCmd.Flags().DurationVar(&monitorTimeout, "timeout", time.Minute*20, "Time after which the monitor will stop polling and throw an error.")
|
||||
startSLAPercentageDrainCmd.Flags().Float64Var(&percent, "percent", 75.0, "Percentage of instances that should be running to meet SLA.")
|
||||
startSLAPercentageDrainCmd.Flags().DurationVar(&duration, "duration", time.Second*45, "Minimum time duration a task needs to be `RUNNING` to be treated as active.")
|
||||
startSLAPercentageDrainCmd.Flags().DurationVar(&forceDrainTimeout, "sla-limit", time.Minute*60, "Time limit after which SLA-Aware drain sheds SLA Awareness.")
|
||||
startSLAPercentageDrainCmd.cmd.Flags().DurationVar(&startSLACountDrainCmd.monitorInterval, "interval", time.Second*10, "Interval at which to poll scheduler.")
|
||||
startSLAPercentageDrainCmd.cmd.Flags().DurationVar(&startSLACountDrainCmd.monitorTimeout, "timeout", time.Minute*20, "Time after which the monitor will stop polling and throw an error.")
|
||||
startSLAPercentageDrainCmd.cmd.Flags().Float64Var(&percent, "percent", 75.0, "Percentage of instances that should be running to meet SLA.")
|
||||
startSLAPercentageDrainCmd.cmd.Flags().DurationVar(&duration, "duration", time.Second*45, "Minimum time duration a task needs to be `RUNNING` to be treated as active.")
|
||||
startSLAPercentageDrainCmd.cmd.Flags().DurationVar(&forceDrainTimeout, "sla-limit", time.Minute*60, "Time limit after which SLA-Aware drain sheds SLA Awareness.")
|
||||
|
||||
startCmd.AddCommand(startMaintenanceCmd)
|
||||
startCmd.AddCommand(startMaintenanceCmd.cmd)
|
||||
startMaintenanceCmd.cmd.Run = maintenance
|
||||
|
||||
// SLA Maintenance specific flags
|
||||
startMaintenanceCmd.Flags().DurationVar(&monitorInterval, "interval", time.Second*5, "Interval at which to poll scheduler.")
|
||||
startMaintenanceCmd.Flags().DurationVar(&monitorTimeout, "timeout", time.Minute*10, "Time after which the monitor will stop polling and throw an error.")
|
||||
startMaintenanceCmd.cmd.Flags().DurationVar(&startMaintenanceCmd.monitorInterval, "interval", time.Second*5, "Interval at which to poll scheduler.")
|
||||
startMaintenanceCmd.cmd.Flags().DurationVar(&startMaintenanceCmd.monitorTimeout, "timeout", time.Minute*10, "Time after which the monitor will stop polling and throw an error.")
|
||||
}
|
||||
|
||||
var startCmd = &cobra.Command{
|
||||
|
|
@ -50,21 +54,16 @@ var startCmd = &cobra.Command{
|
|||
Short: "Start a service, maintenance on a host (DRAIN), a snapshot, or a backup.",
|
||||
}
|
||||
|
||||
var startDrainCmd = &cobra.Command{
|
||||
Use: "drain [space separated host list]",
|
||||
Short: "Place a list of space separated Mesos Agents into draining mode.",
|
||||
Long: `Adds a Mesos Agent to Aurora's Drain list. Agents in this list
|
||||
var startDrainCmd = MonitorCmdConfig{
|
||||
cmd: &cobra.Command{
|
||||
Use: "drain [space separated host list]",
|
||||
Short: "Place a list of space separated Mesos Agents into draining mode.",
|
||||
Long: `Adds a Mesos Agent to Aurora's Drain list. Agents in this list
|
||||
are not allowed to schedule new tasks and any tasks already running on this Agent
|
||||
are killed and rescheduled in an Agent that is not in maintenance mode. Command
|
||||
expects a space separated list of hosts to place into maintenance mode.`,
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
PreRun: func(cmd *cobra.Command, args []string) {
|
||||
// Manually initializing default values for this command as the default value for shared variables will
|
||||
// be dependent on the order in which all commands were initialized
|
||||
monitorTimeout = time.Minute * 10
|
||||
monitorInterval = time.Second * 5
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
},
|
||||
Run: drain,
|
||||
}
|
||||
|
||||
var startSLADrainCmd = &cobra.Command{
|
||||
|
|
@ -76,50 +75,35 @@ are killed and rescheduled in an Agent that is not in maintenance mode. Command
|
|||
expects a space separated list of hosts to place into maintenance mode.`,
|
||||
}
|
||||
|
||||
var startSLACountDrainCmd = &cobra.Command{
|
||||
Use: "count [space separated host list]",
|
||||
Short: "Place a list of space separated Mesos Agents into maintenance mode using the count SLA aware policy as a fallback.",
|
||||
Long: `Adds a Mesos Agent to Aurora's Drain list. Tasks will be drained using the count SLA policy as a fallback
|
||||
var startSLACountDrainCmd = MonitorCmdConfig{
|
||||
cmd: &cobra.Command{
|
||||
Use: "count [space separated host list]",
|
||||
Short: "Place a list of space separated Mesos Agents into maintenance mode using the count SLA aware policy as a fallback.",
|
||||
Long: `Adds a Mesos Agent to Aurora's Drain list. Tasks will be drained using the count SLA policy as a fallback
|
||||
when a Job does not have a defined SLA policy.`,
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
PreRun: func(cmd *cobra.Command, args []string) {
|
||||
// Manually initializing default values for this command as the default value for shared variables will
|
||||
// be dependent on the order in which all commands were initialized
|
||||
monitorTimeout = time.Minute * 20
|
||||
monitorInterval = time.Second * 10
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
},
|
||||
Run: SLACountDrain,
|
||||
}
|
||||
|
||||
var startSLAPercentageDrainCmd = &cobra.Command{
|
||||
Use: "percentage [space separated host list]",
|
||||
Short: "Place a list of space separated Mesos Agents into maintenance mode using the percentage SLA aware policy as a fallback.",
|
||||
Long: `Adds a Mesos Agent to Aurora's Drain list. Tasks will be drained using the percentage SLA policy as a fallback
|
||||
var startSLAPercentageDrainCmd = MonitorCmdConfig{
|
||||
cmd: &cobra.Command{
|
||||
Use: "percentage [space separated host list]",
|
||||
Short: "Place a list of space separated Mesos Agents into maintenance mode using the percentage SLA aware policy as a fallback.",
|
||||
Long: `Adds a Mesos Agent to Aurora's Drain list. Tasks will be drained using the percentage SLA policy as a fallback
|
||||
when a Job does not have a defined SLA policy.`,
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
PreRun: func(cmd *cobra.Command, args []string) {
|
||||
// Manually initializing default values for this command as the default value for shared variables will
|
||||
// be dependent on the order in which all commands were initialized
|
||||
monitorTimeout = time.Minute * 20
|
||||
monitorInterval = time.Second * 10
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
},
|
||||
Run: SLAPercentageDrain,
|
||||
}
|
||||
|
||||
var startMaintenanceCmd = &cobra.Command{
|
||||
Use: "maintenance [space separated host list]",
|
||||
Short: "Place a list of space separated Mesos Agents into maintenance mode.",
|
||||
Long: `Places Mesos Agent into Maintenance mode. Agents in this list
|
||||
var startMaintenanceCmd = MonitorCmdConfig{
|
||||
cmd: &cobra.Command{
|
||||
Use: "maintenance [space separated host list]",
|
||||
Short: "Place a list of space separated Mesos Agents into maintenance mode.",
|
||||
Long: `Places Mesos Agent into Maintenance mode. Agents in this list
|
||||
are de-prioritized for scheduling a task. Command
|
||||
expects a space separated list of hosts to place into maintenance mode.`,
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
PreRun: func(cmd *cobra.Command, args []string) {
|
||||
// Manually initializing default values for this command as the default value for shared variables will
|
||||
// be dependent on the order in which all commands were initialized
|
||||
monitorTimeout = time.Minute * 1
|
||||
monitorInterval = time.Second * 5
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
},
|
||||
Run: maintenance,
|
||||
}
|
||||
|
||||
func drain(cmd *cobra.Command, args []string) {
|
||||
|
|
@ -132,12 +116,13 @@ func drain(cmd *cobra.Command, args []string) {
|
|||
|
||||
log.Debugln(result)
|
||||
|
||||
log.Infof("Monitoring for %v at %v intervals", monitorHostCmd.monitorTimeout, monitorHostCmd.monitorInterval)
|
||||
// Monitor change to DRAINING and DRAINED mode
|
||||
hostResult, err := client.HostMaintenanceMonitor(
|
||||
args,
|
||||
[]aurora.MaintenanceMode{aurora.MaintenanceMode_DRAINED},
|
||||
monitorInterval,
|
||||
monitorTimeout)
|
||||
startDrainCmd.monitorInterval,
|
||||
startDrainCmd.monitorTimeout)
|
||||
|
||||
maintenanceMonitorPrint(hostResult, []aurora.MaintenanceMode{aurora.MaintenanceMode_DRAINED})
|
||||
|
||||
|
|
@ -146,7 +131,7 @@ func drain(cmd *cobra.Command, args []string) {
|
|||
}
|
||||
}
|
||||
|
||||
func slaDrain(policy *aurora.SlaPolicy, hosts ...string) {
|
||||
func slaDrain(policy *aurora.SlaPolicy, interval, timeout time.Duration, hosts ...string) {
|
||||
|
||||
result, err := client.SLADrainHosts(policy, int64(forceDrainTimeout.Seconds()), hosts...)
|
||||
if err != nil {
|
||||
|
|
@ -155,12 +140,13 @@ func slaDrain(policy *aurora.SlaPolicy, hosts ...string) {
|
|||
|
||||
log.Debugln(result)
|
||||
|
||||
log.Infof("Monitoring for %v at %v intervals", monitorHostCmd.monitorTimeout, monitorHostCmd.monitorInterval)
|
||||
// Monitor change to DRAINING and DRAINED mode
|
||||
hostResult, err := client.HostMaintenanceMonitor(
|
||||
hosts,
|
||||
[]aurora.MaintenanceMode{aurora.MaintenanceMode_DRAINED},
|
||||
monitorInterval,
|
||||
monitorTimeout)
|
||||
interval,
|
||||
timeout)
|
||||
|
||||
maintenanceMonitorPrint(hostResult, []aurora.MaintenanceMode{aurora.MaintenanceMode_DRAINED})
|
||||
|
||||
|
|
@ -175,6 +161,8 @@ func SLACountDrain(cmd *cobra.Command, args []string) {
|
|||
|
||||
slaDrain(&aurora.SlaPolicy{
|
||||
CountSlaPolicy: &aurora.CountSlaPolicy{Count: count, DurationSecs: int64(duration.Seconds())}},
|
||||
startSLACountDrainCmd.monitorInterval,
|
||||
startSLACountDrainCmd.monitorTimeout,
|
||||
args...)
|
||||
}
|
||||
|
||||
|
|
@ -184,6 +172,8 @@ func SLAPercentageDrain(cmd *cobra.Command, args []string) {
|
|||
|
||||
slaDrain(&aurora.SlaPolicy{
|
||||
PercentageSlaPolicy: &aurora.PercentageSlaPolicy{Percentage: percent, DurationSecs: int64(duration.Seconds())}},
|
||||
startSLAPercentageDrainCmd.monitorInterval,
|
||||
startSLAPercentageDrainCmd.monitorTimeout,
|
||||
args...)
|
||||
}
|
||||
|
||||
|
|
@ -197,12 +187,14 @@ func maintenance(cmd *cobra.Command, args []string) {
|
|||
|
||||
log.Debugln(result)
|
||||
|
||||
log.Infof("Monitoring for %v at %v intervals", monitorHostCmd.monitorTimeout, monitorHostCmd.monitorInterval)
|
||||
|
||||
// Monitor change to DRAINING and DRAINED mode
|
||||
hostResult, err := client.HostMaintenanceMonitor(
|
||||
args,
|
||||
[]aurora.MaintenanceMode{aurora.MaintenanceMode_SCHEDULED},
|
||||
monitorInterval,
|
||||
monitorTimeout)
|
||||
startMaintenanceCmd.monitorInterval,
|
||||
startMaintenanceCmd.monitorTimeout)
|
||||
|
||||
maintenanceMonitorPrint(hostResult, []aurora.MaintenanceMode{aurora.MaintenanceMode_SCHEDULED})
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue