/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package cmd import ( "encoding/json" "errors" "fmt" "io/ioutil" "net/http" "strings" "github.com/aurora-scheduler/australis/internal" realis "github.com/aurora-scheduler/gorealis/v2" "github.com/aurora-scheduler/gorealis/v2/gen-go/apache/aurora" "github.com/spf13/cobra" "github.com/spf13/pflag" ) const ( localAgentStateURL = "http://127.0.0.1:5051/state" ) type mesosAgentState struct { Flags mesosAgentFlags `json:"flags,omitempty"` } type mesosAgentFlags struct { Master string `json:"master,omitempty"` hasMaster bool // indicates if the master flag contains direct Master's address } func init() { rootCmd.AddCommand(fetchCmd) // Sub-commands // Fetch Task Config fetchCmd.AddCommand(fetchTaskCmd) // Fetch Task Config fetchTaskCmd.AddCommand(taskConfigCmd) taskConfigCmd.Flags().StringVarP(env, "environment", "e", "", "Aurora Environment") taskConfigCmd.Flags().StringVarP(role, "role", "r", "", "Aurora Role") taskConfigCmd.Flags().StringVarP(name, "name", "n", "", "Aurora Name") // Fetch Task Status fetchTaskCmd.AddCommand(taskStatusCmd) taskStatusCmd.Flags().StringVarP(env, "environment", "e", "", "Aurora Environment") taskStatusCmd.Flags().StringVarP(role, "role", "r", "", "Aurora Role") taskStatusCmd.Flags().StringVarP(name, "name", "n", "", "Aurora Name") /* Fetch Leader */ leaderCmd.Flags().String("zkPath", "/aurora/scheduler", "Zookeeper node path where leader election happens") fetchCmd.AddCommand(leaderCmd) // Hijack help function to hide unnecessary global flags help := leaderCmd.HelpFunc() leaderCmd.SetHelpFunc(func(cmd *cobra.Command, s []string) { if cmd.HasInheritedFlags() { cmd.InheritedFlags().VisitAll(func(f *pflag.Flag) { if f.Name != "logLevel" { f.Hidden = true } }) } help(cmd, s) }) mesosLeaderCmd.Flags().String("zkPath", "/mesos", "Zookeeper node path where mesos leader election happens") mesosCmd.AddCommand(mesosLeaderCmd) fetchCmd.AddCommand(mesosCmd) // Hijack help function to hide unnecessary global flags mesosCmd.SetHelpFunc(func(cmd *cobra.Command, s []string) { if cmd.HasInheritedFlags() { cmd.InheritedFlags().VisitAll(func(f *pflag.Flag) { if f.Name != "logLevel" { f.Hidden = true } }) } help(cmd, s) }) /* Fetch Master nodes/Leader */ masterCmd.Flags().String("zkPath", "/aurora/scheduler", "Zookeeper node path to get master nodes/leader") fetchCmd.AddCommand(masterCmd) // Hijack help function to hide unnecessary global flags masterCmd.SetHelpFunc(func(cmd *cobra.Command, s []string) { if cmd.HasInheritedFlags() { cmd.InheritedFlags().VisitAll(func(f *pflag.Flag) { if f.Name != "logLevel" { f.Hidden = true } }) } help(cmd, s) }) mesosMasterCmd.Flags().String("zkPath", "/mesos", "Zookeeper node path to get mesos master nodes/leader") mesosCmd.AddCommand(mesosMasterCmd) // Hijack help function to hide unnecessary global flags mesosMasterCmd.SetHelpFunc(func(cmd *cobra.Command, s []string) { if cmd.HasInheritedFlags() { cmd.InheritedFlags().VisitAll(func(f *pflag.Flag) { if f.Name != "logLevel" { f.Hidden = true } }) } help(cmd, s) }) // Fetch jobs fetchJobsCmd.Flags().StringVarP(role, "role", "r", "", "Aurora Role") fetchCmd.AddCommand(fetchJobsCmd) // Fetch Status fetchCmd.AddCommand(fetchStatusCmd) // fetch quota fetchCmd.AddCommand(fetchQuotaCmd) // fetch capacity fetchCmd.AddCommand(fetchAvailCapacityCmd) // Hijack help function to hide unnecessary global flags fetchAvailCapacityCmd.SetHelpFunc(func(cmd *cobra.Command, s []string) { if cmd.HasInheritedFlags() { cmd.InheritedFlags().VisitAll(func(f *pflag.Flag) { if f.Name != "logLevel" { f.Hidden = true } }) } help(cmd, s) }) // fetch tasks with status fetchCmd.AddCommand(fetchTasksWithStatusCmd) fetchTasksWithStatusCmd.Flags().StringVarP(taskStatus, "status", "x", "", "Task Status") fetchTasksWithStatusCmd.MarkFlagRequired("status") fetchTasksWithStatusCmd.Flags().StringVarP(env, "environment", "e", "", "Aurora Environment") fetchTasksWithStatusCmd.Flags().StringVarP(role, "role", "r", "", "Aurora Role") fetchTasksWithStatusCmd.Flags().StringVarP(name, "name", "n", "", "Aurora Name") // Hijack help function to hide unnecessary global flags fetchTasksWithStatusCmd.SetHelpFunc(func(cmd *cobra.Command, s []string) { if cmd.HasInheritedFlags() { cmd.InheritedFlags().VisitAll(func(f *pflag.Flag) { if f.Name != "logLevel" { f.Hidden = true } }) } help(cmd, s) }) } var fetchCmd = &cobra.Command{ Use: "fetch", Short: "Fetch information from Aurora", } var fetchTaskCmd = &cobra.Command{ Use: "task", Short: "Task information from Aurora", } var taskConfigCmd = &cobra.Command{ Use: "config", Short: "Fetch a list of task configurations from Aurora.", Long: `To be written.`, Run: fetchTasksConfig, } var taskStatusCmd = &cobra.Command{ Use: "status", Short: "Fetch task status for a Job key.", Long: `To be written.`, Run: fetchTasksStatus, } var leaderCmd = &cobra.Command{ Use: "leader [zkNode0, zkNode1, ...zkNodeN]", PersistentPreRun: func(cmd *cobra.Command, args []string) {}, // We don't need a realis client for this cmd PersistentPostRun: func(cmd *cobra.Command, args []string) {}, // We don't need a realis client for this cmd PreRun: setConfig, Args: cobra.MinimumNArgs(1), Short: "Fetch current Aurora leader given Zookeeper nodes. ", Long: `Gets the current leading aurora scheduler instance using information from Zookeeper path. Pass Zookeeper nodes separated by a space as an argument to this command.`, Run: fetchLeader, } var masterCmd = &cobra.Command{ Use: "master [zkNode0 zkNode1 ...zkNodeN]", PersistentPreRun: func(cmd *cobra.Command, args []string) {}, // We don't need a realis client for this cmd PersistentPostRun: func(cmd *cobra.Command, args []string) {}, // We don't need a realis client for this cmd PreRun: setConfig, Args: cobra.MinimumNArgs(1), Short: "Fetch current Aurora master nodes/leader given Zookeeper nodes. ", Long: `Gets the current aurora master nodes/leader using information from Zookeeper path. Pass Zookeeper nodes separated by a space as an argument to this command.`, Run: fetchMaster, } var mesosCmd = &cobra.Command{ Use: "mesos", PreRun: setConfig, Short: "Fetch information from Mesos.", } var mesosLeaderCmd = &cobra.Command{ Use: "leader [zkNode0, zkNode1, ...zkNodeN]", PersistentPreRun: func(cmd *cobra.Command, args []string) {}, // We don't need a realis client for this cmd PersistentPostRun: func(cmd *cobra.Command, args []string) {}, // We don't need a realis client for this cmd PreRun: setConfig, Short: "Fetch current Mesos-master leader given Zookeeper nodes.", Long: `Gets the current leading Mesos-master instance using information from Zookeeper path. Pass Zookeeper nodes separated by a space as an argument to this command. If no nodes are provided, it fetches leader from local Mesos agent or Zookeeper`, Run: fetchMesosLeader, } var mesosMasterCmd = &cobra.Command{ Use: "master [zkNode0 zkNode1 ...zkNodeN]", PersistentPreRun: func(cmd *cobra.Command, args []string) {}, // We don't need a realis client for this cmd PersistentPostRun: func(cmd *cobra.Command, args []string) {}, // We don't need a realis client for this cmd PreRun: setConfig, Short: "Fetch current Mesos-master nodes/leader given Zookeeper nodes.", Long: `Gets the current Mesos-master instances using information from Zookeeper path. Pass Zookeeper nodes separated by a space as an argument to this command. If no nodes are provided, it fetches Mesos-master nodes/leader from local Mesos agent or Zookeeper`, Run: fetchMesosMaster, } var fetchJobsCmd = &cobra.Command{ Use: "jobs", Short: "Fetch a list of task Aurora running under a role.", Long: `To be written.`, Run: fetchJobs, } var fetchStatusCmd = &cobra.Command{ Use: "status", Short: "Fetch the maintenance status of a node from Aurora", Long: `This command will print the actual status of the mesos agent nodes in Aurora server`, Run: fetchHostStatus, } var fetchQuotaCmd = &cobra.Command{ Use: "quota", Short: "Fetch the quotas of given roles", Long: `This command will print list of resource quotas with the aggregated resources for the given roles`, Run: fetchQuota, } var fetchAvailCapacityCmd = &cobra.Command{ Use: "capacity", PreRun: setConfig, Short: "Fetch capacity report", Long: `This command will show detailed capacity report of the cluster`, Run: fetchAvailCapacity, } var fetchTasksWithStatusCmd = &cobra.Command{ Use: "tasks", Short: "Fetch tasks with status", Long: `This command will return the list of tasks with a given status`, Run: fetchTasksWithStatus, } func fetchTasksConfig(cmd *cobra.Command, args []string) { log.Infof("Fetching job configuration for [%s/%s/%s] \n", *env, *role, *name) // Task Query takes nil for values it shouldn't need to match against. // This allows us to potentially more expensive calls for specific environments, roles, or job names. if *env == "" { env = nil } if *role == "" { role = nil } if *role == "" { role = nil } //TODO: Add filtering down by status taskQuery := &aurora.TaskQuery{Environment: env, Role: role, JobName: name} tasks, err := client.GetTasksWithoutConfigs(taskQuery) if err != nil { log.Fatalf("error: %+v", err) } if toJson { fmt.Println(internal.ToJSON(tasks)) } else { for _, t := range tasks { fmt.Println(t) } } } func fetchTasksStatus(cmd *cobra.Command, args []string) { log.Infof("Fetching task status for [%s/%s/%s] \n", *env, *role, *name) // Task Query takes nil for values it shouldn't need to match against. // This allows us to potentially more expensive calls for specific environments, roles, or job names. if *env == "" { env = nil } if *role == "" { role = nil } if *role == "" { role = nil } // TODO(rdelvalle): Add filtering down by status taskQuery := &aurora.TaskQuery{ Environment: env, Role: role, JobName: name, Statuses: aurora.LIVE_STATES} tasks, err := client.GetTaskStatus(taskQuery) if err != nil { log.Fatalf("error: %+v", err) } if toJson { fmt.Println(internal.ToJSON(tasks)) } else { for _, t := range tasks { fmt.Println(t) } } } func fetchHostStatus(cmd *cobra.Command, args []string) { log.Infof("Fetching maintenance status for %v \n", args) result, err := client.MaintenanceStatus(args...) if err != nil { log.Fatalf("error: %+v\n", err) } if toJson { fmt.Println(internal.ToJSON(result.Statuses)) } else { for _, k := range result.GetStatuses() { fmt.Printf("Result: %s:%s\n", k.Host, k.Mode) } } } func fetchLeader(cmd *cobra.Command, args []string) { log.Infof("Fetching leader from %v \n", args) if len(args) < 1 { log.Fatalln("At least one Zookeeper node address must be passed in.") } url, err := realis.LeaderFromZKOpts(realis.ZKEndpoints(args...), realis.ZKPath(cmd.Flag("zkPath").Value.String())) if err != nil { log.Fatalf("error: %+v\n", err) } fmt.Println(url) } func fetchMesosLeader(cmd *cobra.Command, args []string) { if len(args) < 1 { mesosAgentFlags, err := fetchMasterFromAgent(localAgentStateURL) if err != nil || mesosAgentFlags.Master == "" { log.Debugf("unable to fetch Mesos leader via local Mesos agent: %v", err) args = append(args, "localhost") } else if mesosAgentFlags.hasMaster { fmt.Println(mesosAgentFlags.Master) return } else { args = append(args, strings.Split(mesosAgentFlags.Master, ",")...) } } log.Infof("Fetching Mesos-master leader from Zookeeper node(s): %v \n", args) url, err := realis.MesosFromZKOpts(realis.ZKEndpoints(args...), realis.ZKPath(cmd.Flag("zkPath").Value.String())) if err != nil { log.Fatalf("error: %+v\n", err) } fmt.Println(url) } func fetchMaster(cmd *cobra.Command, args []string) { log.Infof("Fetching master nodes from %v \n", args) if len(args) < 1 { log.Fatalln("At least one Zookeeper node address must be passed in.") } masterMap, err := realis.MasterNodesFromZKOpts(realis.ZKEndpoints(args...), realis.ZKPath(cmd.Flag("zkPath").Value.String())) if err != nil { log.Fatalf("error: %+v\n", err) } if toJson { fmt.Println(internal.ToJSON(masterMap)) } else { for key, masterNodes := range masterMap { for _, masterNode := range masterNodes { fmt.Println(key + "=" + masterNode) } } } } func fetchMesosMaster(cmd *cobra.Command, args []string) { if len(args) < 1 { mesosAgentFlags, err := fetchMasterFromAgent(localAgentStateURL) if err != nil || mesosAgentFlags.Master == "" { log.Debugf("unable to fetch Mesos master nodes via local Mesos agent: %v", err) args = append(args, "localhost") } else { args = append(args, strings.Split(mesosAgentFlags.Master, ",")...) } } log.Infof("Fetching Mesos-master nodes from Zookeeper node(s): %v \n", args) mesosMasterMap, err := realis.MesosMasterNodesFromZKOpts(realis.ZKEndpoints(args...), realis.ZKPath(cmd.Flag("zkPath").Value.String())) if err != nil { log.Fatalf("error: %+v\n", err) } if toJson { fmt.Println(internal.ToJSON(mesosMasterMap)) } else { for key, mesosMasterNodes := range mesosMasterMap { for _, mesosMasterNode := range mesosMasterNodes { fmt.Println(key + "=" + mesosMasterNode) } } } } func fetchMasterFromAgent(url string) (mesosAgentFlags mesosAgentFlags, err error) { resp, err := http.Get(url) if err != nil { return } if resp.StatusCode != 200 { return } defer resp.Body.Close() state := &mesosAgentState{} err = json.NewDecoder(resp.Body).Decode(state) if err != nil { return } mesosAgentFlags = state.Flags err = updateMasterFlag(&mesosAgentFlags) return } /* Master flag can be passed as one of : host:port zk://host1:port1,host2:port2,.../path zk://username:password@host1:port1,host2:port2,.../path file:///path/to/file This function takes care of all the above cases and updates flags with parsed values */ func updateMasterFlag(flags *mesosAgentFlags) error { zkPathPrefix := "zk://" filePathPrefix := "file://" if strings.HasPrefix(flags.Master, zkPathPrefix) { beginIndex := len(zkPathPrefix) if strings.Contains(flags.Master, "@") { beginIndex = strings.Index(flags.Master, "@") + 1 } flags.Master = flags.Master[beginIndex:strings.LastIndex(flags.Master, "/")] } else if strings.HasPrefix(flags.Master, filePathPrefix) { content, err := ioutil.ReadFile(flags.Master) if err != nil { return err } if strings.Contains(string(content), filePathPrefix) { return errors.New("invalid master file content") } flags.Master = string(content) return updateMasterFlag(flags) } else { flags.hasMaster = true } return nil } // TODO: Expand this to be able to filter by job name and environment. func fetchJobs(cmd *cobra.Command, args []string) { log.Infof("Fetching tasks under role: %s \n", *role) if *role == "" { log.Fatalln("Role must be specified.") } if *role == "*" { log.Warnln("This is an expensive operation.") *role = "" } result, err := client.GetJobs(*role) if err != nil { log.Fatalf("error: %+v", err) } if toJson { var configSlice []*aurora.JobConfiguration for _, config := range result.GetConfigs() { configSlice = append(configSlice, config) } fmt.Println(internal.ToJSON(configSlice)) } else { for jobConfig := range result.GetConfigs() { fmt.Println(jobConfig) } } } //fetchQuota gets quotas for roles in args func fetchQuota(cmd *cobra.Command, args []string) { for _, role := range args { log.Infof("Fetching quota for role: %s \n", role) result, err := client.GetQuota(role) if err != nil { log.Fatalf("error: %+v\n", err) } if toJson { fmt.Println(internal.ToJSON(result)) } else { fmt.Printf(" Quota: %v\n", internal.ToJSON(result.Quota.GetResources())) fmt.Printf(" Aggregated Resources: \n") fmt.Printf(" ProdSharedConsumption: %v\n", internal.ToJSON(result.ProdSharedConsumption.GetResources())) fmt.Printf(" NonProdSharedConsumption: %v\n", internal.ToJSON(result.NonProdSharedConsumption.GetResources())) fmt.Printf(" ProdDedicatedConsumption: %v\n", internal.ToJSON(result.ProdDedicatedConsumption.GetResources())) fmt.Printf(" NonProdDedicatedConsumption: %v\n", internal.ToJSON(result.NonProdDedicatedConsumption.GetResources())) } } } //fetchAvailCapacity reports free capacity in details func fetchAvailCapacity(cmd *cobra.Command, args []string) { log.Infof("Fetching available capacity from %s/offers\n", client.GetSchedulerURL()) report, err := client.AvailOfferReport() if err != nil { log.Fatalf("error: %+v\n", err) } // convert report to user-friendly structure capacity := map[string]map[string]map[string]int64{} for g, gv := range report { if _, ok := capacity[g]; !ok { capacity[g] = map[string]map[string]int64{} } for r, rc := range gv { if _, ok := capacity[g][r]; !ok { capacity[g][r] = map[string]int64{} } for v, c := range rc { capacity[g][r][fmt.Sprint(v)] = c } } } if toJson { fmt.Println(internal.ToJSON(capacity)) if err != nil { log.Fatalf("error: %+v\n", err) } } else { fmt.Println(capacity) } } //fetchTasksWithStatus returns lists of tasks for a given set of status func fetchTasksWithStatus(cmd *cobra.Command, args []string) { status := *taskStatus log.Infof("Fetching tasks for role/environment/job:[%s/%s/%s] \n", *role, *env, *name) log.Infof("Fetching tasks for a given status: %v \n", status) // This Query takes nil for values it shouldn't need to match against. // This allows us to potentially avoid expensive calls for specific environments, roles, or job names. if *env == "" { env = nil } if *role == "" { role = nil } if *name == "" { name = nil } // role needs to be specified if env is specified if env != nil { if role == nil { log.Fatalln("Role must be specified when env is specified.") } } // role or env needs to be specified if name is specified if name != nil { if role == nil && env == nil { log.Fatalln("Role or env must be specified when name is specified.") } } queryStatuses, err := scheduleStatusFromString(status) if err != nil { log.Fatalf("error: %+v", err) } taskQuery := &aurora.TaskQuery{Environment: env, Role: role, JobName: name, Statuses: queryStatuses} tasks, err := client.GetTasksWithoutConfigs(taskQuery) if err != nil { log.Fatalf("error: %+v", err) } if toJson { taskStatus := strings.ToUpper(status) // convert task lists to a list of task id like role-env-name-[instance-id] taskIdsMap := map[string][]string{} var taskIds []string for _, task := range tasks { taskIds = append(taskIds, task.AssignedTask.TaskId) } taskIdsMap[taskStatus] = taskIds fmt.Println(internal.ToJSON(taskIdsMap)) } else { fmt.Printf("Tasks for status %s:\n", strings.ToUpper(status)) for _, t := range tasks { fmt.Println(t.AssignedTask.TaskId) } } } // Convert status slice into ScheduleStatus slice func scheduleStatusFromString(status string) ([]aurora.ScheduleStatus, error) { scheduleStatus, err := aurora.ScheduleStatusFromString(strings.ToUpper(status)) if err != nil { return nil, err } result := []aurora.ScheduleStatus{scheduleStatus} return result, nil }