workload validation before registering elektron. #19
13
def/task.go
|
@ -20,6 +20,7 @@ package def
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"github.com/spdfg/elektron/utilities/validation"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
mesos "github.com/mesos/mesos-go/api/v0/mesosproto"
|
mesos "github.com/mesos/mesos-go/api/v0/mesosproto"
|
||||||
|
@ -55,6 +56,18 @@ func TasksFromJSON(uri string) ([]Task, error) {
|
||||||
return nil, errors.Wrap(err, "Error unmarshalling")
|
return nil, errors.Wrap(err, "Error unmarshalling")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Validating task definitions.
|
||||||
|
for _, task := range tasks {
|
||||||
|
err := validation.Validate("invalid task definition",
|
||||||
|
ValidatorForTask(task,
|
||||||
|
withNameValidator(),
|
||||||
|
withImageValidator(),
|
||||||
|
withResourceValidator()))
|
||||||
|
if err != nil {
|
||||||
|
return tasks, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|||||||
initTaskResourceRequirements(tasks)
|
initTaskResourceRequirements(tasks)
|
||||||
return tasks, nil
|
return tasks, nil
|
||||||
}
|
}
|
||||||
|
|
91
def/taskValidators.go
Normal file
|
@ -0,0 +1,91 @@
|
||||||
|
// Copyright (C) 2018 spdfg
|
||||||
|
//
|
||||||
|
// This file is part of Elektron.
|
||||||
|
//
|
||||||
|
// Elektron is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// Elektron is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License
|
||||||
|
// along with Elektron. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
//
|
||||||
|
|
||||||
|
package def
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"github.com/spdfg/elektron/utilities/validation"
|
||||||
|
"regexp"
|
||||||
|
)
|
||||||
|
|
||||||
|
// taskValidator is a validator that validates one or more attributes of a task.
|
||||||
|
type taskValidator func(Task) error
|
||||||
|
|
||||||
|
// ValidatorForTask returns a validator that runs all provided taskValidators and
|
||||||
|
// returns an error corresponding to the first taskValidator that failed.
|
||||||
|
func ValidatorForTask(t Task, taskValidators ...taskValidator) validation.Validator {
|
||||||
|
return func() error {
|
||||||
|
for _, tv := range taskValidators {
|
||||||
|
if err := tv(t); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// withNameValidator returns a taskValidator that checks whether the task name is valid.
|
||||||
|
func withNameValidator() taskValidator {
|
||||||
|
return func(t Task) error {
|
||||||
|
// Task name cannot be empty string.
|
||||||
|
if t.Name == "" {
|
||||||
|
return errors.New("task name cannot be empty string")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Task name cannot contain tabs or spaces.
|
||||||
|
matched, _ := regexp.MatchString("\\t+|\\s+", t.Name)
|
||||||
|
if matched {
|
||||||
|
return errors.New("task name cannot contain tabs or spaces")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// withResourceValidator returns a taskValidator that checks whether the resource requirements are valid.
|
||||||
|
// Currently, only requirements for traditional resources (cpu and memory) are validated.
|
||||||
|
func withResourceValidator() taskValidator {
|
||||||
|
return func(t Task) error {
|
||||||
|
// CPU value cannot be 0.
|
||||||
|
if t.CPU == 0.0 {
|
||||||
|
return errors.New("cpu resource for task cannot be 0")
|
||||||
|
}
|
||||||
|
|
||||||
|
// RAM value cannot be 0.
|
||||||
|
if t.RAM == 0.0 {
|
||||||
|
return errors.New("memory resource for task cannot be 0")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// withImageValidator returns a taskValidator that checks whether a valid image has been
|
||||||
|
// provided for the task.
|
||||||
|
func withImageValidator() taskValidator {
|
||||||
|
return func(t Task) error {
|
||||||
|
// Image cannot be empty.
|
||||||
|
if t.Image == "" {
|
||||||
|
return errors.New("valid image needs to be provided for task")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
75
def/taskValidators_test.go
Normal file
|
@ -0,0 +1,75 @@
|
||||||
|
// Copyright (C) 2018 spdfg
|
||||||
|
//
|
||||||
|
// This file is part of Elektron.
|
||||||
|
//
|
||||||
|
// Elektron is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// Elektron is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License
|
||||||
|
// along with Elektron. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
//
|
||||||
|
|
||||||
|
package def
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/spdfg/elektron/utilities/validation"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidatorForTask(t *testing.T) {
|
||||||
|
getValidator := func(t Task) validation.Validator {
|
||||||
|
return ValidatorForTask(t,
|
||||||
|
withNameValidator(),
|
||||||
|
withImageValidator(),
|
||||||
|
withResourceValidator())
|
||||||
|
}
|
||||||
|
|
||||||
|
test := func(task Task, expectErr bool, message string) {
|
||||||
|
validator := getValidator(task)
|
||||||
|
if expectErr {
|
||||||
|
assert.Error(t, validation.Validate(message, validator))
|
||||||
|
} else {
|
||||||
|
assert.NoError(t, validation.Validate(message, validator))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
inst := 10
|
||||||
|
validTask := Task{
|
||||||
|
Name: "minife",
|
||||||
|
CPU: 3.0,
|
||||||
|
RAM: 4096,
|
||||||
|
Watts: 50,
|
||||||
|
Image: "rdelvalle/minife:electron1",
|
||||||
|
CMD: "cd src && mpirun -np 1 miniFE.x -nx 100 -ny 100 -nz 100",
|
||||||
|
Instances: &inst,
|
||||||
|
}
|
||||||
|
test(validTask, false, "invalid task definition")
|
||||||
|
// Task with empty name.
|
||||||
|
invalidTaskEmptyName := validTask
|
||||||
|
invalidTaskEmptyName.Name = ""
|
||||||
|
test(invalidTaskEmptyName, true, "invalid task definition")
|
||||||
|
// Task with name that contains spaces.
|
||||||
|
invalidTaskNameWithSpaces := validTask
|
||||||
|
invalidTaskNameWithSpaces.Name = "my task"
|
||||||
|
test(invalidTaskNameWithSpaces, true, "invalid task definition")
|
||||||
|
// Task with invalid image.
|
||||||
|
invalidTaskImage := validTask
|
||||||
|
invalidTaskImage.Image = ""
|
||||||
|
test(invalidTaskImage, true, "invalid task definition")
|
||||||
|
// Task with invalid cpu resources.
|
||||||
|
invalidTaskResourcesCPU := validTask
|
||||||
|
invalidTaskResourcesCPU.CPU = 0
|
||||||
|
test(invalidTaskResourcesCPU, true, "invalid task definition")
|
||||||
|
// Task with invalid memory resources.
|
||||||
|
invalidTaskResourcesRAM := validTask
|
||||||
|
invalidTaskResourcesRAM.RAM = 0
|
||||||
|
test(invalidTaskResourcesRAM, true, "invalid task definition")
|
||||||
|
}
|
|
@ -208,7 +208,7 @@ func main() {
|
||||||
}
|
}
|
||||||
tasks, err := def.TasksFromJSON(*tasksFile)
|
tasks, err := def.TasksFromJSON(*tasksFile)
|
||||||
if err != nil || len(tasks) == 0 {
|
if err != nil || len(tasks) == 0 {
|
||||||
log.Fatal("Invalid tasks specification file provided.")
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
schedOptions = append(schedOptions, schedulers.WithTasks(tasks))
|
schedOptions = append(schedOptions, schedulers.WithTasks(tasks))
|
||||||
|
|
||||||
|
|
40
utilities/validation/validate.go
Normal file
|
@ -0,0 +1,40 @@
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
// Copyright (C) 2018 spdfg
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
//
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
// This file is part of Elektron.
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
//
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
// Elektron is free software: you can redistribute it and/or modify
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
// it under the terms of the GNU General Public License as published by
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
// (at your option) any later version.
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
//
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
// Elektron is distributed in the hope that it will be useful,
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
// GNU General Public License for more details.
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
//
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
// You should have received a copy of the GNU General Public License
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
// along with Elektron. If not, see <http://www.gnu.org/licenses/>.
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
//
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
// Package validation contains utilities to help run validators.
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
package validation
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
import "github.com/pkg/errors"
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
// Validator is a function that performs some sort of validation.
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
// To keep things generic, this function does not accept any arguments.
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
// In practice, a validator could be a closure.
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
type Validator func() error
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
// Validate a list of validators.
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
// If validation fails, then wrap the returned error with the given base
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
// error message.
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
func Validate(baseErrMsg string, validators ...Validator) error {
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
for _, v := range validators {
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
if err := v(); err != nil {
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
return errors.Wrap(err, baseErrMsg)
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
}
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
}
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
return nil
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
|||||||
|
}
|
||||||
![]() Do we need to keep this around? Do we need to keep this around?
![]() Yeah, the example might be an overkill. I'll get rid of it. Yeah, the example might be an overkill. I'll get rid of it.
|
Awesome idea