workload validation before registering elektron. #19

Merged
pradykaushik merged 2 commits from issues/issue-18-workload-validation into master 2019-11-26 08:08:12 +00:00
5 changed files with 220 additions and 1 deletions

View file

@ -20,6 +20,7 @@ package def
import (
"encoding/json"
"github.com/spdfg/elektron/utilities/validation"
"os"
mesos "github.com/mesos/mesos-go/api/v0/mesosproto"
@ -55,6 +56,18 @@ func TasksFromJSON(uri string) ([]Task, error) {
return nil, errors.Wrap(err, "Error unmarshalling")
}
// Validating task definitions.
for _, task := range tasks {
err := validation.Validate("invalid task definition",
ValidatorForTask(task,
withNameValidator(),
withImageValidator(),
withResourceValidator()))
if err != nil {
return tasks, err
}
}
ridv commented 2019-11-25 22:46:17 +00:00 (Migrated from github.com)

Awesome idea

Awesome idea
initTaskResourceRequirements(tasks)
return tasks, nil
}

91
def/taskValidators.go Normal file
View file

@ -0,0 +1,91 @@
// Copyright (C) 2018 spdfg
//
// This file is part of Elektron.
//
// Elektron is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Elektron is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Elektron. If not, see <http://www.gnu.org/licenses/>.
//
package def
import (
"github.com/pkg/errors"
"github.com/spdfg/elektron/utilities/validation"
"regexp"
)
// taskValidator is a validator that validates one or more attributes of a task.
type taskValidator func(Task) error
// ValidatorForTask returns a validator that runs all provided taskValidators and
// returns an error corresponding to the first taskValidator that failed.
func ValidatorForTask(t Task, taskValidators ...taskValidator) validation.Validator {
return func() error {
for _, tv := range taskValidators {
if err := tv(t); err != nil {
return err
}
}
return nil
}
}
// withNameValidator returns a taskValidator that checks whether the task name is valid.
func withNameValidator() taskValidator {
return func(t Task) error {
// Task name cannot be empty string.
if t.Name == "" {
return errors.New("task name cannot be empty string")
}
// Task name cannot contain tabs or spaces.
matched, _ := regexp.MatchString("\\t+|\\s+", t.Name)
if matched {
return errors.New("task name cannot contain tabs or spaces")
}
return nil
}
}
// withResourceValidator returns a taskValidator that checks whether the resource requirements are valid.
// Currently, only requirements for traditional resources (cpu and memory) are validated.
func withResourceValidator() taskValidator {
return func(t Task) error {
// CPU value cannot be 0.
if t.CPU == 0.0 {
return errors.New("cpu resource for task cannot be 0")
}
// RAM value cannot be 0.
if t.RAM == 0.0 {
return errors.New("memory resource for task cannot be 0")
}
return nil
}
}
// withImageValidator returns a taskValidator that checks whether a valid image has been
// provided for the task.
func withImageValidator() taskValidator {
return func(t Task) error {
// Image cannot be empty.
if t.Image == "" {
return errors.New("valid image needs to be provided for task")
}
return nil
}
}

View file

@ -0,0 +1,75 @@
// Copyright (C) 2018 spdfg
//
// This file is part of Elektron.
//
// Elektron is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Elektron is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Elektron. If not, see <http://www.gnu.org/licenses/>.
//
package def
import (
"github.com/spdfg/elektron/utilities/validation"
"github.com/stretchr/testify/assert"
"testing"
)
func TestValidatorForTask(t *testing.T) {
getValidator := func(t Task) validation.Validator {
return ValidatorForTask(t,
withNameValidator(),
withImageValidator(),
withResourceValidator())
}
test := func(task Task, expectErr bool, message string) {
validator := getValidator(task)
if expectErr {
assert.Error(t, validation.Validate(message, validator))
} else {
assert.NoError(t, validation.Validate(message, validator))
}
}
inst := 10
validTask := Task{
Name: "minife",
CPU: 3.0,
RAM: 4096,
Watts: 50,
Image: "rdelvalle/minife:electron1",
CMD: "cd src && mpirun -np 1 miniFE.x -nx 100 -ny 100 -nz 100",
Instances: &inst,
}
test(validTask, false, "invalid task definition")
// Task with empty name.
invalidTaskEmptyName := validTask
invalidTaskEmptyName.Name = ""
test(invalidTaskEmptyName, true, "invalid task definition")
// Task with name that contains spaces.
invalidTaskNameWithSpaces := validTask
invalidTaskNameWithSpaces.Name = "my task"
test(invalidTaskNameWithSpaces, true, "invalid task definition")
// Task with invalid image.
invalidTaskImage := validTask
invalidTaskImage.Image = ""
test(invalidTaskImage, true, "invalid task definition")
// Task with invalid cpu resources.
invalidTaskResourcesCPU := validTask
invalidTaskResourcesCPU.CPU = 0
test(invalidTaskResourcesCPU, true, "invalid task definition")
// Task with invalid memory resources.
invalidTaskResourcesRAM := validTask
invalidTaskResourcesRAM.RAM = 0
test(invalidTaskResourcesRAM, true, "invalid task definition")
}

View file

@ -208,7 +208,7 @@ func main() {
}
tasks, err := def.TasksFromJSON(*tasksFile)
if err != nil || len(tasks) == 0 {
log.Fatal("Invalid tasks specification file provided.")
log.Fatal(err)
}
schedOptions = append(schedOptions, schedulers.WithTasks(tasks))

View file

@ -0,0 +1,40 @@
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
// Copyright (C) 2018 spdfg
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
//
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
// This file is part of Elektron.
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
//
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
// Elektron is free software: you can redistribute it and/or modify
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
// it under the terms of the GNU General Public License as published by
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
// the Free Software Foundation, either version 3 of the License, or
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
// (at your option) any later version.
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
//
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
// Elektron is distributed in the hope that it will be useful,
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
// but WITHOUT ANY WARRANTY; without even the implied warranty of
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
// GNU General Public License for more details.
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
//
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
// You should have received a copy of the GNU General Public License
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
// along with Elektron. If not, see <http://www.gnu.org/licenses/>.
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
//
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
// Package validation contains utilities to help run validators.
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
package validation
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
import "github.com/pkg/errors"
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
// Validator is a function that performs some sort of validation.
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
// To keep things generic, this function does not accept any arguments.
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
// In practice, a validator could be a closure.
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
type Validator func() error
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
// Validate a list of validators.
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
// If validation fails, then wrap the returned error with the given base
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
// error message.
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
func Validate(baseErrMsg string, validators ...Validator) error {
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
for _, v := range validators {
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
if err := v(); err != nil {
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
return errors.Wrap(err, baseErrMsg)
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
}
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
}
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
return nil
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.
}
ridv commented 2019-11-25 22:54:02 +00:00 (Migrated from github.com)

Do we need to keep this around?

Do we need to keep this around?
pradykaushik commented 2019-11-26 07:59:30 +00:00 (Migrated from github.com)

Yeah, the example might be an overkill. I'll get rid of it.

Yeah, the example might be an overkill. I'll get rid of it.