├── .gitignore ├── LICENSE ├── README.adoc ├── bash_completion └── test161 ├── build.go ├── build_test.go ├── collab.go ├── commands.go ├── commands_test.go ├── conf.go ├── conf_test.go ├── environment.go ├── expect ├── expect.go └── logger.go ├── fixtures ├── .gitignore ├── README.adoc ├── commands │ ├── commands.tc │ └── misc.tc ├── keys │ ├── id_rsa.pub │ └── test@test161.ops-class.org │ │ └── id_rsa ├── overlays │ └── simple │ │ ├── SECRET │ │ └── kern │ │ └── include │ │ └── kern │ │ └── secret.h ├── sys161 │ └── .gitignore ├── tags │ └── all.td ├── targets │ ├── asst1.tt │ ├── entire.tt │ ├── full.tt │ ├── meta.1.tt │ ├── meta.2.tt │ ├── meta.tt │ ├── partial.tt │ └── simple.tt └── tests │ ├── boot.t │ ├── cycle │ ├── boot.t │ ├── sync │ │ ├── semu1.t │ │ ├── sy1.t │ │ ├── sy2.t │ │ ├── sy3.t │ │ ├── sy4.t │ │ └── sy5.t │ └── threads │ │ ├── tt1.t │ │ ├── tt2.t │ │ └── tt3.t │ └── nocycle │ ├── boot.t │ ├── panics │ ├── deppanic.t │ └── panic.t │ ├── sync │ ├── all.t │ ├── cvt1.t │ ├── cvt2.t │ ├── cvt3.t │ ├── cvt4.t │ ├── fail.t │ ├── lt1.t │ ├── lt2.t │ ├── lt3.t │ ├── multi.t │ ├── sem1.t │ └── semu1.t │ └── threads │ ├── tt1.t │ ├── tt2.t │ └── tt3.t ├── graph ├── graph.go └── graph_test.go ├── groups.go ├── groups_test.go ├── logger.go ├── man ├── test161-server.1 └── test161.1 ├── manager.go ├── mongopersist.go ├── mongopersist_test.go ├── output.go ├── persistence.go ├── run.go ├── run_test.go ├── runners.go ├── runners_test.go ├── stats.go ├── stats_test.go ├── submission.go ├── submission_test.go ├── tags.go ├── tags_test.go ├── target.go ├── target_test.go ├── test161-server ├── .gitignore ├── comm.go ├── control.go ├── routes.go ├── server.go ├── submissions.go ├── uploads.go └── usage.go ├── test161 ├── .gitignore ├── comm.go ├── conf.go ├── consolepersist.go ├── errors.go ├── fixtures │ └── files │ │ ├── usage.json.gz │ │ ├── usage_013999999999.json.gz │ │ └── usage_014999999999.json.gz ├── git.go ├── listcmd.go ├── main.go ├── printing.go ├── runcmd.go ├── submitcmd.go ├── usage.go └── usage_test.go ├── testing_persistence.go ├── usage.go ├── usage_test.go ├── version.go └── version_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | *.swo 2 | *.swp 3 | /src 4 | /pkg 5 | /bin 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Scott Haseley 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /bash_completion/test161: -------------------------------------------------------------------------------- 1 | _test161() 2 | { 3 | local cur prev opts cmd 4 | COMPREPLY=() 5 | cur="${COMP_WORDS[COMP_CWORD]}" 6 | prev="${COMP_WORDS[COMP_CWORD-1]}" 7 | cmd="${COMP_WORDS[1]}" 8 | opts="run submit list config version" 9 | 10 | case "$cmd" in 11 | version) 12 | COMPREPLY=() 13 | return 0 14 | ;; 15 | 16 | run) 17 | case "$cur" in 18 | -*) 19 | local runopts tests 20 | runopts="-dry-run -explain -sequential -no-dependencies -verbose -tag" 21 | COMPREPLY=( $(compgen -W "${runopts}" -- $cur) ) 22 | return 0 23 | ;; 24 | 25 | *) 26 | local tests 27 | tests=$(test161 list all 2>/dev/null) 28 | COMPREPLY=( $(compgen -W "${tests}" -- $cur) ) 29 | return 0 30 | ;; 31 | esac 32 | ;; 33 | 34 | submit) 35 | case "$cur" in 36 | -*) 37 | local submitopts 38 | submitopts="-debug -verify -no-cache" 39 | COMPREPLY=( $(compgen -W "${submitopts}" -- $cur) ) 40 | return 0 41 | ;; 42 | esac 43 | 44 | case "$prev" in 45 | submit|-debug|-verify|-no-cache) 46 | local targets 47 | targets=$(test161 list targets | awk 'NR>3' | cut -f 1 -d " ") 48 | COMPREPLY=( $(compgen -W "${targets}" -- $cur) ) 49 | return 0 50 | ;; 51 | esac 52 | ;; 53 | config) 54 | case "$prev" in 55 | test161dir) 56 | COMPREPLY=( $(compgen -o dirnames -- $cur) ) 57 | esac 58 | esac 59 | 60 | case "$prev" in 61 | list) 62 | local listopts 63 | listopts="targets tags tests" 64 | COMPREPLY=( $(compgen -W "${listopts}" -- $cur) ) 65 | return 0 66 | ;; 67 | 68 | targets) 69 | COMPREPLY=( $(compgen -W "-remote" -- $cur) ) 70 | return 0 71 | ;; 72 | 73 | config) 74 | COMPREPLY=( $(compgen -W "add-user del-user change-token test161dir" -- $cur) ) 75 | return 0 76 | ;; 77 | 78 | tags) 79 | local tags 80 | tags=$(test161 list tagnames 2>/dev/null) 81 | COMPREPLY=( $(compgen -W "${tags} -short" -- $cur) ) 82 | return 0 83 | ;; 84 | 85 | -verbose) 86 | COMPREPLY=( $(compgen -W "loud quiet whisper" -- $cur) ) 87 | return 0 88 | ;; 89 | 90 | esac 91 | 92 | if [[ ${COMP_CWORD} -eq 1 ]]; then 93 | COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) 94 | fi 95 | return 0 96 | } 97 | complete -F _test161 test161 98 | -------------------------------------------------------------------------------- /build_test.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestBuildFull(t *testing.T) { 9 | t.Parallel() 10 | assert := assert.New(t) 11 | 12 | conf := &BuildConf{ 13 | Repo: "https://github.com/ops-class/os161.git", 14 | CommitID: "HEAD", 15 | KConfig: "DUMBVM", 16 | } 17 | 18 | env := defaultEnv.CopyEnvironment() 19 | env.RootDir = "./fixtures/root" 20 | 21 | test, err := conf.ToBuildTest(env) 22 | assert.Nil(err) 23 | assert.NotNil(test) 24 | 25 | if test == nil { 26 | t.Log(err) 27 | t.FailNow() 28 | } 29 | 30 | _, err = test.Run(env) 31 | assert.Nil(err) 32 | 33 | t.Log(test.OutputJSON()) 34 | 35 | for k, v := range env.keyMap { 36 | t.Log(k, v) 37 | } 38 | 39 | } 40 | 41 | type confDetail struct { 42 | repo string 43 | commit string 44 | config string 45 | reqCommit string 46 | } 47 | 48 | func TestBuildFailures(t *testing.T) { 49 | t.Parallel() 50 | assert := assert.New(t) 51 | 52 | configs := []confDetail{ 53 | confDetail{"https://notgithub.com/ops-class/os161111.git", "HEAD", "DUMBVM", ""}, 54 | confDetail{"https://github.com/ops-class/os161.git", "aaaaaaaaaaa111111112222", "FOO", ""}, 55 | confDetail{"https://github.com/ops-class/os161.git", "HEAD", "FOO", ""}, 56 | confDetail{"https://github.com/ops-class/os161.git", "HEAD", "DUMBVM", "notavalidcommitit"}, 57 | } 58 | 59 | for _, c := range configs { 60 | 61 | conf := &BuildConf{ 62 | Repo: c.repo, 63 | CommitID: c.commit, 64 | KConfig: c.config, 65 | RequiredCommit: c.reqCommit, 66 | } 67 | 68 | test, err := conf.ToBuildTest(defaultEnv) 69 | assert.NotNil(test) 70 | 71 | res, err := test.Run(defaultEnv) 72 | assert.NotNil(err) 73 | assert.Nil(res) 74 | } 75 | } 76 | 77 | func TestHexString(t *testing.T) { 78 | t.Parallel() 79 | assert := assert.New(t) 80 | 81 | testCases := []struct { 82 | input string 83 | expected bool 84 | }{ 85 | {"0123456789abcdef", true}, 86 | {"0123456789ABCDEF", true}, 87 | {"e1a2fbd038c618b6d9e636a94e1907dc92e94ca6", true}, 88 | {"", false}, 89 | {"foo", false}, 90 | {"0123456789abcdefg", false}, 91 | {"!", false}, 92 | } 93 | 94 | for _, test := range testCases { 95 | t.Log(test.input) 96 | assert.Equal(test.expected, isHexString(test.input)) 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /collab.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | var collabMsgs = map[string]string{ 4 | "asst1": CollabMsgAsst1, 5 | "asst2": CollabMsgAsst2, 6 | "asst2.1": CollabMsgAsst2, 7 | "asst2.2": CollabMsgAsst2, 8 | "asst3": CollabMsgAsst3, 9 | "asst3.1": CollabMsgAsst3, 10 | "asst3.2": CollabMsgAsst3, 11 | "asst3.3": CollabMsgAsst3, 12 | } 13 | 14 | const CollabMsgAsst1 = ` 15 | - Pair programming to complete the implementation tasks is strongly 16 | encouraged. 17 | 18 | - Answering the code reading questions side-by-side with your partner is 19 | strongly encouraged. 20 | 21 | - Helping other students with Git, GDB, editing, and with other parts of the 22 | OS/161 toolchain is encouraged. 23 | 24 | - Discussing the code reading questions and browsing the source tree with 25 | other students is encouraged. 26 | 27 | - Dividing the code reading questions and development tasks between partners 28 | is discouraged. 29 | 30 | - Copying any answers from anyone who is not your partner or anywhere else and 31 | submitting them as your own is cheating. 32 | 33 | - You may not refer to or incorporate any external sources without explicit 34 | permission.` 35 | 36 | const CollabMsgAsst2 = ` 37 | - Pair programming to complete the implementation tasks is strongly 38 | encouraged. 39 | 40 | - Writing a design document with your partner is strongly encouraged. 41 | 42 | - Having one partner work on the file system system calls while the other 43 | partner works on process support is a good division of labor. The partner 44 | working on the file system system calls may finish first, at which point they 45 | can help and continue testing. 46 | 47 | - Answering the code reading questions side-by-side with your partner is 48 | strongly encouraged. 49 | 50 | - Discussing the code reading questions and browsing the source tree with other 51 | students is encouraged. 52 | 53 | - Dividing the code reading questions and development tasks between partners is 54 | discouraged. 55 | 56 | - Any arrangement that results in one partner writing the entire design 57 | document is cheating. 58 | 59 | - Any arrangement that results in one partner writing all or almost all of the 60 | code is cheating. 61 | 62 | - Copying any answers from anyone who is not your partner or anywhere else and 63 | submitting them as your own is cheating. 64 | 65 | - You may not refer to or incorporate any external sources without explicit 66 | permission.` 67 | 68 | const CollabMsgAsst3 = ` 69 | - Pair programming to complete the implementation tasks is strongly 70 | encouraged. 71 | 72 | - Writing a design document with your partner is strongly encouraged. 73 | 74 | - Answering the code reading questions side-by-side with your partner is 75 | strongly encouraged. 76 | 77 | - Discussing the code reading questions and browsing the source tree with other 78 | students is encouraged. 79 | 80 | - Dividing the code reading questions and development tasks between partners is 81 | discouraged. 82 | 83 | - Any arrangement that results in one partner writing the entire design 84 | document is cheating. 85 | 86 | - Any arrangement that results in one partner writing all or almost all of the 87 | code is cheating. 88 | 89 | - Copying any answers from anyone who is not your partner or anywhere else and 90 | submitting them as your own is cheating. 91 | 92 | - You may not refer to or incorporate any external sources without explicit 93 | permission.` 94 | -------------------------------------------------------------------------------- /commands.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "github.com/imdario/mergo" 8 | yaml "gopkg.in/yaml.v2" 9 | "io/ioutil" 10 | "math/rand" 11 | "strconv" 12 | "strings" 13 | "text/template" 14 | "time" 15 | ) 16 | 17 | // This file has everything for creating command instances from command templates. 18 | // In test161, a command is either a command run from the kernel menu (sy1), or 19 | // a single userspace program (i.e. /testbin/huge). 20 | // 21 | // Command Templates: Some commands, (e.g. argtest, factorial), have output that 22 | // depends on the input. For this reason, output may be specified as a golang 23 | // template, with various functions provided. Furthermore, random inputs can also be 24 | // generated using a template, which can be overriden in assignment files if needed. 25 | // 26 | // Command Instances: The input/expected output for an instance of a command instance 27 | // is created by executing the templates. 28 | // 29 | // Composite Commands: Some commands may execute other commands, i.e. triple*. For 30 | // these cases, there is an "external" property of the output line, which when set to 31 | // "true", specifies that the text property should be used to look up the output from 32 | // another command. 33 | // 34 | 35 | // Functions and function map & functions for command template evaluation 36 | 37 | func add(a int, b int) int { 38 | return a + b 39 | } 40 | 41 | func atoi(s string) (int, error) { 42 | return strconv.Atoi(s) 43 | } 44 | 45 | func randInt(min, max int) (int, error) { 46 | if min >= max { 47 | return 0, errors.New("max must be greater than min") 48 | } 49 | 50 | // between 0 and max-min 51 | temp := rand.Intn(max - min) 52 | 53 | // between min and mix 54 | return min + temp, nil 55 | } 56 | 57 | const stringChars string = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*()" 58 | 59 | func randString(min, max int) (string, error) { 60 | // Get the length of the string 61 | l, err := randInt(min, max) 62 | if err != nil { 63 | return "", err 64 | } 65 | 66 | // Make it 67 | b := make([]byte, l) 68 | for i := 0; i < l; i++ { 69 | b[i] = stringChars[rand.Intn(len(stringChars))] 70 | } 71 | 72 | return string(b), nil 73 | } 74 | 75 | func factorial(n int) int { 76 | if n == 0 { 77 | return 1 78 | } else { 79 | return n * factorial(n-1) 80 | } 81 | } 82 | 83 | // Get something of length n that we can range over. This is useful 84 | // for creating random inputs. 85 | func ranger(n int) []int { 86 | res := make([]int, n) 87 | for i := 0; i < n; i++ { 88 | res[i] = i 89 | } 90 | return res 91 | } 92 | 93 | // Functions we provide to the command templates. 94 | var funcMap template.FuncMap = template.FuncMap{ 95 | "add": add, 96 | "atoi": atoi, 97 | "factorial": factorial, 98 | "randInt": randInt, 99 | "randString": randString, 100 | "ranger": ranger, 101 | } 102 | 103 | // Data that we provide for command templates. 104 | type templateData struct { 105 | Args []string 106 | ArgLen int 107 | } 108 | 109 | // Command options for panics and timesout 110 | const ( 111 | CMD_OPT_NO = "no" 112 | CMD_OPT_MAYBE = "maybe" 113 | CMD_OPT_YES = "yes" 114 | ) 115 | 116 | // Template for commands instances. These get expanded depending on the command environment. 117 | type CommandTemplate struct { 118 | Name string `yaml:"name"` 119 | Output []*TemplOutputLine `yaml:"output"` 120 | Input []string `yaml:"input"` 121 | Panic string `yaml:"panics"` // CMD_OPT 122 | TimesOut string `yaml:"timesout"` // CMD_OPT 123 | Timeout float32 `yaml:"timeout"` // Timeout in sec. A timeout of 0.0 uses the test default. 124 | } 125 | 126 | // An expected line of output, which may either be expanded or not. 127 | type TemplOutputLine struct { 128 | Text string `yaml:"text"` 129 | Trusted string `yaml:"trusted"` 130 | External string `yaml:"external"` 131 | } 132 | 133 | // Command instance expected output line. The difference here is that we store the name 134 | // of the key that we need to verify the output. 135 | type ExpectedOutputLine struct { 136 | Text string 137 | Trusted bool 138 | KeyName string 139 | } 140 | 141 | func (ct *CommandTemplate) Clone() *CommandTemplate { 142 | clone := *ct 143 | clone.Output = make([]*TemplOutputLine, 0, len(ct.Output)) 144 | clone.Input = make([]string, 0, len(ct.Input)) 145 | 146 | for _, o := range ct.Output { 147 | copy := *o 148 | clone.Output = append(clone.Output, ©) 149 | } 150 | 151 | for _, s := range ct.Input { 152 | clone.Input = append(clone.Input, s) 153 | } 154 | 155 | return &clone 156 | } 157 | 158 | // Expand the golang text template using the provided tempate data. 159 | // We do this on a per-command instance basis, since output can change 160 | // depending on input. 161 | func expandLine(t string, templdata interface{}) ([]string, error) { 162 | 163 | res := make([]string, 0) 164 | bb := &bytes.Buffer{} 165 | 166 | if tmpl, err := template.New("CommandInstance").Funcs(funcMap).Parse(t); err != nil { 167 | return nil, err 168 | } else if tmpl.Execute(bb, templdata); err != nil { 169 | return nil, err 170 | } else { 171 | lines := strings.Split(bb.String(), "\n") 172 | for _, l := range lines { 173 | if strings.TrimSpace(l) != "" { 174 | res = append(res, l) 175 | } 176 | } 177 | } 178 | 179 | return res, nil 180 | } 181 | 182 | // Expand template output lines into the actual expected output. This may be 183 | // called recursively if the output line references another command. The 184 | // 'processed' map takes care of checking for cycles so we don't get stuck. 185 | func expandOutput(id string, tmpl *CommandTemplate, td *templateData, 186 | processed map[string]bool, env *TestEnvironment) ([]*ExpectedOutputLine, error) { 187 | 188 | var ok bool 189 | 190 | // Check for cycles 191 | if _, ok = processed[id]; ok { 192 | return nil, errors.New("Cycle detected in command template output. ID: " + id) 193 | } 194 | processed[id] = true 195 | 196 | // expected output 197 | expected := make([]*ExpectedOutputLine, 0) 198 | 199 | // Expand each expected output line, possibly referencing external commands 200 | for _, origline := range tmpl.Output { 201 | if origline.External == "true" { 202 | copy := make(map[string]bool) 203 | for key, _ := range processed { 204 | copy[key] = true 205 | } 206 | if otherTmpl, ok := env.Commands[origline.Text]; ok { 207 | if more, err := expandOutput(origline.Text, otherTmpl, td, copy, env); err != nil { 208 | return nil, err 209 | } else { 210 | expected = append(expected, more...) 211 | } 212 | } 213 | } else { 214 | if lines, err := expandLine(origline.Text, td); err != nil { 215 | return nil, err 216 | } else { 217 | for _, expandedline := range lines { 218 | expectedline := &ExpectedOutputLine{ 219 | Text: expandedline, 220 | } 221 | if origline.Trusted == "true" { 222 | expectedline.Trusted = true 223 | expectedline.KeyName = id 224 | } else { 225 | expectedline.Trusted = false 226 | expectedline.KeyName = "" 227 | } 228 | expected = append(expected, expectedline) 229 | } 230 | } 231 | } 232 | } 233 | 234 | return expected, nil 235 | } 236 | 237 | func (c *Command) Id() string { 238 | _, id, _ := (&c.Input).splitCommand() 239 | return id 240 | } 241 | 242 | // Instantiate the command (input, expected output) using the command template. 243 | // This needs to be must be done prior to executing the command. 244 | func (c *Command) Instantiate(env *TestEnvironment) error { 245 | 246 | pfx, id, args := (&c.Input).splitCommand() 247 | tmpl, ok := env.Commands[id] 248 | if !ok { 249 | return fmt.Errorf("Command '%v' is not recognized by test161.\n", id) 250 | } 251 | 252 | // Individual tests can override the template in the commands files. 253 | // This merges the overrides on top of a copy of the command template. 254 | tmpl = tmpl.Clone() 255 | if err := mergo.Merge(tmpl, &c.Config, mergo.WithOverride); err != nil { 256 | return err 257 | } 258 | c.Panic = tmpl.Panic 259 | c.TimesOut = tmpl.TimesOut 260 | c.Timeout = tmpl.Timeout 261 | 262 | // Input 263 | 264 | // Check if we need to create some input. If args haven't already been 265 | // specified, and there is an input template, use that to create input. 266 | if len(args) == 0 && len(tmpl.Input) > 0 { 267 | args = make([]string, 0) 268 | for _, line := range tmpl.Input { 269 | if temp, err := expandLine(line, "No data"); err != nil { 270 | return err 271 | } else { 272 | args = append(args, temp...) 273 | } 274 | } 275 | } else if len(args) > 0 { 276 | // Expand the provided arguments as well 277 | argLine := "" 278 | for i, arg := range args { 279 | if i > 0 { 280 | argLine += " " 281 | } 282 | argLine += arg 283 | } 284 | if temp, err := expandLine(argLine, "No data"); err != nil { 285 | return err 286 | } else { 287 | // Break 288 | args = []string{} 289 | for _, arg := range temp { 290 | args = append(args, splitArgs(arg)...) 291 | } 292 | } 293 | } 294 | 295 | // Output 296 | 297 | // template data for the output 298 | td := &templateData{args, len(args)} 299 | processed := make(map[string]bool) 300 | 301 | if expected, err := expandOutput(id, tmpl, td, processed, env); err != nil { 302 | return err 303 | } else { 304 | // Piece back together a command line for the command 305 | commandLine := "" 306 | 307 | if len(pfx) > 0 { 308 | commandLine += pfx + " " 309 | } 310 | 311 | commandLine += id 312 | 313 | for _, arg := range args { 314 | commandLine += " " + arg 315 | } 316 | 317 | c.Input.Line = commandLine 318 | c.ExpectedOutput = expected 319 | 320 | return nil 321 | } 322 | 323 | } 324 | 325 | // CommandTemplate Collection. We just use this for loading and move the 326 | // references into a map in the global environment. 327 | type CommandTemplates struct { 328 | Templates []*CommandTemplate `yaml:"templates"` 329 | } 330 | 331 | func CommandTemplatesFromFile(file string) (*CommandTemplates, error) { 332 | data, err := ioutil.ReadFile(file) 333 | if err != nil { 334 | return nil, fmt.Errorf("Error reading commands file %v: %v", file, err) 335 | } 336 | 337 | c, err := CommandTemplatesFromString(string(data)) 338 | if err != nil { 339 | err = fmt.Errorf("Error loading commands file %v: %v", file, err) 340 | } 341 | return c, err 342 | } 343 | 344 | func CommandTemplatesFromString(text string) (*CommandTemplates, error) { 345 | cmds := &CommandTemplates{} 346 | err := yaml.Unmarshal([]byte(text), cmds) 347 | 348 | if err != nil { 349 | return nil, err 350 | } 351 | 352 | for _, t := range cmds.Templates { 353 | t.fixDefaults() 354 | } 355 | 356 | return cmds, nil 357 | } 358 | 359 | func (t *CommandTemplate) fixDefaults() { 360 | 361 | // Fix poor default value support in go-yaml/golang. 362 | // 363 | // If we find an empty output line, delete it - these are commands 364 | // that do not expect output. If we find a command with no expected 365 | // output, add the default expected output. 366 | 367 | // Default for panic is to not allow it, i.e. must return to prompt 368 | if t.Panic != CMD_OPT_MAYBE && t.Panic != CMD_OPT_YES { 369 | t.Panic = CMD_OPT_NO 370 | } 371 | 372 | if t.TimesOut != CMD_OPT_MAYBE && t.TimesOut != CMD_OPT_YES { 373 | t.TimesOut = CMD_OPT_NO 374 | } 375 | 376 | if len(t.Output) == 1 && strings.TrimSpace(t.Output[0].Text) == "" { 377 | t.Output = nil 378 | } else if len(t.Output) == 0 { 379 | t.Output = []*TemplOutputLine{ 380 | &TemplOutputLine{ 381 | Trusted: "true", 382 | External: "false", 383 | Text: t.Name + ": SUCCESS", 384 | }, 385 | } 386 | } else { 387 | for _, line := range t.Output { 388 | if line.Trusted != "false" { 389 | line.Trusted = "true" 390 | } 391 | 392 | if line.External != "true" { 393 | line.External = "false" 394 | } 395 | } 396 | } 397 | } 398 | 399 | // Seed random for the random input templates and sys161 400 | func init() { 401 | rand.Seed(time.Now().UnixNano()) 402 | } 403 | -------------------------------------------------------------------------------- /commands_test.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stretchr/testify/assert" 6 | "strings" 7 | "testing" 8 | ) 9 | 10 | // A random string input generator. This generates between 2 and 10 11 | // strings, each of length between 5 and 10 characters. 12 | const input_template = "{{$x := randInt 2 10 | ranger}}{{range $index, $element := $x}}{{randString 5 10}}\n{{end}}" 13 | 14 | func TestCommandArgTest(t *testing.T) { 15 | t.Parallel() 16 | assert := assert.New(t) 17 | 18 | // Create a test 19 | args := []string{"arg1", "arg2", "arg3", "arg4"} 20 | cmdline := "$ /testbin/argtest" 21 | for _, a := range args { 22 | cmdline += " " + a 23 | } 24 | test, err := TestFromString(cmdline) 25 | assert.Nil(err) 26 | 27 | // Set the commands for argtest 28 | var argtest *Command 29 | for _, c := range test.Commands { 30 | if c.Id() == "/testbin/argtest" { 31 | argtest = c 32 | break 33 | } 34 | } 35 | 36 | assert.NotNil(argtest) 37 | if argtest == nil { 38 | t.FailNow() 39 | } 40 | 41 | // Create the command instance 42 | err = argtest.Instantiate(defaultEnv) 43 | 44 | if err != nil { 45 | t.Log(err) 46 | t.FailNow() 47 | } 48 | 49 | for _, o := range argtest.ExpectedOutput { 50 | t.Log(o.Text) 51 | } 52 | 53 | // Assertions 54 | assert.Equal(3+len(args), len(argtest.ExpectedOutput)) 55 | 56 | if len(argtest.ExpectedOutput) == 7 { 57 | assert.Equal("argc: 5", argtest.ExpectedOutput[0].Text) 58 | assert.Equal("argv[0]: /testbin/argtest", argtest.ExpectedOutput[1].Text) 59 | for i, arg := range args { 60 | assert.Equal(fmt.Sprintf("argv[%d]: %v", i+1, arg), argtest.ExpectedOutput[i+2].Text) 61 | } 62 | } 63 | 64 | } 65 | 66 | func TestCommandAdd(t *testing.T) { 67 | t.Parallel() 68 | assert := assert.New(t) 69 | 70 | // Create a test 71 | test, err := TestFromString("$ /testbin/add 70 200") 72 | assert.Nil(err) 73 | 74 | // Set the commands for argtest 75 | var add *Command 76 | for _, c := range test.Commands { 77 | if c.Id() == "/testbin/add" { 78 | add = c 79 | break 80 | } 81 | } 82 | 83 | assert.NotNil(add) 84 | if add == nil { 85 | t.FailNow() 86 | } 87 | 88 | // Create the command instance 89 | err = add.Instantiate(defaultEnv) 90 | 91 | if err != nil { 92 | t.Log(err) 93 | t.FailNow() 94 | } 95 | 96 | for _, o := range add.ExpectedOutput { 97 | t.Log(o.Text) 98 | } 99 | 100 | // Assertions 101 | assert.Equal(1, len(add.ExpectedOutput)) 102 | if len(add.ExpectedOutput) == 1 { 103 | assert.Equal("270", add.ExpectedOutput[0].Text) 104 | } 105 | } 106 | 107 | func TestCommandFactorial(t *testing.T) { 108 | t.Parallel() 109 | assert := assert.New(t) 110 | 111 | // Create a test 112 | test, err := TestFromString("$ /testbin/factorial 8") 113 | assert.Nil(err) 114 | 115 | // Set the commands for argtest 116 | var factorial *Command 117 | for _, c := range test.Commands { 118 | if c.Id() == "/testbin/factorial" { 119 | factorial = c 120 | break 121 | } 122 | } 123 | 124 | assert.NotNil(factorial) 125 | if factorial == nil { 126 | t.FailNow() 127 | } 128 | 129 | // Create the command instance 130 | err = factorial.Instantiate(defaultEnv) 131 | 132 | if err != nil { 133 | t.Log(err) 134 | t.FailNow() 135 | } 136 | 137 | for _, o := range factorial.ExpectedOutput { 138 | t.Log(o.Text) 139 | } 140 | 141 | // Assertions 142 | assert.Equal(1, len(factorial.ExpectedOutput)) 143 | if len(factorial.ExpectedOutput) == 1 { 144 | assert.Equal("40320", factorial.ExpectedOutput[0].Text) 145 | } 146 | } 147 | 148 | func addInputTest() (*TestEnvironment, error) { 149 | 150 | env, err := NewEnvironment("./fixtures", &DoNothingPersistence{}) 151 | if err != nil { 152 | return nil, err 153 | } 154 | env.TestDir = "./fixtures/tests/nocycle/" 155 | 156 | // Create the Command Template for (fake) randinput. 157 | c := &CommandTemplate{ 158 | Name: "randinput", 159 | Input: []string{input_template}, 160 | } 161 | 162 | env.Commands["randinput"] = c 163 | return env, nil 164 | } 165 | 166 | func TestCommandInput(t *testing.T) { 167 | t.Parallel() 168 | assert := assert.New(t) 169 | 170 | env, err := addInputTest() 171 | 172 | assert.Nil(err) 173 | if err != nil { 174 | t.Log(err) 175 | t.FailNow() 176 | } 177 | 178 | // Create a test 179 | test, err := TestFromString("randinput") 180 | assert.Nil(err) 181 | 182 | var randinput *Command 183 | for _, c := range test.Commands { 184 | if c.Id() == "randinput" { 185 | randinput = c 186 | break 187 | } 188 | } 189 | 190 | assert.NotNil(randinput) 191 | if randinput == nil { 192 | t.FailNow() 193 | } 194 | 195 | // Create the command instance 196 | err = randinput.Instantiate(env) 197 | 198 | t.Log(randinput.Input.Line) 199 | 200 | for _, o := range randinput.ExpectedOutput { 201 | t.Log(o.Text) 202 | } 203 | 204 | _, id, args := randinput.Input.splitCommand() 205 | 206 | t.Log(args) 207 | t.Log(id) 208 | 209 | // Assertions 210 | assert.True(len(args) >= 2) 211 | assert.True(len(args) <= 10) 212 | 213 | for _, o := range args { 214 | assert.True(len(o) >= 5) 215 | assert.True(len(o) <= 10) 216 | } 217 | 218 | // Now, check override 219 | randinput.Input.Line = "randinput 1" 220 | randinput.ExpectedOutput = nil 221 | 222 | randinput.Instantiate(defaultEnv) 223 | 224 | _, id, args = randinput.Input.splitCommand() 225 | 226 | assert.Equal(0, len(randinput.ExpectedOutput)) 227 | assert.Equal(1, len(args)) 228 | if len(args) == 1 { 229 | assert.Equal("1", args[0]) 230 | } 231 | assert.Equal("randinput", id) 232 | assert.Equal("randinput 1", randinput.Input.Line) 233 | } 234 | 235 | func TestCommandTemplateLoad(t *testing.T) { 236 | t.Parallel() 237 | assert := assert.New(t) 238 | 239 | text := `--- 240 | templates: 241 | - name: sy1 242 | - name: sy2 243 | - name: sy3 244 | - name: sy4 245 | - name: sy5 246 | ` 247 | cmds, err := CommandTemplatesFromString(text) 248 | if err != nil { 249 | t.Log(err) 250 | t.FailNow() 251 | } 252 | 253 | assert.Equal(5, len(cmds.Templates)) 254 | if len(cmds.Templates) == 5 { 255 | for i, tmpl := range cmds.Templates { 256 | assert.Equal(fmt.Sprintf("sy%v", i+1), tmpl.Name) 257 | assert.Equal(1, len(tmpl.Output)) 258 | if len(tmpl.Output) == 1 { 259 | assert.Equal(tmpl.Name+": SUCCESS", tmpl.Output[0].Text) 260 | assert.Equal("true", tmpl.Output[0].Trusted) 261 | assert.Equal("false", tmpl.Output[0].External) 262 | } 263 | } 264 | } 265 | } 266 | 267 | func addExternalCmd() (*TestEnvironment, error) { 268 | env, err := NewEnvironment("./fixtures", &DoNothingPersistence{}) 269 | if err != nil { 270 | return nil, err 271 | } 272 | env.TestDir = "./fixtures/tests/nocycle/" 273 | 274 | // Create the Command Template for (fake) randinput. 275 | c := &CommandTemplate{ 276 | Name: "external", 277 | Output: []*TemplOutputLine{ 278 | &TemplOutputLine{ 279 | Text: "sem1", 280 | Trusted: "true", 281 | External: "true", 282 | }, 283 | &TemplOutputLine{ 284 | Text: "lt1", 285 | Trusted: "true", 286 | External: "true", 287 | }, 288 | }, 289 | } 290 | 291 | env.Commands["external"] = c 292 | return env, nil 293 | } 294 | 295 | func TestCommandExternal(t *testing.T) { 296 | t.Parallel() 297 | assert := assert.New(t) 298 | 299 | env, err := addExternalCmd() 300 | assert.Nil(err) 301 | if err != nil { 302 | t.Log(err) 303 | t.FailNow() 304 | } 305 | 306 | test, err := TestFromString("external") 307 | assert.Nil(err) 308 | 309 | var cmd *Command 310 | for _, c := range test.Commands { 311 | if c.Input.Line == "external" { 312 | cmd = c 313 | break 314 | } 315 | } 316 | 317 | assert.NotNil(cmd) 318 | if cmd == nil { 319 | t.FailNow() 320 | } 321 | 322 | // Create the command instance 323 | err = cmd.Instantiate(env) 324 | 325 | for _, o := range cmd.ExpectedOutput { 326 | t.Log(o.Text) 327 | } 328 | 329 | t.Log(cmd.ExpectedOutput) 330 | 331 | // Assertions 332 | assert.Equal(2, len(cmd.ExpectedOutput)) 333 | if len(cmd.ExpectedOutput) != 2 { 334 | t.FailNow() 335 | } 336 | 337 | assert.Equal("sem1: SUCCESS", cmd.ExpectedOutput[0].Text) 338 | assert.True(cmd.ExpectedOutput[0].Trusted) 339 | assert.Equal("sem1", cmd.ExpectedOutput[0].KeyName) 340 | 341 | assert.Equal("lt1: SUCCESS", cmd.ExpectedOutput[1].Text) 342 | assert.True(cmd.ExpectedOutput[1].Trusted) 343 | assert.Equal("lt1", cmd.ExpectedOutput[1].KeyName) 344 | } 345 | 346 | func TestCommandID(t *testing.T) { 347 | 348 | t.Parallel() 349 | assert := assert.New(t) 350 | 351 | tests := [][]string{ 352 | []string{ 353 | "/hello/world", "/hello/world", 354 | }, 355 | []string{ 356 | "/hello/world ", "/hello/world", 357 | }, 358 | []string{ 359 | `/testbin/argtest 1 2 3`, "/testbin/argtest", 360 | }, 361 | []string{ 362 | `/bin/space test`, "/bin/space", 363 | }, 364 | []string{ 365 | `/bin/space\ test`, `/bin/space\ test`, 366 | }, 367 | []string{ 368 | `"/bin/space test" 1 2 3`, `"/bin/space test"`, 369 | }, 370 | []string{ 371 | `p /bin/something 1 2 3`, `/bin/something`, 372 | }, 373 | []string{ 374 | `p s 1 2 3`, `s`, 375 | }, 376 | } 377 | 378 | for _, test := range tests { 379 | line := &InputLine{Line: test[0]} 380 | pfx, base, args := line.splitCommand() 381 | assert.Equal(test[1], base) 382 | if strings.HasPrefix(test[0], "p ") { 383 | assert.Equal("p", pfx) 384 | } 385 | t.Log(base, test[1]) 386 | t.Log(args) 387 | } 388 | 389 | } 390 | 391 | func TestCommandReplaceArgs(t *testing.T) { 392 | t.Parallel() 393 | assert := assert.New(t) 394 | 395 | type replaceTest struct { 396 | original string 397 | newArgs []string 398 | expected string 399 | } 400 | 401 | tests := []*replaceTest{ 402 | &replaceTest{ 403 | "/hello/world", []string{"arg1"}, "/hello/world arg1", 404 | }, 405 | &replaceTest{ 406 | "/foo/bar *", []string{"-r", "25"}, "/foo/bar -r 25", 407 | }, 408 | &replaceTest{ 409 | "/foo/bar -t 100", []string{}, "/foo/bar", 410 | }, 411 | &replaceTest{ 412 | "p /hello/world", []string{"arg1"}, "p /hello/world arg1", 413 | }, 414 | &replaceTest{ 415 | "p /foo/bar *", []string{"-r", "25"}, "p /foo/bar -r 25", 416 | }, 417 | &replaceTest{ 418 | "p /foo/bar -t 100", []string{}, "p /foo/bar", 419 | }, 420 | } 421 | 422 | for _, test := range tests { 423 | line := &InputLine{Line: test.original} 424 | line.replaceArgs(test.newArgs) 425 | assert.Equal(line.Line, test.expected) 426 | t.Log(test.original, test.newArgs, test.expected) 427 | } 428 | } 429 | 430 | func TestCommandOverride(t *testing.T) { 431 | t.Parallel() 432 | assert := assert.New(t) 433 | 434 | testString := `--- 435 | commandoverrides: 436 | - name: sem1 437 | output: 438 | - text: "Override SUCCESS" 439 | --- 440 | sem1 441 | ` 442 | test, err := TestFromString(testString) 443 | assert.Nil(err) 444 | 445 | // find the sem1 command 446 | var cmd *Command 447 | for _, c := range test.Commands { 448 | if c.Id() == "sem1" { 449 | cmd = c 450 | break 451 | } 452 | } 453 | 454 | err = cmd.Instantiate(defaultEnv) 455 | assert.Nil(err) 456 | 457 | if cmd == nil { 458 | t.Fatalf("cmd == nil)") 459 | } 460 | 461 | if len(cmd.ExpectedOutput) != 1 { 462 | t.Fatalf("Command output != 1") 463 | } 464 | 465 | assert.Equal("Override SUCCESS", cmd.ExpectedOutput[0].Text) 466 | 467 | // These are overrides only, so if these aren't specified, they get the 468 | // default values. 469 | assert.Equal(false, cmd.ExpectedOutput[0].Trusted) 470 | assert.Equal("", cmd.ExpectedOutput[0].KeyName) 471 | 472 | testString = `--- 473 | commandoverrides: 474 | - name: sem1 475 | timeout: 1000.0 476 | --- 477 | sem1 478 | ` 479 | 480 | test, err = TestFromString(testString) 481 | assert.Nil(err) 482 | 483 | // find the sem1 command 484 | cmd = nil 485 | for _, c := range test.Commands { 486 | if c.Id() == "sem1" { 487 | cmd = c 488 | break 489 | } 490 | } 491 | 492 | err = cmd.Instantiate(defaultEnv) 493 | assert.Nil(err) 494 | 495 | if cmd == nil { 496 | t.Fatalf("cmd == nil)") 497 | } 498 | 499 | if len(cmd.ExpectedOutput) != 1 { 500 | t.Fatalf("Command output != 1") 501 | } 502 | 503 | assert.Equal("sem1: SUCCESS", cmd.ExpectedOutput[0].Text) 504 | assert.Equal(true, cmd.ExpectedOutput[0].Trusted) 505 | assert.Equal("sem1", cmd.ExpectedOutput[0].KeyName) 506 | 507 | assert.Equal(float32(1000.0), cmd.Timeout) 508 | 509 | } 510 | -------------------------------------------------------------------------------- /environment.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | "path" 9 | "path/filepath" 10 | "strings" 11 | ) 12 | 13 | // TestEnvironment encapsultes the environment tests runs in. Much of the 14 | // environment is global - commands, targets, etc. However, some state 15 | // is local, such as the secure keyMap and OS/161 root directory. 16 | type TestEnvironment struct { 17 | // These do not depend on the TestGroup/Target 18 | TestDir string 19 | Commands map[string]*CommandTemplate 20 | Targets map[string]*Target 21 | 22 | // Optional - added in version 1.2.6 23 | Tags map[string]*TagDescription 24 | 25 | manager *manager 26 | 27 | CacheDir string 28 | OverlayRoot string 29 | KeyDir string 30 | Persistence PersistenceManager 31 | 32 | Log *log.Logger 33 | 34 | // These depend on the TestGroup/Target 35 | keyMap map[string]string 36 | RootDir string 37 | } 38 | 39 | // Create a new TestEnvironment by copying the global state from an existing 40 | // environment. Local test state will be initialized to default values. 41 | func (env *TestEnvironment) CopyEnvironment() *TestEnvironment { 42 | 43 | // Global 44 | copy := *env 45 | 46 | // Local 47 | copy.keyMap = make(map[string]string) 48 | copy.RootDir = "" 49 | 50 | return © 51 | } 52 | 53 | // Handle a single commands file (.tc) and load it into the TestEnvironment. 54 | func envCommandHandler(env *TestEnvironment, f string) error { 55 | if templates, err := CommandTemplatesFromFile(f); err != nil { 56 | return err 57 | } else { 58 | // If we already know about the command, it's an error 59 | for _, templ := range templates.Templates { 60 | if _, ok := env.Commands[templ.Name]; ok { 61 | return fmt.Errorf("Duplicate command (%v) in file %v", templ.Name, f) 62 | } 63 | env.Commands[templ.Name] = templ 64 | } 65 | return nil 66 | } 67 | } 68 | 69 | // Handle a single targets file (.tt) and load it into the TestEnvironment. 70 | func envTargetHandler(env *TestEnvironment, f string) error { 71 | if t, err := TargetFromFile(f); err != nil { 72 | return err 73 | } else { 74 | // Only track the most recent version, and only track active targets. 75 | if t.Active == "true" { 76 | prev, ok := env.Targets[t.Name] 77 | if !ok || t.Version > prev.Version { 78 | env.Targets[t.Name] = t 79 | } 80 | } 81 | 82 | if env.Persistence != nil { 83 | return env.Persistence.Notify(t, MSG_TARGET_LOAD, 0) 84 | } else { 85 | return nil 86 | } 87 | } 88 | } 89 | 90 | // Handle a single tag description file (.td) and load it into the 91 | // TestEnvironment. 92 | func envTagDescHandler(env *TestEnvironment, f string) error { 93 | if tags, err := TagDescriptionsFromFile(f); err != nil { 94 | return err 95 | } else { 96 | // If we already know about the tag, it's an error 97 | for _, tag := range tags.Tags { 98 | if _, ok := env.Tags[tag.Name]; ok { 99 | return fmt.Errorf("Duplicate tag (%v) in file %v", tag.Name, f) 100 | } 101 | env.Tags[tag.Name] = tag 102 | } 103 | return nil 104 | } 105 | } 106 | 107 | // envReadLoop searches a directory for files with a certain extention. When it 108 | // finds one, it calls handler(). 109 | func (env *TestEnvironment) envReadLoop(searchDir, ext string, 110 | handler func(env *TestEnvironment, f string) error) error { 111 | 112 | dir, err := ioutil.ReadDir(searchDir) 113 | if err != nil { 114 | return err 115 | } 116 | 117 | for _, f := range dir { 118 | if f.Mode().IsRegular() { 119 | if strings.HasSuffix(f.Name(), ext) { 120 | if err := handler(env, filepath.Join(searchDir, f.Name())); err != nil { 121 | return err 122 | } 123 | } 124 | } 125 | } 126 | return nil 127 | } 128 | 129 | // Create a new TestEnvironment from the given test161 directory. The directory 130 | // must contain these subdirectories: commands/ targets/ tests/ 131 | // In addition to loading tests, commands, and targets, a logger is set up that 132 | // writes to os.Stderr. This can be changed by changing env.Log. 133 | func NewEnvironment(test161Dir string, pm PersistenceManager) (*TestEnvironment, error) { 134 | 135 | cmdDir := path.Join(test161Dir, "commands") 136 | testDir := path.Join(test161Dir, "tests") 137 | targetDir := path.Join(test161Dir, "targets") 138 | tagDir := path.Join(test161Dir, "tags") 139 | 140 | env := &TestEnvironment{ 141 | TestDir: testDir, 142 | manager: testManager, 143 | Commands: make(map[string]*CommandTemplate), 144 | Targets: make(map[string]*Target), 145 | Tags: make(map[string]*TagDescription), 146 | keyMap: make(map[string]string), 147 | Log: log.New(os.Stderr, "test161: ", log.Ldate|log.Ltime|log.Lshortfile), 148 | Persistence: pm, 149 | } 150 | 151 | resChan := make(chan error) 152 | 153 | go func() { 154 | resChan <- env.envReadLoop(targetDir, ".tt", envTargetHandler) 155 | }() 156 | 157 | go func() { 158 | resChan <- env.envReadLoop(cmdDir, ".tc", envCommandHandler) 159 | }() 160 | 161 | // Tags are optional 162 | numExpected := 2 163 | if _, err := os.Stat(tagDir); err == nil { 164 | numExpected += 1 165 | go func() { 166 | resChan <- env.envReadLoop(tagDir, ".td", envTagDescHandler) 167 | }() 168 | } 169 | 170 | // Get the results 171 | var err error = nil 172 | for i := 0; i < numExpected; i++ { 173 | // Let the other finish, but just return one error 174 | temp := <-resChan 175 | if err == nil { 176 | err = temp 177 | } 178 | } 179 | 180 | if err == nil { 181 | err = env.linkMetaTargets() 182 | } 183 | 184 | return env, err 185 | } 186 | 187 | func (env *TestEnvironment) linkMetaTargets() error { 188 | // First, if the target is a subtarget, link it to its metatarget 189 | // and sibling subtargets. 190 | for _, target := range env.Targets { 191 | if len(target.MetaName) > 0 { 192 | if err := target.initAsSubTarget(env); err != nil { 193 | return err 194 | } 195 | } 196 | } 197 | 198 | // Next, validate the metatargets 199 | for _, target := range env.Targets { 200 | if target.IsMetaTarget { 201 | if err := target.initAsMetaTarget(env); err != nil { 202 | return err 203 | } 204 | } 205 | } 206 | 207 | return nil 208 | } 209 | 210 | func (env *TestEnvironment) TargetList() *TargetList { 211 | list := &TargetList{} 212 | list.Targets = make([]*TargetListItem, 0, len(env.Targets)) 213 | 214 | for _, t := range env.Targets { 215 | list.Targets = append(list.Targets, &TargetListItem{ 216 | Name: t.Name, 217 | Version: t.Version, 218 | PrintName: t.PrintName, 219 | Description: t.Description, 220 | Active: t.Active, 221 | Points: t.Points, 222 | Type: t.Type, 223 | FileName: t.FileName, 224 | FileHash: t.FileHash, 225 | CollabMsg: collabMsgs[t.Name], 226 | }) 227 | } 228 | return list 229 | } 230 | 231 | // Helper function for logging persistence errors 232 | func (env *TestEnvironment) notifyAndLogErr(desc string, entity interface{}, msg, what int) { 233 | if env.Persistence != nil { 234 | err := env.Persistence.Notify(entity, msg, what) 235 | if err != nil { 236 | if env.Log != nil { 237 | env.Log.Printf("(%v) Error writing data: %v\n", desc, err) 238 | } 239 | } 240 | } 241 | } 242 | 243 | func (env *TestEnvironment) SetNullLogger() { 244 | env.Log.SetFlags(0) 245 | env.Log.SetOutput(ioutil.Discard) 246 | } 247 | -------------------------------------------------------------------------------- /expect/expect.go: -------------------------------------------------------------------------------- 1 | package expect 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "os" 7 | "os/exec" 8 | "regexp" 9 | "sync" 10 | "syscall" 11 | "time" 12 | 13 | "github.com/kr/pty" 14 | ) 15 | 16 | // Expect is a program interaction session. 17 | type Expect struct { 18 | timeout time.Duration 19 | pty io.ReadWriteCloser 20 | Killer func() 21 | buffer []byte 22 | 23 | // channel for receiving read events 24 | readChan chan readEvent 25 | readStatus error 26 | 27 | logger Logger 28 | } 29 | 30 | // Match is returned from exp.Expect*() when a match is found. 31 | type Match struct { 32 | Before string 33 | Groups []string 34 | } 35 | 36 | type readEvent struct { 37 | buf []byte 38 | status error 39 | } 40 | 41 | // ErrTimeout is returned from exp.Expect*() when a timeout is reached. 42 | var ErrTimeout = errors.New("Expect Timeout") 43 | 44 | // READ_SIZE is the largest amount of data expect attempts to read in a single I/O operation. 45 | // This likely needs some research and tuning. 46 | const READ_SIZE = 4094 47 | 48 | // Create an Expect instance from a command. 49 | // Effectively the same as Create(pty.Start(exec.Command(name, args...))) 50 | func Spawn(name string, args ...string) (*Expect, error) { 51 | cmd := exec.Command(name, args...) 52 | pty, err := pty.Start(cmd) 53 | if err != nil { 54 | return nil, err 55 | } 56 | killer := func() { 57 | cmd.Process.Kill() 58 | } 59 | return Create(pty, killer, nil, 0), nil 60 | } 61 | 62 | // Create an Expect instance from something that we can do read/writes off of. 63 | // 64 | // Note: Close() must be called to cleanup this process. 65 | func Create(pty io.ReadWriteCloser, killer func(), logger Logger, timeout time.Duration) (exp *Expect) { 66 | if timeout == 0 { 67 | timeout = time.Hour * 24 * 356 68 | } 69 | rv := Expect{ 70 | timeout: timeout, 71 | pty: pty, 72 | readChan: make(chan readEvent), 73 | logger: logger, 74 | Killer: killer, 75 | } 76 | 77 | // Start up processes 78 | rv.startReader() 79 | 80 | // Done 81 | return &rv 82 | } 83 | 84 | // Timeout() returns amount of time an Expect() call will wait for the output to appear. 85 | func (exp *Expect) Timeout() time.Duration { 86 | return exp.timeout 87 | } 88 | 89 | // SetTimeout(Duration) sets the amount of time an Expect() call will wait for the output to appear. 90 | func (exp *Expect) SetTimeout(d time.Duration) { 91 | exp.timeout = d 92 | } 93 | 94 | // Return the current buffer. 95 | // 96 | // Note: This is not all data received off the network, but data that has been received for processing. 97 | func (exp *Expect) Buffer() []byte { 98 | return exp.buffer 99 | } 100 | 101 | // Kill & close off process. 102 | // 103 | // Note: This *must* be run to cleanup the process 104 | func (exp *Expect) Close() error { 105 | exp.Killer() 106 | err := exp.pty.Close() 107 | for readEvent := range exp.readChan { 108 | exp.mergeRead(readEvent) 109 | } 110 | return err 111 | } 112 | 113 | // Send data to program 114 | func (exp *Expect) Send(s string) error { 115 | return exp.send([]byte(s), false) 116 | } 117 | 118 | // Send data, but mark it as masked to observers. Use this for passwords 119 | func (exp *Expect) SendMasked(s string) error { 120 | return exp.send([]byte(s), true) 121 | } 122 | 123 | // Send several lines data (separated by \n) to the process 124 | func (exp *Expect) SendLn(lines ...string) error { 125 | for _, l := range lines { 126 | if err := exp.Send(l + "\n"); err != nil { 127 | return err 128 | } 129 | } 130 | return nil 131 | } 132 | 133 | func (exp *Expect) send(arr []byte, masked bool) error { 134 | for len(arr) > 0 { 135 | if n, err := exp.pty.Write(arr); err == nil { 136 | if masked { 137 | exp.logger.SendMasked(time.Now(), arr[0:n]) 138 | } else { 139 | exp.logger.Send(time.Now(), arr[0:n]) 140 | } 141 | arr = arr[n:] 142 | } else { 143 | return err 144 | } 145 | } 146 | return nil 147 | } 148 | 149 | // ExpectRegexp searches the I/O read stream for a pattern within .Timeout() 150 | func (exp *Expect) ExpectRegexp(pat *regexp.Regexp) (Match, error) { 151 | exp.logger.ExpectCall(time.Now(), pat) 152 | 153 | // Read error happened. 154 | if exp.readStatus != nil { 155 | exp.logger.ExpectReturn(time.Now(), Match{}, exp.readStatus) 156 | return Match{}, exp.readStatus 157 | } 158 | 159 | // Calculate absolute timeout 160 | giveUpTime := time.Now().Add(exp.timeout) 161 | 162 | // Loop until we match or read some data 163 | for first := true; first || time.Now().Before(giveUpTime); first = false { 164 | // Read some data 165 | if !first { 166 | if !exp.readData(giveUpTime) { 167 | return Match{}, io.EOF 168 | } 169 | } 170 | 171 | // Check for match 172 | if m, found := exp.checkForMatch(pat); found { 173 | exp.logger.ExpectReturn(time.Now(), m, nil) 174 | return m, nil 175 | } 176 | 177 | // If no match, check for read error (likely io.EOF) 178 | if exp.readStatus != nil { 179 | exp.logger.ExpectReturn(time.Now(), Match{}, exp.readStatus) 180 | return Match{}, exp.readStatus 181 | } 182 | } 183 | 184 | // Time is up 185 | exp.logger.ExpectReturn(time.Now(), Match{}, ErrTimeout) 186 | return Match{}, ErrTimeout 187 | } 188 | 189 | func (exp *Expect) checkForMatch(pat *regexp.Regexp) (m Match, found bool) { 190 | 191 | matches := pat.FindSubmatchIndex(exp.buffer) 192 | if matches != nil { 193 | found = true 194 | groupCount := len(matches) / 2 195 | m.Groups = make([]string, groupCount) 196 | 197 | for i := 0; i < groupCount; i++ { 198 | start := matches[2*i] 199 | end := matches[2*i+1] 200 | if start >= 0 && end >= 0 { 201 | m.Groups[i] = string(exp.buffer[start:end]) 202 | } 203 | } 204 | m.Before = string(exp.buffer[0:matches[0]]) 205 | exp.buffer = exp.buffer[matches[1]:] 206 | } 207 | return 208 | } 209 | 210 | func (exp *Expect) readData(giveUpTime time.Time) bool { 211 | wait := giveUpTime.Sub(time.Now()) 212 | select { 213 | case read, ok := <-exp.readChan: 214 | if ok { 215 | exp.mergeRead(read) 216 | } else { 217 | return false 218 | } 219 | 220 | case <-time.After(wait): 221 | // Timeout & return 222 | } 223 | return true 224 | } 225 | 226 | func (exp *Expect) mergeRead(read readEvent) { 227 | exp.buffer = append(exp.buffer, read.buf...) 228 | exp.readStatus = read.status 229 | //exp.fixNewLines() 230 | 231 | if len(read.buf) > 0 { 232 | exp.logger.Recv(time.Now(), read.buf) 233 | } 234 | 235 | if read.status == io.EOF { 236 | exp.logger.RecvEOF(time.Now()) 237 | } 238 | } 239 | 240 | var newLineRegexp *regexp.Regexp 241 | var newLineOnce sync.Once 242 | 243 | // fixNewLines will change various newlines combinations to \n 244 | func (exp *Expect) fixNewLines() { 245 | newLineOnce.Do(func() { newLineRegexp = regexp.MustCompile("\r\n") }) 246 | 247 | // This code could probably be optimized 248 | exp.buffer = newLineRegexp.ReplaceAllLiteral(exp.buffer, []byte("\n")) 249 | } 250 | 251 | // Expect(s string) is equivalent to exp.ExpectRegexp(regexp.MustCompile(s)) 252 | func (exp *Expect) Expect(expr string) (m Match, err error) { 253 | return exp.ExpectRegexp(regexp.MustCompile(expr)) 254 | } 255 | 256 | // Wait for EOF 257 | func (exp *Expect) ExpectEOF() error { 258 | _, err := exp.Expect("$EOF") 259 | return err 260 | } 261 | 262 | func (exp *Expect) startReader() { 263 | bufferInput := make(chan readEvent) 264 | 265 | // Buffer shim 266 | go func() { 267 | queue := make([]readEvent, 0) 268 | done := false 269 | 270 | // Normal I/O loop 271 | for !done { 272 | var sendItem readEvent 273 | var sendChan chan readEvent = nil 274 | 275 | // Set up send operation if we have data to send 276 | if len(queue) > 0 { 277 | sendItem = queue[0] 278 | sendChan = exp.readChan 279 | } 280 | 281 | // I/O 282 | select { 283 | case sendChan <- sendItem: 284 | queue = queue[1:] 285 | case read, ok := <-bufferInput: 286 | if ok { 287 | queue = append(queue, read) 288 | } else { 289 | done = true 290 | } 291 | } 292 | } 293 | 294 | // Drain buffer 295 | for _, read := range queue { 296 | exp.readChan <- read 297 | } 298 | 299 | // Close output 300 | close(exp.readChan) 301 | }() 302 | 303 | // Reader process 304 | go func() { 305 | done := false 306 | for !done { 307 | buf := make([]byte, READ_SIZE) 308 | n, err := exp.pty.Read(buf) 309 | buf = buf[0:n] 310 | 311 | // OSX: Closed FD returns io.EOF 312 | // Linux: Closed FD returns syscall.EIO, translate to io.EOF 313 | pathErr, ok := err.(*os.PathError) 314 | if ok && pathErr.Err == syscall.EIO { 315 | err = io.EOF 316 | } 317 | 318 | exp.logger.RecvNet(time.Now(), buf) 319 | bufferInput <- readEvent{buf, err} 320 | 321 | if err != nil { 322 | done = true 323 | } 324 | } 325 | close(bufferInput) 326 | }() 327 | 328 | } 329 | -------------------------------------------------------------------------------- /expect/logger.go: -------------------------------------------------------------------------------- 1 | package expect 2 | 3 | import ( 4 | "regexp" 5 | "time" 6 | ) 7 | 8 | type Logger interface { 9 | 10 | // API user sent an item 11 | Send(time.Time, []byte) 12 | 13 | // API user sent a masked item. The masked data is included, but the API user is advised to 14 | // not log this data in production. 15 | SendMasked(time.Time, []byte) 16 | 17 | // Data is received by the same goroutine as the API user. 18 | Recv(time.Time, []byte) 19 | 20 | // Data is received off the network 21 | RecvNet(time.Time, []byte) 22 | 23 | // EOF has been reached. Time is when the EOF was received off the network 24 | RecvEOF(time.Time) 25 | 26 | // API user ran some form of Expect* call 27 | ExpectCall(time.Time, *regexp.Regexp) 28 | 29 | // API user got a return back from an Expect* call 30 | ExpectReturn(time.Time, Match, error) 31 | 32 | // Close the log file / this is the last item 33 | Close(time.Time) 34 | } 35 | -------------------------------------------------------------------------------- /fixtures/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | -------------------------------------------------------------------------------- /fixtures/README.adoc: -------------------------------------------------------------------------------- 1 | = Testing Fixtures 2 | 3 | Testing requires a solution kernel for 4 | `ops-class.org`[https://www.ops-class.org] ASST3. Please email 5 | shaseley@buffalo.edu`[mailto:shaseley@buffalo.edu]. 6 | -------------------------------------------------------------------------------- /fixtures/commands/commands.tc: -------------------------------------------------------------------------------- 1 | --- 2 | templates: 3 | - name: sem1 4 | - name: lt1 5 | - name: lt2 6 | panics: yes 7 | output: 8 | - text: "lt2: Should panic..." 9 | - name: lt3 10 | panics: yes 11 | output: 12 | - text: "lt3: Should panic..." 13 | - name: cvt1 14 | - name: cvt2 15 | - name: cvt3 16 | panics: yes 17 | output: 18 | - text: "cvt3: Should panic..." 19 | - name: cvt4 20 | panics: yes 21 | output: 22 | - text: "cvt4: Should panic..." 23 | - name: /testbin/argtest 24 | output: 25 | - text: "argc: {{add 1 .ArgLen}}" 26 | - text: "argv[0]: /testbin/argtest" 27 | - text: "{{range $index, $element := .Args}}argv[{{add $index 1}}]: {{$element}}\n{{end}}" 28 | - text: "argv[{{add 1 .ArgLen}}]: [NULL]" 29 | - name: /testbin/add 30 | output: 31 | - text: "{{$x:= index .Args 0 | atoi}}{{$y := index .Args 1 | atoi}}{{add $x $y}}\n" 32 | - name: /testbin/factorial 33 | output: 34 | - text: "{{$n:= index .Args 0 | atoi}}{{factorial $n}}\n" 35 | - name: /testbin/forktest 36 | -------------------------------------------------------------------------------- /fixtures/commands/misc.tc: -------------------------------------------------------------------------------- 1 | # These commands expect no output, but must not panic 2 | templates: 3 | - name: tt1 4 | output: 5 | - text: "" 6 | 7 | - name: tt2 8 | output: 9 | - text: "" 10 | 11 | - name: tt3 12 | output: 13 | - text: "" 14 | 15 | - name: khu 16 | output: 17 | - text: "" 18 | 19 | - name: q 20 | output: 21 | - text: "" 22 | 23 | - name: s 24 | output: 25 | - text: "" 26 | 27 | - name: boot 28 | output: 29 | - text: 30 | 31 | - name: exit 32 | output: 33 | - text: "" 34 | 35 | - name: panic 36 | output: 37 | - text: "" 38 | 39 | - name: /testbin/waiter 40 | output: 41 | - text: "" 42 | 43 | - name: ll1 44 | output: 45 | - text: "" 46 | 47 | - name: ll16 48 | output: 49 | - text: "" 50 | 51 | - name: dl 52 | output: 53 | - text: "" 54 | 55 | - name: /bin/true 56 | output: 57 | - text: "" 58 | 59 | - name: /bin/space 60 | output: 61 | - text: "" 62 | 63 | - name: /testbin/shll 64 | output: 65 | - text: "" 66 | 67 | - name: semu1 68 | output: 69 | - text: "" 70 | -------------------------------------------------------------------------------- /fixtures/keys/id_rsa.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDb9CX+IuNQNsMvdQ2AcULcPHeLTN5YzcqszY+98fljcAzTxlv4Gv8fMDcqubDUwTdOBu5uUubVyGnLgKRzhM4nvLm4JW9ArS9suWcs/TFl2Uyvd8ay8/NbEy3j2rAUKBLBYO2e0eeJBqHz8dX+VDCqCo6V/eJKvzcNbH37JfagC0TC4HhMUyTo6lBx0LHyPYygmgb7VS/lLl42WGm1NI6Z2zGEecRZBD7Qh2ohCie0Lw2RcgM3q8mWoxeY3+7r9a9oYSU6gekupTljkFR6zI0qjOyyIvIe6BFjwodA1Q7aLRXwNWjh+bQ8oiK6thZVxuyw2PvASWCVv+L+KV0N/Oep test161 2 | -------------------------------------------------------------------------------- /fixtures/keys/test@test161.ops-class.org/id_rsa: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEowIBAAKCAQEA2/Ql/iLjUDbDL3UNgHFC3Dx3i0zeWM3KrM2PvfH5Y3AM08Zb 3 | +Br/HzA3Krmw1ME3TgbublLm1chpy4Ckc4TOJ7y5uCVvQK0vbLlnLP0xZdlMr3fG 4 | svPzWxMt49qwFCgSwWDtntHniQah8/HV/lQwqgqOlf3iSr83DWx9+yX2oAtEwuB4 5 | TFMk6OpQcdCx8j2MoJoG+1Uv5S5eNlhptTSOmdsxhHnEWQQ+0IdqIQontC8NkXID 6 | N6vJlqMXmN/u6/WvaGElOoHpLqU5Y5BUesyNKozssiLyHugRY8KHQNUO2i0V8DVo 7 | 4fm0PKIiurYWVcbssNj7wElglb/i/ildDfznqQIDAQABAoIBAGqWux6Xq46Aqz/B 8 | OqTGvj7Z8piHzKw+NfxdtU24CEG+2Ah8dK8b7gwgImvLBr1jULi4NS0zcXeiIlqi 9 | 4Y61ie3J0DpDsdEm2/eVNUGhBSI1FqtloN2xyHmXJsLnhDKQZN5faWdwkwJdO3hq 10 | 9mfERrzwVr6rBNFyLmyL6dUeRZCyXw5Mv3z1mreavoMj6DwfprCAU/gltoY/jGe8 11 | 5fseLIy3dgBgz2IxC3ADH44OTpURKEal58pHuB7I+eOY5cxq5TBolOO7nyXT5I72 12 | 7f6Mr5Vy6XpcvSHuMLv1o0iYK+QWeFlJiO2O8UuHPyvOAQuITbTytZpLYzwSOLFX 13 | WDvNDD0CgYEA+BSAP4pL9EvnIkzwFtw91/CHuaexa+NNRIeXDvTuNIjleMCr9nSr 14 | KVMnVJDgOWPazArCnIjMWMievu78rGPEvL8e4Vn9JxOjRmfI/HH53UgbScA4QeLf 15 | 2BpHnaLCdwBlhKTtHwrr/zZxpicATfrsSdEfiFPTIN4hrTdUUxCum5sCgYEA4vnG 16 | /UfDNWAiXb5TXwJdJ7RslAPp7889yP4eWmeCUW8iDMI7zXb6E0v5jCacJV+bNBtK 17 | eC5v/XXEgAcJ+vdsYfYBtmzZ8+rDkoGM3SEGGN/JtVhqOp+WWlWwaNZ2L/ZT4oQF 18 | yz7y9G8YWvSS/mv68Q19YMd7xjbqxcbU9jtEKAsCgYEAq6V7qvskXI6cCOaVBCw0 19 | +hEpx7IYl155Wt46DZY4rs69f1RIZ0kIGJq5TtDC49KMU7tqNeaNBS0icVdoKlsJ 20 | h9LxKdkayIvU3+T1cn3l9U5r2xaNlkDEwoBEZvRzeuUiWKnIiz8CVN41ulGn60yf 21 | at+v4qKlJUusn232AVc8iNsCgYA7jGCCjtNOK5yYj5h78rjR8+oQoz465lpFYzY/ 22 | baypBMkgI81gyHgvm90qwe5xd7XWY9qT0UscaktVc4NQzp0mzk4AuGouLkeFJmv4 23 | j/NzjzLyWvHz02604IpZ1vpG9w9m/FAw1KEVNBhltIjkKxw5JdrhCzUT+dB6dwHk 24 | YAQvpQKBgB9gy9gJUNhtIu6eLX75T1x4biMxKyg587uttOu9y9GgfW0dELXLtsLl 25 | r5yC8Oo1Irezu2J3K9e5Uk/3O+LjmOsFzv4yjSja9XuQ9JV7eCl1zIglCxo6lTlf 26 | jb89ulQUyV6VcX4IOhHYlw4AFtpZskmTULS/GVYcOMHqVFfVEbJw 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /fixtures/overlays/simple/SECRET: -------------------------------------------------------------------------------- 1 | kern/test/synchtest.c 2 | kern/vm/kmalloc.c 3 | kern/test/rwtest.c 4 | kern/test/kmalloctest.c 5 | -------------------------------------------------------------------------------- /fixtures/overlays/simple/kern/include/kern/secret.h: -------------------------------------------------------------------------------- 1 | #ifndef _SECRET_H_ 2 | #define _SECRET_H_ 3 | 4 | /* Enable secret based testing. */ 5 | #define SECRET_TESTING 6 | 7 | /* Make sure we overwrite all secrets during injection. */ 8 | #undef SECRET 9 | 10 | #endif /* _SECRET_H_ */ 11 | -------------------------------------------------------------------------------- /fixtures/sys161/.gitignore: -------------------------------------------------------------------------------- 1 | /* 2 | -------------------------------------------------------------------------------- /fixtures/tags/all.td: -------------------------------------------------------------------------------- 1 | tags: 2 | - name: tag1 3 | desc: "This is desc1" 4 | - name: tag2 5 | desc: "This is desc2" 6 | - name: tag3 7 | desc: "This is desc3" 8 | - name: tag4 9 | desc: "This is desc4" 10 | - name: tag5 11 | desc: "This is desc5" 12 | -------------------------------------------------------------------------------- /fixtures/targets/asst1.tt: -------------------------------------------------------------------------------- 1 | --- 2 | name: asst1 3 | version: 1 4 | points: 50 5 | type: asst 6 | kconfig: ASST1 7 | overlay: asst1 8 | tests: 9 | - id: sync/lt1.t 10 | points: 10 11 | - id: sync/lt2.t 12 | points: 5 13 | - id: sync/lt3.t 14 | points: 5 15 | - id: sync/cvt1.t 16 | points: 10 17 | - id: sync/cvt2.t 18 | points: 10 19 | - id: sync/cvt3.t 20 | points: 5 21 | - id: sync/cvt4.t 22 | points: 5 23 | -------------------------------------------------------------------------------- /fixtures/targets/entire.tt: -------------------------------------------------------------------------------- 1 | --- 2 | name: entire 3 | points: 60 4 | type: asst 5 | kconfig: ASST1 6 | tests: 7 | - id: sync/sem1.t 8 | points: 10 9 | - id: sync/lt1.t 10 | points: 10 11 | - id: sync/fail.t 12 | points: 40 13 | -------------------------------------------------------------------------------- /fixtures/targets/full.tt: -------------------------------------------------------------------------------- 1 | --- 2 | name: full 3 | points: 60 4 | type: asst 5 | kconfig: ASST1 6 | tests: 7 | - id: sync/sem1.t 8 | points: 10 9 | - id: sync/lt1.t 10 | points: 10 11 | - id: sync/multi.t 12 | points: 40 13 | -------------------------------------------------------------------------------- /fixtures/targets/meta.1.tt: -------------------------------------------------------------------------------- 1 | --- 2 | name: meta.1 3 | version: 1 4 | points: 25 5 | type: asst 6 | kconfig: ASST1 7 | meta_name: metatest 8 | tests: 9 | - id: sync/sem1.t 10 | points: 25 11 | -------------------------------------------------------------------------------- /fixtures/targets/meta.2.tt: -------------------------------------------------------------------------------- 1 | --- 2 | name: meta.2 3 | version: 1 4 | points: 75 5 | type: asst 6 | kconfig: ASST1 7 | meta_name: metatest 8 | tests: 9 | - id: sync/lt1.t 10 | points: 75 11 | -------------------------------------------------------------------------------- /fixtures/targets/meta.tt: -------------------------------------------------------------------------------- 1 | --- 2 | name: metatest 3 | points: 100 4 | type: asst 5 | kconfig: ASST1 6 | is_meta_target: true 7 | sub_target_names: [meta.1, meta.2] 8 | -------------------------------------------------------------------------------- /fixtures/targets/partial.tt: -------------------------------------------------------------------------------- 1 | --- 2 | name: partial 3 | points: 60 4 | type: asst 5 | kconfig: ASST1 6 | tests: 7 | - id: sync/sem1.t 8 | points: 10 9 | - id: sync/lt1.t 10 | points: 10 11 | - id: sync/fail.t 12 | points: 40 13 | scoring: partial 14 | commands: 15 | - id: sem1 16 | points: 10 17 | - id: lt1 18 | points: 10 19 | - id: cvt1 20 | points: 20 21 | -------------------------------------------------------------------------------- /fixtures/targets/simple.tt: -------------------------------------------------------------------------------- 1 | --- 2 | name: simple 3 | version: 1 4 | points: 50 5 | type: asst 6 | kconfig: DUMBVM 7 | tests: 8 | - id: sync/sem1.t 9 | points: 50 10 | -------------------------------------------------------------------------------- /fixtures/tests/boot.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: Kernel Boot 3 | tags: 4 | - boot 5 | stat: 6 | resolution: 0.01 7 | window: 100 8 | misc: 9 | prompttimeout: 30.0 10 | --- 11 | q 12 | -------------------------------------------------------------------------------- /fixtures/tests/cycle/boot.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: Kernel Boot 3 | tags: 4 | - boot 5 | depends: 6 | - /sync/sy4.t 7 | stat: 8 | resolution: 0.01 9 | window: 100 10 | misc: 11 | prompttimeout: 30.0 12 | --- 13 | q 14 | -------------------------------------------------------------------------------- /fixtures/tests/cycle/sync/semu1.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: Semaphore Unit Test 1 3 | tags: 4 | - sync 5 | - sem 6 | depends: 7 | - ../threads/*.t 8 | stat: 9 | resolution: 0.01 10 | window: 100 11 | misc: 12 | prompttimeout: 30.0 13 | --- 14 | sy1 15 | -------------------------------------------------------------------------------- /fixtures/tests/cycle/sync/sy1.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: Semaphore Test 3 | tags: 4 | - sync 5 | - sem 6 | depends: 7 | - /**/tt*.t 8 | stat: 9 | resolution: 0.01 10 | window: 100 11 | misc: 12 | prompttimeout: 30.0 13 | --- 14 | sy1 15 | -------------------------------------------------------------------------------- /fixtures/tests/cycle/sync/sy2.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: Lock Test 3 | tags: 4 | - sync 5 | - locks 6 | depends: 7 | - threads 8 | stat: 9 | resolution: 0.01 10 | window: 100 11 | misc: 12 | prompttimeout: 30.0 13 | --- 14 | sy2 15 | -------------------------------------------------------------------------------- /fixtures/tests/cycle/sync/sy3.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: CV Test 3 | tags: 4 | - sync 5 | - cv 6 | depends: 7 | - threads 8 | - locks 9 | stat: 10 | resolution: 0.01 11 | window: 100 12 | misc: 13 | prompttimeout: 30.0 14 | --- 15 | sy3 16 | -------------------------------------------------------------------------------- /fixtures/tests/cycle/sync/sy4.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: CV Test 2 3 | tags: 4 | - sync 5 | - cv 6 | depends: 7 | - threads 8 | - locks 9 | - sy3.t 10 | stat: 11 | resolution: 0.01 12 | window: 100 13 | misc: 14 | prompttimeout: 30.0 15 | --- 16 | sy4 17 | -------------------------------------------------------------------------------- /fixtures/tests/cycle/sync/sy5.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: RW Lock Test 3 | tags: 4 | - sync 5 | - rwlock 6 | depends: 7 | - threads 8 | stat: 9 | resolution: 0.01 10 | window: 100 11 | misc: 12 | prompttimeout: 30.0 13 | --- 14 | sy5 15 | -------------------------------------------------------------------------------- /fixtures/tests/cycle/threads/tt1.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: Thread Test 1 3 | tags: 4 | - threads 5 | depends: 6 | - boot 7 | stat: 8 | resolution: 0.01 9 | window: 100 10 | misc: 11 | prompttimeout: 30.0 12 | --- 13 | tt1 14 | -------------------------------------------------------------------------------- /fixtures/tests/cycle/threads/tt2.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: Thread Test 2 3 | tags: 4 | - threads 5 | depends: 6 | - /*.t 7 | stat: 8 | resolution: 0.01 9 | window: 100 10 | misc: 11 | prompttimeout: 30.0 12 | --- 13 | tt2 14 | -------------------------------------------------------------------------------- /fixtures/tests/cycle/threads/tt3.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: Thread Test 3 3 | tags: 4 | - threads 5 | depends: 6 | - /boot.t 7 | stat: 8 | resolution: 0.01 9 | window: 100 10 | misc: 11 | prompttimeout: 30.0 12 | --- 13 | tt3 14 | -------------------------------------------------------------------------------- /fixtures/tests/nocycle/boot.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: Kernel Boot 3 | tags: 4 | - boot 5 | stat: 6 | resolution: 0.01 7 | window: 100 8 | misc: 9 | prompttimeout: 30.0 10 | --- 11 | q 12 | -------------------------------------------------------------------------------- /fixtures/tests/nocycle/panics/deppanic.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: Waiting for Panic! 3 | depends: 4 | - panic.t 5 | stat: 6 | resolution: 0.01 7 | window: 100 8 | misc: 9 | prompttimeout: 30.0 10 | --- 11 | q 12 | -------------------------------------------------------------------------------- /fixtures/tests/nocycle/panics/panic.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: Panic! 3 | depends: 4 | - boot 5 | stat: 6 | resolution: 0.01 7 | window: 100 8 | misc: 9 | prompttimeout: 30.0 10 | --- 11 | panic 12 | -------------------------------------------------------------------------------- /fixtures/tests/nocycle/sync/all.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: Test All Sync 3 | tags: 4 | - sync 5 | depends: 6 | - threads 7 | stat: 8 | resolution: 0.01 9 | window: 100 10 | misc: 11 | prompttimeout: 30.0 12 | --- 13 | sem1 14 | lt1 15 | cvt1 16 | cvt2 17 | -------------------------------------------------------------------------------- /fixtures/tests/nocycle/sync/cvt1.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: cv_test_1 3 | tags: 4 | - sync 5 | - cv 6 | depends: 7 | - threads 8 | - locks 9 | stat: 10 | resolution: 0.01 11 | window: 100 12 | misc: 13 | prompttimeout: 30.0 14 | --- 15 | cvt1 16 | -------------------------------------------------------------------------------- /fixtures/tests/nocycle/sync/cvt2.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: cv_test_2 3 | tags: 4 | - sync 5 | - cv 6 | depends: 7 | - threads 8 | - locks 9 | stat: 10 | resolution: 0.01 11 | window: 100 12 | misc: 13 | prompttimeout: 30.0 14 | --- 15 | cvt2 16 | -------------------------------------------------------------------------------- /fixtures/tests/nocycle/sync/cvt3.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: cv_test_3 3 | tags: 4 | - sync 5 | - cv 6 | depends: 7 | - threads 8 | - locks 9 | stat: 10 | resolution: 0.01 11 | window: 100 12 | misc: 13 | prompttimeout: 30.0 14 | --- 15 | cvt3 16 | -------------------------------------------------------------------------------- /fixtures/tests/nocycle/sync/cvt4.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: cv_test_4 3 | tags: 4 | - sync 5 | - cv 6 | depends: 7 | - threads 8 | - locks 9 | stat: 10 | resolution: 0.01 11 | window: 100 12 | misc: 13 | prompttimeout: 30.0 14 | --- 15 | cvt4 16 | -------------------------------------------------------------------------------- /fixtures/tests/nocycle/sync/fail.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: Test a failure 3 | tags: 4 | - sync 5 | depends: 6 | - threads 7 | stat: 8 | resolution: 0.01 9 | window: 100 10 | misc: 11 | prompttimeout: 30.0 12 | --- 13 | sem1 14 | panic 15 | lt1 16 | cvt1 17 | -------------------------------------------------------------------------------- /fixtures/tests/nocycle/sync/lt1.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: lock_test_1 3 | tags: 4 | - sync 5 | - locks 6 | depends: 7 | - threads 8 | stat: 9 | resolution: 0.01 10 | window: 100 11 | misc: 12 | prompttimeout: 30.0 13 | --- 14 | lt1 15 | -------------------------------------------------------------------------------- /fixtures/tests/nocycle/sync/lt2.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: lock_test_2 3 | tags: 4 | - sync 5 | - locks 6 | depends: 7 | - threads 8 | stat: 9 | resolution: 0.01 10 | window: 100 11 | misc: 12 | prompttimeout: 30.0 13 | --- 14 | lt2 15 | -------------------------------------------------------------------------------- /fixtures/tests/nocycle/sync/lt3.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: lock_test_3 3 | tags: 4 | - sync 5 | - locks 6 | depends: 7 | - threads 8 | stat: 9 | resolution: 0.01 10 | window: 100 11 | misc: 12 | prompttimeout: 30.0 13 | --- 14 | lt3 15 | -------------------------------------------------------------------------------- /fixtures/tests/nocycle/sync/multi.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: multi_test_1 3 | tags: 4 | - sync 5 | depends: 6 | - threads 7 | stat: 8 | resolution: 0.01 9 | window: 100 10 | misc: 11 | prompttimeout: 30.0 12 | --- 13 | sem1 14 | lt1 15 | -------------------------------------------------------------------------------- /fixtures/tests/nocycle/sync/sem1.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: semaphore_test_1 3 | tags: 4 | - sync 5 | - sem 6 | depends: 7 | - /**/tt*.t 8 | stat: 9 | resolution: 0.01 10 | window: 100 11 | misc: 12 | prompttimeout: 30.0 13 | --- 14 | sem1 15 | -------------------------------------------------------------------------------- /fixtures/tests/nocycle/sync/semu1.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: Semaphore Unit Test 1 3 | tags: 4 | - sync 5 | - sem 6 | depends: 7 | - ../threads/*.t 8 | stat: 9 | resolution: 0.01 10 | window: 100 11 | misc: 12 | prompttimeout: 30.0 13 | --- 14 | semu1 15 | -------------------------------------------------------------------------------- /fixtures/tests/nocycle/threads/tt1.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: Thread Test 1 3 | tags: 4 | - threads 5 | depends: 6 | - boot 7 | stat: 8 | resolution: 0.01 9 | window: 100 10 | misc: 11 | prompttimeout: 30.0 12 | --- 13 | tt1 14 | -------------------------------------------------------------------------------- /fixtures/tests/nocycle/threads/tt2.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: Thread Test 2 3 | tags: 4 | - threads 5 | depends: 6 | - /*.t 7 | stat: 8 | resolution: 0.01 9 | window: 100 10 | misc: 11 | prompttimeout: 30.0 12 | --- 13 | tt2 14 | -------------------------------------------------------------------------------- /fixtures/tests/nocycle/threads/tt3.t: -------------------------------------------------------------------------------- 1 | --- 2 | name: Thread Test 3 3 | tags: 4 | - threads 5 | depends: 6 | - /boot.t 7 | stat: 8 | resolution: 0.01 9 | window: 100 10 | misc: 11 | prompttimeout: 60.0 12 | --- 13 | tt3 14 | -------------------------------------------------------------------------------- /graph/graph.go: -------------------------------------------------------------------------------- 1 | package graph 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | // Our Graph type consists of only a map of Nodes, 8 | // indexed by strings. The graph's edge data is stored 9 | // in the Nodes themselves, which makes cycle detection 10 | // a bit easier. 11 | type Graph struct { 12 | NodeMap map[string]*Node 13 | } 14 | 15 | // A Node of a directed graph, with incoming and outgoing edges. 16 | type Node struct { 17 | Name string 18 | Value Keyer 19 | EdgesOut map[string]*Node 20 | EdgesIn map[string]*Node 21 | } 22 | 23 | type Keyer interface { 24 | Key() string 25 | } 26 | 27 | type StringNode string 28 | 29 | func (s StringNode) Key() string { 30 | return string(s) 31 | } 32 | 33 | // Add an incoming edge. This needs to be paired with addEdgeOut. 34 | func (n *Node) addEdgeIn(edgeNode *Node) { 35 | n.EdgesIn[edgeNode.Name] = edgeNode 36 | } 37 | 38 | // Add an outgoing edge. This needs to be paired with addEdgeIn. 39 | func (n *Node) addEdgeOut(edgeNode *Node) { 40 | n.EdgesOut[edgeNode.Name] = edgeNode 41 | } 42 | 43 | // Remove an incoming edge. This needs to be paired with removeEdgeOut. 44 | func (n *Node) removeEdgeIn(edgeNode *Node) { 45 | delete(n.EdgesIn, edgeNode.Name) 46 | } 47 | 48 | // Remove an outgoing edge. This needs to be paired with removeEdgeIn. 49 | func (n *Node) removeEdgeOut(edgeNode *Node) { 50 | delete(n.EdgesOut, edgeNode.Name) 51 | } 52 | 53 | // TopSort creates a topological sort of the Nodes of a Graph. 54 | // If there is a cycle, an error is returned, otherwise the 55 | // topological sort is returned as a list of node names. 56 | func (g *Graph) TopSort() ([]string, error) { 57 | sorted := make([]string, 0) 58 | copy := g.copy() 59 | 60 | // Initially, add all nodes without dependencies 61 | empty := make([]*Node, 0) 62 | for _, node := range copy.NodeMap { 63 | if len(node.EdgesIn) == 0 { 64 | empty = append(empty, node) 65 | } 66 | } 67 | 68 | for len(empty) > 0 { 69 | node := empty[0] 70 | sorted = append(sorted, node.Name) 71 | empty = empty[1:] 72 | for _, outgoing := range node.EdgesOut { 73 | // delete the edge from node -> outgoing 74 | outgoing.removeEdgeIn(node) 75 | if len(outgoing.EdgesIn) == 0 { 76 | empty = append(empty, outgoing) 77 | } 78 | } 79 | node.EdgesOut = nil 80 | } 81 | 82 | // if there are any edges left, we have a cycle 83 | for _, n := range copy.NodeMap { 84 | if len(n.EdgesIn) > 0 || len(n.EdgesOut) > 0 { 85 | return nil, errors.New("Cycle!") 86 | } 87 | } 88 | return sorted, nil 89 | } 90 | 91 | // Copy an existing graph into an independent structure 92 | // (i.e. new nodes/edges are created - pointers aren't copied) 93 | func (g *Graph) copy() *Graph { 94 | // Copy nodes 95 | nodes := make([]Keyer, 0, len(g.NodeMap)) 96 | for _, node := range g.NodeMap { 97 | nodes = append(nodes, node.Value) 98 | } 99 | other := New(nodes) 100 | 101 | // Copy edges 102 | for fromId, node := range g.NodeMap { 103 | for toId := range node.EdgesOut { 104 | other.addEdge(other.NodeMap[fromId], other.NodeMap[toId]) 105 | } 106 | } 107 | 108 | return other 109 | } 110 | 111 | // Create a new graph consisting of a set of nodes with no edges. 112 | func New(nodes []Keyer) *Graph { 113 | g := &Graph{} 114 | g.NodeMap = make(map[string]*Node) 115 | 116 | for _, s := range nodes { 117 | g.AddNode(s) 118 | } 119 | return g 120 | } 121 | 122 | func (g *Graph) addEdge(from *Node, to *Node) { 123 | from.addEdgeOut(to) 124 | to.addEdgeIn(from) 125 | } 126 | 127 | func (g *Graph) AddNode(node Keyer) { 128 | g.NodeMap[node.Key()] = &Node{node.Key(), node, make(map[string]*Node), make(map[string]*Node)} 129 | } 130 | 131 | // Add an edge from to 132 | func (g *Graph) AddEdge(from Keyer, to Keyer) error { 133 | if from == to { 134 | return errors.New("From node cannot be the same as To node") 135 | } 136 | 137 | fromNode, ok1 := g.NodeMap[from.Key()] 138 | toNode, ok2 := g.NodeMap[to.Key()] 139 | 140 | if !ok1 { 141 | return errors.New("from node not found") 142 | } else if !ok2 { 143 | return errors.New("to node not found") 144 | } else { 145 | g.addEdge(fromNode, toNode) 146 | return nil 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /graph/graph_test.go: -------------------------------------------------------------------------------- 1 | package graph 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestGraphCycles(t *testing.T) { 9 | t.Parallel() 10 | 11 | assert := assert.New(t) 12 | 13 | nodes := []Keyer{ 14 | StringNode("shell"), 15 | StringNode("boot"), 16 | StringNode("badcall2"), 17 | StringNode("randcall"), 18 | StringNode("badcall"), 19 | StringNode("shll"), 20 | StringNode("badcall3"), 21 | } 22 | 23 | graph := New(nodes) 24 | assert.Equal(len(nodes), len(graph.NodeMap)) 25 | if len(nodes) == len(graph.NodeMap) { 26 | for _, s := range nodes { 27 | var sn StringNode = s.(StringNode) 28 | key := string(sn) 29 | assert.NotNil(graph.NodeMap[key]) 30 | } 31 | } 32 | 33 | graph.AddEdge(StringNode("shell"), StringNode("boot")) 34 | graph.AddEdge(StringNode("badcall"), StringNode("shell")) 35 | graph.AddEdge(StringNode("randcall"), StringNode("shell")) 36 | graph.AddEdge(StringNode("badcall2"), StringNode("badcall")) 37 | graph.AddEdge(StringNode("shll"), StringNode("boot")) 38 | graph.AddEdge(StringNode("badcall3"), StringNode("badcall2")) 39 | 40 | sorted, err := graph.TopSort() 41 | assert.Nil(err) 42 | t.Log(sorted) 43 | 44 | graph.AddEdge(StringNode("shell"), StringNode("badcall3")) 45 | _, err = graph.TopSort() 46 | assert.NotNil(err) 47 | t.Log(err) 48 | } 49 | 50 | func TestGraphForest(t *testing.T) { 51 | t.Parallel() 52 | 53 | assert := assert.New(t) 54 | 55 | nodes := []Keyer{ 56 | StringNode("shell"), 57 | StringNode("boot"), 58 | StringNode("badcall2"), 59 | StringNode("randcall"), 60 | StringNode("badcall"), 61 | StringNode("shll"), 62 | StringNode("badcall3"), 63 | StringNode("boot2"), 64 | StringNode("shell2"), 65 | } 66 | 67 | graph := New(nodes) 68 | graph.AddEdge(StringNode("shell"), StringNode("boot")) 69 | graph.AddEdge(StringNode("badcall"), StringNode("shell")) 70 | graph.AddEdge(StringNode("randcall"), StringNode("shell")) 71 | graph.AddEdge(StringNode("badcall2"), StringNode("badcall")) 72 | graph.AddEdge(StringNode("shll"), StringNode("boot")) 73 | graph.AddEdge(StringNode("badcall3"), StringNode("badcall2")) 74 | graph.AddEdge(StringNode("shell2"), StringNode("boot2")) 75 | 76 | sorted, err := graph.TopSort() 77 | assert.Nil(err) 78 | t.Log(sorted) 79 | 80 | graph.AddEdge(StringNode("boot2"), StringNode("shell2")) 81 | _, err = graph.TopSort() 82 | assert.NotNil(err) 83 | t.Log(err) 84 | } 85 | -------------------------------------------------------------------------------- /groups_test.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | import ( 4 | //"github.com/ops-class/test161/graph" 5 | "github.com/stretchr/testify/assert" 6 | "path/filepath" 7 | "sort" 8 | "testing" 9 | ) 10 | 11 | const TEST_DIR string = "fixtures/tests/nocycle" 12 | const CYCLE_DIR string = "fixtures/tests/cycle" 13 | 14 | func testsToSortedSlice(tests []*Test) []string { 15 | res := make([]string, len(tests)) 16 | for i, t := range tests { 17 | res[i] = t.DependencyID 18 | } 19 | sort.Strings(res) 20 | return res 21 | } 22 | 23 | func TestTestMapLoad(t *testing.T) { 24 | t.Parallel() 25 | assert := assert.New(t) 26 | 27 | tm, errs := newTestMap(TEST_DIR) 28 | assert.NotNil(tm) 29 | assert.Equal(0, len(errs)) 30 | 31 | expected := []string{ 32 | "boot.t", 33 | "panics/panic.t", 34 | "panics/deppanic.t", 35 | "threads/tt1.t", 36 | "threads/tt2.t", 37 | "threads/tt3.t", 38 | "sync/all.t", 39 | "sync/cvt1.t", 40 | "sync/cvt2.t", 41 | "sync/cvt3.t", 42 | "sync/cvt4.t", 43 | "sync/fail.t", 44 | "sync/lt1.t", 45 | "sync/lt2.t", 46 | "sync/lt3.t", 47 | "sync/multi.t", 48 | "sync/sem1.t", 49 | "sync/semu1.t", 50 | } 51 | 52 | assert.Equal(len(expected), len(tm.Tests)) 53 | 54 | for _, id := range expected { 55 | _, ok := tm.Tests[id] 56 | assert.True(ok) 57 | } 58 | 59 | expected = []string{ 60 | "boot", "threads", "sync", 61 | "sem", "locks", "cv", 62 | } 63 | 64 | assert.Equal(len(expected), len(tm.Tags)) 65 | 66 | for _, id := range expected { 67 | _, ok := tm.Tags[id] 68 | assert.True(ok) 69 | } 70 | } 71 | 72 | func TestTestMapGlobs(t *testing.T) { 73 | t.Parallel() 74 | assert := assert.New(t) 75 | 76 | abs, err := filepath.Abs(TEST_DIR) 77 | assert.Nil(err) 78 | 79 | tm, errs := newTestMap(TEST_DIR) 80 | assert.NotNil(tm) 81 | assert.Equal(0, len(errs)) 82 | 83 | // Glob 84 | tests, err := tm.testsFromGlob("**/cv*.t", abs) 85 | expected := []string{ 86 | "sync/cvt1.t", 87 | "sync/cvt2.t", 88 | "sync/cvt3.t", 89 | "sync/cvt4.t", 90 | } 91 | 92 | assert.Nil(err) 93 | assert.Equal(len(expected), len(tests)) 94 | 95 | actual := testsToSortedSlice(tests) 96 | assert.Equal(expected, actual) 97 | 98 | // Single test 99 | single := "threads/tt2.t" 100 | tests, err = tm.testsFromGlob(single, abs) 101 | assert.Nil(err) 102 | assert.Equal(1, len(tests)) 103 | if len(tests) == 1 { 104 | assert.Equal(single, tests[0].DependencyID) 105 | } 106 | 107 | // Empty 108 | tests, err = tm.testsFromGlob("foo/bar*.t", abs) 109 | assert.NotNil(err) 110 | assert.Equal(0, len(tests)) 111 | 112 | } 113 | 114 | func TestTestMapTags(t *testing.T) { 115 | t.Parallel() 116 | assert := assert.New(t) 117 | 118 | tm, errs := newTestMap(TEST_DIR) 119 | assert.NotNil(tm) 120 | assert.Equal(0, len(errs)) 121 | 122 | expected := []string{ 123 | "threads/tt1.t", 124 | "threads/tt2.t", 125 | "threads/tt3.t", 126 | } 127 | tests, ok := tm.Tags["threads"] 128 | assert.True(ok) 129 | assert.Equal(len(expected), len(tests)) 130 | 131 | actual := testsToSortedSlice(tests) 132 | sort.Strings(actual) 133 | assert.Equal(expected, actual) 134 | 135 | expected = []string{ 136 | "sync/cvt1.t", 137 | "sync/cvt2.t", 138 | "sync/cvt3.t", 139 | "sync/cvt4.t", 140 | } 141 | tests, ok = tm.Tags["cv"] 142 | assert.True(ok) 143 | assert.Equal(len(expected), len(tests)) 144 | 145 | actual = testsToSortedSlice(tests) 146 | sort.Strings(actual) 147 | assert.Equal(expected, actual) 148 | 149 | } 150 | 151 | var DEP_MAP = map[string][]string{ 152 | "boot.t": []string{}, 153 | "threads/tt1.t": []string{"boot.t"}, 154 | "threads/tt2.t": []string{"boot.t"}, 155 | "threads/tt3.t": []string{"boot.t"}, 156 | 157 | "sync/semu1.t": []string{"threads/tt1.t", "threads/tt2.t", "threads/tt2.t"}, 158 | "sync/sem1.t": []string{"threads/tt1.t", "threads/tt2.t", "threads/tt2.t"}, 159 | "sync/lt1.t": []string{"threads/tt1.t", "threads/tt2.t", "threads/tt2.t"}, 160 | "sync/lt2.t": []string{"threads/tt1.t", "threads/tt2.t", "threads/tt2.t"}, 161 | "sync/lt3.t": []string{"threads/tt1.t", "threads/tt2.t", "threads/tt2.t"}, 162 | "sync/cvt1.t": []string{"threads/tt1.t", "threads/tt2.t", "threads/tt2.t", "sync/lt1.t", "sync/lt2.t", "sync/lt3.t"}, 163 | "sync/cvt2.t": []string{"threads/tt1.t", "threads/tt2.t", "threads/tt2.t", "sync/lt1.t", "sync/lt2.t", "sync/lt3.t"}, 164 | "sync/cvt3.t": []string{"threads/tt1.t", "threads/tt2.t", "threads/tt2.t", "sync/lt1.t", "sync/lt2.t", "sync/lt3.t"}, 165 | "sync/cvt4.t": []string{"threads/tt1.t", "threads/tt2.t", "threads/tt2.t", "sync/lt1.t", "sync/lt2.t", "sync/lt3.t"}, 166 | 167 | "sync/multi.t": []string{"threads/tt1.t", "threads/tt2.t", "threads/tt2.t"}, 168 | "sync/all.t": []string{"threads/tt1.t", "threads/tt2.t", "threads/tt2.t"}, 169 | "sync/fail.t": []string{"threads/tt1.t", "threads/tt2.t", "threads/tt2.t"}, 170 | 171 | "panics/panic.t": []string{"boot.t"}, 172 | "panics/deppanic.t": []string{"panics/panic.t"}, 173 | } 174 | 175 | func TestTestMapDependencies(t *testing.T) { 176 | t.Parallel() 177 | assert := assert.New(t) 178 | 179 | tm, errs := newTestMap(TEST_DIR) 180 | assert.NotNil(tm) 181 | assert.Equal(0, len(errs)) 182 | 183 | errs = tm.expandAllDeps() 184 | assert.Equal(0, len(errs)) 185 | if len(errs) > 0 { 186 | t.Log(errs) 187 | } 188 | 189 | // Now, test the dependencies by hand. We have a mix of 190 | // glob and tag deps in the test directory 191 | 192 | assert.Equal(len(DEP_MAP), len(tm.Tests)) 193 | 194 | for k, v := range DEP_MAP { 195 | test, ok := tm.Tests[k] 196 | assert.True(ok) 197 | if ok { 198 | assert.Equal(len(v), len(test.ExpandedDeps)) 199 | for _, id := range v { 200 | dep, ok := test.ExpandedDeps[id] 201 | assert.True(ok) 202 | if !ok { 203 | t.Log(id) 204 | t.FailNow() 205 | } 206 | assert.Equal(id, dep.DependencyID) 207 | } 208 | } 209 | } 210 | } 211 | 212 | func TestDependencyGraph(t *testing.T) { 213 | t.Parallel() 214 | assert := assert.New(t) 215 | 216 | tm, errs := newTestMap(TEST_DIR) 217 | assert.NotNil(tm) 218 | assert.Equal(0, len(errs)) 219 | 220 | g, errs := tm.dependencyGraph() 221 | assert.Equal(0, len(errs)) 222 | if len(errs) > 0 { 223 | t.Log(errs) 224 | } 225 | 226 | assert.NotNil(g) 227 | 228 | // Now, test the dependencies by hand. We have a mix of 229 | // glob and tag deps in the test directory 230 | 231 | assert.Equal(len(DEP_MAP), len(g.NodeMap)) 232 | 233 | for k, v := range DEP_MAP { 234 | node, ok := g.NodeMap[k] 235 | assert.True(ok) 236 | if ok { 237 | assert.Equal(len(v), len(node.EdgesOut)) 238 | for _, id := range v { 239 | depNode, ok := node.EdgesOut[id] 240 | assert.True(ok) 241 | if !ok { 242 | t.Log(id) 243 | t.FailNow() 244 | } 245 | assert.Equal(id, depNode.Name) 246 | } 247 | } 248 | } 249 | } 250 | 251 | func TestDependencyCycle(t *testing.T) { 252 | t.Parallel() 253 | assert := assert.New(t) 254 | 255 | tm, errs := newTestMap(TEST_DIR) 256 | assert.NotNil(tm) 257 | assert.Equal(0, len(errs)) 258 | 259 | g, errs := tm.dependencyGraph() 260 | assert.Equal(0, len(errs)) 261 | if len(errs) > 0 { 262 | t.Log(errs) 263 | } 264 | 265 | assert.NotNil(g) 266 | _, err := g.TopSort() 267 | assert.Nil(err) 268 | 269 | tm, errs = newTestMap(CYCLE_DIR) 270 | assert.NotNil(tm) 271 | assert.Equal(0, len(errs)) 272 | 273 | g, errs = tm.dependencyGraph() 274 | assert.Equal(0, len(errs)) 275 | if len(errs) > 0 { 276 | t.Log(errs) 277 | } 278 | 279 | assert.NotNil(g) 280 | _, err = g.TopSort() 281 | assert.NotNil(err) 282 | } 283 | 284 | func TestGroupFromConfg(t *testing.T) { 285 | t.Parallel() 286 | assert := assert.New(t) 287 | 288 | // Test config with dependencies 289 | config := &GroupConfig{ 290 | Name: "Test", 291 | UseDeps: true, 292 | Tests: []string{"sync/cvt1.t"}, 293 | Env: defaultEnv, 294 | } 295 | 296 | expected := []string{ 297 | "boot.t", "threads/tt1.t", "threads/tt2.t", 298 | "threads/tt3.t", "sync/lt1.t", "sync/lt2.t", "sync/lt3.t", 299 | "sync/cvt1.t", 300 | } 301 | 302 | tg, errs := GroupFromConfig(config) 303 | assert.Equal(0, len(errs)) 304 | assert.NotNil(tg) 305 | 306 | assert.Equal(len(expected), len(tg.Tests)) 307 | for _, id := range expected { 308 | test, ok := tg.Tests[id] 309 | assert.True(ok) 310 | if ok { 311 | assert.Equal(id, test.DependencyID) 312 | } 313 | } 314 | 315 | t.Log(tg) 316 | 317 | // Test same config without dependencies 318 | config.UseDeps = false 319 | tg, errs = GroupFromConfig(config) 320 | assert.Equal(0, len(errs)) 321 | assert.NotNil(tg) 322 | assert.Equal(1, len(tg.Tests)) 323 | id := config.Tests[0] 324 | test, ok := tg.Tests[id] 325 | assert.True(ok) 326 | if ok { 327 | assert.Equal(id, test.DependencyID) 328 | } 329 | 330 | t.Log(tg) 331 | } 332 | 333 | func TestGroupConfigInvalid(t *testing.T) { 334 | t.Parallel() 335 | assert := assert.New(t) 336 | 337 | tests := []string{ 338 | "threads/tt1.t", 339 | "threads/tt2.t", 340 | "thread/tt3.t", 341 | } 342 | 343 | // Test config with dependencies 344 | config := &GroupConfig{ 345 | Name: "Test", 346 | UseDeps: false, 347 | Tests: tests, 348 | Env: defaultEnv, 349 | } 350 | 351 | tg, errs := GroupFromConfig(config) 352 | assert.NotEqual(0, len(errs)) 353 | t.Log(errs) 354 | assert.Nil(tg) 355 | } 356 | -------------------------------------------------------------------------------- /logger.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | import ( 4 | "github.com/ops-class/test161/expect" 5 | "regexp" 6 | "time" 7 | ) 8 | 9 | // Recv processes new sys161 output and restarts the progress timer 10 | func (t *Test) Recv(receivedTime time.Time, received []byte) { 11 | 12 | // This is a slightly hacky way to ensure that getStats isn't started until 13 | // sys161 has began to run. (Starting it too early causes the unix socket 14 | // connect to fail.) statStarted is only used once and doesn't need to be 15 | // protected. 16 | if !t.statStarted { 17 | go t.getStats() 18 | t.statStarted = true 19 | } 20 | 21 | // Parse some new incoming data. Frequently just a single byte but sometimes 22 | // more. 23 | t.L.Lock() 24 | defer t.L.Unlock() 25 | 26 | // Mark progress for the progress timeout. 27 | t.progressTime = float64(t.SimTime) 28 | 29 | for _, b := range received { 30 | // Add timestamps to the beginning of each line. 31 | if t.currentOutput.WallTime == 0.0 { 32 | t.currentOutput.WallTime = t.getWallTime() 33 | t.currentOutput.SimTime = t.SimTime 34 | } 35 | t.currentOutput.Buffer.WriteByte(b) 36 | if b == '\n' { 37 | t.currentOutput.Line = t.currentOutput.Buffer.String() 38 | t.outputLineComplete() 39 | t.currentCommand.Output = append(t.currentCommand.Output, t.currentOutput) 40 | t.env.notifyAndLogErr("Update Command Output", t.currentCommand, MSG_PERSIST_UPDATE, MSG_FIELD_OUTPUT) 41 | t.currentOutput = &OutputLine{} 42 | } 43 | } 44 | } 45 | 46 | // Unused parts of the expect.Logger interface 47 | func (t *Test) Send(time.Time, []byte) {} 48 | func (t *Test) SendMasked(time.Time, []byte) {} 49 | func (t *Test) RecvNet(time.Time, []byte) {} 50 | func (t *Test) RecvEOF(time.Time) {} 51 | func (t *Test) ExpectCall(time.Time, *regexp.Regexp) {} 52 | func (t *Test) ExpectReturn(time.Time, expect.Match, error) {} 53 | func (t *Test) Close(time.Time) {} 54 | -------------------------------------------------------------------------------- /man/test161-server.1: -------------------------------------------------------------------------------- 1 | .\" Manpage for test161-server. 2 | .\" Contact shaseley@buffalo.edu to correct errors or typos. 3 | .TH man 1 "07 January 2017" "1.2.5" "test161-server Manual" 4 | .SH NAME 5 | test161-server \- OS/161 Testing Tool 6 | .SH SYNOPSIS 7 | test161-server [ARGS] 8 | .SH DESCRIPTION 9 | .SH OPTIONS 10 | .SH SEE ALSO 11 | .SH BUGS 12 | No known bugs. 13 | .SH AUTHOR 14 | Scott Haseley (shaseley@buffalo.edu) 15 | -------------------------------------------------------------------------------- /man/test161.1: -------------------------------------------------------------------------------- 1 | .\" Manpage for test161. 2 | .\" Contact shaseley@buffalo.edu to correct errors or typos. 3 | .TH man 1 "07 January 2017" "1.2.5" "test161 Manual" 4 | .SH NAME 5 | test161 \- OS/161 Testing Tool 6 | .SH SYNOPSIS 7 | test161 [ARGS] 8 | .SH DESCRIPTION 9 | .SH OPTIONS 10 | .SH SEE ALSO 11 | .SH BUGS 12 | No known bugs. 13 | .SH AUTHOR 14 | Scott Haseley (shaseley@buffalo.edu) 15 | -------------------------------------------------------------------------------- /manager.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "sync" 7 | "time" 8 | ) 9 | 10 | // This file defines test161's test manager. The manager is responsible for 11 | // keeping track of the number of running tests and limiting that number to 12 | // a configurable capacity. 13 | // 14 | // There is a global test manager (testManager) that listens for new job 15 | // requests on its SubmitChan. In the current implementation, this can 16 | // only be accessed from within the package by one of the TestRunners. 17 | 18 | // A test161Job consists of the test to run, the directory to find the 19 | // binaries, and a channel to communicate the results on. 20 | type test161Job struct { 21 | Test *Test 22 | Env *TestEnvironment 23 | DoneChan chan *Test161JobResult 24 | } 25 | 26 | // A Test161JobResult consists of the completed test and any error that 27 | // occurred while running the test. 28 | type Test161JobResult struct { 29 | Test *Test 30 | Err error 31 | } 32 | 33 | type manager struct { 34 | SubmitChan chan *test161Job 35 | Capacity uint 36 | 37 | // protected by L 38 | statsCond *sync.Cond 39 | queueCond *sync.Cond 40 | isRunning bool 41 | 42 | stats ManagerStats 43 | } 44 | 45 | type ManagerStats struct { 46 | // protected by manager.L 47 | Running uint `json:"running"` 48 | HighRunning uint `json:"high_running"` 49 | Queued uint `json:"queued"` 50 | HighQueued uint `json:"high_queued"` 51 | Finished uint `json:"finished"` 52 | MaxWait int64 `json:"max_wait_ms"` 53 | AvgWait int64 `json:"avg_wait_ms"` 54 | StartTime time.Time 55 | total int64 // denominator for avg 56 | } 57 | 58 | // Combined submission and tests statistics since the service started 59 | type Test161Stats struct { 60 | Status string `json:"status"` 61 | SubmissionStats ManagerStats `json:"submission_stats"` 62 | TestStats ManagerStats `json:"test_stats"` 63 | } 64 | 65 | const DEFAULT_MGR_CAPACITY uint = 0 66 | 67 | func newManager() *manager { 68 | m := &manager{ 69 | SubmitChan: nil, 70 | Capacity: DEFAULT_MGR_CAPACITY, 71 | statsCond: sync.NewCond(&sync.Mutex{}), 72 | queueCond: sync.NewCond(&sync.Mutex{}), 73 | isRunning: false, 74 | } 75 | return m 76 | } 77 | 78 | // The global test manager. NewEnvironment has a reference to 79 | // this manager, which should be used by clients of this library. 80 | // Unit tests use their own manager/environment for runner/assertion 81 | // isolation. 82 | var testManager *manager = newManager() 83 | 84 | // Clear state and start listening for job requests 85 | func (m *manager) start() { 86 | m.statsCond.L.Lock() 87 | defer m.statsCond.L.Unlock() 88 | 89 | if m.isRunning { 90 | return 91 | } 92 | 93 | m.stats = ManagerStats{ 94 | StartTime: time.Now(), 95 | } 96 | m.SubmitChan = make(chan *test161Job) 97 | m.isRunning = true 98 | 99 | // Listener goroutine 100 | go func() { 101 | // We simply spawn a worker that blocks until it can run. 102 | for job := range m.SubmitChan { 103 | go m.runOrQueueJob(job) 104 | } 105 | }() 106 | } 107 | 108 | // Queue the job if we're at capacity, and run it once we're under. 109 | func (m *manager) runOrQueueJob(job *test161Job) { 110 | 111 | m.statsCond.L.Lock() 112 | queued := false 113 | start := time.Now() 114 | 115 | for m.Capacity > 0 && m.stats.Running >= m.Capacity { 116 | if !queued { 117 | queued = true 118 | 119 | // Update queued stats 120 | m.stats.Queued += 1 121 | if m.stats.Queued > m.stats.HighQueued { 122 | m.stats.HighQueued = m.stats.Queued 123 | } 124 | } 125 | 126 | // Wait for a finished test to signal us 127 | m.statsCond.Wait() 128 | } 129 | 130 | // We've got the green light... (and the stats lock) 131 | if queued { 132 | // Update the queue count and signal the submission manager (if there is one) 133 | m.queueCond.L.Lock() 134 | m.stats.Queued -= 1 135 | m.queueCond.Signal() 136 | m.queueCond.L.Unlock() 137 | queued = false 138 | 139 | // Max and average waits 140 | curWait := int64(time.Now().Sub(start).Nanoseconds() / 1e6) 141 | if m.stats.MaxWait < curWait { 142 | m.stats.MaxWait = curWait 143 | } 144 | m.stats.AvgWait = (m.stats.total*m.stats.AvgWait + curWait) / (m.stats.total + 1) 145 | m.stats.total += 1 146 | } 147 | 148 | m.stats.Running += 1 149 | if m.stats.Running > m.stats.HighRunning { 150 | m.stats.HighRunning = m.stats.Running 151 | } 152 | 153 | m.statsCond.L.Unlock() 154 | 155 | // Go! 156 | err := job.Test.Run(job.Env) 157 | 158 | // And... we're done. 159 | 160 | // Update stats 161 | m.statsCond.L.Lock() 162 | m.stats.Running -= 1 163 | m.stats.Finished += 1 164 | 165 | m.statsCond.Signal() 166 | m.statsCond.L.Unlock() 167 | 168 | // Pass the completed test back to the caller 169 | // (Blocking call, we need to make sure the caller gets the result.) 170 | job.DoneChan <- &Test161JobResult{job.Test, err} 171 | } 172 | 173 | // Shut it down 174 | func (m *manager) stop() { 175 | m.statsCond.L.Lock() 176 | defer m.statsCond.L.Unlock() 177 | 178 | if !m.isRunning { 179 | return 180 | } 181 | 182 | m.isRunning = false 183 | close(m.SubmitChan) 184 | } 185 | 186 | // Exported shared test manger functions 187 | 188 | // Start the shared test manager 189 | func StartManager() { 190 | testManager.start() 191 | } 192 | 193 | // Stop the shared test manager 194 | func StopManager() { 195 | testManager.stop() 196 | } 197 | 198 | func SetManagerCapacity(capacity uint) { 199 | testManager.Capacity = capacity 200 | } 201 | 202 | func ManagerCapacity() uint { 203 | return testManager.Capacity 204 | } 205 | 206 | func (m *manager) Stats() *ManagerStats { 207 | // Lock so we at least get a consistent view of the stats 208 | testManager.statsCond.L.Lock() 209 | defer testManager.statsCond.L.Unlock() 210 | 211 | // copy 212 | var res ManagerStats = testManager.stats 213 | return &res 214 | } 215 | 216 | // Return a copy of the current shared test manager stats 217 | func GetManagerStats() *ManagerStats { 218 | return testManager.Stats() 219 | } 220 | 221 | //////// Submission Manager 222 | // 223 | // SubmissionManager handles running multiple submissions, which is useful for the server. 224 | // The (Test)Manager already handles rate limiting tests, but we also need to be careful 225 | // about rate limiting submissions. In particular, we don't need to build all new 226 | // if we have queued tests. This just wastes cycles and I/O that the tests could use. 227 | // Plus, the student will see the status go from building to running, but the boot test 228 | // will just get queued. 229 | 230 | const ( 231 | SM_ACCEPTING = iota 232 | SM_NOT_ACCEPTING 233 | SM_STAFF_ONLY 234 | ) 235 | 236 | type SubmissionManager struct { 237 | env *TestEnvironment 238 | runlock *sync.Mutex // Block everything from running 239 | l *sync.Mutex // Synchronize other state 240 | status int 241 | stats ManagerStats 242 | } 243 | 244 | func NewSubmissionManager(env *TestEnvironment) *SubmissionManager { 245 | mgr := &SubmissionManager{ 246 | env: env, 247 | runlock: &sync.Mutex{}, 248 | l: &sync.Mutex{}, 249 | status: SM_ACCEPTING, 250 | stats: ManagerStats{ 251 | StartTime: time.Now(), 252 | }, 253 | } 254 | return mgr 255 | } 256 | 257 | func (sm *SubmissionManager) CombinedStats() *Test161Stats { 258 | stats := &Test161Stats{ 259 | SubmissionStats: *sm.Stats(), 260 | TestStats: *sm.env.manager.Stats(), 261 | } 262 | 263 | switch sm.Status() { 264 | case SM_ACCEPTING: 265 | stats.Status = "accepting submissions" 266 | case SM_NOT_ACCEPTING: 267 | stats.Status = "not accepting submissions" 268 | case SM_STAFF_ONLY: 269 | stats.Status = "accepting submissions from staff only" 270 | } 271 | 272 | return stats 273 | } 274 | 275 | func (sm *SubmissionManager) Stats() *ManagerStats { 276 | sm.l.Lock() 277 | defer sm.l.Unlock() 278 | copy := sm.stats 279 | return © 280 | } 281 | 282 | func (sm *SubmissionManager) Run(s *Submission) error { 283 | 284 | // The test manager we're associated with 285 | mgr := sm.env.manager 286 | 287 | sm.l.Lock() 288 | 289 | // Check to see if we've been paused or stopped. The server checks too, but there's delay. 290 | abort := false 291 | abortMsg := "" 292 | 293 | if sm.status == SM_NOT_ACCEPTING { 294 | abort = true 295 | abortMsg = "The submission server is not accepting new submissions at this time" 296 | } else if sm.status == SM_STAFF_ONLY { 297 | for _, student := range s.students { 298 | if isStaff, _ := student.IsStaff(sm.env); !isStaff { 299 | abort = true 300 | abortMsg = "The submission server is not accepting new submissions from students at this time" 301 | } 302 | } 303 | } 304 | 305 | if abort { 306 | sm.l.Unlock() 307 | s.Status = SUBMISSION_ABORTED 308 | err := errors.New(abortMsg) 309 | s.Errors = append(s.Errors, fmt.Sprintf("%v", err)) 310 | sm.env.notifyAndLogErr("Submissions Closed", s, MSG_PERSIST_COMPLETE, 0) 311 | return err 312 | } 313 | 314 | // Update Queued 315 | sm.stats.Queued += 1 316 | start := time.Now() 317 | if sm.stats.HighQueued < sm.stats.Queued { 318 | sm.stats.HighQueued = sm.stats.Queued 319 | } 320 | sm.l.Unlock() 321 | 322 | /////////// 323 | // Queued here 324 | sm.runlock.Lock() 325 | 326 | // Still queued, but on deck. Wait on the manager's queue condition variable so we 327 | // get notifications when the count changes. 328 | mgr.queueCond.L.Lock() 329 | for mgr.stats.Queued > 0 { 330 | mgr.queueCond.Wait() 331 | } 332 | mgr.queueCond.L.Unlock() 333 | // We may get a rush of builds here. Eventually we'll get a queue again, and 334 | // the test manager handles this. 335 | // TODO: Consider better build rate limiting 336 | /////////// 337 | 338 | // Update run stats 339 | sm.l.Lock() 340 | sm.stats.Queued -= 1 341 | sm.stats.Running += 1 342 | if sm.stats.HighRunning < sm.stats.Running { 343 | sm.stats.HighRunning = sm.stats.Running 344 | } 345 | 346 | // Max and average waits 347 | curWait := int64(time.Now().Sub(start).Nanoseconds() / 1e6) 348 | if sm.stats.MaxWait < curWait { 349 | sm.stats.MaxWait = curWait 350 | } 351 | sm.stats.AvgWait = (sm.stats.total*sm.stats.AvgWait + curWait) / (sm.stats.total + 1) 352 | sm.stats.total += 1 353 | 354 | sm.l.Unlock() 355 | 356 | // Run the submission 357 | sm.runlock.Unlock() 358 | err := s.Run() 359 | 360 | // Update stats 361 | sm.l.Lock() 362 | sm.stats.Running -= 1 363 | sm.stats.Finished += 1 364 | sm.l.Unlock() 365 | 366 | return err 367 | } 368 | 369 | func (sm *SubmissionManager) Pause() { 370 | sm.l.Lock() 371 | defer sm.l.Unlock() 372 | sm.status = SM_NOT_ACCEPTING 373 | } 374 | 375 | func (sm *SubmissionManager) Resume() { 376 | sm.l.Lock() 377 | defer sm.l.Unlock() 378 | sm.status = SM_ACCEPTING 379 | } 380 | 381 | func (sm *SubmissionManager) Status() int { 382 | sm.l.Lock() 383 | defer sm.l.Unlock() 384 | return sm.status 385 | } 386 | 387 | func (sm *SubmissionManager) SetStaffOnly() { 388 | sm.l.Lock() 389 | defer sm.l.Unlock() 390 | sm.status = SM_STAFF_ONLY 391 | } 392 | -------------------------------------------------------------------------------- /mongopersist.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/kevinburke/go.uuid" 7 | "gopkg.in/mgo.v2" 8 | "gopkg.in/mgo.v2/bson" 9 | ) 10 | 11 | type MongoPersistence struct { 12 | session *mgo.Session 13 | dbName string 14 | } 15 | 16 | const ( 17 | COLLECTION_SUBMISSIONS = "submissions" 18 | COLLECTION_TESTS = "tests" 19 | COLLECTION_STUDENTS = "students" 20 | COLLECTION_TARGETS = "targets" 21 | COLLECTION_USERS = "users" 22 | COLLECTION_USAGE = "usage" 23 | ) 24 | 25 | func NewMongoPersistence(dial *mgo.DialInfo) (PersistenceManager, error) { 26 | var err error 27 | 28 | m := &MongoPersistence{} 29 | 30 | if m.session, err = mgo.DialWithInfo(dial); err != nil { 31 | return nil, fmt.Errorf("Mongo Create Session: %s\n", err) 32 | } 33 | m.dbName = dial.Database 34 | 35 | return m, nil 36 | } 37 | 38 | func (m *MongoPersistence) insertDocument(s *mgo.Session, collection string, data interface{}) error { 39 | c := s.DB(m.dbName).C(collection) 40 | err := c.Insert(data) 41 | return err 42 | } 43 | 44 | func (m *MongoPersistence) updateDocumentByID(s *mgo.Session, collection string, id, data interface{}) error { 45 | c := s.DB(m.dbName).C(collection) 46 | err := c.UpdateId(id, data) 47 | return err 48 | } 49 | 50 | func (m *MongoPersistence) updateDocument(s *mgo.Session, collection string, selector, data interface{}) error { 51 | c := s.DB(m.dbName).C(collection) 52 | err := c.Update(selector, data) 53 | return err 54 | } 55 | 56 | // Update if it exists, otherwise insert 57 | func (m *MongoPersistence) upsertDocument(s *mgo.Session, collection string, selector, data interface{}) error { 58 | c := s.DB(m.dbName).C(collection) 59 | _, err := c.Upsert(selector, data) 60 | return err 61 | } 62 | 63 | func (m *MongoPersistence) Close() { 64 | m.session.Close() 65 | } 66 | 67 | func getTestUpdateMap(test *Test, what int) bson.M { 68 | changes := bson.M{} 69 | 70 | if what&MSG_FIELD_SCORE == MSG_FIELD_SCORE { 71 | changes["points_earned"] = test.PointsEarned 72 | } 73 | if what&MSG_FIELD_STATUS == MSG_FIELD_STATUS { 74 | changes["result"] = test.Result 75 | } 76 | 77 | return changes 78 | } 79 | 80 | func (m *MongoPersistence) Notify(t interface{}, msg, what int) (err error) { 81 | 82 | session := m.session.Copy() 83 | defer session.Close() 84 | 85 | switch t.(type) { 86 | default: 87 | { 88 | err = fmt.Errorf("Unexpected type in Notify(): %T", t) 89 | } 90 | case *Test: 91 | { 92 | test := t.(*Test) 93 | switch msg { 94 | case MSG_PERSIST_CREATE: 95 | err = m.insertDocument(session, COLLECTION_TESTS, test) 96 | case MSG_PERSIST_COMPLETE: 97 | err = m.updateDocumentByID(session, COLLECTION_TESTS, test.ID, test) 98 | case MSG_PERSIST_UPDATE: 99 | changes := bson.M{} 100 | if what&MSG_FIELD_SCORE == MSG_FIELD_SCORE { 101 | changes["points_earned"] = test.PointsEarned 102 | } 103 | 104 | if what&MSG_FIELD_STATUS == MSG_FIELD_STATUS { 105 | changes["result"] = test.Result 106 | } 107 | 108 | if len(changes) > 0 { 109 | err = m.updateDocumentByID(session, COLLECTION_TESTS, test.ID, bson.M{"$set": changes}) 110 | } 111 | } 112 | } 113 | case *Command: 114 | { 115 | cmd := t.(*Command) 116 | switch msg { 117 | case MSG_PERSIST_UPDATE: 118 | selector := bson.M{ 119 | "_id": cmd.Test.ID, 120 | "commands._id": cmd.ID, 121 | } 122 | changes := bson.M{} 123 | 124 | if what&MSG_FIELD_OUTPUT == MSG_FIELD_OUTPUT { 125 | changes["commands.$.output"] = cmd.Output 126 | } 127 | 128 | if what&MSG_FIELD_SCORE == MSG_FIELD_SCORE { 129 | changes["commands.$.points_earned"] = cmd.PointsEarned 130 | } 131 | 132 | if what&MSG_FIELD_STATUS == MSG_FIELD_STATUS { 133 | changes["commands.$.status"] = cmd.Status 134 | } 135 | 136 | err = m.updateDocument(session, COLLECTION_TESTS, selector, bson.M{"$set": changes}) 137 | 138 | } 139 | } 140 | case *Submission: 141 | { 142 | submission := t.(*Submission) 143 | switch msg { 144 | case MSG_PERSIST_CREATE: 145 | err = m.insertDocument(session, COLLECTION_SUBMISSIONS, submission) 146 | case MSG_PERSIST_COMPLETE: 147 | fallthrough 148 | case MSG_PERSIST_UPDATE: 149 | err = m.updateDocumentByID(session, COLLECTION_SUBMISSIONS, submission.ID, submission) 150 | } 151 | } 152 | case *BuildTest: 153 | { 154 | test := t.(*BuildTest) 155 | switch msg { 156 | case MSG_PERSIST_CREATE: 157 | err = m.insertDocument(session, COLLECTION_TESTS, test) 158 | case MSG_PERSIST_COMPLETE: 159 | err = m.updateDocumentByID(session, COLLECTION_TESTS, test.ID, test) 160 | case MSG_PERSIST_UPDATE: 161 | changes := bson.M{} 162 | if what&MSG_FIELD_STATUS == MSG_FIELD_STATUS { 163 | changes["result"] = test.Result 164 | } 165 | if len(changes) > 0 { 166 | err = m.updateDocumentByID(session, COLLECTION_TESTS, test.ID, bson.M{"$set": changes}) 167 | } 168 | } 169 | } 170 | case *BuildCommand: 171 | { 172 | cmd := t.(*BuildCommand) 173 | switch msg { 174 | case MSG_PERSIST_UPDATE: 175 | selector := bson.M{ 176 | "_id": cmd.test.ID, 177 | "commands._id": cmd.ID, 178 | } 179 | changes := bson.M{} 180 | 181 | if what&MSG_FIELD_OUTPUT == MSG_FIELD_OUTPUT { 182 | changes["commands.$.output"] = cmd.Output 183 | } 184 | 185 | if what&MSG_FIELD_STATUS == MSG_FIELD_STATUS { 186 | changes["commands.$.status"] = cmd.Status 187 | } 188 | 189 | err = m.updateDocument(session, COLLECTION_TESTS, selector, bson.M{"$set": changes}) 190 | 191 | } 192 | } 193 | case *Student: 194 | { 195 | student := t.(*Student) 196 | switch msg { 197 | case MSG_PERSIST_UPDATE: 198 | err = m.updateDocumentByID(session, COLLECTION_STUDENTS, student.ID, student) 199 | } 200 | } 201 | case *Target: 202 | { 203 | target := t.(*Target) 204 | switch msg { 205 | case MSG_TARGET_LOAD: 206 | c := session.DB(m.dbName).C(COLLECTION_TARGETS) 207 | targets := []*Target{} 208 | c.Find(bson.M{ 209 | "name": target.Name, 210 | "version": target.Version, 211 | }).All(&targets) 212 | if len(targets) == 0 { 213 | // Insert 214 | target.ID = uuid.NewV4().String() 215 | err = m.insertDocument(session, COLLECTION_TARGETS, target) 216 | } else if len(targets) == 1 { 217 | target.ID = targets[0].ID 218 | if target.FileHash != targets[0].FileHash { 219 | // Sanity checks to make sure no one changed a target that has a submission. 220 | // (If this happens in testing, just clear the DB manually) 221 | changeErr := targets[0].isChangeAllowed(target) 222 | 223 | // Figure out if there are any submissions with this id. If so, fail. 224 | var submissions []*Submission 225 | subColl := session.DB(m.dbName).C(COLLECTION_SUBMISSIONS) 226 | err = subColl.Find(bson.M{"target_id": target.ID}).Limit(1).All(&submissions) 227 | if err == nil { 228 | if len(submissions) > 0 && changeErr != nil { 229 | err = fmt.Errorf( 230 | "Target details changed and previous submissions exist. Increment the version number of the new target.\n%v", changeErr) 231 | } else { 232 | // Just update it with the new version 233 | err = m.updateDocumentByID(session, COLLECTION_TARGETS, target.ID, target) 234 | } 235 | } 236 | } 237 | } else { 238 | err = errors.New("Multiple targets exist in DB for '" + target.Name + "'") 239 | } 240 | } 241 | } 242 | case *UsageStat: 243 | { 244 | stat := t.(*UsageStat) 245 | switch msg { 246 | case MSG_PERSIST_CREATE: 247 | if len(stat.ID) == 0 { 248 | return errors.New("ID required to upsert UsageStat") 249 | } 250 | selector := bson.M{ 251 | "_id": stat.ID, 252 | } 253 | err = m.upsertDocument(session, COLLECTION_USAGE, selector, stat) 254 | } 255 | } 256 | } 257 | return 258 | } 259 | 260 | func (m *MongoPersistence) CanRetrieve() bool { 261 | return true 262 | } 263 | 264 | func (m *MongoPersistence) Retrieve(what int, who map[string]interface{}, filter map[string]interface{}, res interface{}) error { 265 | session := m.session.Copy() 266 | defer session.Close() 267 | 268 | collection := "" 269 | 270 | switch what { 271 | case PERSIST_TYPE_STUDENTS: 272 | collection = COLLECTION_STUDENTS 273 | case PERSIST_TYPE_USERS: 274 | collection = COLLECTION_USERS 275 | default: 276 | return errors.New("Persistence: Invalid data type") 277 | } 278 | 279 | c := session.DB(m.dbName).C(collection) 280 | query := c.Find(bson.M(who)) 281 | if filter != nil { 282 | query = query.Select(bson.M(filter)) 283 | } 284 | return query.All(res) 285 | } 286 | -------------------------------------------------------------------------------- /mongopersist_test.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "gopkg.in/mgo.v2" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | // MongoDB test connection 11 | var mongoTestDialInfo = &mgo.DialInfo{ 12 | Addrs: []string{"localhost:27017"}, 13 | Timeout: 60 * time.Second, 14 | Database: "test", 15 | Username: "", 16 | Password: "", 17 | } 18 | 19 | func TestMongoBoot(t *testing.T) { 20 | t.Parallel() 21 | 22 | if !testFlagDB { 23 | t.Skip("Skipping MongoDB Test") 24 | } 25 | 26 | assert := assert.New(t) 27 | 28 | mongo, err := NewMongoPersistence(mongoTestDialInfo) 29 | assert.Nil(err) 30 | assert.NotNil(mongo) 31 | 32 | if err != nil { 33 | t.FailNow() 34 | } 35 | defer mongo.Close() 36 | 37 | test, err := TestFromString("q") 38 | assert.Nil(err) 39 | assert.Nil(test.MergeConf(TEST_DEFAULTS)) 40 | 41 | env := defaultEnv.CopyEnvironment() 42 | env.Persistence = mongo 43 | env.RootDir = defaultEnv.RootDir 44 | 45 | assert.Nil(env.Persistence.Notify(test, MSG_PERSIST_CREATE, 0)) 46 | assert.Nil(test.Run(env)) 47 | } 48 | 49 | func TestMongoGetUser(t *testing.T) { 50 | t.Parallel() 51 | 52 | if !testFlagDB { 53 | t.Skip("Skipping MongoDB Test") 54 | } 55 | 56 | assert := assert.New(t) 57 | 58 | // MongoDB test connection 59 | var di = &mgo.DialInfo{ 60 | Addrs: []string{"localhost:27017"}, 61 | Timeout: 60 * time.Second, 62 | Database: "test161", 63 | Username: "", 64 | Password: "", 65 | } 66 | 67 | mongo, err := NewMongoPersistence(di) 68 | assert.Nil(err) 69 | assert.NotNil(mongo) 70 | 71 | if err != nil { 72 | t.FailNow() 73 | } 74 | defer mongo.Close() 75 | 76 | staff := "services.auth0.user_metadata.staff" 77 | who := map[string]interface{}{"services.auth0.email": "admin@ops-class.org"} 78 | filter := map[string]interface{}{staff: 1} 79 | res := make([]interface{}, 0) 80 | 81 | err = mongo.Retrieve(PERSIST_TYPE_USERS, who, filter, &res) 82 | assert.Nil(err) 83 | assert.True(len(res) > 0) 84 | t.Log(res) 85 | } 86 | -------------------------------------------------------------------------------- /output.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strings" 7 | ) 8 | 9 | // OutputJSON serializes the test object and all related output. 10 | func (t *Test) OutputJSON() (string, error) { 11 | outputBytes, err := json.MarshalIndent(t, "", " ") 12 | if err != nil { 13 | return "", err 14 | } 15 | return string(outputBytes), nil 16 | } 17 | 18 | // OutputString prints test output in a human readable form. 19 | func (t *Test) OutputString() string { 20 | var output string 21 | for _, conf := range strings.Split(t.ConfString, "\n") { 22 | conf = strings.TrimSpace(conf) 23 | output += fmt.Sprintf("conf: %s\n", conf) 24 | } 25 | for i, command := range t.Commands { 26 | for j, outputLine := range command.Output { 27 | if i == 0 || j != 0 { 28 | output += fmt.Sprintf("%.6f\t%s\n", outputLine.SimTime, outputLine.Line) 29 | } else { 30 | output += fmt.Sprintf("%s\n", outputLine.Line) 31 | } 32 | } 33 | } 34 | if len(output) > 0 && string(output[len(output)-1]) != "\n" { 35 | output += "\n" 36 | } 37 | if len(t.Status) > 0 { 38 | status := t.Status[len(t.Status)-1] 39 | output += fmt.Sprintf("%.6f\t%s", t.SimTime, status.Status) 40 | if status.Message != "" { 41 | output += fmt.Sprintf(": %s", status.Message) 42 | } 43 | output += "\n" 44 | } 45 | return output 46 | } 47 | 48 | func (tg *TestGroup) OutputJSON() (string, error) { 49 | outputBytes, err := json.MarshalIndent(tg, "", " ") 50 | if err != nil { 51 | return "", err 52 | } 53 | return string(outputBytes), nil 54 | } 55 | 56 | func (tg *TestGroup) OutputString() string { 57 | var output string 58 | output += fmt.Sprintf("\ngroup: name = %v\n", tg.Config.Name) 59 | output += fmt.Sprintf("group: rootdir = %v\n", tg.Config.Env.RootDir) 60 | output += fmt.Sprintf("group: testdir = %v\n", tg.Config.Env.TestDir) 61 | output += fmt.Sprintf("group: usedeps = %v\n", tg.Config.UseDeps) 62 | output += fmt.Sprintf("group: tests = %v\n", tg.Config.Tests) 63 | 64 | for _, test := range tg.Tests { 65 | output += "\n" 66 | output += test.OutputString() 67 | } 68 | 69 | return output 70 | } 71 | 72 | func (t *BuildTest) OutputJSON() (string, error) { 73 | outputBytes, err := json.MarshalIndent(t, "", " ") 74 | if err != nil { 75 | return "", err 76 | } 77 | return string(outputBytes), nil 78 | } 79 | -------------------------------------------------------------------------------- /persistence.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | const ( 4 | MSG_PERSIST_CREATE = iota // The object has been created 5 | MSG_PERSIST_UPDATE // Generic update message. 6 | MSG_PERSIST_OUTPUT // Added an output line (command types only) 7 | MSG_PERSIST_COMPLETE // We won't update the object any more 8 | MSG_TARGET_LOAD // When a target is loaded 9 | ) 10 | 11 | // Inidividual field updates 12 | const ( 13 | MSG_FIELD_SCORE = 1 << iota 14 | MSG_FIELD_STATUS 15 | MSG_FIELD_TESTS 16 | MSG_FIELD_OUTPUT 17 | MSG_FIELD_STATUSES 18 | ) 19 | 20 | const ( 21 | PERSIST_TYPE_STUDENTS = 1 << iota 22 | PERSIST_TYPE_USERS 23 | ) 24 | 25 | // Each Submission has at most one PersistenceManager, and it is pinged when a 26 | // variety of events occur. These callbacks are invoked synchronously, so it's 27 | // up to the PersistenceManager to not slow down the tests. We do this because 28 | // the PersistenceManager can create goroutines if applicable, but we can't 29 | // make an asynchronous call synchronous when it might be needed. So, be kind 30 | // ye PersistenceManagers. 31 | type PersistenceManager interface { 32 | Close() 33 | Notify(entity interface{}, msg, what int) error 34 | CanRetrieve() bool 35 | 36 | // what should be PERSIST_TYPE_* 37 | // who is a map of field:value 38 | // res is where to deserialize the data 39 | Retrieve(what int, who map[string]interface{}, filter map[string]interface{}, res interface{}) error 40 | } 41 | 42 | type DoNothingPersistence struct { 43 | } 44 | 45 | func (d *DoNothingPersistence) Close() { 46 | } 47 | 48 | func (d *DoNothingPersistence) Notify(entity interface{}, msg, what int) error { 49 | return nil 50 | } 51 | 52 | func (d *DoNothingPersistence) CanRetrieve() bool { 53 | return false 54 | } 55 | 56 | func (d *DoNothingPersistence) Retrieve(what int, who map[string]interface{}, 57 | filter map[string]interface{}, res interface{}) error { 58 | return nil 59 | } 60 | -------------------------------------------------------------------------------- /runners.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | // A TestRunner is responsible for running a TestGroup and sending the 4 | // results back on a read-only channel. test161 runners close the results 5 | // channel when finished so clients can range over it. test161 runners also 6 | // return as soon as they are able to and let tests run asynchronously. 7 | type TestRunner interface { 8 | Group() *TestGroup 9 | Run() <-chan *Test161JobResult 10 | } 11 | 12 | // Create a TestRunner from a GroupConfig. config.UseDeps determines the 13 | // type of runner created. 14 | func TestRunnerFromConfig(config *GroupConfig) (TestRunner, []error) { 15 | if tg, errs := GroupFromConfig(config); len(errs) > 0 { 16 | return nil, errs 17 | } else if config.UseDeps { 18 | return NewDependencyRunner(tg), nil 19 | } else { 20 | return NewSimpleRunner(tg), nil 21 | } 22 | } 23 | 24 | // Factory function to create a new SimpleRunner. 25 | func NewSimpleRunner(group *TestGroup) TestRunner { 26 | return &SimpleRunner{group} 27 | } 28 | 29 | // Factory function to create a new DependencyRunner. 30 | func NewDependencyRunner(group *TestGroup) TestRunner { 31 | return &DependencyRunner{group} 32 | } 33 | 34 | // A simple runner that tries to run everything as fast as it's allowed to, 35 | // i.e. it doesn't care about dependencies. 36 | type SimpleRunner struct { 37 | group *TestGroup 38 | } 39 | 40 | func (r *SimpleRunner) Group() *TestGroup { 41 | return r.group 42 | } 43 | 44 | func (r *SimpleRunner) Run() <-chan *Test161JobResult { 45 | 46 | // We create 2 channels, one to receive the results from the test 47 | // manager and one to transmit the results to the caller. We 48 | // don't just pass the caller channel to the test manager because 49 | // we promise to close the channel when we're done so clients can 50 | // range over the channel if they'd like 51 | 52 | // For retrieving results from the test manager 53 | resChan := make(chan *Test161JobResult) 54 | 55 | // Buffered channel for our client. 56 | callbackChan := make(chan *Test161JobResult, len(r.group.Tests)) 57 | 58 | env := r.group.Config.Env 59 | 60 | // Spawn every job at once (no dependency tracking) 61 | for _, test := range r.group.Tests { 62 | job := &test161Job{test, env, resChan} 63 | env.manager.SubmitChan <- job 64 | } 65 | 66 | go func() { 67 | for i, count := 0, len(r.group.Tests); i < count; i++ { 68 | // Always block recieving the test result 69 | res := <-resChan 70 | 71 | // But, never block sending it back 72 | select { 73 | case callbackChan <- res: 74 | default: 75 | } 76 | } 77 | close(callbackChan) 78 | }() 79 | 80 | return callbackChan 81 | } 82 | 83 | // This runner has mad respect for dependencies. 84 | type DependencyRunner struct { 85 | group *TestGroup 86 | } 87 | 88 | func (r *DependencyRunner) Group() *TestGroup { 89 | return r.group 90 | } 91 | 92 | // Holding pattern. An individual test waits here until all of its 93 | // dependencies have been met or failed, in which case it runs or aborts. 94 | func waitForDeps(test *Test, depChan, readyChan, abortChan chan *Test) { 95 | // Copy deps 96 | deps := make(map[string]bool) 97 | for id := range test.ExpandedDeps { 98 | deps[id] = true 99 | } 100 | 101 | for len(deps) > 0 { 102 | res := <-depChan 103 | if _, ok := deps[res.DependencyID]; ok { 104 | if res.Result == TEST_RESULT_CORRECT { 105 | delete(deps, res.DependencyID) 106 | } else { 107 | test.Result = TEST_RESULT_SKIP 108 | abortChan <- test 109 | return 110 | } 111 | } 112 | } 113 | 114 | // We're clear 115 | readyChan <- test 116 | } 117 | 118 | func (r *DependencyRunner) Run() <-chan *Test161JobResult { 119 | 120 | // Everything that's still waiting. 121 | // We make it big enough that it can hold all the results 122 | waiting := make(map[string]chan *Test, len(r.group.Tests)) 123 | 124 | // The channel that our goroutines message us back on 125 | // to let us know they're ready 126 | readyChan := make(chan *Test) 127 | 128 | // The channel we use to get results back from the manager 129 | resChan := make(chan *Test161JobResult) 130 | 131 | // The channel that our goroutines message us back on 132 | // when they can't run because a dependency failed. 133 | abortChan := make(chan *Test) 134 | 135 | // A function to broadcast a single test result 136 | bcast := func(test *Test) { 137 | for _, ch := range waiting { 138 | // Non-blocking because it is buffered to hold the max number of tests 139 | select { 140 | case ch <- test: 141 | default: 142 | } 143 | } 144 | } 145 | 146 | // Buffered channel for our client. 147 | callbackChan := make(chan *Test161JobResult, len(r.group.Tests)) 148 | 149 | // A function to send the result back to the caller 150 | callback := func(res *Test161JobResult) { 151 | // Don't block, not that we should since we're buffered 152 | select { 153 | case callbackChan <- res: 154 | default: 155 | } 156 | } 157 | 158 | // Spawn all the tests and put them in a waiting pattern 159 | for id, test := range r.group.Tests { 160 | // Buffer this so we eliminate races during setup 161 | waiting[id] = make(chan *Test, len(r.group.Tests)) 162 | go waitForDeps(test, waiting[id], readyChan, abortChan) 163 | } 164 | 165 | // Main goroutine responsible for directing traffic. 166 | // - Results from the manager and abort channels are broadcast to the 167 | // remaining blocked tests and caller 168 | // - Tests coming in on the ready channel get sent to the manager 169 | go func() { 170 | //We're done as soon as we recieve the final result from the manager. 171 | results := 0 172 | env := r.group.Config.Env 173 | 174 | for results < len(r.group.Tests) { 175 | select { 176 | case res := <-resChan: 177 | bcast(res.Test) 178 | callback(res) 179 | results += 1 180 | 181 | case test := <-abortChan: 182 | // Abort! 183 | delete(waiting, test.DependencyID) 184 | bcast(test) 185 | callback(&Test161JobResult{test, nil}) 186 | results += 1 187 | 188 | case test := <-readyChan: 189 | // We have a test that can run. 190 | delete(waiting, test.DependencyID) 191 | job := &test161Job{test, env, resChan} 192 | env.manager.SubmitChan <- job 193 | } 194 | } 195 | close(callbackChan) 196 | }() 197 | 198 | return callbackChan 199 | } 200 | -------------------------------------------------------------------------------- /runners_test.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | func runnerFromConfig(t *testing.T, config *GroupConfig, expected []string) TestRunner { 10 | 11 | // Create a test the group 12 | r, errs := TestRunnerFromConfig(config) 13 | assert.Equal(t, 0, len(errs)) 14 | if len(errs) > 0 { 15 | t.Log(errs) 16 | return nil 17 | } 18 | 19 | tg := r.Group() 20 | assert.NotNil(t, r.Group()) 21 | assert.Equal(t, len(expected), len(tg.Tests)) 22 | 23 | switch r.(type) { 24 | case *SimpleRunner: 25 | assert.False(t, config.UseDeps) 26 | case *DependencyRunner: 27 | assert.True(t, config.UseDeps) 28 | default: 29 | t.Errorf("Unexpected type for runner r") 30 | } 31 | 32 | // Make sure it has what we're expecting 33 | for _, id := range expected { 34 | test, ok := tg.Tests[id] 35 | assert.True(t, ok) 36 | if ok { 37 | assert.Equal(t, id, test.DependencyID) 38 | } 39 | } 40 | 41 | t.Log(tg) 42 | 43 | return r 44 | } 45 | 46 | func TestRunnerCapacity(t *testing.T) { 47 | t.Parallel() 48 | assert := assert.New(t) 49 | 50 | // Copy the default environment so we can have our own manager 51 | env := defaultEnv.CopyEnvironment() 52 | env.manager = newManager() 53 | env.RootDir = "./fixtures/root" 54 | 55 | expected := []string{ 56 | "boot.t", "threads/tt1.t", "threads/tt2.t", "threads/tt3.t", 57 | "sync/lt1.t", "sync/cvt1.t", "sync/semu1.t", 58 | } 59 | 60 | config := &GroupConfig{ 61 | Name: "Test", 62 | UseDeps: false, 63 | Tests: expected, 64 | Env: env, 65 | } 66 | 67 | caps := []uint{0, 1, 3, 5} 68 | 69 | for i := 0; i < 4; i++ { 70 | env.manager.Capacity = caps[i] 71 | r := runnerFromConfig(t, config, expected) 72 | 73 | env.manager.start() 74 | 75 | done := r.Run() 76 | count := 0 77 | 78 | for res := range done { 79 | assert.Nil(res.Err) 80 | assert.Equal(TEST_RESULT_CORRECT, res.Test.Result) 81 | if res.Test.Result != TEST_RESULT_CORRECT { 82 | t.Log(res.Err) 83 | t.Log(res.Test.OutputJSON()) 84 | t.Log(res.Test.OutputString()) 85 | } 86 | t.Log(fmt.Sprintf("test: %v status: %v", res.Test.DependencyID, res.Test.Result)) 87 | count += 1 88 | } 89 | 90 | assert.Equal(len(expected), count) 91 | assert.Equal(uint(len(expected)), env.manager.stats.Finished) 92 | 93 | if env.manager.Capacity > 0 { 94 | assert.True(env.manager.stats.HighRunning <= env.manager.Capacity) 95 | } 96 | 97 | t.Log(fmt.Sprintf("High count: %v High queue: %v Finished: %v", 98 | env.manager.stats.HighRunning, env.manager.stats.HighQueued, env.manager.stats.Finished)) 99 | 100 | env.manager.stop() 101 | } 102 | } 103 | 104 | func TestRunnerSimple(t *testing.T) { 105 | t.Parallel() 106 | assert := assert.New(t) 107 | 108 | env := defaultEnv.CopyEnvironment() 109 | env.manager = newManager() 110 | env.RootDir = "./fixtures/root" 111 | 112 | expected := []string{ 113 | "threads/tt1.t", "sync/lt1.t", 114 | } 115 | 116 | // Test config with dependencies 117 | config := &GroupConfig{ 118 | Name: "Test", 119 | UseDeps: false, 120 | Tests: expected, 121 | Env: env, 122 | } 123 | 124 | r := runnerFromConfig(t, config, expected) 125 | 126 | env.manager.start() 127 | 128 | done := r.Run() 129 | count := 0 130 | 131 | for res := range done { 132 | assert.Nil(res.Err) 133 | assert.Equal(TEST_RESULT_CORRECT, res.Test.Result) 134 | if TEST_RESULT_CORRECT != res.Test.Result { 135 | t.Log(res.Err) 136 | t.Log(res.Test.OutputJSON()) 137 | t.Log(res.Test.OutputString()) 138 | } 139 | count += 1 140 | } 141 | 142 | assert.Equal(len(expected), count) 143 | assert.Equal(uint(len(expected)), env.manager.stats.Finished) 144 | 145 | // Shut it down 146 | env.manager.stop() 147 | } 148 | 149 | func TestRunnerDependency(t *testing.T) { 150 | t.Parallel() 151 | assert := assert.New(t) 152 | 153 | expected := []string{ 154 | "boot.t", "threads/tt1.t", "threads/tt2.t", "threads/tt3.t", 155 | "sync/lt1.t", "sync/lt2.t", "sync/lt3.t", "sync/cvt1.t", 156 | } 157 | 158 | env := defaultEnv.CopyEnvironment() 159 | env.manager = newManager() 160 | env.RootDir = "./fixtures/root" 161 | 162 | config := &GroupConfig{ 163 | Name: "Test", 164 | UseDeps: true, 165 | Tests: []string{"sync/cvt1.t"}, 166 | Env: env, 167 | } 168 | 169 | r := runnerFromConfig(t, config, expected) 170 | env.manager.Capacity = 0 171 | env.manager.start() 172 | done := r.Run() 173 | 174 | results := make([]string, 0) 175 | count := 0 176 | 177 | for res := range done { 178 | assert.Nil(res.Err) 179 | assert.Equal(TEST_RESULT_CORRECT, res.Test.Result) 180 | if res.Test.Result != TEST_RESULT_CORRECT { 181 | t.Log(res.Err) 182 | t.Log(res.Test.OutputJSON()) 183 | t.Log(res.Test.OutputString()) 184 | } 185 | count += 1 186 | results = append(results, res.Test.DependencyID) 187 | } 188 | 189 | assert.Equal(len(expected), count) 190 | 191 | if len(expected) != count { 192 | t.FailNow() 193 | } 194 | 195 | // Boot has to be first, and since cvt1 depends on locks depends on threads, 196 | // cvt1 needs to be last and lt1 - lt3 needs to be before this. 197 | assert.Equal(expected[0], results[0]) 198 | 199 | threads := []bool{false, false, false} 200 | locks := []bool{false, false, false} 201 | 202 | // Check locks, they should be 1-3 203 | for i := 1; i <= 3; i++ { 204 | switch results[i] { 205 | case "threads/tt1.t": 206 | assert.False(threads[0]) 207 | threads[0] = true 208 | case "threads/tt2.t": 209 | assert.False(threads[1]) 210 | threads[1] = true 211 | case "threads/tt3.t": 212 | assert.False(threads[2]) 213 | threads[2] = true 214 | 215 | } 216 | } 217 | // Check locks, they should be 4-6 218 | for i := 4; i <= 6; i++ { 219 | switch results[i] { 220 | case "sync/lt1.t": 221 | assert.False(locks[0]) 222 | locks[0] = true 223 | case "sync/lt2.t": 224 | assert.False(locks[1]) 225 | locks[1] = true 226 | case "sync/lt3.t": 227 | assert.False(locks[2]) 228 | locks[2] = true 229 | } 230 | } 231 | 232 | // Now, check CV 233 | assert.Equal(expected[len(expected)-1], results[len(expected)-1]) 234 | 235 | env.manager.stop() 236 | } 237 | 238 | func TestRunnerAbort(t *testing.T) { 239 | t.Parallel() 240 | assert := assert.New(t) 241 | 242 | expected := []string{ 243 | "boot.t", "panics/panic.t", "panics/deppanic.t", 244 | } 245 | 246 | env := defaultEnv.CopyEnvironment() 247 | env.manager = newManager() 248 | env.RootDir = "./fixtures/root" 249 | 250 | config := &GroupConfig{ 251 | Name: "Test", 252 | UseDeps: true, 253 | Tests: []string{"panics/deppanic.t"}, 254 | Env: env, 255 | } 256 | 257 | r := runnerFromConfig(t, config, expected) 258 | env.manager.Capacity = 0 259 | env.manager.start() 260 | done := r.Run() 261 | 262 | count := 0 263 | for res := range done { 264 | assert.Nil(res.Err) 265 | assert.Equal(expected[count], res.Test.DependencyID) 266 | 267 | var expected TestResult 268 | 269 | switch count { 270 | case 0: // boot 271 | expected = TEST_RESULT_CORRECT 272 | case 1: // panic 273 | expected = TEST_RESULT_INCORRECT 274 | case 2: // deppanic 275 | expected = TEST_RESULT_SKIP 276 | } 277 | 278 | count += 1 279 | 280 | assert.Equal(expected, res.Test.Result) 281 | if expected != res.Test.Result { 282 | t.Log(res.Err) 283 | t.Log(res.Test.OutputJSON()) 284 | t.Log(res.Test.OutputString()) 285 | } 286 | } 287 | 288 | assert.Equal(len(expected), count) 289 | 290 | env.manager.stop() 291 | } 292 | 293 | func TestRunnersParallel(t *testing.T) { 294 | t.Parallel() 295 | assert := assert.New(t) 296 | 297 | env := defaultEnv.CopyEnvironment() 298 | env.manager = newManager() 299 | env.RootDir = "./fixtures/root" 300 | 301 | tests := [][]string{ 302 | []string{ 303 | "boot.t", "sync/lt1.t", "sync/lt2.t", "threads/tt1.t", 304 | }, 305 | []string{ 306 | "boot.t", "threads/tt1.t", "threads/tt3.t", "sync/lt1.t", "sync/sem1.t", 307 | }, 308 | []string{ 309 | "boot.t", "threads/tt1.t", "threads/tt3.t", "sync/lt2.t", "sync/lt1.t", "sync/sem1.t", 310 | }, 311 | []string{ 312 | "boot.t", "threads/tt1.t", "threads/tt3.t", "sync/lt1.t", "sync/cvt1.t", "sync/cvt2.t", "sync/cvt3.t", 313 | }, 314 | []string{ 315 | "boot.t", "threads/tt1.t", "threads/tt3.t", "sync/lt1.t", "sync/cvt2.t", "sync/cvt1.t", "sync/semu1.t", 316 | }, 317 | } 318 | 319 | runners := make([]TestRunner, 0, len(tests)) 320 | 321 | for _, group := range tests { 322 | config := &GroupConfig{ 323 | Name: "Test", 324 | UseDeps: false, 325 | Tests: group, 326 | Env: env, 327 | } 328 | r := runnerFromConfig(t, config, group) 329 | runners = append(runners, r) 330 | } 331 | 332 | syncChan := make(chan int) 333 | 334 | env.manager.Capacity = 10 335 | env.manager.start() 336 | 337 | for index, runner := range runners { 338 | go func(r TestRunner, i int) { 339 | done := r.Run() 340 | count := 0 341 | 342 | for res := range done { 343 | assert.Nil(res.Err) 344 | assert.Equal(TEST_RESULT_CORRECT, res.Test.Result) 345 | if res.Test.Result != TEST_RESULT_CORRECT { 346 | t.Log(res.Err) 347 | t.Log(res.Test.OutputJSON()) 348 | t.Log(res.Test.OutputString()) 349 | } 350 | 351 | count += 1 352 | } 353 | 354 | // Done with this test group 355 | assert.Equal(len(tests[i]), count) 356 | syncChan <- 1 357 | }(runner, index) 358 | } 359 | 360 | // Let all the workers finish 361 | for i, count := 0, len(runners); i < count; i++ { 362 | <-syncChan 363 | } 364 | 365 | env.manager.stop() 366 | 367 | if env.manager.Capacity > 0 { 368 | assert.True(env.manager.stats.HighRunning <= env.manager.Capacity) 369 | } 370 | 371 | t.Log(fmt.Sprintf("High count: %v High queue: %v Finished: %v", 372 | env.manager.stats.HighRunning, env.manager.stats.HighQueued, env.manager.stats.Finished)) 373 | 374 | } 375 | -------------------------------------------------------------------------------- /stats_test.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "strings" 6 | "testing" 7 | ) 8 | 9 | func TestStatsKernelDeadlock(t *testing.T) { 10 | t.Parallel() 11 | assert := assert.New(t) 12 | 13 | test, err := TestFromString("dl") 14 | assert.Nil(err) 15 | assert.Nil(test.MergeConf(TEST_DEFAULTS)) 16 | test.Monitor.Enabled = "true" 17 | test.Monitor.Kernel.EnableMin = "true" 18 | test.Monitor.ProgressTimeout = 8.0 19 | assert.Nil(test.Run(defaultEnv)) 20 | 21 | assert.Equal(len(test.Commands), 2) 22 | if len(test.Commands) == 2 { 23 | assert.Equal(test.Commands[0].Type, "kernel") 24 | assert.Equal(test.Commands[0].Input.Line, "boot") 25 | assert.Equal(test.Commands[1].Type, "kernel") 26 | assert.Equal(test.Commands[1].Input.Line, "dl") 27 | } 28 | 29 | assert.Equal(len(test.Status), 3) 30 | if len(test.Status) == 3 { 31 | assert.Equal(test.Status[0].Status, "started") 32 | assert.Equal(test.Status[1].Status, "monitor") 33 | assert.True(strings.HasPrefix(test.Status[1].Message, "insufficient kernel instructions")) 34 | assert.Equal(test.Status[2].Status, "shutdown") 35 | assert.True(strings.HasPrefix(test.Status[2].Message, "unexpected")) 36 | } 37 | 38 | assert.True(test.SimTime < 8.0) 39 | 40 | t.Log(test.OutputJSON()) 41 | t.Log(test.OutputString()) 42 | } 43 | 44 | func TestStatsKernelLivelock(t *testing.T) { 45 | t.Parallel() 46 | assert := assert.New(t) 47 | 48 | test, err := TestFromString("ll16") 49 | assert.Nil(err) 50 | assert.Nil(test.MergeConf(TEST_DEFAULTS)) 51 | test.Monitor.Enabled = "true" 52 | test.Monitor.Kernel.Max = 0.99 53 | test.Sys161.CPUs = 1 54 | test.Monitor.ProgressTimeout = 8.0 55 | assert.Nil(test.Run(defaultEnv)) 56 | 57 | assert.Equal(len(test.Commands), 2) 58 | if len(test.Commands) == 2 { 59 | assert.Equal(test.Commands[0].Type, "kernel") 60 | assert.Equal(test.Commands[0].Input.Line, "boot") 61 | assert.Equal(test.Commands[1].Type, "kernel") 62 | assert.Equal(test.Commands[1].Input.Line, "ll16") 63 | } 64 | 65 | assert.Equal(len(test.Status), 3) 66 | if len(test.Status) == 3 { 67 | assert.Equal(test.Status[0].Status, "started") 68 | assert.Equal(test.Status[1].Status, "monitor") 69 | assert.True(strings.HasPrefix(test.Status[1].Message, "too many kernel instructions")) 70 | assert.Equal(test.Status[2].Status, "shutdown") 71 | assert.True(strings.HasPrefix(test.Status[2].Message, "unexpected")) 72 | } 73 | 74 | assert.True(test.SimTime < 8.0) 75 | 76 | t.Log(test.OutputJSON()) 77 | t.Log(test.OutputString()) 78 | } 79 | 80 | func TestStatsUserDeadlock(t *testing.T) { 81 | t.Parallel() 82 | assert := assert.New(t) 83 | 84 | test, err := TestFromString("p /testbin/waiter") 85 | assert.Nil(err) 86 | assert.Nil(test.MergeConf(TEST_DEFAULTS)) 87 | test.Monitor.Enabled = "true" 88 | test.Monitor.Kernel.EnableMin = "false" 89 | test.Monitor.User.EnableMin = "true" 90 | test.Misc.PromptTimeout = 8.0 91 | assert.Nil(test.Run(defaultEnv)) 92 | 93 | assert.Equal(len(test.Commands), 2) 94 | if len(test.Commands) == 2 { 95 | assert.Equal(test.Commands[0].Type, "kernel") 96 | assert.Equal(test.Commands[0].Input.Line, "boot") 97 | assert.Equal(test.Commands[1].Type, "user") 98 | assert.Equal(test.Commands[1].Input.Line, "p /testbin/waiter") 99 | } 100 | 101 | assert.Equal(len(test.Status), 3) 102 | if len(test.Status) == 3 { 103 | assert.Equal(test.Status[0].Status, "started") 104 | assert.Equal(test.Status[1].Status, "monitor") 105 | assert.True(strings.HasPrefix(test.Status[1].Message, "insufficient user instructions")) 106 | assert.Equal(test.Status[2].Status, "shutdown") 107 | assert.True(strings.HasPrefix(test.Status[2].Message, "unexpected")) 108 | } 109 | 110 | assert.True(test.SimTime < 8.0) 111 | 112 | t.Log(test.OutputJSON()) 113 | t.Log(test.OutputString()) 114 | } 115 | 116 | func TestStatsKernelProgress(t *testing.T) { 117 | t.Parallel() 118 | assert := assert.New(t) 119 | 120 | test, err := TestFromString("ll1") 121 | assert.Nil(err) 122 | assert.Nil(test.MergeConf(TEST_DEFAULTS)) 123 | test.Monitor.Enabled = "true" 124 | test.Monitor.Kernel.EnableMin = "false" 125 | test.Monitor.User.EnableMin = "false" 126 | test.Monitor.ProgressTimeout = 2.0 127 | test.Misc.PromptTimeout = 10.0 128 | assert.Nil(test.Run(defaultEnv)) 129 | 130 | assert.Equal(len(test.Commands), 2) 131 | if len(test.Commands) == 2 { 132 | assert.Equal(test.Commands[0].Type, "kernel") 133 | assert.Equal(test.Commands[0].Input.Line, "boot") 134 | assert.Equal(test.Commands[1].Type, "kernel") 135 | assert.Equal(test.Commands[1].Input.Line, "ll1") 136 | } 137 | 138 | assert.Equal(len(test.Status), 3) 139 | if len(test.Status) == 3 { 140 | assert.Equal(test.Status[0].Status, "started") 141 | assert.Equal(test.Status[1].Status, "monitor") 142 | assert.True(strings.HasPrefix(test.Status[1].Message, "no progress")) 143 | assert.Equal(test.Status[2].Status, "shutdown") 144 | assert.True(strings.HasPrefix(test.Status[2].Message, "unexpected")) 145 | } 146 | 147 | assert.True(test.SimTime < 6.0) 148 | 149 | t.Log(test.OutputJSON()) 150 | t.Log(test.OutputString()) 151 | } 152 | 153 | func TestStatsKernelProgressOK(t *testing.T) { 154 | t.Parallel() 155 | assert := assert.New(t) 156 | 157 | test, err := TestFromString("tt3") 158 | assert.Nil(err) 159 | assert.Nil(test.MergeConf(TEST_DEFAULTS)) 160 | test.Monitor.Enabled = "true" 161 | test.Monitor.Kernel.EnableMin = "false" 162 | test.Monitor.User.EnableMin = "false" 163 | test.Monitor.ProgressTimeout = 1.0 164 | test.Misc.PromptTimeout = 120.0 165 | assert.Nil(test.Run(defaultEnv)) 166 | 167 | assert.Equal(len(test.Commands), 3) 168 | if len(test.Commands) == 3 { 169 | assert.Equal(test.Commands[0].Type, "kernel") 170 | assert.Equal(test.Commands[0].Input.Line, "boot") 171 | assert.Equal(test.Commands[1].Type, "kernel") 172 | assert.Equal(test.Commands[1].Input.Line, "tt3") 173 | assert.Equal(test.Commands[2].Type, "kernel") 174 | assert.Equal(test.Commands[2].Input.Line, "q") 175 | } 176 | 177 | assert.Equal(len(test.Status), 2) 178 | if len(test.Status) == 2 { 179 | assert.Equal(test.Status[0].Status, "started") 180 | assert.Equal(test.Status[1].Status, "shutdown") 181 | assert.True(strings.HasPrefix(test.Status[1].Message, "normal")) 182 | } 183 | 184 | assert.True(test.SimTime < 100.0) 185 | 186 | t.Log(test.OutputJSON()) 187 | t.Log(test.OutputString()) 188 | } 189 | 190 | func TestStatsUserProgress(t *testing.T) { 191 | t.Parallel() 192 | assert := assert.New(t) 193 | 194 | test, err := TestFromString("p /testbin/waiter") 195 | assert.Nil(err) 196 | assert.Nil(test.MergeConf(TEST_DEFAULTS)) 197 | test.Monitor.Enabled = "true" 198 | test.Monitor.Kernel.EnableMin = "false" 199 | test.Monitor.User.EnableMin = "false" 200 | test.Monitor.ProgressTimeout = 2.0 201 | test.Misc.PromptTimeout = 10.0 202 | assert.Nil(test.Run(defaultEnv)) 203 | 204 | assert.Equal(len(test.Commands), 2) 205 | if len(test.Commands) == 2 { 206 | assert.Equal(test.Commands[0].Type, "kernel") 207 | assert.Equal(test.Commands[0].Input.Line, "boot") 208 | assert.Equal(test.Commands[1].Type, "user") 209 | assert.Equal(test.Commands[1].Input.Line, "p /testbin/waiter") 210 | } 211 | 212 | assert.Equal(len(test.Status), 3) 213 | if len(test.Status) == 3 { 214 | assert.Equal(test.Status[0].Status, "started") 215 | assert.Equal(test.Status[1].Status, "monitor") 216 | assert.True(strings.HasPrefix(test.Status[1].Message, "no progress")) 217 | assert.Equal(test.Status[2].Status, "shutdown") 218 | assert.True(strings.HasPrefix(test.Status[2].Message, "unexpected")) 219 | } 220 | 221 | assert.True(test.SimTime < 6.0) 222 | 223 | t.Log(test.OutputJSON()) 224 | t.Log(test.OutputString()) 225 | } 226 | -------------------------------------------------------------------------------- /submission_test.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/stretchr/testify/require" 7 | "testing" 8 | ) 9 | 10 | func genTestSubmissionRequest(targetName string) *SubmissionRequest { 11 | req := &SubmissionRequest{ 12 | Target: targetName, 13 | Users: []*SubmissionUserInfo{ 14 | &SubmissionUserInfo{ 15 | Email: testStudent.Email, 16 | Token: testStudent.Token, 17 | }, 18 | }, 19 | Repository: "git@github.com:ops-class/os161.git", 20 | CommitID: "HEAD", 21 | ClientVersion: Version, 22 | } 23 | return req 24 | } 25 | 26 | func TestSubmissionRun(t *testing.T) { 27 | assert := assert.New(t) 28 | 29 | var err error 30 | 31 | req := genTestSubmissionRequest("simple") 32 | env := defaultEnv.CopyEnvironment() 33 | 34 | if testFlagDB { 35 | dialInfo := *mongoTestDialInfo 36 | dialInfo.Database = "test161" 37 | mongo, err := NewMongoPersistence(&dialInfo) 38 | assert.Nil(err) 39 | assert.NotNil(mongo) 40 | if err != nil { 41 | t.FailNow() 42 | } 43 | env.Persistence = mongo 44 | defer mongo.Close() 45 | } else { 46 | env.Persistence = &TestingPersistence{} 47 | } 48 | 49 | env.manager = newManager() 50 | 51 | s, errs := NewSubmission(req, env) 52 | assert.Equal(0, len(errs)) 53 | assert.NotNil(s) 54 | 55 | if s == nil || len(errs) > 0 { 56 | t.Log(errs) 57 | t.FailNow() 58 | } 59 | 60 | env.manager.start() 61 | 62 | err = s.Run() 63 | assert.Nil(err) 64 | 65 | assert.Equal(uint(50), s.Score) 66 | assert.True(len(s.OverlayCommitID) > 0) 67 | assert.True(isHexString(s.OverlayCommitID)) 68 | assert.True(s.IsStaff) 69 | 70 | students := retrieveTestStudent(env.Persistence) 71 | assert.Equal(1, len(students)) 72 | if len(students) != 1 { 73 | t.FailNow() 74 | } 75 | 76 | // The submission makes a copy of of env with a fresh keymap, so use that 77 | assert.True(len(s.Env.keyMap) > 0) 78 | t.Log(s.Env.keyMap) 79 | 80 | outputBytes, err := json.MarshalIndent(s, "", " ") 81 | if err != nil { 82 | t.Log(err) 83 | t.FailNow() 84 | } 85 | t.Log(string(outputBytes)) 86 | 87 | stat := students[0].getStat("simple") 88 | assert.NotNil(stat) 89 | assert.Equal(uint(50), stat.MaxScore) 90 | assert.Equal(uint(50), stat.HighScore) 91 | 92 | // Check test ids 93 | assert.True(len(s.ID) > 0) 94 | assert.Equal(s.ID, s.BuildTest.SubmissionID) 95 | for _, test := range s.Tests.Tests { 96 | assert.Equal(s.ID, test.SubmissionID) 97 | } 98 | 99 | env.manager.stop() 100 | } 101 | 102 | func TestMetaTargetSubmissionRun(t *testing.T) { 103 | assert := assert.New(t) 104 | require := require.New(t) 105 | 106 | var err error 107 | 108 | req := genTestSubmissionRequest("meta.2") 109 | env := defaultEnv.CopyEnvironment() 110 | 111 | if testFlagDB { 112 | dialInfo := *mongoTestDialInfo 113 | dialInfo.Database = "test161" 114 | mongo, err := NewMongoPersistence(&dialInfo) 115 | assert.Nil(err) 116 | assert.NotNil(mongo) 117 | if err != nil { 118 | t.FailNow() 119 | } 120 | env.Persistence = mongo 121 | defer mongo.Close() 122 | } else { 123 | env.Persistence = &TestingPersistence{} 124 | } 125 | 126 | env.manager = newManager() 127 | 128 | s, errs := NewSubmission(req, env) 129 | assert.Equal(0, len(errs)) 130 | assert.NotNil(s) 131 | 132 | if s == nil || len(errs) > 0 { 133 | t.Log(errs) 134 | t.FailNow() 135 | } 136 | 137 | // Test target name flip 138 | assert.Equal("metatest", s.TargetName) 139 | assert.Equal("meta.2", s.SubmittedTargetName) 140 | assert.Equal(uint(100), s.PointsAvailable) 141 | 142 | env.manager.start() 143 | 144 | // Note: lt1 will fail when running this from the base sources. 145 | err = s.Run() 146 | assert.Nil(err) 147 | 148 | assert.Equal(uint(25), s.Score) 149 | sub, ok := s.subSubmissions["meta.1"] 150 | require.True(ok) 151 | assert.Equal(uint(25), sub.Score) 152 | 153 | sub, ok = s.subSubmissions["meta.2"] 154 | require.True(ok) 155 | assert.Equal(uint(0), sub.Score) 156 | 157 | require.Equal(2, len(s.SubSubmissionIDs)) 158 | 159 | env.manager.stop() 160 | } 161 | 162 | func TestSubmissionMetaSplit(t *testing.T) { 163 | require := require.New(t) 164 | 165 | req := genTestSubmissionRequest("meta.2") 166 | 167 | s, errs := NewSubmission(req, defaultEnv) 168 | require.NotNil(s) 169 | require.Equal(0, len(errs)) 170 | 171 | splits := s.split() 172 | 173 | // Test target name flip 174 | require.Equal("metatest", s.TargetName) 175 | require.Equal("meta.2", s.SubmittedTargetName) 176 | require.Equal(uint(100), s.PointsAvailable) 177 | 178 | // Should have one for orig, and one for meta.1 179 | require.Equal(2, len(splits)) 180 | sMap := make(map[string]*Submission) 181 | for _, sub := range splits { 182 | sMap[sub.TargetName] = sub 183 | } 184 | 185 | sub, ok := sMap["meta.1"] 186 | require.True(ok) 187 | require.Equal(uint(25), sub.PointsAvailable) 188 | require.Equal(s.ID, sub.OrigSubmissionID) 189 | require.NotEqual(s.ID, sub.ID) 190 | 191 | sub, ok = sMap["meta.2"] 192 | require.True(ok) 193 | require.Equal(uint(75), sub.PointsAvailable) 194 | require.Equal(s.ID, sub.OrigSubmissionID) 195 | require.NotEqual(s.ID, sub.ID) 196 | } 197 | 198 | func retrieveTestStudent(persist PersistenceManager) []*Student { 199 | students := []*Student{} 200 | request := map[string]interface{}{ 201 | "email": testStudent.Email, 202 | "token": testStudent.Token, 203 | } 204 | 205 | persist.Retrieve(PERSIST_TYPE_STUDENTS, request, nil, &students) 206 | 207 | return students 208 | } 209 | 210 | func TestMetaTargetSubmissionRunMeta(t *testing.T) { 211 | assert := assert.New(t) 212 | require := require.New(t) 213 | 214 | var err error 215 | 216 | req := genTestSubmissionRequest("metatest") 217 | env := defaultEnv.CopyEnvironment() 218 | 219 | if testFlagDB { 220 | dialInfo := *mongoTestDialInfo 221 | dialInfo.Database = "test161" 222 | mongo, err := NewMongoPersistence(&dialInfo) 223 | assert.Nil(err) 224 | assert.NotNil(mongo) 225 | if err != nil { 226 | t.FailNow() 227 | } 228 | env.Persistence = mongo 229 | defer mongo.Close() 230 | } else { 231 | env.Persistence = &TestingPersistence{} 232 | } 233 | 234 | env.manager = newManager() 235 | 236 | s, errs := NewSubmission(req, env) 237 | assert.Equal(0, len(errs)) 238 | assert.NotNil(s) 239 | 240 | if s == nil || len(errs) > 0 { 241 | t.Log(errs) 242 | t.FailNow() 243 | } 244 | 245 | // Test target name flip 246 | assert.Equal("metatest", s.TargetName) 247 | assert.Equal("metatest", s.SubmittedTargetName) 248 | assert.Equal(uint(100), s.PointsAvailable) 249 | 250 | env.manager.start() 251 | 252 | // Note: lt1 will fail when running this from the base sources. 253 | err = s.Run() 254 | assert.Nil(err) 255 | 256 | assert.Equal(uint(25), s.Score) 257 | 258 | sub, ok := s.subSubmissions["meta.1"] 259 | require.True(ok) 260 | assert.Equal(uint(25), sub.Score) 261 | 262 | sub, ok = s.subSubmissions["meta.2"] 263 | require.True(ok) 264 | assert.Equal(uint(0), sub.Score) 265 | 266 | require.Equal(2, len(s.SubSubmissionIDs)) 267 | 268 | env.manager.stop() 269 | } 270 | -------------------------------------------------------------------------------- /tags.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | import ( 4 | "fmt" 5 | yaml "gopkg.in/yaml.v2" 6 | "io/ioutil" 7 | ) 8 | 9 | type TagDescription struct { 10 | Name string `yaml:"name"` 11 | Description string `yaml:"desc"` 12 | } 13 | 14 | // TagDescription Collection. We just use this for loading and move the 15 | // references into a map in the global environment. 16 | type TagDescriptions struct { 17 | Tags []*TagDescription `yaml:"tags"` 18 | } 19 | 20 | func TagDescriptionsFromFile(file string) (*TagDescriptions, error) { 21 | data, err := ioutil.ReadFile(file) 22 | if err != nil { 23 | return nil, fmt.Errorf("Error reading tags file %v: %v", file, err) 24 | } 25 | 26 | tags, err := TagDescriptionsFromString(string(data)) 27 | if err != nil { 28 | err = fmt.Errorf("Error loading tags file %v: %v", file, err) 29 | } 30 | return tags, err 31 | } 32 | 33 | func TagDescriptionsFromString(text string) (*TagDescriptions, error) { 34 | tags := &TagDescriptions{} 35 | err := yaml.Unmarshal([]byte(text), tags) 36 | 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | return tags, nil 42 | } 43 | -------------------------------------------------------------------------------- /tags_test.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | func TestTagDescriptionsLoad(t *testing.T) { 10 | t.Parallel() 11 | assert := assert.New(t) 12 | 13 | text := `tags: 14 | - name: tag1 15 | desc: "This is desc1" 16 | - name: tag2 17 | desc: "This is desc2" 18 | - name: tag3 19 | desc: "This is desc3" 20 | - name: tag4 21 | desc: "This is desc4" 22 | - name: tag5 23 | desc: "This is desc5" 24 | ` 25 | all, err := TagDescriptionsFromString(text) 26 | if err != nil { 27 | t.Log(err) 28 | t.FailNow() 29 | } 30 | 31 | assert.Equal(5, len(all.Tags)) 32 | if len(all.Tags) == 5 { 33 | for i, tag := range all.Tags { 34 | assert.Equal(fmt.Sprintf("tag%v", i+1), tag.Name) 35 | assert.Equal(fmt.Sprintf("This is desc%v", i+1), tag.Description) 36 | } 37 | } 38 | } 39 | 40 | func TestTagDescriptionsLoadFile(t *testing.T) { 41 | t.Parallel() 42 | assert := assert.New(t) 43 | 44 | assert.Equal(5, len(defaultEnv.Tags)) 45 | for i := 1; i <= 5; i++ { 46 | key := fmt.Sprintf("tag%v", i) 47 | tag, ok := defaultEnv.Tags[key] 48 | assert.True(ok) 49 | if ok { 50 | assert.Equal(key, tag.Name) 51 | assert.Equal(fmt.Sprintf("This is desc%v", i), tag.Description) 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /target_test.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "github.com/stretchr/testify/require" 6 | "testing" 7 | ) 8 | 9 | func TestTargetLoad(t *testing.T) { 10 | t.Parallel() 11 | assert := assert.New(t) 12 | 13 | text := `--- 14 | name: asst1 15 | points: 90 16 | type: asst 17 | tests: 18 | - id: sync/sem1.t 19 | points: 20 20 | - id: sync/lt1.t 21 | points: 30 22 | - id: sync/multi.t 23 | points: 40 24 | scoring: partial 25 | commands: 26 | - id: sem1 27 | points: 25 28 | - id: lt1 29 | points: 15 30 | ` 31 | target, err := TargetFromString(text) 32 | if err != nil { 33 | t.Log(err) 34 | t.FailNow() 35 | } 36 | 37 | assert.Equal(TARGET_ASST, target.Type) 38 | assert.Equal(3, len(target.Tests)) 39 | if len(target.Tests) == 3 { 40 | assert.Equal("sync/sem1.t", target.Tests[0].Id) 41 | assert.Equal(TEST_SCORING_ENTIRE, target.Tests[0].Scoring) 42 | assert.Equal(uint(20), target.Tests[0].Points) 43 | 44 | assert.Equal("sync/lt1.t", target.Tests[1].Id) 45 | assert.Equal(TEST_SCORING_ENTIRE, target.Tests[1].Scoring) 46 | assert.Equal(uint(30), target.Tests[1].Points) 47 | 48 | assert.Equal("sync/multi.t", target.Tests[2].Id) 49 | assert.Equal(TEST_SCORING_PARTIAL, target.Tests[2].Scoring) 50 | assert.Equal(uint(40), target.Tests[2].Points) 51 | } 52 | 53 | tg, errs := target.Instance(defaultEnv) 54 | assert.Equal(0, len(errs)) 55 | assert.NotNil(tg) 56 | t.Log(errs) 57 | t.Log(tg.OutputJSON()) 58 | 59 | } 60 | 61 | type expectedCmdResults struct { 62 | points uint 63 | status string 64 | } 65 | 66 | type expectedTestResults struct { 67 | points uint 68 | result TestResult 69 | cmdPoints map[string]*expectedCmdResults 70 | } 71 | 72 | func runTargetTest(t *testing.T, testPoints map[string]*expectedTestResults, targetId string) *TestGroup { 73 | assert := assert.New(t) 74 | t.Log(targetId) 75 | 76 | env := defaultEnv.CopyEnvironment() 77 | env.manager = newManager() 78 | env.RootDir = "./fixtures/root" 79 | env.manager.Capacity = 10 80 | 81 | target, ok := defaultEnv.Targets[targetId] 82 | assert.True(ok) 83 | 84 | if !ok { 85 | t.FailNow() 86 | } 87 | 88 | tg, errs := target.Instance(env) 89 | assert.NotNil(tg) 90 | if tg == nil { 91 | t.Log(errs) 92 | t.FailNow() 93 | } 94 | 95 | if len(target.MetaName) == 0 { 96 | assert.Equal(target.Points, tg.TotalPoints()) 97 | } else { 98 | // For metatarget subtargets, we need to sum the points for 99 | // all targets that will be run. 100 | totalPoints := target.Points 101 | for _, other := range target.previousSubTargets { 102 | totalPoints += other.Points 103 | } 104 | assert.Equal(totalPoints, tg.TotalPoints()) 105 | } 106 | 107 | totalExpected := uint(0) 108 | for _, v := range testPoints { 109 | totalExpected += v.points 110 | } 111 | 112 | // Run it and make sure we get the correct results 113 | env.manager.start() 114 | 115 | r := NewDependencyRunner(tg) 116 | done := r.Run() 117 | 118 | for res := range done { 119 | id := res.Test.DependencyID 120 | t.Log(id + " completed") 121 | if res.Err != nil { 122 | t.Log(res.Err) 123 | } 124 | exp, ok := testPoints[id] 125 | if !ok { 126 | assert.Equal(uint(0), res.Test.PointsEarned) 127 | assert.Equal(uint(0), res.Test.PointsAvailable) 128 | for _, c := range res.Test.Commands { 129 | assert.Equal(uint(0), c.PointsAvailable) 130 | assert.Equal(uint(0), c.PointsEarned) 131 | } 132 | } else { 133 | assert.Equal(exp.points, res.Test.PointsEarned) 134 | if exp.points != res.Test.PointsEarned { 135 | t.Log(res.Test.OutputJSON()) 136 | t.FailNow() 137 | } 138 | 139 | assert.Equal(string(exp.result), string(res.Test.Result)) 140 | for _, c := range res.Test.Commands { 141 | if cmd, ok2 := exp.cmdPoints[c.Id()]; !ok2 { 142 | assert.Equal(uint(0), c.PointsAvailable) 143 | assert.Equal(uint(0), c.PointsEarned) 144 | } else { 145 | assert.Equal(cmd.points, c.PointsEarned) 146 | assert.Equal(cmd.status, c.Status) 147 | } 148 | } 149 | } 150 | } 151 | 152 | env.manager.stop() 153 | 154 | assert.Equal(totalExpected, tg.EarnedPoints()) 155 | 156 | return tg 157 | } 158 | 159 | func getPartialTestPoints() map[string]*expectedTestResults { 160 | testPoints := make(map[string]*expectedTestResults) 161 | 162 | // sem1 163 | testPoints["sync/sem1.t"] = &expectedTestResults{ 164 | points: 10, 165 | result: TEST_RESULT_CORRECT, 166 | } 167 | 168 | // lt1 169 | testPoints["sync/lt1.t"] = &expectedTestResults{ 170 | points: 10, 171 | result: TEST_RESULT_CORRECT, 172 | } 173 | 174 | // fail 175 | testPoints["sync/fail.t"] = &expectedTestResults{ 176 | points: 10, 177 | result: TEST_RESULT_INCORRECT, 178 | cmdPoints: map[string]*expectedCmdResults{ 179 | "sem1": &expectedCmdResults{ 180 | status: COMMAND_STATUS_CORRECT, 181 | points: 10, 182 | }, 183 | "panic": &expectedCmdResults{ 184 | status: COMMAND_STATUS_INCORRECT, 185 | points: 0, 186 | }, 187 | "lt1": &expectedCmdResults{ 188 | status: COMMAND_STATUS_NONE, 189 | points: 0, 190 | }, 191 | "cvt1": &expectedCmdResults{ 192 | status: COMMAND_STATUS_NONE, 193 | points: 0, 194 | }, 195 | }, 196 | } 197 | 198 | return testPoints 199 | } 200 | 201 | func TestTargetScorePartial(t *testing.T) { 202 | // t.Parallel() 203 | testPoints := getPartialTestPoints() 204 | runTargetTest(t, testPoints, "partial") 205 | } 206 | 207 | func TestTargetScoreEntire(t *testing.T) { 208 | // t.Parallel() 209 | testPoints := make(map[string]*expectedTestResults) 210 | 211 | // sem1 212 | testPoints["sync/sem1.t"] = &expectedTestResults{ 213 | points: 10, 214 | result: TEST_RESULT_CORRECT, 215 | } 216 | 217 | // lt1 218 | testPoints["sync/lt1.t"] = &expectedTestResults{ 219 | points: 10, 220 | result: TEST_RESULT_CORRECT, 221 | } 222 | 223 | // fail 224 | testPoints["sync/fail.t"] = &expectedTestResults{ 225 | points: 0, 226 | result: TEST_RESULT_INCORRECT, 227 | cmdPoints: map[string]*expectedCmdResults{ 228 | "sem1": &expectedCmdResults{ 229 | status: COMMAND_STATUS_CORRECT, 230 | points: 0, 231 | }, 232 | "panic": &expectedCmdResults{ 233 | status: COMMAND_STATUS_INCORRECT, 234 | points: 0, 235 | }, 236 | "lt1": &expectedCmdResults{ 237 | status: COMMAND_STATUS_NONE, 238 | points: 0, 239 | }, 240 | "cvt1": &expectedCmdResults{ 241 | status: COMMAND_STATUS_NONE, 242 | points: 0, 243 | }, 244 | }, 245 | } 246 | 247 | runTargetTest(t, testPoints, "entire") 248 | } 249 | 250 | func TestTargetScoreFull(t *testing.T) { 251 | t.Parallel() 252 | testPoints := make(map[string]*expectedTestResults) 253 | 254 | // sem1 255 | testPoints["sync/sem1.t"] = &expectedTestResults{ 256 | points: 10, 257 | result: TEST_RESULT_CORRECT, 258 | } 259 | 260 | // lt1 261 | testPoints["sync/lt1.t"] = &expectedTestResults{ 262 | points: 10, 263 | result: TEST_RESULT_CORRECT, 264 | } 265 | 266 | // multi 267 | testPoints["sync/multi.t"] = &expectedTestResults{ 268 | points: 40, 269 | result: TEST_RESULT_CORRECT, 270 | cmdPoints: map[string]*expectedCmdResults{ 271 | "sem1": &expectedCmdResults{ 272 | status: COMMAND_STATUS_CORRECT, 273 | points: 0, 274 | }, 275 | "lt1": &expectedCmdResults{ 276 | status: COMMAND_STATUS_CORRECT, 277 | points: 0, 278 | }, 279 | }, 280 | } 281 | 282 | runTargetTest(t, testPoints, "full") 283 | } 284 | 285 | func TestMetaTargetLoad(t *testing.T) { 286 | t.Parallel() 287 | assert := assert.New(t) 288 | 289 | text := `--- 290 | name: asst0 291 | points: 100 292 | type: asst 293 | sub_target_names: [asst0.1, asst0.2, asst0.3] 294 | ` 295 | target, err := TargetFromString(text) 296 | if err != nil { 297 | t.Log(err) 298 | t.FailNow() 299 | } 300 | 301 | assert.Equal(3, len(target.SubTargetNames)) 302 | if len(target.Tests) == 3 { 303 | assert.Equal("asst0.1", target.SubTargetNames[0]) 304 | assert.Equal("asst0.2", target.SubTargetNames[1]) 305 | assert.Equal("asst0.3", target.SubTargetNames[2]) 306 | } 307 | 308 | tg, errs := target.Instance(defaultEnv) 309 | assert.Equal(1, len(errs)) 310 | assert.Nil(tg) 311 | } 312 | 313 | func TestMetaTargetLoadFile(t *testing.T) { 314 | assert := assert.New(t) 315 | 316 | target, ok := defaultEnv.Targets["metatest"] 317 | t.Log(defaultEnv.Targets) 318 | assert.True(ok) 319 | if !ok { 320 | t.FailNow() 321 | } 322 | 323 | assert.Equal(2, len(target.SubTargetNames)) 324 | if len(target.SubTargetNames) != 2 { 325 | t.FailNow() 326 | } 327 | assert.Equal("meta.1", target.SubTargetNames[0]) 328 | assert.Equal("meta.2", target.SubTargetNames[1]) 329 | assert.True(target.IsMetaTarget) 330 | } 331 | 332 | func TestMetaTargetLoadSubTarget(t *testing.T) { 333 | assert := assert.New(t) 334 | 335 | target, ok := defaultEnv.Targets["meta.2"] 336 | assert.True(ok) 337 | if !ok { 338 | t.FailNow() 339 | } 340 | 341 | assert.NotNil(target.metaTarget) 342 | if target.metaTarget == nil { 343 | t.FailNow() 344 | } 345 | assert.Equal(1, len(target.previousSubTargets)) 346 | assert.Equal("metatest", target.MetaName) 347 | assert.Equal("metatest", target.metaTarget.Name) 348 | 349 | assert.False(target.IsMetaTarget) 350 | 351 | tg, errs := target.Instance(defaultEnv) 352 | t.Log(errs) 353 | assert.Equal(0, len(errs)) 354 | assert.NotNil(tg) 355 | 356 | if tg == nil { 357 | t.FailNow() 358 | } 359 | for _, test := range tg.Tests { 360 | assert.True(len(test.requiredBy) > 0) 361 | t.Log(test.DependencyID, test.requiredBy) 362 | } 363 | } 364 | 365 | func TestMetaTargetRun(t *testing.T) { 366 | t.Parallel() 367 | assert := assert.New(t) 368 | 369 | testPoints := make(map[string]*expectedTestResults) 370 | 371 | // sem1 372 | testPoints["sync/sem1.t"] = &expectedTestResults{ 373 | points: 25, 374 | result: TEST_RESULT_CORRECT, 375 | } 376 | 377 | // lt1 378 | testPoints["sync/lt1.t"] = &expectedTestResults{ 379 | points: 75, 380 | result: TEST_RESULT_CORRECT, 381 | } 382 | 383 | tg := runTargetTest(t, testPoints, "meta.2") 384 | 385 | count := 0 386 | for _, test := range tg.Tests { 387 | if test.DependencyID == "sync/sem1.t" { 388 | assert.Equal("meta.1", test.TargetName) 389 | count += 1 390 | } else if test.DependencyID == "sync/lt1.t" { 391 | assert.Equal("meta.2", test.TargetName) 392 | count += 1 393 | } 394 | } 395 | 396 | assert.Equal(2, count) 397 | } 398 | 399 | func TestMetaTargetInconsistent(t *testing.T) { 400 | // Not parallel 401 | require := require.New(t) 402 | 403 | target := defaultEnv.Targets["metatest"] 404 | require.NotNil(target) 405 | 406 | err := target.initAsMetaTarget(defaultEnv) 407 | require.Nil(err) 408 | 409 | // Point total 410 | target.Points += 1 411 | err = target.initAsMetaTarget(defaultEnv) 412 | require.NotNil(err) 413 | target.Points -= 1 414 | err = target.initAsMetaTarget(defaultEnv) 415 | require.Nil(err) 416 | 417 | subtarget := defaultEnv.Targets["meta.2"] 418 | require.NotNil(subtarget) 419 | 420 | // Different userland requirement 421 | subtarget.RequiresUserland = true 422 | require.NotNil(target.initAsMetaTarget(defaultEnv)) 423 | subtarget.RequiresUserland = false 424 | require.Nil(target.initAsMetaTarget(defaultEnv)) 425 | 426 | // Different configs 427 | prev := subtarget.KConfig 428 | subtarget.KConfig += "111" 429 | require.NotNil(target.initAsMetaTarget(defaultEnv)) 430 | subtarget.KConfig = prev 431 | require.Nil(target.initAsMetaTarget(defaultEnv)) 432 | 433 | // Different types 434 | subtarget.Type = "perf" 435 | require.NotNil(target.initAsMetaTarget(defaultEnv)) 436 | subtarget.Type = "asst" 437 | require.Nil(target.initAsMetaTarget(defaultEnv)) 438 | 439 | } 440 | -------------------------------------------------------------------------------- /test161-server/.gitignore: -------------------------------------------------------------------------------- 1 | /test161-server 2 | -------------------------------------------------------------------------------- /test161-server/comm.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | ) 7 | 8 | const JsonHeader = "application/json; charset=UTF-8" 9 | const TextHeader = "text/plain; charset=UTF-8" 10 | 11 | func sendErrorCode(w http.ResponseWriter, code int, err error) { 12 | w.Header().Set("Content-Type", TextHeader) 13 | w.WriteHeader(code) 14 | fmt.Fprintf(w, "%v", err) 15 | } 16 | -------------------------------------------------------------------------------- /test161-server/control.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "github.com/ops-class/test161" 6 | "log" 7 | "net" 8 | "net/http" 9 | "net/rpc" 10 | "strconv" 11 | ) 12 | 13 | const ( 14 | CTRL_PAUSE = iota 15 | CTRL_RESUME 16 | CTRL_STATUS 17 | CTRL_SETCAPACITY 18 | CTRL_GETCAPACITY 19 | CTRL_STAFF_ONLY 20 | ) 21 | 22 | type ControlRequest struct { 23 | Message int 24 | NewCapacity uint 25 | } 26 | 27 | type ServerCtrl int 28 | 29 | func (sc *ServerCtrl) Control(msg *ControlRequest, reply *int) error { 30 | 31 | *reply = 0 32 | 33 | if submissionServer == nil || submissionServer.submissionMgr == nil { 34 | return errors.New("SubmissionManager is not initialized") 35 | } 36 | 37 | submissionMgr := submissionServer.submissionMgr 38 | 39 | switch msg.Message { 40 | case CTRL_PAUSE: 41 | submissionMgr.Pause() 42 | return nil 43 | case CTRL_RESUME: 44 | submissionMgr.Resume() 45 | return nil 46 | case CTRL_STAFF_ONLY: 47 | submissionMgr.SetStaffOnly() 48 | return nil 49 | case CTRL_STATUS: 50 | *reply = submissionMgr.Status() 51 | return nil 52 | case CTRL_GETCAPACITY: 53 | cap := test161.ManagerCapacity() 54 | *reply = int(cap) 55 | return nil 56 | case CTRL_SETCAPACITY: 57 | test161.SetManagerCapacity(msg.NewCapacity) 58 | *reply = 0 59 | return nil 60 | default: 61 | return errors.New("Unrecongnized control message") 62 | } 63 | } 64 | 65 | type ControlServer struct { 66 | } 67 | 68 | func (cs *ControlServer) Start() { 69 | server := rpc.NewServer() 70 | server.Register(new(ServerCtrl)) 71 | server.HandleHTTP("/test161/control", "/debug/test161/control") 72 | l, e := net.Listen("tcp", "127.0.0.1:4001") 73 | if e != nil { 74 | log.Fatal("listen error:", e) 75 | } 76 | http.Serve(l, nil) 77 | } 78 | 79 | func (server *ControlServer) Stop() { 80 | } 81 | 82 | func doCtrlRequest(msg interface{}, reply *int) error { 83 | 84 | req := ControlRequest{} 85 | 86 | switch msg.(type) { 87 | case int: 88 | req.Message = msg.(int) 89 | case ControlRequest: 90 | req = msg.(ControlRequest) 91 | default: 92 | return errors.New("Unexpected type in doCtrlRequest") 93 | } 94 | 95 | client, err := rpc.DialHTTPPath("tcp", "127.0.0.1:4001", "/test161/control") 96 | if err != nil { 97 | return err 98 | } 99 | 100 | // Synchronous call 101 | err = client.Call("ServerCtrl.Control", req, reply) 102 | return err 103 | } 104 | 105 | func CtrlPause() error { 106 | var reply int 107 | return doCtrlRequest(CTRL_PAUSE, &reply) 108 | } 109 | 110 | func CtrlResume() error { 111 | var reply int 112 | return doCtrlRequest(CTRL_RESUME, &reply) 113 | } 114 | 115 | func CtrlSetStaffOnly() error { 116 | var reply int 117 | return doCtrlRequest(CTRL_STAFF_ONLY, &reply) 118 | } 119 | 120 | func CtrlStatus() (int, error) { 121 | var reply int 122 | err := doCtrlRequest(CTRL_STATUS, &reply) 123 | return reply, err 124 | } 125 | 126 | func CtrlGetCapacity() (int, error) { 127 | var reply int 128 | err := doCtrlRequest(CTRL_GETCAPACITY, &reply) 129 | return reply, err 130 | } 131 | 132 | func CtrlSetCapacity(argCap string) error { 133 | newCap, err := strconv.Atoi(argCap) 134 | if err != nil { 135 | return err 136 | } 137 | 138 | var reply int 139 | err = doCtrlRequest(ControlRequest{ 140 | Message: CTRL_SETCAPACITY, 141 | NewCapacity: uint(newCap), 142 | }, &reply) 143 | 144 | return err 145 | } 146 | -------------------------------------------------------------------------------- /test161-server/routes.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/gorilla/mux" 5 | "net/http" 6 | "time" 7 | ) 8 | 9 | // API Goo 10 | 11 | type Route struct { 12 | Name string 13 | Method string 14 | Pattern string 15 | HandlerFunc http.HandlerFunc 16 | } 17 | 18 | var routes = []Route{ 19 | Route{ 20 | "Usage", 21 | "GET", 22 | "/api-v1/", 23 | apiUsage, 24 | }, 25 | Route{ 26 | "Submit", 27 | "POST", 28 | "/api-v1/submit", 29 | createSubmission, 30 | }, 31 | Route{ 32 | "ListTargets", 33 | "GET", 34 | "/api-v1/targets", 35 | listTargets, 36 | }, 37 | Route{ 38 | "stats", 39 | "GET", 40 | "/api-v1/stats", 41 | getStats, 42 | }, 43 | Route{ 44 | "keygen", 45 | "POST", 46 | "/api-v1/keygen", 47 | keygen, 48 | }, 49 | Route{ 50 | "validate", 51 | "POST", 52 | "/api-v1/validate", 53 | validateSubmission, 54 | }, 55 | Route{ 56 | "upload", 57 | "POST", 58 | "/api-v1/upload", 59 | uploadFiles, 60 | }, 61 | } 62 | 63 | func NewRouter() *mux.Router { 64 | 65 | router := mux.NewRouter().StrictSlash(true) 66 | for _, route := range routes { 67 | var handler http.Handler 68 | 69 | handler = route.HandlerFunc 70 | handler = Logger(handler, route.Name) 71 | 72 | router. 73 | Methods(route.Method). 74 | Path(route.Pattern). 75 | Name(route.Name). 76 | Handler(handler) 77 | } 78 | 79 | return router 80 | } 81 | 82 | func Logger(inner http.Handler, name string) http.Handler { 83 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 84 | start := time.Now() 85 | 86 | inner.ServeHTTP(w, r) 87 | 88 | logger.Printf( 89 | "%s\t%s\t%s\t%s", 90 | r.Method, 91 | r.RequestURI, 92 | name, 93 | time.Since(start), 94 | ) 95 | }) 96 | } 97 | -------------------------------------------------------------------------------- /test161-server/server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/ops-class/test161" 7 | "os" 8 | "os/signal" 9 | ) 10 | 11 | // test161 Submission Server 12 | 13 | // Run cleanup when signal is received 14 | type test161Server interface { 15 | Start() 16 | Stop() 17 | } 18 | 19 | var servers = []test161Server{} 20 | 21 | // Modified from http://nathanleclaire.com/blog/2014/08/24/handling-ctrl-c-interrupt-signal-in-golang-programs/ 22 | func waitForSignal() { 23 | signalChan := make(chan os.Signal, 1) 24 | doneChan := make(chan bool) 25 | signal.Notify(signalChan, os.Interrupt) 26 | signal.Notify(signalChan, os.Kill) 27 | 28 | go func() { 29 | for _ = range signalChan { 30 | for _, s := range servers { 31 | s.Stop() 32 | } 33 | fmt.Println("Killing...") 34 | doneChan <- true 35 | } 36 | }() 37 | 38 | <-doneChan 39 | } 40 | 41 | func main() { 42 | // TODO: Usage 43 | 44 | if len(os.Args) > 1 { 45 | var err error 46 | var status int 47 | 48 | switch os.Args[1] { 49 | case "status": 50 | status, err = CtrlStatus() 51 | if err == nil { 52 | switch status { 53 | case test161.SM_ACCEPTING: 54 | fmt.Println("test161 server: accepting submissions") 55 | case test161.SM_NOT_ACCEPTING: 56 | fmt.Println("test161 server: not accepting submissions") 57 | case test161.SM_STAFF_ONLY: 58 | fmt.Println("test161 server: accepting submissions for staff only") 59 | default: 60 | fmt.Printf("test161 server: unknown status %v\n", status) 61 | } 62 | } 63 | case "pause": 64 | err = CtrlPause() 65 | case "resume": 66 | err = CtrlResume() 67 | case "staff-only": 68 | err = CtrlSetStaffOnly() 69 | case "set-capacity": 70 | if len(os.Args) != 3 { 71 | err = errors.New("Wrong number of arguments to set-capacity") 72 | } else { 73 | err = CtrlSetCapacity(os.Args[2]) 74 | } 75 | case "get-capacity": 76 | var capacity int 77 | capacity, err = CtrlGetCapacity() 78 | if err == nil { 79 | fmt.Println("Current test capacity:", capacity) 80 | } 81 | case "version": 82 | fmt.Printf("test161-server version: %v\n", test161.Version) 83 | err = nil 84 | 85 | default: 86 | fmt.Println("Unknown command:", os.Args[1]) 87 | os.Exit(2) 88 | } 89 | 90 | if err != nil { 91 | fmt.Println("Error processing request:", err) 92 | os.Exit(1) 93 | } else { 94 | os.Exit(0) 95 | } 96 | } 97 | 98 | // Create Submission Server 99 | server, err := NewSubmissionServer() 100 | if err != nil { 101 | fmt.Println("Error creating submission server:", err) 102 | return 103 | } 104 | servers = append(servers, server) 105 | submissionServer = server.(*SubmissionServer) 106 | 107 | ctrl := &ControlServer{} 108 | servers = append(servers, ctrl) 109 | 110 | for _, s := range servers { 111 | go s.Start() 112 | } 113 | 114 | waitForSignal() 115 | } 116 | -------------------------------------------------------------------------------- /test161-server/uploads.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "compress/gzip" 6 | "encoding/json" 7 | "errors" 8 | "fmt" 9 | "github.com/ops-class/test161" 10 | "mime/multipart" 11 | "net/http" 12 | ) 13 | 14 | type UploadHandler interface { 15 | HandleFile(*multipart.FileHeader, *test161.UploadRequest, []*test161.Student) error 16 | FileComplete(error) 17 | } 18 | 19 | type UploadFileReader struct { 20 | Compressed bool 21 | LineChan chan string 22 | } 23 | 24 | func NewUploadFileReader(gzipped bool, lineChan chan string) *UploadFileReader { 25 | return &UploadFileReader{ 26 | Compressed: gzipped, 27 | LineChan: lineChan, 28 | } 29 | } 30 | 31 | type UploadTypeManager interface { 32 | test161Server 33 | GetFileHandler() UploadHandler 34 | } 35 | 36 | func (handler *UploadFileReader) HandleFile(header *multipart.FileHeader) error { 37 | file, err := header.Open() 38 | if err != nil { 39 | return err 40 | } 41 | 42 | defer file.Close() 43 | 44 | var scanner *bufio.Scanner 45 | 46 | if handler.Compressed { 47 | if gz, err := gzip.NewReader(file); err != nil { 48 | return err 49 | } else { 50 | scanner = bufio.NewScanner(gz) 51 | defer gz.Close() 52 | } 53 | } else { 54 | scanner = bufio.NewScanner(file) 55 | } 56 | 57 | for scanner.Scan() { 58 | handler.LineChan <- scanner.Text() 59 | } 60 | 61 | return scanner.Err() 62 | } 63 | 64 | const MAX_FILE_DOWNLOAD = 50 * 1024 * 1024 65 | 66 | // Upload manager/handlers. Currently we just have one (stats). 67 | type UploadHandlerFactory func() UploadHandler 68 | 69 | var uploadManagers map[int]UploadHandlerFactory 70 | 71 | func initUploadManagers() { 72 | uploadManagers = make(map[int]UploadHandlerFactory) 73 | uploadManagers[test161.UPLOAD_TYPE_USAGE] = GetUsageStatHandler 74 | } 75 | 76 | func uploadFiles(w http.ResponseWriter, r *http.Request) { 77 | 78 | // The upload request has 2 pieces: 79 | // 1) A json string with user info so we can validate users 80 | // 2) A list of files and their contents 81 | // 82 | // If we are able to validate the users, we'll process the file contents. 83 | 84 | var req test161.UploadRequest 85 | var students []*test161.Student 86 | var err error 87 | 88 | if err = r.ParseMultipartForm(MAX_FILE_DOWNLOAD); err != nil { 89 | logger.Println("Error parsing multi-part form:", err) 90 | sendErrorCode(w, http.StatusBadRequest, err) 91 | return 92 | } 93 | 94 | if data, ok := r.MultipartForm.Value["request"]; !ok { 95 | logger.Println("request field not found") 96 | sendErrorCode(w, http.StatusBadRequest, errors.New("Missing form field: request")) 97 | return 98 | } else if len(data) != 1 { 99 | logger.Println("Invalid request field. Length != 1: ", data) 100 | sendErrorCode(w, http.StatusNotAcceptable, errors.New("Invalid request field. Length != 1")) 101 | return 102 | } else { 103 | json.Unmarshal([]byte(data[0]), &req) 104 | if students, err = req.Validate(submissionServer.GetEnv()); err != nil { 105 | sendErrorCode(w, 422, err) 106 | return 107 | } 108 | } 109 | 110 | var handler UploadHandler 111 | 112 | if generator, ok := uploadManagers[req.UploadType]; !ok { 113 | sendErrorCode(w, http.StatusBadRequest, fmt.Errorf("Invalid upload type %v: ", req.UploadType)) 114 | logger.Println("Invalid upload type request:", req.UploadType) 115 | return 116 | } else { 117 | handler = generator() 118 | } 119 | 120 | for fname, headers := range r.MultipartForm.File { 121 | fmt.Println("Processing", fname+"...") 122 | for _, fheader := range headers { 123 | err := handler.HandleFile(fheader, &req, students) 124 | handler.FileComplete(err) 125 | } 126 | } 127 | 128 | r.MultipartForm.RemoveAll() 129 | 130 | w.Header().Set("Content-Type", JsonHeader) 131 | w.WriteHeader(http.StatusOK) 132 | } 133 | -------------------------------------------------------------------------------- /test161-server/usage.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/ops-class/test161" 7 | "io" 8 | "mime/multipart" 9 | "os" 10 | "path" 11 | "time" 12 | ) 13 | 14 | var usageFailDir string 15 | 16 | func getFailedUsageFileName() string { 17 | res := path.Join(usageFailDir, fmt.Sprintf("usage_%v.json.gz", time.Now().Unix())) 18 | return res 19 | } 20 | 21 | type UsageStatsFileHandler struct { 22 | hasError bool 23 | header *multipart.FileHeader 24 | } 25 | 26 | func (handler *UsageStatsFileHandler) HandleFile(header *multipart.FileHeader, 27 | req *test161.UploadRequest, students []*test161.Student) error { 28 | 29 | // We'll defer grunt work to upload reader and just collect inflated lines. 30 | lineChan := make(chan string) 31 | reader := NewUploadFileReader(true, lineChan) 32 | handler.header = header 33 | handler.hasError = false 34 | staff := false 35 | env := submissionServer.GetEnv() 36 | 37 | if len(students) > 0 { 38 | staff, _ = students[0].IsStaff(env) 39 | } 40 | 41 | // We'll replace the user info with what comes in the validated request. 42 | users := make([]string, 0) 43 | for _, u := range req.Users { 44 | users = append(users, u.Email) 45 | } 46 | 47 | go func() { 48 | for line := range lineChan { 49 | // Each line is a single usage log 50 | var usageStat test161.UsageStat 51 | 52 | if err := json.Unmarshal([]byte(line), &usageStat); err != nil { 53 | handler.hasError = true 54 | logger.Println("Invalid usage JSON:", line) 55 | } else { 56 | usageStat.Users = users 57 | usageStat.IsStaff = staff 58 | if err = usageStat.Persist(env); err != nil { 59 | logger.Println("Error saving stat:", err) 60 | } 61 | } 62 | } 63 | }() 64 | 65 | return reader.HandleFile(header) 66 | } 67 | 68 | func (handler *UsageStatsFileHandler) tryCopyFile() { 69 | 70 | file, err := handler.header.Open() 71 | if err != nil { 72 | logger.Println("Failed to open the file for reading") 73 | } 74 | defer file.Close() 75 | 76 | outFile := getFailedUsageFileName() 77 | out, err := os.Create(outFile) 78 | if err != nil { 79 | logger.Printf("Failed to open '%v' file for writing.\n", outFile) 80 | return 81 | } 82 | defer out.Close() 83 | 84 | _, err = io.Copy(out, file) 85 | if err != nil { 86 | logger.Println("Failed to copy file:", err) 87 | } 88 | } 89 | 90 | func (handler *UsageStatsFileHandler) FileComplete(err error) { 91 | if err != nil || handler.hasError { 92 | handler.tryCopyFile() 93 | } 94 | } 95 | 96 | // Generator function for usage stats uploads 97 | func GetUsageStatHandler() UploadHandler { 98 | return &UsageStatsFileHandler{} 99 | } 100 | -------------------------------------------------------------------------------- /test161/.gitignore: -------------------------------------------------------------------------------- 1 | /test161 2 | fixtures 3 | -------------------------------------------------------------------------------- /test161/comm.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/parnurzeal/gorequest" 6 | "net/http" 7 | ) 8 | 9 | type PostEndpoint string 10 | 11 | const ( 12 | ApiEndpointSubmit PostEndpoint = "/api-v1/submit" 13 | ApiEndpointValidate = "/api-v1/validate" 14 | ApiEndpointUpload = "/api-v1/upload" 15 | ) 16 | 17 | type SupportedPostType string 18 | 19 | const ( 20 | PostTypeJSON SupportedPostType = "json" 21 | PostTypeMultipart = "multipart" 22 | PostTypeText = "text" 23 | ) 24 | 25 | type PostRequest struct { 26 | Endpoint string 27 | SA *gorequest.SuperAgent 28 | } 29 | 30 | func NewPostRequest(endpoint PostEndpoint) *PostRequest { 31 | ep := clientConf.Server + string(endpoint) 32 | 33 | return &PostRequest{ 34 | Endpoint: ep, 35 | SA: gorequest.New().Post(ep), 36 | } 37 | } 38 | 39 | func (pr *PostRequest) SetType(t SupportedPostType) { 40 | pr.SA.Type(string(t)) 41 | } 42 | 43 | func (pr *PostRequest) QueueJSON(obj interface{}, fieldname string) error { 44 | if bytes, err := json.Marshal(obj); err != nil { 45 | return err 46 | } else { 47 | // A limitiation here is that we can't send complicated JSON objects. 48 | // For example, sending an obj with a list as a field truncates things. 49 | // Instead, we'll send a single field (a=b) where (a) is the expected 50 | // field name, and (b) is a JSON string. 51 | // 52 | // For compatibility sake, if fieldname is empty, we'll just send the 53 | // JSON string. 54 | if len(fieldname) > 0 { 55 | pr.SA.Send(fieldname + "=" + string(bytes)) 56 | } else { 57 | pr.SA.Send(string(bytes)) 58 | } 59 | return nil 60 | } 61 | } 62 | 63 | func (pr *PostRequest) QueueFile(fileOnDisk, fileFieldName string) { 64 | pr.SA.SendFile(fileOnDisk, "", fileFieldName) 65 | } 66 | 67 | func (pr *PostRequest) Submit() (*http.Response, string, []error) { 68 | return pr.SA.End() 69 | } 70 | -------------------------------------------------------------------------------- /test161/consolepersist.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/ops-class/test161" 6 | "strings" 7 | ) 8 | 9 | // A PersistenceManager that "persists" to the console 10 | 11 | type ConsolePersistence struct { 12 | width int 13 | } 14 | 15 | func (c *ConsolePersistence) Close() { 16 | } 17 | 18 | const lineFmt = "[%-*v]\t%.6f\t%s" 19 | 20 | func (c *ConsolePersistence) Notify(entity interface{}, msg, what int) error { 21 | 22 | if msg == test161.MSG_PERSIST_UPDATE && what == test161.MSG_FIELD_OUTPUT { 23 | 24 | switch entity.(type) { 25 | case *test161.Command: 26 | { 27 | cmd := entity.(*test161.Command) 28 | line := cmd.Output[len(cmd.Output)-1] 29 | // Learn the width on the fly (submissions) 30 | if c.width < len(cmd.Test.DependencyID) { 31 | c.width = len(cmd.Test.DependencyID) 32 | } 33 | output := fmt.Sprintf(lineFmt, c.width, cmd.Test.DependencyID, line.SimTime, line.Line) 34 | fmt.Println(output) 35 | } 36 | case *test161.BuildCommand: 37 | { 38 | cmd := entity.(*test161.BuildCommand) 39 | for _, line := range cmd.Output { 40 | output := fmt.Sprintf("%.6f\t%s", line.SimTime, line.Line) 41 | fmt.Println(output) 42 | } 43 | } 44 | } 45 | } else if msg == test161.MSG_PERSIST_UPDATE && what == test161.MSG_FIELD_STATUSES { 46 | switch entity.(type) { 47 | case *test161.Test: 48 | { 49 | test := entity.(*test161.Test) 50 | index := len(test.Status) - 1 51 | if index > 0 { 52 | status := test.Status[index] 53 | str := fmt.Sprintf(lineFmt, c.width, test.DependencyID, test.SimTime, status.Status) 54 | if status.Message != "" { 55 | str += fmt.Sprintf(": %s", status.Message) 56 | } 57 | fmt.Println(str) 58 | } 59 | } 60 | } 61 | } else if msg == test161.MSG_PERSIST_UPDATE && what == test161.MSG_FIELD_STATUS { 62 | switch entity.(type) { 63 | case *test161.Test: 64 | { 65 | test := entity.(*test161.Test) 66 | if test.Result == test161.TEST_RESULT_RUNNING { 67 | lines := strings.Split(strings.TrimSpace(test.ConfString), "\n") 68 | for _, line := range lines { 69 | output := fmt.Sprintf(lineFmt, c.width, test.DependencyID, 0.0, line) 70 | fmt.Println(output) 71 | } 72 | } 73 | } 74 | } 75 | } 76 | 77 | return nil 78 | } 79 | 80 | func (d *ConsolePersistence) Retrieve(what int, who map[string]interface{}, 81 | filter map[string]interface{}, res interface{}) error { 82 | return nil 83 | } 84 | 85 | func (d *ConsolePersistence) CanRetrieve() bool { 86 | return false 87 | } 88 | -------------------------------------------------------------------------------- /test161/errors.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | color "gopkg.in/fatih/color.v0" 7 | "os" 8 | ) 9 | 10 | var errText = color.New(color.Bold).SprintFunc()("Error:") 11 | 12 | func __printRunError(err error) { 13 | fmt.Fprintf(os.Stderr, "%v %v\n", errText, err) 14 | } 15 | 16 | func printRunError(err error) { 17 | __printRunError(err) 18 | } 19 | 20 | func printRunErrors(errs []error) { 21 | if len(errs) > 0 { 22 | for _, e := range errs { 23 | __printRunError(e) 24 | } 25 | } 26 | } 27 | 28 | func connectionError(endpoint string, errs []error) []error { 29 | msg := fmt.Sprintf("Unable to connect to server '%v'", endpoint) 30 | for _, e := range errs { 31 | msg += fmt.Sprintf("\n %v", e) 32 | } 33 | return []error{errors.New(msg)} 34 | } 35 | -------------------------------------------------------------------------------- /test161/fixtures/files/usage.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ops-class/test161/c05cbd4b6ee6cf04978cb69d1ada446b59ca27d7/test161/fixtures/files/usage.json.gz -------------------------------------------------------------------------------- /test161/fixtures/files/usage_013999999999.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ops-class/test161/c05cbd4b6ee6cf04978cb69d1ada446b59ca27d7/test161/fixtures/files/usage_013999999999.json.gz -------------------------------------------------------------------------------- /test161/fixtures/files/usage_014999999999.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ops-class/test161/c05cbd4b6ee6cf04978cb69d1ada446b59ca27d7/test161/fixtures/files/usage_014999999999.json.gz -------------------------------------------------------------------------------- /test161/listcmd.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "flag" 7 | "fmt" 8 | "github.com/ops-class/test161" 9 | "github.com/parnurzeal/gorequest" 10 | color "gopkg.in/fatih/color.v0" 11 | "net/http" 12 | "os" 13 | "sort" 14 | "strings" 15 | ) 16 | 17 | var listRemoteFlag bool 18 | 19 | var ( 20 | listTagsShort bool 21 | listTagsList []string 22 | ) 23 | 24 | func doListCommand() int { 25 | if len(os.Args) < 3 { 26 | fmt.Fprintf(os.Stderr, "Missing argument to list command\n") 27 | return 1 28 | } 29 | 30 | switch os.Args[2] { 31 | case "targets": 32 | return doListTargets() 33 | case "tags": 34 | return doListTags() 35 | case "tests": 36 | return doListTests() 37 | case "all": 38 | return doListAll() 39 | case "tagnames": 40 | return doListTagnames() 41 | default: 42 | fmt.Fprintf(os.Stderr, "Invalid option to 'test161 list'. Must be one of (targets, tags, tests)\n") 43 | return 1 44 | } 45 | } 46 | 47 | type targetsByName []*test161.TargetListItem 48 | 49 | func (t targetsByName) Len() int { return len(t) } 50 | func (t targetsByName) Swap(i, j int) { t[i], t[j] = t[j], t[i] } 51 | func (t targetsByName) Less(i, j int) bool { return t[i].Name < t[j].Name } 52 | 53 | func doListTargets() int { 54 | if err := getListArgs(); err != nil { 55 | printRunError(err) 56 | return 1 57 | } 58 | 59 | var targets *test161.TargetList 60 | 61 | if listRemoteFlag { 62 | var errs []error 63 | if targets, errs = getRemoteTargets(); len(errs) > 0 { 64 | printRunErrors(errs) 65 | return 1 66 | } 67 | } else { 68 | targets = env.TargetList() 69 | } 70 | 71 | sort.Sort(targetsByName(targets.Targets)) 72 | 73 | printTargets(targets) 74 | 75 | return 0 76 | } 77 | 78 | func getRemoteTargets() (*test161.TargetList, []error) { 79 | if len(clientConf.Server) == 0 { 80 | return nil, []error{errors.New("server field missing in .test161.conf")} 81 | } 82 | 83 | endpoint := clientConf.Server + "/api-v1/targets" 84 | request := gorequest.New() 85 | 86 | resp, body, errs := request.Get(endpoint).End() 87 | if errs != nil { 88 | return nil, connectionError(endpoint, errs) 89 | } 90 | 91 | if resp.StatusCode != http.StatusOK { 92 | return nil, []error{fmt.Errorf("Unable to retrieve remote targets: %v", resp.Status)} 93 | } 94 | 95 | targets := &test161.TargetList{} 96 | 97 | if err := json.Unmarshal([]byte(body), targets); err != nil { 98 | return nil, []error{err} 99 | } 100 | 101 | return targets, nil 102 | } 103 | 104 | func printTargets(list *test161.TargetList) { 105 | var desc string 106 | if listRemoteFlag { 107 | desc = "Remote Target" 108 | } else { 109 | desc = "Local Target" 110 | } 111 | 112 | pd := &PrintData{ 113 | Headings: []*Heading{ 114 | &Heading{ 115 | Text: desc, 116 | MinWidth: 20, 117 | }, 118 | &Heading{ 119 | Text: "Type", 120 | }, 121 | &Heading{ 122 | Text: "Version", 123 | }, 124 | &Heading{ 125 | Text: "Points", 126 | RightJustified: true, 127 | }, 128 | }, 129 | Rows: make(Rows, 0), 130 | Config: defaultPrintConf, 131 | } 132 | 133 | for _, t := range list.Targets { 134 | row := []*Cell{ 135 | &Cell{Text: t.Name}, 136 | &Cell{Text: t.Type}, 137 | &Cell{Text: fmt.Sprintf("v%v", t.Version)}, 138 | &Cell{Text: fmt.Sprintf("%v", t.Points)}, 139 | } 140 | pd.Rows = append(pd.Rows, row) 141 | } 142 | 143 | fmt.Println() 144 | pd.Print() 145 | fmt.Println() 146 | } 147 | 148 | func getListArgs() error { 149 | 150 | listFlags := flag.NewFlagSet("test161 list-targets", flag.ExitOnError) 151 | listFlags.Usage = usage 152 | listFlags.BoolVar(&listRemoteFlag, "remote", false, "") 153 | listFlags.BoolVar(&listRemoteFlag, "r", false, "") 154 | 155 | listFlags.Parse(os.Args[3:]) // this may exit 156 | 157 | if len(listFlags.Args()) > 0 { 158 | return errors.New("test161 list-targets does not support positional arguments") 159 | } 160 | 161 | return nil 162 | } 163 | 164 | func getTagArgs() error { 165 | flags := flag.NewFlagSet("test161 list-tags", flag.ExitOnError) 166 | flags.Usage = usage 167 | flags.BoolVar(&listTagsShort, "short", false, "") 168 | flags.BoolVar(&listTagsShort, "s", false, "") 169 | 170 | flags.Parse(os.Args[3:]) // this may exit 171 | listTagsList = flags.Args() 172 | 173 | return nil 174 | 175 | } 176 | 177 | func getAllTests() ([]*test161.Test, []error) { 178 | conf := &test161.GroupConfig{ 179 | Tests: []string{"**/*.t"}, 180 | Env: env, 181 | } 182 | 183 | tg, errs := test161.GroupFromConfig(conf) 184 | if len(errs) > 0 { 185 | return nil, errs 186 | } 187 | 188 | // Sort the tests by ID 189 | tests := make([]*test161.Test, 0) 190 | for _, t := range tg.Tests { 191 | tests = append(tests, t) 192 | } 193 | sort.Sort(testsByID(tests)) 194 | 195 | return tests, nil 196 | } 197 | 198 | // Hidden option for autocomplete 199 | func doListTagnames() int { 200 | // Load every test file 201 | tests, errs := getAllTests() 202 | if len(errs) > 0 { 203 | printRunErrors(errs) 204 | return 1 205 | } 206 | 207 | tags := make(map[string]bool) 208 | 209 | for _, test := range tests { 210 | for _, tag := range test.Tags { 211 | tags[tag] = true 212 | } 213 | } 214 | 215 | // Print tags 216 | for key, _ := range tags { 217 | fmt.Println(key) 218 | } 219 | 220 | return 0 221 | } 222 | 223 | func doListTags() int { 224 | if err := getTagArgs(); err != nil { 225 | printRunError(err) 226 | return 1 227 | } 228 | 229 | tags := make(map[string][]*test161.Test) 230 | 231 | desired := make(map[string]bool) 232 | for _, t := range listTagsList { 233 | desired[t] = true 234 | } 235 | 236 | // Load every test file 237 | tests, errs := getAllTests() 238 | if len(errs) > 0 { 239 | printRunErrors(errs) 240 | return 1 241 | } 242 | 243 | // Get a tagmap of tag name -> list of tests 244 | for _, test := range tests { 245 | for _, tag := range test.Tags { 246 | if _, ok := tags[tag]; !ok { 247 | tags[tag] = make([]*test161.Test, 0) 248 | } 249 | tags[tag] = append(tags[tag], test) 250 | } 251 | } 252 | 253 | sorted := make([]string, 0) 254 | for tag, _ := range tags { 255 | sorted = append(sorted, tag) 256 | } 257 | sort.Strings(sorted) 258 | 259 | // Printing 260 | fmt.Println() 261 | 262 | if listTagsShort { 263 | // For the short version, we'll print a table to align the descriptions 264 | pd := &PrintData{ 265 | Headings: []*Heading{ 266 | &Heading{ 267 | Text: "Tag", 268 | }, 269 | &Heading{ 270 | Text: "Description", 271 | }, 272 | }, 273 | Config: defaultPrintConf, 274 | Rows: make(Rows, 0), 275 | } 276 | 277 | for _, tag := range sorted { 278 | if len(desired) > 0 && !desired[tag] { 279 | continue 280 | } 281 | 282 | desc := "" 283 | if info, ok := env.Tags[tag]; ok { 284 | desc = info.Description 285 | } 286 | 287 | pd.Rows = append(pd.Rows, []*Cell{ 288 | &Cell{Text: tag}, 289 | &Cell{Text: desc}, 290 | }) 291 | } 292 | 293 | if len(pd.Rows) > 0 { 294 | pd.Print() 295 | } 296 | fmt.Println() 297 | 298 | } else { 299 | bold := color.New(color.Bold) 300 | 301 | for _, tag := range sorted { 302 | if len(desired) > 0 && !desired[tag] { 303 | continue 304 | } 305 | 306 | if info, ok := env.Tags[tag]; ok { 307 | bold.Printf("%v:", tag) 308 | fmt.Printf(" %v\n", info.Description) 309 | } else { 310 | bold.Print(tag) 311 | } 312 | 313 | for _, test := range tags[tag] { 314 | fmt.Println(" ", test.DependencyID) 315 | } 316 | fmt.Println() 317 | } 318 | } 319 | 320 | return 0 321 | } 322 | 323 | func doListTests() int { 324 | pd := &PrintData{ 325 | Headings: []*Heading{ 326 | &Heading{ 327 | Text: "Test ID", 328 | }, 329 | &Heading{ 330 | Text: "Name", 331 | }, 332 | &Heading{ 333 | Text: "Description", 334 | }, 335 | }, 336 | Rows: make(Rows, 0), 337 | Config: defaultPrintConf, 338 | } 339 | 340 | // Load every test file 341 | tests, errs := getAllTests() 342 | if len(errs) > 0 { 343 | printRunErrors(errs) 344 | return 1 345 | } 346 | 347 | // Print ID, line, description for each tests 348 | for _, test := range tests { 349 | row := Row{ 350 | &Cell{Text: test.DependencyID}, 351 | &Cell{Text: test.Name}, 352 | &Cell{Text: strings.TrimSpace(test.Description)}, 353 | } 354 | pd.Rows = append(pd.Rows, row) 355 | } 356 | 357 | fmt.Println() 358 | pd.Print() 359 | fmt.Println() 360 | 361 | return 0 362 | } 363 | 364 | func doListAll() int { 365 | // Load every test file 366 | tests, errs := getAllTests() 367 | if len(errs) > 0 { 368 | printRunErrors(errs) 369 | return 1 370 | } 371 | 372 | tags := make(map[string]bool) 373 | 374 | for _, test := range tests { 375 | fmt.Println(test.DependencyID) 376 | for _, tag := range test.Tags { 377 | tags[tag] = true 378 | } 379 | } 380 | 381 | // Print tags 382 | for key, _ := range tags { 383 | fmt.Println(key) 384 | } 385 | 386 | // Print targets 387 | for _, target := range env.Targets { 388 | fmt.Println(target.Name) 389 | } 390 | return 0 391 | } 392 | -------------------------------------------------------------------------------- /test161/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/ops-class/test161" 6 | "os" 7 | ) 8 | 9 | var env *test161.TestEnvironment 10 | var clientConf *ClientConf 11 | 12 | func envInit(cmd *test161Command) { 13 | var err error 14 | 15 | // Get our bearings 16 | clientConf, err = inferConf() 17 | if err != nil || clientConf == nil { 18 | fmt.Fprintf(os.Stderr, "Unable to determine your test161 configuration:\n%v\n", err) 19 | os.Exit(1) 20 | } 21 | 22 | // Now load their actual conf file to get user info. 23 | confFile, err := getConfFromFile() 24 | if err != nil { 25 | // An error here means we couldn't load the file, either bad yaml or I/O problem 26 | fmt.Fprintf(os.Stderr, "An error occurred reading your %v file: %v\n", CONF_FILE, err) 27 | os.Exit(1) 28 | } 29 | 30 | // OK if confFile is nil, but they won't be able to submit. 31 | if confFile != nil { 32 | clientConf.Users = confFile.Users 33 | // The test161dir, if present, overrides the inferred directory. 34 | if confFile.Test161Dir != "" { 35 | clientConf.Test161Dir = confFile.Test161Dir 36 | } 37 | } 38 | 39 | // Environment variable overrides 40 | clientConf.OverlayDir = os.Getenv("TEST161_OVERLAY") 41 | if server := os.Getenv("TEST161_SERVER"); server != "" { 42 | clientConf.Server = server 43 | } 44 | 45 | // Test all the paths before trying to load the environment. Only the overlay 46 | // should really be a problem since we're figuring everything else out from 47 | // the cwd. 48 | if err = clientConf.checkPaths(cmd); err != nil { 49 | fmt.Fprintf(os.Stderr, "%v\n", err) 50 | os.Exit(1) 51 | } 52 | 53 | // Lastly, create the acutal test environment, loading the targets, commands, and 54 | // tests, but only if the test161 dir is set. 55 | if cmd.reqTests { 56 | if env, err = test161.NewEnvironment(clientConf.Test161Dir, nil); err != nil { 57 | fmt.Fprintf(os.Stderr, "Unable to create your test161 test environment:\n%v\n", err) 58 | os.Exit(1) 59 | } 60 | env.RootDir = clientConf.RootDir 61 | env.OverlayRoot = clientConf.OverlayDir 62 | } 63 | } 64 | 65 | func usage() { 66 | fmt.Fprintf(os.Stdout, `usage: 67 | test161 68 | 69 | test161 run [-dry-run | -d] [-explain | -x] [sequential | -s] 70 | [-no-dependencies | -n] [-verbose | -v (whisper|quiet|loud*)] 71 | [-tag] 72 | 73 | test161 submit [-debug] [-verify] [-no-cache] 74 | 75 | test161 list tags [-s | -short] [tags] 76 | test161 list targets [-remote | -r] 77 | test161 list tests 78 | 79 | test161 config [-debug] [(add-user|del-user|change-token)] 80 | test161 config test161dir 81 | 82 | test161 version 83 | 84 | test161 help for a detailed commands description 85 | `) 86 | } 87 | 88 | func doHelp() int { 89 | usage() 90 | fmt.Fprintf(os.Stdout, ` 91 | Commands Description: 92 | 93 | 'test161 run' runs a single target, or a group of tests. By default, all 94 | dependencies for the test group will also be run. For single tests and tags, 95 | specifying -no-dependencies will only run the test and tags given on the command 96 | line. 97 | 98 | Specififying Tests: Individual tests may be specified by name, doublestar 99 | globbing, or by tag. Because naming conflicts could arise between tags and 100 | targets, adding the -tag flag forces test161 to interpet a single positional 101 | argument as a tag. This flag can be safely omitted as long there as there are no 102 | conflicts between tag and target name. 103 | 104 | Output: Unless specified by -sequential, all output is interleaved with a 105 | summary at the end. You can disable test output lines with -v quiet, and hide 106 | everything except pass/fail with -v whisper. Specifying -dry-run will show you 107 | the tests that would be run, without running them. Similarly, -explain will show 108 | you more detailed information about the tests and what they expect, without 109 | running them. This option is very useful when writing your own tests. 110 | 111 | 112 | 'test161 submit' creates a submission for on the test161.ops-class.org 113 | server. This command will return a status, but will not block while evaluating 114 | the target on the server. 115 | 116 | Specifying a Commit: 'test161 submit' needs to send a Git commit id to the 117 | server so that it can run your kernel. If omitted, test161 will use the commit 118 | id corresponding the tip of your current branch. The submit command will also 119 | recognize sha commit ids, branches, and tags. For example, the following are all 120 | valid: 121 | test161 submit asst1 origin/master # origin/master 122 | test161 submit asst1 asst1 # asst1 is a tag 123 | 124 | Debugging: Adding the -verify flag will validate the submission will by checking 125 | for local and remote issues, without submitting. This is useful for debugging 126 | username and token, deployment key, and repository issues. Adding -debug will 127 | print the git commands that submit uses to determine the status of your 128 | repository. Adding -no-cache will clone the repo locally instead of using a 129 | previously cached copy. 130 | 131 | 132 | 'test161 list' prints a variety of useful information. 'test161 list targets' 133 | shows the local targets available to test161; adding -r will show the remote 134 | targets instead. 'test161 list tags' shows a listing of tags, their 135 | descriptions, and tests for each tag. Adding -shprt will print the tests for a 136 | concise table of tag names and descriptions. 'test161 list tests' lists all 137 | tests available to test161 along with their descriptions. 138 | 139 | 140 | 'test161 config' is used to view and change test161 configuration. When run with 141 | no arguments, this command shows test161 path, user, and Git repository 142 | information. Add -debug to see the Git commands that are run. 143 | 144 | User Configuration: 'test161 config' can also edit user/token data. There are 145 | three commands available to modify user configuration: 146 | 147 | 'test161 config add-user ' 148 | 'test161 config del-user ' 149 | 'test161 config change-token ' 150 | 151 | Test161 Directory: 'test161 config' can also configure your test161 directory, 152 | which is the directory that targets and tests are found. To specify a specific 153 | directory, use: 154 | 155 | 'test161 config test161dir ' 156 | `) 157 | 158 | return 0 159 | } 160 | 161 | type test161Command struct { 162 | cmd func() int 163 | reqEnv bool 164 | reqSource bool 165 | reqRoot bool 166 | reqTests bool 167 | } 168 | 169 | var cmdTable = map[string]*test161Command{ 170 | "run": &test161Command{ 171 | cmd: doRun, 172 | reqEnv: true, 173 | reqRoot: true, 174 | reqTests: true, 175 | }, 176 | "submit": &test161Command{ 177 | cmd: doSubmit, 178 | reqEnv: true, 179 | reqSource: true, 180 | reqTests: true, 181 | }, 182 | "list": &test161Command{ 183 | cmd: doListCommand, 184 | reqEnv: true, 185 | reqTests: true, 186 | }, 187 | "config": &test161Command{ 188 | cmd: doConfig, 189 | reqEnv: true, 190 | }, 191 | "version": &test161Command{ 192 | cmd: doVersion, 193 | }, 194 | "help": &test161Command{ 195 | cmd: doHelp, 196 | }, 197 | "upload-usage": &test161Command{ 198 | cmd: doUploadUsage, 199 | reqEnv: true, 200 | }, 201 | } 202 | 203 | func doVersion() int { 204 | fmt.Printf("test161 version: %v\n", test161.Version) 205 | return 0 206 | } 207 | 208 | func main() { 209 | exitcode := 2 210 | 211 | if len(os.Args) == 1 { 212 | usage() 213 | } else { 214 | // Get the sub-command 215 | if cmd, ok := cmdTable[os.Args[1]]; ok { 216 | if cmd.reqEnv { 217 | envInit(cmd) // This might exit 218 | } 219 | exitcode = cmd.cmd() 220 | } else { 221 | fmt.Fprintf(os.Stderr, "'%v' is not a recognized test161 command\n", os.Args[1]) 222 | usage() 223 | } 224 | } 225 | os.Exit(exitcode) 226 | } 227 | -------------------------------------------------------------------------------- /test161/printing.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | color "gopkg.in/fatih/color.v0" 7 | "os" 8 | "os/exec" 9 | "strconv" 10 | "strings" 11 | ) 12 | 13 | type Heading struct { 14 | Text string 15 | RightJustified bool 16 | MinWidth int 17 | 18 | // Calculated/working values 19 | width int 20 | format string 21 | min int 22 | max int 23 | } 24 | 25 | type Cell struct { 26 | Text string 27 | CellColor *color.Color 28 | } 29 | 30 | type Row []*Cell 31 | type Rows []Row 32 | 33 | type PrintConfig struct { 34 | NumSpaceSep int 35 | UnderlineChar string 36 | BoldHeadings bool 37 | } 38 | 39 | type PrintData struct { 40 | Headings []*Heading 41 | Rows Rows 42 | Config PrintConfig 43 | } 44 | 45 | //////////////////////////////////////////////////////////////////////////////// 46 | 47 | var defaultPrintConf = PrintConfig{ 48 | NumSpaceSep: 3, 49 | UnderlineChar: "-", 50 | BoldHeadings: true, 51 | } 52 | 53 | // Get the number of columns available on the terminal. We'll try to squeeze 54 | // the results into this width. 55 | func numTTYColumns() int { 56 | cmd := exec.Command("stty", "size") 57 | cmd.Stdin = os.Stdin 58 | res, err := cmd.Output() 59 | if err == nil { 60 | tokens := strings.Split(strings.TrimSpace(string(res)), " ") 61 | if len(tokens) == 2 { 62 | cols, _ := strconv.Atoi(tokens[1]) 63 | return cols 64 | } 65 | } 66 | 67 | return -1 68 | } 69 | 70 | // Get the length of the longest word in a line 71 | func longestWordLen(line string) int { 72 | start, longest, pos := 0, 0, 0 73 | for pos <= len(line) { 74 | // Break char or EOL 75 | if pos == len(line) || line[pos] == ' ' { 76 | // (pos-1) - start + 1 77 | wordlen := pos - start 78 | start = pos + 1 79 | pos = start 80 | 81 | if wordlen > longest { 82 | longest = wordlen 83 | } 84 | } else { 85 | pos += 1 86 | } 87 | } 88 | return longest 89 | } 90 | 91 | // Calculate the final widths of each column. 92 | func (pd *PrintData) calcWidths() { 93 | // First pass, get the max and min widths of each column. Best case scenerio, 94 | // we fit within the real estate we have. Worst case scenerio, the min width 95 | // is smaller than what we have to work with. 96 | 97 | // Start with the headings, and don't break them up 98 | for _, h := range pd.Headings { 99 | h.min = h.MinWidth 100 | if h.min < len(h.Text) { 101 | h.min = len(h.Text) 102 | } 103 | h.max = h.min 104 | } 105 | 106 | // Now, the rows. The min column width should be the shortest word we find. 107 | // The max width is the width of the longest cell. 108 | for _, row := range pd.Rows { 109 | for col, cell := range row { 110 | lw := longestWordLen(cell.Text) 111 | if pd.Headings[col].min < lw { 112 | pd.Headings[col].min = lw 113 | } 114 | if len(cell.Text) > pd.Headings[col].max { 115 | pd.Headings[col].max = len(cell.Text) 116 | } 117 | } 118 | } 119 | 120 | // Figure out how many columns we have to work with. But, at least give 121 | // us something to work with. 122 | remaining := numTTYColumns() 123 | if remaining < 80 { 124 | remaining = 80 125 | } 126 | 127 | // Deduct the column spacers 128 | remaining -= (len(pd.Headings) - 1) * pd.Config.NumSpaceSep 129 | 130 | // Start out by setting the widths to the min widths. 131 | // (OK if remaining goes negative) 132 | for _, h := range pd.Headings { 133 | h.width = h.min 134 | remaining -= h.width 135 | } 136 | 137 | // Divide up the remaining columns equally 138 | for remaining > 0 { 139 | didOne := false 140 | for _, h := range pd.Headings { 141 | if h.width < h.max && remaining > 0 { 142 | remaining -= 1 143 | h.width += 1 144 | didOne = true 145 | } 146 | } 147 | if !didOne { 148 | break 149 | } 150 | } 151 | } 152 | 153 | // Split a single-line cell into (possibly) mutiple cells, with one line per cell. 154 | func (cell *Cell) split(width int) []*Cell { 155 | // We currently use the simple (and common) greedy algorithm for 156 | // filling each row. 157 | // TODO: Change this to Knuth's algorithm for minumum raggedness 158 | 159 | res := make([]*Cell, 0) 160 | remaining := width 161 | line := "" 162 | words := strings.Split(cell.Text, " ") 163 | 164 | for _, word := range words { 165 | if len(line) == 0 { 166 | // First word 167 | line = word 168 | remaining = width - len(word) 169 | } else if remaining >= len(word)+1 { 170 | // The word + space fits in the current line 171 | remaining -= (len(word) + 1) 172 | line += " " + word 173 | } else { 174 | // The word doesn't fit; finish the old line and start a new one. 175 | res = append(res, &Cell{line, cell.CellColor}) 176 | remaining = width - len(word) 177 | line = word 178 | } 179 | } 180 | 181 | // Make sure the last line gets added 182 | if len(line) > 0 { 183 | res = append(res, &Cell{line, cell.CellColor}) 184 | } 185 | 186 | return res 187 | } 188 | 189 | // Split this row if it that has any cells that are too long for their column. 190 | func (row Row) split(headings []*Heading) []Row { 191 | newRows := make([]Row, 0) 192 | 193 | splits := make([][]*Cell, len(row)) 194 | numLines := 0 195 | for i, cell := range row { 196 | splits[i] = cell.split(headings[i].width) 197 | if len(splits[i]) > numLines { 198 | numLines = len(splits[i]) 199 | } 200 | } 201 | 202 | // At this point we have something like this: 203 | // [ ] [ ] [ ] 204 | // [ ] [ ] 205 | // [ ] 206 | // [ ] 207 | // 208 | // Each cell has been broken up into columnar slice, and we now have 209 | // to piece back together rows. We need to make sure each new row 210 | // has the right number of columns, even if we have blank cells. 211 | // We iterate over the rows, and if the cell split has that row, 212 | // we add it, otherwise we just add a placeholder. 213 | for i := 0; i < numLines; i++ { 214 | // The new row has to have the same number of columns 215 | newRow := make(Row, len(row)) 216 | 217 | for col, _ := range row { 218 | if i < len(splits[col]) { 219 | newRow[col] = splits[col][i] 220 | } else { 221 | newRow[col] = &Cell{"", nil} 222 | } 223 | } 224 | newRows = append(newRows, newRow) 225 | } 226 | 227 | return newRows 228 | } 229 | 230 | // Split any rows that have cells that are too long for their column. 231 | func (pd *PrintData) splitRows() { 232 | newRows := make(Rows, 0) 233 | 234 | for _, row := range pd.Rows { 235 | rows := row.split(pd.Headings) 236 | newRows = append(newRows, rows...) 237 | } 238 | 239 | pd.Rows = newRows 240 | } 241 | 242 | func (pd *PrintData) Print() error { 243 | // Do we have the right number of columns in the data? This is a 244 | // programming error if we don't. 245 | for _, row := range pd.Rows { 246 | if len(row) != len(pd.Headings) { 247 | return errors.New("Wrong number of columns") 248 | } 249 | } 250 | 251 | // Calculate min/max column widths and split up cells if needed 252 | pd.calcWidths() 253 | pd.splitRows() 254 | 255 | // Next compute the format string for each cell 256 | fmtStrings := make([]string, 0, len(pd.Headings)) 257 | for i, h := range pd.Headings { 258 | fmtStr := "%" 259 | if !h.RightJustified { 260 | fmtStr += "-" 261 | } 262 | fmtStr += fmt.Sprintf("%v", h.width) 263 | fmtStr += "v" 264 | if i+1 < len(pd.Headings) && pd.Config.NumSpaceSep > 0 { 265 | fmtStr += strings.Repeat(" ", pd.Config.NumSpaceSep) 266 | } else { 267 | fmtStr += "\n" 268 | } 269 | fmtStrings = append(fmtStrings, fmtStr) 270 | } 271 | 272 | // Print heading 273 | bold := color.New(color.Bold) 274 | 275 | for i, h := range pd.Headings { 276 | if pd.Config.BoldHeadings { 277 | bold.Printf(fmtStrings[i], h.Text) 278 | } else { 279 | fmt.Printf(fmtStrings[i], h.Text) 280 | } 281 | } 282 | 283 | // Underlines 284 | if pd.Config.UnderlineChar != "" { 285 | for i, h := range pd.Headings { 286 | fmt.Printf(fmtStrings[i], strings.Repeat(pd.Config.UnderlineChar, h.width)) 287 | } 288 | } 289 | 290 | for _, row := range pd.Rows { 291 | for col, cell := range row { 292 | if cell.CellColor != nil { 293 | cell.CellColor.Printf(fmtStrings[col], cell.Text) 294 | } else { 295 | fmt.Printf(fmtStrings[col], cell.Text) 296 | } 297 | } 298 | } 299 | 300 | return nil 301 | } 302 | -------------------------------------------------------------------------------- /test161/usage.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "compress/gzip" 6 | "fmt" 7 | "github.com/kardianos/osext" 8 | "github.com/ops-class/test161" 9 | "io/ioutil" 10 | "net/http" 11 | "os" 12 | "os/exec" 13 | "path" 14 | "sort" 15 | "strings" 16 | "syscall" 17 | "time" 18 | ) 19 | 20 | // 1 MB 21 | const MAX_FILE_SIZE = 1 * 1024 * 1024 22 | 23 | // Actual number is +1 for current 24 | const MAX_NUM_FILES = 20 25 | 26 | const USAGE_FILE_PREFIX = "usage" 27 | const USAGE_FILE_EXT = ".json.gz" 28 | const USAGE_FILE = USAGE_FILE_PREFIX + USAGE_FILE_EXT 29 | 30 | // This is less than the server accepts, so we have room to play with. 31 | const MAX_UPLOAD_SIZE = 10 * 1024 * 1024 32 | 33 | // Log a single usage entry for either a test group, target, or tag 34 | func logUsageStat(tg *test161.TestGroup, tagTarget string, start, end time.Time) error { 35 | // Check if we should save off the old one. If the uploader is running, 36 | // don't bother. 37 | err := lockFile(USAGE_LOCK_FILE, false) 38 | if err == nil { 39 | checkMoveUsgaeFile(false) 40 | unlockFile(USAGE_LOCK_FILE) 41 | } 42 | 43 | // Lock the current usage file while log the entry. We set the wait flag, 44 | // so this should block until finished. 45 | err = lockFile(CUR_USAGE_LOCK_FILE, true) 46 | if err != nil { 47 | return err 48 | } 49 | defer unlockFile(CUR_USAGE_LOCK_FILE) 50 | 51 | // It's possible the user info isn't filled in when the tests are run. The 52 | // server can fill in the blanks with the auth info. 53 | users := make([]string, 0) 54 | for _, user := range clientConf.Users { 55 | users = append(users, user.Email) 56 | } 57 | 58 | stat := test161.NewTestGroupUsageStat(users, tagTarget, tg, start, end) 59 | err = saveUsageStat(stat) 60 | if err != nil { 61 | printRunError(err) 62 | } 63 | return err 64 | } 65 | 66 | // Serialize the stat and append to the current usage file 67 | func saveUsageStat(stat *test161.UsageStat) error { 68 | fname := getCurUsageFilename() 69 | f, err := os.OpenFile(fname, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) 70 | if err != nil { 71 | return err 72 | } 73 | 74 | defer f.Close() 75 | 76 | if j, err := stat.JSON(); err != nil { 77 | return err 78 | } else { 79 | gz := gzip.NewWriter(f) 80 | writer := bufio.NewWriter(gz) 81 | _, err = writer.WriteString(j + "\n") 82 | writer.Flush() 83 | gz.Close() 84 | return err 85 | } 86 | } 87 | 88 | // Check if we should move the usage file due to space concerns. 89 | // If force is true, do it regardless of size -- and don't try 90 | // to prune anything. 91 | func checkMoveUsgaeFile(force bool) { 92 | source := getCurUsageFilename() 93 | dest := path.Join(USAGE_DIR, fmt.Sprintf("%v_%011d%v", USAGE_FILE_PREFIX, 94 | time.Now().Unix(), USAGE_FILE_EXT)) 95 | 96 | fi, err := os.Stat(source) 97 | if err == nil && (force || fi.Size() >= MAX_FILE_SIZE) { 98 | os.Rename(source, dest) 99 | if !force { 100 | checkPruneUsageFiles() 101 | } 102 | } 103 | } 104 | 105 | // See if we hit our space limit, and prune old files if we did. 106 | func checkPruneUsageFiles() { 107 | files, err := getAllUsageFiles() 108 | if err != nil { 109 | return 110 | } 111 | 112 | if len(files) > MAX_NUM_FILES { 113 | numDeletes := len(files) - MAX_NUM_FILES 114 | // The timestamps are 0-padded, so sorting ascending will put the 115 | // oldest files first. But, we want to ignore the current usage file. 116 | sort.Strings(files) 117 | pos := 0 118 | for pos < len(files) && numDeletes > 0 { 119 | file := files[pos] 120 | os.Remove(file) 121 | numDeletes -= 1 122 | pos += 1 123 | } 124 | } 125 | } 126 | 127 | // Spawn the uploader in the background. 128 | func runTest161Uploader() { 129 | // Try to use the version of test161 that is currently running. 130 | exename, _ := osext.Executable() 131 | if len(exename) == 0 { 132 | exename = "test161" 133 | } 134 | 135 | cmd := exec.Command(exename, "upload-usage") 136 | cmd.Start() 137 | } 138 | 139 | func handleUploadError(err error) { 140 | // TODO: Log this somewhere 141 | fmt.Println(err) 142 | return 143 | } 144 | 145 | func handleUploadErrors(errors []error) { 146 | // TODO: Log this somewhere 147 | fmt.Println(errors) 148 | return 149 | } 150 | 151 | type fileSizeInfo struct { 152 | name string 153 | size int64 154 | } 155 | 156 | func getFileSizes(files []string) ([]*fileSizeInfo, error) { 157 | 158 | res := make([]*fileSizeInfo, 0) 159 | 160 | for _, f := range files { 161 | if fi, err := os.Stat(f); err != nil { 162 | return nil, err 163 | } else { 164 | res = append(res, &fileSizeInfo{f, fi.Size()}) 165 | } 166 | } 167 | 168 | return res, nil 169 | } 170 | 171 | // Get the next group of files to upload, so that the total of their size is 172 | // less than or equal to maxChunkSize. 173 | // We use a simple greedy approach rather than optimally packing the upload. 174 | func nextUploadChunk(files []*fileSizeInfo, curPos int, maxChunkSize int64) ([]string, int) { 175 | if curPos >= len(files) { 176 | return nil, curPos 177 | } 178 | 179 | res := make([]string, 0) 180 | chunkSize := int64(0) 181 | 182 | for curPos < len(files) { 183 | fi := files[curPos] 184 | if chunkSize+fi.size <= maxChunkSize { 185 | res = append(res, files[curPos].name) 186 | chunkSize += fi.size 187 | } else { 188 | break 189 | } 190 | curPos += 1 191 | } 192 | return res, curPos 193 | } 194 | 195 | // Upload a group of usage files to the test161 server. 196 | func uploadStatFiles(files []string) []error { 197 | 198 | pos := 0 199 | allFileInfo, err := getFileSizes(files) 200 | if err != nil { 201 | return []error{err} 202 | } 203 | 204 | for pos < len(files) { 205 | chunk, newPos := nextUploadChunk(allFileInfo, pos, MAX_UPLOAD_SIZE) 206 | if newPos == pos { 207 | // We have a file that's too big 208 | // TODO: log this too 209 | pos += 1 210 | continue 211 | } 212 | 213 | pos = newPos 214 | 215 | // Upload this batch of files 216 | pr := NewPostRequest(ApiEndpointUpload) 217 | pr.SetType(PostTypeMultipart) 218 | 219 | req := &test161.UploadRequest{ 220 | UploadType: test161.UPLOAD_TYPE_USAGE, 221 | Users: clientConf.Users, 222 | } 223 | 224 | if err := pr.QueueJSON(req, "request"); err != nil { 225 | return []error{err} 226 | } 227 | 228 | for _, f := range chunk { 229 | pr.QueueFile(f, "usage") 230 | } 231 | 232 | resp, body, errs := pr.Submit() 233 | 234 | if len(errs) > 0 { 235 | return errs 236 | } else if resp.StatusCode != http.StatusOK { 237 | return []error{ 238 | fmt.Errorf("Upload failed. Status code: %v Msg: %v", resp.StatusCode, body), 239 | } 240 | } else { 241 | for _, f := range chunk { 242 | os.Remove(f) 243 | } 244 | } 245 | } 246 | 247 | return nil 248 | } 249 | 250 | // Entry point for running test161 upload-usage 251 | func doUploadUsage() int { 252 | // First, acquire the usage lock so we don't compete with another 253 | // uploader process. 254 | if err := lockFile(USAGE_LOCK_FILE, true); err != nil { 255 | // Already running 256 | return 1 257 | } 258 | defer unlockFile(USAGE_LOCK_FILE) 259 | 260 | // Next, move the current usage file out of the way so we don't compte with 261 | // 'test161 run ...' logging a new stat. 262 | if err := moveCurrentUsage(); err != nil { 263 | // Current usage file in use 264 | return 1 265 | } 266 | 267 | usageFiles, err := getAllUsageFiles() 268 | if err != nil { 269 | handleUploadError(err) 270 | return 1 271 | } 272 | 273 | // OK, we can upload now without competition from uploading or logging. 274 | if len(usageFiles) > 0 { 275 | if errs := uploadStatFiles(usageFiles); len(errs) > 0 { 276 | handleUploadErrors(errs) 277 | return 1 278 | } 279 | } 280 | return 0 281 | } 282 | 283 | // Make a copy of the current usage file so we can get out of the way and let 284 | // the user continue to run tests. 285 | func moveCurrentUsage() error { 286 | fname := getCurUsageFilename() 287 | if _, err := os.Stat(fname); err != nil { 288 | return nil 289 | } 290 | 291 | if lockErr := lockFile(CUR_USAGE_LOCK_FILE, true); lockErr != nil { 292 | return lockErr 293 | } 294 | defer unlockFile(CUR_USAGE_LOCK_FILE) 295 | 296 | // We'll just move the current file so it gets queued 297 | checkMoveUsgaeFile(true) 298 | 299 | return nil 300 | } 301 | 302 | // Get a list of all usage files in the usage directory, EXCEPT FOR 303 | // the current usage file. 304 | func getAllUsageFiles() ([]string, error) { 305 | files, err := ioutil.ReadDir(USAGE_DIR) 306 | if err != nil { 307 | return nil, err 308 | } 309 | 310 | usageFiles := make([]string, 0) 311 | 312 | for _, file := range files { 313 | name := file.Name() 314 | // Adding "_" will skip the current usage file. 315 | if strings.HasPrefix(name, USAGE_FILE_PREFIX+"_") && strings.HasSuffix(name, USAGE_FILE_EXT) { 316 | 317 | usageFiles = append(usageFiles, path.Join(USAGE_DIR, name)) 318 | } 319 | } 320 | 321 | return usageFiles, nil 322 | } 323 | 324 | // flock() a file in the filesystem 325 | func lockFile(fileName string, block bool) error { 326 | 327 | file, err := os.OpenFile(fileName, os.O_CREATE+os.O_APPEND, 0666) 328 | if err != nil { 329 | return err 330 | } 331 | 332 | op := syscall.LOCK_EX 333 | if block { 334 | op += syscall.LOCK_NB 335 | } 336 | 337 | fd := file.Fd() 338 | return syscall.Flock(int(fd), op) 339 | } 340 | 341 | // un-flock() a file in the filesystem 342 | func unlockFile(fileName string) error { 343 | file, err := os.OpenFile(fileName, os.O_CREATE+os.O_APPEND, 0666) 344 | if err != nil { 345 | return err 346 | } 347 | 348 | fd := file.Fd() 349 | return syscall.Flock(int(fd), syscall.LOCK_UN) 350 | } 351 | 352 | func getCurUsageFilename() string { 353 | return path.Join(USAGE_DIR, USAGE_FILE) 354 | } 355 | -------------------------------------------------------------------------------- /test161/usage_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestUploadChunks(t *testing.T) { 9 | assert := assert.New(t) 10 | sizes := []*fileSizeInfo{ 11 | &fileSizeInfo{ 12 | name: "foo", 13 | size: 1024, 14 | }, 15 | &fileSizeInfo{ 16 | name: "bar", 17 | size: 1024, 18 | }, 19 | &fileSizeInfo{ 20 | name: "food", 21 | size: 2048, 22 | }, 23 | &fileSizeInfo{ 24 | name: "bars", 25 | size: 1, 26 | }, 27 | } 28 | 29 | files, pos := nextUploadChunk(sizes, 0, 2048) 30 | assert.Equal(2, pos) 31 | assert.Equal(2, len(files)) 32 | assert.Equal("foo", files[0]) 33 | assert.Equal("bar", files[1]) 34 | 35 | files, pos = nextUploadChunk(sizes, pos, 2048) 36 | assert.Equal(3, pos) 37 | assert.Equal(1, len(files)) 38 | assert.Equal("food", files[0]) 39 | 40 | files, pos = nextUploadChunk(sizes, pos, 2048) 41 | assert.Equal(4, pos) 42 | assert.Equal(1, len(files)) 43 | assert.Equal("bars", files[0]) 44 | 45 | files, pos = nextUploadChunk(sizes, 0, 512) 46 | assert.Equal(0, len(files)) 47 | assert.Equal(0, pos) 48 | } 49 | 50 | func TestGetUsageFiles(t *testing.T) { 51 | assert := assert.New(t) 52 | 53 | USAGE_DIR = "./fixtures/files/" 54 | 55 | assert.Equal("fixtures/files/usage.json.gz", getCurUsageFilename()) 56 | 57 | files, err := getAllUsageFiles() 58 | assert.Nil(err) 59 | assert.Equal(2, len(files)) 60 | if len(files) != 2 { 61 | t.FailNow() 62 | } 63 | 64 | sizes, err := getFileSizes(files) 65 | assert.Nil(err) 66 | assert.Equal(2, len(sizes)) 67 | 68 | sizeMap := make(map[string]int64) 69 | for _, fi := range sizes { 70 | sizeMap[fi.name] = fi.size 71 | } 72 | 73 | sz, ok := sizeMap["fixtures/files/usage_013999999999.json.gz"] 74 | assert.True(ok) 75 | assert.Equal(int64(490), sz) 76 | 77 | sz, ok = sizeMap["fixtures/files/usage_014999999999.json.gz"] 78 | assert.True(ok) 79 | assert.Equal(int64(490), sz) 80 | } 81 | 82 | func TestFileLock(t *testing.T) { 83 | assert := assert.New(t) 84 | 85 | lock_file := "fixtures/files/test161.lock" 86 | err := lockFile(lock_file, true) 87 | assert.Nil(err) 88 | 89 | err = lockFile(lock_file, true) 90 | assert.NotNil(err) 91 | t.Log(err) 92 | 93 | err = unlockFile(lock_file) 94 | assert.Nil(err) 95 | } 96 | -------------------------------------------------------------------------------- /testing_persistence.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | // A fake persistence to use for test code 8 | 9 | var testStudent = &Student{ 10 | Email: "test@test161.ops-class.org", 11 | Token: "TestToken4$5^", 12 | } 13 | 14 | type TestingPersistence struct { 15 | Verbose bool 16 | } 17 | 18 | func (p *TestingPersistence) Close() { 19 | } 20 | 21 | func (p *TestingPersistence) Notify(entity interface{}, msg, what int) error { 22 | return nil 23 | } 24 | 25 | func (d *TestingPersistence) CanRetrieve() bool { 26 | return true 27 | } 28 | 29 | func (d *TestingPersistence) Retrieve(what int, who map[string]interface{}, 30 | filter map[string]interface{}, res interface{}) error { 31 | 32 | switch what { 33 | case PERSIST_TYPE_STUDENTS: 34 | if email, _ := who["email"]; email == testStudent.Email { 35 | if token, _ := who["token"]; token == testStudent.Token { 36 | students := res.(*[]*Student) 37 | *students = append(*students, testStudent) 38 | } 39 | } 40 | 41 | return nil 42 | 43 | case PERSIST_TYPE_USERS: 44 | // Only currently used to get the staff flag, and only the length is 45 | // checked. Since it's like 4 levels deep, we'll just fake it. 46 | if id, _ := who["services.auth0.email"]; id == testStudent.Email { 47 | results := res.(*[]interface{}) 48 | *results = append(*results, 1) 49 | } 50 | 51 | return nil 52 | default: 53 | return errors.New("Persistence: Invalid data type") 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /usage.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/kevinburke/go.uuid" 6 | "time" 7 | ) 8 | 9 | const CUR_USAGE_VERSION = 1 10 | 11 | type UsageStat struct { 12 | ID string `bson:"_id"` 13 | Users []string `bson:"users" json:"users"` 14 | Timestamp time.Time `bson:"timestamp" json:"timestamp"` 15 | Version int `bson:"version" json:"version"` 16 | Test161Version string `bson:"test161_version" json:"test161_version"` 17 | IsStaff bool `bson:"is_staff" json:"-"` 18 | GroupInfo *GroupStat `bson:"group_info" json:"group_info"` 19 | } 20 | 21 | type GroupStat struct { 22 | TargetTagName string `bson:"target_tag_name" json:"target_tag_name"` 23 | PointsAvailable uint `bson:"max_score" json:"max_score"` 24 | Status string `bson:"status" json:"status"` 25 | Score uint `bson:"score" json:"score"` 26 | Tests []*TestStat `bson:"tests" json:"tests"` 27 | Errors []string `bson:"errors" json:"errors"` 28 | SubmissionTime time.Time `bson:"submission_time" json:"submission_time"` 29 | CompletionTime time.Time `bson:"completion_time" json:"completion_time"` 30 | } 31 | 32 | type TestStat struct { 33 | Name string `json:"name" bson:"name"` 34 | Result TestResult `json:"result" bson:"result"` 35 | PointsAvailable uint `json:"points_avail" bson:"points_avail"` 36 | PointsEarned uint `json:"points_earned" bson:"points_earned"` 37 | MemLeakBytes int `json:"mem_leak_bytes" bson:"mem_leak_bytes"` 38 | MemLeakPoints uint `json:"mem_leak_points" bson:"mem_leak_points"` 39 | MemLeakDeducted uint `json:"mem_leak_deducted" bson:"mem_leak_deducted"` 40 | } 41 | 42 | func (stat *UsageStat) JSON() (string, error) { 43 | if data, err := json.Marshal(stat); err != nil { 44 | return "", err 45 | } else { 46 | return string(data), nil 47 | } 48 | } 49 | 50 | func (stat *UsageStat) Persist(env *TestEnvironment) error { 51 | return env.Persistence.Notify(stat, MSG_PERSIST_CREATE, 0) 52 | } 53 | 54 | func newUsageStat(users []string) *UsageStat { 55 | stat := &UsageStat{ 56 | ID: uuid.NewV4().String(), 57 | Users: users, 58 | Timestamp: time.Now(), 59 | Test161Version: Version.String(), 60 | Version: CUR_USAGE_VERSION, 61 | } 62 | return stat 63 | } 64 | 65 | func NewTestGroupUsageStat(users []string, targetOrTag string, tg *TestGroup, 66 | startTime, endTime time.Time) *UsageStat { 67 | 68 | stat := newUsageStat(users) 69 | stat.GroupInfo = &GroupStat{ 70 | TargetTagName: targetOrTag, 71 | PointsAvailable: 0, 72 | Status: "", 73 | Score: 0, 74 | Errors: []string{}, 75 | SubmissionTime: startTime, 76 | CompletionTime: endTime, 77 | } 78 | stat.GroupInfo.Tests = testStatsFromGroup(tg) 79 | for _, t := range stat.GroupInfo.Tests { 80 | stat.GroupInfo.PointsAvailable += t.PointsAvailable 81 | stat.GroupInfo.Score += t.PointsEarned 82 | } 83 | 84 | return stat 85 | } 86 | 87 | func testStatsFromGroup(group *TestGroup) []*TestStat { 88 | tests := make([]*TestStat, 0, len(group.Tests)) 89 | for _, test := range group.Tests { 90 | tests = append(tests, newTestStat(test)) 91 | } 92 | return tests 93 | } 94 | 95 | func newTestStat(t *Test) *TestStat { 96 | stat := &TestStat{ 97 | Name: t.Name, 98 | Result: t.Result, 99 | PointsAvailable: t.PointsAvailable, 100 | PointsEarned: t.PointsEarned, 101 | MemLeakBytes: t.MemLeakBytes, 102 | MemLeakPoints: t.MemLeakPoints, 103 | MemLeakDeducted: t.MemLeakDeducted, 104 | } 105 | return stat 106 | } 107 | -------------------------------------------------------------------------------- /usage_test.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestUsageStatsSubmission(t *testing.T) { 10 | t.Parallel() 11 | testPoints := getPartialTestPoints() 12 | 13 | targetName := "partial" 14 | 15 | start := time.Now() 16 | tg := runTargetTest(t, testPoints, targetName) 17 | end := time.Now() 18 | stats := NewTestGroupUsageStat([]string{"test161"}, targetName, tg, start, end) 19 | 20 | assert := assert.New(t) 21 | 22 | assert.Equal(1, len(stats.Users)) 23 | if len(stats.Users) == 1 { 24 | assert.Equal("test161", stats.Users[0]) 25 | } 26 | assert.Equal(Version.String(), stats.Test161Version) 27 | assert.Equal(CUR_USAGE_VERSION, stats.Version) 28 | 29 | assert.Equal(uint(60), stats.GroupInfo.PointsAvailable) 30 | assert.Equal(uint(30), stats.GroupInfo.Score) 31 | assert.Equal(targetName, stats.GroupInfo.TargetTagName) 32 | assert.Equal(start, stats.GroupInfo.SubmissionTime) 33 | assert.Equal(end, stats.GroupInfo.CompletionTime) 34 | } 35 | -------------------------------------------------------------------------------- /version.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type ProgramVersion struct { 8 | Major uint `yaml:"major"` 9 | Minor uint `yaml:"minor"` 10 | Revision uint `yaml:"revision"` 11 | } 12 | 13 | var Version = ProgramVersion{ 14 | Major: 1, 15 | Minor: 3, 16 | Revision: 2, 17 | } 18 | 19 | func (v ProgramVersion) String() string { 20 | return fmt.Sprintf("%v.%v.%v", v.Major, v.Minor, v.Revision) 21 | } 22 | 23 | // Returns 1 if this > other, 0 if this == other, and -1 if this < other 24 | func (this ProgramVersion) CompareTo(other ProgramVersion) int { 25 | 26 | if this.Major > other.Major { 27 | return 1 28 | } else if this.Major < other.Major { 29 | return -1 30 | } else if this.Minor > other.Minor { 31 | return 1 32 | } else if this.Minor < other.Minor { 33 | return -1 34 | } else if this.Revision > other.Revision { 35 | return 1 36 | } else if this.Revision < other.Revision { 37 | return -1 38 | } else { 39 | return 0 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /version_test.go: -------------------------------------------------------------------------------- 1 | package test161 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestVersion(t *testing.T) { 9 | t.Parallel() 10 | assert := assert.New(t) 11 | 12 | testVer := ProgramVersion{2, 5, 1} 13 | 14 | vers := []ProgramVersion{ 15 | ProgramVersion{2, 5, 1}, 16 | ProgramVersion{2, 5, 2}, 17 | ProgramVersion{2, 6, 1}, 18 | ProgramVersion{3, 0, 0}, 19 | ProgramVersion{2, 4, 9}, 20 | ProgramVersion{2, 5, 0}, 21 | ProgramVersion{1, 9, 9}, 22 | } 23 | expected := []int{0, -1, -1, -1, 1, 1, 1} 24 | 25 | for i, version := range vers { 26 | t.Log(version) 27 | assert.Equal(expected[i], testVer.CompareTo(version)) 28 | } 29 | } 30 | --------------------------------------------------------------------------------