├── testrunner ├── _._ ├── Events │ ├── TestMethodEndEvent.cs │ ├── TestRunBeginEvent.cs │ ├── ClassCleanupMethodEndEvent.cs │ ├── TestCleanupMethodEndEvent.cs │ ├── TestContextSetterEndEvent.cs │ ├── AssemblyCleanupMethodEndEvent.cs │ ├── ClassInitializeMethodEndEvent.cs │ ├── TestInitializeMethodEndEvent.cs │ ├── TestBeginEvent.cs │ ├── ErrorOutputEvent.cs │ ├── ProgramUsageEvent.cs │ ├── TraceOutputEvent.cs │ ├── ProgramBannerEvent.cs │ ├── StandardOutputEvent.cs │ ├── TestClassBeginEvent.cs │ ├── ProgramUserErrorEvent.cs │ ├── TestAssemblyBeginEvent.cs │ ├── TestAssemblyNotTestEvent.cs │ ├── TestContextSetterBeginEvent.cs │ ├── TestIgnoredEvent.cs │ ├── TestMethodBeginEvent.cs │ ├── AssemblyInitializeMethodEndEvent.cs │ ├── ClassCleanupMethodBeginEvent.cs │ ├── TestAssemblyNotDotNetEvent.cs │ ├── TestAssemblyNotFoundEvent.cs │ ├── AssemblyCleanupMethodBeginEvent.cs │ ├── TestClassIgnoredEvent.cs │ ├── TestCleanupMethodBeginEvent.cs │ ├── TestInitializeMethodBeginEvent.cs │ ├── TestAssemblyConfigFileSwitchedEvent.cs │ ├── TestEndEvent.cs │ ├── ProgramInternalErrorEvent.cs │ ├── TestRunEndEvent.cs │ ├── MethodEndEvent.cs │ ├── MethodUnexpectedExceptionEvent.cs │ ├── TestClassEndEvent.cs │ ├── ClassInitializeMethodBeginEvent.cs │ ├── TestAssemblyEndEvent.cs │ ├── MethodExpectedExceptionEvent.cs │ ├── AssemblyInitializeMethodBeginEvent.cs │ └── TestRunnerEvent.cs ├── Results │ ├── TestRunResult.cs │ ├── StackFrameInfo.cs │ ├── TestAssemblyResult.cs │ ├── MethodResult.cs │ ├── TestResult.cs │ ├── TestClassResult.cs │ └── ExceptionInfo.cs ├── Program │ ├── OutputFormats.cs │ ├── Program.cs │ └── ArgumentParser.cs ├── MSTest │ ├── IgnoreAttribute.cs │ ├── TestClassAttribute.cs │ ├── TestMethodAttribute.cs │ ├── TestCleanupAttribute.cs │ ├── ClassCleanupAttribute.cs │ ├── TestInitializeAttribute.cs │ ├── AssemblyCleanupAttribute.cs │ ├── ClassInitializeAttribute.cs │ ├── AssemblyInitializeAttribute.cs │ ├── AttributeBase.cs │ ├── ExpectedExceptionAttribute.cs │ ├── UnitTestOutcome.cs │ ├── TestMethod.cs │ ├── TestContext.cs │ ├── TestAssembly.cs │ ├── TestClass.cs │ └── TestContextProxy.cs ├── EventHandlers │ ├── MachineOutputEventHandler.cs │ ├── EventTraceListener.cs │ ├── ResultAccumulatingEventHandler.cs │ ├── ContextTrackingEventHandler.cs │ ├── TestContextEventHandler.cs │ ├── TestResultEventHandler.cs │ ├── EventHandlerPipeline.cs │ ├── TestClassResultEventHandler.cs │ ├── TestAssemblyResultEventHandler.cs │ ├── MethodResultEventHandler.cs │ ├── EventHandler.cs │ ├── MachineReadableEventSerializer.cs │ └── HumanOutputEventHandler.cs ├── Infrastructure │ ├── Disposable.cs │ ├── ProcessExecuteResults.cs │ ├── StringExtensions.cs │ ├── UserException.cs │ ├── Guard.cs │ ├── MemberInfoExtensions.cs │ └── ProcessExtensions.cs ├── testrunner.csproj ├── testrunner.nuspec └── Runners │ ├── TestClassRunner.cs │ ├── ConfigFileSwitcher.cs │ ├── TestMethodRunner.cs │ ├── TestAssemblyRunner.cs │ └── MethodRunner.cs ├── .gitattributes ├── testrunner.Tests.FakeDll └── FakeDll.dll ├── .editorconfig ├── .gitignore ├── .produce ├── testrunner.Tests.MSTest ├── testrunner.Tests.MSTest.dll.config ├── EmptyTestClass.cs ├── IgnoredTestClass.cs ├── testrunner.Tests.MSTest.csproj ├── MSTestTests.Constants.cs └── MSTestTests.cs ├── testrunner.Tests.DifferentConfigValue ├── testrunner.Tests.DifferentConfigValue.dll.config ├── DifferentConfigValueTests.cs └── testrunner.Tests.DifferentConfigValue.csproj ├── testrunner.Tests.Pass ├── PassTests.cs └── testrunner.Tests.Pass.csproj ├── testrunner.Tests.ReferencedAssembly ├── ReferencedClass.cs └── testrunner.Tests.ReferencedAssembly.csproj ├── testrunner.Tests.Fail ├── testrunner.Tests.Fail.csproj └── FailTests.cs ├── testrunner.Tests.IncludeExclude ├── testrunner.Tests.IncludeExclude.csproj └── Tests.cs ├── .travis.yml ├── license.txt ├── changelog.md ├── testrunner.Tests └── testrunner.Tests.csproj ├── hacking.md ├── testrunner.sln └── readme.md /testrunner/_._: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | * -text 2 | -------------------------------------------------------------------------------- /testrunner.Tests.FakeDll/FakeDll.dll: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | guidelines = 120 5 | 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.vs 2 | /.vscode 3 | /*/bin 4 | /*/obj 5 | *.user 6 | -------------------------------------------------------------------------------- /.produce: -------------------------------------------------------------------------------- 1 | program: testrunner/bin/Debug/netcoreapp2.0/publish/testrunner.dll 2 | 3 | -------------------------------------------------------------------------------- /testrunner/Events/TestMethodEndEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class TestMethodEndEvent : MethodEndEvent 4 | { 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /testrunner/Events/TestRunBeginEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class TestRunBeginEvent : TestRunnerEvent 4 | { 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /testrunner/Events/ClassCleanupMethodEndEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class ClassCleanupMethodEndEvent : MethodEndEvent 4 | { 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /testrunner/Events/TestCleanupMethodEndEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class TestCleanupMethodEndEvent : MethodEndEvent 4 | { 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /testrunner/Events/TestContextSetterEndEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class TestContextSetterEndEvent : MethodEndEvent 4 | { 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /testrunner/Events/AssemblyCleanupMethodEndEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class AssemblyCleanupMethodEndEvent : MethodEndEvent 4 | { 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /testrunner/Events/ClassInitializeMethodEndEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class ClassInitializeMethodEndEvent : MethodEndEvent 4 | { 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /testrunner/Events/TestInitializeMethodEndEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class TestInitializeMethodEndEvent : MethodEndEvent 4 | { 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /testrunner/Results/TestRunResult.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Results 2 | { 3 | public class TestRunResult 4 | { 5 | public bool Success { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Events/TestBeginEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class TestBeginEvent : TestRunnerEvent 4 | { 5 | public string Name { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Events/ErrorOutputEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class ErrorOutputEvent : TestRunnerEvent 4 | { 5 | public string Message { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Events/ProgramUsageEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class ProgramUsageEvent : TestRunnerEvent 4 | { 5 | public string[] Lines { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Events/TraceOutputEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class TraceOutputEvent : TestRunnerEvent 4 | { 5 | public string Message { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Events/ProgramBannerEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class ProgramBannerEvent : TestRunnerEvent 4 | { 5 | public string[] Lines { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Events/StandardOutputEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class StandardOutputEvent : TestRunnerEvent 4 | { 5 | public string Message { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Events/TestClassBeginEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class TestClassBeginEvent : TestRunnerEvent 4 | { 5 | public string FullName { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Events/ProgramUserErrorEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class ProgramUserErrorEvent : TestRunnerEvent 4 | { 5 | public string Message { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Events/TestAssemblyBeginEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class TestAssemblyBeginEvent : TestRunnerEvent 4 | { 5 | public string Path { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Events/TestAssemblyNotTestEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class TestAssemblyNotTestEvent : TestRunnerEvent 4 | { 5 | public string Path { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Events/TestContextSetterBeginEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class TestContextSetterBeginEvent : TestRunnerEvent 4 | { 5 | public string MethodName; 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Events/TestIgnoredEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class TestIgnoredEvent : TestRunnerEvent 4 | { 5 | public bool IgnoredFromCommandLine { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Events/TestMethodBeginEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class TestMethodBeginEvent : TestRunnerEvent 4 | { 5 | public string MethodName { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Events/AssemblyInitializeMethodEndEvent.cs: -------------------------------------------------------------------------------- 1 | using TestRunner.Results; 2 | 3 | namespace TestRunner.Events 4 | { 5 | public class AssemblyInitializeMethodEndEvent : MethodEndEvent 6 | { 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /testrunner/Events/ClassCleanupMethodBeginEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class ClassCleanupMethodBeginEvent : TestRunnerEvent 4 | { 5 | public string MethodName; 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Events/TestAssemblyNotDotNetEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class TestAssemblyNotDotNetEvent : TestRunnerEvent 4 | { 5 | public string Path { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Events/TestAssemblyNotFoundEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class TestAssemblyNotFoundEvent : TestRunnerEvent 4 | { 5 | public string Path { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Events/AssemblyCleanupMethodBeginEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class AssemblyCleanupMethodBeginEvent : TestRunnerEvent 4 | { 5 | public string MethodName; 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Events/TestClassIgnoredEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class TestClassIgnoredEvent : TestRunnerEvent 4 | { 5 | public bool IgnoredFromCommandLine { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Events/TestCleanupMethodBeginEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class TestCleanupMethodBeginEvent : TestRunnerEvent 4 | { 5 | public string MethodName { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Events/TestInitializeMethodBeginEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class TestInitializeMethodBeginEvent : TestRunnerEvent 4 | { 5 | public string MethodName { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Events/TestAssemblyConfigFileSwitchedEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class TestAssemblyConfigFileSwitchedEvent : TestRunnerEvent 4 | { 5 | public string Path { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /testrunner/Results/StackFrameInfo.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Results 2 | { 3 | public class StackFrameInfo 4 | { 5 | 6 | public string At { get; set; } 7 | public string In { get; set; } 8 | 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /testrunner/Results/TestAssemblyResult.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Results 2 | { 3 | public class TestAssemblyResult 4 | { 5 | public string TestAssemblyPath { get; set; } 6 | public bool Success { get; set; } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /testrunner.Tests.MSTest/testrunner.Tests.MSTest.dll.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /testrunner/Events/TestEndEvent.cs: -------------------------------------------------------------------------------- 1 | using TestRunner.Results; 2 | 3 | namespace TestRunner.Events 4 | { 5 | public class TestEndEvent : TestRunnerEvent 6 | { 7 | public TestResult Result { get; set; } = new TestResult(); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /testrunner/Events/ProgramInternalErrorEvent.cs: -------------------------------------------------------------------------------- 1 | using TestRunner.Results; 2 | 3 | namespace TestRunner.Events 4 | { 5 | public class ProgramInternalErrorEvent : TestRunnerEvent 6 | { 7 | public ExceptionInfo Exception { get; set; } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /testrunner/Events/TestRunEndEvent.cs: -------------------------------------------------------------------------------- 1 | using TestRunner.Results; 2 | 3 | namespace TestRunner.Events 4 | { 5 | public class TestRunEndEvent : TestRunnerEvent 6 | { 7 | public TestRunResult Result { get; set; } = new TestRunResult(); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /testrunner/Events/MethodEndEvent.cs: -------------------------------------------------------------------------------- 1 | using TestRunner.Results; 2 | 3 | namespace TestRunner.Events 4 | { 5 | public abstract class MethodEndEvent : TestRunnerEvent 6 | { 7 | public MethodResult Result { get; set; } = new MethodResult(); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /testrunner/Events/MethodUnexpectedExceptionEvent.cs: -------------------------------------------------------------------------------- 1 | using TestRunner.Results; 2 | 3 | namespace TestRunner.Events 4 | { 5 | public class MethodUnexpectedExceptionEvent : TestRunnerEvent 6 | { 7 | public ExceptionInfo Exception { get; set; } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /testrunner/Events/TestClassEndEvent.cs: -------------------------------------------------------------------------------- 1 | using TestRunner.Results; 2 | 3 | namespace TestRunner.Events 4 | { 5 | public class TestClassEndEvent : TestRunnerEvent 6 | { 7 | public TestClassResult Result { get; set; } = new TestClassResult(); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /testrunner.Tests.DifferentConfigValue/testrunner.Tests.DifferentConfigValue.dll.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /testrunner/Events/ClassInitializeMethodBeginEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class ClassInitializeMethodBeginEvent : TestRunnerEvent 4 | { 5 | public string MethodName { get; set; } 6 | public string FirstTestMethodName { get; set; } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /testrunner/Events/TestAssemblyEndEvent.cs: -------------------------------------------------------------------------------- 1 | using TestRunner.Results; 2 | 3 | namespace TestRunner.Events 4 | { 5 | public class TestAssemblyEndEvent : TestRunnerEvent 6 | { 7 | public TestAssemblyResult Result { get; set; } = new TestAssemblyResult(); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /testrunner/Results/MethodResult.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Results 2 | { 3 | public class MethodResult 4 | { 5 | 6 | public bool Success { get; set; } 7 | public ExceptionInfo Exception { get; set; } 8 | public long ElapsedMilliseconds { get; set; } = -1; 9 | 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /testrunner/Events/MethodExpectedExceptionEvent.cs: -------------------------------------------------------------------------------- 1 | using TestRunner.Results; 2 | 3 | namespace TestRunner.Events 4 | { 5 | public class MethodExpectedExceptionEvent : TestRunnerEvent 6 | { 7 | public string ExpectedFullName { get; set; } 8 | public ExceptionInfo Exception { get; set; } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /testrunner/Program/OutputFormats.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Program 2 | { 3 | 4 | /// 5 | /// Program output formats 6 | /// 7 | /// 8 | public static class OutputFormats 9 | { 10 | public const string Human = "human"; 11 | public const string Machine = "machine"; 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /testrunner.Tests.Pass/PassTests.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.VisualStudio.TestTools.UnitTesting; 2 | 3 | namespace TestRunner.Tests.Pass 4 | { 5 | 6 | [TestClass] 7 | public class PassTests 8 | { 9 | 10 | [TestMethod] 11 | public void PassTest() 12 | { 13 | // This passes 14 | } 15 | 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /testrunner/Events/AssemblyInitializeMethodBeginEvent.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Events 2 | { 3 | public class AssemblyInitializeMethodBeginEvent : TestRunnerEvent 4 | { 5 | public string MethodName { get; set; } 6 | public string FirstTestClassFullName { get; set; } 7 | public string FirstTestMethodName { get; set; } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /testrunner.Tests.ReferencedAssembly/ReferencedClass.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Tests.ReferencedAssembly 2 | { 3 | public static class TestReferencedClass 4 | { 5 | public static string TestReferencedMethod() 6 | { 7 | return "TestRunner.Tests.ReferencedAssembly.TestReferencedClass.TestReferencedMethod()"; 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /testrunner.Tests.ReferencedAssembly/testrunner.Tests.ReferencedAssembly.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.0;net461 5 | 1.10.1-master 6 | 1.10.1.0 7 | 1.0.0.0 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /testrunner/Results/TestResult.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Results 2 | { 3 | public class TestResult 4 | { 5 | public string TestAssemblyPath { get; set; } 6 | public string TestClassFullName { get; set; } 7 | public string TestName { get; set; } 8 | public bool Success { get; set; } 9 | public bool Ignored { get; set; } 10 | public bool IgnoredFromCommandLine { get; set; } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /testrunner/Events/TestRunnerEvent.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | 4 | namespace TestRunner.Events 5 | { 6 | public abstract class TestRunnerEvent 7 | { 8 | 9 | public TestRunnerEvent() 10 | { 11 | ProcessId = Process.GetCurrentProcess().Id; 12 | Timestamp = DateTime.Now; 13 | } 14 | 15 | 16 | public int ProcessId { get; set; } 17 | public DateTime Timestamp { get; set; } 18 | 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /testrunner/MSTest/IgnoreAttribute.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace TestRunner.MSTest 4 | { 5 | 6 | class IgnoreAttribute : AttributeBase 7 | { 8 | 9 | public static IgnoreAttribute TryCreate(Attribute attribute) 10 | { 11 | return TryCreate( 12 | attribute, 13 | "Microsoft.VisualStudio.TestTools.UnitTesting.IgnoreAttribute", 14 | a => new IgnoreAttribute(a)); 15 | } 16 | 17 | 18 | IgnoreAttribute(Attribute attribute) 19 | : base(attribute) 20 | { 21 | } 22 | 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /testrunner/MSTest/TestClassAttribute.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace TestRunner.MSTest 4 | { 5 | 6 | class TestClassAttribute : AttributeBase 7 | { 8 | 9 | public static TestClassAttribute TryCreate(Attribute attribute) 10 | { 11 | return TryCreate( 12 | attribute, 13 | "Microsoft.VisualStudio.TestTools.UnitTesting.TestClassAttribute", 14 | a => new TestClassAttribute(a)); 15 | } 16 | 17 | 18 | TestClassAttribute(Attribute attribute) 19 | : base(attribute) 20 | { 21 | } 22 | 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /testrunner/MSTest/TestMethodAttribute.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace TestRunner.MSTest 4 | { 5 | 6 | class TestMethodAttribute : AttributeBase 7 | { 8 | 9 | public static TestMethodAttribute TryCreate(Attribute attribute) 10 | { 11 | return TryCreate( 12 | attribute, 13 | "Microsoft.VisualStudio.TestTools.UnitTesting.TestMethodAttribute", 14 | a => new TestMethodAttribute(a)); 15 | } 16 | 17 | 18 | TestMethodAttribute(Attribute attribute) 19 | : base(attribute) 20 | { 21 | } 22 | 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /testrunner.Tests.Fail/testrunner.Tests.Fail.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.0;net461 5 | 1.10.1-master 6 | 1.10.1.0 7 | 1.0.0.0 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /testrunner.Tests.Pass/testrunner.Tests.Pass.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.0;net461 5 | 1.10.1-master 6 | 1.10.1.0 7 | 1.0.0.0 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /testrunner/MSTest/TestCleanupAttribute.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace TestRunner.MSTest 4 | { 5 | 6 | class TestCleanupAttribute : AttributeBase 7 | { 8 | 9 | public static TestCleanupAttribute TryCreate(Attribute attribute) 10 | { 11 | return TryCreate( 12 | attribute, 13 | "Microsoft.VisualStudio.TestTools.UnitTesting.TestCleanupAttribute", 14 | a => new TestCleanupAttribute(a)); 15 | } 16 | 17 | 18 | TestCleanupAttribute(Attribute attribute) 19 | : base(attribute) 20 | { 21 | } 22 | 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /testrunner/MSTest/ClassCleanupAttribute.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace TestRunner.MSTest 4 | { 5 | 6 | class ClassCleanupAttribute : AttributeBase 7 | { 8 | 9 | public static ClassCleanupAttribute TryCreate(Attribute attribute) 10 | { 11 | return TryCreate( 12 | attribute, 13 | "Microsoft.VisualStudio.TestTools.UnitTesting.ClassCleanupAttribute", 14 | a => new ClassCleanupAttribute(a)); 15 | } 16 | 17 | 18 | ClassCleanupAttribute(Attribute attribute) 19 | : base(attribute) 20 | { 21 | } 22 | 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /testrunner.Tests.MSTest/EmptyTestClass.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.VisualStudio.TestTools.UnitTesting; 3 | 4 | namespace TestRunner.Tests.MSTest 5 | { 6 | 7 | [TestClass] 8 | public class EmptyTestClass 9 | { 10 | 11 | [ClassInitialize] 12 | public static void ClassInitialize(TestContext testContext) 13 | { 14 | Console.WriteLine(MSTestTests.EmptyClassInitializeMessage); 15 | } 16 | 17 | 18 | [ClassCleanup] 19 | public static void ClassCleanup() 20 | { 21 | Console.WriteLine(MSTestTests.EmptyClassCleanupMessage); 22 | } 23 | 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /testrunner.Tests.IncludeExclude/testrunner.Tests.IncludeExclude.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.0;net461 5 | 1.10.1-master 6 | 1.10.1.0 7 | 1.0.0.0 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /testrunner/MSTest/TestInitializeAttribute.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace TestRunner.MSTest 4 | { 5 | 6 | class TestInitializeAttribute : AttributeBase 7 | { 8 | 9 | public static TestInitializeAttribute TryCreate(Attribute attribute) 10 | { 11 | return TryCreate( 12 | attribute, 13 | "Microsoft.VisualStudio.TestTools.UnitTesting.TestInitializeAttribute", 14 | a => new TestInitializeAttribute(a)); 15 | } 16 | 17 | 18 | TestInitializeAttribute(Attribute attribute) 19 | : base(attribute) 20 | { 21 | } 22 | 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /testrunner/MSTest/AssemblyCleanupAttribute.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace TestRunner.MSTest 4 | { 5 | 6 | class AssemblyCleanupAttribute : AttributeBase 7 | { 8 | 9 | public static AssemblyCleanupAttribute TryCreate(Attribute attribute) 10 | { 11 | return TryCreate( 12 | attribute, 13 | "Microsoft.VisualStudio.TestTools.UnitTesting.AssemblyCleanupAttribute", 14 | a => new AssemblyCleanupAttribute(a)); 15 | } 16 | 17 | 18 | AssemblyCleanupAttribute(Attribute attribute) 19 | : base(attribute) 20 | { 21 | } 22 | 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /testrunner/MSTest/ClassInitializeAttribute.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace TestRunner.MSTest 4 | { 5 | 6 | class ClassInitializeAttribute : AttributeBase 7 | { 8 | 9 | public static ClassInitializeAttribute TryCreate(Attribute attribute) 10 | { 11 | return TryCreate( 12 | attribute, 13 | "Microsoft.VisualStudio.TestTools.UnitTesting.ClassInitializeAttribute", 14 | a => new ClassInitializeAttribute(a)); 15 | } 16 | 17 | 18 | ClassInitializeAttribute(Attribute attribute) 19 | : base(attribute) 20 | { 21 | } 22 | 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /testrunner/MSTest/AssemblyInitializeAttribute.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace TestRunner.MSTest 4 | { 5 | 6 | class AssemblyInitializeAttribute : AttributeBase 7 | { 8 | 9 | public static AssemblyInitializeAttribute TryCreate(Attribute attribute) 10 | { 11 | return TryCreate( 12 | attribute, 13 | "Microsoft.VisualStudio.TestTools.UnitTesting.AssemblyInitializeAttribute", 14 | a => new AssemblyInitializeAttribute(a)); 15 | } 16 | 17 | 18 | AssemblyInitializeAttribute(Attribute attribute) 19 | : base(attribute) 20 | { 21 | } 22 | 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /testrunner/EventHandlers/MachineOutputEventHandler.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using TestRunner.Events; 3 | using TestRunner.Infrastructure; 4 | 5 | namespace TestRunner.EventHandlers 6 | { 7 | 8 | /// 9 | /// Event handler that outputs events in a single-line machine-readable JSON-based text format 10 | /// 11 | /// 12 | public class MachineOutputEventHandler : EventHandler 13 | { 14 | 15 | public override void Handle(TestRunnerEvent e) 16 | { 17 | Guard.NotNull(e, nameof(e)); 18 | Console.Out.WriteLine(MachineReadableEventSerializer.Serialize(e)); 19 | base.Handle(e); 20 | } 21 | 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /testrunner.Tests.Fail/FailTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.VisualStudio.TestTools.UnitTesting; 3 | 4 | namespace TestRunner.Tests.Fail 5 | { 6 | 7 | [TestClass] 8 | public class FailTests 9 | { 10 | 11 | public TestContext TestContext { get; set; } 12 | 13 | 14 | [TestCleanup] 15 | public void TestCleanup() 16 | { 17 | if (TestContext.CurrentTestOutcome == UnitTestOutcome.Failed) 18 | { 19 | Console.Out.WriteLine("Failed UnitTestOutcome"); 20 | } 21 | } 22 | 23 | 24 | [TestMethod] 25 | public void FailTest() 26 | { 27 | throw new Exception("Fail"); 28 | } 29 | 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /testrunner/Infrastructure/Disposable.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace TestRunner.Infrastructure 4 | { 5 | 6 | /// 7 | /// An that performs an action when disposed 8 | /// 9 | /// 10 | public class Disposable : IDisposable 11 | { 12 | 13 | public Disposable(Action disposeAction) 14 | { 15 | Guard.NotNull(disposeAction, nameof(disposeAction)); 16 | this.disposeAction = disposeAction; 17 | } 18 | 19 | 20 | Action disposeAction; 21 | bool disposed; 22 | 23 | 24 | void IDisposable.Dispose() 25 | { 26 | if (disposed) return; 27 | disposeAction(); 28 | disposed = true; 29 | } 30 | 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /testrunner.Tests.MSTest/IgnoredTestClass.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.VisualStudio.TestTools.UnitTesting; 3 | 4 | namespace TestRunner.Tests.MSTest 5 | { 6 | 7 | [Ignore] 8 | [TestClass] 9 | public class IgnoredTestClass 10 | { 11 | 12 | [ClassInitialize] 13 | public static void ClassInitialize(TestContext testContext) 14 | { 15 | Console.WriteLine(MSTestTests.IgnoredClassInitializeMessage); 16 | } 17 | 18 | 19 | [TestMethod] 20 | public void IgnoredTestClassMethod() 21 | { 22 | Console.WriteLine(MSTestTests.IgnoredClassTestMessage); 23 | } 24 | 25 | 26 | [ClassCleanup] 27 | public static void ClassCleanup() 28 | { 29 | Console.WriteLine(MSTestTests.IgnoredClassCleanupMessage); 30 | } 31 | 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /testrunner/Results/TestClassResult.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Results 2 | { 3 | public class TestClassResult 4 | { 5 | public string TestAssemblyPath { get; set; } 6 | public string TestClassFullName { get; set; } 7 | public bool Success { get; set; } 8 | public bool ClassIgnored { get; set; } 9 | public bool ClassIgnoredFromCommandLine { get; set; } 10 | public bool InitializePresent { get; set; } 11 | public bool InitializeSucceeded { get; set; } 12 | public int TestsTotal { get; set; } 13 | public int TestsRan { get; set; } 14 | public int TestsIgnored { get; set; } 15 | public int TestsPassed { get; set; } 16 | public int TestsFailed { get; set; } 17 | public bool CleanupPresent { get; set; } 18 | public bool CleanupSucceeded { get; set; } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /testrunner/testrunner.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp2.0;net461 6 | 1.10.1-master 7 | 1.10.1.0 8 | 1.0.0.0 9 | Copyright (c) 2012-2019 Ron MacNeil 10 | testrunner.nuspec 11 | $(NuspecProperties);version=$(Version) 12 | $(NuspecProperties);copyright=$(Copyright) 13 | $(NuspecProperties);configuration=$(Configuration) 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /testrunner/Infrastructure/ProcessExecuteResults.cs: -------------------------------------------------------------------------------- 1 | 2 | namespace TestRunner.Infrastructure 3 | { 4 | public class ProcessExecuteResults 5 | { 6 | 7 | public ProcessExecuteResults(string standardOutput, string errorOutput, string output, int exitCode) 8 | { 9 | Guard.NotNull(standardOutput, nameof(standardOutput)); 10 | Guard.NotNull(errorOutput, nameof(errorOutput)); 11 | Guard.NotNull(output, nameof(output)); 12 | StandardOutput = standardOutput; 13 | ErrorOutput = errorOutput; 14 | Output = output; 15 | ExitCode = exitCode; 16 | } 17 | 18 | 19 | public string StandardOutput { get; private set; } 20 | public string ErrorOutput { get; private set; } 21 | public string Output { get; private set; } 22 | public int ExitCode { get; private set; } 23 | 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /testrunner.Tests.DifferentConfigValue/DifferentConfigValueTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | #if NET461 3 | using System.Configuration; 4 | #endif 5 | using Microsoft.VisualStudio.TestTools.UnitTesting; 6 | 7 | namespace TestRunner.Tests.DifferentConfigValue 8 | { 9 | 10 | [TestClass] 11 | public class DifferentConfigValueTests 12 | { 13 | 14 | [TestMethod] 15 | public void TestAssembly_Config_File_Is_Used() 16 | { 17 | #if NET461 18 | // 19 | // Config file switching doesn't work on Mono 20 | // See https://bugzilla.xamarin.com/show_bug.cgi?id=15741 21 | // 22 | if (Type.GetType("Mono.Runtime") != null) return; 23 | Assert.AreEqual( 24 | "DifferentConfigFileValue", 25 | ConfigurationManager.AppSettings["ConfigFileKey"]); 26 | #endif 27 | } 28 | 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: csharp 2 | 3 | branches: 4 | only: 5 | - master 6 | - /-master$/ 7 | - /^\d+\.\d+\.\d+$/ 8 | 9 | matrix: 10 | include: 11 | - dotnet: 2.1.4 12 | mono: none 13 | script: 14 | - dotnet publish -f netcoreapp2.0 15 | - dotnet testrunner/bin/Debug/netcoreapp2.0/publish/testrunner.dll testrunner.Tests/bin/Debug/netcoreapp2.0/publish/testrunner.Tests.dll 16 | - mono: latest 17 | dotnet: none 18 | script: 19 | - msbuild /t:restore /p:TargetFramework=net461 20 | - msbuild /t:rebuild /p:TargetFramework=net461 21 | - msbuild /t:publish /p:TargetFramework=net461 22 | - mono --debug testrunner/bin/Debug/net461/publish/testrunner.exe testrunner.Tests/bin/Debug/net461/publish/testrunner.Tests.dll 23 | -------------------------------------------------------------------------------- /testrunner/Infrastructure/StringExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | 4 | namespace TestRunner.Infrastructure 5 | { 6 | 7 | public static class StringExtensions 8 | { 9 | 10 | public static string Indent(string value) 11 | { 12 | Guard.NotNull(value, nameof(value)); 13 | var lines = SplitLines(value); 14 | var indentedLines = lines.Select(s => " " + s); 15 | return string.Join(Environment.NewLine, indentedLines); 16 | } 17 | 18 | 19 | public static string[] SplitLines(string value) 20 | { 21 | Guard.NotNull(value, nameof(value)); 22 | value = value.Replace("\r\n", "\n").Replace("\r", "\n"); 23 | if (value.EndsWith("\n", StringComparison.Ordinal)) 24 | { 25 | value = value.Substring(0, value.Length-1); 26 | } 27 | return value.Split('\n'); 28 | } 29 | 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /testrunner.Tests.DifferentConfigValue/testrunner.Tests.DifferentConfigValue.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.0;net461 5 | 1.10.1-master 6 | 1.10.1.0 7 | 1.0.0.0 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | Always 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /testrunner/Infrastructure/UserException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace TestRunner.Infrastructure 4 | { 5 | 6 | /// 7 | /// A user-facing error 8 | /// 9 | /// 10 | [System.Diagnostics.CodeAnalysis.SuppressMessage( 11 | "Microsoft.Usage", 12 | "CA2237:MarkISerializableTypesWithSerializable", 13 | Justification = "Doesn't need to be serializable for this application")] 14 | [System.Diagnostics.CodeAnalysis.SuppressMessage( 15 | "Microsoft.Design", 16 | "CA1032:ImplementStandardExceptionConstructors", 17 | Justification = "At least a message is required")] 18 | public class UserException : Exception 19 | { 20 | 21 | public UserException(string message) 22 | : this(message, null) 23 | { 24 | } 25 | 26 | 27 | public UserException(string message, Exception innerException) 28 | : base(message, innerException) 29 | { 30 | } 31 | 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /testrunner.Tests.IncludeExclude/Tests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.VisualStudio.TestTools.UnitTesting; 3 | 4 | namespace A 5 | { 6 | [TestClass] public class A 7 | { 8 | [TestMethod] public void a() { Console.WriteLine("A.A.a() is running"); } 9 | [TestMethod] public void b() { Console.WriteLine("A.A.b() is running"); } 10 | } 11 | [TestClass] public class B 12 | { 13 | [TestMethod] public void a() { Console.WriteLine("A.B.a() is running"); } 14 | [TestMethod] public void b() { Console.WriteLine("A.B.b() is running"); } 15 | } 16 | } 17 | namespace B 18 | { 19 | [TestClass] public class A 20 | { 21 | [TestMethod] public void a() { Console.WriteLine("B.A.a() is running"); } 22 | [TestMethod] public void b() { Console.WriteLine("B.A.b() is running"); } 23 | } 24 | [TestClass] public class B 25 | { 26 | [TestMethod] public void a() { Console.WriteLine("B.B.a() is running"); } 27 | [TestMethod] public void b() { Console.WriteLine("B.B.b() is running"); } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /testrunner/MSTest/AttributeBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using TestRunner.Infrastructure; 3 | 4 | namespace TestRunner.MSTest 5 | { 6 | public abstract class AttributeBase 7 | { 8 | 9 | protected static TAttribute TryCreate( 10 | Attribute attribute, 11 | string typeName, 12 | Func constructor) 13 | where TAttribute : class 14 | { 15 | Guard.NotNull(attribute, nameof(attribute)); 16 | Guard.NotNull(typeName, nameof(typeName)); 17 | Guard.NotNull(constructor, nameof(constructor)); 18 | 19 | if (attribute.GetType().FullName != typeName) 20 | return null; 21 | 22 | return constructor(attribute); 23 | } 24 | 25 | 26 | protected AttributeBase(Attribute attribute) 27 | { 28 | Attribute = attribute; 29 | } 30 | 31 | 32 | public Attribute Attribute 33 | { 34 | get; 35 | private set; 36 | } 37 | 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /testrunner/Infrastructure/Guard.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace TestRunner.Infrastructure 4 | { 5 | 6 | public static class Guard 7 | { 8 | 9 | [System.Diagnostics.CodeAnalysis.SuppressMessage( 10 | "Microsoft.Naming", 11 | "CA1704:IdentifiersShouldBeSpelledCorrectly", 12 | MessageId = "param", 13 | Justification = "Consistency with ArgumentException classes in BCL")] 14 | public static void NotNull(object value, string paramName) 15 | { 16 | if (paramName == null) throw new ArgumentNullException(nameof(paramName)); 17 | if (value == null) throw new ArgumentNullException(paramName); 18 | } 19 | 20 | 21 | public static void NotNullOrWhiteSpace(string value, string paramName) 22 | { 23 | NotNull(value, paramName); 24 | if (string.IsNullOrWhiteSpace(value)) 25 | { 26 | throw new ArgumentException("Argument is empty or whitespace-only", paramName); 27 | } 28 | } 29 | 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /testrunner.Tests.MSTest/testrunner.Tests.MSTest.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.0;net461 5 | 1.10.1-master 6 | 1.10.1.0 7 | 1.0.0.0 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | Always 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /testrunner/MSTest/ExpectedExceptionAttribute.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace TestRunner.MSTest 4 | { 5 | 6 | class ExpectedExceptionAttribute : AttributeBase 7 | { 8 | 9 | public static ExpectedExceptionAttribute TryCreate(Attribute attribute) 10 | { 11 | return TryCreate( 12 | attribute, 13 | "Microsoft.VisualStudio.TestTools.UnitTesting.ExpectedExceptionAttribute", 14 | a => new ExpectedExceptionAttribute(a)); 15 | } 16 | 17 | 18 | ExpectedExceptionAttribute(Attribute attribute) 19 | : base(attribute) 20 | { 21 | ExceptionType = (Type)attribute.GetType().GetProperty("ExceptionType").GetValue(attribute, null); 22 | AllowDerivedTypes = (bool)attribute.GetType().GetProperty("AllowDerivedTypes").GetValue(attribute, null); 23 | } 24 | 25 | 26 | public Type ExceptionType 27 | { 28 | get; 29 | private set; 30 | } 31 | 32 | 33 | public bool AllowDerivedTypes 34 | { 35 | get; 36 | private set; 37 | } 38 | 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /license.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016-2019 Ron MacNeil 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /testrunner.Tests.MSTest/MSTestTests.Constants.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.Tests.MSTest 2 | { 3 | 4 | public partial class MSTestTests 5 | { 6 | 7 | public static readonly string TestCleanupMessage = "[TestCleanup] is running"; 8 | public static readonly string ClassCleanupMessage = "[ClassCleanup] is running"; 9 | public static readonly string AssemblyCleanupMessage = "[AssemblyCleanup] is running"; 10 | public static readonly string IgnoredTestMessage = "[Ignore]d test method is running"; 11 | public static readonly string IgnoredClassInitializeMessage = "[ClassInitialize] on [Ignore]d [TestClass] is running"; 12 | public static readonly string IgnoredClassTestMessage = "[Ignore]d test class method is running"; 13 | public static readonly string IgnoredClassCleanupMessage = "[ClassCleanup] on [Ignore]d [TestClass] is running"; 14 | public static readonly string EmptyClassInitializeMessage = "[ClassInitialize] on empty [TestClass] is running"; 15 | public static readonly string EmptyClassCleanupMessage = "[ClassCleanup] on empty [TestClass] is running"; 16 | public static readonly string TraceTestMessage = "Trace test message"; 17 | 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /testrunner/testrunner.nuspec: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | TestRunner 5 | $version$ 6 | A console MSTest runner 7 | macro187 8 | 9 | mstest test runner console 10 | false 11 | 12 | $copyright$ 13 | MIT 14 | https://github.com/macro187/testrunner 15 | https://github.com/macro187/testrunner/blob/master/changelog.md 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /testrunner/EventHandlers/EventTraceListener.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics; 2 | using TestRunner.Events; 3 | 4 | namespace TestRunner.EventHandlers 5 | { 6 | 7 | /// 8 | /// A that redirects output to the pipeline 9 | /// 10 | /// 11 | public class EventTraceListener : TraceListener 12 | { 13 | 14 | string buffer = ""; 15 | 16 | 17 | public override void WriteLine(string message) 18 | { 19 | Write(message + "\n"); 20 | } 21 | 22 | 23 | public override void Write(string message) 24 | { 25 | message = message ?? ""; 26 | message = message.Replace("\r\n", "\n").Replace("\r", "\n"); 27 | 28 | int i; 29 | while ((i = message.IndexOf('\n')) >= 0) 30 | { 31 | var line = message.Substring(0, i); 32 | if (buffer != "") 33 | { 34 | line = buffer + line; 35 | buffer = ""; 36 | } 37 | EventHandlerPipeline.Raise(new TraceOutputEvent() { Message = line }); 38 | 39 | message = message.Substring(i + 1); 40 | } 41 | 42 | buffer = buffer + message; 43 | } 44 | 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /testrunner/EventHandlers/ResultAccumulatingEventHandler.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using TestRunner.Events; 3 | using TestRunner.Results; 4 | 5 | namespace TestRunner.EventHandlers 6 | { 7 | 8 | /// 9 | /// Event handler that remembers results 10 | /// 11 | /// 12 | public class ResultAccumulatingEventHandler : EventHandler 13 | { 14 | 15 | public IList TestResults { get; private set; } = new List(); 16 | public IList TestClassResults { get; private set; } = new List(); 17 | public IList TestAssemblyResults { get; private set; } = new List(); 18 | public IList TestRunResults { get; private set; } = new List(); 19 | 20 | 21 | protected override void Handle(TestEndEvent e) 22 | { 23 | TestResults.Add(e.Result); 24 | } 25 | 26 | 27 | protected override void Handle(TestClassEndEvent e) 28 | { 29 | TestClassResults.Add(e.Result); 30 | } 31 | 32 | 33 | protected override void Handle(TestAssemblyEndEvent e) 34 | { 35 | TestAssemblyResults.Add(e.Result); 36 | } 37 | 38 | 39 | protected override void Handle(TestRunEndEvent e) 40 | { 41 | TestRunResults.Add(e.Result); 42 | } 43 | 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /testrunner/EventHandlers/ContextTrackingEventHandler.cs: -------------------------------------------------------------------------------- 1 | using TestRunner.Events; 2 | 3 | namespace TestRunner.EventHandlers 4 | { 5 | 6 | /// 7 | /// Event handler that tracks the currently-running assembly, class, and test 8 | /// 9 | /// 10 | public class ContextTrackingEventHandler : EventHandler 11 | { 12 | 13 | public string CurrentTestAssemblyPath { get; private set; } 14 | public string CurrentTestClassFullName { get; private set; } 15 | public string CurrentTestName { get; private set; } 16 | 17 | 18 | protected override void Handle(TestAssemblyBeginEvent e) 19 | { 20 | CurrentTestAssemblyPath = e.Path; 21 | } 22 | 23 | 24 | protected override void Handle(TestAssemblyEndEvent e) 25 | { 26 | CurrentTestAssemblyPath = null; 27 | } 28 | 29 | 30 | protected override void Handle(TestClassBeginEvent e) 31 | { 32 | CurrentTestClassFullName = e.FullName; 33 | } 34 | 35 | 36 | protected override void Handle(TestClassEndEvent e) 37 | { 38 | CurrentTestClassFullName = null; 39 | } 40 | 41 | 42 | protected override void Handle(TestBeginEvent e) 43 | { 44 | CurrentTestName = e.Name; 45 | } 46 | 47 | 48 | protected override void Handle(TestEndEvent e) 49 | { 50 | CurrentTestName = null; 51 | } 52 | 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /testrunner/MSTest/UnitTestOutcome.cs: -------------------------------------------------------------------------------- 1 | namespace TestRunner.MSTest 2 | { 3 | public enum UnitTestOutcome : int 4 | { 5 | 6 | /// 7 | /// Test was executed, but there were issues 8 | /// 9 | /// 10 | /// 11 | /// Issues may involve exceptions or failed assertions. 12 | /// 13 | /// 14 | Failed, 15 | 16 | /// 17 | /// Test has completed, but we can't say if it passed or failed 18 | /// 19 | /// 20 | /// 21 | /// May be used for aborted tests. 22 | /// 23 | /// 24 | Inconclusive, 25 | 26 | /// 27 | /// Test was executed without any issues 28 | /// 29 | /// 30 | Passed, 31 | 32 | /// 33 | /// Test is currently executing 34 | /// 35 | /// 36 | InProgress, 37 | 38 | /// 39 | /// There was a system error while we were trying to execute a test 40 | /// 41 | /// 42 | Error, 43 | 44 | /// 45 | /// The test timed out 46 | /// 47 | /// 48 | Timeout, 49 | 50 | /// 51 | /// Test was aborted by the user 52 | /// 53 | /// 54 | Aborted, 55 | 56 | /// 57 | /// Test is in an unknown state 58 | /// 59 | /// 60 | Unknown, 61 | 62 | /// 63 | /// Test cannot be executed 64 | /// 65 | /// 66 | NotRunnable 67 | 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /testrunner/Infrastructure/MemberInfoExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Reflection; 5 | 6 | namespace TestRunner.Infrastructure 7 | { 8 | 9 | public static class MemberInfoExtensions 10 | { 11 | 12 | public static bool HasCustomAttribute( 13 | this MemberInfo memberInfo, 14 | Func tryCreate) 15 | where TAttribute : class 16 | { 17 | return memberInfo.GetCustomAttribute(tryCreate) != null; 18 | } 19 | 20 | 21 | public static TAttribute GetCustomAttribute( 22 | this MemberInfo memberInfo, 23 | Func tryCreate) 24 | where TAttribute : class 25 | { 26 | var atts = memberInfo.GetCustomAttributes(tryCreate).ToList(); 27 | 28 | if (atts.Count > 1) 29 | { 30 | var memberName = $"{memberInfo.DeclaringType.FullName}.{memberInfo.Name}"; 31 | if (memberInfo is MethodInfo) memberName += "()"; 32 | 33 | var attributeName = typeof(TAttribute).Name; 34 | 35 | throw new UserException( 36 | $"{memberName}(): Encountered multiple [{attributeName}] where a maximum of one was expected"); 37 | } 38 | 39 | if (atts.Count == 0) 40 | return null; 41 | 42 | return atts[0]; 43 | } 44 | 45 | 46 | public static IEnumerable GetCustomAttributes( 47 | this MemberInfo memberInfo, 48 | Func tryCreate) 49 | { 50 | Guard.NotNull(memberInfo, nameof(memberInfo)); 51 | Guard.NotNull(tryCreate, nameof(tryCreate)); 52 | 53 | return 54 | memberInfo.GetCustomAttributes(false) 55 | .Cast() 56 | .Select(a => tryCreate(a)) 57 | .Where(a => a != null); 58 | } 59 | 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /testrunner/Runners/TestClassRunner.cs: -------------------------------------------------------------------------------- 1 | using TestRunner.MSTest; 2 | using TestRunner.Events; 3 | using TestRunner.Infrastructure; 4 | using TestRunner.EventHandlers; 5 | using TestRunner.Program; 6 | 7 | namespace TestRunner.Runners 8 | { 9 | static class TestClassRunner 10 | { 11 | 12 | /// 13 | /// Run tests in a [TestClass] 14 | /// 15 | /// 16 | static public void Run(TestClass testClass) 17 | { 18 | Guard.NotNull(testClass, nameof(testClass)); 19 | 20 | EventHandlerPipeline.Raise(new TestClassBeginEvent() { FullName = testClass.FullName }); 21 | 22 | do 23 | { 24 | // 25 | // Handle exclusion from the command line 26 | // 27 | if (!ArgumentParser.ClassShouldRun(testClass.FullName)) 28 | { 29 | EventHandlerPipeline.Raise(new TestClassIgnoredEvent() { IgnoredFromCommandLine = true }); 30 | break; 31 | } 32 | 33 | // 34 | // Handle [Ignored] [TestClass] 35 | // 36 | if (testClass.IsIgnored) 37 | { 38 | EventHandlerPipeline.Raise(new TestClassIgnoredEvent()); 39 | break; 40 | } 41 | 42 | // 43 | // Run [ClassInitialize] method 44 | // 45 | if (!MethodRunner.RunClassInitializeMethod(testClass)) break; 46 | 47 | // 48 | // Run [TestMethod]s 49 | // 50 | foreach (var testMethod in testClass.TestMethods) 51 | { 52 | TestMethodRunner.Run(testMethod); 53 | } 54 | 55 | // 56 | // Run [ClassCleanup] method 57 | // 58 | MethodRunner.RunClassCleanupMethod(testClass); 59 | } 60 | while (false); 61 | 62 | EventHandlerPipeline.Raise(new TestClassEndEvent()); 63 | } 64 | 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /changelog.md: -------------------------------------------------------------------------------- 1 | TestRunner Changelog 2 | ==================== 3 | 4 | 5 | v1.10.0 6 | ------- 7 | 8 | Add --method command line option 9 | 10 | 11 | v1.9.0 12 | ------ 13 | 14 | Switch to MIT license 15 | 16 | Add --class command line option 17 | 18 | Add experimental machine-readable output format 19 | 20 | Correctly capture stderr output from child processes 21 | 22 | Correctly propagate --outputformat to child processes 23 | 24 | 25 | v1.8.0 26 | ------ 27 | 28 | Add --outputformat command line option 29 | 30 | Add --help command line option 31 | 32 | Correctly consider \[Ignore\]d \[TestClass\]es to be successful 33 | 34 | 35 | v1.7.1 36 | ------ 37 | 38 | Stop occasionally printing output from child processes out of order, by 39 | correctly waiting until child processes are completely finished before 40 | proceeding 41 | 42 | 43 | v1.7.0 44 | ------ 45 | 46 | Add .NET Core support 47 | 48 | 49 | v1.6.0 50 | ------ 51 | 52 | Add support for `TestContext` `.CurrentTestOutcome`, 53 | `.FullyQualifiedTestClassName`, and `.TestName` 54 | 55 | 56 | v1.5.1 57 | ------ 58 | 59 | Fix build on Mono 5 which no longer supports `__MonoCS__` preprocessor 60 | variable 61 | 62 | 63 | v1.5 64 | ---- 65 | 66 | Support multiple test assemblies in a single `TestRunner` invocation 67 | 68 | 69 | v1.4 70 | ---- 71 | 72 | Stop hanging indefinitely if there are leftover threads, by exiting the 73 | program using 74 | [Environment.Exit](https://msdn.microsoft.com/en-us/library/system.environment.exit.aspx) 75 | 76 | 77 | v1.3 78 | ---- 79 | 80 | Eliminate dependency on 81 | `Microsoft.VisualStudio.QualityTools.UnitTestFramework.dll` by using 82 | reflection to discover and run tests 83 | 84 | 85 | v1.2 86 | ---- 87 | 88 | Don't try to run test or cleanup methods if relevant initialize methods fail 89 | 90 | 91 | v1.1 92 | ---- 93 | 94 | Improve \[ExpectedException\] output 95 | 96 | Don't crash in environments where the internal framework details required for 97 | test assembly `.config` file loading aren't present 98 | 99 | 100 | v1.0 101 | ---- 102 | 103 | 104 | -------------------------------------------------------------------------------- /testrunner/MSTest/TestMethod.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Reflection; 3 | using TestRunner.Infrastructure; 4 | 5 | namespace TestRunner.MSTest 6 | { 7 | 8 | public class TestMethod 9 | { 10 | 11 | internal static TestMethod TryCreate(TestClass testClass, MethodInfo methodInfo) 12 | { 13 | Guard.NotNull(methodInfo, nameof(methodInfo)); 14 | Guard.NotNull(testClass, nameof(testClass)); 15 | return 16 | methodInfo.HasCustomAttribute(TestMethodAttribute.TryCreate) 17 | ? new TestMethod(testClass, methodInfo) 18 | : null; 19 | } 20 | 21 | 22 | TestMethod(TestClass testClass, MethodInfo methodInfo) 23 | { 24 | TestClass = testClass; 25 | MethodInfo = methodInfo; 26 | IsIgnored = MethodInfo.HasCustomAttribute(IgnoreAttribute.TryCreate); 27 | var eea = methodInfo.GetCustomAttribute(ExpectedExceptionAttribute.TryCreate); 28 | ExpectedException = eea != null ? eea.ExceptionType : null; 29 | AllowDerivedExpectedExceptionTypes = eea != null ? eea.AllowDerivedTypes : false; 30 | } 31 | 32 | 33 | public TestClass TestClass 34 | { 35 | get; 36 | private set; 37 | } 38 | 39 | 40 | public MethodInfo MethodInfo 41 | { 42 | get; 43 | private set; 44 | } 45 | 46 | 47 | public bool IsIgnored 48 | { 49 | get; 50 | private set; 51 | } 52 | 53 | 54 | public Type ExpectedException 55 | { 56 | get; 57 | private set; 58 | } 59 | 60 | 61 | public bool AllowDerivedExpectedExceptionTypes 62 | { 63 | get; 64 | private set; 65 | } 66 | 67 | 68 | public string Name 69 | { 70 | get 71 | { 72 | return MethodInfo.Name; 73 | } 74 | } 75 | 76 | 77 | public string FullName 78 | { 79 | get 80 | { 81 | return $"{TestClass.FullName}.{Name}"; 82 | } 83 | } 84 | 85 | } 86 | 87 | } 88 | -------------------------------------------------------------------------------- /testrunner.Tests/testrunner.Tests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.0;net461 5 | 1.10.1-master 6 | 1.10.1.0 7 | 1.0.0.0 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | false 26 | 27 | 28 | false 29 | 30 | 31 | false 32 | 33 | 34 | false 35 | 36 | 37 | false 38 | 39 | 40 | false 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /testrunner/Runners/ConfigFileSwitcher.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | using TestRunner.EventHandlers; 3 | #if NET461 4 | using System; 5 | using System.Configuration; 6 | using System.Linq; 7 | using System.Reflection; 8 | using TestRunner.Events; 9 | #endif 10 | 11 | namespace TestRunner.Runners 12 | { 13 | 14 | static class ConfigFileSwitcher 15 | { 16 | 17 | /// 18 | /// Switch to using a specified assembly .config file (if present) 19 | /// 20 | /// 21 | static public void SwitchTo(string configPath) 22 | { 23 | if (!File.Exists(configPath)) return; 24 | 25 | #if NET461 26 | AppDomain.CurrentDomain.SetData("APP_CONFIG_FILE", configPath); 27 | 28 | // 29 | // The following hackery forces the new config file to take effect 30 | // 31 | // See http://stackoverflow.com/questions/6150644/change-default-app-config-at-runtime/6151688#6151688 32 | // 33 | var initStateField = 34 | typeof(ConfigurationManager).GetField("s_initState", BindingFlags.NonPublic | BindingFlags.Static); 35 | if (initStateField != null) 36 | { 37 | initStateField.SetValue(null, 0); 38 | } 39 | 40 | var configSystemField = 41 | typeof(ConfigurationManager).GetField("s_configSystem", BindingFlags.NonPublic | BindingFlags.Static); 42 | if (configSystemField != null) 43 | { 44 | configSystemField.SetValue(null, null); 45 | } 46 | 47 | var clientConfigPathsType = 48 | typeof(ConfigurationManager) 49 | .Assembly 50 | .GetTypes() 51 | .FirstOrDefault(x => x.FullName == "System.Configuration.ClientConfigPaths"); 52 | var currentField = 53 | clientConfigPathsType != null 54 | ? clientConfigPathsType.GetField("s_current", BindingFlags.NonPublic | BindingFlags.Static) 55 | : null; 56 | if (currentField != null) 57 | { 58 | currentField.SetValue(null, null); 59 | } 60 | 61 | EventHandlerPipeline.Raise(new TestAssemblyConfigFileSwitchedEvent() { Path = configPath }); 62 | #endif 63 | } 64 | 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /testrunner/Runners/TestMethodRunner.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using TestRunner.MSTest; 3 | using TestRunner.Events; 4 | using TestRunner.EventHandlers; 5 | using TestRunner.Program; 6 | 7 | namespace TestRunner.Runners 8 | { 9 | static class TestMethodRunner 10 | { 11 | 12 | /// 13 | /// Run a test method (plus its intialize and cleanup methods, if present) 14 | /// 15 | /// 16 | /// 17 | /// If the test method is decorated with [Ignore], nothing is run 18 | /// 19 | /// 20 | static public void Run(TestMethod testMethod) 21 | { 22 | EventHandlerPipeline.Raise(new TestBeginEvent() { Name = testMethod.Name }); 23 | 24 | do 25 | { 26 | // 27 | // Handle exclusion from the command line 28 | // 29 | if (!ArgumentParser.MethodShouldRun(testMethod.FullName)) 30 | { 31 | EventHandlerPipeline.Raise(new TestIgnoredEvent() { IgnoredFromCommandLine = true }); 32 | break; 33 | } 34 | 35 | // 36 | // Handle [Ignored] [TestMethod] 37 | // 38 | if (testMethod.IsIgnored) 39 | { 40 | EventHandlerPipeline.Raise(new TestIgnoredEvent()); 41 | break; 42 | } 43 | 44 | // 45 | // Create instance of [TestClass] 46 | // 47 | var instance = Activator.CreateInstance(testMethod.TestClass.Type); 48 | 49 | // 50 | // Set TestContext property (if present) 51 | // 52 | MethodRunner.RunTestContextSetter(testMethod.TestClass, instance); 53 | 54 | // 55 | // Run [TestInitialize] method 56 | // 57 | if (!MethodRunner.RunTestInitializeMethod(testMethod.TestClass, instance)) break; 58 | 59 | // 60 | // Run [TestMethod] 61 | // 62 | MethodRunner.RunTestMethod(testMethod, instance); 63 | 64 | // 65 | // Run [TestCleanup] method 66 | // 67 | MethodRunner.RunTestCleanupMethod(testMethod.TestClass, instance); 68 | } 69 | while (false); 70 | 71 | EventHandlerPipeline.Raise(new TestEndEvent()); 72 | } 73 | 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /testrunner/EventHandlers/TestContextEventHandler.cs: -------------------------------------------------------------------------------- 1 | using TestRunner.MSTest; 2 | using System.Linq; 3 | using TestRunner.Events; 4 | 5 | namespace TestRunner.EventHandlers 6 | { 7 | 8 | /// 9 | /// Event handler that maintains state 10 | /// 11 | /// 12 | public class TestContextEventHandler : EventHandler 13 | { 14 | 15 | protected override void Handle(TestClassBeginEvent e) 16 | { 17 | TestContext.FullyQualifiedTestClassName = e.FullName; 18 | } 19 | 20 | 21 | protected override void Handle(TestClassEndEvent e) 22 | { 23 | TestContext.FullyQualifiedTestClassName = null; 24 | } 25 | 26 | 27 | protected override void Handle(TestBeginEvent e) 28 | { 29 | TestContext.TestName = e.Name; 30 | } 31 | 32 | 33 | protected override void Handle(TestEndEvent e) 34 | { 35 | TestContext.TestName = null; 36 | TestContext.CurrentTestOutcome = UnitTestOutcome.Unknown; 37 | } 38 | 39 | 40 | protected override void Handle(AssemblyInitializeMethodBeginEvent e) 41 | { 42 | TestContext.FullyQualifiedTestClassName = e.FirstTestClassFullName; 43 | TestContext.TestName = e.FirstTestMethodName; 44 | TestContext.CurrentTestOutcome = UnitTestOutcome.InProgress; 45 | } 46 | 47 | 48 | protected override void Handle(AssemblyInitializeMethodEndEvent e) 49 | { 50 | TestContext.FullyQualifiedTestClassName = null; 51 | TestContext.TestName = null; 52 | TestContext.CurrentTestOutcome = UnitTestOutcome.Unknown; 53 | } 54 | 55 | 56 | protected override void Handle(ClassInitializeMethodBeginEvent e) 57 | { 58 | TestContext.TestName = e.FirstTestMethodName; 59 | TestContext.CurrentTestOutcome = UnitTestOutcome.InProgress; 60 | } 61 | 62 | 63 | protected override void Handle(ClassInitializeMethodEndEvent e) 64 | { 65 | TestContext.TestName = null; 66 | TestContext.CurrentTestOutcome = UnitTestOutcome.Unknown; 67 | } 68 | 69 | 70 | protected override void Handle(TestInitializeMethodBeginEvent e) 71 | { 72 | TestContext.CurrentTestOutcome = UnitTestOutcome.InProgress; 73 | } 74 | 75 | 76 | protected override void Handle(TestMethodEndEvent e) 77 | { 78 | TestContext.CurrentTestOutcome = e.Result.Success ? UnitTestOutcome.Passed : UnitTestOutcome.Failed; 79 | } 80 | 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /testrunner/EventHandlers/TestResultEventHandler.cs: -------------------------------------------------------------------------------- 1 | using TestRunner.Events; 2 | using TestRunner.Results; 3 | 4 | namespace TestRunner.EventHandlers 5 | { 6 | 7 | /// 8 | /// Event handler that accumulates test execution information and populates results 9 | /// 10 | /// 11 | public class TestResultEventHandler : ContextTrackingEventHandler 12 | { 13 | 14 | bool ignored; 15 | bool ignoredFromCommandLine; 16 | MethodResult testInitializeMethodResult; 17 | MethodResult testMethodResult; 18 | MethodResult testCleanupMethodResult; 19 | 20 | 21 | protected override void Handle(TestBeginEvent e) 22 | { 23 | base.Handle(e); 24 | ignored = false; 25 | ignoredFromCommandLine = false; 26 | testInitializeMethodResult = null; 27 | testMethodResult = null; 28 | testCleanupMethodResult = null; 29 | } 30 | 31 | 32 | protected override void Handle(TestIgnoredEvent e) 33 | { 34 | base.Handle(e); 35 | ignored = true; 36 | ignoredFromCommandLine = e.IgnoredFromCommandLine; 37 | } 38 | 39 | 40 | protected override void Handle(TestInitializeMethodEndEvent e) 41 | { 42 | base.Handle(e); 43 | testInitializeMethodResult = e.Result; 44 | } 45 | 46 | 47 | protected override void Handle(TestMethodEndEvent e) 48 | { 49 | base.Handle(e); 50 | testMethodResult = e.Result; 51 | } 52 | 53 | 54 | protected override void Handle(TestCleanupMethodEndEvent e) 55 | { 56 | base.Handle(e); 57 | testCleanupMethodResult = e.Result; 58 | } 59 | 60 | 61 | protected override void Handle(TestEndEvent e) 62 | { 63 | base.Handle(e); 64 | e.Result.TestAssemblyPath = CurrentTestAssemblyPath; 65 | e.Result.TestClassFullName = CurrentTestClassFullName; 66 | e.Result.TestName = CurrentTestName; 67 | e.Result.Success = GetSuccess(); 68 | e.Result.Ignored = ignored; 69 | e.Result.IgnoredFromCommandLine = ignoredFromCommandLine; 70 | } 71 | 72 | 73 | bool GetSuccess() 74 | { 75 | if (ignored) return true; 76 | if (testInitializeMethodResult?.Success == false) return false; 77 | if (testMethodResult?.Success == false) return false; 78 | if (testCleanupMethodResult?.Success == false) return false; 79 | return true; 80 | } 81 | 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /testrunner/Results/ExceptionInfo.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.Collections.Generic; 4 | using TestRunner.Infrastructure; 5 | 6 | namespace TestRunner.Results 7 | { 8 | public class ExceptionInfo 9 | { 10 | 11 | public ExceptionInfo(Exception ex) 12 | : this() 13 | { 14 | ParseException(ex); 15 | } 16 | 17 | 18 | public ExceptionInfo() 19 | { 20 | Data = new Dictionary(); 21 | StackTrace = new List(); 22 | } 23 | 24 | 25 | void ParseException(Exception ex) 26 | { 27 | Guard.NotNull(ex, nameof(ex)); 28 | 29 | FullName = ex.GetType().FullName; 30 | Message = ex.Message; 31 | Source = ex.Source; 32 | HelpLink = ex.HelpLink; 33 | ParseData(ex.Data); 34 | ParseStackTrace(ex.StackTrace); 35 | if (ex.InnerException != null) InnerException = new ExceptionInfo(ex.InnerException); 36 | } 37 | 38 | 39 | void ParseData(IDictionary data) 40 | { 41 | if (data == null) return; 42 | foreach (DictionaryEntry de in data) 43 | { 44 | Data.Add(Convert.ToString(de.Key), Convert.ToString(de.Value)); 45 | } 46 | } 47 | 48 | 49 | void ParseStackTrace(string stackTrace) 50 | { 51 | if (stackTrace == null) return; 52 | var lines = StringExtensions.SplitLines(stackTrace); 53 | foreach (var line in lines) 54 | { 55 | ParseStackFrame(line); 56 | } 57 | } 58 | 59 | 60 | void ParseStackFrame(string line) 61 | { 62 | var atpart = line.Trim(); 63 | if (atpart == "") return; 64 | if (atpart.StartsWith("at ", StringComparison.Ordinal)) 65 | { 66 | atpart = atpart.Substring(3); 67 | } 68 | 69 | var inpart = ""; 70 | var inpos = atpart.IndexOf(" in ", StringComparison.Ordinal); 71 | if (inpos >= 0) 72 | { 73 | inpart = atpart.Substring(inpos + 4); 74 | atpart = atpart.Substring(0, inpos); 75 | } 76 | 77 | StackTrace.Add(new StackFrameInfo() { At = atpart, In = inpart }); 78 | } 79 | 80 | 81 | public string FullName { get; set; } 82 | public string Message { get; set; } 83 | public string Source { get; set; } 84 | public string HelpLink { get; set; } 85 | public Dictionary Data; 86 | public List StackTrace { get; set; } 87 | public ExceptionInfo InnerException { get; set; } 88 | 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /testrunner/MSTest/TestContext.cs: -------------------------------------------------------------------------------- 1 | #if NETCOREAPP2_0 2 | using System.Collections.Generic; 3 | #else 4 | using System.Collections; 5 | #endif 6 | using System.Data; 7 | using System.Data.Common; 8 | 9 | namespace TestRunner.MSTest 10 | { 11 | 12 | /// 13 | /// Property values and method implementations for the 14 | /// provided to test methods 15 | /// 16 | /// 17 | public static class TestContext 18 | { 19 | 20 | static TestContext() 21 | { 22 | Clear(); 23 | } 24 | 25 | public static UnitTestOutcome CurrentTestOutcome { get; set; } 26 | 27 | public static DbConnection DataConnection { get; set; } 28 | 29 | public static DataRow DataRow { get; set; } 30 | 31 | public static string DeploymentDirectory { get; set; } 32 | 33 | public static string FullyQualifiedTestClassName { get; set; } 34 | 35 | #if NETCOREAPP2_0 36 | public static IDictionary Properties { get; set; } 37 | #else 38 | public static IDictionary Properties { get; set; } 39 | #endif 40 | 41 | public static string ResultsDirectory { get; set; } 42 | 43 | public static string TestDeploymentDir { get; set; } 44 | 45 | public static string TestDir { get; set; } 46 | 47 | public static string TestLogsDir { get; set; } 48 | 49 | public static string TestName { get; set; } 50 | 51 | public static string TestResultsDirectory { get; set; } 52 | 53 | public static string TestRunDirectory { get; set; } 54 | 55 | public static string TestRunResultsDirectory { get; set; } 56 | 57 | public static void AddResultFile(string fileName) 58 | { 59 | } 60 | 61 | public static void BeginTimer(string timerName) 62 | { 63 | } 64 | 65 | public static void EndTimer(string timerName) 66 | { 67 | } 68 | 69 | public static void WriteLine(string message) 70 | { 71 | } 72 | 73 | public static void WriteLine(string format, params object[] args) 74 | { 75 | } 76 | 77 | public static void Clear() 78 | { 79 | CurrentTestOutcome = UnitTestOutcome.Unknown; 80 | DataConnection = null; 81 | DataRow = null; 82 | DeploymentDirectory = null; 83 | FullyQualifiedTestClassName = null; 84 | Properties = null; 85 | ResultsDirectory = null; 86 | TestDeploymentDir = null; 87 | TestDir = null; 88 | TestLogsDir = null; 89 | TestName = null; 90 | TestResultsDirectory = null; 91 | TestRunDirectory = null; 92 | TestRunResultsDirectory = null; 93 | } 94 | 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /testrunner/MSTest/TestAssembly.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Collections.ObjectModel; 3 | using System.Linq; 4 | using System.Reflection; 5 | using TestRunner.Infrastructure; 6 | 7 | namespace TestRunner.MSTest 8 | { 9 | 10 | public class TestAssembly 11 | { 12 | 13 | public static TestAssembly TryCreate(Assembly assembly) 14 | { 15 | Guard.NotNull(assembly, nameof(assembly)); 16 | var testAssembly = new TestAssembly(assembly); 17 | return testAssembly.TestClasses.Any() ? testAssembly : null; 18 | } 19 | 20 | 21 | TestAssembly(Assembly testAssembly) 22 | { 23 | Assembly = testAssembly; 24 | 25 | FindTestClasses(); 26 | FindAssemblyInitializeMethod(); 27 | FindAssemblyCleanupMethod(); 28 | } 29 | 30 | 31 | public Assembly Assembly 32 | { 33 | get; 34 | private set; 35 | } 36 | 37 | 38 | public ICollection TestClasses 39 | { 40 | get; 41 | private set; 42 | } 43 | 44 | 45 | public MethodInfo AssemblyInitializeMethod 46 | { 47 | get; 48 | private set; 49 | } 50 | 51 | 52 | public MethodInfo AssemblyCleanupMethod 53 | { 54 | get; 55 | private set; 56 | } 57 | 58 | 59 | void FindTestClasses() 60 | { 61 | TestClasses = 62 | new ReadOnlyCollection( 63 | Assembly.GetTypes() 64 | .Select(t => TestClass.TryCreate(this, t)) 65 | .Where(t => t != null) 66 | .ToList()); 67 | } 68 | 69 | 70 | void FindAssemblyInitializeMethod() 71 | { 72 | var methods = 73 | TestClasses 74 | .Select(c => c.AssemblyInitializeMethod) 75 | .Where(m => m != null) 76 | .ToList(); 77 | 78 | if (methods.Count > 1) 79 | throw new UserException( 80 | $"[TestAssembly] {Assembly.FullName} contains more than one [AssemblyInitialize] method"); 81 | 82 | if (methods.Count == 0) 83 | return; 84 | 85 | AssemblyInitializeMethod = methods[0]; 86 | } 87 | 88 | 89 | void FindAssemblyCleanupMethod() 90 | { 91 | var methods = 92 | TestClasses 93 | .Select(c => c.AssemblyCleanupMethod) 94 | .Where(c => c != null) 95 | .ToList(); 96 | 97 | if (methods.Count > 1) 98 | throw new UserException( 99 | $"[TestAssembly] {Assembly.FullName} contains more than one [AssemblyCleanup] method"); 100 | 101 | if (methods.Count == 0) 102 | return; 103 | 104 | AssemblyCleanupMethod = methods[0]; 105 | } 106 | 107 | } 108 | 109 | } 110 | -------------------------------------------------------------------------------- /hacking.md: -------------------------------------------------------------------------------- 1 | Building 2 | ======== 3 | 4 | .NET Framework 5 | ``` 6 | dotnet publish -f net461 7 | ``` 8 | 9 | .NET Core 10 | ``` 11 | dotnet publish -f netcoreapp2.0 12 | ``` 13 | 14 | Mono 15 | ``` 16 | msbuild /p:TargetFramework=net461 /t:Restore 17 | msbuild /p:TargetFramework=net461 /t:Rebuild 18 | msbuild /p:TargetFramework=net461 /t:Publish 19 | ``` 20 | 21 | 22 | 23 | Continuous Integration 24 | ====================== 25 | 26 | Appveyor (Windows) 27 | ------------------ 28 | 29 | [![Build status](https://ci.appveyor.com/api/projects/status/v8s72ij64an7kr87?svg=true)](https://ci.appveyor.com/project/macro187/testrunner) 30 | 31 | net461 build and tests 32 | 33 | netcoreapp2.0 build and tests 34 | 35 | 36 | Travis (Linux) 37 | -------------- 38 | 39 | [![Build Status](https://travis-ci.org/macro187/testrunner.svg?branch=master)](https://travis-ci.org/macro187/testrunner) 40 | 41 | net461 build and tests (Mono) 42 | 43 | netcoreapp2.0 build and tests (.NET Core) 44 | 45 | 46 | 47 | Design 48 | ====== 49 | 50 | Program 51 | ------- 52 | 53 | The `Program` namespace contains the `Main()` program code that sets up error 54 | handlers, parses command line arguments, and takes appropriate top-level 55 | actions. 56 | 57 | The initial parent `testrunner` process runs the test file(s) specified on the 58 | command-line by reinvoking separate child `testrunner` processes for each. 59 | Child processes are instructed via command-line options to produce output in a 60 | machine-readable format, which the parent interprets and combines into a unified 61 | event stream for final analysis and output. 62 | 63 | 64 | Runners 65 | ------- 66 | 67 | Child processes run test files by delegating to runner routines in the `Runners` 68 | namespace. These routines activate test assembly `.config` files, locate test 69 | classes, and run initialize, test, and cleanup methods in the right order. 70 | 71 | 72 | MSTest 73 | ------ 74 | 75 | The runners interpret and interact with test assemblies through types in the 76 | `MSTest` namespace, which use reflection to discover, bind, and interact with 77 | test assembly elements at runtime. These elements include the test assemblies 78 | themselves, test classes, initialization, test, and cleanup methods, MSTest 79 | attributes, and a `TestContext` implementation. 80 | 81 | 82 | Events 83 | ------ 84 | 85 | As the runners execute tests, they emit events from the `Events` namespace... 86 | 87 | 88 | EventHandlers 89 | ------------- 90 | 91 | ...into a pipeline of event handlers in the `EventHandlers` namespace. 92 | Individual handlers focus on single supporting responsibilities like 93 | measurement, analysis, aggregation, and output. Distributing responsibility 94 | across the runners and handlers keeps down their individual complexity. 95 | 96 | 97 | Results 98 | ------- 99 | 100 | As tests run, event handlers record results in types from the `Results` 101 | namespace. 102 | 103 | 104 | Infrastructure 105 | -------------- 106 | 107 | The `Infrastructure` namespace contains general support functionality used 108 | throughout the rest of the application. 109 | 110 | -------------------------------------------------------------------------------- /testrunner/EventHandlers/EventHandlerPipeline.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using TestRunner.Events; 3 | using TestRunner.Infrastructure; 4 | 5 | namespace TestRunner.EventHandlers 6 | { 7 | 8 | /// 9 | /// Event handler pipeline 10 | /// 11 | /// 12 | /// 13 | /// The TestRunner program raises events as it runs. Each handler in the pipeline has the opportunity to perform 14 | /// actions and/or modify the event before propagating to the next one. 15 | /// 16 | /// 17 | public static class EventHandlerPipeline 18 | { 19 | 20 | static EventHandlerPipeline() 21 | { 22 | Append(new EventHandler()); 23 | } 24 | 25 | 26 | private static EventHandler First; 27 | 28 | 29 | private static EventHandler Last; 30 | 31 | 32 | /// 33 | /// Raise an event 34 | /// 35 | /// 36 | public static void Raise(TestRunnerEvent e) 37 | { 38 | First.Handle(e); 39 | } 40 | 41 | 42 | /// 43 | /// Add an event handler to the beginning of the pipeline 44 | /// 45 | /// 46 | /// 47 | /// An that, when disposed, removes from the pipeline 48 | /// 49 | /// 50 | public static IDisposable Prepend(EventHandler handler) 51 | { 52 | Guard.NotNull(handler, nameof(handler)); 53 | handler.Next = First; 54 | First = handler; 55 | if (Last == null) Last = handler; 56 | return new Disposable(() => Remove(handler)); 57 | } 58 | 59 | 60 | /// 61 | /// Add an event handler to the end of the pipeline 62 | /// 63 | /// 64 | /// 65 | /// An that, when disposed, removes from the pipeline 66 | /// 67 | /// 68 | public static IDisposable Append(EventHandler handler) 69 | { 70 | Guard.NotNull(handler, nameof(handler)); 71 | if (First == null) First = handler; 72 | if (Last != null) Last.Next = handler; 73 | Last = handler; 74 | return new Disposable(() => Remove(handler)); 75 | } 76 | 77 | 78 | static void Remove(EventHandler handler) 79 | { 80 | Guard.NotNull(handler, nameof(handler)); 81 | 82 | for (EventHandler h = First, prev = null; h != null; prev = h, h = h.Next) 83 | { 84 | if (h != handler) continue; 85 | 86 | if (handler == First) 87 | { 88 | First = handler.Next; 89 | } 90 | 91 | if (prev != null) 92 | { 93 | prev.Next = handler.Next; 94 | } 95 | 96 | if (handler == Last) 97 | { 98 | Last = prev; 99 | } 100 | 101 | handler.Next = null; 102 | } 103 | } 104 | 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /testrunner/EventHandlers/TestClassResultEventHandler.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using TestRunner.Events; 4 | using TestRunner.Results; 5 | 6 | namespace TestRunner.EventHandlers 7 | { 8 | 9 | /// 10 | /// Event handler that accumulates test class execution information and populates 11 | /// results 12 | /// 13 | /// 14 | public class TestClassResultEventHandler : ContextTrackingEventHandler 15 | { 16 | 17 | bool ignored; 18 | bool ignoredFromCommandLine; 19 | MethodResult classInitializeMethodResult; 20 | IList testResults; 21 | MethodResult classCleanupMethodResult; 22 | 23 | 24 | protected override void Handle(TestClassBeginEvent e) 25 | { 26 | base.Handle(e); 27 | ignored = false; 28 | ignoredFromCommandLine = false; 29 | classInitializeMethodResult = null; 30 | testResults = new List(); 31 | classCleanupMethodResult = null; 32 | } 33 | 34 | 35 | protected override void Handle(TestClassIgnoredEvent e) 36 | { 37 | base.Handle(e); 38 | ignored = true; 39 | ignoredFromCommandLine = e.IgnoredFromCommandLine; 40 | } 41 | 42 | 43 | protected override void Handle(ClassInitializeMethodEndEvent e) 44 | { 45 | base.Handle(e); 46 | classInitializeMethodResult = e.Result; 47 | } 48 | 49 | 50 | protected override void Handle(TestEndEvent e) 51 | { 52 | base.Handle(e); 53 | testResults.Add(e.Result); 54 | } 55 | 56 | 57 | protected override void Handle(ClassCleanupMethodEndEvent e) 58 | { 59 | base.Handle(e); 60 | classCleanupMethodResult = e.Result; 61 | } 62 | 63 | 64 | protected override void Handle(TestClassEndEvent e) 65 | { 66 | base.Handle(e); 67 | e.Result.TestAssemblyPath = CurrentTestAssemblyPath; 68 | e.Result.TestClassFullName = CurrentTestClassFullName; 69 | e.Result.Success = GetSuccess(); 70 | e.Result.ClassIgnored = ignored; 71 | e.Result.ClassIgnoredFromCommandLine = ignoredFromCommandLine; 72 | e.Result.InitializePresent = classInitializeMethodResult != null; 73 | e.Result.InitializeSucceeded = classInitializeMethodResult?.Success ?? false; 74 | e.Result.TestsTotal = testResults.Count; 75 | e.Result.TestsRan = testResults.Count - testResults.Count(r => r.Ignored); 76 | e.Result.TestsIgnored = testResults.Count(r => r.Ignored); 77 | e.Result.TestsPassed = testResults.Count(r => !r.Ignored && r.Success); 78 | e.Result.TestsFailed = testResults.Count(r => !r.Success); 79 | e.Result.CleanupPresent = classCleanupMethodResult != null; 80 | e.Result.CleanupSucceeded = classCleanupMethodResult?.Success ?? false; 81 | } 82 | 83 | 84 | bool GetSuccess() 85 | { 86 | if (ignored) return true; 87 | if (classInitializeMethodResult?.Success == false) return false; 88 | if (testResults.Any(r => !r.Success)) return false; 89 | if (classCleanupMethodResult?.Success == false) return false; 90 | return true; 91 | } 92 | 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /testrunner/EventHandlers/TestAssemblyResultEventHandler.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using TestRunner.Events; 4 | using TestRunner.Results; 5 | 6 | namespace TestRunner.EventHandlers 7 | { 8 | 9 | /// 10 | /// Event handler that accumulates test assembly execution information and populates 11 | /// results 12 | /// 13 | /// 14 | public class TestAssemblyResultEventHandler : ContextTrackingEventHandler 15 | { 16 | 17 | bool assemblyNotFound; 18 | bool assemblyNotDotNet; 19 | bool assemblyNotTest; 20 | string configFilePath; 21 | MethodResult assemblyInitializeResult; 22 | IList testClassResults; 23 | MethodResult assemblyCleanupResult; 24 | 25 | 26 | protected override void Handle(TestAssemblyBeginEvent e) 27 | { 28 | base.Handle(e); 29 | assemblyNotFound = false; 30 | assemblyNotDotNet = false; 31 | assemblyNotTest = false; 32 | configFilePath = null; 33 | assemblyInitializeResult = null; 34 | testClassResults = new List(); 35 | assemblyCleanupResult = null; 36 | } 37 | 38 | 39 | protected override void Handle(TestAssemblyNotFoundEvent e) 40 | { 41 | base.Handle(e); 42 | assemblyNotFound = true; 43 | } 44 | 45 | 46 | protected override void Handle(TestAssemblyNotDotNetEvent e) 47 | { 48 | base.Handle(e); 49 | assemblyNotDotNet = true; 50 | } 51 | 52 | 53 | protected override void Handle(TestAssemblyNotTestEvent e) 54 | { 55 | base.Handle(e); 56 | assemblyNotTest = true; 57 | } 58 | 59 | 60 | protected override void Handle(TestAssemblyConfigFileSwitchedEvent e) 61 | { 62 | base.Handle(e); 63 | configFilePath = e.Path; 64 | } 65 | 66 | 67 | protected override void Handle(AssemblyInitializeMethodEndEvent e) 68 | { 69 | base.Handle(e); 70 | assemblyInitializeResult = e.Result; 71 | } 72 | 73 | 74 | protected override void Handle(TestClassEndEvent e) 75 | { 76 | base.Handle(e); 77 | testClassResults.Add(e.Result); 78 | } 79 | 80 | 81 | protected override void Handle(AssemblyCleanupMethodEndEvent e) 82 | { 83 | base.Handle(e); 84 | assemblyCleanupResult = e.Result; 85 | } 86 | 87 | 88 | protected override void Handle(TestAssemblyEndEvent e) 89 | { 90 | base.Handle(e); 91 | e.Result.TestAssemblyPath = CurrentTestAssemblyPath; 92 | e.Result.Success = GetSuccess(); 93 | } 94 | 95 | 96 | bool GetSuccess() 97 | { 98 | if (assemblyNotFound) return false; 99 | if (assemblyNotDotNet) return true; 100 | if (assemblyNotTest) return true; 101 | if (assemblyInitializeResult?.Success == false) return false; 102 | if (testClassResults.Any(r => !r.Success)) return false; 103 | if (assemblyCleanupResult?.Success == false) return false; 104 | return true; 105 | } 106 | 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /testrunner/EventHandlers/MethodResultEventHandler.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | using TestRunner.Events; 4 | using TestRunner.Results; 5 | 6 | namespace TestRunner.EventHandlers 7 | { 8 | 9 | /// 10 | /// Event handler that accumulates method execution information and populates end event results with it 11 | /// 12 | /// 13 | public class MethodResultEventHandler : EventHandler 14 | { 15 | 16 | readonly Stopwatch Stopwatch = new Stopwatch(); 17 | bool inMethod; 18 | ExceptionInfo exception; 19 | bool exceptionWasExpected; 20 | 21 | 22 | protected override void Handle(AssemblyInitializeMethodBeginEvent e) { HandleBegin(); } 23 | protected override void Handle(AssemblyInitializeMethodEndEvent e) { HandleEnd(e); } 24 | protected override void Handle(AssemblyCleanupMethodBeginEvent e) { HandleBegin(); } 25 | protected override void Handle(AssemblyCleanupMethodEndEvent e) { HandleEnd(e); } 26 | protected override void Handle(ClassInitializeMethodBeginEvent e) { HandleBegin(); } 27 | protected override void Handle(ClassInitializeMethodEndEvent e) { HandleEnd(e); } 28 | protected override void Handle(ClassCleanupMethodBeginEvent e) { HandleBegin(); } 29 | protected override void Handle(ClassCleanupMethodEndEvent e) { HandleEnd(e); } 30 | protected override void Handle(TestContextSetterBeginEvent e) { HandleBegin(); } 31 | protected override void Handle(TestContextSetterEndEvent e) { HandleEnd(e); } 32 | protected override void Handle(TestInitializeMethodBeginEvent e) { HandleBegin(); } 33 | protected override void Handle(TestInitializeMethodEndEvent e) { HandleEnd(e); } 34 | protected override void Handle(TestMethodBeginEvent e) { HandleBegin(); } 35 | protected override void Handle(TestMethodEndEvent e) { HandleEnd(e); } 36 | protected override void Handle(TestCleanupMethodBeginEvent e) { HandleBegin(); } 37 | protected override void Handle(TestCleanupMethodEndEvent e) { HandleEnd(e); } 38 | 39 | 40 | protected override void Handle(MethodExpectedExceptionEvent e) 41 | { 42 | exception = e.Exception; 43 | exceptionWasExpected = true; 44 | } 45 | 46 | 47 | protected override void Handle(MethodUnexpectedExceptionEvent e) 48 | { 49 | exception = e.Exception; 50 | exceptionWasExpected = false; 51 | } 52 | 53 | 54 | void HandleBegin() 55 | { 56 | if (inMethod) throw new InvalidOperationException("Method began before previous one ended"); 57 | inMethod = true; 58 | exception = null; 59 | exceptionWasExpected = false; 60 | StartStopwatch(); 61 | } 62 | 63 | 64 | void HandleEnd(MethodEndEvent e) 65 | { 66 | if (!inMethod) throw new InvalidOperationException("Method ended before it started"); 67 | e.Result.ElapsedMilliseconds = StopStopwatch(); 68 | e.Result.Exception = exception; 69 | e.Result.Success = exception == null || exceptionWasExpected; 70 | inMethod = false; 71 | } 72 | 73 | 74 | void StartStopwatch() 75 | { 76 | Stopwatch.Reset(); 77 | Stopwatch.Start(); 78 | } 79 | 80 | 81 | long StopStopwatch() 82 | { 83 | Stopwatch.Stop(); 84 | return Stopwatch.ElapsedMilliseconds; 85 | } 86 | 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /testrunner/Runners/TestAssemblyRunner.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Reflection; 4 | using TestRunner.MSTest; 5 | using TestRunner.Infrastructure; 6 | using TestRunner.Events; 7 | using TestRunner.EventHandlers; 8 | 9 | namespace TestRunner.Runners 10 | { 11 | static class TestAssemblyRunner 12 | { 13 | 14 | /// 15 | /// Run tests in a test assembly 16 | /// 17 | /// 18 | [System.Diagnostics.CodeAnalysis.SuppressMessage( 19 | "Microsoft.Reliability", 20 | "CA2001:AvoidCallingProblematicMethods", 21 | MessageId = "System.Reflection.Assembly.LoadFrom", 22 | Justification = "Need to load assemblies in order to run tests")] 23 | public static void Run(string assemblyPath) 24 | { 25 | Guard.NotNull(assemblyPath, nameof(assemblyPath)); 26 | 27 | EventHandlerPipeline.Raise(new TestAssemblyBeginEvent() { Path = assemblyPath }); 28 | 29 | do 30 | { 31 | // 32 | // Resolve full path to test assembly file 33 | // 34 | string fullAssemblyPath = 35 | Path.IsPathRooted(assemblyPath) 36 | ? assemblyPath 37 | : Path.Combine(Environment.CurrentDirectory, assemblyPath); 38 | 39 | if (!File.Exists(fullAssemblyPath)) 40 | { 41 | EventHandlerPipeline.Raise(new TestAssemblyNotFoundEvent() { Path = fullAssemblyPath }); 42 | break; 43 | } 44 | 45 | // 46 | // Load assembly 47 | // 48 | Assembly assembly; 49 | try 50 | { 51 | assembly = Assembly.LoadFrom(fullAssemblyPath); 52 | } 53 | catch (BadImageFormatException) 54 | { 55 | EventHandlerPipeline.Raise(new TestAssemblyNotDotNetEvent() { Path = fullAssemblyPath }); 56 | break; 57 | } 58 | 59 | // 60 | // Interpret as test assembly 61 | // 62 | var testAssembly = TestAssembly.TryCreate(assembly); 63 | if (testAssembly == null) 64 | { 65 | EventHandlerPipeline.Raise(new TestAssemblyNotTestEvent() { Path = fullAssemblyPath }); 66 | break; 67 | } 68 | 69 | // 70 | // Activate the test assembly's .config file if present 71 | // 72 | ConfigFileSwitcher.SwitchTo(testAssembly.Assembly.Location + ".config"); 73 | 74 | // 75 | // Run [AssemblyInitialize] method 76 | // 77 | if (!MethodRunner.RunAssemblyInitializeMethod(testAssembly)) break; 78 | 79 | // 80 | // Run tests in each [TestClass] 81 | // 82 | foreach (var testClass in testAssembly.TestClasses) 83 | { 84 | TestClassRunner.Run(testClass); 85 | } 86 | 87 | // 88 | // Run [AssemblyCleanup] method 89 | // 90 | MethodRunner.RunAssemblyCleanupMethod(testAssembly); 91 | } 92 | while (false); 93 | 94 | EventHandlerPipeline.Raise(new TestAssemblyEndEvent()); 95 | } 96 | 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /testrunner/EventHandlers/EventHandler.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using TestRunner.Events; 3 | using TestRunner.Infrastructure; 4 | 5 | namespace TestRunner.EventHandlers 6 | { 7 | public class EventHandler 8 | { 9 | 10 | public EventHandler Next { get; set; } 11 | 12 | 13 | public virtual void Handle(TestRunnerEvent e) 14 | { 15 | Guard.NotNull(e, nameof(e)); 16 | 17 | var method = GetType().GetMethod( 18 | "Handle", 19 | BindingFlags.NonPublic | BindingFlags.Instance, 20 | null, 21 | new[]{ e.GetType() }, 22 | null); 23 | 24 | method.Invoke(this, new[]{ e }); 25 | 26 | Next?.Handle(e); 27 | } 28 | 29 | 30 | protected virtual void Handle(ProgramBannerEvent e) {} 31 | protected virtual void Handle(ProgramUsageEvent e) {} 32 | protected virtual void Handle(ProgramUserErrorEvent e) {} 33 | protected virtual void Handle(ProgramInternalErrorEvent e) {} 34 | protected virtual void Handle(TestRunBeginEvent e) {} 35 | protected virtual void Handle(TestRunEndEvent e) {} 36 | protected virtual void Handle(TestAssemblyBeginEvent e) {} 37 | protected virtual void Handle(TestAssemblyNotFoundEvent e) {} 38 | protected virtual void Handle(TestAssemblyNotDotNetEvent e) {} 39 | protected virtual void Handle(TestAssemblyNotTestEvent e) {} 40 | protected virtual void Handle(TestAssemblyConfigFileSwitchedEvent e) {} 41 | protected virtual void Handle(TestAssemblyEndEvent e) {} 42 | protected virtual void Handle(TestClassBeginEvent e) {} 43 | protected virtual void Handle(TestClassIgnoredEvent e) {} 44 | protected virtual void Handle(TestClassEndEvent e) {} 45 | protected virtual void Handle(TestBeginEvent e) {} 46 | protected virtual void Handle(TestIgnoredEvent e) {} 47 | protected virtual void Handle(TestEndEvent e) {} 48 | protected virtual void Handle(AssemblyInitializeMethodBeginEvent e) {} 49 | protected virtual void Handle(AssemblyInitializeMethodEndEvent e) {} 50 | protected virtual void Handle(AssemblyCleanupMethodBeginEvent e) {} 51 | protected virtual void Handle(AssemblyCleanupMethodEndEvent e) {} 52 | protected virtual void Handle(ClassInitializeMethodBeginEvent e) {} 53 | protected virtual void Handle(ClassInitializeMethodEndEvent e) {} 54 | protected virtual void Handle(ClassCleanupMethodBeginEvent e) {} 55 | protected virtual void Handle(ClassCleanupMethodEndEvent e) {} 56 | protected virtual void Handle(TestContextSetterBeginEvent e) {} 57 | protected virtual void Handle(TestContextSetterEndEvent e) {} 58 | protected virtual void Handle(TestInitializeMethodBeginEvent e) {} 59 | protected virtual void Handle(TestInitializeMethodEndEvent e) {} 60 | protected virtual void Handle(TestMethodBeginEvent e) {} 61 | protected virtual void Handle(TestMethodEndEvent e) {} 62 | protected virtual void Handle(TestCleanupMethodBeginEvent e) {} 63 | protected virtual void Handle(TestCleanupMethodEndEvent e) {} 64 | protected virtual void Handle(MethodExpectedExceptionEvent e) {} 65 | protected virtual void Handle(MethodUnexpectedExceptionEvent e) {} 66 | protected virtual void Handle(StandardOutputEvent e) {} 67 | protected virtual void Handle(ErrorOutputEvent e) {} 68 | protected virtual void Handle(TraceOutputEvent e) {} 69 | 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /testrunner/Infrastructure/ProcessExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | using System.IO; 4 | using System.Text; 5 | using System.Threading; 6 | 7 | namespace TestRunner.Infrastructure 8 | { 9 | public static class ProcessExtensions 10 | { 11 | 12 | static ProcessExtensions() 13 | { 14 | var driverPath = Process.GetCurrentProcess().MainModule.FileName; 15 | var driverName = Path.GetFileNameWithoutExtension(driverPath).ToLowerInvariant(); 16 | switch (driverName) 17 | { 18 | case "dotnet": 19 | case "mono": 20 | DotnetDriver = driverPath; 21 | break; 22 | } 23 | } 24 | 25 | 26 | /// 27 | /// Path to the .NET "driver" program (dotnet or mono) running the process, or null if none 28 | /// 29 | /// 30 | static string DotnetDriver { get; } 31 | 32 | 33 | public static ProcessExecuteResults ExecuteDotnet(string fileName, string arguments) 34 | { 35 | if (DotnetDriver != null) 36 | { 37 | arguments = fileName + " " + arguments; 38 | fileName = DotnetDriver; 39 | } 40 | return Execute(fileName, arguments); 41 | } 42 | 43 | 44 | public static int ExecuteDotnet( 45 | string fileName, 46 | string arguments, 47 | Action onStandardOutput, 48 | Action onErrorOutput) 49 | { 50 | if (DotnetDriver != null) 51 | { 52 | arguments = fileName + " " + arguments; 53 | fileName = DotnetDriver; 54 | } 55 | return Execute(fileName, arguments, onStandardOutput, onErrorOutput); 56 | } 57 | 58 | 59 | static ProcessExecuteResults Execute(string fileName, string arguments) 60 | { 61 | var standardOutput = new StringBuilder(); 62 | var errorOutput = new StringBuilder(); 63 | var output = new StringBuilder(); 64 | var exitCode = Execute( 65 | fileName, 66 | arguments, 67 | (_, line) => { 68 | Console.Out.WriteLine(line); 69 | standardOutput.AppendLine(line); 70 | output.AppendLine(line); 71 | }, 72 | (_, line) => { 73 | Console.Error.WriteLine(line); 74 | errorOutput.AppendLine(line); 75 | output.AppendLine(line); 76 | }); 77 | return new ProcessExecuteResults( 78 | standardOutput.ToString(), 79 | errorOutput.ToString(), 80 | output.ToString(), 81 | exitCode); 82 | } 83 | 84 | 85 | static int Execute( 86 | string fileName, 87 | string arguments, 88 | Action onStandardOutput, 89 | Action onErrorOutput) 90 | { 91 | Guard.NotNull(fileName, nameof(fileName)); 92 | Guard.NotNull(arguments, nameof(arguments)); 93 | Guard.NotNull(onStandardOutput, nameof(onStandardOutput)); 94 | Guard.NotNull(onErrorOutput, nameof(onErrorOutput)); 95 | 96 | using (var proc = new Process()) 97 | { 98 | bool exited = false; 99 | 100 | proc.StartInfo.FileName = fileName; 101 | proc.StartInfo.Arguments = arguments; 102 | proc.StartInfo.UseShellExecute = false; 103 | proc.StartInfo.RedirectStandardOutput = true; 104 | proc.StartInfo.RedirectStandardError = true; 105 | proc.OutputDataReceived += (_,e) => { 106 | onStandardOutput(proc, e.Data ?? ""); 107 | }; 108 | proc.ErrorDataReceived += (_,e) => { 109 | onErrorOutput(proc, e.Data ?? ""); 110 | }; 111 | proc.EnableRaisingEvents = true; 112 | proc.Exited += (_,__) => { 113 | exited = true; 114 | }; 115 | 116 | proc.Start(); 117 | proc.BeginOutputReadLine(); 118 | proc.BeginErrorReadLine(); 119 | while (!exited) Thread.Yield(); 120 | proc.WaitForExit(); 121 | 122 | return proc.ExitCode; 123 | } 124 | } 125 | 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /testrunner.sln: -------------------------------------------------------------------------------- 1 | Microsoft Visual Studio Solution File, Format Version 12.00 2 | # Visual Studio Version 16 3 | VisualStudioVersion = 16.0.29123.88 4 | MinimumVisualStudioVersion = 15.0.26124.0 5 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "testrunner", "testrunner\testrunner.csproj", "{924F65F6-9446-4059-8715-1427FFA0DB75}" 6 | EndProject 7 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "testrunner.Tests.DifferentConfigValue", "testrunner.Tests.DifferentConfigValue\testrunner.Tests.DifferentConfigValue.csproj", "{E1A8A4FB-04C7-4891-9733-78ADD081C4CA}" 8 | EndProject 9 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "testrunner.Tests.Fail", "testrunner.Tests.Fail\testrunner.Tests.Fail.csproj", "{0095A6EA-1395-47FD-B2AB-C8EB5573EB8E}" 10 | EndProject 11 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "testrunner.Tests.MSTest", "testrunner.Tests.MSTest\testrunner.Tests.MSTest.csproj", "{0C19969A-4AE7-48F2-8BB8-A552F1537F3B}" 12 | EndProject 13 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "testrunner.Tests.Pass", "testrunner.Tests.Pass\testrunner.Tests.Pass.csproj", "{9F12C883-D4D4-4AA3-A9E5-50E799FDA758}" 14 | EndProject 15 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "testrunner.Tests.ReferencedAssembly", "testrunner.Tests.ReferencedAssembly\testrunner.Tests.ReferencedAssembly.csproj", "{3F792DDB-DAA9-4A4E-8550-FA5CA0B919C8}" 16 | EndProject 17 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{F158A9B5-CB1E-4C17-92F2-3914A2985157}" 18 | ProjectSection(SolutionItems) = preProject 19 | .editorconfig = .editorconfig 20 | .gitignore = .gitignore 21 | .produce = .produce 22 | .travis.yml = .travis.yml 23 | changelog.md = changelog.md 24 | hacking.md = hacking.md 25 | license.txt = license.txt 26 | readme.md = readme.md 27 | EndProjectSection 28 | EndProject 29 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "testrunner.Tests", "testrunner.Tests\testrunner.Tests.csproj", "{C084FF39-C166-4238-83DF-F752A24738D3}" 30 | EndProject 31 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "testrunner.Tests.IncludeExclude", "testrunner.Tests.IncludeExclude\testrunner.Tests.IncludeExclude.csproj", "{16767246-C565-47B2-83FE-684F79CD33AC}" 32 | EndProject 33 | Global 34 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 35 | Debug|Any CPU = Debug|Any CPU 36 | Release|Any CPU = Release|Any CPU 37 | EndGlobalSection 38 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 39 | {924F65F6-9446-4059-8715-1427FFA0DB75}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 40 | {924F65F6-9446-4059-8715-1427FFA0DB75}.Debug|Any CPU.Build.0 = Debug|Any CPU 41 | {924F65F6-9446-4059-8715-1427FFA0DB75}.Release|Any CPU.ActiveCfg = Release|Any CPU 42 | {924F65F6-9446-4059-8715-1427FFA0DB75}.Release|Any CPU.Build.0 = Release|Any CPU 43 | {E1A8A4FB-04C7-4891-9733-78ADD081C4CA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 44 | {E1A8A4FB-04C7-4891-9733-78ADD081C4CA}.Debug|Any CPU.Build.0 = Debug|Any CPU 45 | {E1A8A4FB-04C7-4891-9733-78ADD081C4CA}.Release|Any CPU.ActiveCfg = Release|Any CPU 46 | {E1A8A4FB-04C7-4891-9733-78ADD081C4CA}.Release|Any CPU.Build.0 = Release|Any CPU 47 | {0095A6EA-1395-47FD-B2AB-C8EB5573EB8E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 48 | {0095A6EA-1395-47FD-B2AB-C8EB5573EB8E}.Debug|Any CPU.Build.0 = Debug|Any CPU 49 | {0095A6EA-1395-47FD-B2AB-C8EB5573EB8E}.Release|Any CPU.ActiveCfg = Release|Any CPU 50 | {0095A6EA-1395-47FD-B2AB-C8EB5573EB8E}.Release|Any CPU.Build.0 = Release|Any CPU 51 | {0C19969A-4AE7-48F2-8BB8-A552F1537F3B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 52 | {0C19969A-4AE7-48F2-8BB8-A552F1537F3B}.Debug|Any CPU.Build.0 = Debug|Any CPU 53 | {0C19969A-4AE7-48F2-8BB8-A552F1537F3B}.Release|Any CPU.ActiveCfg = Release|Any CPU 54 | {0C19969A-4AE7-48F2-8BB8-A552F1537F3B}.Release|Any CPU.Build.0 = Release|Any CPU 55 | {9F12C883-D4D4-4AA3-A9E5-50E799FDA758}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 56 | {9F12C883-D4D4-4AA3-A9E5-50E799FDA758}.Debug|Any CPU.Build.0 = Debug|Any CPU 57 | {9F12C883-D4D4-4AA3-A9E5-50E799FDA758}.Release|Any CPU.ActiveCfg = Release|Any CPU 58 | {9F12C883-D4D4-4AA3-A9E5-50E799FDA758}.Release|Any CPU.Build.0 = Release|Any CPU 59 | {3F792DDB-DAA9-4A4E-8550-FA5CA0B919C8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 60 | {3F792DDB-DAA9-4A4E-8550-FA5CA0B919C8}.Debug|Any CPU.Build.0 = Debug|Any CPU 61 | {3F792DDB-DAA9-4A4E-8550-FA5CA0B919C8}.Release|Any CPU.ActiveCfg = Release|Any CPU 62 | {3F792DDB-DAA9-4A4E-8550-FA5CA0B919C8}.Release|Any CPU.Build.0 = Release|Any CPU 63 | {C084FF39-C166-4238-83DF-F752A24738D3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 64 | {C084FF39-C166-4238-83DF-F752A24738D3}.Debug|Any CPU.Build.0 = Debug|Any CPU 65 | {C084FF39-C166-4238-83DF-F752A24738D3}.Release|Any CPU.ActiveCfg = Release|Any CPU 66 | {C084FF39-C166-4238-83DF-F752A24738D3}.Release|Any CPU.Build.0 = Release|Any CPU 67 | {16767246-C565-47B2-83FE-684F79CD33AC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 68 | {16767246-C565-47B2-83FE-684F79CD33AC}.Debug|Any CPU.Build.0 = Debug|Any CPU 69 | {16767246-C565-47B2-83FE-684F79CD33AC}.Release|Any CPU.ActiveCfg = Release|Any CPU 70 | {16767246-C565-47B2-83FE-684F79CD33AC}.Release|Any CPU.Build.0 = Release|Any CPU 71 | EndGlobalSection 72 | GlobalSection(SolutionProperties) = preSolution 73 | HideSolutionNode = FALSE 74 | EndGlobalSection 75 | GlobalSection(ExtensibilityGlobals) = postSolution 76 | SolutionGuid = {50FFC009-FF2E-4BB6-9E37-EAB80C64B4E0} 77 | EndGlobalSection 78 | EndGlobal 79 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | TestRunner 2 | ========== 3 | 4 | A console **MSTest** runner 5 | 6 | 7 | 8 | Features 9 | ======== 10 | 11 | **Lightweight**, with no external dependencies. 12 | 13 | **Cross-platform**, tested on .NET Framework (Windows), .NET Core (Windows and 14 | Linux), and Mono (Linux). 15 | 16 | **Process isolation** runs test assemblies in separate processes. 17 | 18 | **Reflection-based test discovery** supports test assemblies built against any 19 | variant or version of the MSTest dll. 20 | 21 | **Test output** captured from Console.Out, Console.Error, and 22 | System.Diagnostics.Trace. 23 | 24 | **Timings** measured for all test, initialize, and cleanup methods. 25 | 26 | **Exception details** provided for expected exceptions in tests, and unexpected 27 | failures in test, initialize, and cleanup methods. 28 | 29 | **Test attributes** supported include 30 | [\[TestClass\]](https://docs.microsoft.com/en-gb/dotnet/api/microsoft.visualstudio.testtools.unittesting.testclassattribute), 31 | [\[TestMethod\]](https://docs.microsoft.com/en-gb/dotnet/api/microsoft.visualstudio.testtools.unittesting.testmethodattribute), 32 | [\[TestInitialize\]](https://docs.microsoft.com/en-gb/dotnet/api/microsoft.visualstudio.testtools.unittesting.testinitializeattribute), 33 | [\[TestCleanup\]](https://docs.microsoft.com/en-gb/dotnet/api/microsoft.visualstudio.testtools.unittesting.testcleanupattribute), 34 | [\[ClassInitialize\]](https://docs.microsoft.com/en-gb/dotnet/api/microsoft.visualstudio.testtools.unittesting.classinitializeattribute), 35 | [\[ClassCleanup\]](https://docs.microsoft.com/en-gb/dotnet/api/microsoft.visualstudio.testtools.unittesting.classcleanupattribute), 36 | [\[AssemblyInitialize\]](https://docs.microsoft.com/en-gb/dotnet/api/microsoft.visualstudio.testtools.unittesting.assemblyinitializeattribute), 37 | [\[AssemblyCleanup\]](https://docs.microsoft.com/en-gb/dotnet/api/microsoft.visualstudio.testtools.unittesting.assemblycleanupattribute), 38 | [\[ExpectedException\]](https://docs.microsoft.com/en-gb/dotnet/api/microsoft.visualstudio.testtools.unittesting.expectedexceptionattribute), 39 | and 40 | [\[Ignore\]](https://docs.microsoft.com/en-gb/dotnet/api/microsoft.visualstudio.testtools.unittesting.ignoreattribute). 41 | 42 | **TestContext** members supported include 43 | [CurrentTestOutcome](https://docs.microsoft.com/en-gb/dotnet/api/microsoft.visualstudio.testtools.unittesting.testcontext.currenttestoutcome), 44 | [FullyQualifiedTestClassName](https://docs.microsoft.com/en-gb/dotnet/api/microsoft.visualstudio.testtools.unittesting.testcontext.fullyqualifiedtestclassname), 45 | and 46 | [TestName](https://docs.microsoft.com/en-gb/dotnet/api/microsoft.visualstudio.testtools.unittesting.testcontext.testname). 47 | 48 | **Assembly .config files** are supported. 49 | 50 | 51 | 52 | Limitations 53 | =========== 54 | 55 | Some test attributes are unsupported and are ignored and have no effect. 56 | 57 | Some TestContext members are unsupported and return `null` at runtime. 58 | 59 | Assembly .config files don't work on Mono. 60 | [Issue #17](https://github.com/macro187/testrunner/issues/17). 61 | 62 | ``s in assembly .config files have no effect. 63 | 64 | 65 | 66 | Requirements 67 | ============ 68 | 69 | .NET Framework 4.6.1 (or newer) 70 | 71 | .NET Core 2.0 (or newer) 72 | 73 | Mono 5.0.0 (or newer) 74 | 75 | 76 | 77 | NuGet Package 78 | ============= 79 | 80 | 81 | 82 | Includes `net461` binaries for .NET Framework and Mono, and `netcoreapp2.0` 83 | binaries for .NET Core. 84 | 85 | 86 | 87 | Usage 88 | ===== 89 | 90 | ``` 91 | SYNOPSIS 92 | 93 | testrunner.exe [options] ... 94 | testrunner.exe --help 95 | 96 | DESCRIPTION 97 | 98 | Run tests in (s) 99 | 100 | OPTIONS 101 | 102 | --outputformat 103 | Set the output format 104 | 105 | human 106 | Human-readable text format (default) 107 | 108 | machine 109 | Machine-readable JSON-based format (experimental) 110 | 111 | --class . 112 | --class 113 | Run the specified test class. 114 | 115 | If is omitted, run all test classes with the specified 116 | name. 117 | 118 | If not specified, run all test classes. 119 | 120 | Can be specified multiple times. 121 | 122 | Case-sensitive. 123 | 124 | Does not override [Ignore] attributes. 125 | 126 | --method .. 127 | --method 128 | Run the specified test method. 129 | 130 | If and are omitted, run all test methods with 131 | the specified name (constrained by --class). 132 | 133 | If not specified, run all test methods (constrained by --class). 134 | 135 | Can be specified multiple times. 136 | 137 | Case-sensitive. 138 | 139 | Does not override [Ignore] attributes. 140 | 141 | --help 142 | Show usage information 143 | 144 | EXIT STATUS 145 | 146 | 0 if all specified test files succeed, non-zero otherwise. 147 | 148 | Test files succeed if all test, initialization, and cleanup methods run 149 | successfully. 150 | 151 | Test files succeed if they contain no tests. 152 | 153 | Test files succeed if they are not .NET assemblies. 154 | 155 | Test files fail if any test, initialization, or cleanup methods fail. 156 | 157 | Test files fail if they do not exist. 158 | 159 | EXAMPLES 160 | 161 | .NET Framework 162 | 163 | testrunner.exe C:\Path\To\TestAssembly.dll C:\Path\To\AnotherTestAssembly.dll 164 | 165 | .NET Core 166 | 167 | dotnet testrunner.dll C:\Path\To\TestAssembly.dll C:\Path\To\AnotherTestAssembly.dll 168 | 169 | Mono 170 | 171 | mono --debug testrunner.exe /path/to/TestAssembly.dll /path/to/AnotherTestAssembly.dll 172 | ``` 173 | 174 | 175 | 176 | History 177 | ======= 178 | 179 | Forked from [Bernd Rickenberg](https://github.com/rickenberg)'s 180 | revision 87713 on September 24th, 2016. 181 | 182 | -------------------------------------------------------------------------------- /testrunner/EventHandlers/MachineReadableEventSerializer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Runtime.Serialization; 4 | using System.Runtime.Serialization.Json; 5 | using System.Text; 6 | using TestRunner.MSTest; 7 | using TestRunner.Infrastructure; 8 | using TestRunner.Results; 9 | using TestRunner.Events; 10 | 11 | namespace TestRunner.EventHandlers 12 | { 13 | 14 | /// 15 | /// Serialize events to and from a single-line machine-readable JSON-based text format 16 | /// 17 | /// 18 | public static class MachineReadableEventSerializer 19 | { 20 | 21 | const string Prefix = "[TestRunnerEvent] "; 22 | 23 | static readonly DataContractJsonSerializerSettings JsonSerializerSettings = 24 | new DataContractJsonSerializerSettings() { 25 | EmitTypeInformation = EmitTypeInformation.Never, 26 | DateTimeFormat = new DateTimeFormat("o"), // ISO8601 27 | KnownTypes = new[]{ 28 | typeof(UnitTestOutcome), 29 | typeof(ExceptionInfo), 30 | typeof(StackFrameInfo), 31 | typeof(TestRunnerEvent), 32 | typeof(ProgramBannerEvent), 33 | typeof(ProgramUsageEvent), 34 | typeof(ProgramUserErrorEvent), 35 | typeof(ProgramInternalErrorEvent), 36 | typeof(TestRunBeginEvent), 37 | typeof(TestRunEndEvent), 38 | typeof(TestAssemblyBeginEvent), 39 | typeof(TestAssemblyNotFoundEvent), 40 | typeof(TestAssemblyNotDotNetEvent), 41 | typeof(TestAssemblyNotTestEvent), 42 | typeof(TestAssemblyConfigFileSwitchedEvent), 43 | typeof(TestAssemblyEndEvent), 44 | typeof(TestClassBeginEvent), 45 | typeof(TestClassIgnoredEvent), 46 | typeof(TestClassEndEvent), 47 | typeof(TestBeginEvent), 48 | typeof(TestIgnoredEvent), 49 | typeof(TestEndEvent), 50 | typeof(AssemblyInitializeMethodBeginEvent), 51 | typeof(AssemblyInitializeMethodEndEvent), 52 | typeof(AssemblyCleanupMethodBeginEvent), 53 | typeof(AssemblyCleanupMethodEndEvent), 54 | typeof(ClassInitializeMethodBeginEvent), 55 | typeof(ClassInitializeMethodEndEvent), 56 | typeof(ClassCleanupMethodBeginEvent), 57 | typeof(ClassCleanupMethodEndEvent), 58 | typeof(TestContextSetterBeginEvent), 59 | typeof(TestContextSetterEndEvent), 60 | typeof(TestInitializeMethodBeginEvent), 61 | typeof(TestInitializeMethodEndEvent), 62 | typeof(TestMethodBeginEvent), 63 | typeof(TestMethodEndEvent), 64 | typeof(TestCleanupMethodBeginEvent), 65 | typeof(TestCleanupMethodEndEvent), 66 | typeof(MethodExpectedExceptionEvent), 67 | typeof(MethodUnexpectedExceptionEvent), 68 | typeof(StandardOutputEvent), 69 | typeof(ErrorOutputEvent), 70 | typeof(TraceOutputEvent), 71 | }, 72 | }; 73 | 74 | 75 | /// 76 | /// Serialize a into a line of text 77 | /// 78 | /// 79 | public static string Serialize(TestRunnerEvent e) 80 | { 81 | Guard.NotNull(e, nameof(e)); 82 | var name = e.GetType().Name; 83 | var json = SerializeJson(e); 84 | return $"{Prefix}{name} {json}"; 85 | } 86 | 87 | 88 | /// 89 | /// Try to deserialize a line of text into a 90 | /// 91 | /// 92 | /// 93 | /// A 94 | /// - OR - 95 | /// null if does not appear to be a serialized event 96 | /// 97 | /// 98 | /// 99 | /// appears to be a serialized event but deserialization fails 100 | /// 101 | /// 102 | public static TestRunnerEvent TryDeserialize(string line) 103 | { 104 | Guard.NotNull(line, nameof(line)); 105 | if (!line.StartsWith(Prefix, StringComparison.Ordinal)) return null; 106 | line = line.Substring(Prefix.Length); 107 | var i = line.IndexOf(' '); 108 | if (i < 0) throw new FormatException("No space separator in serialized event"); 109 | var name = line.Substring(0, i); 110 | if (string.IsNullOrWhiteSpace(name)) throw new FormatException("No event name in serialized event"); 111 | var json = line.Substring(i + 1); 112 | if (string.IsNullOrWhiteSpace(json)) throw new FormatException("No event data in serialized event"); 113 | var type = Type.GetType($"TestRunner.Events.{name}"); 114 | if (type == null) throw new FormatException($"Unknown serialized event '{name}'"); 115 | return DeserializeJson(type, json); 116 | } 117 | 118 | 119 | static string SerializeJson(TestRunnerEvent e) 120 | { 121 | using (var stream = new MemoryStream()) 122 | { 123 | GetSerializer(typeof(TestRunnerEvent)).WriteObject(stream, e); 124 | return Encoding.UTF8.GetString(stream.GetBuffer(), 0, (int)stream.Position); 125 | } 126 | } 127 | 128 | 129 | static TestRunnerEvent DeserializeJson(Type type, string json) 130 | { 131 | using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(json))) 132 | { 133 | return (TestRunnerEvent)GetSerializer(type).ReadObject(stream); 134 | } 135 | } 136 | 137 | 138 | static DataContractJsonSerializer GetSerializer(Type type) 139 | { 140 | return new DataContractJsonSerializer(type, JsonSerializerSettings); 141 | } 142 | 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /testrunner/Runners/MethodRunner.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Reflection; 4 | using TestRunner.MSTest; 5 | using TestRunner.Events; 6 | using TestRunner.Infrastructure; 7 | using TestRunner.Results; 8 | using System.Diagnostics; 9 | using TestRunner.EventHandlers; 10 | 11 | namespace TestRunner.Runners 12 | { 13 | static class MethodRunner 14 | { 15 | 16 | static public bool RunAssemblyInitializeMethod(TestAssembly testAssembly) 17 | { 18 | Guard.NotNull(testAssembly, nameof(testAssembly)); 19 | var method = testAssembly.AssemblyInitializeMethod; 20 | if (method == null) return true; 21 | EventHandlerPipeline.Raise( 22 | new AssemblyInitializeMethodBeginEvent() { 23 | MethodName = method.Name, 24 | FirstTestClassFullName = testAssembly.TestClasses.First().FullName, 25 | FirstTestMethodName = testAssembly.TestClasses.First().TestMethods.First().Name, 26 | }); 27 | var success = Run(method, null, true, null, false); 28 | EventHandlerPipeline.Raise(new AssemblyInitializeMethodEndEvent()); 29 | return success; 30 | } 31 | 32 | 33 | static public bool RunAssemblyCleanupMethod(TestAssembly testAssembly) 34 | { 35 | Guard.NotNull(testAssembly, nameof(testAssembly)); 36 | var method = testAssembly.AssemblyCleanupMethod; 37 | if (method == null) return true; 38 | EventHandlerPipeline.Raise(new AssemblyCleanupMethodBeginEvent() { MethodName = method.Name }); 39 | var success = Run(method, null, false, null, false); 40 | EventHandlerPipeline.Raise(new AssemblyCleanupMethodEndEvent()); 41 | return success; 42 | } 43 | 44 | 45 | static public bool RunClassInitializeMethod(TestClass testClass) 46 | { 47 | Guard.NotNull(testClass, nameof(testClass)); 48 | var method = testClass.ClassInitializeMethod; 49 | if (method == null) return true; 50 | EventHandlerPipeline.Raise( 51 | new ClassInitializeMethodBeginEvent() { 52 | MethodName = method.Name, 53 | FirstTestMethodName = testClass.TestMethods.First().Name, 54 | }); 55 | var success = Run(method, null, true, null, false); 56 | EventHandlerPipeline.Raise(new ClassInitializeMethodEndEvent()); 57 | return success; 58 | } 59 | 60 | 61 | static public bool RunClassCleanupMethod(TestClass testClass) 62 | { 63 | Guard.NotNull(testClass, nameof(testClass)); 64 | var method = testClass.ClassCleanupMethod; 65 | if (method == null) return true; 66 | EventHandlerPipeline.Raise(new ClassCleanupMethodBeginEvent() { MethodName = method.Name }); 67 | var success = Run(method, null, false, null, false); 68 | EventHandlerPipeline.Raise(new ClassCleanupMethodEndEvent()); 69 | return success; 70 | } 71 | 72 | 73 | static public void RunTestContextSetter(TestClass testClass, object instance) 74 | { 75 | Guard.NotNull(testClass, nameof(testClass)); 76 | Guard.NotNull(instance, nameof(instance)); 77 | var method = testClass.TestContextSetter; 78 | if (method == null) return; 79 | EventHandlerPipeline.Raise(new TestContextSetterBeginEvent() { MethodName = method.Name }); 80 | var success = Run(method, instance, true, null, false); 81 | EventHandlerPipeline.Raise(new TestContextSetterEndEvent()); 82 | } 83 | 84 | 85 | static public bool RunTestInitializeMethod(TestClass testClass, object instance) 86 | { 87 | Guard.NotNull(testClass, nameof(testClass)); 88 | Guard.NotNull(instance, nameof(instance)); 89 | var method = testClass.TestInitializeMethod; 90 | if (method == null) return true; 91 | EventHandlerPipeline.Raise(new TestInitializeMethodBeginEvent() { MethodName = method.Name }); 92 | var success = Run(method, instance, false, null, false); 93 | EventHandlerPipeline.Raise(new TestInitializeMethodEndEvent()); 94 | return success; 95 | } 96 | 97 | 98 | static public bool RunTestMethod(TestMethod testMethod, object instance) 99 | { 100 | Guard.NotNull(testMethod, nameof(testMethod)); 101 | Guard.NotNull(instance, nameof(instance)); 102 | EventHandlerPipeline.Raise(new TestMethodBeginEvent() { MethodName = testMethod.Name }); 103 | var success = Run( 104 | testMethod.MethodInfo, 105 | instance, 106 | false, 107 | testMethod.ExpectedException, 108 | testMethod.AllowDerivedExpectedExceptionTypes); 109 | EventHandlerPipeline.Raise(new TestMethodEndEvent()); 110 | return success; 111 | } 112 | 113 | 114 | static public bool RunTestCleanupMethod(TestClass testClass, object instance) 115 | { 116 | Guard.NotNull(testClass, nameof(testClass)); 117 | Guard.NotNull(instance, nameof(instance)); 118 | var method = testClass.TestCleanupMethod; 119 | if (method == null) return true; 120 | EventHandlerPipeline.Raise(new TestCleanupMethodBeginEvent() { MethodName = method.Name }); 121 | var success = Run(method, instance, false, null, false); 122 | EventHandlerPipeline.Raise(new TestCleanupMethodEndEvent()); 123 | return success; 124 | } 125 | 126 | 127 | static bool Run( 128 | MethodInfo method, 129 | object instance, 130 | bool takesTestContext, 131 | Type expectedException, 132 | bool expectedExceptionAllowDerived) 133 | { 134 | Guard.NotNull(method, nameof(method)); 135 | 136 | var parameters = takesTestContext ? new object[] { TestContextProxy.Proxy } : null; 137 | 138 | Exception exception = null; 139 | bool exceptionWasExpected = true; 140 | 141 | var traceListener = new EventTraceListener(); 142 | Trace.Listeners.Add(traceListener); 143 | try 144 | { 145 | method.Invoke(instance, parameters); 146 | } 147 | catch (TargetInvocationException tie) 148 | { 149 | exception = tie.InnerException; 150 | } 151 | finally 152 | { 153 | Trace.Listeners.Remove(traceListener); 154 | } 155 | 156 | if (exception == null) return true; 157 | 158 | var isExactExpectedException = 159 | expectedException != null && 160 | exception.GetType() == expectedException; 161 | 162 | var isDerivedExpectedException = 163 | expectedException != null && 164 | expectedExceptionAllowDerived && 165 | exception.GetType().IsSubclassOf(expectedException); 166 | 167 | exceptionWasExpected = isExactExpectedException || isDerivedExpectedException; 168 | 169 | if (exceptionWasExpected) 170 | { 171 | EventHandlerPipeline.Raise( 172 | new MethodExpectedExceptionEvent() { 173 | ExpectedFullName = expectedException.FullName, 174 | Exception = new ExceptionInfo(exception), 175 | }); 176 | } 177 | else 178 | { 179 | EventHandlerPipeline.Raise( 180 | new MethodUnexpectedExceptionEvent() { 181 | Exception = new ExceptionInfo(exception) 182 | }); 183 | } 184 | 185 | return exceptionWasExpected; 186 | } 187 | 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /testrunner/MSTest/TestClass.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Collections.ObjectModel; 4 | using System.Linq; 5 | using System.Reflection; 6 | using TestRunner.Infrastructure; 7 | 8 | namespace TestRunner.MSTest 9 | { 10 | 11 | public class TestClass 12 | { 13 | 14 | internal static TestClass TryCreate(TestAssembly testAssembly, Type type) 15 | { 16 | Guard.NotNull(testAssembly, nameof(testAssembly)); 17 | Guard.NotNull(type, nameof(type)); 18 | if (!type.HasCustomAttribute(TestClassAttribute.TryCreate)) return null; 19 | var testClass = new TestClass(testAssembly, type); 20 | if (testClass.TestMethods.Count == 0) return null; 21 | return testClass; 22 | } 23 | 24 | 25 | TestClass(TestAssembly testAssembly, Type testClass) 26 | { 27 | TestAssembly = testAssembly; 28 | Type = testClass; 29 | IsIgnored = Type.HasCustomAttribute(IgnoreAttribute.TryCreate); 30 | FindTestMethods(); 31 | FindAssemblyInitializeMethod(); 32 | FindAssemblyCleanupMethod(); 33 | FindClassInitializeMethod(); 34 | FindClassCleanupMethod(); 35 | FindTestInitializeMethod(); 36 | FindTestCleanupMethod(); 37 | FindTestContextSetter(); 38 | } 39 | 40 | 41 | public TestAssembly TestAssembly 42 | { 43 | get; 44 | private set; 45 | } 46 | 47 | 48 | [System.Diagnostics.CodeAnalysis.SuppressMessage( 49 | "Microsoft.Naming", 50 | "CA1721:PropertyNamesShouldNotMatchGetMethods", 51 | Justification = "This is the most appropriate name")] 52 | public Type Type 53 | { 54 | get; 55 | private set; 56 | } 57 | 58 | 59 | public bool IsIgnored 60 | { 61 | get; 62 | private set; 63 | } 64 | 65 | 66 | public ICollection TestMethods 67 | { 68 | get; 69 | private set; 70 | } 71 | 72 | 73 | public MethodInfo AssemblyInitializeMethod 74 | { 75 | get; 76 | private set; 77 | } 78 | 79 | 80 | public MethodInfo AssemblyCleanupMethod 81 | { 82 | get; 83 | private set; 84 | } 85 | 86 | 87 | public MethodInfo ClassInitializeMethod 88 | { 89 | get; 90 | private set; 91 | } 92 | 93 | 94 | public MethodInfo ClassCleanupMethod 95 | { 96 | get; 97 | private set; 98 | } 99 | 100 | 101 | public MethodInfo TestInitializeMethod 102 | { 103 | get; 104 | private set; 105 | } 106 | 107 | 108 | public MethodInfo TestCleanupMethod 109 | { 110 | get; 111 | private set; 112 | } 113 | 114 | 115 | /// 116 | /// Setter method for the test class' TestContext property OR null if it doesn't have one OR 117 | /// null if it is read-only 118 | /// 119 | /// 120 | public MethodInfo TestContextSetter 121 | { 122 | get; 123 | private set; 124 | } 125 | 126 | 127 | public string Name 128 | { 129 | get 130 | { 131 | return Type.Name; 132 | } 133 | } 134 | 135 | 136 | public string FullName 137 | { 138 | get 139 | { 140 | return Type.FullName; 141 | } 142 | } 143 | 144 | 145 | void FindTestMethods() 146 | { 147 | TestMethods = 148 | new ReadOnlyCollection( 149 | Type.GetMethods(BindingFlags.Public | BindingFlags.Instance) 150 | .Select(m => TestMethod.TryCreate(this, m)) 151 | .Where(m => m != null) 152 | .ToList()); 153 | } 154 | 155 | 156 | void FindAssemblyInitializeMethod() 157 | { 158 | var methods = 159 | Type.GetMethods(BindingFlags.Public | BindingFlags.Static) 160 | .Where(m => m.HasCustomAttribute(AssemblyInitializeAttribute.TryCreate)) 161 | .ToList(); 162 | 163 | if (methods.Count > 1) 164 | throw new UserException( 165 | $"[TestClass] {Type.FullName} contains more than one [AssemblyInitialize] method"); 166 | 167 | if (methods.Count == 0) 168 | return; 169 | 170 | AssemblyInitializeMethod = methods[0]; 171 | } 172 | 173 | 174 | void FindAssemblyCleanupMethod() 175 | { 176 | var methods = 177 | Type.GetMethods(BindingFlags.Public | BindingFlags.Static) 178 | .Where(m => m.HasCustomAttribute(AssemblyCleanupAttribute.TryCreate)) 179 | .ToList(); 180 | 181 | if (methods.Count > 1) 182 | throw new UserException( 183 | $"[TestClass] {Type.FullName} contains more than one [AssemblyCleanup] method"); 184 | 185 | if (methods.Count == 0) 186 | return; 187 | 188 | AssemblyCleanupMethod = methods[0]; 189 | } 190 | 191 | 192 | void FindClassInitializeMethod() 193 | { 194 | var methods = 195 | Type.GetMethods(BindingFlags.Public | BindingFlags.Static) 196 | .Where(m => m.HasCustomAttribute(ClassInitializeAttribute.TryCreate)) 197 | .ToList(); 198 | 199 | if (methods.Count > 1) 200 | throw new UserException( 201 | $"[TestClass] {Type.FullName} contains more than one [ClassInitialize] method"); 202 | 203 | if (methods.Count == 0) 204 | return; 205 | 206 | ClassInitializeMethod = methods[0]; 207 | } 208 | 209 | 210 | void FindClassCleanupMethod() 211 | { 212 | var methods = 213 | Type.GetMethods(BindingFlags.Public | BindingFlags.Static) 214 | .Where(m => m.HasCustomAttribute(ClassCleanupAttribute.TryCreate)) 215 | .ToList(); 216 | 217 | if (methods.Count > 1) 218 | throw new UserException( 219 | $"[TestClass] {Type.FullName} contains more than one [ClassCleanup] method"); 220 | 221 | if (methods.Count == 0) 222 | return; 223 | 224 | ClassCleanupMethod = methods[0]; 225 | } 226 | 227 | 228 | void FindTestInitializeMethod() 229 | { 230 | var methods = 231 | Type.GetMethods(BindingFlags.Public | BindingFlags.Instance) 232 | .Where(m => m.HasCustomAttribute(TestInitializeAttribute.TryCreate)) 233 | .ToList(); 234 | 235 | if (methods.Count > 1) 236 | throw new UserException( 237 | $"[TestClass] {Type.FullName} contains more than one [TestInitialize] method"); 238 | 239 | if (methods.Count == 0) 240 | return; 241 | 242 | TestInitializeMethod = methods[0]; 243 | } 244 | 245 | 246 | void FindTestCleanupMethod() 247 | { 248 | var methods = 249 | Type.GetMethods(BindingFlags.Public | BindingFlags.Instance) 250 | .Where(m => m.HasCustomAttribute(TestCleanupAttribute.TryCreate)) 251 | .ToList(); 252 | 253 | if (methods.Count > 1) 254 | throw new UserException( 255 | $"[TestClass] {Type.FullName} contains more than one [TestCleanup] method"); 256 | 257 | if (methods.Count == 0) 258 | return; 259 | 260 | TestCleanupMethod = methods[0]; 261 | } 262 | 263 | 264 | void FindTestContextSetter() 265 | { 266 | var property = Type.GetProperty("TestContext", BindingFlags.Instance | BindingFlags.Public); 267 | if (property == null) return; 268 | 269 | var type = property.PropertyType.FullName; 270 | if (type != "Microsoft.VisualStudio.TestTools.UnitTesting.TestContext") return; 271 | 272 | var setter = property.SetMethod; 273 | if (setter == null) return; 274 | 275 | TestContextSetter = setter; 276 | } 277 | 278 | } 279 | 280 | } 281 | -------------------------------------------------------------------------------- /testrunner/Program/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.Linq; 5 | using System.Reflection; 6 | using TestRunner.Infrastructure; 7 | using TestRunner.Runners; 8 | using TestRunner.Events; 9 | using TestRunner.Results; 10 | using TestRunner.EventHandlers; 11 | 12 | namespace TestRunner.Program 13 | { 14 | static class Program 15 | { 16 | 17 | /// 18 | /// 1. Run the program and exit assertively killing any background threads 19 | /// 20 | /// 21 | [STAThread] 22 | static void Main(string[] args) 23 | { 24 | Environment.Exit(Main2(args)); 25 | } 26 | 27 | 28 | /// 29 | /// 2. Set up required event handlers 30 | /// 31 | /// 32 | static int Main2(string[] args) 33 | { 34 | EventHandlerPipeline.Append(new MethodResultEventHandler()); 35 | EventHandlerPipeline.Append(new TestResultEventHandler()); 36 | EventHandlerPipeline.Append(new TestClassResultEventHandler()); 37 | EventHandlerPipeline.Append(new TestAssemblyResultEventHandler()); 38 | EventHandlerPipeline.Append(new TestContextEventHandler()); 39 | return Main3(args); 40 | } 41 | 42 | 43 | /// 44 | /// 3. Set up error handlers 45 | /// 46 | /// 47 | [System.Diagnostics.CodeAnalysis.SuppressMessage( 48 | "Microsoft.Design", 49 | "CA1031:DoNotCatchGeneralExceptionTypes", 50 | Justification = "Required to handle unexpected exceptions")] 51 | static int Main3(string[] args) 52 | { 53 | try 54 | { 55 | return Main4(args); 56 | } 57 | catch (UserException ue) 58 | { 59 | HandleUserException(ue); 60 | return 1; 61 | } 62 | catch (Exception e) 63 | { 64 | HandleInternalException(e); 65 | return 1; 66 | } 67 | } 68 | 69 | 70 | /// 71 | /// 4. Parse arguments and take action 72 | /// 73 | /// 74 | static int Main4(string[] args) 75 | { 76 | ArgumentParser.Parse(args); 77 | 78 | switch(ArgumentParser.OutputFormat) 79 | { 80 | case OutputFormats.Human: 81 | EventHandlerPipeline.Append(new HumanOutputEventHandler()); 82 | break; 83 | case OutputFormats.Machine: 84 | EventHandlerPipeline.Append(new MachineOutputEventHandler()); 85 | break; 86 | default: 87 | throw new Exception($"Unrecognised from parser {ArgumentParser.OutputFormat}"); 88 | } 89 | 90 | if (!ArgumentParser.Success) 91 | { 92 | throw ArgumentParseError(); 93 | } 94 | 95 | if (ArgumentParser.Help) 96 | { 97 | return Help(); 98 | } 99 | 100 | if (ArgumentParser.InProc) 101 | { 102 | return InProc(); 103 | } 104 | 105 | return Main5(); 106 | } 107 | 108 | 109 | /// 110 | /// 5. Run test file(s) 111 | /// 112 | // 113 | static int Main5() 114 | { 115 | Banner(); 116 | EventHandlerPipeline.Raise(new TestRunBeginEvent() {}); 117 | bool success = true; 118 | foreach (var testFile in ArgumentParser.TestFiles) 119 | { 120 | if (!Reinvoke(testFile)) success = false; 121 | } 122 | EventHandlerPipeline.Raise( new TestRunEndEvent() { Result = new TestRunResult() { Success = success } }); 123 | return success ? 0 : 1; 124 | } 125 | 126 | 127 | /// 128 | /// Reinvoke testrunner to run an individual test file in its own process 129 | /// 130 | /// 131 | static bool Reinvoke(string testFile) 132 | { 133 | var args = new List(); 134 | 135 | args.Add("--inproc"); 136 | args.Add("--outputformat machine"); 137 | 138 | foreach (var @class in ArgumentParser.Classes) 139 | { 140 | args.Add($"--class {@class}"); 141 | } 142 | 143 | foreach (var method in ArgumentParser.Methods) 144 | { 145 | args.Add($"--method {method}"); 146 | } 147 | 148 | args.Add($"\"{testFile}\""); 149 | 150 | var exitCode = 151 | ProcessExtensions.ExecuteDotnet( 152 | Assembly.GetExecutingAssembly().Location, 153 | string.Join(" ", args), 154 | (proc, line) => { 155 | var e = MachineReadableEventSerializer.TryDeserialize(line); 156 | EventHandlerPipeline.Raise( 157 | e ?? 158 | new StandardOutputEvent() { 159 | ProcessId = proc.Id, 160 | Message = line, 161 | }); 162 | }, 163 | (proc, line) => { 164 | EventHandlerPipeline.Raise( 165 | new ErrorOutputEvent() { 166 | ProcessId = proc.Id, 167 | Message = line, 168 | }); 169 | }); 170 | 171 | return exitCode == 0; 172 | } 173 | 174 | 175 | /// 176 | /// --help: Print out brief program usage information 177 | /// 178 | /// 179 | static int Help() 180 | { 181 | Banner(); 182 | Usage(); 183 | return 0; 184 | } 185 | 186 | 187 | /// 188 | /// --inproc: Run an individual test file in-process 189 | /// 190 | /// 191 | static int InProc() 192 | { 193 | var eventHandler = new ResultAccumulatingEventHandler(); 194 | using (EventHandlerPipeline.Append(eventHandler)) 195 | { 196 | TestAssemblyRunner.Run(ArgumentParser.TestFiles[0]); 197 | } 198 | return eventHandler.TestAssemblyResults.Last().Success ? 0 : 1; 199 | } 200 | 201 | 202 | /// 203 | /// Handle argument parse error 204 | /// 205 | /// 206 | static UserException ArgumentParseError() 207 | { 208 | Banner(); 209 | Usage(); 210 | return new UserException(ArgumentParser.ErrorMessage); 211 | } 212 | 213 | 214 | /// 215 | /// Print program name, version, and copyright banner 216 | /// 217 | /// 218 | static void Banner() 219 | { 220 | var name = FileVersionInfo.GetVersionInfo(Assembly.GetExecutingAssembly().Location).ProductName; 221 | var version = FileVersionInfo.GetVersionInfo(Assembly.GetExecutingAssembly().Location).ProductVersion; 222 | var copyright = FileVersionInfo.GetVersionInfo(Assembly.GetExecutingAssembly().Location).LegalCopyright; 223 | 224 | EventHandlerPipeline.Raise( 225 | new ProgramBannerEvent() { 226 | Lines = new[]{ 227 | $"{name} v{version}", 228 | copyright, 229 | }, 230 | }); 231 | } 232 | 233 | 234 | /// 235 | /// Print program usage information 236 | /// 237 | /// 238 | static void Usage() 239 | { 240 | EventHandlerPipeline.Raise(new ProgramUsageEvent() { Lines = ArgumentParser.GetUsage() }); 241 | } 242 | 243 | 244 | /// 245 | /// Handle user-facing error 246 | /// 247 | /// 248 | static void HandleUserException(UserException ue) 249 | { 250 | EventHandlerPipeline.Raise(new ProgramUserErrorEvent() { Message = ue.Message }); 251 | } 252 | 253 | 254 | /// 255 | /// Handle internal TestRunner error 256 | /// 257 | /// 258 | static void HandleInternalException(Exception e) 259 | { 260 | EventHandlerPipeline.Raise( 261 | new ProgramInternalErrorEvent() { 262 | Exception = new ExceptionInfo(e) 263 | }); 264 | 265 | if (e is ReflectionTypeLoadException rtle) 266 | { 267 | foreach (var le in rtle.LoaderExceptions) 268 | { 269 | EventHandlerPipeline.Raise( 270 | new ProgramInternalErrorEvent() { 271 | Exception = new ExceptionInfo(le) 272 | }); 273 | } 274 | } 275 | } 276 | 277 | } 278 | } 279 | -------------------------------------------------------------------------------- /testrunner/MSTest/TestContextProxy.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | #if NETCOREAPP2_0 3 | using System.Collections.Generic; 4 | #else 5 | using System.Collections; 6 | #endif 7 | using System.Reflection; 8 | using System.Reflection.Emit; 9 | using TestRunner.Infrastructure; 10 | 11 | namespace TestRunner.MSTest 12 | { 13 | 14 | /// 15 | /// Provides a dynamically created instance 16 | /// that behaves as a proxy to 17 | /// 18 | /// 19 | static class TestContextProxy 20 | { 21 | 22 | /// 23 | /// The instance 24 | /// 25 | /// 26 | public static object Proxy 27 | { 28 | get 29 | { 30 | if (_proxy == null) _proxy = BuildProxy(); 31 | return _proxy; 32 | } 33 | } 34 | 35 | static object _proxy; 36 | 37 | 38 | static object BuildProxy() 39 | { 40 | var testContextType = GetTestContextType(); 41 | var typeBuilder = GetProxyTypeBuilder(testContextType); 42 | 43 | BuildProxyProperty(typeBuilder, testContextType, "CurrentTestOutcome", GetUnitTestOutcomeType()); 44 | BuildProxyProperty(typeBuilder, testContextType, "DataConnection", typeof(System.Data.Common.DbConnection)); 45 | BuildProxyProperty(typeBuilder, testContextType, "DataRow", typeof(System.Data.DataRow)); 46 | BuildProxyProperty(typeBuilder, testContextType, "DeploymentDirectory", typeof(string)); 47 | BuildProxyProperty(typeBuilder, testContextType, "FullyQualifiedTestClassName", typeof(string)); 48 | #if NETCOREAPP2_0 49 | BuildProxyProperty(typeBuilder, testContextType, "Properties", typeof(IDictionary)); 50 | #else 51 | BuildProxyProperty(typeBuilder, testContextType, "Properties", typeof(IDictionary)); 52 | #endif 53 | BuildProxyProperty(typeBuilder, testContextType, "ResultsDirectory", typeof(string)); 54 | BuildProxyProperty(typeBuilder, testContextType, "TestDeploymentDir", typeof(string)); 55 | BuildProxyProperty(typeBuilder, testContextType, "TestDir", typeof(string)); 56 | BuildProxyProperty(typeBuilder, testContextType, "TestLogsDir", typeof(string)); 57 | BuildProxyProperty(typeBuilder, testContextType, "TestName", typeof(string)); 58 | BuildProxyProperty(typeBuilder, testContextType, "TestResultsDirectory", typeof(string)); 59 | BuildProxyProperty(typeBuilder, testContextType, "TestRunDirectory", typeof(string)); 60 | BuildProxyProperty(typeBuilder, testContextType, "TestRunResultsDirectory", typeof(string)); 61 | 62 | BuildProxyMethod(typeBuilder, testContextType, "AddResultFile", null, typeof(string)); 63 | BuildProxyMethod(typeBuilder, testContextType, "BeginTimer", null, typeof(string)); 64 | BuildProxyMethod(typeBuilder, testContextType, "EndTimer", null, typeof(string)); 65 | BuildProxyMethod(typeBuilder, testContextType, "WriteLine", null, typeof(string)); 66 | BuildProxyMethod(typeBuilder, testContextType, "WriteLine", null, typeof(string), typeof(object[])); 67 | 68 | return Activator.CreateInstance(GetTypeFromTypeBuilder(typeBuilder)); 69 | } 70 | 71 | 72 | static void BuildProxyMethod( 73 | TypeBuilder typeBuilder, 74 | Type baseType, 75 | string name, 76 | Type returnType, 77 | params Type[] parameterTypes) 78 | { 79 | parameterTypes = parameterTypes ?? new Type[0]; 80 | 81 | var baseMethod = baseType.GetMethod(name, parameterTypes); 82 | if (baseMethod == null) return; 83 | if (baseMethod.ReturnType.FullName != (returnType?.FullName ?? "System.Void")) return; 84 | 85 | var method = typeBuilder.DefineMethod( 86 | name, 87 | MethodAttributes.Public | 88 | MethodAttributes.Virtual | 89 | MethodAttributes.HideBySig, 90 | returnType, 91 | parameterTypes); 92 | 93 | var target = typeof(TestContext).GetMethod(name, parameterTypes); 94 | if (target == null) throw new Exception("Target TestContext method " + name + " not found"); 95 | 96 | var il = method.GetILGenerator(); 97 | if (parameterTypes.Length > 0) il.Emit(OpCodes.Ldarg_0); 98 | if (parameterTypes.Length > 1) il.Emit(OpCodes.Ldarg_1); 99 | if (parameterTypes.Length > 2) il.Emit(OpCodes.Ldarg_2); 100 | if (parameterTypes.Length > 3) il.Emit(OpCodes.Ldarg_3); 101 | if (parameterTypes.Length > 4) throw new NotSupportedException(); 102 | 103 | il.Emit(OpCodes.Call, target); 104 | il.Emit(OpCodes.Ret); 105 | } 106 | 107 | 108 | static void BuildProxyProperty(TypeBuilder typeBuilder, Type baseType, string name, Type type) 109 | { 110 | var getterName = "get_" + name; 111 | 112 | var baseGetter = baseType.GetMethod(getterName); 113 | if (baseGetter == null) return; 114 | if (baseGetter.ReturnType.FullName != (type?.FullName ?? "System.Void")) return; 115 | 116 | var getter = typeBuilder.DefineMethod( 117 | getterName, 118 | MethodAttributes.Public | 119 | MethodAttributes.Virtual | 120 | MethodAttributes.HideBySig | 121 | MethodAttributes.SpecialName, 122 | type, 123 | Type.EmptyTypes); 124 | 125 | var target = typeof(TestContext).GetProperty(name).GetMethod; 126 | if (target == null) throw new Exception("Target TestContext property " + name + " getter not found"); 127 | 128 | var il = getter.GetILGenerator(); 129 | il.Emit(OpCodes.Call, target); 130 | il.Emit(OpCodes.Ret); 131 | 132 | var property = typeBuilder.DefineProperty( 133 | name, 134 | PropertyAttributes.None, 135 | type, 136 | null); 137 | 138 | property.SetGetMethod(getter); 139 | } 140 | 141 | 142 | static TypeBuilder GetProxyTypeBuilder(Type testContextType) 143 | { 144 | return GetProxyAssemblyBuilder() 145 | .DefineDynamicModule("MainModule") 146 | .DefineType( 147 | "TestContextProxy", 148 | TypeAttributes.Public | 149 | TypeAttributes.Class | 150 | TypeAttributes.AutoClass | 151 | TypeAttributes.AnsiClass | 152 | TypeAttributes.BeforeFieldInit | 153 | TypeAttributes.AutoLayout, 154 | testContextType); 155 | } 156 | 157 | 158 | static AssemblyBuilder GetProxyAssemblyBuilder() 159 | { 160 | #if NETCOREAPP2_0 161 | return AssemblyBuilder.DefineDynamicAssembly( 162 | new AssemblyName("TestContextProxyAssembly"), 163 | AssemblyBuilderAccess.Run); 164 | #else 165 | return AppDomain.CurrentDomain.DefineDynamicAssembly( 166 | new AssemblyName("TestContextProxyAssembly"), 167 | AssemblyBuilderAccess.Run); 168 | #endif 169 | } 170 | 171 | 172 | static Type GetTestContextType() 173 | { 174 | var type = 175 | GetMSTestExtensionsAssembly() 176 | .GetType("Microsoft.VisualStudio.TestTools.UnitTesting.TestContext", false); 177 | 178 | if (type == null) 179 | throw new UserException("No TestContext type found in linked MSTest DLL"); 180 | 181 | return type; 182 | } 183 | 184 | 185 | static Type GetUnitTestOutcomeType() 186 | { 187 | var type = 188 | GetMSTestAssembly() 189 | .GetType("Microsoft.VisualStudio.TestTools.UnitTesting.UnitTestOutcome", false); 190 | 191 | if (type == null) 192 | throw new UserException("No UnitTestOutcome type found in linked MSTest DLL"); 193 | 194 | return type; 195 | } 196 | 197 | 198 | static Assembly GetMSTestAssembly() 199 | { 200 | Assembly assembly = null; 201 | foreach (Assembly a in AppDomain.CurrentDomain.GetAssemblies()) 202 | { 203 | var name = a.GetName().Name; 204 | 205 | // Old-style MSTest 206 | if (name == "Microsoft.VisualStudio.QualityTools.UnitTestFramework") 207 | { 208 | assembly = a; 209 | break; 210 | } 211 | 212 | // New-style MSTest 213 | if (name == "Microsoft.VisualStudio.TestPlatform.TestFramework") 214 | { 215 | assembly = a; 216 | break; 217 | } 218 | } 219 | 220 | if (assembly == null) 221 | throw new UserException("Test DLL doesn't appear to be linked to an MSTest DLL"); 222 | 223 | return assembly; 224 | } 225 | 226 | 227 | static Assembly GetMSTestExtensionsAssembly() 228 | { 229 | Assembly assembly = null; 230 | foreach (Assembly a in AppDomain.CurrentDomain.GetAssemblies()) 231 | { 232 | var name = a.GetName().Name; 233 | 234 | // In old-style MSTest everything is in one assembly 235 | if (name == "Microsoft.VisualStudio.QualityTools.UnitTestFramework") 236 | { 237 | assembly = a; 238 | break; 239 | } 240 | 241 | // In new-style MSTest some stuff is in a separate .Extensions assembly 242 | if (name == "Microsoft.VisualStudio.TestPlatform.TestFramework") 243 | { 244 | assembly = Assembly.LoadFrom(a.Location.Substring(0, a.Location.Length - 4) + ".Extensions.dll"); 245 | break; 246 | } 247 | } 248 | 249 | if (assembly == null) 250 | throw new UserException("Test DLL doesn't appear to be linked to an MSTest DLL"); 251 | 252 | return assembly; 253 | } 254 | 255 | 256 | static Type GetTypeFromTypeBuilder(TypeBuilder typeBuilder) 257 | { 258 | #if NETCOREAPP2_0 259 | return typeBuilder.CreateTypeInfo().AsType(); 260 | #else 261 | return typeBuilder.CreateType(); 262 | #endif 263 | } 264 | 265 | } 266 | } 267 | -------------------------------------------------------------------------------- /testrunner.Tests.MSTest/MSTestTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Reflection; 3 | #if NET461 4 | using System.Configuration; 5 | #endif 6 | using Microsoft.VisualStudio.TestTools.UnitTesting; 7 | using TestRunner.Tests.ReferencedAssembly; 8 | 9 | namespace TestRunner.Tests.MSTest 10 | { 11 | 12 | [TestClass] 13 | public partial class MSTestTests 14 | { 15 | 16 | static readonly string FullyQualifiedTestClassName = typeof(MSTestTests).FullName; 17 | 18 | static bool assemblyInitializeRan = false; 19 | static bool assemblyInitializeReceivedTestContext = false; 20 | static string assemblyInitializeTestName; 21 | static string assemblyInitializeFullyQualifiedTestClassName; 22 | static UnitTestOutcome? assemblyInitializeCurrentTestOutcome; 23 | 24 | static int classInitializeCount = 0; 25 | static bool classInitializeReceivedTestContext = false; 26 | static string classInitializeTestName; 27 | static string classInitializeFullyQualifiedTestClassName; 28 | static UnitTestOutcome? classInitializeCurrentTestOutcome; 29 | 30 | static int testInitializeCount = 0; 31 | bool testInitializeTestContextAvailable; 32 | string testInitializeTestName; 33 | string testInitializeFullyQualifiedTestClassName; 34 | UnitTestOutcome? testInitializeCurrentTestOutcome; 35 | 36 | static int testCleanupCount = 0; 37 | static UnitTestOutcome? testCleanupCurrentTestOutcome; 38 | 39 | bool isInstanceNew = true; 40 | object isInstanceNewLock = new object(); 41 | 42 | 43 | public TestContext TestContext { get; set; } 44 | 45 | 46 | [AssemblyInitialize] 47 | public static void AssemblyInitialize(TestContext testContext) 48 | { 49 | assemblyInitializeReceivedTestContext = testContext != null; 50 | assemblyInitializeTestName = testContext?.TestName; 51 | assemblyInitializeFullyQualifiedTestClassName = testContext?.FullyQualifiedTestClassName; 52 | assemblyInitializeCurrentTestOutcome = testContext?.CurrentTestOutcome; 53 | assemblyInitializeRan = true; 54 | } 55 | 56 | 57 | [ClassInitialize] 58 | public static void ClassInitialize(TestContext testContext) 59 | { 60 | classInitializeReceivedTestContext = testContext != null; 61 | classInitializeTestName = testContext?.TestName; 62 | classInitializeFullyQualifiedTestClassName = testContext?.FullyQualifiedTestClassName; 63 | classInitializeCurrentTestOutcome = testContext?.CurrentTestOutcome; 64 | classInitializeCount++; 65 | } 66 | 67 | 68 | [TestInitialize] 69 | public void TestInitialize() 70 | { 71 | testInitializeTestContextAvailable = TestContext != null; 72 | testInitializeTestName = TestContext?.TestName; 73 | testInitializeFullyQualifiedTestClassName = TestContext?.FullyQualifiedTestClassName; 74 | testInitializeCurrentTestOutcome = TestContext?.CurrentTestOutcome; 75 | testInitializeCount++; 76 | } 77 | 78 | 79 | [TestMethod] 80 | public void AssemblyInitialize_Runs() 81 | { 82 | Assert.IsTrue(assemblyInitializeRan, "[AssemblyInitialize] method did not run"); 83 | } 84 | 85 | 86 | [TestMethod] 87 | public void ClassInitialize_Runs_Once() 88 | { 89 | Assert.AreEqual(1, classInitializeCount); 90 | } 91 | [TestMethod] 92 | public void ClassInitialize_Runs_Once_2() 93 | { 94 | Assert.AreEqual(1, classInitializeCount); 95 | } 96 | 97 | 98 | [TestMethod] 99 | public void TestInitialize_Runs() 100 | { 101 | Assert.IsTrue(testInitializeCount > 0, "[TestInitialize] method did not run"); 102 | } 103 | 104 | 105 | // 106 | // Run the same check twice to make sure we've completed at least one full [TestMethod] 107 | // 108 | [TestMethod] 109 | public void TestCleanup_Runs() 110 | { 111 | if (testInitializeCount < 2) return; 112 | Assert.IsTrue(testCleanupCount > 0, "[TestCleanup] did not run"); 113 | } 114 | [TestMethod] 115 | public void TestCleanup_Runs_2() 116 | { 117 | if (testInitializeCount < 2) return; 118 | Assert.IsTrue(testCleanupCount > 0, "[TestCleanup] did not run"); 119 | } 120 | 121 | 122 | [TestMethod] 123 | public void AssemblyInitialize_Receives_TestContext() 124 | { 125 | Assert.IsTrue( 126 | assemblyInitializeReceivedTestContext, 127 | "[AssemblyInitialize] method did not receive a TestContext instance"); 128 | } 129 | 130 | 131 | [TestMethod] 132 | public void AssemblyInitialize_Receives_Random_TestName() 133 | { 134 | Assert.IsNotNull(assemblyInitializeTestName); 135 | Assert.AreNotEqual("", assemblyInitializeTestName); 136 | } 137 | 138 | 139 | [TestMethod] 140 | public void AssemblyInitialize_Receives_Random_FullyQualifiedTestClassName() 141 | { 142 | Assert.IsNotNull(assemblyInitializeFullyQualifiedTestClassName); 143 | Assert.AreNotEqual("", assemblyInitializeFullyQualifiedTestClassName); 144 | } 145 | 146 | 147 | [TestMethod] 148 | public void AssemblyInitialize_Receives_InProgress_CurrentTestOutcome() 149 | { 150 | Assert.AreEqual(UnitTestOutcome.InProgress, assemblyInitializeCurrentTestOutcome); 151 | } 152 | 153 | 154 | [TestMethod] 155 | public void ClassInitialize_Receives_TestContext() 156 | { 157 | Assert.IsTrue( 158 | classInitializeReceivedTestContext, 159 | "[ClassInitialize] method did not receive a TestContext instance"); 160 | } 161 | 162 | 163 | [TestMethod] 164 | public void ClassInitialize_Receives_Random_TestName() 165 | { 166 | Assert.IsNotNull(classInitializeTestName); 167 | Assert.AreNotEqual("", classInitializeTestName); 168 | } 169 | 170 | 171 | [TestMethod] 172 | public void ClassInitialize_Receives_Correct_FullyQualifiedTestClassName() 173 | { 174 | Assert.AreEqual(FullyQualifiedTestClassName, classInitializeFullyQualifiedTestClassName); 175 | } 176 | 177 | 178 | [TestMethod] 179 | public void ClassInitialize_Receives_InProgress_CurrentTestOutcome() 180 | { 181 | Assert.AreEqual(UnitTestOutcome.InProgress, classInitializeCurrentTestOutcome); 182 | } 183 | 184 | 185 | [TestMethod] 186 | public void TestContext_Available_During_TestMethod() 187 | { 188 | Assert.IsNotNull(TestContext, "TestContext not available during [TestMethod]"); 189 | } 190 | 191 | 192 | [TestMethod] 193 | public void TestContext_Available_During_TestInitialize() 194 | { 195 | Assert.IsTrue(testInitializeTestContextAvailable); 196 | } 197 | 198 | 199 | [TestMethod] 200 | public void TestContext_CurrentTestOutcome_InProgress_During_TestInitialize() 201 | { 202 | Assert.AreEqual( 203 | UnitTestOutcome.InProgress, 204 | testInitializeCurrentTestOutcome); 205 | } 206 | 207 | 208 | [TestMethod] 209 | public void TestContext_CurrentTestOutcome_InProgress_During_TestMethod() 210 | { 211 | Assert.AreEqual( 212 | UnitTestOutcome.InProgress, 213 | TestContext.CurrentTestOutcome); 214 | } 215 | 216 | 217 | // 218 | // Run the same check twice to make sure we've completed at least one full [TestMethod] 219 | // 220 | [TestMethod] 221 | public void TestContext_CurrentTestOutcome_Passed_During_TestCleanup_After_Passed_TestMethod() 222 | { 223 | if (testInitializeCount < 2) return; 224 | Assert.AreEqual( 225 | UnitTestOutcome.Passed, 226 | testCleanupCurrentTestOutcome); 227 | } 228 | [TestMethod] 229 | public void TestContext_CurrentTestOutcome_Passed_During_TestCleanup_After_Passed_TestMethod_2() 230 | { 231 | if (testInitializeCount < 2) return; 232 | Assert.AreEqual( 233 | UnitTestOutcome.Passed, 234 | testCleanupCurrentTestOutcome); 235 | } 236 | 237 | 238 | [TestMethod] 239 | public void TestContext_FullyQualifiedTestClassName_Correct_During_TestInitialize() 240 | { 241 | Assert.AreEqual(FullyQualifiedTestClassName, testInitializeFullyQualifiedTestClassName); 242 | } 243 | 244 | 245 | [TestMethod] 246 | public void TestContext_FullyQualifiedTestClassName_Correct_During_TestMethod() 247 | { 248 | Assert.AreEqual(FullyQualifiedTestClassName, TestContext?.FullyQualifiedTestClassName); 249 | } 250 | 251 | 252 | [TestMethod] 253 | public void TestContext_TestName_Correct_During_TestInitialize() 254 | { 255 | var thisTestName = MethodBase.GetCurrentMethod().Name; 256 | Assert.AreEqual(thisTestName, testInitializeTestName); 257 | } 258 | 259 | 260 | [TestMethod] 261 | public void TestContext_TestName_Correct_During_TestMethod() 262 | { 263 | var thisTestName = MethodBase.GetCurrentMethod().Name; 264 | Assert.AreEqual(thisTestName, TestContext?.TestName); 265 | } 266 | 267 | 268 | [TestMethod] 269 | public void Print_Trace_Test_Message() 270 | { 271 | System.Diagnostics.Trace.WriteLine(TraceTestMessage); 272 | } 273 | 274 | 275 | [Ignore] 276 | [TestMethod] 277 | public void IgnoredTestMethod() 278 | { 279 | Console.WriteLine(IgnoredTestMessage); 280 | } 281 | 282 | 283 | [TestMethod] 284 | [ExpectedException(typeof(ArgumentException), AllowDerivedTypes = false)] 285 | public void ExpectedException_Works() 286 | { 287 | throw new ArgumentException(); 288 | } 289 | 290 | 291 | [TestMethod] 292 | [ExpectedException(typeof(ArgumentException), AllowDerivedTypes = true)] 293 | public void ExpectedException_AllowDerivedTypes_Works() 294 | { 295 | throw new ArgumentNullException(); 296 | } 297 | 298 | 299 | [TestMethod] 300 | public void TestAssembly_Config_File_Is_Used() 301 | { 302 | #if NET461 303 | // 304 | // Config file switching doesn't work on Mono 305 | // See https://bugzilla.xamarin.com/show_bug.cgi?id=15741 306 | // 307 | if (Type.GetType("Mono.Runtime") != null) return; 308 | 309 | Assert.AreEqual( 310 | "ConfigFileValue", 311 | ConfigurationManager.AppSettings["ConfigFileKey"]); 312 | #endif 313 | } 314 | 315 | 316 | // 317 | // If each [TestMethod] doesn't get its own [TestClass] instance, the second of these two tests to run will fail 318 | // 319 | [TestMethod] 320 | public void Each_TestMethod_Gets_New_TestClass_Instance_Part1() 321 | { 322 | lock (isInstanceNewLock) 323 | { 324 | Assert.IsTrue(isInstanceNew, "Not a new [TestClass] instance"); 325 | isInstanceNew = false; 326 | } 327 | } 328 | [TestMethod] 329 | public void Each_TestMethod_Gets_New_TestClass_Instance_Part2() 330 | { 331 | lock (isInstanceNewLock) 332 | { 333 | Assert.IsTrue(isInstanceNew, "Not a new [TestClass] instance"); 334 | isInstanceNew = false; 335 | } 336 | } 337 | 338 | 339 | [TestMethod] 340 | public void Use_Referenced_Assembly() 341 | { 342 | Console.WriteLine(TestReferencedClass.TestReferencedMethod()); 343 | } 344 | 345 | 346 | [TestCleanup] 347 | public void TestCleanup() 348 | { 349 | // 350 | // No way to directly test that [TestCleanup] runs, so print a message so it can be confirmed by examining 351 | // the output 352 | // 353 | Console.WriteLine(TestCleanupMessage); 354 | testCleanupCurrentTestOutcome = TestContext?.CurrentTestOutcome; 355 | testCleanupCount++; 356 | } 357 | 358 | 359 | [ClassCleanup] 360 | public static void ClassCleanup() 361 | { 362 | // 363 | // No way to directly test that [ClassCleanup] runs, so print a message so it can be confirmed by examining 364 | // the output 365 | // 366 | Console.WriteLine(ClassCleanupMessage); 367 | } 368 | 369 | 370 | [AssemblyCleanup] 371 | public static void AssemblyCleanup() 372 | { 373 | // 374 | // No way to directly test that [AssemblyCleanup] runs, so print a message so it can be confirmed by 375 | // examining the output 376 | // 377 | Console.WriteLine(AssemblyCleanupMessage); 378 | } 379 | 380 | } 381 | 382 | } 383 | -------------------------------------------------------------------------------- /testrunner/Program/ArgumentParser.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Collections.ObjectModel; 4 | using System.IO; 5 | using System.Linq; 6 | using System.Reflection; 7 | using TestRunner.Infrastructure; 8 | 9 | namespace TestRunner.Program 10 | { 11 | 12 | /// 13 | /// Command line argument parser 14 | /// 15 | /// 16 | static class ArgumentParser 17 | { 18 | 19 | static List _classes = new List(); 20 | static List _methods = new List(); 21 | static List _testFiles = new List(); 22 | 23 | 24 | /// 25 | /// Were the command line arguments valid? 26 | /// 27 | /// 28 | static public bool Success 29 | { 30 | get; 31 | private set; 32 | } 33 | 34 | 35 | /// 36 | /// User-facing error message if not 37 | /// 38 | /// 39 | static public string ErrorMessage 40 | { 41 | get; 42 | private set; 43 | } 44 | 45 | 46 | /// 47 | /// The specified --class options 48 | /// 49 | /// 50 | static public IReadOnlyCollection Classes { get; } = new ReadOnlyCollection(_classes); 51 | 52 | 53 | /// 54 | /// The specified --method options 55 | /// 56 | /// 57 | static public IReadOnlyCollection Methods { get; } = new ReadOnlyCollection(_methods); 58 | 59 | 60 | /// 61 | /// The specifed --outputformat option (default: human) 62 | /// 63 | /// 64 | static public string OutputFormat 65 | { 66 | get; 67 | private set; 68 | } 69 | 70 | 71 | /// 72 | /// Was the --help option specified? 73 | /// 74 | /// 75 | static public bool Help 76 | { 77 | get; 78 | private set; 79 | } 80 | 81 | 82 | /// 83 | /// Was the --inproc option specified? 84 | /// 85 | /// 86 | static public bool InProc 87 | { 88 | get; 89 | private set; 90 | } 91 | 92 | 93 | /// 94 | /// Path(s) to test assemblies listed on the command line 95 | /// 96 | /// 97 | static public IReadOnlyList TestFiles { get; } = new ReadOnlyCollection(_testFiles); 98 | 99 | 100 | /// 101 | /// Produce user-facing command line usage information 102 | /// 103 | /// 104 | static public string[] GetUsage() 105 | { 106 | var fileName = Path.GetFileName(Assembly.GetExecutingAssembly().Location); 107 | bool isUnix = new[] { PlatformID.Unix, PlatformID.MacOSX }.Contains(Environment.OSVersion.Platform); 108 | 109 | var shellPrefix = 110 | isUnix 111 | ? "$" 112 | : "C:\\>"; 113 | 114 | var examplePath = 115 | isUnix 116 | ? "/path/to/" 117 | : "C:\\path\\to\\"; 118 | 119 | return 120 | new[] { 121 | $"SYNOPSIS", 122 | $"", 123 | $" {fileName} [options] ...", 124 | $" {fileName} --help", 125 | $"", 126 | $"DESCRIPTION", 127 | $"", 128 | $" Run tests in (s)", 129 | $"", 130 | $"OPTIONS", 131 | $"", 132 | $" --outputformat ", 133 | $" Set the output format", 134 | $"", 135 | $" human", 136 | $" Human-readable text format (default)", 137 | $"", 138 | $" machine", 139 | $" Machine-readable JSON-based format (experimental)", 140 | $"", 141 | $" --class .", 142 | $" --class ", 143 | $" Run the specified test class.", 144 | $"", 145 | $" If is omitted, run all test classes with the specified", 146 | $" name.", 147 | $"", 148 | $" If not specified, run all test classes.", 149 | $"", 150 | $" Can be specified multiple times.", 151 | $"", 152 | $" Case-sensitive.", 153 | $"", 154 | $" Does not override [Ignore] attributes.", 155 | $"", 156 | $" --method ..", 157 | $" --method ", 158 | $" Run the specified test method.", 159 | $"", 160 | $" If and are omitted, run all test methods with", 161 | $" the specified name (constrained by --class).", 162 | $"", 163 | $" If not specified, run all test methods (constrained by --class).", 164 | $"", 165 | $" Can be specified multiple times.", 166 | $"", 167 | $" Case-sensitive.", 168 | $"", 169 | $" Does not override [Ignore] attributes.", 170 | $"", 171 | $" --help", 172 | $" Show usage information", 173 | $"", 174 | $"EXAMPLES", 175 | $"", 176 | $" {shellPrefix} {fileName} TestAssembly.dll AnotherTestAssembly.dll", 177 | $"", 178 | $" {shellPrefix} {fileName} {examplePath}TestAssembly.dll {examplePath}AnotherTestAssembly.dll", 179 | }; 180 | } 181 | 182 | 183 | /// 184 | /// Decide whether a test class should run given the specified --class options 185 | /// 186 | /// 187 | static public bool ClassShouldRun(string fullClassName) 188 | { 189 | Guard.NotNullOrWhiteSpace(fullClassName, nameof(fullClassName)); 190 | 191 | if (!Classes.Any()) 192 | { 193 | return true; 194 | } 195 | 196 | if (WasFullClassSpecified(fullClassName)) 197 | { 198 | return true; 199 | } 200 | 201 | if (WasClassSpecified(fullClassName)) 202 | { 203 | return true; 204 | } 205 | 206 | if (WasFullMethodInClassSpecified(fullClassName)) 207 | { 208 | return true; 209 | } 210 | 211 | return false; 212 | } 213 | 214 | 215 | /// 216 | /// Decide whether a test method should run given the specified --class and --method options 217 | /// 218 | /// 219 | static public bool MethodShouldRun(string fullMethodName) 220 | { 221 | Guard.NotNullOrWhiteSpace(fullMethodName, nameof(fullMethodName)); 222 | 223 | if (!Methods.Any()) 224 | { 225 | return true; 226 | } 227 | 228 | if (WasFullMethodSpecified(fullMethodName)) 229 | { 230 | return true; 231 | } 232 | 233 | if (!WasMethodSpecified(fullMethodName)) 234 | { 235 | return false; 236 | } 237 | 238 | var a = fullMethodName.Split('.'); 239 | var fullClassName = string.Join(".", a.Take(a.Length - 1)); 240 | 241 | if (!Classes.Any()) 242 | { 243 | return true; 244 | } 245 | 246 | if (WasFullClassSpecified(fullClassName)) 247 | { 248 | return true; 249 | } 250 | 251 | if (WasClassSpecified(fullClassName)) 252 | { 253 | return true; 254 | } 255 | 256 | return false; 257 | } 258 | 259 | 260 | /// 261 | /// Parse command line arguments 262 | /// 263 | /// 264 | static public void Parse(string[] args) 265 | { 266 | Success = false; 267 | ErrorMessage = ""; 268 | OutputFormat = OutputFormats.Human; 269 | InProc = false; 270 | Help = false; 271 | Parse(new Queue(args)); 272 | } 273 | 274 | 275 | static void Parse(Queue args) 276 | { 277 | if (args.Count == 1 && args.Peek() == "--help") 278 | { 279 | Help = true; 280 | Success = true; 281 | return; 282 | } 283 | 284 | for (;;) 285 | { 286 | if (args.Count == 0) break; 287 | if (!args.Peek().StartsWith("--", StringComparison.Ordinal)) break; 288 | var s = args.Dequeue(); 289 | switch (s) 290 | { 291 | case "--outputformat": 292 | ParseOutputFormat(args); 293 | if (ErrorMessage != "") return; 294 | break; 295 | 296 | case "--class": 297 | ParseClass(args); 298 | if (ErrorMessage != "") return; 299 | break; 300 | 301 | case "--method": 302 | ParseMethod(args); 303 | if (ErrorMessage != "") return; 304 | break; 305 | 306 | case "--inproc": 307 | InProc = true; 308 | break; 309 | 310 | case "--help": 311 | ErrorMessage = $"Unexpected switch {s}"; 312 | return; 313 | 314 | default: 315 | ErrorMessage = $"Unrecognised switch {s}"; 316 | return; 317 | } 318 | } 319 | 320 | while (args.Count > 0) 321 | { 322 | _testFiles.Add(args.Dequeue()); 323 | } 324 | 325 | if (TestFiles.Count == 0) 326 | { 327 | ErrorMessage = "No s specified"; 328 | return; 329 | } 330 | 331 | if (InProc && TestFiles.Count > 1) 332 | { 333 | ErrorMessage = "Only one allowed when --inproc"; 334 | return; 335 | } 336 | 337 | Success = true; 338 | } 339 | 340 | 341 | static void ParseClass(Queue args) 342 | { 343 | if (args.Count == 0) 344 | { 345 | ErrorMessage = "Expected "; 346 | return; 347 | } 348 | 349 | _classes.Add(args.Dequeue()); 350 | } 351 | 352 | 353 | static void ParseMethod(Queue args) 354 | { 355 | if (args.Count == 0) 356 | { 357 | ErrorMessage = "Expected "; 358 | return; 359 | } 360 | 361 | _methods.Add(args.Dequeue()); 362 | } 363 | 364 | 365 | static void ParseOutputFormat(Queue args) 366 | { 367 | if (args.Count == 0) 368 | { 369 | ErrorMessage = "Expected "; 370 | return; 371 | } 372 | 373 | var s = args.Dequeue(); 374 | 375 | switch (s) 376 | { 377 | case OutputFormats.Human: 378 | case OutputFormats.Machine: 379 | OutputFormat = s; 380 | break; 381 | default: 382 | ErrorMessage = $"Unrecognised {s}"; 383 | break; 384 | } 385 | } 386 | 387 | 388 | static bool WasFullClassSpecified(string fullClassName) 389 | { 390 | return Classes.Contains(fullClassName); 391 | } 392 | 393 | 394 | static bool WasClassSpecified(string fullClassName) 395 | { 396 | var className = fullClassName.Split('.').Last(); 397 | return Classes.Contains(className); 398 | } 399 | 400 | 401 | static bool WasFullMethodInClassSpecified(string fullClassName) 402 | { 403 | return Methods.Any(m => m.StartsWith($"{fullClassName}.", StringComparison.Ordinal)); 404 | } 405 | 406 | 407 | static bool WasFullMethodSpecified(string fullMethodName) 408 | { 409 | return Methods.Contains(fullMethodName); 410 | } 411 | 412 | 413 | static bool WasMethodSpecified(string fullMethodName) 414 | { 415 | var methodName = fullMethodName.Split('.').Last(); 416 | return Methods.Contains(methodName); 417 | } 418 | 419 | } 420 | } 421 | -------------------------------------------------------------------------------- /testrunner/EventHandlers/HumanOutputEventHandler.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Text; 4 | using TestRunner.Events; 5 | using TestRunner.Infrastructure; 6 | using TestRunner.Results; 7 | 8 | namespace TestRunner.EventHandlers 9 | { 10 | 11 | /// 12 | /// Event handler that produces output in human-readable format 13 | /// 14 | /// 15 | public class HumanOutputEventHandler : EventHandler 16 | { 17 | 18 | static void WriteMethodBegin(string name, string prefix) 19 | { 20 | Guard.NotNull(name, nameof(name)); 21 | Guard.NotNull(prefix, nameof(prefix)); 22 | prefix = prefix != "" ? prefix + " " : prefix; 23 | WriteOut(); 24 | WriteOut($"{prefix}{name}()"); 25 | } 26 | 27 | 28 | static void WriteMethodEnd(bool success, long elapsedMilliseconds) 29 | { 30 | var result = success ? "Succeeded" : "Failed"; 31 | WriteOut($" {result} ({elapsedMilliseconds:N0} ms)"); 32 | } 33 | 34 | 35 | static void WriteHeadingOut(params string[] lines) 36 | { 37 | lines = lines ?? new string[0]; 38 | lines = FormatHeading('=', lines); 39 | foreach (var line in lines) WriteOut(line); 40 | } 41 | 42 | 43 | static void WriteHeadingError(params string[] lines) 44 | { 45 | lines = lines ?? new string[0]; 46 | lines = FormatHeading('=', lines); 47 | foreach (var line in lines) WriteError(line); 48 | } 49 | 50 | 51 | static void WriteSubheadingOut(params string[] lines) 52 | { 53 | lines = lines ?? new string[0]; 54 | lines = FormatHeading('-', lines); 55 | foreach (var line in lines) WriteOut(line); 56 | } 57 | 58 | 59 | static void WriteOut(string message = "") 60 | { 61 | message = message ?? ""; 62 | foreach (var line in StringExtensions.SplitLines(message)) Console.Out.WriteLine(line); 63 | } 64 | 65 | 66 | static void WriteError(string[] lines) 67 | { 68 | Guard.NotNull(lines, nameof(lines)); 69 | foreach (var line in lines) WriteError(line); 70 | } 71 | 72 | 73 | static void WriteError(string message = "") 74 | { 75 | message = message ?? ""; 76 | foreach (var line in StringExtensions.SplitLines(message)) Console.Error.WriteLine(line); 77 | } 78 | 79 | 80 | static string[] FormatHeading(char ruleCharacter, params string[] lines) 81 | { 82 | if (lines == null) return new string[0]; 83 | if (lines.Length == 0) return new string [0]; 84 | 85 | var longestLine = lines.Max(line => line.Length); 86 | var rule = new string(ruleCharacter, longestLine); 87 | 88 | return 89 | Enumerable.Empty() 90 | .Concat(new[]{ rule }) 91 | .Concat(lines) 92 | .Concat(new[]{ rule }) 93 | .ToArray(); 94 | } 95 | 96 | 97 | static string FormatException(ExceptionInfo ex) 98 | { 99 | if (ex == null) return ""; 100 | var sb = new StringBuilder(); 101 | sb.AppendLine(ex.Message); 102 | sb.AppendLine($"Type: {ex.FullName}"); 103 | foreach (var kvp in ex.Data) 104 | { 105 | sb.AppendLine($"Data.{kvp.Key}: {kvp.Value}"); 106 | } 107 | if (!string.IsNullOrWhiteSpace(ex.Source)) 108 | { 109 | sb.AppendLine("Source: " + ex.Source); 110 | } 111 | if (!string.IsNullOrWhiteSpace(ex.HelpLink)) 112 | { 113 | sb.AppendLine("HelpLink: " + ex.HelpLink); 114 | } 115 | if (ex.StackTrace.Count > 0) 116 | { 117 | sb.AppendLine("StackTrace:"); 118 | foreach (var frame in ex.StackTrace) 119 | { 120 | sb.AppendLine(" " + frame.At); 121 | if (frame.In == "") continue; 122 | sb.AppendLine(" " + frame.In); 123 | } 124 | } 125 | if (ex.InnerException != null) 126 | { 127 | sb.AppendLine("InnerException:"); 128 | sb.AppendLine(StringExtensions.Indent(FormatException(ex.InnerException))); 129 | } 130 | return sb.ToString(); 131 | } 132 | 133 | 134 | static string FormatStackTrace(string stackTrace) 135 | { 136 | return string.Join( 137 | Environment.NewLine, 138 | StringExtensions.SplitLines(stackTrace) 139 | .Select(line => line.Trim()) 140 | .SelectMany(line => { 141 | var i = line.IndexOf(" in ", StringComparison.Ordinal); 142 | if (i <= 0) return new[] {line}; 143 | var inPart = line.Substring(i + 1); 144 | var atPart = line.Substring(0, i); 145 | return new[] {atPart, StringExtensions.Indent(inPart)}; 146 | })); 147 | } 148 | 149 | 150 | protected override void Handle(ProgramBannerEvent e) 151 | { 152 | WriteError(); 153 | WriteError(); 154 | WriteHeadingError(e.Lines); 155 | } 156 | 157 | 158 | protected override void Handle(ProgramUsageEvent e) 159 | { 160 | WriteError(); 161 | WriteError(e.Lines); 162 | WriteError(); 163 | } 164 | 165 | 166 | protected override void Handle(ProgramUserErrorEvent e) 167 | { 168 | WriteError(); 169 | WriteError(e.Message); 170 | } 171 | 172 | 173 | protected override void Handle(ProgramInternalErrorEvent e) 174 | { 175 | WriteError(); 176 | WriteError("An internal error occurred:"); 177 | WriteError(FormatException(e.Exception)); 178 | } 179 | 180 | 181 | protected override void Handle(TestAssemblyBeginEvent e) 182 | { 183 | WriteOut(); 184 | WriteHeadingOut(e.Path); 185 | } 186 | 187 | 188 | protected override void Handle(TestAssemblyNotFoundEvent e) 189 | { 190 | WriteOut(); 191 | WriteOut($"Test assembly not found: {e.Path}"); 192 | } 193 | 194 | 195 | protected override void Handle(TestAssemblyNotDotNetEvent e) 196 | { 197 | WriteOut(); 198 | WriteOut($"Not a .NET assembly: {e.Path}"); 199 | } 200 | 201 | 202 | protected override void Handle(TestAssemblyNotTestEvent e) 203 | { 204 | WriteOut(); 205 | WriteOut($"Not a test assembly: {e.Path}"); 206 | } 207 | 208 | 209 | protected override void Handle(TestAssemblyConfigFileSwitchedEvent e) 210 | { 211 | WriteOut(); 212 | WriteOut("Configuration File:"); 213 | WriteOut(e.Path); 214 | 215 | if (Type.GetType("Mono.Runtime") != null) 216 | { 217 | WriteOut(); 218 | WriteOut("WARNING: Running on Mono, configuration file will probably not take effect"); 219 | WriteOut("See https://bugzilla.xamarin.com/show_bug.cgi?id=15741"); 220 | } 221 | 222 | } 223 | 224 | 225 | protected override void Handle(TestAssemblyEndEvent e) 226 | { 227 | } 228 | 229 | 230 | protected override void Handle(TestClassBeginEvent e) 231 | { 232 | WriteOut(); 233 | WriteHeadingOut(e.FullName); 234 | } 235 | 236 | 237 | protected override void Handle(TestClassEndEvent e) 238 | { 239 | var initializeResult = 240 | e.Result.InitializePresent 241 | ? e.Result.ClassIgnored 242 | ? "Ignored" 243 | : e.Result.InitializeSucceeded 244 | ? "Succeeded" 245 | : "Failed" 246 | : "Not present"; 247 | 248 | var cleanupResult = 249 | e.Result.CleanupPresent 250 | ? e.Result.ClassIgnored 251 | ? "Ignored" 252 | : e.Result.CleanupSucceeded 253 | ? "Succeeded" 254 | : "Failed" 255 | : "Not present"; 256 | 257 | WriteOut(); 258 | WriteSubheadingOut("Summary"); 259 | 260 | if (e.Result.ClassIgnored) 261 | { 262 | WriteOut(); 263 | if (e.Result.ClassIgnoredFromCommandLine) 264 | { 265 | WriteOut("Ignored all tests because class is excluded by command line option(s)"); 266 | } 267 | else 268 | { 269 | WriteOut("Ignored all tests because class is decorated with [Ignore]"); 270 | } 271 | } 272 | 273 | WriteOut(); 274 | WriteOut($"ClassInitialize: {initializeResult}"); 275 | WriteOut($"Total: {e.Result.TestsTotal} tests"); 276 | WriteOut($"Ignored: {e.Result.TestsIgnored} tests"); 277 | WriteOut($"Ran: {e.Result.TestsRan} tests"); 278 | WriteOut($"Passed: {e.Result.TestsPassed} tests"); 279 | WriteOut($"Failed: {e.Result.TestsFailed} tests"); 280 | WriteOut($"ClassCleanup: {cleanupResult}"); 281 | } 282 | 283 | 284 | protected override void Handle(TestBeginEvent e) 285 | { 286 | WriteOut(); 287 | WriteSubheadingOut(e.Name.Replace("_", " ")); 288 | } 289 | 290 | 291 | protected override void Handle(TestEndEvent e) 292 | { 293 | WriteOut(); 294 | if (e.Result.Ignored && e.Result.IgnoredFromCommandLine) 295 | { 296 | WriteOut("Ignored because method is excluded by command line option(s)"); 297 | } 298 | else if (e.Result.Ignored) 299 | { 300 | WriteOut("Ignored because method is decorated with [Ignore]"); 301 | } 302 | else if (e.Result.Success) 303 | { 304 | WriteOut("Passed"); 305 | } 306 | else 307 | { 308 | WriteOut("FAILED"); 309 | } 310 | } 311 | 312 | 313 | protected override void Handle(AssemblyInitializeMethodBeginEvent e) 314 | { 315 | WriteMethodBegin(e.MethodName, "[AssemblyInitialize]"); 316 | } 317 | 318 | 319 | protected override void Handle(AssemblyInitializeMethodEndEvent e) 320 | { 321 | WriteMethodEnd(e.Result.Success, e.Result.ElapsedMilliseconds); 322 | } 323 | 324 | 325 | protected override void Handle(AssemblyCleanupMethodBeginEvent e) 326 | { 327 | WriteMethodBegin(e.MethodName, "[AssemblyCleanup]"); 328 | } 329 | 330 | 331 | protected override void Handle(AssemblyCleanupMethodEndEvent e) 332 | { 333 | WriteMethodEnd(e.Result.Success, e.Result.ElapsedMilliseconds); 334 | } 335 | 336 | 337 | protected override void Handle(ClassInitializeMethodBeginEvent e) 338 | { 339 | WriteMethodBegin(e.MethodName, "[ClassInitialize]"); 340 | } 341 | 342 | 343 | protected override void Handle(ClassInitializeMethodEndEvent e) 344 | { 345 | WriteMethodEnd(e.Result.Success, e.Result.ElapsedMilliseconds); 346 | } 347 | 348 | 349 | protected override void Handle(ClassCleanupMethodBeginEvent e) 350 | { 351 | WriteMethodBegin(e.MethodName, "[ClassCleanup]"); 352 | } 353 | 354 | 355 | protected override void Handle(ClassCleanupMethodEndEvent e) 356 | { 357 | WriteMethodEnd(e.Result.Success, e.Result.ElapsedMilliseconds); 358 | } 359 | 360 | 361 | protected override void Handle(TestContextSetterBeginEvent e) 362 | { 363 | WriteMethodBegin(e.MethodName, ""); 364 | } 365 | 366 | 367 | protected override void Handle(TestContextSetterEndEvent e) 368 | { 369 | WriteMethodEnd(e.Result.Success, e.Result.ElapsedMilliseconds); 370 | } 371 | 372 | 373 | protected override void Handle(TestInitializeMethodBeginEvent e) 374 | { 375 | WriteMethodBegin(e.MethodName, "[TestInitialize]"); 376 | } 377 | 378 | 379 | protected override void Handle(TestInitializeMethodEndEvent e) 380 | { 381 | WriteMethodEnd(e.Result.Success, e.Result.ElapsedMilliseconds); 382 | } 383 | 384 | 385 | protected override void Handle(TestMethodBeginEvent e) 386 | { 387 | WriteMethodBegin(e.MethodName, "[TestMethod]"); 388 | } 389 | 390 | 391 | protected override void Handle(TestMethodEndEvent e) 392 | { 393 | WriteMethodEnd(e.Result.Success, e.Result.ElapsedMilliseconds); 394 | } 395 | 396 | 397 | protected override void Handle(TestCleanupMethodBeginEvent e) 398 | { 399 | WriteMethodBegin(e.MethodName, "[TestCleanup]"); 400 | } 401 | 402 | 403 | protected override void Handle(TestCleanupMethodEndEvent e) 404 | { 405 | WriteMethodEnd(e.Result.Success, e.Result.ElapsedMilliseconds); 406 | } 407 | 408 | 409 | protected override void Handle(MethodExpectedExceptionEvent e) 410 | { 411 | WriteOut($" [ExpectedException] {e.ExpectedFullName} occurred:"); 412 | WriteOut(StringExtensions.Indent(FormatException(e.Exception))); 413 | } 414 | 415 | 416 | protected override void Handle(MethodUnexpectedExceptionEvent e) 417 | { 418 | WriteOut(StringExtensions.Indent(FormatException(e.Exception))); 419 | } 420 | 421 | 422 | protected override void Handle(StandardOutputEvent e) 423 | { 424 | WriteOut(e.Message); 425 | } 426 | 427 | 428 | protected override void Handle(ErrorOutputEvent e) 429 | { 430 | WriteError(e.Message); 431 | } 432 | 433 | 434 | protected override void Handle(TraceOutputEvent e) 435 | { 436 | WriteOut(e.Message); 437 | } 438 | 439 | } 440 | } 441 | --------------------------------------------------------------------------------