├── .gitattributes ├── .gitignore ├── LICENSE ├── ParallelTestingSample-dotnet.sln ├── README.md ├── SliceTests ├── SliceTests.cs ├── SliceTests.csproj └── testdata │ └── words.txt ├── azure-pipelines.yml └── create_slicing_filter_condition.sh /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.userosscache 8 | *.sln.docstates 9 | 10 | # User-specific files (MonoDevelop/Xamarin Studio) 11 | *.userprefs 12 | 13 | # Build results 14 | [Dd]ebug/ 15 | [Dd]ebugPublic/ 16 | [Rr]elease/ 17 | [Rr]eleases/ 18 | x64/ 19 | x86/ 20 | bld/ 21 | [Bb]in/ 22 | [Oo]bj/ 23 | [Ll]og/ 24 | 25 | # Visual Studio 2015 cache/options directory 26 | .vs/ 27 | # Uncomment if you have tasks that create the project's static files in wwwroot 28 | #wwwroot/ 29 | 30 | # MSTest test Results 31 | [Tt]est[Rr]esult*/ 32 | [Bb]uild[Ll]og.* 33 | 34 | # NUNIT 35 | *.VisualState.xml 36 | TestResult.xml 37 | 38 | # Build Results of an ATL Project 39 | [Dd]ebugPS/ 40 | [Rr]eleasePS/ 41 | dlldata.c 42 | 43 | # DNX 44 | project.lock.json 45 | project.fragment.lock.json 46 | artifacts/ 47 | 48 | *_i.c 49 | *_p.c 50 | *_i.h 51 | *.ilk 52 | *.meta 53 | *.obj 54 | *.pch 55 | *.pdb 56 | *.pgc 57 | *.pgd 58 | *.rsp 59 | *.sbr 60 | *.tlb 61 | *.tli 62 | *.tlh 63 | *.tmp 64 | *.tmp_proj 65 | *.log 66 | *.vspscc 67 | *.vssscc 68 | .builds 69 | *.pidb 70 | *.svclog 71 | *.scc 72 | 73 | # Chutzpah Test files 74 | _Chutzpah* 75 | 76 | # Visual C++ cache files 77 | ipch/ 78 | *.aps 79 | *.ncb 80 | *.opendb 81 | *.opensdf 82 | *.sdf 83 | *.cachefile 84 | *.VC.db 85 | *.VC.VC.opendb 86 | 87 | # Visual Studio profiler 88 | *.psess 89 | *.vsp 90 | *.vspx 91 | *.sap 92 | 93 | # TFS 2012 Local Workspace 94 | $tf/ 95 | 96 | # Guidance Automation Toolkit 97 | *.gpState 98 | 99 | # ReSharper is a .NET coding add-in 100 | _ReSharper*/ 101 | *.[Rr]e[Ss]harper 102 | *.DotSettings.user 103 | 104 | # JustCode is a .NET coding add-in 105 | .JustCode 106 | 107 | # TeamCity is a build add-in 108 | _TeamCity* 109 | 110 | # DotCover is a Code Coverage Tool 111 | *.dotCover 112 | 113 | # NCrunch 114 | _NCrunch_* 115 | .*crunch*.local.xml 116 | nCrunchTemp_* 117 | 118 | # MightyMoose 119 | *.mm.* 120 | AutoTest.Net/ 121 | 122 | # Web workbench (sass) 123 | .sass-cache/ 124 | 125 | # Installshield output folder 126 | [Ee]xpress/ 127 | 128 | # DocProject is a documentation generator add-in 129 | DocProject/buildhelp/ 130 | DocProject/Help/*.HxT 131 | DocProject/Help/*.HxC 132 | DocProject/Help/*.hhc 133 | DocProject/Help/*.hhk 134 | DocProject/Help/*.hhp 135 | DocProject/Help/Html2 136 | DocProject/Help/html 137 | 138 | # Click-Once directory 139 | publish/ 140 | 141 | # Publish Web Output 142 | *.[Pp]ublish.xml 143 | *.azurePubxml 144 | # TODO: Comment the next line if you want to checkin your web deploy settings 145 | # but database connection strings (with potential passwords) will be unencrypted 146 | #*.pubxml 147 | *.publishproj 148 | 149 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 150 | # checkin your Azure Web App publish settings, but sensitive information contained 151 | # in these scripts will be unencrypted 152 | PublishScripts/ 153 | 154 | # NuGet Packages 155 | *.nupkg 156 | # The packages folder can be ignored because of Package Restore 157 | **/packages/* 158 | # except build/, which is used as an MSBuild target. 159 | !**/packages/build/ 160 | # Uncomment if necessary however generally it will be regenerated when needed 161 | #!**/packages/repositories.config 162 | # NuGet v3's project.json files produces more ignoreable files 163 | *.nuget.props 164 | *.nuget.targets 165 | 166 | # Microsoft Azure Build Output 167 | csx/ 168 | *.build.csdef 169 | 170 | # Microsoft Azure Emulator 171 | ecf/ 172 | rcf/ 173 | 174 | # Windows Store app package directories and files 175 | AppPackages/ 176 | BundleArtifacts/ 177 | Package.StoreAssociation.xml 178 | _pkginfo.txt 179 | 180 | # Visual Studio cache files 181 | # files ending in .cache can be ignored 182 | *.[Cc]ache 183 | # but keep track of directories ending in .cache 184 | !*.[Cc]ache/ 185 | 186 | # Others 187 | ClientBin/ 188 | ~$* 189 | *~ 190 | *.dbmdl 191 | *.dbproj.schemaview 192 | *.jfm 193 | *.pfx 194 | *.publishsettings 195 | node_modules/ 196 | orleans.codegen.cs 197 | 198 | # Since there are multiple workflows, uncomment next line to ignore bower_components 199 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 200 | #bower_components/ 201 | 202 | # RIA/Silverlight projects 203 | Generated_Code/ 204 | 205 | # Backup & report files from converting an old project file 206 | # to a newer Visual Studio version. Backup files are not needed, 207 | # because we have git ;-) 208 | _UpgradeReport_Files/ 209 | Backup*/ 210 | UpgradeLog*.XML 211 | UpgradeLog*.htm 212 | 213 | # SQL Server files 214 | *.mdf 215 | *.ldf 216 | 217 | # Business Intelligence projects 218 | *.rdl.data 219 | *.bim.layout 220 | *.bim_*.settings 221 | 222 | # Microsoft Fakes 223 | FakesAssemblies/ 224 | 225 | # GhostDoc plugin setting file 226 | *.GhostDoc.xml 227 | 228 | # Node.js Tools for Visual Studio 229 | .ntvs_analysis.dat 230 | 231 | # Visual Studio 6 build log 232 | *.plg 233 | 234 | # Visual Studio 6 workspace options file 235 | *.opt 236 | 237 | # Visual Studio LightSwitch build output 238 | **/*.HTMLClient/GeneratedArtifacts 239 | **/*.DesktopClient/GeneratedArtifacts 240 | **/*.DesktopClient/ModelManifest.xml 241 | **/*.Server/GeneratedArtifacts 242 | **/*.Server/ModelManifest.xml 243 | _Pvt_Extensions 244 | 245 | # Paket dependency manager 246 | .paket/paket.exe 247 | paket-files/ 248 | 249 | # FAKE - F# Make 250 | .fake/ 251 | 252 | # JetBrains Rider 253 | .idea/ 254 | *.sln.iml 255 | 256 | # CodeRush 257 | .cr/ 258 | 259 | # Python Tools for Visual Studio (PTVS) 260 | __pycache__/ 261 | *.pyc -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018-2019 idubnori 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /ParallelTestingSample-dotnet.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.28010.2050 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SliceTests", "SliceTests\SliceTests.csproj", "{EEDC36D8-D8EE-494A-81CE-308D12BEC3ED}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Release|Any CPU = Release|Any CPU 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {EEDC36D8-D8EE-494A-81CE-308D12BEC3ED}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 15 | {EEDC36D8-D8EE-494A-81CE-308D12BEC3ED}.Debug|Any CPU.Build.0 = Debug|Any CPU 16 | {EEDC36D8-D8EE-494A-81CE-308D12BEC3ED}.Release|Any CPU.ActiveCfg = Release|Any CPU 17 | {EEDC36D8-D8EE-494A-81CE-308D12BEC3ED}.Release|Any CPU.Build.0 = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(SolutionProperties) = preSolution 20 | HideSolutionNode = FALSE 21 | EndGlobalSection 22 | GlobalSection(ExtensibilityGlobals) = postSolution 23 | SolutionGuid = {3D9244A3-E98F-4BE6-B172-457815363931} 24 | EndGlobalSection 25 | EndGlobal 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ParallelTestingSample-dotnet-core 2 | === 3 | [![GitHub license](https://img.shields.io/github/license/idubnori/ParallelTestingSample-dotnet-core.svg)](https://github.com/idubnori/ParallelTestingSample-dotnet-core/blob/master/LICENSE) 4 | [![Build Status](https://dev.azure.com/idubnori/idubnori/_apis/build/status/ParallelTestingSample-dotnet-CI)](https://dev.azure.com/idubnori/idubnori/_build/latest?definitionId=4) 5 | 6 | This .NET Core parallel testing sample uses `--list-tests` and `--filter` parameters of `dotnet test` to slice the tests. The tests are run using the NUnit. 7 | 8 | This sample has the 100 tests, and slices them to 20 tests in 5 jobs. You can see the pipeline behavior result by clicking the build status badge above. 9 | 10 | ## Overview of *azure-pipelines.yml* 11 | ### Setting up parallel count 12 | ```yml 13 | jobs: 14 | - job: 'ParallelTesting' 15 | pool: 16 | name: Hosted Ubuntu 1604 17 | strategy: 18 | parallel: 5 19 | displayName: Run tests in parallel 20 | ``` 21 | 22 | ### Make slicing condition 23 | - Get test name list of all tests by using `--list-tests` parameter and `grep Test_` 24 | - `create_slicing_filter_condition.sh` makes filter condition to slice the tests, and set into `$(targetTestsFilter)` 25 | ```yml 26 | - bash: | 27 | tests=($(dotnet test . --no-build --list-tests | grep Test_)) 28 | . 'create_slicing_filter_condition.sh' $tests 29 | displayName: 'Create slicing filter condition' 30 | ``` 31 | 32 | ### Run tests using slicing condition 33 | ```yml 34 | - task: DotNetCoreCLI@2 35 | displayName: Test 36 | inputs: 37 | command: test 38 | projects: '**/*Tests/*Tests.csproj' 39 | arguments: '--no-build --filter "$(targetTestsFilter)"' 40 | ``` 41 | 42 | ## References 43 | - [Speed up testing by running tests in parallel - Azure Pipelines & TFS | Microsoft Docs](https://docs.microsoft.com/en-us/azure/devops/pipelines/test/parallel-testing-any-test-runner?view=vsts) -------------------------------------------------------------------------------- /SliceTests/SliceTests.cs: -------------------------------------------------------------------------------- 1 | using System.Collections; 2 | using System.IO; 3 | using FluentAssertions; 4 | using NUnit.Framework; 5 | using OpenQA.Selenium; 6 | using OpenQA.Selenium.Chrome; 7 | 8 | namespace SliceTests 9 | { 10 | public class SliceTests 11 | { 12 | private IWebDriver _webDriver; 13 | 14 | private static IEnumerable Words() 15 | { 16 | using (var reader = new StreamReader("./testdata/words.txt")) 17 | { 18 | while (reader.Peek() >= 0) 19 | { 20 | var word = reader.ReadLine(); 21 | yield return new TestCaseData(word).SetName("{m}_" + word); 22 | } 23 | } 24 | } 25 | 26 | [SetUp] 27 | public void Setup() 28 | { 29 | var options = new ChromeOptions(); 30 | options.AddArguments("--headless", "--no-sandbox", "--disable-dev-shm-usage"); 31 | 32 | _webDriver = new ChromeDriver(Directory.GetCurrentDirectory(), options); 33 | } 34 | 35 | [TearDown] 36 | public void TearDown() 37 | { 38 | _webDriver.Quit(); 39 | } 40 | 41 | [Test] 42 | [TestCaseSource(nameof(Words))] 43 | public void SearchTest_ByWord_WordContainsInTitle(string word) 44 | { 45 | // Arrange 46 | var url = $"https://www.google.co.jp/search?q={word}"; 47 | 48 | // Act 49 | _webDriver.Url = url; 50 | 51 | // Assert 52 | _webDriver.Title.Should().Contain(word); 53 | } 54 | } 55 | } -------------------------------------------------------------------------------- /SliceTests/SliceTests.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | net8.0 5 | 6 | 7 | 8 | 9 | PreserveNewest 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /SliceTests/testdata/words.txt: -------------------------------------------------------------------------------- 1 | bird 2 | notice 3 | banish 4 | physical 5 | rough 6 | minority 7 | elect 8 | reference 9 | cry 10 | cycle 11 | cold 12 | enthusiasm 13 | chimpanzee 14 | project 15 | premature 16 | scramble 17 | module 18 | crop 19 | air 20 | false 21 | chord 22 | ghost 23 | key 24 | formulate 25 | captivate 26 | necklace 27 | neighbour 28 | result 29 | rear 30 | plug 31 | hardship 32 | graze 33 | young 34 | content 35 | hostility 36 | claim 37 | value 38 | retailer 39 | intensify 40 | environmental 41 | touch 42 | cherry 43 | panel 44 | learn 45 | effect 46 | omission 47 | silk 48 | possibility 49 | race 50 | reduction 51 | threaten 52 | council 53 | snow 54 | column 55 | prospect 56 | swell 57 | sign 58 | railroad 59 | trench 60 | financial 61 | stress 62 | duke 63 | conductor 64 | frank 65 | push 66 | coerce 67 | sand 68 | family 69 | stick 70 | chocolate 71 | user 72 | update 73 | attractive 74 | attitude 75 | stock 76 | strategic 77 | theory 78 | book 79 | royalty 80 | foundation 81 | orientation 82 | multimedia 83 | decay 84 | utter 85 | joke 86 | discrimination 87 | funny 88 | prejudice 89 | inside 90 | anticipation 91 | distance 92 | van 93 | deprive 94 | landscape 95 | commission 96 | railcar 97 | confine 98 | like 99 | marine 100 | microphone -------------------------------------------------------------------------------- /azure-pipelines.yml: -------------------------------------------------------------------------------- 1 | schedules: 2 | - cron: "0 3 * * Mon" 3 | displayName: Weekly build 4 | branches: 5 | include: 6 | - master 7 | always: true 8 | 9 | jobs: 10 | - job: 'ParallelTesting' 11 | pool: 12 | vmImage: 'ubuntu-20.04' 13 | strategy: 14 | parallel: 5 15 | displayName: Run tests in parallel 16 | 17 | steps: 18 | - task: UseDotNet@2 19 | displayName: 'Use .NET sdk 8.0.x' 20 | inputs: 21 | version: 8.0.x 22 | performMultiLevelLookup: true 23 | 24 | - task: DotNetCoreCLI@2 25 | displayName: Build 26 | inputs: 27 | command: build 28 | projects: '**/*.csproj' 29 | 30 | - bash: | 31 | tests=($(dotnet test . --no-build --list-tests | grep Test_)) 32 | . 'create_slicing_filter_condition.sh' $tests 33 | displayName: 'Create slicing filter condition' 34 | 35 | - bash: | 36 | echo "Slicing filter condition: $(targetTestsFilter)" 37 | displayName: 'Echo slicing filter condition' 38 | 39 | - task: DotNetCoreCLI@2 40 | displayName: Test 41 | inputs: 42 | command: test 43 | projects: '**/*Tests/*Tests.csproj' 44 | arguments: '--no-build --filter "$(targetTestsFilter)"' -------------------------------------------------------------------------------- /create_slicing_filter_condition.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | filterProperty="Name" 4 | 5 | tests=$1 6 | testCount=${#tests[@]} 7 | totalAgents=$SYSTEM_TOTALJOBSINPHASE 8 | agentNumber=$SYSTEM_JOBPOSITIONINPHASE 9 | 10 | if [ $totalAgents -eq 0 ]; then totalAgents=1; fi 11 | if [ -z "$agentNumber" ]; then agentNumber=1; fi 12 | 13 | echo "Total agents: $totalAgents" 14 | echo "Agent number: $agentNumber" 15 | echo "Total tests: $testCount" 16 | 17 | echo "Target tests:" 18 | for ((i=$agentNumber; i <= $testCount;i=$((i+$totalAgents)))); do 19 | targetTestName=${tests[$i -1]} 20 | echo "$targetTestName" 21 | filter+="|${filterProperty}=${targetTestName}" 22 | done 23 | filter=${filter#"|"} 24 | 25 | echo "##vso[task.setvariable variable=targetTestsFilter]$filter" --------------------------------------------------------------------------------